<commit_before>"""Tests for generating a continuous integration setup."""
from . import pytest_generate_tests  # noqa, pylint: disable=unused-import
from . import verify_file_matches_repo_root

# pylint: disable=too-few-public-methods
class TestCISetup:
    """
    Tests for verifying generated CI setups of this cookiecutter,
    executed several times with different values (test scenarios).
    """
    scenarios = [
        ('bitbucket', {
            'project_slug': 'myproject',
            'vcs_account': 'painless-software',
            'vcs_platform': 'Bitbucket.org',
            'ci_service': 'bitbucket-pipelines.yml',
            'ci_testcommand': '        - tox -e py37',
            'checks': 'flake8,pylint,bandit',
            'tests': 'py35,py36,py37,pypy3,behave',
            'container_platform': 'APPUiO',
            'environment_strategy': 'shared',
            'expected_ci_target': '',
        }),
        ('codeship', {
            'project_slug': 'myproject',
            'vcs_account': 'painless-software',
            'vcs_platform': 'GitHub.com',
            'ci_service': 'codeship-steps.yml',
            'ci_testcommand': '  service: app',
            'checks': 'flake8,pylint,bandit',
            'tests': 'py35,py36,py37,pypy3,behave',
            'container_platform': 'APPUiO',
            'environment_strategy': 'shared',
            'expected_ci_target': '',
        }),
        ('gitlab-shared', {
            'project_slug': 'myproject',
            'vcs_account': 'painless-software',
            'vcs_platform': 'GitLab.com',
            'ci_service': '.gitlab-ci.yml',
            'ci_testcommand': '  script: tox -e py37',
            'checks': 'flake8,pylint,bandit',
            'tests': 'py35,py36,py37,pypy3,behave',
            'container_platform': 'APPUiO',
            'environment_strategy': 'shared',
            'expected_ci_target': \
                '    TARGET: "myproject"',
        }),
        ('gitlab-dedicated', {
            'project_slug': 'myproject',
            'vcs_account': 'painless-software',
            'vcs_platform': 'GitLab.com',
            'ci_service': '.gitlab-ci.yml',
            'ci_testcommand': '  script: tox -e py37',
            'checks': 'flake8,pylint,bandit',
            'tests': 'py35,py36,py37,pypy3,behave',
            'container_platform': 'APPUiO',
            'environment_strategy': 'dedicated',
            'expected_ci_target': \
                '    TARGET: "myproject-${CI_ENVIRONMENT_NAME}"',
        }),
        ('shippable', {
            'project_slug': 'myproject',
            'vcs_account': 'painless-software',
            'vcs_platform': 'Bitbucket.org',
            'ci_service': 'shippable.yml',
            'ci_testcommand': '  - tox',
            'checks': 'flake8,pylint,bandit',
            'tests': 'py35,py36,py37,pypy3,behave',
            'container_platform': 'APPUiO',
            'environment_strategy': 'shared',
            'expected_ci_target': '',
        }),
        ('travis', {
            'project_slug': 'myproject',
            'vcs_account': 'painless-software',
            'vcs_platform': 'GitHub.com',
            'ci_service': '.travis.yml',
            'ci_testcommand': 'script: tox',
            'checks': 'flake8,pylint,bandit',
            'tests': 'py35,py36,py37,pypy3,behave',
            'container_platform': 'APPUiO',
            'environment_strategy': 'shared',
            'expected_ci_target': '',
        }),
    ]

    # pylint: disable=too-many-arguments,too-many-locals,no-self-use
    def test_ci_setup(self, cookies, project_slug, vcs_account, vcs_platform,
                      ci_service, ci_testcommand, checks, tests,
                      container_platform, environment_strategy,
                      expected_ci_target):
        """
        Generate a CI setup with specific settings and verify it is complete.
        """
        result = cookies.bake(extra_context={
            'project_slug': project_slug,
            'vcs_platform': vcs_platform,
            'vcs_account': vcs_account,
            'ci_service': ci_service,
            'checks': checks,
            'tests': tests,
            'container_platform': container_platform,
            'environment_strategy': environment_strategy,
        })

        assert result.exit_code == 0
        assert result.exception is None

        assert result.project.basename == project_slug
        assert result.project.isdir()
        assert result.project.join('README.rst').isfile()

        ci_service_conf = result.project.join(ci_service).readlines(cr=False)
        assert ci_testcommand in ci_service_conf, \
            "Test command not found in CI config: '%s'" % ci_testcommand
        assert expected_ci_target in ci_service_conf, \
            "Deployment target not found in CI config: '%s'\n%s" \
            % (expected_ci_target, '\n'.join(ci_service_conf))

        codeship_services = result.project.join('codeship-services.yml')
        assert (ci_service == 'codeship-steps.yml' and
                codeship_services.isfile()) or not codeship_services.exists()

        # ensure this project itself stays up-to-date with the template
        verify_file_matches_repo_root(result, ci_service,
                                      max_compare_bytes=85)
<commit_msg>Fix flake8 complaints<commit_after>"""Tests for generating a continuous integration setup."""
from . import pytest_generate_tests  # noqa, pylint: disable=unused-import
from . import verify_file_matches_repo_root

# pylint: disable=too-few-public-methods
class TestCISetup:
    """
    Tests for verifying generated CI setups of this cookiecutter,
    executed several times with different values (test scenarios).
    """
    scenarios = [
        ('bitbucket', {
            'project_slug': 'myproject',
            'vcs_account': 'painless-software',
            'vcs_platform': 'Bitbucket.org',
            'ci_service': 'bitbucket-pipelines.yml',
            'ci_testcommand': '        - tox -e py37',
            'checks': 'flake8,pylint,bandit',
            'tests': 'py35,py36,py37,pypy3,behave',
            'container_platform': 'APPUiO',
            'environment_strategy': 'shared',
            'expected_ci_target': '',
        }),
        ('codeship', {
            'project_slug': 'myproject',
            'vcs_account': 'painless-software',
            'vcs_platform': 'GitHub.com',
            'ci_service': 'codeship-steps.yml',
            'ci_testcommand': '  service: app',
            'checks': 'flake8,pylint,bandit',
            'tests': 'py35,py36,py37,pypy3,behave',
            'container_platform': 'APPUiO',
            'environment_strategy': 'shared',
            'expected_ci_target': '',
        }),
        ('gitlab-shared', {
            'project_slug': 'myproject',
            'vcs_account': 'painless-software',
            'vcs_platform': 'GitLab.com',
            'ci_service': '.gitlab-ci.yml',
            'ci_testcommand': '  script: tox -e py37',
            'checks': 'flake8,pylint,bandit',
            'tests': 'py35,py36,py37,pypy3,behave',
            'container_platform': 'APPUiO',
            'environment_strategy': 'shared',
            'expected_ci_target':
                '    TARGET: "myproject"',
        }),
        ('gitlab-dedicated', {
            'project_slug': 'myproject',
            'vcs_account': 'painless-software',
            'vcs_platform': 'GitLab.com',
            'ci_service': '.gitlab-ci.yml',
            'ci_testcommand': '  script: tox -e py37',
            'checks': 'flake8,pylint,bandit',
            'tests': 'py35,py36,py37,pypy3,behave',
            'container_platform': 'APPUiO',
            'environment_strategy': 'dedicated',
            'expected_ci_target':
                '    TARGET: "myproject-${CI_ENVIRONMENT_NAME}"',
        }),
        ('shippable', {
            'project_slug': 'myproject',
            'vcs_account': 'painless-software',
            'vcs_platform': 'Bitbucket.org',
            'ci_service': 'shippable.yml',
            'ci_testcommand': '  - tox',
            'checks': 'flake8,pylint,bandit',
            'tests': 'py35,py36,py37,pypy3,behave',
            'container_platform': 'APPUiO',
            'environment_strategy': 'shared',
            'expected_ci_target': '',
        }),
        ('travis', {
            'project_slug': 'myproject',
            'vcs_account': 'painless-software',
            'vcs_platform': 'GitHub.com',
            'ci_service': '.travis.yml',
            'ci_testcommand': 'script: tox',
            'checks': 'flake8,pylint,bandit',
            'tests': 'py35,py36,py37,pypy3,behave',
            'container_platform': 'APPUiO',
            'environment_strategy': 'shared',
            'expected_ci_target': '',
        }),
    ]

    # pylint: disable=too-many-arguments,too-many-locals,no-self-use
    def test_ci_setup(self, cookies, project_slug, vcs_account, vcs_platform,
                      ci_service, ci_testcommand, checks, tests,
                      container_platform, environment_strategy,
                      expected_ci_target):
        """
        Generate a CI setup with specific settings and verify it is complete.
        """
        result = cookies.bake(extra_context={
            'project_slug': project_slug,
            'vcs_platform': vcs_platform,
            'vcs_account': vcs_account,
            'ci_service': ci_service,
            'checks': checks,
            'tests': tests,
            'container_platform': container_platform,
            'environment_strategy': environment_strategy,
        })

        assert result.exit_code == 0
        assert result.exception is None

        assert result.project.basename == project_slug
        assert result.project.isdir()
        assert result.project.join('README.rst').isfile()

        ci_service_conf = result.project.join(ci_service).readlines(cr=False)
        assert ci_testcommand in ci_service_conf, \
            "Test command not found in CI config: '%s'" % ci_testcommand
        assert expected_ci_target in ci_service_conf, \
            "Deployment target not found in CI config: '%s'\n%s" \
            % (expected_ci_target, '\n'.join(ci_service_conf))

        codeship_services = result.project.join('codeship-services.yml')
        assert (ci_service == 'codeship-steps.yml' and
                codeship_services.isfile()) or not codeship_services.exists()

        # ensure this project itself stays up-to-date with the template
        verify_file_matches_repo_root(result, ci_service,
                                      max_compare_bytes=85)
<|endoftext|>


<commit_before>from __future__ import absolute_import, print_function

from flask import render_template
from flask_mail import Message, sanitize_address

from changes.config import db, mail
from changes.constants import Result, Status
from changes.models import Build, TestGroup, ProjectOption, LogSource, LogChunk
from changes.utils.http import build_uri

def get_test_failures(build):
    return sorted([t.name_sha for t in db.session.query(
        TestGroup.name_sha,
    ).filter(
        TestGroup.build_id == build.id,
        TestGroup.result == Result.failed,
        TestGroup.num_leaves == 0,
    )])

def did_cause_breakage(build):
    """
    Compare with parent build (previous build) and confirm if current
    build provided any change in state (e.g. new failures).
    """
    if build.result != Result.failed:
        return False

    parent = Build.query.filter(
        Build.revision_sha != None,  # NOQA
        Build.patch_id == None,
        Build.revision_sha != build.revision_sha,
        Build.date_created < build.date_created,
        Build.status == Status.finished,
    ).order_by(Build.date_created.desc()).first()

    # if theres no parent, this build must be at fault
    if parent is None:
        return True

    if parent.result == Result.passed:
        return True

    current_failures = get_test_failures(build)
    # if we dont have any testgroup failures, then we cannot identify the cause
    # so we must notify the individual
    if not current_failures:
        return True

    parent_failures = get_test_failures(parent)
    if parent_failures != current_failures:
        return True

    return False

def get_log_clipping(logsource, max_size=5000, max_lines=25):
    queryset = LogChunk.query.filter(
        LogChunk.source_id == logsource.id,
    )
    tail = queryset.order_by(LogChunk.offset.desc()).limit(1).first()

    chunks = list(queryset.filter(
        (LogChunk.offset + LogChunk.size) >= max(tail.offset - max_size, 0),
    ).order_by(LogChunk.offset.asc()))

    clipping = ''.join(l.text for l in chunks).strip()[-max_size:]
    # only return the last 25 lines
    clipping = '\r\n'.join(clipping.splitlines()[-max_lines:])

    return clipping

def send_notification(build, recipients):
    # TODO(dcramer): we should send a clipping of a relevant build log
    test_failures = TestGroup.query.filter(
        TestGroup.build_id == build.id,
        TestGroup.result == Result.failed,
        TestGroup.num_leaves == 0,
    ).order_by(TestGroup.name.asc())
    num_test_failures = test_failures.count()
    test_failures = test_failures[:25]

    # TODO(dcramer): we should probably find a better way to do logs
    primary_log = LogSource.query.filter(
        LogSource.build_id == build.id,
    ).order_by(LogSource.date_created.asc()).first()
    if primary_log:
        log_clipping = get_log_clipping(
            primary_log, max_size=5000, max_lines=50)

    subject = u"Build {result} - {target} ({project})".format(
        result=unicode(build.result),
        target=build.target or build.revision_sha or 'Unknown',
        project=build.project.name,
    )

    for testgroup in test_failures:
        testgroup.uri = build_uri('/testgroups/{0}/'.format(testgroup.id.hex))

    build.uri = build_uri('/builds/{0}/'.format(build.id.hex))

    context = {
        'build': build,
        'total_test_failures': num_test_failures,
        'test_failures': test_failures,
    }

    if primary_log:
        context['build_log'] = {
            'text': log_clipping,
            'name': primary_log.name,
            'link': '{0}logs/{1}/'.format(build.uri, primary_log.id.hex),
        }

    msg = Message(subject, recipients=recipients, extra_headers={
        'Reply-To': ', '.join(sanitize_address(r) for r in recipients),
    })
    msg.body = render_template('listeners/mail/notification.txt', **context)
    msg.html = render_template('listeners/mail/notification.html', **context)

    mail.send(msg)

def build_finished_handler(build, **kwargs):
    # get relevant options
    options = dict(
        db.session.query(
            ProjectOption.name, ProjectOption.value
        ).filter(
            ProjectOption.project_id == build.project_id,
            ProjectOption.name.in_([
                'mail.notify-author', 'mail.notify-addresses',
                'mail.notify-addresses-revisions',
            ])
        )
    )

    recipients = []
    if options.get('mail.notify-author', '1') == '1':
        author = build.author
        if author:
            recipients.append(u'%s <%s>' % (author.name, author.email))

    if options.get('mail.notify-addresses'):
        recipients.extend(
            # XXX(dcramer): we dont have option validators so lets assume people
            # enter slightly incorrect values
            [x.strip() for x in options['mail.notify-addresses'].split(',')]
        )

    if not build.patch_id:
        if options.get('mail.notify-addresses-revisions'):
            recipients.extend(
                [x.strip() for x in options['mail.notify-addresses-revisions'].split(',')]
            )

    if not recipients:
        return

    if not did_cause_breakage(build):
        return

    send_notification(build, recipients)
<commit_msg>Lower lines shown to 25 in emails<commit_after>from __future__ import absolute_import, print_function

from flask import render_template
from flask_mail import Message, sanitize_address

from changes.config import db, mail
from changes.constants import Result, Status
from changes.models import Build, TestGroup, ProjectOption, LogSource, LogChunk
from changes.utils.http import build_uri

def get_test_failures(build):
    return sorted([t.name_sha for t in db.session.query(
        TestGroup.name_sha,
    ).filter(
        TestGroup.build_id == build.id,
        TestGroup.result == Result.failed,
        TestGroup.num_leaves == 0,
    )])

def did_cause_breakage(build):
    """
    Compare with parent build (previous build) and confirm if current
    build provided any change in state (e.g. new failures).
    """
    if build.result != Result.failed:
        return False

    parent = Build.query.filter(
        Build.revision_sha != None,  # NOQA
        Build.patch_id == None,
        Build.revision_sha != build.revision_sha,
        Build.date_created < build.date_created,
        Build.status == Status.finished,
    ).order_by(Build.date_created.desc()).first()

    # if theres no parent, this build must be at fault
    if parent is None:
        return True

    if parent.result == Result.passed:
        return True

    current_failures = get_test_failures(build)
    # if we dont have any testgroup failures, then we cannot identify the cause
    # so we must notify the individual
    if not current_failures:
        return True

    parent_failures = get_test_failures(parent)
    if parent_failures != current_failures:
        return True

    return False

def get_log_clipping(logsource, max_size=5000, max_lines=25):
    queryset = LogChunk.query.filter(
        LogChunk.source_id == logsource.id,
    )
    tail = queryset.order_by(LogChunk.offset.desc()).limit(1).first()

    chunks = list(queryset.filter(
        (LogChunk.offset + LogChunk.size) >= max(tail.offset - max_size, 0),
    ).order_by(LogChunk.offset.asc()))

    clipping = ''.join(l.text for l in chunks).strip()[-max_size:]
    # only return the last 25 lines
    clipping = '\r\n'.join(clipping.splitlines()[-max_lines:])

    return clipping

def send_notification(build, recipients):
    # TODO(dcramer): we should send a clipping of a relevant build log
    test_failures = TestGroup.query.filter(
        TestGroup.build_id == build.id,
        TestGroup.result == Result.failed,
        TestGroup.num_leaves == 0,
    ).order_by(TestGroup.name.asc())
    num_test_failures = test_failures.count()
    test_failures = test_failures[:25]

    # TODO(dcramer): we should probably find a better way to do logs
    primary_log = LogSource.query.filter(
        LogSource.build_id == build.id,
    ).order_by(LogSource.date_created.asc()).first()
    if primary_log:
        log_clipping = get_log_clipping(
            primary_log, max_size=5000, max_lines=25)

    subject = u"Build {result} - {target} ({project})".format(
        result=unicode(build.result),
        target=build.target or build.revision_sha or 'Unknown',
        project=build.project.name,
    )

    for testgroup in test_failures:
        testgroup.uri = build_uri('/testgroups/{0}/'.format(testgroup.id.hex))

    build.uri = build_uri('/builds/{0}/'.format(build.id.hex))

    context = {
        'build': build,
        'total_test_failures': num_test_failures,
        'test_failures': test_failures,
    }

    if primary_log:
        context['build_log'] = {
            'text': log_clipping,
            'name': primary_log.name,
            'link': '{0}logs/{1}/'.format(build.uri, primary_log.id.hex),
        }

    msg = Message(subject, recipients=recipients, extra_headers={
        'Reply-To': ', '.join(sanitize_address(r) for r in recipients),
    })
    msg.body = render_template('listeners/mail/notification.txt', **context)
    msg.html = render_template('listeners/mail/notification.html', **context)

    mail.send(msg)

def build_finished_handler(build, **kwargs):
    # get relevant options
    options = dict(
        db.session.query(
            ProjectOption.name, ProjectOption.value
        ).filter(
            ProjectOption.project_id == build.project_id,
            ProjectOption.name.in_([
                'mail.notify-author', 'mail.notify-addresses',
                'mail.notify-addresses-revisions',
            ])
        )
    )

    recipients = []
    if options.get('mail.notify-author', '1') == '1':
        author = build.author
        if author:
            recipients.append(u'%s <%s>' % (author.name, author.email))

    if options.get('mail.notify-addresses'):
        recipients.extend(
            # XXX(dcramer): we dont have option validators so lets assume people
            # enter slightly incorrect values
            [x.strip() for x in options['mail.notify-addresses'].split(',')]
        )

    if not build.patch_id:
        if options.get('mail.notify-addresses-revisions'):
            recipients.extend(
                [x.strip() for x in options['mail.notify-addresses-revisions'].split(',')]
            )

    if not recipients:
        return

    if not did_cause_breakage(build):
        return

    send_notification(build, recipients)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from mock import Mock
from raven.utils.testutils import TestCase
from raven.utils import six

from raven.utils.stacks import get_culprit, get_stack_info, get_lines_from_file

class Context(object):
    def __init__(self, dict):
        self.dict = dict

    __getitem__ = lambda s, *a: s.dict.__getitem__(*a)
    __setitem__ = lambda s, *a: s.dict.__setitem__(*a)
    iterkeys = lambda s, *a: six.iterkeys(s.dict, *a)

class GetCulpritTest(TestCase):
    def test_empty_module(self):
        culprit = get_culprit([{
            'module': None,
            'function': 'foo',
        }])
        assert culprit == '? in foo'

    def test_empty_function(self):
        culprit = get_culprit([{
            'module': 'foo',
            'function': None,
        }])
        assert culprit == 'foo in ?'

    def test_no_module_or_function(self):
        culprit = get_culprit([{}])
        assert culprit is None

    def test_all_params(self):
        culprit = get_culprit([{
            'module': 'package.name',
            'function': 'foo',
        }])
        assert culprit == 'package.name in foo'

class GetStackInfoTest(TestCase):
    def test_bad_locals_in_frame(self):
        frame = Mock()
        frame.f_locals = Context({
            'foo': 'bar',
            'biz': 'baz',
        })
        frame.f_lineno = 1
        frame.f_globals = {}
        frame.f_code.co_filename = __file__
        frame.f_code.co_name = __name__

        frames = [(frame, 1)]
        results = get_stack_info(frames)
        assert len(results['frames']) == 1
        result = results['frames'][0]
        assert 'vars' in result
        if six.PY3:
            expected = {
                "'foo'": "'bar'",
                "'biz'": "'baz'",
            }
        else:
            expected = {
                "u'foo'": "u'bar'",
                "u'biz'": "u'baz'",
            }
        assert result['vars'] == expected

    def test_max_frames(self):
        frames = []
        for x in xrange(10):
            frame = Mock()
            frame.f_locals = {}
            frame.f_lineno = None
            frame.f_globals = {}
            frame.f_code.co_filename = str(x)
            frame.f_code.co_name = __name__
            frames.append((frame, 1))

        results = get_stack_info(frames, max_frames=4)
        assert results['frames_omitted'] == (3, 9)
        assert len(results['frames']) == 4
        assert results['frames'][0]['filename'] == '0'
        assert results['frames'][1]['filename'] == '1'
        assert results['frames'][2]['filename'] == '8'
        assert results['frames'][3]['filename'] == '9'

class GetLineFromFileTest(TestCase):

    def test_non_ascii_file(self):
        import os.path
        filename = os.path.join(os.path.dirname(__file__), 'utf8_file.txt')
        self.assertEqual(
            get_lines_from_file(filename, 3, 1),
            (['Some code here'], '', ['lorem ipsum']))
<commit_msg>xrange => range<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from mock import Mock
from raven.utils.testutils import TestCase
from raven.utils import six

from raven.utils.stacks import get_culprit, get_stack_info, get_lines_from_file

class Context(object):
    def __init__(self, dict):
        self.dict = dict

    __getitem__ = lambda s, *a: s.dict.__getitem__(*a)
    __setitem__ = lambda s, *a: s.dict.__setitem__(*a)
    iterkeys = lambda s, *a: six.iterkeys(s.dict, *a)

class GetCulpritTest(TestCase):
    def test_empty_module(self):
        culprit = get_culprit([{
            'module': None,
            'function': 'foo',
        }])
        assert culprit == '? in foo'

    def test_empty_function(self):
        culprit = get_culprit([{
            'module': 'foo',
            'function': None,
        }])
        assert culprit == 'foo in ?'

    def test_no_module_or_function(self):
        culprit = get_culprit([{}])
        assert culprit is None

    def test_all_params(self):
        culprit = get_culprit([{
            'module': 'package.name',
            'function': 'foo',
        }])
        assert culprit == 'package.name in foo'

class GetStackInfoTest(TestCase):
    def test_bad_locals_in_frame(self):
        frame = Mock()
        frame.f_locals = Context({
            'foo': 'bar',
            'biz': 'baz',
        })
        frame.f_lineno = 1
        frame.f_globals = {}
        frame.f_code.co_filename = __file__
        frame.f_code.co_name = __name__

        frames = [(frame, 1)]
        results = get_stack_info(frames)
        assert len(results['frames']) == 1
        result = results['frames'][0]
        assert 'vars' in result
        if six.PY3:
            expected = {
                "'foo'": "'bar'",
                "'biz'": "'baz'",
            }
        else:
            expected = {
                "u'foo'": "u'bar'",
                "u'biz'": "u'baz'",
            }
        assert result['vars'] == expected

    def test_max_frames(self):
        frames = []
        for x in range(10):
            frame = Mock()
            frame.f_locals = {}
            frame.f_lineno = None
            frame.f_globals = {}
            frame.f_code.co_filename = str(x)
            frame.f_code.co_name = __name__
            frames.append((frame, 1))

        results = get_stack_info(frames, max_frames=4)
        assert results['frames_omitted'] == (3, 9)
        assert len(results['frames']) == 4
        assert results['frames'][0]['filename'] == '0'
        assert results['frames'][1]['filename'] == '1'
        assert results['frames'][2]['filename'] == '8'
        assert results['frames'][3]['filename'] == '9'

class GetLineFromFileTest(TestCase):

    def test_non_ascii_file(self):
        import os.path
        filename = os.path.join(os.path.dirname(__file__), 'utf8_file.txt')
        self.assertEqual(
            get_lines_from_file(filename, 3, 1),
            (['Some code here'], '', ['lorem ipsum']))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from typing import Dict, Text, Any, List, Union, Optional

from rasa_core_sdk import Tracker
from rasa_core_sdk.executor import CollectingDispatcher
from rasa_core_sdk.forms import FormAction

class RestaurantForm(FormAction):
    """Example of a custom form action"""

    def name(self) -> Text:
        """Unique identifier of the form"""

        return "restaurant_form"

    @staticmethod
    def required_slots(tracker: Tracker) -> List[Text]:
        """A list of required slots that the form has to fill"""

        return ["cuisine", "num_people", "outdoor_seating",
                "preferences", "feedback"]

    def slot_mappings(self) -> Dict[Text: Union[Dict, List[Dict]]]:
        """A dictionary to map required slots to
            - an extracted entity
            - intent: value pairs
            - a whole message
            or a list of them, where a first match will be picked"""

        return {"cuisine": self.from_entity(entity="cuisine",
                                            not_intent="chitchat"),
                "num_people": [self.from_entity(entity="num_people",
                                                intent=["inform",
                                                        "request_restaurant"]),
                               self.from_entity(entity="number")],
                "outdoor_seating": [self.from_entity(entity="seating"),
                                    self.from_intent(intent='affirm',
                                                     value=True),
                                    self.from_intent(intent='deny',
                                                     value=False)],
                "preferences": [self.from_intent(intent='deny',
                                                 value="no additional "
                                                       "preferences"),
                                self.from_text(not_intent="affirm")],
                "feedback": [self.from_entity(entity="feedback"),
                             self.from_text()]}

    @staticmethod
    def cuisine_db() -> List[Text]:
        """Database of supported cuisines"""

        return ["caribbean",
                "chinese",
                "french",
                "greek",
                "indian",
                "italian",
                "mexican"]

    @staticmethod
    def is_int(string: Text) -> bool:
        """Check if a string is an integer"""

        try:
            int(string)
            return True
        except ValueError:
            return False

    def validate_cuisine(self,
                         value: Text,
                         dispatcher: CollectingDispatcher,
                         tracker: Tracker,
                         domain: Dict[Text, Any]) -> Optional[Text]:
        """Validate cuisine value."""

        if value.lower() in self.cuisine_db():
            # validation succeeded
            return value
        else:
            dispatcher.utter_template('utter_wrong_cuisine', tracker)
            # validation failed, set this slot to None, meaning the
            # user will be asked for the slot again
            return None

    def validate_num_people(self,
                            value: Text,
                            dispatcher: CollectingDispatcher,
                            tracker: Tracker,
                            domain: Dict[Text, Any]) -> Optional[Text]:
        """Validate num_people value."""

        if self.is_int(value) and int(value) > 0:
            return value
        else:
            dispatcher.utter_template('utter_wrong_num_people', tracker)
            # validation failed, set slot to None
            return None

    @staticmethod
    def validate_outdoor_seating(value: Text,
                                 dispatcher: CollectingDispatcher,
                                 tracker: Tracker,
                                 domain: Dict[Text, Any]) -> Any:
        """Validate outdoor_seating value."""

        if isinstance(value, str):
            if 'out' in value:
                # convert "out..." to True
                return True
            elif 'in' in value:
                # convert "in..." to False
                return False
            else:
                dispatcher.utter_template('utter_wrong_outdoor_seating',
                                          tracker)
                # validation failed, set slot to None
                return None

        else:
            # affirm/deny was picked up as T/F
            return value

    def submit(self,
               dispatcher: CollectingDispatcher,
               tracker: Tracker,
               domain: Dict[Text, Any]) -> List[Dict]:
        """Define what the form has to do
            after all required slots are filled"""

        # utter submit template
        dispatcher.utter_template('utter_submit', tracker)
        return []
<commit_msg>type fixed<commit_after># -*- coding: utf-8 -*-
from typing import Dict, Text, Any, List, Union, Optional

from rasa_core_sdk import Tracker
from rasa_core_sdk.executor import CollectingDispatcher
from rasa_core_sdk.forms import FormAction

class RestaurantForm(FormAction):
    """Example of a custom form action"""

    def name(self) -> Text:
        """Unique identifier of the form"""

        return "restaurant_form"

    @staticmethod
    def required_slots(tracker: Tracker) -> List[Text]:
        """A list of required slots that the form has to fill"""

        return ["cuisine", "num_people", "outdoor_seating",
                "preferences", "feedback"]

    def slot_mappings(self) -> Dict[Text, Union[Dict, List[Dict]]]:
        """A dictionary to map required slots to
            - an extracted entity
            - intent: value pairs
            - a whole message
            or a list of them, where a first match will be picked"""

        return {"cuisine": self.from_entity(entity="cuisine",
                                            not_intent="chitchat"),
                "num_people": [self.from_entity(entity="num_people",
                                                intent=["inform",
                                                        "request_restaurant"]),
                               self.from_entity(entity="number")],
                "outdoor_seating": [self.from_entity(entity="seating"),
                                    self.from_intent(intent='affirm',
                                                     value=True),
                                    self.from_intent(intent='deny',
                                                     value=False)],
                "preferences": [self.from_intent(intent='deny',
                                                 value="no additional "
                                                       "preferences"),
                                self.from_text(not_intent="affirm")],
                "feedback": [self.from_entity(entity="feedback"),
                             self.from_text()]}

    @staticmethod
    def cuisine_db() -> List[Text]:
        """Database of supported cuisines"""

        return ["caribbean",
                "chinese",
                "french",
                "greek",
                "indian",
                "italian",
                "mexican"]

    @staticmethod
    def is_int(string: Text) -> bool:
        """Check if a string is an integer"""

        try:
            int(string)
            return True
        except ValueError:
            return False

    def validate_cuisine(self,
                         value: Text,
                         dispatcher: CollectingDispatcher,
                         tracker: Tracker,
                         domain: Dict[Text, Any]) -> Optional[Text]:
        """Validate cuisine value."""

        if value.lower() in self.cuisine_db():
            # validation succeeded
            return value
        else:
            dispatcher.utter_template('utter_wrong_cuisine', tracker)
            # validation failed, set this slot to None, meaning the
            # user will be asked for the slot again
            return None

    def validate_num_people(self,
                            value: Text,
                            dispatcher: CollectingDispatcher,
                            tracker: Tracker,
                            domain: Dict[Text, Any]) -> Optional[Text]:
        """Validate num_people value."""

        if self.is_int(value) and int(value) > 0:
            return value
        else:
            dispatcher.utter_template('utter_wrong_num_people', tracker)
            # validation failed, set slot to None
            return None

    @staticmethod
    def validate_outdoor_seating(value: Text,
                                 dispatcher: CollectingDispatcher,
                                 tracker: Tracker,
                                 domain: Dict[Text, Any]) -> Optional[bool]:
        """Validate outdoor_seating value."""

        if isinstance(value, str):
            if 'out' in value:
                # convert "out..." to True
                return True
            elif 'in' in value:
                # convert "in..." to False
                return False
            else:
                dispatcher.utter_template('utter_wrong_outdoor_seating',
                                          tracker)
                # validation failed, set slot to None
                return None

        else:
            # affirm/deny was picked up as T/F
            return value

    def submit(self,
               dispatcher: CollectingDispatcher,
               tracker: Tracker,
               domain: Dict[Text, Any]) -> List[Dict]:
        """Define what the form has to do
            after all required slots are filled"""

        # utter submit template
        dispatcher.utter_template('utter_submit', tracker)
        return []
<|endoftext|>


<commit_before>#!/usr/bin/env python

from os.path import join, dirname

from cloudify import ctx

ctx.download_resource(
    join('components', 'utils.py'),
    join(dirname(__file__), 'utils.py'))
import utils  # NOQA

SERVICE_NAME = 'mgmtworker'
AMQP_SSL_PORT = '5671'

# Some runtime properties to be used in teardown
runtime_props = ctx.instance.runtime_properties
runtime_props['service_name'] = SERVICE_NAME

HOME_DIR = join('/opt', SERVICE_NAME)
LOG_DIR = join(utils.BASE_LOG_DIR, SERVICE_NAME)
runtime_props['files_to_remove'] = [HOME_DIR, LOG_DIR]

# Used in the service template
runtime_props['home_dir'] = HOME_DIR
runtime_props['log_dir'] = LOG_DIR
CLOUDIFY_USER = utils.CLOUDIFY_USER
CLOUDIFY_GROUP = utils.CLOUDIFY_GROUP

ctx_properties = utils.ctx_factory.create(SERVICE_NAME)

def allow_snapshot_db_secrets_fixer():
    command = '/opt/mgmtworker/resources/cloudify/fix_snapshot_ssh_db'
    description = 'snapshot_ssh_db_fix'
    utils.allow_user_to_sudo_command(command, description)

def _install_optional(mgmtworker_venv):
    rest_props = utils.ctx_factory.get('restservice')
    rest_client_source_url = rest_props['rest_client_module_source_url']
    plugins_common_source_url = rest_props['plugins_common_module_source_url']
    script_plugin_source_url = rest_props['script_plugin_module_source_url']
    rest_service_source_url = rest_props['rest_service_module_source_url']
    agent_source_url = rest_props['agent_module_source_url']

    # this allows to upgrade modules if necessary.
    ctx.logger.info('Installing Optional Packages if supplied...')
    if rest_client_source_url:
        utils.install_python_package(rest_client_source_url, mgmtworker_venv)
    if plugins_common_source_url:
        utils.install_python_package(
            plugins_common_source_url, mgmtworker_venv)
    if script_plugin_source_url:
        utils.install_python_package(script_plugin_source_url, mgmtworker_venv)
    if agent_source_url:
        utils.install_python_package(agent_source_url, mgmtworker_venv)

    if rest_service_source_url:
        ctx.logger.info('Downloading cloudify-manager Repository...')
        manager_repo = \
            utils.download_cloudify_resource(rest_service_source_url,
                                             SERVICE_NAME)

        ctx.logger.info('Extracting Manager Repository...')
        tmp_dir = utils.untar(manager_repo, unique_tmp_dir=True)
        workflows_dir = join(tmp_dir, 'workflows')
        riemann_dir = join(tmp_dir, 'plugins/riemann-controller')

        ctx.logger.info('Installing Management Worker Plugins...')
        utils.install_python_package(riemann_dir, mgmtworker_venv)
        utils.install_python_package(workflows_dir, mgmtworker_venv)

        utils.remove(tmp_dir)

def install_mgmtworker():
    riemann_dir = '/opt/riemann'

    management_worker_rpm_source_url = \
        ctx_properties['management_worker_rpm_source_url']

    runtime_props['rabbitmq_endpoint_ip'] = utils.get_rabbitmq_endpoint_ip()

    # Fix possible injections in json of rabbit credentials
    # See json.org for string spec
    for key in ['rabbitmq_username', 'rabbitmq_password']:
        # We will not escape newlines or other control characters,
        # we will accept them breaking
        # things noisily, e.g. on newlines and backspaces.
        # TODO: add:
        # sed 's/"/\\"/' | sed 's/\\/\\\\/' | sed s-/-\\/- | sed 's/\t/\\t/'
        runtime_props[key] = ctx_properties[key]

    runtime_props['rabbitmq_ssl_enabled'] = True
    utils.set_service_as_cloudify_service(runtime_props)

    ctx.logger.info('Installing Management Worker...')
    utils.set_selinux_permissive()

    utils.copy_notice(SERVICE_NAME)
    utils.mkdir(HOME_DIR)
    utils.mkdir(join(HOME_DIR, 'config'))
    utils.mkdir(join(HOME_DIR, 'work'))
    utils.mkdir(LOG_DIR)
    utils.mkdir(riemann_dir)

    mgmtworker_venv = join(HOME_DIR, 'env')

    # this create the mgmtworker_venv and installs the relevant
    # modules into it.
    utils.yum_install(management_worker_rpm_source_url,
                      service_name=SERVICE_NAME)
    _install_optional(mgmtworker_venv)

    # Add certificate and select port, as applicable
    runtime_props['broker_cert_path'] = utils.INTERNAL_CERT_PATH
    # Use SSL port
    runtime_props['broker_port'] = AMQP_SSL_PORT

    utils.chown(CLOUDIFY_USER, CLOUDIFY_GROUP, HOME_DIR)
    utils.chown(CLOUDIFY_USER, CLOUDIFY_GROUP, LOG_DIR)
    # Changing perms on workdir and venv in case they are put outside homedir
    utils.chown(CLOUDIFY_USER, CLOUDIFY_GROUP, mgmtworker_venv)
    # Prepare riemann dir. We will change the owner to riemann later, but the
    # management worker will still need access to it
    utils.chown(CLOUDIFY_USER, CLOUDIFY_GROUP, riemann_dir)
    utils.chmod('770', riemann_dir)

    ctx.logger.info("Using broker port: {0}".format(
        ctx.instance.runtime_properties['broker_port']))

install_mgmtworker()
allow_snapshot_db_secrets_fixer()
<commit_msg>CFY-6791 Remove snapshot db secrets fixer from sudoers (#583)<commit_after>#!/usr/bin/env python

from os.path import join, dirname

from cloudify import ctx

ctx.download_resource(
    join('components', 'utils.py'),
    join(dirname(__file__), 'utils.py'))
import utils  # NOQA

SERVICE_NAME = 'mgmtworker'
AMQP_SSL_PORT = '5671'

# Some runtime properties to be used in teardown
runtime_props = ctx.instance.runtime_properties
runtime_props['service_name'] = SERVICE_NAME

HOME_DIR = join('/opt', SERVICE_NAME)
LOG_DIR = join(utils.BASE_LOG_DIR, SERVICE_NAME)
runtime_props['files_to_remove'] = [HOME_DIR, LOG_DIR]

# Used in the service template
runtime_props['home_dir'] = HOME_DIR
runtime_props['log_dir'] = LOG_DIR
CLOUDIFY_USER = utils.CLOUDIFY_USER
CLOUDIFY_GROUP = utils.CLOUDIFY_GROUP

ctx_properties = utils.ctx_factory.create(SERVICE_NAME)

def _install_optional(mgmtworker_venv):
    rest_props = utils.ctx_factory.get('restservice')
    rest_client_source_url = rest_props['rest_client_module_source_url']
    plugins_common_source_url = rest_props['plugins_common_module_source_url']
    script_plugin_source_url = rest_props['script_plugin_module_source_url']
    rest_service_source_url = rest_props['rest_service_module_source_url']
    agent_source_url = rest_props['agent_module_source_url']

    # this allows to upgrade modules if necessary.
    ctx.logger.info('Installing Optional Packages if supplied...')
    if rest_client_source_url:
        utils.install_python_package(rest_client_source_url, mgmtworker_venv)
    if plugins_common_source_url:
        utils.install_python_package(
            plugins_common_source_url, mgmtworker_venv)
    if script_plugin_source_url:
        utils.install_python_package(script_plugin_source_url, mgmtworker_venv)
    if agent_source_url:
        utils.install_python_package(agent_source_url, mgmtworker_venv)

    if rest_service_source_url:
        ctx.logger.info('Downloading cloudify-manager Repository...')
        manager_repo = \
            utils.download_cloudify_resource(rest_service_source_url,
                                             SERVICE_NAME)

        ctx.logger.info('Extracting Manager Repository...')
        tmp_dir = utils.untar(manager_repo, unique_tmp_dir=True)
        workflows_dir = join(tmp_dir, 'workflows')
        riemann_dir = join(tmp_dir, 'plugins/riemann-controller')

        ctx.logger.info('Installing Management Worker Plugins...')
        utils.install_python_package(riemann_dir, mgmtworker_venv)
        utils.install_python_package(workflows_dir, mgmtworker_venv)

        utils.remove(tmp_dir)

def install_mgmtworker():
    riemann_dir = '/opt/riemann'

    management_worker_rpm_source_url = \
        ctx_properties['management_worker_rpm_source_url']

    runtime_props['rabbitmq_endpoint_ip'] = utils.get_rabbitmq_endpoint_ip()

    # Fix possible injections in json of rabbit credentials
    # See json.org for string spec
    for key in ['rabbitmq_username', 'rabbitmq_password']:
        # We will not escape newlines or other control characters,
        # we will accept them breaking
        # things noisily, e.g. on newlines and backspaces.
        # TODO: add:
        # sed 's/"/\\"/' | sed 's/\\/\\\\/' | sed s-/-\\/- | sed 's/\t/\\t/'
        runtime_props[key] = ctx_properties[key]

    runtime_props['rabbitmq_ssl_enabled'] = True
    utils.set_service_as_cloudify_service(runtime_props)

    ctx.logger.info('Installing Management Worker...')
    utils.set_selinux_permissive()

    utils.copy_notice(SERVICE_NAME)
    utils.mkdir(HOME_DIR)
    utils.mkdir(join(HOME_DIR, 'config'))
    utils.mkdir(join(HOME_DIR, 'work'))
    utils.mkdir(LOG_DIR)
    utils.mkdir(riemann_dir)

    mgmtworker_venv = join(HOME_DIR, 'env')

    # this create the mgmtworker_venv and installs the relevant
    # modules into it.
    utils.yum_install(management_worker_rpm_source_url,
                      service_name=SERVICE_NAME)
    _install_optional(mgmtworker_venv)

    # Add certificate and select port, as applicable
    runtime_props['broker_cert_path'] = utils.INTERNAL_CERT_PATH
    # Use SSL port
    runtime_props['broker_port'] = AMQP_SSL_PORT

    utils.chown(CLOUDIFY_USER, CLOUDIFY_GROUP, HOME_DIR)
    utils.chown(CLOUDIFY_USER, CLOUDIFY_GROUP, LOG_DIR)
    # Changing perms on workdir and venv in case they are put outside homedir
    utils.chown(CLOUDIFY_USER, CLOUDIFY_GROUP, mgmtworker_venv)
    # Prepare riemann dir. We will change the owner to riemann later, but the
    # management worker will still need access to it
    utils.chown(CLOUDIFY_USER, CLOUDIFY_GROUP, riemann_dir)
    utils.chmod('770', riemann_dir)

    ctx.logger.info("Using broker port: {0}".format(
        ctx.instance.runtime_properties['broker_port']))

install_mgmtworker()
<|endoftext|>


<commit_before># pylint: disable=C0111
from urllib.parse import urlparse, urlunparse, parse_qs

import requests
from collections import namedtuple
from unittest.mock import patch, Mock, ANY, call as mock_call

from django.test import TestCase
from django_rq import get_connection
from rest_framework.reverse import reverse
from rq.job import Job

from converters.boundaries import BBox
from converters.converter import Options
from converters.gis_converter.extract.excerpt import Excerpt
from manager.rq_helper import rq_enqueue_with_settings
from shared import ConversionProgress
from worker import converter_job
from tests.redis_test_helpers import perform_all_jobs_sync
from worker.converter_job import Notifier

class WorkerTest(TestCase):
    pbf_file_path = '/some_path/to/pbf.pbf'

    @patch('os.remove')
    @patch.object(Excerpt, '_copy_statistics_file_to_format_dir', return_value=None)
    @patch.object(Excerpt, '_get_statistics', return_value=None)
    @patch.object(Excerpt, '_export_from_db_to_format', return_value=None)
    @patch('converters.gis_converter.bootstrap.bootstrap.boostrap', return_value=None)
    @patch('converters.osm_cutter.cut_osm_extent', return_value=pbf_file_path)
    def test_convert_calls_cut_osm_extent_and_bootstrap(  # pylint: disable=W0613
            self,
            cut_osm_extent_mock, bootstrap_mock, _export_from_db_to_format_mock,
            *args, **kwargs
    ):
        geometry = BBox(29.525547623634335, 40.77546776498174, 29.528980851173397, 40.77739734768811)
        format_options = Options(output_formats=['fgdb', 'spatialite', 'shp', 'gpkg'])
        converter_job.convert(
            geometry=geometry, format_options=format_options, callback_url=None, output_directory='/tmp/',
            protocol='http'
        )
        cut_osm_extent_mock.assert_called_once_with(geometry)
        bootstrap_mock.assert_called_once_with(self.pbf_file_path)
        self.assert_mock_has_exactly_calls(
            _export_from_db_to_format_mock,
            [mock_call(ANY, f) for f in format_options.get_output_formats()],
            any_order=True,
        )

    def assert_mock_has_exactly_calls(self, mocked_function, calls, any_order=False):
        assert_calls_equal = self.assertCountEqual if any_order else self.assertEqual
        assert_calls_equal(mocked_function.mock_calls, list(calls))

    def test_set_progress_on_job_sets_the_progress(self):
        job = rq_enqueue_with_settings(converter_job.set_progress_on_job, ConversionProgress.SUCCESSFUL)
        perform_all_jobs_sync()
        job_fetched = Job.fetch(job.id, connection=get_connection())
        self.assertDictEqual(job_fetched.meta, {'progress': ConversionProgress.SUCCESSFUL})

class ConvertTest(TestCase):
    pbf_file_path = '/some_path/to/pbf.pbf'
    mocked_job_id = '0' * 36
    rq_job_stub = namedtuple('RQJob', ['id', 'meta', 'save'])(
        id=mocked_job_id,
        meta={'progress': ''},
        save=Mock(),
    )

    @patch('os.remove')
    @patch.object(Excerpt, '_copy_statistics_file_to_format_dir', return_value=None)
    @patch.object(Excerpt, '_get_statistics', return_value=None)
    @patch.object(Excerpt, '_export_from_db_to_format', return_value=None)
    @patch('converters.gis_converter.bootstrap.bootstrap.boostrap', return_value=None)
    @patch('converters.osm_cutter.cut_osm_extent', return_value=pbf_file_path)
    @patch('rq.get_current_job', return_value=rq_job_stub)
    @patch('worker.converter_job.Notifier')
    def test_convert_with_host_and_job_assembles_url_correctly(self, notifier_mock, rq_job_stub, *args, **kwargs):
        format_options = Options(output_formats=['fgdb', 'spatialite', 'shp', 'gpkg'])
        geometry, output_directory, callback_url = None, '/tmp/', None
        host = 'converter-host.example.com'
        protocol = 'http'
        converter_job.convert(geometry, format_options, output_directory, callback_url, protocol, host)
        expected_url = protocol + '://' + host + reverse(viewname='conversion_job_result-detail', kwargs={'rq_job_id': self.mocked_job_id})
        notifier_mock.assert_called_with(callback_url, expected_url)

class NotifierTest(TestCase):
    @patch.object(requests.sessions.Session, 'send')
    def test_notify_calls_url_with_status_parameter(self, requests_send_mock):
        notifier = Notifier(
            callback_url='http://osmaxx-ui.example.com/update_job_status/',
            status_url='https://osmaxx-conversion.example.com/example_job/status'
        )
        notifier.notify()
        requests_send_args, requests_send_kwargs = requests_send_mock.call_args
        request = self._get_request(*requests_send_args, **requests_send_kwargs)
        url_components = urlparse(request.url)
        self.assertEqual(urlunparse(url_components._replace(query='')), notifier.callback_url)
        self.assertEqual(parse_qs(url_components.query), dict(status=[notifier.status_url]))

    def _get_request(self, request, **kwargs):
        return request
<commit_msg>test notifier without callback URL, too<commit_after># pylint: disable=C0111
from urllib.parse import urlparse, urlunparse, parse_qs

import requests
from collections import namedtuple
from unittest.mock import patch, Mock, ANY, call as mock_call

from django.test import TestCase
from django_rq import get_connection
from rest_framework.reverse import reverse
from rq.job import Job

from converters.boundaries import BBox
from converters.converter import Options
from converters.gis_converter.extract.excerpt import Excerpt
from manager.rq_helper import rq_enqueue_with_settings
from shared import ConversionProgress
from worker import converter_job
from tests.redis_test_helpers import perform_all_jobs_sync
from worker.converter_job import Notifier

class WorkerTest(TestCase):
    pbf_file_path = '/some_path/to/pbf.pbf'

    @patch('os.remove')
    @patch.object(Excerpt, '_copy_statistics_file_to_format_dir', return_value=None)
    @patch.object(Excerpt, '_get_statistics', return_value=None)
    @patch.object(Excerpt, '_export_from_db_to_format', return_value=None)
    @patch('converters.gis_converter.bootstrap.bootstrap.boostrap', return_value=None)
    @patch('converters.osm_cutter.cut_osm_extent', return_value=pbf_file_path)
    def test_convert_calls_cut_osm_extent_and_bootstrap(  # pylint: disable=W0613
            self,
            cut_osm_extent_mock, bootstrap_mock, _export_from_db_to_format_mock,
            *args, **kwargs
    ):
        geometry = BBox(29.525547623634335, 40.77546776498174, 29.528980851173397, 40.77739734768811)
        format_options = Options(output_formats=['fgdb', 'spatialite', 'shp', 'gpkg'])
        converter_job.convert(
            geometry=geometry, format_options=format_options, callback_url=None, output_directory='/tmp/',
            protocol='http'
        )
        cut_osm_extent_mock.assert_called_once_with(geometry)
        bootstrap_mock.assert_called_once_with(self.pbf_file_path)
        self.assert_mock_has_exactly_calls(
            _export_from_db_to_format_mock,
            [mock_call(ANY, f) for f in format_options.get_output_formats()],
            any_order=True,
        )

    def assert_mock_has_exactly_calls(self, mocked_function, calls, any_order=False):
        assert_calls_equal = self.assertCountEqual if any_order else self.assertEqual
        assert_calls_equal(mocked_function.mock_calls, list(calls))

    def test_set_progress_on_job_sets_the_progress(self):
        job = rq_enqueue_with_settings(converter_job.set_progress_on_job, ConversionProgress.SUCCESSFUL)
        perform_all_jobs_sync()
        job_fetched = Job.fetch(job.id, connection=get_connection())
        self.assertDictEqual(job_fetched.meta, {'progress': ConversionProgress.SUCCESSFUL})

class ConvertTest(TestCase):
    pbf_file_path = '/some_path/to/pbf.pbf'
    mocked_job_id = '0' * 36
    rq_job_stub = namedtuple('RQJob', ['id', 'meta', 'save'])(
        id=mocked_job_id,
        meta={'progress': ''},
        save=Mock(),
    )

    @patch('os.remove')
    @patch.object(Excerpt, '_copy_statistics_file_to_format_dir', return_value=None)
    @patch.object(Excerpt, '_get_statistics', return_value=None)
    @patch.object(Excerpt, '_export_from_db_to_format', return_value=None)
    @patch('converters.gis_converter.bootstrap.bootstrap.boostrap', return_value=None)
    @patch('converters.osm_cutter.cut_osm_extent', return_value=pbf_file_path)
    @patch('rq.get_current_job', return_value=rq_job_stub)
    @patch('worker.converter_job.Notifier')
    def test_convert_with_host_and_job_assembles_url_correctly(self, notifier_mock, rq_job_stub, *args, **kwargs):
        format_options = Options(output_formats=['fgdb', 'spatialite', 'shp', 'gpkg'])
        geometry, output_directory, callback_url = None, '/tmp/', None
        host = 'converter-host.example.com'
        protocol = 'http'
        converter_job.convert(geometry, format_options, output_directory, callback_url, protocol, host)
        expected_url = protocol + '://' + host + reverse(viewname='conversion_job_result-detail', kwargs={'rq_job_id': self.mocked_job_id})
        notifier_mock.assert_called_with(callback_url, expected_url)

class NotifierTest(TestCase):
    @patch.object(requests.sessions.Session, 'send')
    def test_notify_calls_url_with_status_parameter(self, requests_send_mock):
        notifier = Notifier(
            callback_url='http://osmaxx-ui.example.com/update_job_status/',
            status_url='https://osmaxx-conversion.example.com/example_job/status'
        )
        notifier.notify()
        requests_send_args, requests_send_kwargs = requests_send_mock.call_args
        request = self._get_request(*requests_send_args, **requests_send_kwargs)
        url_components = urlparse(request.url)
        self.assertEqual(urlunparse(url_components._replace(query='')), notifier.callback_url)
        self.assertEqual(parse_qs(url_components.query), dict(status=[notifier.status_url]))

    @patch.object(requests.sessions.Session, 'send')
    def test_notify_without_callback_url_sends_no_request(self, requests_send_mock):
        notifier = Notifier(
            callback_url=None,
            status_url='https://osmaxx-conversion.example.com/example_job/status'
        )
        notifier.notify()
        self.assertEqual(requests_send_mock.call_count, 0)

    def _get_request(self, request, **kwargs):
        return request
<|endoftext|>


<commit_before># Copyright (c) 2006-2009 Sippy Software, Inc. All rights reserved.
#
# This file is part of SIPPY, a free RFC3261 SIP stack and B2BUA.
#
# SIPPY is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# For a license to use the SIPPY software under conditions
# other than those described here, or to purchase support for this
# software, please contact Sippy Software, Inc. by e-mail at the
# following addresses: sales@sippysoft.com.
#
# SIPPY is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA.

from twisted.internet import reactor
from errno import ECONNRESET, ENOTCONN, ESHUTDOWN, EWOULDBLOCK, ENOBUFS, EAGAIN, \
  EINTR
from datetime import datetime
from time import sleep
from threading import Thread, Condition
import socket
import sys, traceback

MAX_WORKERS = 30

class AsyncSender(Thread):
    userv = None

    def __init__(self, userv):
        Thread.__init__(self)
        self.userv = userv
        self.setDaemon(True)
        self.start()

    def run(self):
        while True:
            self.userv.wi_available.acquire()
            while len(self.userv.wi) == 0:
                self.userv.wi_available.wait()
            wi = self.userv.wi.pop(0)
            if wi == None:
                # Shutdown request, relay it further
                self.userv.wi.append(None)
                self.userv.wi_available.notify()
            self.userv.wi_available.release()
            if wi == None:
                break
            data, address = wi
            try:
                ai = socket.getaddrinfo(address[0], None, self.userv.family)
            except:
                continue
            if self.userv.family == socket.AF_INET:
                address = (ai[0][4][0], address[1])
            else:
                address = (ai[0][4][0], address[1], ai[0][4][2], ai[0][4][3])
            for i in range(0, 20):
                try:
                    if self.userv.skt.sendto(data, address) == len(data):
                        break
                except socket.error, why:
                    if why[0] not in (EWOULDBLOCK, ENOBUFS, EAGAIN):
                        break
                sleep(0.01)
        self.userv = None

class AsyncReceiver(Thread):
    userv = None

    def __init__(self, userv):
        Thread.__init__(self)
        self.userv = userv
        self.setDaemon(True)
        self.start()

    def run(self):
        maxemptydata = 100
        while True:
            try:
                data, address = self.userv.skt.recvfrom(8192)
                if not data:
                    # Ugly hack to detect socket being closed under us on Linux.
                    # The problem is that even call on non-closed socket can
                    # sometimes return empty data buffer, making AsyncReceiver
                    # to exit prematurely.
                    maxemptydata -= 1
                    if maxemptydata == 0:
                        break
                    continue
                else:
                    maxemptydata = 100
            except Exception, why:
                if isinstance(why, socket.error) and why[0] in (ECONNRESET, ENOTCONN, ESHUTDOWN):
                    break
                if isinstance(why, socket.error) and why[0] in (EINTR,):
                    continue
                else:
                    print datetime.now(), 'Udp_server: unhandled exception when receiving incoming data'
                    print '-' * 70
                    traceback.print_exc(file = sys.stdout)
                    print '-' * 70
                    sys.stdout.flush()
                    sleep(1)
                    continue
            if self.userv.family == socket.AF_INET6:
                address = ('[%s]' % address[0], address[1])
            reactor.callFromThread(self.userv.handle_read, data, address)
        self.userv = None

_DEFAULT_FLAGS = socket.SO_REUSEADDR
if hasattr(socket, 'SO_REUSEPORT'):
    _DEFAULT_FLAGS |= socket.SO_REUSEPORT

class Udp_server(object):
    skt = None
    family = None
    data_callback = None
    laddress = None
    sendqueue = None
    stats = None
    wi_available = None
    wi = None

    def __init__(self, global_config, address, data_callback, family = None, \
      flags = _DEFAULT_FLAGS):
        self.laddress = address
        if family == None:
            if address != None and address[0].startswith('['):
                family = socket.AF_INET6
                address = (address[0][1:-1], address[1])
            else:
                family = socket.AF_INET
        self.family = family
        self.skt = socket.socket(family, socket.SOCK_DGRAM)
        if address != None:
            ai = socket.getaddrinfo(address[0], None, family)
            if family == socket.AF_INET:
                address = (ai[0][4][0], address[1])
            else:
                address = (ai[0][4][0], address[1], ai[0][4][2], ai[0][4][3])
            if (flags & socket.SO_REUSEADDR) != 0:
                self.skt.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
            if hasattr(socket, 'SO_REUSEPORT') and \
              (flags & socket.SO_REUSEPORT) != 0:
                self.skt.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
            self.skt.bind(address)
        self.data_callback = data_callback
        self.sendqueue = []
        self.stats = [0, 0, 0]
        self.wi_available = Condition()
        self.wi = []
        for i in range(0, MAX_WORKERS):
            AsyncSender(self)
            AsyncReceiver(self)

    def send_to(self, data, address):
        if self.family == socket.AF_INET6:
            if not address[0].startswith('['):
                raise Exception('Invalid IPv6 address: %s' % address[0])
            address = (address[0][1:-1], address[1])
        self.wi_available.acquire()
        self.wi.append((data, address))
        self.wi_available.notify()
        self.wi_available.release()

    def handle_read(self, data, address):
        self.stats[2] += 1
        try:
            self.data_callback(data, address, self)
        except:
            print datetime.now(), 'Udp_server: unhandled exception when processing incoming data'
            print '-' * 70
            traceback.print_exc(file = sys.stdout)
            print '-' * 70
            sys.stdout.flush()

    def shutdown(self):
        self.skt.shutdown(socket.SHUT_RDWR)
        self.wi_available.acquire()
        self.wi.append(None)
        self.wi_available.notify()
        self.wi_available.release()
        self.data_callback = None

if __name__ == '__main__':
    from sys import exit
    npongs = 2

    def ping_received(data, address, udp_server):
        print 'ping_received'
        if not (data == 'ping!' and address == ('127.0.0.1', 54321)):
            exit(1)
        udp_server.send_to('pong!', address)

    def pong_received(data, address, udp_server):
        print 'pong_received'
        if not (data == 'pong!' and address == ('127.0.0.1', 12345)):
            exit(1)
        global npongs
        npongs -= 1
        if npongs == 0:
            reactor.stop()

    def ping_received6(data, address, udp_server):
        print 'ping_received6', address
        if not (data == 'ping!' and address == ('[::1]', 54321)):
            exit(1)
        udp_server.send_to('pong!', address)

    def pong_received6(data, address, udp_server):
        print 'pong_received6', address
        if not (data == 'pong!' and address == ('[::1]', 12345)):
            exit(1)
        global npongs
        npongs -= 1
        if npongs == 0:
            reactor.stop()

    udp_server_ping = Udp_server({}, ('127.0.0.1', 12345), ping_received)
    udp_server_pong = Udp_server({}, ('127.0.0.1', 54321), pong_received)
    udp_server_pong.send_to('ping!', ('127.0.0.1', 12345))
    udp_server_ping6 = Udp_server({}, ('[::1]', 12345), ping_received6)
    udp_server_pong6 = Udp_server({}, ('::1', 54321), pong_received6, socket.AF_INET6)
    udp_server_pong6.send_to('ping!', ('[::1]', 12345))
    reactor.run()
<commit_msg>Properly join() worker threads on shutdown().<commit_after># Copyright (c) 2006-2009 Sippy Software, Inc. All rights reserved.
#
# This file is part of SIPPY, a free RFC3261 SIP stack and B2BUA.
#
# SIPPY is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# For a license to use the SIPPY software under conditions
# other than those described here, or to purchase support for this
# software, please contact Sippy Software, Inc. by e-mail at the
# following addresses: sales@sippysoft.com.
#
# SIPPY is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA.

from twisted.internet import reactor
from errno import ECONNRESET, ENOTCONN, ESHUTDOWN, EWOULDBLOCK, ENOBUFS, EAGAIN, \
  EINTR
from datetime import datetime
from time import sleep
from threading import Thread, Condition
import socket
import sys, traceback

MAX_WORKERS = 30

class AsyncSender(Thread):
    userv = None

    def __init__(self, userv):
        Thread.__init__(self)
        self.userv = userv
        self.setDaemon(True)
        self.start()

    def run(self):
        while True:
            self.userv.wi_available.acquire()
            while len(self.userv.wi) == 0:
                self.userv.wi_available.wait()
            wi = self.userv.wi.pop(0)
            if wi == None:
                # Shutdown request, relay it further
                self.userv.wi.append(None)
                self.userv.wi_available.notify()
            self.userv.wi_available.release()
            if wi == None:
                break
            data, address = wi
            try:
                ai = socket.getaddrinfo(address[0], None, self.userv.family)
            except:
                continue
            if self.userv.family == socket.AF_INET:
                address = (ai[0][4][0], address[1])
            else:
                address = (ai[0][4][0], address[1], ai[0][4][2], ai[0][4][3])
            for i in range(0, 20):
                try:
                    if self.userv.skt.sendto(data, address) == len(data):
                        break
                except socket.error, why:
                    if why[0] not in (EWOULDBLOCK, ENOBUFS, EAGAIN):
                        break
                sleep(0.01)
        self.userv = None

class AsyncReceiver(Thread):
    userv = None

    def __init__(self, userv):
        Thread.__init__(self)
        self.userv = userv
        self.setDaemon(True)
        self.start()

    def run(self):
        maxemptydata = 100
        while True:
            try:
                data, address = self.userv.skt.recvfrom(8192)
                if not data:
                    # Ugly hack to detect socket being closed under us on Linux.
                    # The problem is that even call on non-closed socket can
                    # sometimes return empty data buffer, making AsyncReceiver
                    # to exit prematurely.
                    maxemptydata -= 1
                    if maxemptydata == 0:
                        break
                    continue
                else:
                    maxemptydata = 100
            except Exception, why:
                if isinstance(why, socket.error) and why[0] in (ECONNRESET, ENOTCONN, ESHUTDOWN):
                    break
                if isinstance(why, socket.error) and why[0] in (EINTR,):
                    continue
                else:
                    print datetime.now(), 'Udp_server: unhandled exception when receiving incoming data'
                    print '-' * 70
                    traceback.print_exc(file = sys.stdout)
                    print '-' * 70
                    sys.stdout.flush()
                    sleep(1)
                    continue
            if self.userv.family == socket.AF_INET6:
                address = ('[%s]' % address[0], address[1])
            reactor.callFromThread(self.userv.handle_read, data, address)
        self.userv = None

_DEFAULT_FLAGS = socket.SO_REUSEADDR
if hasattr(socket, 'SO_REUSEPORT'):
    _DEFAULT_FLAGS |= socket.SO_REUSEPORT

class Udp_server(object):
    skt = None
    family = None
    data_callback = None
    laddress = None
    sendqueue = None
    stats = None
    wi_available = None
    wi = None
    asenders = None
    areceivers = None

    def __init__(self, global_config, address, data_callback, family = None, \
      flags = _DEFAULT_FLAGS):
        self.laddress = address
        if family == None:
            if address != None and address[0].startswith('['):
                family = socket.AF_INET6
                address = (address[0][1:-1], address[1])
            else:
                family = socket.AF_INET
        self.family = family
        self.skt = socket.socket(family, socket.SOCK_DGRAM)
        if address != None:
            ai = socket.getaddrinfo(address[0], None, family)
            if family == socket.AF_INET:
                address = (ai[0][4][0], address[1])
            else:
                address = (ai[0][4][0], address[1], ai[0][4][2], ai[0][4][3])
            if (flags & socket.SO_REUSEADDR) != 0:
                self.skt.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
            if hasattr(socket, 'SO_REUSEPORT') and \
              (flags & socket.SO_REUSEPORT) != 0:
                self.skt.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
            self.skt.bind(address)
        self.data_callback = data_callback
        self.sendqueue = []
        self.stats = [0, 0, 0]
        self.wi_available = Condition()
        self.wi = []
        self.asenders = []
        self.areceivers = []
        for i in range(0, MAX_WORKERS):
            self.asenders.append(AsyncSender(self))
            self.areceivers.append(AsyncReceiver(self))

    def send_to(self, data, address):
        if self.family == socket.AF_INET6:
            if not address[0].startswith('['):
                raise Exception('Invalid IPv6 address: %s' % address[0])
            address = (address[0][1:-1], address[1])
        self.wi_available.acquire()
        self.wi.append((data, address))
        self.wi_available.notify()
        self.wi_available.release()

    def handle_read(self, data, address):
        self.stats[2] += 1
        try:
            self.data_callback(data, address, self)
        except:
            print datetime.now(), 'Udp_server: unhandled exception when processing incoming data'
            print '-' * 70
            traceback.print_exc(file = sys.stdout)
            print '-' * 70
            sys.stdout.flush()

    def shutdown(self):
        self.skt.shutdown(socket.SHUT_RDWR)
        self.wi_available.acquire()
        self.wi.append(None)
        self.wi_available.notify()
        self.wi_available.release()
        self.data_callback = None
        for worker in self.asenders + self.areceivers:
            worker.join()
        self.asenders = None
        self.areceivers = None

if __name__ == '__main__':
    from sys import exit
    npongs = 2

    def ping_received(data, address, udp_server):
        print 'ping_received'
        if not (data == 'ping!' and address == ('127.0.0.1', 54321)):
            exit(1)
        udp_server.send_to('pong!', address)

    def pong_received(data, address, udp_server):
        print 'pong_received'
        if not (data == 'pong!' and address == ('127.0.0.1', 12345)):
            exit(1)
        global npongs
        npongs -= 1
        if npongs == 0:
            reactor.stop()

    def ping_received6(data, address, udp_server):
        print 'ping_received6', address
        if not (data == 'ping!' and address == ('[::1]', 54321)):
            exit(1)
        udp_server.send_to('pong!', address)

    def pong_received6(data, address, udp_server):
        print 'pong_received6', address
        if not (data == 'pong!' and address == ('[::1]', 12345)):
            exit(1)
        global npongs
        npongs -= 1
        if npongs == 0:
            reactor.stop()

    udp_server_ping = Udp_server({}, ('127.0.0.1', 12345), ping_received)
    udp_server_pong = Udp_server({}, ('127.0.0.1', 54321), pong_received)
    udp_server_pong.send_to('ping!', ('127.0.0.1', 12345))
    udp_server_ping6 = Udp_server({}, ('[::1]', 12345), ping_received6)
    udp_server_pong6 = Udp_server({}, ('::1', 54321), pong_received6, socket.AF_INET6)
    udp_server_pong6.send_to('ping!', ('[::1]', 12345))
    reactor.run()
    udp_server_ping.shutdown()
    udp_server_pong.shutdown()
    udp_server_ping6.shutdown()
    udp_server_pong6.shutdown()
<|endoftext|>