<commit_before>#!/usr/bin/env python3

# Copyright 2015 Open Source Robotics Foundation, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import collections
import os
import sys

try:
    import ros_buildfarm  # noqa
except ImportError:
    sys.exit("Could not import ros_buildfarm, please add it to the PYTHONPATH.")

try:
    import jenkinsapi  # noqa
except ImportError:
    sys.exit("Could not import jenkinsapi, please install it with pip or apt-get.")

from ros_buildfarm.jenkins import configure_job
from ros_buildfarm.jenkins import connect
from ros_buildfarm.templates import expand_template
try:
    from ros_buildfarm.templates import template_prefix_path
except ImportError:
    sys.exit("Could not import symbol from ros_buildfarm, please update ros_buildfarm.")

DEFAULT_REPOS_URL = 'https://raw.githubusercontent.com/ros2/ros2/master/ros2.repos'
DEFAULT_MAIL_RECIPIENTS = 'ros2-buildfarm@googlegroups.com'
PERIODIC_JOB_SPEC = '30 7 * * *'

template_prefix_path[:] = \
    [os.path.join(os.path.abspath(os.path.dirname(__file__)), 'job_templates')]

def main(argv=None):
    if argv is None:
        argv = sys.argv[1:]
    parser = argparse.ArgumentParser(description="Creates the ros2 jobs on Jenkins")
    parser.add_argument(
        '--jenkins-url', '-u', default='https://ci.ros2.org',
        help="Url of the jenkins server to which the job should be added")
    parser.add_argument(
        '--ci-scripts-repository', default='git@github.com:ros2/ci.git',
        help="repository from which ci scripts should be cloned"
    )
    parser.add_argument(
        '--ci-scripts-default-branch', default='master',
        help="default branch of the ci repository to get ci scripts from (this is a job parameter)"
    )
    parser.add_argument(
        '--commit', action='store_true',
        help='Actually modify the Jenkis jobs instead of only doing a dry run',
    )
    args = parser.parse_args(argv)

    data = {
        'ci_scripts_repository': args.ci_scripts_repository,
        'ci_scripts_default_branch': args.ci_scripts_default_branch,
        'default_repos_url': DEFAULT_REPOS_URL,
        'supplemental_repos_url': '',
        'time_trigger_spec': '',
        'mailer_recipients': '',
        'ignore_rmw_default': {
            'rmw_connext_dynamic_cpp',
            'rmw_fastrtps_dynamic_cpp',
            'rmw_opensplice_cpp'},
        'use_connext_debs_default': 'false',
        'use_isolated_default': 'true',
        'build_args_default': '--event-handler console_cohesion+ --cmake-args -DINSTALL_EXAMPLES=OFF -DSECURITY=ON',
        'test_args_default': '--event-handler console_direct+ --executor sequential --retest-until-pass 10',
        'enable_c_coverage_default': 'false',
        'dont_notify_every_unstable_build': 'false',
        'turtlebot_demo': False,
        'build_timeout_mins': 0,
        'ubuntu_distro': 'bionic',
    }

    jenkins = connect(args.jenkins_url)

    os_configs = {
        'linux': {
            'label_expression': 'linux',
            'shell_type': 'Shell',
        },
        'osx': {
            'label_expression': 'macos',
            'shell_type': 'Shell',
            # the current OS X agent can't handle  git@github urls
            'ci_scripts_repository': args.ci_scripts_repository.replace(
                'git@github.com:', 'https://github.com/'),
        },
        'windows': {
            'label_expression': 'windows',
            'shell_type': 'BatchFile',
            'use_isolated_default': 'false',
        },
        'linux-aarch64': {
            'label_expression': 'linux_aarch64',
            'shell_type': 'Shell',
            'ignore_rmw_default': data['ignore_rmw_default'] | {'rmw_connext_cpp', 'rmw_connext_dynamic_cpp'},
        },
    }

    jenkins_kwargs = {}
    if not args.commit:
        jenkins_kwargs['dry_run'] = True

    def create_job(os_name, job_name, template_file, additional_dict):
        job_data = dict(data)
        job_data['os_name'] = os_name
        job_data.update(os_configs[os_name])
        job_data.update(additional_dict)
        job_config = expand_template(template_file, job_data)
        configure_job(jenkins, job_name, job_config, **jenkins_kwargs)

    # configure os specific jobs
    for os_name in sorted(os_configs.keys()):
        # We need the keep the paths short on Windows, so on that platform make
        # the os_name shorter just for the jobs
        job_os_name = os_name
        if os_name == 'windows':
            job_os_name = 'win'

        # configure manual triggered job
        create_job(os_name, 'ci_' + os_name, 'ci_job.xml.em', {
            'cmake_build_type': 'None',
        })
        # configure test jobs for experimenting with job config changes
        # Keep parameters the same as the manual triggered job above.
        create_job(os_name, 'test_ci_' + os_name, 'ci_job.xml.em', {
            'cmake_build_type': 'None',
        })

        # configure a manual version of the packaging job
        create_job(os_name, 'ci_packaging_' + os_name, 'packaging_job.xml.em', {
            'cmake_build_type': 'RelWithDebInfo',
            'test_bridge_default': 'true',
            'ignore_rmw_default': {'rmw_connext_cpp', 'rmw_connext_dynamic_cpp'} if os_name =='linux-aarch64' else set(),
            'use_connext_debs_default': 'true',
        })

        # configure packaging job
        create_job(os_name, 'packaging_' + os_name, 'packaging_job.xml.em', {
            'cmake_build_type': 'RelWithDebInfo',
            'test_bridge_default': 'true',
            'time_trigger_spec': PERIODIC_JOB_SPEC,
            'mailer_recipients': DEFAULT_MAIL_RECIPIENTS,
            'ignore_rmw_default': {'rmw_connext_cpp', 'rmw_connext_dynamic_cpp'} if os_name == 'linux-aarch64' else set(),
            'use_connext_debs_default': 'true',
        })

        # create a nightly Debug packaging job on Windows
        if os_name == 'windows':
            create_job(os_name, 'packaging_' + os_name + '_debug', 'packaging_job.xml.em', {
                'cmake_build_type': 'Debug',
                'test_bridge_default': 'true',
                'time_trigger_spec': PERIODIC_JOB_SPEC,
                'mailer_recipients': DEFAULT_MAIL_RECIPIENTS,
                'ignore_rmw_default': {'rmw_connext_cpp', 'rmw_connext_dynamic_cpp'} if os_name == 'linux-aarch64' else set(),
                'use_connext_debs_default': 'true',
            })

        # configure nightly triggered job
        job_name = 'nightly_' + job_os_name + '_debug'
        if os_name == 'windows':
            job_name = job_name[:15]
        create_job(os_name, job_name, 'ci_job.xml.em', {
            'cmake_build_type': 'Debug',
            'time_trigger_spec': PERIODIC_JOB_SPEC,
            'mailer_recipients': DEFAULT_MAIL_RECIPIENTS,
        })

        # configure a manually triggered version of the coverage job
        if os_name == 'linux':
            create_job(os_name, 'ci_' + os_name + '_coverage', 'ci_job.xml.em', {
                'cmake_build_type': 'Debug',
                'enable_c_coverage_default': 'true',
            })
            create_job(os_name, 'test_' + os_name + '_coverage', 'ci_job.xml.em', {
                'cmake_build_type': 'Debug',
                'enable_c_coverage_default': 'true',
            })

        # configure nightly coverage job on x86 Linux only
        if os_name == 'linux':
            create_job(os_name, 'nightly_' + os_name + '_coverage', 'ci_job.xml.em', {
                'cmake_build_type': 'Debug',
                'enable_c_coverage_default': 'true',
                'time_trigger_spec': PERIODIC_JOB_SPEC,
                'mailer_recipients': DEFAULT_MAIL_RECIPIENTS,
            })

        # configure nightly triggered job using opensplice
        job_name = 'nightly_' + job_os_name + '_extra_rmw' + '_release'
        create_job(os_name, job_name, 'ci_job.xml.em', {
            'cmake_build_type': 'Release',
            'time_trigger_spec': PERIODIC_JOB_SPEC,
            'mailer_recipients': DEFAULT_MAIL_RECIPIENTS,
            'ignore_rmw_default': {
                'rmw_connext_cpp',
                'rmw_connext_dynamic_cpp'},
        })

        # configure nightly triggered job
        job_name = 'nightly_' + job_os_name + '_release'
        if os_name == 'windows':
            job_name = job_name[:15]
        create_job(os_name, job_name, 'ci_job.xml.em', {
            'cmake_build_type': 'Release',
            'time_trigger_spec': PERIODIC_JOB_SPEC,
            'mailer_recipients': DEFAULT_MAIL_RECIPIENTS,
        })

        # configure nightly triggered job with repeated testing
        job_name = 'nightly_' + job_os_name + '_repeated'
        if os_name == 'windows':
            job_name = job_name[:15]
        create_job(os_name, job_name, 'ci_job.xml.em', {
            'cmake_build_type': 'None',
            'time_trigger_spec': PERIODIC_JOB_SPEC,
            'mailer_recipients': DEFAULT_MAIL_RECIPIENTS,
            'test_args_default': '--event-handler console_direct+ --executor sequential --retest-until-fail 10 --ctest-args -LE linter --pytest-args -m "not linter"',
        })

        # configure turtlebot jobs on Linux only for now
        if os_name in ['linux', 'linux-aarch64']:
            create_job(os_name, 'ci_turtlebot-demo_' + os_name, 'ci_job.xml.em', {
                'cmake_build_type': 'None',
                'turtlebot_demo': True,
                'supplemental_repos_url': 'https://raw.githubusercontent.com/ros2/turtlebot2_demo/master/turtlebot2_demo.repos',
            })
            create_job(os_name, 'nightly_turtlebot-demo_' + os_name + '_release', 'ci_job.xml.em', {
                'cmake_build_type': 'Release',
                'turtlebot_demo': True,
                'supplemental_repos_url': 'https://raw.githubusercontent.com/ros2/turtlebot2_demo/master/turtlebot2_demo.repos',
                'time_trigger_spec': PERIODIC_JOB_SPEC,
                'mailer_recipients': DEFAULT_MAIL_RECIPIENTS,
            })

    for os_name in ['linux', 'linux-aarch64']:
        # configure a nightly triggered job for xenial using all RMW implementations
        ubuntu_distro = 'xenial'
        job_name = 'nightly_{0}_{1}_release'.format(ubuntu_distro, os_name)
        create_job(os_name, job_name, 'ci_job.xml.em', {
            'cmake_build_type': 'Release',
            'time_trigger_spec': PERIODIC_JOB_SPEC,
            'mailer_recipients': DEFAULT_MAIL_RECIPIENTS,
            'ignore_rmw_default': {'rmw_connext_cpp', 'rmw_connext_dynamic_cpp'} if os_name == 'linux-aarch64' else set(),
            'ubuntu_distro': ubuntu_distro,
        })

        ubuntu_distro = 'xenial'
        # configure a nightly xenial packaging job
        job_name = 'packaging_{0}_{1}'.format(ubuntu_distro, os_name)
        create_job(os_name, job_name, 'packaging_job.xml.em', {
            'cmake_build_type': 'RelWithDebInfo',
            'test_bridge_default': 'true',
            'time_trigger_spec': PERIODIC_JOB_SPEC,
            'mailer_recipients': DEFAULT_MAIL_RECIPIENTS,
            'ubuntu_distro': ubuntu_distro,
            'ignore_rmw_default': {'rmw_connext_cpp', 'rmw_connext_dynamic_cpp'} if os_name == 'linux-aarch64' else set(),
            'use_connext_debs_default': 'true',
        })

    # configure the launch job
    os_specific_data = collections.OrderedDict()
    for os_name in sorted(os_configs.keys()):
        os_specific_data[os_name] = dict(data)
        os_specific_data[os_name].update(os_configs[os_name])
        os_specific_data[os_name]['job_name'] = 'ci_' + os_name
    job_data = dict(data)
    job_data['ci_scripts_default_branch'] = args.ci_scripts_default_branch
    job_data['label_expression'] = 'master'
    job_data['os_specific_data'] = os_specific_data
    job_data['cmake_build_type'] = 'None'
    job_config = expand_template('ci_launcher_job.xml.em', job_data)
    configure_job(jenkins, 'ci_launcher', job_config, **jenkins_kwargs)

if __name__ == '__main__':
    main()
<commit_msg>Print package list (#213)<commit_after>#!/usr/bin/env python3

# Copyright 2015 Open Source Robotics Foundation, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import collections
import os
import sys

try:
    import ros_buildfarm  # noqa
except ImportError:
    sys.exit("Could not import ros_buildfarm, please add it to the PYTHONPATH.")

try:
    import jenkinsapi  # noqa
except ImportError:
    sys.exit("Could not import jenkinsapi, please install it with pip or apt-get.")

from ros_buildfarm.jenkins import configure_job
from ros_buildfarm.jenkins import connect
from ros_buildfarm.templates import expand_template
try:
    from ros_buildfarm.templates import template_prefix_path
except ImportError:
    sys.exit("Could not import symbol from ros_buildfarm, please update ros_buildfarm.")

DEFAULT_REPOS_URL = 'https://raw.githubusercontent.com/ros2/ros2/master/ros2.repos'
DEFAULT_MAIL_RECIPIENTS = 'ros2-buildfarm@googlegroups.com'
PERIODIC_JOB_SPEC = '30 7 * * *'

template_prefix_path[:] = \
    [os.path.join(os.path.abspath(os.path.dirname(__file__)), 'job_templates')]

def main(argv=None):
    if argv is None:
        argv = sys.argv[1:]
    parser = argparse.ArgumentParser(description="Creates the ros2 jobs on Jenkins")
    parser.add_argument(
        '--jenkins-url', '-u', default='https://ci.ros2.org',
        help="Url of the jenkins server to which the job should be added")
    parser.add_argument(
        '--ci-scripts-repository', default='git@github.com:ros2/ci.git',
        help="repository from which ci scripts should be cloned"
    )
    parser.add_argument(
        '--ci-scripts-default-branch', default='master',
        help="default branch of the ci repository to get ci scripts from (this is a job parameter)"
    )
    parser.add_argument(
        '--commit', action='store_true',
        help='Actually modify the Jenkis jobs instead of only doing a dry run',
    )
    args = parser.parse_args(argv)

    data = {
        'ci_scripts_repository': args.ci_scripts_repository,
        'ci_scripts_default_branch': args.ci_scripts_default_branch,
        'default_repos_url': DEFAULT_REPOS_URL,
        'supplemental_repos_url': '',
        'time_trigger_spec': '',
        'mailer_recipients': '',
        'ignore_rmw_default': {
            'rmw_connext_dynamic_cpp',
            'rmw_fastrtps_dynamic_cpp',
            'rmw_opensplice_cpp'},
        'use_connext_debs_default': 'false',
        'use_isolated_default': 'true',
        'build_args_default': '--event-handlers console_cohesion+ console_package_list+ --cmake-args -DINSTALL_EXAMPLES=OFF -DSECURITY=ON',
        'test_args_default': '--event-handlers console_direct+ --executor sequential --retest-until-pass 10',
        'enable_c_coverage_default': 'false',
        'dont_notify_every_unstable_build': 'false',
        'turtlebot_demo': False,
        'build_timeout_mins': 0,
        'ubuntu_distro': 'bionic',
    }

    jenkins = connect(args.jenkins_url)

    os_configs = {
        'linux': {
            'label_expression': 'linux',
            'shell_type': 'Shell',
        },
        'osx': {
            'label_expression': 'macos',
            'shell_type': 'Shell',
            # the current OS X agent can't handle  git@github urls
            'ci_scripts_repository': args.ci_scripts_repository.replace(
                'git@github.com:', 'https://github.com/'),
        },
        'windows': {
            'label_expression': 'windows',
            'shell_type': 'BatchFile',
            'use_isolated_default': 'false',
        },
        'linux-aarch64': {
            'label_expression': 'linux_aarch64',
            'shell_type': 'Shell',
            'ignore_rmw_default': data['ignore_rmw_default'] | {'rmw_connext_cpp', 'rmw_connext_dynamic_cpp'},
        },
    }

    jenkins_kwargs = {}
    if not args.commit:
        jenkins_kwargs['dry_run'] = True

    def create_job(os_name, job_name, template_file, additional_dict):
        job_data = dict(data)
        job_data['os_name'] = os_name
        job_data.update(os_configs[os_name])
        job_data.update(additional_dict)
        job_config = expand_template(template_file, job_data)
        configure_job(jenkins, job_name, job_config, **jenkins_kwargs)

    # configure os specific jobs
    for os_name in sorted(os_configs.keys()):
        # We need the keep the paths short on Windows, so on that platform make
        # the os_name shorter just for the jobs
        job_os_name = os_name
        if os_name == 'windows':
            job_os_name = 'win'

        # configure manual triggered job
        create_job(os_name, 'ci_' + os_name, 'ci_job.xml.em', {
            'cmake_build_type': 'None',
        })
        # configure test jobs for experimenting with job config changes
        # Keep parameters the same as the manual triggered job above.
        create_job(os_name, 'test_ci_' + os_name, 'ci_job.xml.em', {
            'cmake_build_type': 'None',
        })

        # configure a manual version of the packaging job
        create_job(os_name, 'ci_packaging_' + os_name, 'packaging_job.xml.em', {
            'cmake_build_type': 'RelWithDebInfo',
            'test_bridge_default': 'true',
            'ignore_rmw_default': {'rmw_connext_cpp', 'rmw_connext_dynamic_cpp'} if os_name =='linux-aarch64' else set(),
            'use_connext_debs_default': 'true',
        })

        # configure packaging job
        create_job(os_name, 'packaging_' + os_name, 'packaging_job.xml.em', {
            'cmake_build_type': 'RelWithDebInfo',
            'test_bridge_default': 'true',
            'time_trigger_spec': PERIODIC_JOB_SPEC,
            'mailer_recipients': DEFAULT_MAIL_RECIPIENTS,
            'ignore_rmw_default': {'rmw_connext_cpp', 'rmw_connext_dynamic_cpp'} if os_name == 'linux-aarch64' else set(),
            'use_connext_debs_default': 'true',
        })

        # create a nightly Debug packaging job on Windows
        if os_name == 'windows':
            create_job(os_name, 'packaging_' + os_name + '_debug', 'packaging_job.xml.em', {
                'cmake_build_type': 'Debug',
                'test_bridge_default': 'true',
                'time_trigger_spec': PERIODIC_JOB_SPEC,
                'mailer_recipients': DEFAULT_MAIL_RECIPIENTS,
                'ignore_rmw_default': {'rmw_connext_cpp', 'rmw_connext_dynamic_cpp'} if os_name == 'linux-aarch64' else set(),
                'use_connext_debs_default': 'true',
            })

        # configure nightly triggered job
        job_name = 'nightly_' + job_os_name + '_debug'
        if os_name == 'windows':
            job_name = job_name[:15]
        create_job(os_name, job_name, 'ci_job.xml.em', {
            'cmake_build_type': 'Debug',
            'time_trigger_spec': PERIODIC_JOB_SPEC,
            'mailer_recipients': DEFAULT_MAIL_RECIPIENTS,
        })

        # configure a manually triggered version of the coverage job
        if os_name == 'linux':
            create_job(os_name, 'ci_' + os_name + '_coverage', 'ci_job.xml.em', {
                'cmake_build_type': 'Debug',
                'enable_c_coverage_default': 'true',
            })
            create_job(os_name, 'test_' + os_name + '_coverage', 'ci_job.xml.em', {
                'cmake_build_type': 'Debug',
                'enable_c_coverage_default': 'true',
            })

        # configure nightly coverage job on x86 Linux only
        if os_name == 'linux':
            create_job(os_name, 'nightly_' + os_name + '_coverage', 'ci_job.xml.em', {
                'cmake_build_type': 'Debug',
                'enable_c_coverage_default': 'true',
                'time_trigger_spec': PERIODIC_JOB_SPEC,
                'mailer_recipients': DEFAULT_MAIL_RECIPIENTS,
            })

        # configure nightly triggered job using opensplice
        job_name = 'nightly_' + job_os_name + '_extra_rmw' + '_release'
        create_job(os_name, job_name, 'ci_job.xml.em', {
            'cmake_build_type': 'Release',
            'time_trigger_spec': PERIODIC_JOB_SPEC,
            'mailer_recipients': DEFAULT_MAIL_RECIPIENTS,
            'ignore_rmw_default': {
                'rmw_connext_cpp',
                'rmw_connext_dynamic_cpp'},
        })

        # configure nightly triggered job
        job_name = 'nightly_' + job_os_name + '_release'
        if os_name == 'windows':
            job_name = job_name[:15]
        create_job(os_name, job_name, 'ci_job.xml.em', {
            'cmake_build_type': 'Release',
            'time_trigger_spec': PERIODIC_JOB_SPEC,
            'mailer_recipients': DEFAULT_MAIL_RECIPIENTS,
        })

        # configure nightly triggered job with repeated testing
        job_name = 'nightly_' + job_os_name + '_repeated'
        if os_name == 'windows':
            job_name = job_name[:15]
        create_job(os_name, job_name, 'ci_job.xml.em', {
            'cmake_build_type': 'None',
            'time_trigger_spec': PERIODIC_JOB_SPEC,
            'mailer_recipients': DEFAULT_MAIL_RECIPIENTS,
            'test_args_default': '--event-handlers console_direct+ --executor sequential --retest-until-fail 10 --ctest-args -LE linter --pytest-args -m "not linter"',
        })

        # configure turtlebot jobs on Linux only for now
        if os_name in ['linux', 'linux-aarch64']:
            create_job(os_name, 'ci_turtlebot-demo_' + os_name, 'ci_job.xml.em', {
                'cmake_build_type': 'None',
                'turtlebot_demo': True,
                'supplemental_repos_url': 'https://raw.githubusercontent.com/ros2/turtlebot2_demo/master/turtlebot2_demo.repos',
            })
            create_job(os_name, 'nightly_turtlebot-demo_' + os_name + '_release', 'ci_job.xml.em', {
                'cmake_build_type': 'Release',
                'turtlebot_demo': True,
                'supplemental_repos_url': 'https://raw.githubusercontent.com/ros2/turtlebot2_demo/master/turtlebot2_demo.repos',
                'time_trigger_spec': PERIODIC_JOB_SPEC,
                'mailer_recipients': DEFAULT_MAIL_RECIPIENTS,
            })

    for os_name in ['linux', 'linux-aarch64']:
        # configure a nightly triggered job for xenial using all RMW implementations
        ubuntu_distro = 'xenial'
        job_name = 'nightly_{0}_{1}_release'.format(ubuntu_distro, os_name)
        create_job(os_name, job_name, 'ci_job.xml.em', {
            'cmake_build_type': 'Release',
            'time_trigger_spec': PERIODIC_JOB_SPEC,
            'mailer_recipients': DEFAULT_MAIL_RECIPIENTS,
            'ignore_rmw_default': {'rmw_connext_cpp', 'rmw_connext_dynamic_cpp'} if os_name == 'linux-aarch64' else set(),
            'ubuntu_distro': ubuntu_distro,
        })

        ubuntu_distro = 'xenial'
        # configure a nightly xenial packaging job
        job_name = 'packaging_{0}_{1}'.format(ubuntu_distro, os_name)
        create_job(os_name, job_name, 'packaging_job.xml.em', {
            'cmake_build_type': 'RelWithDebInfo',
            'test_bridge_default': 'true',
            'time_trigger_spec': PERIODIC_JOB_SPEC,
            'mailer_recipients': DEFAULT_MAIL_RECIPIENTS,
            'ubuntu_distro': ubuntu_distro,
            'ignore_rmw_default': {'rmw_connext_cpp', 'rmw_connext_dynamic_cpp'} if os_name == 'linux-aarch64' else set(),
            'use_connext_debs_default': 'true',
        })

    # configure the launch job
    os_specific_data = collections.OrderedDict()
    for os_name in sorted(os_configs.keys()):
        os_specific_data[os_name] = dict(data)
        os_specific_data[os_name].update(os_configs[os_name])
        os_specific_data[os_name]['job_name'] = 'ci_' + os_name
    job_data = dict(data)
    job_data['ci_scripts_default_branch'] = args.ci_scripts_default_branch
    job_data['label_expression'] = 'master'
    job_data['os_specific_data'] = os_specific_data
    job_data['cmake_build_type'] = 'None'
    job_config = expand_template('ci_launcher_job.xml.em', job_data)
    configure_job(jenkins, 'ci_launcher', job_config, **jenkins_kwargs)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from argparse import ArgumentParser
import os
import pprint
import sys

from grab_post_titles import get_all_posts
from utils import *

"""
Feed in a text file to create/update a Wordpress post.

Wordpress login credentials specified by the .password file read by utils.login().

Uses grab_post_titles.get_all_posts() to check for an
existing post with the same title (case-insensitive,
includes both draft + published). If one exists, it updates
it based on the filename and contents, otherwise it creates
a new post.

e.g.

  python create_update_post '../posts/foo bar.txt' -> 'foo bar'

see http://python-wordpress-xmlrpc.readthedocs.org/en/latest/examples/posts.html
"""

def create_update_post(filen, post_status=None):
    # remove the extension from the filename to get the title,
    # e.g. '/yadda/foo bar.txt' -> 'foo bar'
    title = os.path.basename(os.path.splitext(filen)[0])

    posts = get_all_posts()
    existing_post = check_if_title_exists(posts, title)

    if existing_post:
        post = update_post(existing_post, filen, title=title, post_status=post_status, verbose=True)
    else:
        post = create_post(filen, title, post_status=post_status, verbose=True)
    created = not not existing_post
    return post, created

def check_if_title_exists(posts, title):
    existing_post = [post for post in posts
                     if post.title.lower() == title.lower()]
    if existing_post:
        assert len(existing_post) == 1
        return existing_post[0]

def display_post(post, msg, filen):
    print msg
    print pprint.pformat({'filename': filen,
                          'title': post.title,
                          'content': post.content[:20] + '...',
                          'post_status': post.post_status,
                          })

def create_post(filen, title, post_status=None, verbose=False):
    content = open(filen, 'r').read()
    post = WordPressPost()
    post.title = title
    post.content = content
    post.post_status = post_status or 'draft'
    # post.terms_names = {
    #   'post_tag': ['test', 'firstpost'],
    #   'category': ['Introductions', 'Tests']
    # }
    if verbose: display_post(post, 'Creating', filen)
    wp.call(NewPost(post))
    return post

def update_post(post, filen, title=None, post_status=None, verbose=False):
    """
    If POST_STATUS is None, does not alter the POST_STATUS,
    so if it was draft before, it'll still be draft now.
    """
    old_content = post.content
    new_content = open(filen, 'r').read()
    old_title = post.title
    new_title = title or old_title
    post.title = new_title
    post.content = new_content
    if post_status is not None: post.post_status = post_status
    if verbose: display_post(post, 'Updating', filen)
    wp.call(EditPost(post.id, post))
    # raise NotImplementedError
    return post

wp = login()

if __name__ == "__main__":

    parser = ArgumentParser(description=__doc__)
    parser.add_argument('-f', '--filename',
                        type=str,
                        required=True,
                        help='The filename to use as the basis of the post'
                        )
    parser.add_argument('--post-status',
                        type=str,
                        default=None,
                        help='If specified, will set POST_STATUS, otherwise will leave as is'
                        )
    args = vars(parser.parse_args())

    assert os.path.exists(args['filename'])

    post, created = create_update_post(filen=args['filename'],
                                       post_status=args['post_status'])

<commit_msg>Fixed bug in returning 'created'<commit_after>from argparse import ArgumentParser
import os
import pprint
import sys

from grab_post_titles import get_all_posts
from utils import *

"""
Feed in a text file to create/update a Wordpress post.

Wordpress login credentials specified by the .password file read by utils.login().

Uses grab_post_titles.get_all_posts() to check for an
existing post with the same title (case-insensitive,
includes both draft + published). If one exists, it updates
it based on the filename and contents, otherwise it creates
a new post.

e.g.

  python create_update_post '../posts/foo bar.txt' -> 'foo bar'

see http://python-wordpress-xmlrpc.readthedocs.org/en/latest/examples/posts.html
"""

def create_update_post(filen, post_status=None):
    # remove the extension from the filename to get the title,
    # e.g. '/yadda/foo bar.txt' -> 'foo bar'
    title = os.path.basename(os.path.splitext(filen)[0])

    posts = get_all_posts()
    existing_post = check_if_title_exists(posts, title)

    if existing_post:
        post = update_post(existing_post, filen, title=title, post_status=post_status, verbose=True)
    else:
        post = create_post(filen, title, post_status=post_status, verbose=True)
    created = not existing_post
    return post, created

def check_if_title_exists(posts, title):
    existing_post = [post for post in posts
                     if post.title.lower() == title.lower()]
    if existing_post:
        assert len(existing_post) == 1
        return existing_post[0]

def display_post(post, msg, filen):
    print msg
    print pprint.pformat({'filename': filen,
                          'title': post.title,
                          'content': post.content[:20] + '...',
                          'post_status': post.post_status,
                          })

def create_post(filen, title, post_status=None, verbose=False):
    content = open(filen, 'r').read()
    post = WordPressPost()
    post.title = title
    post.content = content
    post.post_status = post_status or 'draft'
    # post.terms_names = {
    #   'post_tag': ['test', 'firstpost'],
    #   'category': ['Introductions', 'Tests']
    # }
    if verbose: display_post(post, 'Creating', filen)
    wp.call(NewPost(post))
    return post

def update_post(post, filen, title=None, post_status=None, verbose=False):
    """
    If POST_STATUS is None, does not alter the POST_STATUS,
    so if it was draft before, it'll still be draft now.
    """
    old_content = post.content
    new_content = open(filen, 'r').read()
    old_title = post.title
    new_title = title or old_title
    post.title = new_title
    post.content = new_content
    if post_status is not None: post.post_status = post_status
    if verbose: display_post(post, 'Updating', filen)
    wp.call(EditPost(post.id, post))
    # raise NotImplementedError
    return post

wp = login()

if __name__ == "__main__":

    parser = ArgumentParser(description=__doc__)
    parser.add_argument('-f', '--filename',
                        type=str,
                        required=True,
                        help='The filename to use as the basis of the post'
                        )
    parser.add_argument('--post-status',
                        type=str,
                        default=None,
                        help='If specified, will set POST_STATUS, otherwise will leave as is'
                        )
    args = vars(parser.parse_args())

    assert os.path.exists(args['filename'])

    post, created = create_update_post(filen=args['filename'],
                                       post_status=args['post_status'])

<|endoftext|>


<commit_before>from __future__ import with_statement
import inspect, traceback
import logging
import sys

from django.conf import settings
from django.forms.forms import BoundField
from django.template import Context
from django.template.loader import get_template
from django.utils.html import conditional_escape
try:
    from django.utils.lru_cache import lru_cache
    def memoize(function, *args):
        return lru_cache()(function)
except:
    from django.utils.functional import memoize

from .base import KeepContext
from .compatibility import text_type, PY2

# Global field template, default template used for rendering a field.

TEMPLATE_PACK = getattr(settings, 'CRISPY_TEMPLATE_PACK', 'bootstrap')

DEBUG_VERSION_TUPLE = False  # False when production-ready
def version_tuple(version):
    '''
    convert a version string to a tuple for more accurate comparisons

    >>> version_tuple('1.7.3.final.0')
    (1, 7, 3, 'final', 0)
    >>> '11.2.3' > '2.3.4'
    False
    >>> version_tuple('11.2.3') > version_tuple('2.3.4')
    True
    '''
    return tuple([int(p) if p.isdigit() else p for p in version.split('.')])
if DEBUG_VERSION_TUPLE:
    version_tuple.debug = logging.info
else:
    version_tuple.debug = lambda *args: None

# By memoizeing we avoid loading the template every time render_field
# is called without a template
def default_field_template(template_pack=TEMPLATE_PACK):
    return get_template("%s/field.html" % template_pack)
default_field_template = memoize(default_field_template, {}, 1)

def set_hidden(widget):
    '''
    set widget to hidden

    different starting in Django 1.7, when is_hidden ceases to be an
    attribute and is determined by the input_type attribute
    '''
    if not callable(widget.is_hidden):
        widget.is_hidden = True
    elif hasattr(widget, 'input_type'):
        widget.input_type = 'hidden'

def render_field(
    field, form, form_style, context, template=None, labelclass=None,
    layout_object=None, attrs=None, template_pack=TEMPLATE_PACK,
    extra_context=None, **kwargs
):
    """
    Renders a django-crispy-forms field

    :param field: Can be a string or a Layout object like `Row`. If it's a layout
        object, we call its render method, otherwise we instantiate a BoundField
        and render it using default template 'CRISPY_TEMPLATE_PACK/field.html'
        The field is added to a list that the form holds called `rendered_fields`
        to avoid double rendering fields.
    :param form: The form/formset to which that field belongs to.
    :param form_style: A way to pass style name to the CSS framework used.
    :template: Template used for rendering the field.
    :layout_object: If passed, it points to the Layout object that is being rendered.
        We use it to store its bound fields in a list called `layout_object.bound_fields`
    :attrs: Attributes for the field's widget
    :template_pack: Name of the template pack to be used for rendering `field`
    :extra_context: Dictionary to be added to context, added variables by the layout object
    """
    logging.debug('rendering field %s from form %s' % (field, repr(form)))
    added_keys = [] if extra_context is None else extra_context.keys()
    with KeepContext(context, added_keys):
        if field is None:
            return ''

        FAIL_SILENTLY = getattr(settings, 'CRISPY_FAIL_SILENTLY', True)

        if hasattr(field, 'render'):
            return field.render(
                form, form_style, context, template_pack=template_pack,
            )
        else:
            # In Python 2 form field names cannot contain unicode characters without ASCII mapping
            if PY2:
                # This allows fields to be unicode strings, always they don't use non ASCII
                try:
                    if isinstance(field, text_type):
                        field = field.encode('ascii').decode()
                    # If `field` is not unicode then we turn it into a unicode string, otherwise doing
                    # str(field) would give no error and the field would not be resolved, causing confusion
                    else:
                        field = text_type(field)

                except (UnicodeEncodeError, UnicodeDecodeError):
                    raise Exception("Field '%s' is using forbidden unicode characters" % field)

        try:
            # Injecting HTML attributes into field's widget, Django handles rendering these
            field_instance = form.fields[field]
            if attrs is not None:
                widgets = getattr(field_instance.widget, 'widgets', [field_instance.widget])

                # We use attrs as a dictionary later, so here we make a copy
                list_attrs = attrs
                if isinstance(attrs, dict):
                    list_attrs = [attrs] * len(widgets)

                for index, (widget, attr) in enumerate(zip(widgets, list_attrs)):
                    if hasattr(field_instance.widget, 'widgets'):
                        if 'type' in attr and attr['type'] == "hidden":
                            logging.debug('subwidget %s.is_hidden = %s' % (field_instance.widget.widgets[index], field_instance.widget.widgets[index].is_hidden))
                            if False:
                                for line in traceback.format_stack():
                                    logging.debug(line.rstrip())
                            set_hidden(field_instance.widget.widgets[index])
                            field_instance.widget.widgets[index] = field_instance.hidden_widget()

                        field_instance.widget.widgets[index].attrs.update(attr)
                    else:
                        if 'type' in attr and attr['type'] == "hidden":
                            logging.debug('widget %s.is_hidden = %s' % (field_instance.widget, field_instance.widget.is_hidden))
                            if False:
                                for line in traceback.format_stack():
                                    logging.debug(line.rstrip())
                            set_hidden(field_instance.widget)
                            field_instance.widget = field_instance.hidden_widget()

                        field_instance.widget.attrs.update(attr)

        except KeyError:
            if not FAIL_SILENTLY:
                raise Exception("Could not resolve form field '%s'." % field)
            else:
                field_instance = None
                logging.warning("Could not resolve form field '%s'." % field, exc_info=sys.exc_info())

        if hasattr(form, 'rendered_fields'):
            if not field in form.rendered_fields:
                form.rendered_fields.add(field)
            else:
                if not FAIL_SILENTLY:
                    raise Exception("A field should only be rendered once: %s" % field)
                else:
                    logging.warning("A field should only be rendered once: %s" % field, exc_info=sys.exc_info())

        if field_instance is None:
            html = ''
        else:
            bound_field = BoundField(form, field_instance, field)

            if template is None:
                if form.crispy_field_template is None:
                    template = default_field_template(template_pack)
                else:   # FormHelper.field_template set
                    template = get_template(form.crispy_field_template)
            else:
                logging.debug('fetching template %s' % template)
                template = get_template(template)

            # We save the Layout object's bound fields in the layout object's `bound_fields` list
            if layout_object is not None:
                if hasattr(layout_object, 'bound_fields') and isinstance(layout_object.bound_fields, list):
                    layout_object.bound_fields.append(bound_field)
                else:
                    layout_object.bound_fields = [bound_field]

            context.update({
                'field': bound_field,
                'labelclass': labelclass,
                'flat_attrs': flatatt(attrs if isinstance(attrs, dict) else {}),
            })
            if extra_context is not None:
                context.update(extra_context)
            html = template.render(context)

        return html

def flatatt(attrs):
    """
    Taken from django.core.utils
    Convert a dictionary of attributes to a single string.
    The returned string will contain a leading space followed by key="value",
    XML-style pairs.  It is assumed that the keys do not need to be XML-escaped.
    If the passed dictionary is empty, then return an empty string.
    """
    return u''.join([u' %s="%s"' % (k.replace('_', '-'), conditional_escape(v)) for k, v in attrs.items()])

def render_crispy_form(form, helper=None, context=None):
    """
    Renders a form and returns its HTML output.

    This function wraps the template logic in a function easy to use in a Django view.
    """
    from crispy_forms.templatetags.crispy_forms_tags import CrispyFormNode

    if helper is not None:
        node = CrispyFormNode('form', 'helper')
    else:
        node = CrispyFormNode('form', None)

    node_context = Context(context)
    node_context.update({
        'form': form,
        'helper': helper
    })

    return node.render(node_context)
<commit_msg>set_hidden now works under Django 1.6 and 1.7 without warning<commit_after>from __future__ import with_statement
import inspect, traceback
import logging
import sys

from django.conf import settings
from django.forms.forms import BoundField
from django.template import Context
from django.template.loader import get_template
from django.utils.html import conditional_escape
try:
    from django.utils.lru_cache import lru_cache
    def memoize(function, *args):
        return lru_cache()(function)
except:
    from django.utils.functional import memoize

from .base import KeepContext
from .compatibility import text_type, PY2

# Global field template, default template used for rendering a field.

TEMPLATE_PACK = getattr(settings, 'CRISPY_TEMPLATE_PACK', 'bootstrap')

DEBUG_VERSION_TUPLE = False  # False when production-ready
def version_tuple(version):
    '''
    convert a version string to a tuple for more accurate comparisons

    >>> version_tuple('1.7.3.final.0')
    (1, 7, 3, 'final', 0)
    >>> '11.2.3' > '2.3.4'
    False
    >>> version_tuple('11.2.3') > version_tuple('2.3.4')
    True
    '''
    return tuple([int(p) if p.isdigit() else p for p in version.split('.')])
if DEBUG_VERSION_TUPLE:
    version_tuple.debug = logging.info
else:
    version_tuple.debug = lambda *args: None

# By memoizeing we avoid loading the template every time render_field
# is called without a template
def default_field_template(template_pack=TEMPLATE_PACK):
    return get_template("%s/field.html" % template_pack)
default_field_template = memoize(default_field_template, {}, 1)

def set_hidden(widget):
    '''
    set widget to hidden

    different starting in Django 1.7, when is_hidden ceases to be a
    true attribute and is determined by the input_type attribute
    '''
    if hasattr(widget, 'input_type'):
        widget.input_type = 'hidden'
    if not widget.is_hidden:
        widget.is_hidden = True

def render_field(
    field, form, form_style, context, template=None, labelclass=None,
    layout_object=None, attrs=None, template_pack=TEMPLATE_PACK,
    extra_context=None, **kwargs
):
    """
    Renders a django-crispy-forms field

    :param field: Can be a string or a Layout object like `Row`. If it's a layout
        object, we call its render method, otherwise we instantiate a BoundField
        and render it using default template 'CRISPY_TEMPLATE_PACK/field.html'
        The field is added to a list that the form holds called `rendered_fields`
        to avoid double rendering fields.
    :param form: The form/formset to which that field belongs to.
    :param form_style: A way to pass style name to the CSS framework used.
    :template: Template used for rendering the field.
    :layout_object: If passed, it points to the Layout object that is being rendered.
        We use it to store its bound fields in a list called `layout_object.bound_fields`
    :attrs: Attributes for the field's widget
    :template_pack: Name of the template pack to be used for rendering `field`
    :extra_context: Dictionary to be added to context, added variables by the layout object
    """
    logging.debug('rendering field %s from form %s' % (field, repr(form)))
    added_keys = [] if extra_context is None else extra_context.keys()
    with KeepContext(context, added_keys):
        if field is None:
            return ''

        FAIL_SILENTLY = getattr(settings, 'CRISPY_FAIL_SILENTLY', True)

        if hasattr(field, 'render'):
            return field.render(
                form, form_style, context, template_pack=template_pack,
            )
        else:
            # In Python 2 form field names cannot contain unicode characters without ASCII mapping
            if PY2:
                # This allows fields to be unicode strings, always they don't use non ASCII
                try:
                    if isinstance(field, text_type):
                        field = field.encode('ascii').decode()
                    # If `field` is not unicode then we turn it into a unicode string, otherwise doing
                    # str(field) would give no error and the field would not be resolved, causing confusion
                    else:
                        field = text_type(field)

                except (UnicodeEncodeError, UnicodeDecodeError):
                    raise Exception("Field '%s' is using forbidden unicode characters" % field)

        try:
            # Injecting HTML attributes into field's widget, Django handles rendering these
            field_instance = form.fields[field]
            if attrs is not None:
                widgets = getattr(field_instance.widget, 'widgets', [field_instance.widget])

                # We use attrs as a dictionary later, so here we make a copy
                list_attrs = attrs
                if isinstance(attrs, dict):
                    list_attrs = [attrs] * len(widgets)

                for index, (widget, attr) in enumerate(zip(widgets, list_attrs)):
                    if hasattr(field_instance.widget, 'widgets'):
                        if 'type' in attr and attr['type'] == "hidden":
                            logging.debug('subwidget %s.is_hidden = %s' % (field_instance.widget.widgets[index], field_instance.widget.widgets[index].is_hidden))
                            if False:
                                for line in traceback.format_stack():
                                    logging.debug(line.rstrip())
                            set_hidden(field_instance.widget.widgets[index])
                            field_instance.widget.widgets[index] = field_instance.hidden_widget()

                        field_instance.widget.widgets[index].attrs.update(attr)
                    else:
                        if 'type' in attr and attr['type'] == "hidden":
                            logging.debug('widget %s.is_hidden = %s' % (field_instance.widget, field_instance.widget.is_hidden))
                            if False:
                                for line in traceback.format_stack():
                                    logging.debug(line.rstrip())
                            set_hidden(field_instance.widget)
                            field_instance.widget = field_instance.hidden_widget()

                        field_instance.widget.attrs.update(attr)

        except KeyError:
            if not FAIL_SILENTLY:
                raise Exception("Could not resolve form field '%s'." % field)
            else:
                field_instance = None
                logging.warning("Could not resolve form field '%s'." % field, exc_info=sys.exc_info())

        if hasattr(form, 'rendered_fields'):
            if not field in form.rendered_fields:
                form.rendered_fields.add(field)
            else:
                if not FAIL_SILENTLY:
                    raise Exception("A field should only be rendered once: %s" % field)
                else:
                    logging.warning("A field should only be rendered once: %s" % field, exc_info=sys.exc_info())

        if field_instance is None:
            html = ''
        else:
            bound_field = BoundField(form, field_instance, field)

            if template is None:
                if form.crispy_field_template is None:
                    template = default_field_template(template_pack)
                else:   # FormHelper.field_template set
                    template = get_template(form.crispy_field_template)
            else:
                logging.debug('fetching template %s' % template)
                template = get_template(template)

            # We save the Layout object's bound fields in the layout object's `bound_fields` list
            if layout_object is not None:
                if hasattr(layout_object, 'bound_fields') and isinstance(layout_object.bound_fields, list):
                    layout_object.bound_fields.append(bound_field)
                else:
                    layout_object.bound_fields = [bound_field]

            context.update({
                'field': bound_field,
                'labelclass': labelclass,
                'flat_attrs': flatatt(attrs if isinstance(attrs, dict) else {}),
            })
            if extra_context is not None:
                context.update(extra_context)
            html = template.render(context)

        return html

def flatatt(attrs):
    """
    Taken from django.core.utils
    Convert a dictionary of attributes to a single string.
    The returned string will contain a leading space followed by key="value",
    XML-style pairs.  It is assumed that the keys do not need to be XML-escaped.
    If the passed dictionary is empty, then return an empty string.
    """
    return u''.join([u' %s="%s"' % (k.replace('_', '-'), conditional_escape(v)) for k, v in attrs.items()])

def render_crispy_form(form, helper=None, context=None):
    """
    Renders a form and returns its HTML output.

    This function wraps the template logic in a function easy to use in a Django view.
    """
    from crispy_forms.templatetags.crispy_forms_tags import CrispyFormNode

    if helper is not None:
        node = CrispyFormNode('form', 'helper')
    else:
        node = CrispyFormNode('form', None)

    node_context = Context(context)
    node_context.update({
        'form': form,
        'helper': helper
    })

    return node.render(node_context)
<|endoftext|>


<commit_before>import json
from django.core.urlresolvers import reverse
from tastypie import authorization
from tastypie.authentication import MultiAuthentication
from tastypie.exceptions import BadRequest

from crits.services.handlers import add_result, add_log, finish_task
from crits.services.service import CRITsService
from crits.core.api import CRITsApiKeyAuthentication, CRITsSessionAuthentication
from crits.core.api import CRITsSerializer, CRITsAPIResource

class ServiceResource(CRITsAPIResource):
    """
    Class to handle everything related to the Services API.

    Currently supports POST.
    """

    class Meta:
        object_class = CRITsService
        allowed_methods = ('post',)
        resource_name = "services"
        authentication = MultiAuthentication(CRITsApiKeyAuthentication(),
                                             CRITsSessionAuthentication())
        authorization = authorization.Authorization()
        serializer = CRITsSerializer()

    def get_object_list(self, request):
        """
        Use the CRITsAPIResource to get our objects but provide the class to get
        the objects from.

        :param request: The incoming request.
        :type request: :class:`django.http.HttpRequest`
        :returns: Resulting objects in the specified format (JSON by default).
        """

        return super(ServiceResource, self).get_object_list(request,
                                                            CRITsService,
                                                            False)

    def obj_create(self, bundle, **kwargs):
        """
        Handles creating service result entries through the API.

        :param bundle: Bundle containing the service results to add.
        :type bundle: Tastypie Bundle object.
        :returns: HttpResponse.

        """
        analyst = bundle.request.user.username
        object_type = bundle.data.get('object_type', None)
        object_id = bundle.data.get('object_id', None)
        analysis_id = bundle.data.get('analysis_id', None)
        result = bundle.data.get('result', None)
        result_type = bundle.data.get('result_type', None)
        result_subtype = bundle.data.get('result_subtype', None)
        result_is_batch = bundle.data.get('result_is_batch', False)
        log_message = bundle.data.get('log_message', None)
        log_level = bundle.data.get('log_level', 'info')
        status = bundle.data.get('status', None)
        finish = bundle.data.get('finish', False)

        success = True
        message = ""

        content = {'return_code': 1,
                   'type': object_type}

        if not object_type or not object_id or not analysis_id:
            content['message'] = 'Need an object type, object id, and analysis id.'
            self.crits_response(content)
        if result:
            if not result_is_batch:
                if not result_type or not result_subtype:
                    content['message'] = 'When adding a result, also need type and subtype'
                    self.crits_response(content)
                result = add_result(object_type, object_id, analysis_id,
                                    result, result_type, result_subtype, analyst)
                if not result['success']:
                    message += ", %s" % result['message']
                    success = False
            else:
                j_result = json.loads(result)
                j_result_type = json.loads(result_type)
                j_result_subtype = json.loads(result_subtype)

                if not (len(j_result) == len(j_result_type) == len(j_result_subtype)):
                    content['message'] = 'When adding results in batch result, result_type, ' +
                                         'and result_subtype must have the same length!'
                    self.crits_response(content)
                for key, r in enumerate(j_result):
                    result = add_result(object_type, object_id, analysis_id,
                                        r, j_result_type[key], j_result_subtype[key], analyst)
                    if not result['success']:
                        message += ", %s" % result['message']
                        success = False
        if log_message:
            result = add_log(object_type, object_id, analysis_id,
                             log_message, log_level, analyst)
            if not result['success']:
                message += ", %s" % result['message']
                success = False
        if finish:
            result = finish_task(object_type, object_id, analysis_id,
                                     status, analyst)
            if not result['success']:
                message += ", %s" % result['message']
                success = False

        content['message'] = message
        content['id'] = object_id
        rname = self.resource_name_from_type(object_type)
        url = reverse('api_dispatch_detail',
                        kwargs={'resource_name': rname,
                                'api_name': 'v1',
                                'pk': object_id})
        content['url'] = url
        if success:
            content['return_code'] = 0
        self.crits_response(content)
<commit_msg>fixed small syntax err<commit_after>import json
from django.core.urlresolvers import reverse
from tastypie import authorization
from tastypie.authentication import MultiAuthentication
from tastypie.exceptions import BadRequest

from crits.services.handlers import add_result, add_log, finish_task
from crits.services.service import CRITsService
from crits.core.api import CRITsApiKeyAuthentication, CRITsSessionAuthentication
from crits.core.api import CRITsSerializer, CRITsAPIResource

class ServiceResource(CRITsAPIResource):
    """
    Class to handle everything related to the Services API.

    Currently supports POST.
    """

    class Meta:
        object_class = CRITsService
        allowed_methods = ('post',)
        resource_name = "services"
        authentication = MultiAuthentication(CRITsApiKeyAuthentication(),
                                             CRITsSessionAuthentication())
        authorization = authorization.Authorization()
        serializer = CRITsSerializer()

    def get_object_list(self, request):
        """
        Use the CRITsAPIResource to get our objects but provide the class to get
        the objects from.

        :param request: The incoming request.
        :type request: :class:`django.http.HttpRequest`
        :returns: Resulting objects in the specified format (JSON by default).
        """

        return super(ServiceResource, self).get_object_list(request,
                                                            CRITsService,
                                                            False)

    def obj_create(self, bundle, **kwargs):
        """
        Handles creating service result entries through the API.

        :param bundle: Bundle containing the service results to add.
        :type bundle: Tastypie Bundle object.
        :returns: HttpResponse.

        """
        analyst = bundle.request.user.username
        object_type = bundle.data.get('object_type', None)
        object_id = bundle.data.get('object_id', None)
        analysis_id = bundle.data.get('analysis_id', None)
        result = bundle.data.get('result', None)
        result_type = bundle.data.get('result_type', None)
        result_subtype = bundle.data.get('result_subtype', None)
        result_is_batch = bundle.data.get('result_is_batch', False)
        log_message = bundle.data.get('log_message', None)
        log_level = bundle.data.get('log_level', 'info')
        status = bundle.data.get('status', None)
        finish = bundle.data.get('finish', False)

        success = True
        message = ""

        content = {'return_code': 1,
                   'type': object_type}

        if not object_type or not object_id or not analysis_id:
            content['message'] = 'Need an object type, object id, and analysis id.'
            self.crits_response(content)
        if result:
            if not result_is_batch:
                if not result_type or not result_subtype:
                    content['message'] = 'When adding a result, also need type and subtype'
                    self.crits_response(content)
                result = add_result(object_type, object_id, analysis_id,
                                    result, result_type, result_subtype, analyst)
                if not result['success']:
                    message += ", %s" % result['message']
                    success = False
            else:
                j_result = json.loads(result)
                j_result_type = json.loads(result_type)
                j_result_subtype = json.loads(result_subtype)

                if not (len(j_result) == len(j_result_type) == len(j_result_subtype)):
                    content['message'] = 'When adding results in batch result, result_type, and result_subtype must have the same length!'
                    self.crits_response(content)
                for key, r in enumerate(j_result):
                    result = add_result(object_type, object_id, analysis_id,
                                        r, j_result_type[key], j_result_subtype[key], analyst)
                    if not result['success']:
                        message += ", %s" % result['message']
                        success = False
        if log_message:
            result = add_log(object_type, object_id, analysis_id,
                             log_message, log_level, analyst)
            if not result['success']:
                message += ", %s" % result['message']
                success = False
        if finish:
            result = finish_task(object_type, object_id, analysis_id,
                                     status, analyst)
            if not result['success']:
                message += ", %s" % result['message']
                success = False

        content['message'] = message
        content['id'] = object_id
        rname = self.resource_name_from_type(object_type)
        url = reverse('api_dispatch_detail',
                        kwargs={'resource_name': rname,
                                'api_name': 'v1',
                                'pk': object_id})
        content['url'] = url
        if success:
            content['return_code'] = 0
        self.crits_response(content)
<|endoftext|>


<commit_before>import logging
import requests

from nameko.dependency_providers import Config
from nameko.events import event_handler
from nameko.utils.retry import retry

from .logger import LoggingDependency
from .storages import RedisStorage

log = logging.info

# See http://docs.python-requests.org/en/latest/user/advanced/#timeouts
# We are waiting 3 sec for the connexion and 9 for the response.
TIMEOUT = (3.1, 9.1)

DELAY_INTERVAL = 10  # in seconds
NB_RETRY = 5
# increase the retry delay by this factor at each try
BACKOFF_FACTOR = 2

class WebhookUnreachableException(Exception):

    def __init__(self, message, url, status, original_exception=None):
        super().__init__(message)
        self.url = url
        self.status = status
        self.original_exception = original_exception

class WebhookService(object):
    name = 'webhook_dispatcher'
    storage = RedisStorage()
    logger = LoggingDependency()
    config = Config()

    def _send(self, url, metadata):
        """POST metadata to url"""
        try:
            response = requests.post(url, json={'data': metadata},
                                     timeout=TIMEOUT)
        except (requests.Timeout, requests.RequestException) as e:
            raise WebhookUnreachableException('Unreachable', url, 503,
                                              original_exception=e)
        if response.status_code < 200 or response.status_code >= 400:
            raise WebhookUnreachableException('Unreachable', url,
                                              response.status_code)
        log('Successfully called webhook {url}'.format(url=url))

    @event_handler('url_crawler', 'url_crawled')
    def send_response(self, metadata):
        """Call a webhook with checked url results"""
        url = metadata.get('checked-url')
        callback_urls = self.storage.get_webhooks_for_url(url)
        if not callback_urls:
            return
        for callback_url in callback_urls:
            log(('Calling webhook url {callback_url} for checked url {url}'
                 .format(callback_url=callback_url, url=url)))
            try:
                send = retry(
                    self._send,
                    for_exceptions=WebhookUnreachableException,
                    max_attempts=self.config.get('WEBHOOK_NB_RETRY', NB_RETRY),
                    delay=self.config.get('WEBHOOK_DELAY_INTERVAL',
                                          DELAY_INTERVAL),
                    backoff=self.config.get('WEBHOOK_BACKOFF_FACTOR',
                                            BACKOFF_FACTOR))
                send(callback_url, metadata)
            except WebhookUnreachableException as e:
                logging.error(('Webhook unreachable: {url} - {code} ({detail})'
                               .format(url=callback_url, code=e.status,
                                       detail=e.original_exception)))
<commit_msg>Reduce default retry nb to 3<commit_after>import logging
import requests

from nameko.dependency_providers import Config
from nameko.events import event_handler
from nameko.utils.retry import retry

from .logger import LoggingDependency
from .storages import RedisStorage

log = logging.info

# See http://docs.python-requests.org/en/latest/user/advanced/#timeouts
# We are waiting 3 sec for the connexion and 9 for the response.
TIMEOUT = (3.1, 9.1)

DELAY_INTERVAL = 10  # in seconds
NB_RETRY = 3
# increase the retry delay by this factor at each try
BACKOFF_FACTOR = 2

class WebhookUnreachableException(Exception):

    def __init__(self, message, url, status, original_exception=None):
        super().__init__(message)
        self.url = url
        self.status = status
        self.original_exception = original_exception

class WebhookService(object):
    name = 'webhook_dispatcher'
    storage = RedisStorage()
    logger = LoggingDependency()
    config = Config()

    def _send(self, url, metadata):
        """POST metadata to url"""
        try:
            response = requests.post(url, json={'data': metadata},
                                     timeout=TIMEOUT)
        except (requests.Timeout, requests.RequestException) as e:
            raise WebhookUnreachableException('Unreachable', url, 503,
                                              original_exception=e)
        if response.status_code < 200 or response.status_code >= 400:
            raise WebhookUnreachableException('Unreachable', url,
                                              response.status_code)
        log('Successfully called webhook {url}'.format(url=url))

    @event_handler('url_crawler', 'url_crawled')
    def send_response(self, metadata):
        """Call a webhook with checked url results"""
        url = metadata.get('checked-url')
        callback_urls = self.storage.get_webhooks_for_url(url)
        if not callback_urls:
            return
        for callback_url in callback_urls:
            log(('Calling webhook url {callback_url} for checked url {url}'
                 .format(callback_url=callback_url, url=url)))
            try:
                send = retry(
                    self._send,
                    for_exceptions=WebhookUnreachableException,
                    max_attempts=self.config.get('WEBHOOK_NB_RETRY', NB_RETRY),
                    delay=self.config.get('WEBHOOK_DELAY_INTERVAL',
                                          DELAY_INTERVAL),
                    backoff=self.config.get('WEBHOOK_BACKOFF_FACTOR',
                                            BACKOFF_FACTOR))
                send(callback_url, metadata)
            except WebhookUnreachableException as e:
                logging.error(('Webhook unreachable: {url} - {code} ({detail})'
                               .format(url=callback_url, code=e.status,
                                       detail=e.original_exception)))
<|endoftext|>


<commit_before>import operator
import six
from django.db.models import Q
from django.http import HttpResponseRedirect
from django.contrib.auth.views import redirect_to_login
from django.contrib.auth import REDIRECT_FIELD_NAME
from django.conf import settings
from django.contrib import messages

from crudbuilder.helpers import plural
from crudbuilder.signals import crudbuilder_signals

if six.PY3:
    from functools import reduce

class LoginRequiredMixin(object):
    """
    View mixin which verifies that login required for the crud builder
    """
    login_url = settings.LOGIN_URL
    redirect_field_name = REDIRECT_FIELD_NAME  # Set by django.contrib.auth

    def handle_login_required(self, request):
        return redirect_to_login(
            request.get_full_path(),
            self.login_url,
            self.redirect_field_name)

    def dispatch(self, request, *args, **kwargs):
        global_login_required = getattr(
            settings, 'LOGIN_REQUIRED_FOR_CRUD', False)

        if global_login_required or self.login_required:
            if not request.user.is_authenticated():
                return self.handle_login_required(request)

        return super(LoginRequiredMixin, self).dispatch(
            request, *args, **kwargs)

class PermissionRequiredMixin(object):
    """
    View mixin which verifies that the logged in user has the specified
    permission.
    """
    def check_permissions(self, request):
        perms = self.permissions
        result = request.user.has_perm(perms) if perms else True
        return result

    def dispatch(self, request, *args, **kwargs):
        has_permission = self.check_permissions(request)
        global_permission_required = getattr(
            settings, 'PERMISSION_REQUIRED_FOR_CRUD', False)

        if global_permission_required or self.permission_required:
            if not has_permission:
                return redirect_to_login(
                    request.get_full_path(),
                    settings.LOGIN_URL,
                    REDIRECT_FIELD_NAME)

        return super(PermissionRequiredMixin, self).dispatch(
            request, *args, **kwargs)

class CrudBuilderMixin(LoginRequiredMixin, PermissionRequiredMixin):
    """
    - Mixin to provide additional context data for the templates
    - app_label : return app_label for CRUD model
    - actual_model_name : returns the actual model name (person -> person)
    - pluralized_model_name : return the pluralized_model_name
    (person -> people)
    """
    def get_context_data(self, **kwargs):
        context = super(CrudBuilderMixin, self).get_context_data(**kwargs)
        model = context['view'].model
        context['app_label'] = model._meta.app_label
        context['actual_model_name'] = model.__name__.lower()
        context['pluralized_model_name'] = plural(model.__name__.lower())
        context['project_name'] = getattr(
            settings, 'PROJECT_NAME', 'CRUDBUILDER')
        return context

    @property
    def get_actual_signal(self):
        view = 'update' if self.object else 'create'
        if self.inlineformset:
            return crudbuilder_signals['inlineformset'][view]
        else:
            return crudbuilder_signals['instance'][view]

class BaseDetailViewMixin(CrudBuilderMixin):
    def get_context_data(self, **kwargs):
        context = super(BaseDetailViewMixin, self).get_context_data(**kwargs)
        context['inlineformset'] = self.inlineformset
        return context

class CreateUpdateViewMixin(CrudBuilderMixin):
    """Common form_valid() method for both Create and Update views"""

    def form_valid(self, form):
        signal = self.get_actual_signal
        instance = form.save(commit=False)
        signal.send(sender=self.model, request=self.request, instance=instance)
        instance.save()
        form.save_m2m()
        return super(CreateUpdateViewMixin, self).form_valid(form)

class BaseListViewMixin(CrudBuilderMixin):
    """
    - Search implementation for tables2 in ListView.
    - search_feilds will be defined in actual model
    - Override get_queryset method of ListView
    """
    def get_queryset(self):
        if self.custom_queryset:
            objects = self.custom_queryset(self.request, **self.kwargs)
        else:
            objects = self.model.objects.all()
        search = self.request.GET.get('search')
        if search:
            q_list = [
                Q(
                    ('{}__icontains'.format(field), search))
                for field in self.crud.search_fields
            ]
            objects = objects.filter(reduce(operator.or_, q_list))
        return objects.order_by('-id')

    def get_context_data(self, **kwargs):
        context = super(BaseListViewMixin, self).get_context_data(**kwargs)
        if self.custom_context:
            context.update(self.custom_context)
        return context

class InlineFormsetViewMixin(CrudBuilderMixin):
    def get_context_data(self, **kwargs):
        context = super(
            InlineFormsetViewMixin, self).get_context_data(**kwargs)
        if self.request.POST:
            context['inlineformset'] = self.inlineformset(
                self.request.POST,
                instance=self.object
            )
        else:
            context['inlineformset'] = self.inlineformset(
                instance=self.object
            )
        return context

    def form_valid(self, form):
        context = self.get_context_data()
        inlineformset = context['inlineformset']

        if inlineformset.is_valid():
            parent = form.save(commit=False)
            inlineformset.instance = parent
            children = inlineformset.save(commit=False)

            # execute post signals
            signal = self.get_actual_signal
            signal.send(
                sender=self.model,
                request=self.request,
                parent=parent,
                children=children)

            parent.save()
            inlineformset.save()
            inlineformset.save_m2m()

            return HttpResponseRedirect(self.success_url)
        else:
            messages.error(
                self.request, inlineformset.non_form_errors())
            return self.render_to_response(
                self.get_context_data(form=form, inlineformset=inlineformset)
            )
<commit_msg>Updated custom context<commit_after>import operator
import six
from django.db.models import Q
from django.http import HttpResponseRedirect
from django.contrib.auth.views import redirect_to_login
from django.contrib.auth import REDIRECT_FIELD_NAME
from django.conf import settings
from django.contrib import messages

from crudbuilder.helpers import plural
from crudbuilder.signals import crudbuilder_signals

if six.PY3:
    from functools import reduce

class LoginRequiredMixin(object):
    """
    View mixin which verifies that login required for the crud builder
    """
    login_url = settings.LOGIN_URL
    redirect_field_name = REDIRECT_FIELD_NAME  # Set by django.contrib.auth

    def handle_login_required(self, request):
        return redirect_to_login(
            request.get_full_path(),
            self.login_url,
            self.redirect_field_name)

    def dispatch(self, request, *args, **kwargs):
        global_login_required = getattr(
            settings, 'LOGIN_REQUIRED_FOR_CRUD', False)

        if global_login_required or self.login_required:
            if not request.user.is_authenticated():
                return self.handle_login_required(request)

        return super(LoginRequiredMixin, self).dispatch(
            request, *args, **kwargs)

class PermissionRequiredMixin(object):
    """
    View mixin which verifies that the logged in user has the specified
    permission.
    """
    def check_permissions(self, request):
        perms = self.permissions
        result = request.user.has_perm(perms) if perms else True
        return result

    def dispatch(self, request, *args, **kwargs):
        has_permission = self.check_permissions(request)
        global_permission_required = getattr(
            settings, 'PERMISSION_REQUIRED_FOR_CRUD', False)

        if global_permission_required or self.permission_required:
            if not has_permission:
                return redirect_to_login(
                    request.get_full_path(),
                    settings.LOGIN_URL,
                    REDIRECT_FIELD_NAME)

        return super(PermissionRequiredMixin, self).dispatch(
            request, *args, **kwargs)

class CrudBuilderMixin(LoginRequiredMixin, PermissionRequiredMixin):
    """
    - Mixin to provide additional context data for the templates
    - app_label : return app_label for CRUD model
    - actual_model_name : returns the actual model name (person -> person)
    - pluralized_model_name : return the pluralized_model_name
    (person -> people)
    """
    def get_context_data(self, **kwargs):
        context = super(CrudBuilderMixin, self).get_context_data(**kwargs)
        model = context['view'].model
        context['app_label'] = model._meta.app_label
        context['actual_model_name'] = model.__name__.lower()
        context['pluralized_model_name'] = plural(model.__name__.lower())
        context['project_name'] = getattr(
            settings, 'PROJECT_NAME', 'CRUDBUILDER')
        return context

    @property
    def get_actual_signal(self):
        view = 'update' if self.object else 'create'
        if self.inlineformset:
            return crudbuilder_signals['inlineformset'][view]
        else:
            return crudbuilder_signals['instance'][view]

class BaseDetailViewMixin(CrudBuilderMixin):
    def get_context_data(self, **kwargs):
        context = super(BaseDetailViewMixin, self).get_context_data(**kwargs)
        context['inlineformset'] = self.inlineformset
        return context

class CreateUpdateViewMixin(CrudBuilderMixin):
    """Common form_valid() method for both Create and Update views"""

    def form_valid(self, form):
        signal = self.get_actual_signal
        instance = form.save(commit=False)
        signal.send(sender=self.model, request=self.request, instance=instance)
        instance.save()
        form.save_m2m()
        return super(CreateUpdateViewMixin, self).form_valid(form)

class BaseListViewMixin(CrudBuilderMixin):
    """
    - Search implementation for tables2 in ListView.
    - search_feilds will be defined in actual model
    - Override get_queryset method of ListView
    """
    def get_queryset(self):
        if self.custom_queryset:
            objects = self.custom_queryset(self.request, **self.kwargs)
        else:
            objects = self.model.objects.all()
        search = self.request.GET.get('search')
        if search:
            q_list = [
                Q(
                    ('{}__icontains'.format(field), search))
                for field in self.crud.search_fields
            ]
            objects = objects.filter(reduce(operator.or_, q_list))
        return objects.order_by('-id')

    def get_context_data(self, **kwargs):
        context = super(BaseListViewMixin, self).get_context_data(**kwargs)
        if self.custom_context:
            custom_context = self.custom_context(self.request, **self.args)
            context.update(custom_context)
        return context

class InlineFormsetViewMixin(CrudBuilderMixin):
    def get_context_data(self, **kwargs):
        context = super(
            InlineFormsetViewMixin, self).get_context_data(**kwargs)
        if self.request.POST:
            context['inlineformset'] = self.inlineformset(
                self.request.POST,
                instance=self.object
            )
        else:
            context['inlineformset'] = self.inlineformset(
                instance=self.object
            )
        return context

    def form_valid(self, form):
        context = self.get_context_data()
        inlineformset = context['inlineformset']

        if inlineformset.is_valid():
            parent = form.save(commit=False)
            inlineformset.instance = parent
            children = inlineformset.save(commit=False)

            # execute post signals
            signal = self.get_actual_signal
            signal.send(
                sender=self.model,
                request=self.request,
                parent=parent,
                children=children)

            parent.save()
            inlineformset.save()
            inlineformset.save_m2m()

            return HttpResponseRedirect(self.success_url)
        else:
            messages.error(
                self.request, inlineformset.non_form_errors())
            return self.render_to_response(
                self.get_context_data(form=form, inlineformset=inlineformset)
            )
<|endoftext|>