<commit_before># coding: utf-8

"""
ARC remote workflow implementation. See http://www.nordugrid.org/arc/ce.
"""

__all__ = ["ARCWorkflow"]

import os
import logging
from abc import abstractmethod
from collections import OrderedDict

from law.workflow.remote import BaseRemoteWorkflow, BaseRemoteWorkflowProxy
from law.job.base import JobArguments
from law.task.proxy import ProxyCommand
from law.target.file import get_path
from law.parameter import CSVParameter
from law.util import law_src_path, merge_dicts, is_number, DotDict

from law.contrib.arc.job import ARCJobManager, ARCJobFileFactory

logger = logging.getLogger(__name__)

class ARCWorkflowProxy(BaseRemoteWorkflowProxy):

    workflow_type = "arc"

    def __init__(self, *args, **kwargs):
        super(ARCWorkflowProxy, self).__init__(*args, **kwargs)

        # check if there is at least one ce
        if not self.task.arc_ce:
            raise Exception("please set at least one arc computing element (--arc-ce)")

    def create_job_manager(self, **kwargs):
        return self.task.arc_create_job_manager(**kwargs)

    def create_job_file_factory(self, **kwargs):
        return self.task.arc_create_job_file_factory(**kwargs)

    def create_job_file(self, job_num, branches):
        task = self.task
        config = self.job_file_factory.Config()

        # the file postfix is pythonic range made from branches, e.g. [0, 1, 2, 4] -> "_0To5"
        postfix = "_{}To{}".format(branches[0], branches[-1] + 1)
        config.postfix = postfix
        pf = lambda s: "__law_job_postfix__:{}".format(s)

        # get the actual wrapper file that will be executed by the remote job
        wrapper_file = get_path(task.arc_wrapper_file())
        config.executable = os.path.basename(wrapper_file)

        # collect task parameters
        proxy_cmd = ProxyCommand(task.as_branch(branches[0]), exclude_task_args={"branch"},
            exclude_global_args=["workers", "local-scheduler"])
        if task.arc_use_local_scheduler():
            proxy_cmd.add_arg("--local-scheduler", "True", overwrite=True)
        for key, value in OrderedDict(task.arc_cmdline_args()).items():
            proxy_cmd.add_arg(key, value, overwrite=True)

        # job script arguments
        job_args = JobArguments(
            task_cls=task.__class__,
            task_params=proxy_cmd.build(skip_run=True),
            branches=branches,
            auto_retry=False,
            dashboard_data=self.dashboard.remote_hook_data(
                job_num, self.submission_data.attempts.get(job_num, 0)),
        )
        config.arguments = job_args.join()

        # meta infos
        config.job_name = task.task_id
        config.output_uri = task.arc_output_uri()

        # prepare render variables
        config.render_variables = {}

        # input files
        config.input_files = [wrapper_file, law_src_path("job", "job.sh")]
        config.render_variables["job_file"] = pf("job.sh")

        # add the bootstrap file
        bootstrap_file = task.arc_bootstrap_file()
        config.input_files.append(bootstrap_file)
        config.render_variables["bootstrap_file"] = pf(os.path.basename(bootstrap_file))

        # add the stageout file
        stageout_file = task.arc_stageout_file()
        if stageout_file:
            config.input_files.append(stageout_file)
            config.render_variables["stageout_file"] = pf(os.path.basename(stageout_file))

        # does the dashboard have a hook file?
        dashboard_file = self.dashboard.remote_hook_file()
        if dashboard_file:
            config.input_files.append(dashboard_file)
            config.render_variables["dashboard_file"] = pf(os.path.basename(dashboard_file))

        # output files
        config.output_files = []

        # custom content
        config.custom_content = []

        # log files
        config.log = None
        if task.transfer_logs:
            log_file = "stdall.txt"
            config.stdout = log_file
            config.stderr = log_file
            config.custom_log_file = log_file
            config.render_variables["log_file"] = pf(log_file)
        else:
            config.stdout = None
            config.stderr = None

        # task hook
        config = task.arc_job_config(config, job_num, branches)

        # determine basenames of input files and add that list to the render data
        input_basenames = [pf(os.path.basename(path)) for path in config.input_files]
        config.render_variables["input_files"] = " ".join(input_basenames)

        # build the job file and get the sanitized config
        job_file, config = self.job_file_factory(**config.__dict__)

        # determine the custom log file uri if set
        abs_log_file = None
        if config.custom_log_file:
            abs_log_file = os.path.join(config.output_uri, config.custom_log_file)

        # return job and log files
        return {"job": job_file, "log": abs_log_file}

    def destination_info(self):
        return "ce: {}".format(",".join(self.task.arc_ce))

class ARCWorkflow(BaseRemoteWorkflow):

    workflow_proxy_cls = ARCWorkflowProxy

    arc_workflow_run_decorators = None
    arc_job_manager_defaults = None
    arc_job_file_factory_defaults = None

    arc_ce = CSVParameter(default=(), significant=False, description="target arc computing "
        "element(s); default: empty")

    arc_job_kwargs = []
    arc_job_kwargs_submit = ["arc_ce"]
    arc_job_kwargs_cancel = None
    arc_job_kwargs_cleanup = None
    arc_job_kwargs_query = None

    exclude_params_branch = {"arc_ce"}

    exclude_index = True

    @abstractmethod
    def arc_output_directory(self):
        return None

    @abstractmethod
    def arc_bootstrap_file(self):
        return None

    def arc_wrapper_file(self):
        return law_src_path("job", "bash_wrapper.sh")

    def arc_stageout_file(self):
        return None

    def arc_workflow_requires(self):
        return DotDict()

    def arc_output_postfix(self):
        return "_" + self.get_branches_repr()

    def arc_output_uri(self):
        return self.arc_output_directory().url()

    def arc_create_job_manager(self, **kwargs):
        kwargs = merge_dicts(self.arc_job_manager_defaults, kwargs)
        return ARCJobManager(**kwargs)

    def arc_create_job_file_factory(self, **kwargs):
        # job file fectory config priority: kwargs > class defaults
        kwargs = merge_dicts({}, self.arc_job_file_factory_defaults, kwargs)
        return ARCJobFileFactory(**kwargs)

    def arc_job_config(self, config, job_num, branches):
        return config

    def arc_use_local_scheduler(self):
        return True

    def arc_cmdline_args(self):
        return {}
<commit_msg>Fix unused imports.<commit_after># coding: utf-8

"""
ARC remote workflow implementation. See http://www.nordugrid.org/arc/ce.
"""

__all__ = ["ARCWorkflow"]

import os
import logging
from abc import abstractmethod
from collections import OrderedDict

from law.workflow.remote import BaseRemoteWorkflow, BaseRemoteWorkflowProxy
from law.job.base import JobArguments
from law.task.proxy import ProxyCommand
from law.target.file import get_path
from law.parameter import CSVParameter
from law.util import law_src_path, merge_dicts, DotDict

from law.contrib.arc.job import ARCJobManager, ARCJobFileFactory

logger = logging.getLogger(__name__)

class ARCWorkflowProxy(BaseRemoteWorkflowProxy):

    workflow_type = "arc"

    def __init__(self, *args, **kwargs):
        super(ARCWorkflowProxy, self).__init__(*args, **kwargs)

        # check if there is at least one ce
        if not self.task.arc_ce:
            raise Exception("please set at least one arc computing element (--arc-ce)")

    def create_job_manager(self, **kwargs):
        return self.task.arc_create_job_manager(**kwargs)

    def create_job_file_factory(self, **kwargs):
        return self.task.arc_create_job_file_factory(**kwargs)

    def create_job_file(self, job_num, branches):
        task = self.task
        config = self.job_file_factory.Config()

        # the file postfix is pythonic range made from branches, e.g. [0, 1, 2, 4] -> "_0To5"
        postfix = "_{}To{}".format(branches[0], branches[-1] + 1)
        config.postfix = postfix
        pf = lambda s: "__law_job_postfix__:{}".format(s)

        # get the actual wrapper file that will be executed by the remote job
        wrapper_file = get_path(task.arc_wrapper_file())
        config.executable = os.path.basename(wrapper_file)

        # collect task parameters
        proxy_cmd = ProxyCommand(task.as_branch(branches[0]), exclude_task_args={"branch"},
            exclude_global_args=["workers", "local-scheduler"])
        if task.arc_use_local_scheduler():
            proxy_cmd.add_arg("--local-scheduler", "True", overwrite=True)
        for key, value in OrderedDict(task.arc_cmdline_args()).items():
            proxy_cmd.add_arg(key, value, overwrite=True)

        # job script arguments
        job_args = JobArguments(
            task_cls=task.__class__,
            task_params=proxy_cmd.build(skip_run=True),
            branches=branches,
            auto_retry=False,
            dashboard_data=self.dashboard.remote_hook_data(
                job_num, self.submission_data.attempts.get(job_num, 0)),
        )
        config.arguments = job_args.join()

        # meta infos
        config.job_name = task.task_id
        config.output_uri = task.arc_output_uri()

        # prepare render variables
        config.render_variables = {}

        # input files
        config.input_files = [wrapper_file, law_src_path("job", "job.sh")]
        config.render_variables["job_file"] = pf("job.sh")

        # add the bootstrap file
        bootstrap_file = task.arc_bootstrap_file()
        config.input_files.append(bootstrap_file)
        config.render_variables["bootstrap_file"] = pf(os.path.basename(bootstrap_file))

        # add the stageout file
        stageout_file = task.arc_stageout_file()
        if stageout_file:
            config.input_files.append(stageout_file)
            config.render_variables["stageout_file"] = pf(os.path.basename(stageout_file))

        # does the dashboard have a hook file?
        dashboard_file = self.dashboard.remote_hook_file()
        if dashboard_file:
            config.input_files.append(dashboard_file)
            config.render_variables["dashboard_file"] = pf(os.path.basename(dashboard_file))

        # output files
        config.output_files = []

        # custom content
        config.custom_content = []

        # log files
        config.log = None
        if task.transfer_logs:
            log_file = "stdall.txt"
            config.stdout = log_file
            config.stderr = log_file
            config.custom_log_file = log_file
            config.render_variables["log_file"] = pf(log_file)
        else:
            config.stdout = None
            config.stderr = None

        # task hook
        config = task.arc_job_config(config, job_num, branches)

        # determine basenames of input files and add that list to the render data
        input_basenames = [pf(os.path.basename(path)) for path in config.input_files]
        config.render_variables["input_files"] = " ".join(input_basenames)

        # build the job file and get the sanitized config
        job_file, config = self.job_file_factory(**config.__dict__)

        # determine the custom log file uri if set
        abs_log_file = None
        if config.custom_log_file:
            abs_log_file = os.path.join(config.output_uri, config.custom_log_file)

        # return job and log files
        return {"job": job_file, "log": abs_log_file}

    def destination_info(self):
        return "ce: {}".format(",".join(self.task.arc_ce))

class ARCWorkflow(BaseRemoteWorkflow):

    workflow_proxy_cls = ARCWorkflowProxy

    arc_workflow_run_decorators = None
    arc_job_manager_defaults = None
    arc_job_file_factory_defaults = None

    arc_ce = CSVParameter(default=(), significant=False, description="target arc computing "
        "element(s); default: empty")

    arc_job_kwargs = []
    arc_job_kwargs_submit = ["arc_ce"]
    arc_job_kwargs_cancel = None
    arc_job_kwargs_cleanup = None
    arc_job_kwargs_query = None

    exclude_params_branch = {"arc_ce"}

    exclude_index = True

    @abstractmethod
    def arc_output_directory(self):
        return None

    @abstractmethod
    def arc_bootstrap_file(self):
        return None

    def arc_wrapper_file(self):
        return law_src_path("job", "bash_wrapper.sh")

    def arc_stageout_file(self):
        return None

    def arc_workflow_requires(self):
        return DotDict()

    def arc_output_postfix(self):
        return "_" + self.get_branches_repr()

    def arc_output_uri(self):
        return self.arc_output_directory().url()

    def arc_create_job_manager(self, **kwargs):
        kwargs = merge_dicts(self.arc_job_manager_defaults, kwargs)
        return ARCJobManager(**kwargs)

    def arc_create_job_file_factory(self, **kwargs):
        # job file fectory config priority: kwargs > class defaults
        kwargs = merge_dicts({}, self.arc_job_file_factory_defaults, kwargs)
        return ARCJobFileFactory(**kwargs)

    def arc_job_config(self, config, job_num, branches):
        return config

    def arc_use_local_scheduler(self):
        return True

    def arc_cmdline_args(self):
        return {}
<|endoftext|>


<commit_before>__version__ = '0.2020.04.24.1833'
<commit_msg>RELEASE: Bump version to 0.2020.04.24.1834<commit_after>__version__ = '0.2020.04.24.1834'
<|endoftext|>


<commit_before>"""
You MUST test pika_subscriber_test.py and pika_publisher_test.py
before you use distributed computing with IoTPy.

When you execute:
                   pika_subscriber_test.py
in one terminal window, and execute
                   pika_publisher_test.py argument
in a different terminal window, you shoud see the argument echoed
in the subscriber window.

Look at:
 https://www.rabbitmq.com/tutorials/tutorial-one-python.html

"""
#!/usr/bin/env python
import pika
import json
import time
# stream is in core
from IoTPy.core.stream import Stream, run
# sink, op, basics are in the agent_types
# sink is in agent_types
from IoTPy.agent_types.sink import sink_list
from IoTPy.agent_types.op import map_element
from IoTPy.helper_functions.recent_values import recent_values
# multicore is in concurrency
from IoTPy.concurrency.multicore import copy_data_to_stream, finished_source
from IoTPy.concurrency.multicore import get_processes
#from pika_subscribe_agent import PikaSubscriber
from IoTPy.concurrency.pika_publication_agent import PikaPublisher
# print_stream is in helper_functions
from IoTPy.helper_functions.print_stream import print_stream

publisher = PikaPublisher(
    routing_key='temperature',
    exchange='publications', host='localhost')

def test():
    publisher = PikaPublisher(
        routing_key='temperature',
        exchange='publications', host='localhost')
    x = Stream('x')
    y = Stream('y')
    map_element(lambda v: 2*v, x, y)
    publisher.publish(y)
    for i in range(3):
        x.extend(list(range(i*4, (i+1)*4)))
        run()
        time.sleep(0.001)

def pika_publisher_test():
    """
    Simple example

    """
    publisher = PikaPublisher(
        routing_key='temperature',
        exchange='publications', host='localhost')

    # Agent function for process named 'p0'
    def f(in_streams, out_streams):
        print ('in f')
        print ('in_streams[0]')
        #print_stream(in_streams[0], 'x')
        publisher.publish(in_streams[0])
        #publisher.close()

    # Source thread target for source stream named 'x'.
    def h(proc):
        proc.copy_stream(data=list(range(10)), stream_name='x')
        proc.finished_source(stream_name='x')

    # The specification
    multicore_specification = [
        # Streams
        [('x', 'i')],
        # Processes
        [
            # Process p0
            {'name': 'p0', 'agent': f, 'inputs':['x'], 'sources': ['x'],
             'source_functions':[h]}
        ]
       ]

    # Execute processes (after including your own non IoTPy processes)
    processes = get_processes(multicore_specification)
    for process in processes: process.start()
    for process in processes: process.join()
    for process in processes: process.terminate()

if __name__ == '__main__':
    test()
    ## pika_publisher_test()
<commit_msg>Moved to new test structure.<commit_after>"""
You MUST test pika_subscriber_test.py and pika_publisher_test.py
before you use distributed computing with IoTPy.

When you execute:
                   pika_subscriber_test.py
in one terminal window, and execute
                   pika_publisher_test.py argument
in a different terminal window, you shoud see the argument echoed
in the subscriber window.

Look at:
 https://www.rabbitmq.com/tutorials/tutorial-one-python.html

"""
#!/usr/bin/env python
import pika
import json
import time

from IoTPy.core.agent import Agent
from IoTPy.core.stream import Stream, StreamArray, _no_value, _multivalue, run
from IoTPy.agent_types.op import map_element
from IoTPy.helper_functions.recent_values import recent_values
from IoTPy.helper_functions.print_stream import print_stream
# multicore imports
from IoTPy.concurrency.multicore import get_processes, get_processes_and_procs
from IoTPy.concurrency.multicore import terminate_stream
from IoTPy.concurrency.multicore import get_proc_that_inputs_source
from IoTPy.concurrency.multicore import extend_stream
from IoTPy.concurrency.PikaPublisher import PikaPublisher

publisher = PikaPublisher(
    routing_key='temperature',
    exchange='publications', host='localhost')

def test():
    publisher = PikaPublisher(
        routing_key='temperature',
        exchange='publications', host='localhost')
    x = Stream('x')
    y = Stream('y')
    map_element(lambda v: 2*v, x, y)
    publisher.publish(y)
    for i in range(3):
        x.extend(list(range(i*4, (i+1)*4)))
        run()
        time.sleep(0.001)

def pika_publisher_test():
    """
    Simple example

    """
    publisher = PikaPublisher(
        routing_key='temperature',
        exchange='publications', host='localhost')

    # Agent function for process named 'p0'
    def f(in_streams, out_streams):
        print ('in f')
        print ('in_streams[0]')
        #print_stream(in_streams[0], 'x')
        publisher.publish(in_streams[0])
        #publisher.close()

    # Source thread target for source stream named 'x'.
    def h(proc):
        proc.copy_stream(data=list(range(10)), stream_name='x')
        proc.finished_source(stream_name='x')

    # The specification
    multicore_specification = [
        # Streams
        [('x', 'i')],
        # Processes
        [
            # Process p0
            {'name': 'p0', 'agent': f, 'inputs':['x'], 'sources': ['x'],
             'source_functions':[h]}
        ]
       ]

    # Execute processes (after including your own non IoTPy processes)
    processes = get_processes(multicore_specification)
    for process in processes: process.start()
    for process in processes: process.join()
    for process in processes: process.terminate()

if __name__ == '__main__':
    test()
    ## pika_publisher_test()
<|endoftext|>


<commit_before>__version__ = '0.2016.10.13.1533'
<commit_msg>Update README for #40, more TODO<commit_after>__version__ = '0.2016.10.13.1544'
<|endoftext|>


<commit_before># -*- coding: utf-8 -*_

import contextlib
import hashlib
import imp
import os
import re
import sys
import warnings

from setuptools import setup, Extension

from IPython.core.magic import Magics, magics_class, cell_magic, line_magic, on_off
from IPython.core.magic_arguments import argument, magic_arguments, parse_argstring

from ipybind.build_ext import build_ext
from ipybind.common import ext_suffix, ext_path, is_kernel, split_args, pybind11_get_include
from ipybind.stream import forward, start_forwarding, stop_forwarding

@magics_class
class Pybind11Magics(Magics):
    @magic_arguments()
    @argument('-f', '--force', action='store_true',
              help='Force recompilation of the module.')
    @argument('-v', '--verbose', action='store_true',
              help='Display compilation output.')
    @argument('-std', choices=['c++11', 'c++14', 'c++1z', 'c++17'],
              help='C++ standard, defaults to C++14 if available.')
    @argument('-i', '--prefix-include', action='store_true',
              help='Add $PREFIX/include to include path.')
    @argument('--cc',
              help='Set CC environment variable.')
    @argument('--cxx',
              help='Set CXX environment variable.')
    @argument('--compiler',
              help='Pass --compiler to distutils.')
    @argument('-c', '--compile-args', action='append', default=[], metavar='ARGS',
              help='Extra flags to pass to the compiler.')
    @cell_magic
    def pybind11(self, line, cell):
        """
        Compile and import everything from a pybind11 C++ code cell.

        The contents of the cell are written to a `.cpp` file in `$IPYTHONDIR/pybind11`
        directory using a filename based on the hash of the code. The file is compiled,
        and the symbols in the produced module are then imported directly into the
        current namespace.
        """

        line = line.strip().rstrip(';')
        args = parse_argstring(self.pybind11, line)
        module = 'pybind11_{}'.format(self.compute_hash(line, cell))
        code = self.format_code(cell)
        libfile = ext_path(module + ext_suffix())
        need_rebuild = not os.path.isfile(libfile) or args.force
        if need_rebuild:
            source = self.save_source(code, module)
            self.build_module(module, source, args)
        self.import_module(module, libfile)

    @line_magic
    def pybind11_capture(self, parameter_s=''):
        """
        Control the automatic capturing of C++ stdout and stderr streams.

        To enable:  `%pybind11_capture 1` or `%pybind11_capture on`.
        To disable: `%pybind11_capture 0` or `%pybind11_capture off`.
        To toggle:  `%pybind11_capture`.
        """

        if not is_kernel():
            print('C++ stdout/stderr capturing is not available in the terminal environment.')
            return

        p = parameter_s.strip().lower()
        if p:
            try:
                capture = {'off': 0, '0': 0, 'on': 1, '1': 1}[p]
            except KeyError:
                print('Incorrect argument. Use on/1, off/0, or nothing for a toggle.')
                return
        else:
            capture = not getattr(self.shell, 'pybind11_capture', False)
        self.shell.pybind11_capture = capture
        (start_forwarding if capture else stop_forwarding)()
        print('C++ stdout/stderr capturing has been turned', on_off(capture))

    def compute_hash(self, line, cell):
        key = cell, line, sys.version_info, sys.executable
        return hashlib.md5(str(key).encode('utf-8')).hexdigest()[:7]

    def format_code(self, cell):
        code = cell.replace('PYBIND11_PLUGIN', '_PYBIND11_PLUGIN')
        code = '#include <pybind11_preamble.h>\n' + code
        code += '\n' * (not code.endswith('\n'))
        return code

    def save_source(self, code, module):
        filename = ext_path(module + '.cpp')
        os.makedirs(os.path.dirname(filename), exist_ok=True)
        with open(filename, 'w') as f:
            f.write(code)
        return filename

    @contextlib.contextmanager
    def with_env(self, **env_vars):
        env_vars = {k: v for k, v in env_vars.items() if v is not None}
        env = os.environ.copy()
        for k, v in env_vars.items():
            os.environ[k] = v
        try:
            yield
        finally:
            for k in env_vars:
                if k not in env:
                    os.environ.pop(k, None)
                else:
                    os.environ[k] = env[k]

    def make_extension(self, module, source, args):
        include_dirs = [os.path.dirname(__file__)]
        include_dirs += pybind11_get_include()
        if args.prefix_include:
            include_dirs.append(os.path.join(sys.prefix, 'include'))
        extension = Extension(
            name=module,
            sources=[source],
            include_dirs=include_dirs,
            library_dirs=[],
            extra_compile_args=split_args(args.compile_args),
            extra_link_args=[],
            libraries=[],
            language='c++',
            define_macros=[
                ('_PYBIND11_MODULE_NAME', module)
            ]
        )
        extension.args = args
        return extension

    def forward_handler(self, source):
        def handler(data):
            data = data.replace(source, '<source>')
            data = re.sub(r'^/.+/(pybind11/[\w_]+\.h:)', r'\1', data, flags=re.MULTILINE)
            return data
        return handler

    def build_module(self, module, source, args):
        with self.with_env(**{'CC': args.cc, 'CXX': args.cxx}):
            workdir = ext_path(module)
            os.makedirs(workdir, exist_ok=True)
            script_args = ['-v' if args.verbose else '-q']
            script_args += ['build_ext', '--inplace', '--build-temp', workdir]
            if args.compiler is not None:
                script_args += ['--compiler', args.compiler]
            with forward(self.forward_handler(source)):
                warnings.filterwarnings('ignore', 'To exit')
                setup(
                    name=module,
                    ext_modules=[self.make_extension(module, source, args)],
                    script_args=script_args,
                    cmdclass={'build_ext': build_ext}
                )

    def import_module(self, module, libfile):
        mod = imp.load_dynamic(module, libfile)
        for k, v in mod.__dict__.items():
            if not k.startswith('__'):
                self.shell.push({k: v})
<commit_msg>Minor cosmetic error processing adjustment<commit_after># -*- coding: utf-8 -*_

import contextlib
import hashlib
import imp
import os
import re
import sys
import warnings

from setuptools import setup, Extension

from IPython.core.magic import Magics, magics_class, cell_magic, line_magic, on_off
from IPython.core.magic_arguments import argument, magic_arguments, parse_argstring

from ipybind.build_ext import build_ext
from ipybind.common import ext_suffix, ext_path, is_kernel, split_args, pybind11_get_include
from ipybind.stream import forward, start_forwarding, stop_forwarding

@magics_class
class Pybind11Magics(Magics):
    @magic_arguments()
    @argument('-f', '--force', action='store_true',
              help='Force recompilation of the module.')
    @argument('-v', '--verbose', action='store_true',
              help='Display compilation output.')
    @argument('-std', choices=['c++11', 'c++14', 'c++1z', 'c++17'],
              help='C++ standard, defaults to C++14 if available.')
    @argument('-i', '--prefix-include', action='store_true',
              help='Add $PREFIX/include to include path.')
    @argument('--cc',
              help='Set CC environment variable.')
    @argument('--cxx',
              help='Set CXX environment variable.')
    @argument('--compiler',
              help='Pass --compiler to distutils.')
    @argument('-c', '--compile-args', action='append', default=[], metavar='ARGS',
              help='Extra flags to pass to the compiler.')
    @cell_magic
    def pybind11(self, line, cell):
        """
        Compile and import everything from a pybind11 C++ code cell.

        The contents of the cell are written to a `.cpp` file in `$IPYTHONDIR/pybind11`
        directory using a filename based on the hash of the code. The file is compiled,
        and the symbols in the produced module are then imported directly into the
        current namespace.
        """

        line = line.strip().rstrip(';')
        args = parse_argstring(self.pybind11, line)
        module = 'pybind11_{}'.format(self.compute_hash(line, cell))
        code = self.format_code(cell)
        libfile = ext_path(module + ext_suffix())
        need_rebuild = not os.path.isfile(libfile) or args.force
        if need_rebuild:
            source = self.save_source(code, module)
            self.build_module(module, source, args)
        self.import_module(module, libfile)

    @line_magic
    def pybind11_capture(self, parameter_s=''):
        """
        Control the automatic capturing of C++ stdout and stderr streams.

        To enable:  `%pybind11_capture 1` or `%pybind11_capture on`.
        To disable: `%pybind11_capture 0` or `%pybind11_capture off`.
        To toggle:  `%pybind11_capture`.
        """

        if not is_kernel():
            print('C++ stdout/stderr capturing is not available in the terminal environment.')
            return

        p = parameter_s.strip().lower()
        if p:
            try:
                capture = {'off': 0, '0': 0, 'on': 1, '1': 1}[p]
            except KeyError:
                print('Incorrect argument. Use on/1, off/0, or nothing for a toggle.')
                return
        else:
            capture = not getattr(self.shell, 'pybind11_capture', False)
        self.shell.pybind11_capture = capture
        (start_forwarding if capture else stop_forwarding)()
        print('C++ stdout/stderr capturing has been turned', on_off(capture))

    def compute_hash(self, line, cell):
        key = cell, line, sys.version_info, sys.executable
        return hashlib.md5(str(key).encode('utf-8')).hexdigest()[:7]

    def format_code(self, cell):
        code = cell.replace('PYBIND11_PLUGIN', '_PYBIND11_PLUGIN')
        code = '#include <pybind11_preamble.h>\n' + code
        code += '\n' * (not code.endswith('\n'))
        return code

    def save_source(self, code, module):
        filename = ext_path(module + '.cpp')
        os.makedirs(os.path.dirname(filename), exist_ok=True)
        with open(filename, 'w') as f:
            f.write(code)
        return filename

    @contextlib.contextmanager
    def with_env(self, **env_vars):
        env_vars = {k: v for k, v in env_vars.items() if v is not None}
        env = os.environ.copy()
        for k, v in env_vars.items():
            os.environ[k] = v
        try:
            yield
        finally:
            for k in env_vars:
                if k not in env:
                    os.environ.pop(k, None)
                else:
                    os.environ[k] = env[k]

    def make_extension(self, module, source, args):
        include_dirs = [os.path.dirname(__file__)]
        include_dirs += pybind11_get_include()
        if args.prefix_include:
            include_dirs.append(os.path.join(sys.prefix, 'include'))
        extension = Extension(
            name=module,
            sources=[source],
            include_dirs=include_dirs,
            library_dirs=[],
            extra_compile_args=split_args(args.compile_args),
            extra_link_args=[],
            libraries=[],
            language='c++',
            define_macros=[
                ('_PYBIND11_MODULE_NAME', module)
            ]
        )
        extension.args = args
        return extension

    def forward_handler(self, source):
        def handler(data):
            data = data.replace(source, '<source>')
            data = re.sub(r'^/.+/(pybind11/[\w_]+\.h:)', r'\1',
                          data, flags=re.MULTILINE)
            data = re.sub(r'^/.+/pybind11_preamble.h:', 'pybind11_preamble.h:',
                          data, flags=re.MULTILINE)
            return data
        return handler

    def build_module(self, module, source, args):
        with self.with_env(**{'CC': args.cc, 'CXX': args.cxx}):
            workdir = ext_path(module)
            os.makedirs(workdir, exist_ok=True)
            script_args = ['-v' if args.verbose else '-q']
            script_args += ['build_ext', '--inplace', '--build-temp', workdir]
            if args.compiler is not None:
                script_args += ['--compiler', args.compiler]
            with forward(self.forward_handler(source)):
                warnings.filterwarnings('ignore', 'To exit')
                setup(
                    name=module,
                    ext_modules=[self.make_extension(module, source, args)],
                    script_args=script_args,
                    cmdclass={'build_ext': build_ext}
                )

    def import_module(self, module, libfile):
        mod = imp.load_dynamic(module, libfile)
        for k, v in mod.__dict__.items():
            if not k.startswith('__'):
                self.shell.push({k: v})
<|endoftext|>


<commit_before># Licensed under a 3-clause BSD style license - see LICENSE.rst
"""
This module provides the tools used to internally run the astropy test suite
from the installed astropy.  It makes use of the `pytest`_ testing framework.
"""
import os
import sys
import types
import pickle
import warnings
import functools

import pytest

from astropy.units import allclose as quantity_allclose  # noqa
from astropy.utils.exceptions import (AstropyDeprecationWarning,
                                      AstropyPendingDeprecationWarning)

# For backward-compatibility with affiliated packages
from .runner import TestRunner  # pylint: disable=W0611  # noqa

__all__ = ['raises', 'enable_deprecations_as_exceptions', 'remote_data',
           'treat_deprecations_as_exceptions', 'catch_warnings',
           'assert_follows_unicode_guidelines',
           'assert_quantity_allclose', 'check_pickling_recovery',
           'pickle_protocol', 'generic_recursive_equality_test']

# pytest marker to mark tests which get data from the web
# This is being maintained for backwards compatibility
remote_data = pytest.mark.remote_data

# distutils expects options to be Unicode strings
def _fix_user_options(options):
    def to_str_or_none(x):
        if x is None:
            return None
        return str(x)

    return [tuple(to_str_or_none(x) for x in y) for y in options]

def _save_coverage(cov, result, rootdir, testing_path):
    """
    This method is called after the tests have been run in coverage mode
    to cleanup and then save the coverage data and report.
    """
    from astropy.utils.console import color_print

    if result != 0:
        return

    # The coverage report includes the full path to the temporary
    # directory, so we replace all the paths with the true source
    # path. Note that this will not work properly for packages that still
    # rely on 2to3.
    try:
        # Coverage 4.0: _harvest_data has been renamed to get_data, the
        # lines dict is private
        cov.get_data()
    except AttributeError:
        # Coverage < 4.0
        cov._harvest_data()
        lines = cov.data.lines
    else:
        lines = cov.data._lines

    for key in list(lines.keys()):
        new_path = os.path.relpath(
            os.path.realpath(key),
            os.path.realpath(testing_path))
        new_path = os.path.abspath(
            os.path.join(rootdir, new_path))
        lines[new_path] = lines.pop(key)

    color_print('Saving coverage data in .coverage...', 'green')
    cov.save()

    color_print('Saving HTML coverage report in htmlcov...', 'green')
    cov.html_report(directory=os.path.join(rootdir, 'htmlcov'))

# TODO: Plan a roadmap of deprecation as pytest.raises has matured over the years.
# See https://github.com/astropy/astropy/issues/6761
class raises:
    """
    A decorator to mark that a test should raise a given exception.
    Use as follows::

        @raises(ZeroDivisionError)
        def test_foo():
            x = 1/0

    This can also be used a context manager, in which case it is just
    an alias for the ``pytest.raises`` context manager (because the
    two have the same name this help avoid confusion by being
    flexible).

    .. note:: Usage of ``pytest.raises`` is preferred.

    """

    # pep-8 naming exception -- this is a decorator class
    def __init__(self, exc):
        self._exc = exc
        self._ctx = None

    def __call__(self, func):
        @functools.wraps(func)
        def run_raises_test(*args, **kwargs):
            pytest.raises(self._exc, func, *args, **kwargs)
        return run_raises_test

    def __enter__(self):
        self._ctx = pytest.raises(self._exc)
        return self._ctx.__enter__()

    def __exit__(self, *exc_info):
        return self._ctx.__exit__(*exc_info)

_deprecations_as_exceptions = False
_include_astropy_deprecations = True
_modules_to_ignore_on_import = set([
    r'compiler',  # A deprecated stdlib module used by pytest
    r'scipy',
    r'pygments',
    r'ipykernel',
    r'IPython',   # deprecation warnings for async and await
    r'setuptools'])
_warnings_to_ignore_entire_module = set([])
_warnings_to_ignore_by_pyver = {
    None: set([  # Python version agnostic
        # https://github.com/astropy/astropy/pull/7372
        (r"Importing from numpy\.testing\.decorators is deprecated, "
         r"import from numpy\.testing instead\.", DeprecationWarning),
        # inspect raises this slightly different warning on Python 3.7.
        # Keeping it since e.g. lxml as of 3.8.0 is still calling getargspec()
        (r"inspect\.getargspec\(\) is deprecated, use "
         r"inspect\.signature\(\) or inspect\.getfullargspec\(\)",
         DeprecationWarning),
        # https://github.com/astropy/pytest-doctestplus/issues/29
        (r"split\(\) requires a non-empty pattern match", FutureWarning),
        # Package resolution warning that we can do nothing about
        (r"can't resolve package from __spec__ or __package__, "
         r"falling back on __name__ and __path__", ImportWarning)]),
    (3, 7): set([
        # Deprecation warning for collections.abc, fixed in Astropy but still
        # used in lxml, and maybe others
        (r"Using or importing the ABCs from 'collections'",
         DeprecationWarning)])
}

def enable_deprecations_as_exceptions(include_astropy_deprecations=True,
                                      modules_to_ignore_on_import=[],
                                      warnings_to_ignore_entire_module=[],
                                      warnings_to_ignore_by_pyver={}):
    """
    Turn on the feature that turns deprecations into exceptions.

    Parameters
    ----------
    include_astropy_deprecations : bool
        If set to `True`, ``AstropyDeprecationWarning`` and
        ``AstropyPendingDeprecationWarning`` are also turned into exceptions.

    modules_to_ignore_on_import : list of str
        List of additional modules that generate deprecation warnings
        on import, which are to be ignored. By default, these are already
        included: ``compiler``, ``scipy``, ``pygments``, ``ipykernel``, and
        ``setuptools``.

    warnings_to_ignore_entire_module : list of str
        List of modules with deprecation warnings to ignore completely,
        not just during import. If ``include_astropy_deprecations=True``
        is given, ``AstropyDeprecationWarning`` and
        ``AstropyPendingDeprecationWarning`` are also ignored for the modules.

    warnings_to_ignore_by_pyver : dict
        Dictionary mapping tuple of ``(major, minor)`` Python version to
        a list of ``(warning_message, warning_class)`` to ignore.
        Python version-agnostic warnings should be mapped to `None` key.
        This is in addition of those already ignored by default
        (see ``_warnings_to_ignore_by_pyver`` values).

    """
    global _deprecations_as_exceptions
    _deprecations_as_exceptions = True

    global _include_astropy_deprecations
    _include_astropy_deprecations = include_astropy_deprecations

    global _modules_to_ignore_on_import
    _modules_to_ignore_on_import.update(modules_to_ignore_on_import)

    global _warnings_to_ignore_entire_module
    _warnings_to_ignore_entire_module.update(warnings_to_ignore_entire_module)

    global _warnings_to_ignore_by_pyver
    for key, val in warnings_to_ignore_by_pyver.items():
        if key in _warnings_to_ignore_by_pyver:
            _warnings_to_ignore_by_pyver[key].update(val)
        else:
            _warnings_to_ignore_by_pyver[key] = set(val)

def treat_deprecations_as_exceptions():
    """
    Turn all DeprecationWarnings (which indicate deprecated uses of
    Python itself or Numpy, but not within Astropy, where we use our
    own deprecation warning class) into exceptions so that we find
    out about them early.

    This completely resets the warning filters and any "already seen"
    warning state.
    """
    # First, totally reset the warning state. The modules may change during
    # this iteration thus we copy the original state to a list to iterate
    # on. See https://github.com/astropy/astropy/pull/5513.
    for module in list(sys.modules.values()):
        try:
            del module.__warningregistry__
        except AttributeError:
            pass

    if not _deprecations_as_exceptions:
        return

    warnings.resetwarnings()

    # Hide the next couple of DeprecationWarnings
    warnings.simplefilter('ignore', DeprecationWarning)
    # Here's the wrinkle: a couple of our third-party dependencies
    # (pytest and scipy) are still using deprecated features
    # themselves, and we'd like to ignore those.  Fortunately, those
    # show up only at import time, so if we import those things *now*,
    # before we turn the warnings into exceptions, we're golden.
    for m in _modules_to_ignore_on_import:
        try:
            __import__(m)
        except ImportError:
            pass

    # Now, start over again with the warning filters
    warnings.resetwarnings()
    # Now, turn these warnings into exceptions
    _all_warns = [DeprecationWarning, FutureWarning, ImportWarning]

    # Only turn astropy deprecation warnings into exceptions if requested
    if _include_astropy_deprecations:
        _all_warns += [AstropyDeprecationWarning,
                       AstropyPendingDeprecationWarning]

    for w in _all_warns:
        warnings.filterwarnings("error", ".*", w)

    # This ignores all specified warnings from given module(s),
    # not just on import, for use of Astropy affiliated packages.
    for m in _warnings_to_ignore_entire_module:
        for w in _all_warns:
            warnings.filterwarnings('ignore', category=w, module=m)

    # This ignores only specified warnings by Python version, if applicable.
    for v in _warnings_to_ignore_by_pyver:
        if v is None or sys.version_info[:2] == v:
            for s in _warnings_to_ignore_by_pyver[v]:
                warnings.filterwarnings("ignore", s[0], s[1])

# TODO: Plan a roadmap of deprecation as pytest.warns has matured over the years.
# See https://github.com/astropy/astropy/issues/6761
class catch_warnings(warnings.catch_warnings):
    """
    A high-powered version of warnings.catch_warnings to use for testing
    and to make sure that there is no dependence on the order in which
    the tests are run.

    This completely blitzes any memory of any warnings that have
    appeared before so that all warnings will be caught and displayed.

    ``*args`` is a set of warning classes to collect.  If no arguments are
    provided, all warnings are collected.

    Use as follows::

        with catch_warnings(MyCustomWarning) as w:
            do.something.bad()
        assert len(w) > 0

    .. note:: Usage of :ref:`pytest.warns <pytest:warns>` is preferred.

    """

    def __init__(self, *classes):
        super().__init__(record=True)
        self.classes = classes

    def __enter__(self):
        warning_list = super().__enter__()
        treat_deprecations_as_exceptions()
        if len(self.classes) == 0:
            warnings.simplefilter('always')
        else:
            warnings.simplefilter('ignore')
            for cls in self.classes:
                warnings.simplefilter('always', cls)
        return warning_list

    def __exit__(self, type, value, traceback):
        treat_deprecations_as_exceptions()

class ignore_warnings(catch_warnings):
    """
    This can be used either as a context manager or function decorator to
    ignore all warnings that occur within a function or block of code.

    An optional category option can be supplied to only ignore warnings of a
    certain category or categories (if a list is provided).
    """

    def __init__(self, category=None):
        super().__init__()

        if isinstance(category, type) and issubclass(category, Warning):
            self.category = [category]
        else:
            self.category = category

    def __call__(self, func):
        @functools.wraps(func)
        def wrapper(*args, **kwargs):
            # Originally this just reused self, but that doesn't work if the
            # function is called more than once so we need to make a new
            # context manager instance for each call
            with self.__class__(category=self.category):
                return func(*args, **kwargs)

        return wrapper

    def __enter__(self):
        retval = super().__enter__()
        if self.category is not None:
            for category in self.category:
                warnings.simplefilter('ignore', category)
        else:
            warnings.simplefilter('ignore')
        return retval

def assert_follows_unicode_guidelines(
        x, roundtrip=None):
    """
    Test that an object follows our Unicode policy.  See
    "Unicode guidelines" in the coding guidelines.

    Parameters
    ----------
    x : object
        The instance to test

    roundtrip : module, optional
        When provided, this namespace will be used to evaluate
        ``repr(x)`` and ensure that it roundtrips.  It will also
        ensure that ``__bytes__(x)`` roundtrip.
        If not provided, no roundtrip testing will be performed.
    """
    from astropy import conf

    with conf.set_temp('unicode_output', False):
        bytes_x = bytes(x)
        unicode_x = str(x)
        repr_x = repr(x)

        assert isinstance(bytes_x, bytes)
        bytes_x.decode('ascii')
        assert isinstance(unicode_x, str)
        unicode_x.encode('ascii')
        assert isinstance(repr_x, str)
        if isinstance(repr_x, bytes):
            repr_x.decode('ascii')
        else:
            repr_x.encode('ascii')

        if roundtrip is not None:
            assert x.__class__(bytes_x) == x
            assert x.__class__(unicode_x) == x
            assert eval(repr_x, roundtrip) == x

    with conf.set_temp('unicode_output', True):
        bytes_x = bytes(x)
        unicode_x = str(x)
        repr_x = repr(x)

        assert isinstance(bytes_x, bytes)
        bytes_x.decode('ascii')
        assert isinstance(unicode_x, str)
        assert isinstance(repr_x, str)
        if isinstance(repr_x, bytes):
            repr_x.decode('ascii')
        else:
            repr_x.encode('ascii')

        if roundtrip is not None:
            assert x.__class__(bytes_x) == x
            assert x.__class__(unicode_x) == x
            assert eval(repr_x, roundtrip) == x

@pytest.fixture(params=[0, 1, -1])
def pickle_protocol(request):
    """
    Fixture to run all the tests for protocols 0 and 1, and -1 (most advanced).
    (Originally from astropy.table.tests.test_pickle)
    """
    return request.param

def generic_recursive_equality_test(a, b, class_history):
    """
    Check if the attributes of a and b are equal. Then,
    check if the attributes of the attributes are equal.
    """
    dict_a = a.__getstate__() if hasattr(a, '__getstate__') else a.__dict__
    dict_b = b.__dict__
    for key in dict_a:
        assert key in dict_b,\
          f"Did not pickle {key}"
        if hasattr(dict_a[key], '__eq__'):
            eq = (dict_a[key] == dict_b[key])
            if '__iter__' in dir(eq):
                eq = (False not in eq)
            assert eq, f"Value of {key} changed by pickling"

        if hasattr(dict_a[key], '__dict__'):
            if dict_a[key].__class__ in class_history:
                # attempt to prevent infinite recursion
                pass
            else:
                new_class_history = [dict_a[key].__class__]
                new_class_history.extend(class_history)
                generic_recursive_equality_test(dict_a[key],
                                                dict_b[key],
                                                new_class_history)

def check_pickling_recovery(original, protocol):
    """
    Try to pickle an object. If successful, make sure
    the object's attributes survived pickling and unpickling.
    """
    f = pickle.dumps(original, protocol=protocol)
    unpickled = pickle.loads(f)
    class_history = [original.__class__]
    generic_recursive_equality_test(original, unpickled,
                                    class_history)

def assert_quantity_allclose(actual, desired, rtol=1.e-7, atol=None,
                             **kwargs):
    """
    Raise an assertion if two objects are not equal up to desired tolerance.

    This is a :class:`~astropy.units.Quantity`-aware version of
    :func:`numpy.testing.assert_allclose`.
    """
    import numpy as np
    from astropy.units.quantity import _unquantify_allclose_arguments
    np.testing.assert_allclose(*_unquantify_allclose_arguments(
        actual, desired, rtol, atol), **kwargs)
<commit_msg>Ensure no unexpected failures in treating deprecations as exceptions.<commit_after># Licensed under a 3-clause BSD style license - see LICENSE.rst
"""
This module provides the tools used to internally run the astropy test suite
from the installed astropy.  It makes use of the `pytest`_ testing framework.
"""
import os
import sys
import types
import pickle
import warnings
import functools

import pytest

from astropy.units import allclose as quantity_allclose  # noqa
from astropy.utils.exceptions import (AstropyDeprecationWarning,
                                      AstropyPendingDeprecationWarning)

# For backward-compatibility with affiliated packages
from .runner import TestRunner  # pylint: disable=W0611  # noqa

__all__ = ['raises', 'enable_deprecations_as_exceptions', 'remote_data',
           'treat_deprecations_as_exceptions', 'catch_warnings',
           'assert_follows_unicode_guidelines',
           'assert_quantity_allclose', 'check_pickling_recovery',
           'pickle_protocol', 'generic_recursive_equality_test']

# pytest marker to mark tests which get data from the web
# This is being maintained for backwards compatibility
remote_data = pytest.mark.remote_data

# distutils expects options to be Unicode strings
def _fix_user_options(options):
    def to_str_or_none(x):
        if x is None:
            return None
        return str(x)

    return [tuple(to_str_or_none(x) for x in y) for y in options]

def _save_coverage(cov, result, rootdir, testing_path):
    """
    This method is called after the tests have been run in coverage mode
    to cleanup and then save the coverage data and report.
    """
    from astropy.utils.console import color_print

    if result != 0:
        return

    # The coverage report includes the full path to the temporary
    # directory, so we replace all the paths with the true source
    # path. Note that this will not work properly for packages that still
    # rely on 2to3.
    try:
        # Coverage 4.0: _harvest_data has been renamed to get_data, the
        # lines dict is private
        cov.get_data()
    except AttributeError:
        # Coverage < 4.0
        cov._harvest_data()
        lines = cov.data.lines
    else:
        lines = cov.data._lines

    for key in list(lines.keys()):
        new_path = os.path.relpath(
            os.path.realpath(key),
            os.path.realpath(testing_path))
        new_path = os.path.abspath(
            os.path.join(rootdir, new_path))
        lines[new_path] = lines.pop(key)

    color_print('Saving coverage data in .coverage...', 'green')
    cov.save()

    color_print('Saving HTML coverage report in htmlcov...', 'green')
    cov.html_report(directory=os.path.join(rootdir, 'htmlcov'))

# TODO: Plan a roadmap of deprecation as pytest.raises has matured over the years.
# See https://github.com/astropy/astropy/issues/6761
class raises:
    """
    A decorator to mark that a test should raise a given exception.
    Use as follows::

        @raises(ZeroDivisionError)
        def test_foo():
            x = 1/0

    This can also be used a context manager, in which case it is just
    an alias for the ``pytest.raises`` context manager (because the
    two have the same name this help avoid confusion by being
    flexible).

    .. note:: Usage of ``pytest.raises`` is preferred.

    """

    # pep-8 naming exception -- this is a decorator class
    def __init__(self, exc):
        self._exc = exc
        self._ctx = None

    def __call__(self, func):
        @functools.wraps(func)
        def run_raises_test(*args, **kwargs):
            pytest.raises(self._exc, func, *args, **kwargs)
        return run_raises_test

    def __enter__(self):
        self._ctx = pytest.raises(self._exc)
        return self._ctx.__enter__()

    def __exit__(self, *exc_info):
        return self._ctx.__exit__(*exc_info)

_deprecations_as_exceptions = False
_include_astropy_deprecations = True
_modules_to_ignore_on_import = set([
    r'compiler',  # A deprecated stdlib module used by pytest
    r'scipy',
    r'pygments',
    r'ipykernel',
    r'IPython',   # deprecation warnings for async and await
    r'setuptools'])
_warnings_to_ignore_entire_module = set([])
_warnings_to_ignore_by_pyver = {
    None: set([  # Python version agnostic
        # https://github.com/astropy/astropy/pull/7372
        (r"Importing from numpy\.testing\.decorators is deprecated, "
         r"import from numpy\.testing instead\.", DeprecationWarning),
        # inspect raises this slightly different warning on Python 3.7.
        # Keeping it since e.g. lxml as of 3.8.0 is still calling getargspec()
        (r"inspect\.getargspec\(\) is deprecated, use "
         r"inspect\.signature\(\) or inspect\.getfullargspec\(\)",
         DeprecationWarning),
        # https://github.com/astropy/pytest-doctestplus/issues/29
        (r"split\(\) requires a non-empty pattern match", FutureWarning),
        # Package resolution warning that we can do nothing about
        (r"can't resolve package from __spec__ or __package__, "
         r"falling back on __name__ and __path__", ImportWarning)]),
    (3, 7): set([
        # Deprecation warning for collections.abc, fixed in Astropy but still
        # used in lxml, and maybe others
        (r"Using or importing the ABCs from 'collections'",
         DeprecationWarning)])
}

def enable_deprecations_as_exceptions(include_astropy_deprecations=True,
                                      modules_to_ignore_on_import=[],
                                      warnings_to_ignore_entire_module=[],
                                      warnings_to_ignore_by_pyver={}):
    """
    Turn on the feature that turns deprecations into exceptions.

    Parameters
    ----------
    include_astropy_deprecations : bool
        If set to `True`, ``AstropyDeprecationWarning`` and
        ``AstropyPendingDeprecationWarning`` are also turned into exceptions.

    modules_to_ignore_on_import : list of str
        List of additional modules that generate deprecation warnings
        on import, which are to be ignored. By default, these are already
        included: ``compiler``, ``scipy``, ``pygments``, ``ipykernel``, and
        ``setuptools``.

    warnings_to_ignore_entire_module : list of str
        List of modules with deprecation warnings to ignore completely,
        not just during import. If ``include_astropy_deprecations=True``
        is given, ``AstropyDeprecationWarning`` and
        ``AstropyPendingDeprecationWarning`` are also ignored for the modules.

    warnings_to_ignore_by_pyver : dict
        Dictionary mapping tuple of ``(major, minor)`` Python version to
        a list of ``(warning_message, warning_class)`` to ignore.
        Python version-agnostic warnings should be mapped to `None` key.
        This is in addition of those already ignored by default
        (see ``_warnings_to_ignore_by_pyver`` values).

    """
    global _deprecations_as_exceptions
    _deprecations_as_exceptions = True

    global _include_astropy_deprecations
    _include_astropy_deprecations = include_astropy_deprecations

    global _modules_to_ignore_on_import
    _modules_to_ignore_on_import.update(modules_to_ignore_on_import)

    global _warnings_to_ignore_entire_module
    _warnings_to_ignore_entire_module.update(warnings_to_ignore_entire_module)

    global _warnings_to_ignore_by_pyver
    for key, val in warnings_to_ignore_by_pyver.items():
        if key in _warnings_to_ignore_by_pyver:
            _warnings_to_ignore_by_pyver[key].update(val)
        else:
            _warnings_to_ignore_by_pyver[key] = set(val)

def treat_deprecations_as_exceptions():
    """
    Turn all DeprecationWarnings (which indicate deprecated uses of
    Python itself or Numpy, but not within Astropy, where we use our
    own deprecation warning class) into exceptions so that we find
    out about them early.

    This completely resets the warning filters and any "already seen"
    warning state.
    """
    # First, totally reset the warning state. The modules may change during
    # this iteration thus we copy the original state to a list to iterate
    # on. See https://github.com/astropy/astropy/pull/5513.
    for module in list(sys.modules.values()):
        try:
            del module.__warningregistry__
        except Exception:
            pass

    if not _deprecations_as_exceptions:
        return

    warnings.resetwarnings()

    # Hide the next couple of DeprecationWarnings
    warnings.simplefilter('ignore', DeprecationWarning)
    # Here's the wrinkle: a couple of our third-party dependencies
    # (pytest and scipy) are still using deprecated features
    # themselves, and we'd like to ignore those.  Fortunately, those
    # show up only at import time, so if we import those things *now*,
    # before we turn the warnings into exceptions, we're golden.
    for m in _modules_to_ignore_on_import:
        try:
            __import__(m)
        except ImportError:
            pass

    # Now, start over again with the warning filters
    warnings.resetwarnings()
    # Now, turn these warnings into exceptions
    _all_warns = [DeprecationWarning, FutureWarning, ImportWarning]

    # Only turn astropy deprecation warnings into exceptions if requested
    if _include_astropy_deprecations:
        _all_warns += [AstropyDeprecationWarning,
                       AstropyPendingDeprecationWarning]

    for w in _all_warns:
        warnings.filterwarnings("error", ".*", w)

    # This ignores all specified warnings from given module(s),
    # not just on import, for use of Astropy affiliated packages.
    for m in _warnings_to_ignore_entire_module:
        for w in _all_warns:
            warnings.filterwarnings('ignore', category=w, module=m)

    # This ignores only specified warnings by Python version, if applicable.
    for v in _warnings_to_ignore_by_pyver:
        if v is None or sys.version_info[:2] == v:
            for s in _warnings_to_ignore_by_pyver[v]:
                warnings.filterwarnings("ignore", s[0], s[1])

# TODO: Plan a roadmap of deprecation as pytest.warns has matured over the years.
# See https://github.com/astropy/astropy/issues/6761
class catch_warnings(warnings.catch_warnings):
    """
    A high-powered version of warnings.catch_warnings to use for testing
    and to make sure that there is no dependence on the order in which
    the tests are run.

    This completely blitzes any memory of any warnings that have
    appeared before so that all warnings will be caught and displayed.

    ``*args`` is a set of warning classes to collect.  If no arguments are
    provided, all warnings are collected.

    Use as follows::

        with catch_warnings(MyCustomWarning) as w:
            do.something.bad()
        assert len(w) > 0

    .. note:: Usage of :ref:`pytest.warns <pytest:warns>` is preferred.

    """

    def __init__(self, *classes):
        super().__init__(record=True)
        self.classes = classes

    def __enter__(self):
        warning_list = super().__enter__()
        treat_deprecations_as_exceptions()
        if len(self.classes) == 0:
            warnings.simplefilter('always')
        else:
            warnings.simplefilter('ignore')
            for cls in self.classes:
                warnings.simplefilter('always', cls)
        return warning_list

    def __exit__(self, type, value, traceback):
        treat_deprecations_as_exceptions()

class ignore_warnings(catch_warnings):
    """
    This can be used either as a context manager or function decorator to
    ignore all warnings that occur within a function or block of code.

    An optional category option can be supplied to only ignore warnings of a
    certain category or categories (if a list is provided).
    """

    def __init__(self, category=None):
        super().__init__()

        if isinstance(category, type) and issubclass(category, Warning):
            self.category = [category]
        else:
            self.category = category

    def __call__(self, func):
        @functools.wraps(func)
        def wrapper(*args, **kwargs):
            # Originally this just reused self, but that doesn't work if the
            # function is called more than once so we need to make a new
            # context manager instance for each call
            with self.__class__(category=self.category):
                return func(*args, **kwargs)

        return wrapper

    def __enter__(self):
        retval = super().__enter__()
        if self.category is not None:
            for category in self.category:
                warnings.simplefilter('ignore', category)
        else:
            warnings.simplefilter('ignore')
        return retval

def assert_follows_unicode_guidelines(
        x, roundtrip=None):
    """
    Test that an object follows our Unicode policy.  See
    "Unicode guidelines" in the coding guidelines.

    Parameters
    ----------
    x : object
        The instance to test

    roundtrip : module, optional
        When provided, this namespace will be used to evaluate
        ``repr(x)`` and ensure that it roundtrips.  It will also
        ensure that ``__bytes__(x)`` roundtrip.
        If not provided, no roundtrip testing will be performed.
    """
    from astropy import conf

    with conf.set_temp('unicode_output', False):
        bytes_x = bytes(x)
        unicode_x = str(x)
        repr_x = repr(x)

        assert isinstance(bytes_x, bytes)
        bytes_x.decode('ascii')
        assert isinstance(unicode_x, str)
        unicode_x.encode('ascii')
        assert isinstance(repr_x, str)
        if isinstance(repr_x, bytes):
            repr_x.decode('ascii')
        else:
            repr_x.encode('ascii')

        if roundtrip is not None:
            assert x.__class__(bytes_x) == x
            assert x.__class__(unicode_x) == x
            assert eval(repr_x, roundtrip) == x

    with conf.set_temp('unicode_output', True):
        bytes_x = bytes(x)
        unicode_x = str(x)
        repr_x = repr(x)

        assert isinstance(bytes_x, bytes)
        bytes_x.decode('ascii')
        assert isinstance(unicode_x, str)
        assert isinstance(repr_x, str)
        if isinstance(repr_x, bytes):
            repr_x.decode('ascii')
        else:
            repr_x.encode('ascii')

        if roundtrip is not None:
            assert x.__class__(bytes_x) == x
            assert x.__class__(unicode_x) == x
            assert eval(repr_x, roundtrip) == x

@pytest.fixture(params=[0, 1, -1])
def pickle_protocol(request):
    """
    Fixture to run all the tests for protocols 0 and 1, and -1 (most advanced).
    (Originally from astropy.table.tests.test_pickle)
    """
    return request.param

def generic_recursive_equality_test(a, b, class_history):
    """
    Check if the attributes of a and b are equal. Then,
    check if the attributes of the attributes are equal.
    """
    dict_a = a.__getstate__() if hasattr(a, '__getstate__') else a.__dict__
    dict_b = b.__dict__
    for key in dict_a:
        assert key in dict_b,\
          f"Did not pickle {key}"
        if hasattr(dict_a[key], '__eq__'):
            eq = (dict_a[key] == dict_b[key])
            if '__iter__' in dir(eq):
                eq = (False not in eq)
            assert eq, f"Value of {key} changed by pickling"

        if hasattr(dict_a[key], '__dict__'):
            if dict_a[key].__class__ in class_history:
                # attempt to prevent infinite recursion
                pass
            else:
                new_class_history = [dict_a[key].__class__]
                new_class_history.extend(class_history)
                generic_recursive_equality_test(dict_a[key],
                                                dict_b[key],
                                                new_class_history)

def check_pickling_recovery(original, protocol):
    """
    Try to pickle an object. If successful, make sure
    the object's attributes survived pickling and unpickling.
    """
    f = pickle.dumps(original, protocol=protocol)
    unpickled = pickle.loads(f)
    class_history = [original.__class__]
    generic_recursive_equality_test(original, unpickled,
                                    class_history)

def assert_quantity_allclose(actual, desired, rtol=1.e-7, atol=None,
                             **kwargs):
    """
    Raise an assertion if two objects are not equal up to desired tolerance.

    This is a :class:`~astropy.units.Quantity`-aware version of
    :func:`numpy.testing.assert_allclose`.
    """
    import numpy as np
    from astropy.units.quantity import _unquantify_allclose_arguments
    np.testing.assert_allclose(*_unquantify_allclose_arguments(
        actual, desired, rtol, atol), **kwargs)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import contextlib
import os
import subprocess
import sys

import distutils.errors
import distutils.spawn

class inject:
    def __init__(self, fn):
        self.orig = fn
        self.set(fn)

    def set(self, fn):
        self.fn = fn

    def reset(self):
        self.fn = self.orig

    def __call__(self, *args, **kwargs):
        return self.fn(*args, **kwargs)

def patch_spawn():
    distutils.spawn.spawn = inject(distutils.spawn.spawn)

def spawn_fn(mode, fmt=None, log_commands=False):
    def spawn(cmd, search_path=True, verbose=False, dry_run=False):
        cmd = list(cmd)
        if search_path:
            cmd[0] = distutils.spawn.find_executable(cmd[0]) or cmd[0]
        if dry_run:
            return
        if log_commands:
            distutils.log.info(' '.join(distutils.spawn._nt_quote_args(list(cmd))))
        try:
            p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
            out, _ = p.communicate()
            if out and (mode == 'always' or (mode == 'on_error' and p.returncode != 0)):
                if fmt is not None:
                    out = fmt(out.decode('utf-8'))
                sep = '-' * 80 + '\n'
                sys.stdout.write(sep)
                sys.stdout.write(out)
                if not out.endswith('\n'):
                    sys.stdout.write('\n')
                sys.stdout.write(sep)
                sys.stdout.flush()
            if p.returncode != 0:
                raise subprocess.CalledProcessError(p.returncode, cmd)
        except OSError as e:
            raise distutils.errors.DistutilsExecError(
                'command {!r} failed with exit status {}: {}'
                .format(os.path.basename(cmd[0]), e.errno, e.strerror)) from None
        except:
            raise distutils.errors.DistutilsExecError(
                'command {!r} failed'
                .format(os.path.basename(cmd[0]))) from None
    return spawn

@contextlib.contextmanager
def spawn_capture(mode='on_error', fmt=None, log_commands=False):
    distutils.spawn.spawn.set(spawn_fn(mode, fmt=fmt, log_commands=log_commands))
    try:
        yield
    finally:
        distutils.spawn.spawn.reset()
<commit_msg>Ensure output is always decoded in spawn_capture()<commit_after># -*- coding: utf-8 -*-

import contextlib
import os
import subprocess
import sys

import distutils.errors
import distutils.spawn

class inject:
    def __init__(self, fn):
        self.orig = fn
        self.set(fn)

    def set(self, fn):
        self.fn = fn

    def reset(self):
        self.fn = self.orig

    def __call__(self, *args, **kwargs):
        return self.fn(*args, **kwargs)

def patch_spawn():
    distutils.spawn.spawn = inject(distutils.spawn.spawn)

def spawn_fn(mode, fmt=None, log_commands=False):
    def spawn(cmd, search_path=True, verbose=False, dry_run=False):
        cmd = list(cmd)
        if search_path:
            cmd[0] = distutils.spawn.find_executable(cmd[0]) or cmd[0]
        if dry_run:
            return
        if log_commands:
            distutils.log.info(' '.join(distutils.spawn._nt_quote_args(list(cmd))))
        try:
            p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
            out, _ = p.communicate()
            if out and (mode == 'always' or (mode == 'on_error' and p.returncode != 0)):
                out = out.decode('utf-8')
                if fmt is not None:
                    out = fmt(out)
                sep = '-' * 80 + '\n'
                sys.stdout.write(sep)
                sys.stdout.write(out)
                if not out.endswith('\n'):
                    sys.stdout.write('\n')
                sys.stdout.write(sep)
                sys.stdout.flush()
            if p.returncode != 0:
                raise subprocess.CalledProcessError(p.returncode, cmd)
        except OSError as e:
            raise distutils.errors.DistutilsExecError(
                'command {!r} failed with exit status {}: {}'
                .format(os.path.basename(cmd[0]), e.errno, e.strerror)) from None
        except:
            raise distutils.errors.DistutilsExecError(
                'command {!r} failed'
                .format(os.path.basename(cmd[0]))) from None
    return spawn

@contextlib.contextmanager
def spawn_capture(mode='on_error', fmt=None, log_commands=False):
    distutils.spawn.spawn.set(spawn_fn(mode, fmt=fmt, log_commands=log_commands))
    try:
        yield
    finally:
        distutils.spawn.spawn.reset()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import contextlib
import os
import subprocess
import sys

import distutils.errors
import distutils.spawn

class inject:
    def __init__(self, fn):
        self.orig = fn
        self.set(fn)

    def set(self, fn):
        self.fn = fn

    def reset(self):
        self.fn = self.orig

    def __call__(self, *args, **kwargs):
        return self.fn(*args, **kwargs)

def patch_spawn():
    distutils.spawn.spawn = inject(distutils.spawn.spawn)

def spawn_fn(mode, fmt=None):
    def spawn(cmd, search_path=True, verbose=False, dry_run=False):
        cmd = list(cmd)
        executable = cmd[0]
        if search_path:
            executable = distutils.spawn.find_executable(executable) or executable
        if dry_run:
            return
        try:
            p = subprocess.Popen([executable] + cmd[1:],
                                 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
            out, _ = p.communicate()
            if mode == 'always' or (mode == 'on_error' and p.returncode != 0):
                if fmt is not None:
                    out = fmt(out.decode('utf-8')).encode('utf-8')
                sys.stdout.write(out)
                sys.stdout.flush()
            if p.returncode != 0:
                raise subprocess.CalledProcessError
        except OSError as e:
            raise distutils.errors.DistutilsExecError(
                'command {!r} failed with exit status {}: {}'
                .format(executable, e.errno, e.strerror)) from None
        except:
            raise distutils.errors.DistutilsExecError(
                'command {!r} failed'
                .format(executable)) from None
    return spawn

@contextlib.contextmanager
def spawn_capture(mode='on_error', fmt=None):
    distutils.spawn.spawn.set(spawn_fn(mode, fmt=fmt))
    try:
        yield
    finally:
        distutils.spawn.spawn.reset()
<commit_msg>spawn_capture(): only report basename if it fails<commit_after># -*- coding: utf-8 -*-

import contextlib
import os
import subprocess
import sys

import distutils.errors
import distutils.spawn

class inject:
    def __init__(self, fn):
        self.orig = fn
        self.set(fn)

    def set(self, fn):
        self.fn = fn

    def reset(self):
        self.fn = self.orig

    def __call__(self, *args, **kwargs):
        return self.fn(*args, **kwargs)

def patch_spawn():
    distutils.spawn.spawn = inject(distutils.spawn.spawn)

def spawn_fn(mode, fmt=None):
    def spawn(cmd, search_path=True, verbose=False, dry_run=False):
        cmd = list(cmd)
        if search_path:
            cmd[0] = distutils.spawn.find_executable(cmd[0]) or cmd[0]
        if dry_run:
            return
        try:
            p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
            out, _ = p.communicate()
            if mode == 'always' or (mode == 'on_error' and p.returncode != 0):
                if fmt is not None:
                    out = fmt(out.decode('utf-8')).encode('utf-8')
                sys.stdout.write(out)
                sys.stdout.flush()
            if p.returncode != 0:
                raise subprocess.CalledProcessError
        except OSError as e:
            raise distutils.errors.DistutilsExecError(
                'command {!r} failed with exit status {}: {}'
                .format(os.path.basename(cmd[0]), e.errno, e.strerror)) from None
        except:
            raise distutils.errors.DistutilsExecError(
                'command {!r} failed'
                .format(os.path.basename(cmd[0]))) from None
    return spawn

@contextlib.contextmanager
def spawn_capture(mode='on_error', fmt=None):
    distutils.spawn.spawn.set(spawn_fn(mode, fmt=fmt))
    try:
        yield
    finally:
        distutils.spawn.spawn.reset()
<|endoftext|>


<commit_before>from django import forms
from django.forms import ModelForm, ModelChoiceField, MultiValueField, NullBooleanField, BooleanField
from django.forms import CharField
from django.forms import EmailField
from django.forms import Textarea
from django.forms import FileField
from django.forms import DecimalField
from django.forms import Form
import json
from models import TRequest, ProductionTask, StepExecution, MCPattern, MCPriority, TProject
from django.forms.widgets import TextInput
from django.forms import widgets

class RequestForm(ModelForm):
    cstatus =  CharField(label='Status', required=False)

    class Meta:
        model = TRequest

def energy_to_str(energy):
    if int(energy)==5023:
        return '5TeV'
    if int(energy)>1000:
        gev = str(int(energy))
        print gev[-3]
        if gev[-3]!='0':
            return gev[0:-3]+'p'+gev[-3:].rstrip('0')+'TeV'
        else:
            return gev[0:-3]+gev[-3:].rstrip('0')+'TeV'

    else:
        return str(energy)+'GeV'

class TRequestCreateCloneConfirmation(ModelForm):
    long_description = CharField(widget=Textarea, required=False)
    cc = CharField(required=False)
    description = CharField(label='Short description', widget=Textarea, required=True)
    cstatus = CharField(widget=forms.HiddenInput, required=False)
    project = ModelChoiceField(queryset=TProject.objects.all(),required=True)
    provenance = CharField(required=True)
    phys_group = CharField(required=True, widget=forms.Select(choices=TRequest.PHYS_GROUPS))
    campaign = CharField(required=True)
    need_approve = NullBooleanField(required=False,initial=True)

    class Meta:
        model = TRequest
        exclude = ['reqid','is_error','jira_reference','info_fields','is_fast']

    def clean(self):
        cleaned_data = super(TRequestCreateCloneConfirmation, self).clean()
        if type(self) == TRequestCreateCloneConfirmation:
            project = cleaned_data.get('project')
            energy = cleaned_data.get('energy_gev')
            if project:
                if 'eV' in str(project):
                    if energy_to_str(energy) not in str(project):
                        msg = "Energy doesn't correspond project"
                        self._errors['project'] = self.error_class([msg])
                        self._errors['energy_gev'] = self.error_class([msg])
                        del cleaned_data['project']
                        del cleaned_data['energy_gev']

        return cleaned_data

class TRequestMCCreateCloneForm(TRequestCreateCloneConfirmation):

    excellink = CharField(required=False, label="Spreadsheet Link")
    excelfile = FileField(required=False, label="Spreadsheet File")
    manager = CharField(widget=forms.HiddenInput, required=False)
    project = ModelChoiceField(queryset=TProject.objects.all(),required=False)
    phys_group = CharField(required=False, widget=forms.Select(choices=TRequest.PHYS_GROUPS))
    campaign = CharField(required=False)
    provenance = CharField(widget=forms.HiddenInput, required=False)
    description = CharField(label='Short description', widget=Textarea, required=False)
    need_approve = NullBooleanField(widget=forms.HiddenInput,required=False,initial=True)

    class Meta:
        model = TRequest
        exclude = ['reqid','is_error','jira_reference','info_fields', 'is_fast']

class TRequestDPDCreateCloneForm(TRequestCreateCloneConfirmation):
    excellink = CharField(required=False, label="DPD link")
    excelfile = FileField(required=False, label="DPD file")
    provenance = CharField(widget=forms.HiddenInput, required=False)
    cstatus = CharField(widget=forms.HiddenInput, required=False)
    request_type = CharField(widget=forms.HiddenInput, required=False)
    project = ModelChoiceField(queryset=TProject.objects.all(),required=False)
    hidden_json_slices = CharField(widget=forms.HiddenInput, required=False, label="Will be hidden")
    phys_group = CharField(required=False, widget=forms.Select(choices=TRequest.PHYS_GROUPS))
    campaign = CharField(required=False)
    description = CharField(label='Short description', widget=Textarea, required=False)
    need_approve = NullBooleanField(widget=forms.HiddenInput,required=False,initial=True)

    class Meta:
        model = TRequest
        exclude = ['reqid','is_error','jira_reference','info_fields' ,'is_fast']

class TRequestHLTCreateCloneForm(TRequestCreateCloneConfirmation):
    excellink = CharField(required=False, label="HLT LIST link")
    excelfile = FileField(required=False, label="HLT LIST file")
    provenance = CharField(widget=forms.HiddenInput, required=False)
    cstatus = CharField(widget=forms.HiddenInput, required=False)
    request_type = CharField(widget=forms.HiddenInput, required=False)
    project = ModelChoiceField(queryset=TProject.objects.all(),required=False)
    phys_group = CharField(required=False, widget=forms.Select(choices=TRequest.PHYS_GROUPS), initial='THLT')
    campaign = CharField(required=False)
    hidden_json_slices = CharField(widget=forms.HiddenInput, required=False, label="Will be hidden")
    description = CharField(label='Short description', widget=Textarea, required=False)
    need_approve = NullBooleanField(widget=forms.HiddenInput,required=False,initial=True)

    class Meta:
        model = TRequest
        exclude = ['reqid','is_error','jira_reference','info_fields','is_fast']

class PatternTextInput(widgets.MultiWidget):
    def __init__(self, attrs={'0':None,'1':None}):

        _widgets = (
            widgets.TextInput(attrs=attrs['0'] ),
            widgets.TextInput(attrs=attrs['1']),
            widgets.NumberInput(attrs=attrs['2']),
        )
        super(PatternTextInput, self).__init__(_widgets, attrs)

    def decompress(self, value):
        if value:
            return [value[0],value[1],value[2]]
        return None

    def format_output(self, rendered_widgets):
        return ''.join(rendered_widgets)

    def value_from_datadict(self, data, files, name):
        datelist = [
            widget.value_from_datadict(data, files, name + '_%s' % i)
            for i, widget in enumerate(self.widgets)]
        try:
            D = [datelist[0], datelist[1], datelist[2]]
        except ValueError:
            return ''
        else:
            return D

class DoubleCharField(MultiValueField):
    def __init__(self, *args, **kwargs):

        fields = (
            CharField(),
            CharField(),
            DecimalField()
        )
        super(DoubleCharField, self).__init__(fields=fields, *args, **kwargs)

    def compress(self, data_list):
        return json.dumps([str(x) for x in data_list])

class TRequestReprocessingCreateCloneForm(TRequestCreateCloneConfirmation):
    excellink = CharField(required=False, label="First step LIST link")
    excelfile = FileField(required=False, label="First step LIST file")
    provenance = CharField(widget=forms.HiddenInput, required=False)
    cstatus = CharField(widget=forms.HiddenInput, required=False)
    request_type = CharField(widget=forms.HiddenInput, required=False)
    project = ModelChoiceField(queryset=TProject.objects.all(),required=False)
    phys_group = CharField(required=False, widget=forms.Select(choices=TRequest.PHYS_GROUPS), initial='REPR')
    campaign = CharField(required=False)
    hidden_json_slices = CharField(widget=forms.HiddenInput, required=False, label="Will be hidden")
    description = CharField(label='Short description', widget=Textarea, required=False)
    need_approve = NullBooleanField(widget=forms.HiddenInput,required=False,initial=True)

    class Meta:
        model = TRequest
        exclude = ['reqid','is_error','jira_reference','info_fields','is_fast']

class MCPatternForm(ModelForm):

    def __init__(self, *args, **kwargs):
        steps = kwargs.pop('steps')
        super(MCPatternForm, self).__init__(*args, **kwargs)
        for step, value in steps:
            #self.fields['custom_%s' % step] = CharField(label=step, required=False)
            self.fields['custom_%s' % step] = DoubleCharField(label=step,
                                                              required=False,
                                                              widget=PatternTextInput(attrs={'0':{'placeholder':'ami tag', 'value':value[0]},
                                                                                '1':{'placeholder':'project mode', 'value':value[1]},
                                                                                '2':{'placeholder':'nEventsPerJob', 'value':value[2]}}))
            # if value:
            #     self.data['custom_%s' % step] = ['a','n',1]

    def steps_dict(self):
        return_dict = {}
        for name, value in self.cleaned_data.items():
            if name.startswith('custom_'):
                return_dict.update({self.fields[name].label: value})
        return return_dict

    class Meta:
        model = MCPattern
        exclude = ['id','pattern_dict']

class MCPatternUpdateForm(MCPatternForm):

    class Meta:
        model = MCPattern
        exclude = ['id','pattern_dict','pattern_name']

class MCPriorityForm(ModelForm):

    def __init__(self, *args, **kwargs):
        steps = kwargs.pop('steps')
        super(MCPriorityForm, self).__init__(*args, **kwargs)
        for step, value in steps:
            self.fields['custom_%s' % step] = DecimalField(label=step, required=True)
            if value:
                self.data['custom_%s' % step] = value

    def steps_dict(self):
        return_dict = {}
        for name, value in self.cleaned_data.items():
            if name.startswith('custom_'):
                return_dict.update({self.fields[name].label: int(value)})
        return return_dict

    class Meta:
        model = MCPriority
        exclude = ['id','priority_dict']

class MCPriorityUpdateForm(MCPriorityForm):

    class Meta:
        model = MCPriority
        exclude = ['id','priority_dict','priority_key']

class RequestUpdateForm(ModelForm):
    class Meta:
        model = TRequest
        widgets = {
            'reqid': TextInput(attrs={'readonly': 'readonly'}),
            'info_fields': TextInput(attrs={'readonly': 'readonly'}),
        }

class StepExecutionForm(ModelForm):
    class Meta:
        model = StepExecution

class ProductionTaskForm(ModelForm):
    class Meta:
        model = ProductionTask

class ProductionTaskCreateCloneForm(ModelForm):
    class Meta:
        model = ProductionTask

class ProductionTaskUpdateForm(ModelForm):
    class Meta:
        model = ProductionTask
        widgets = {
            'id': TextInput(attrs={'readonly': 'readonly'}),
        }<commit_msg>fix energy check<commit_after>from django import forms
from django.forms import ModelForm, ModelChoiceField, MultiValueField, NullBooleanField, BooleanField
from django.forms import CharField
from django.forms import EmailField
from django.forms import Textarea
from django.forms import FileField
from django.forms import DecimalField
from django.forms import Form
import json
from models import TRequest, ProductionTask, StepExecution, MCPattern, MCPriority, TProject
from django.forms.widgets import TextInput
from django.forms import widgets

class RequestForm(ModelForm):
    cstatus =  CharField(label='Status', required=False)

    class Meta:
        model = TRequest

def energy_to_str(energy):
    if (int(energy)==5023)or(int(energy)==5020):
        return '5TeV'
    if int(energy)>1000:
        gev = str(int(energy))
        print gev[-3]
        if gev[-3]!='0':
            return gev[0:-3]+'p'+gev[-3:].rstrip('0')+'TeV'
        else:
            return gev[0:-3]+gev[-3:].rstrip('0')+'TeV'

    else:
        return str(energy)+'GeV'

class TRequestCreateCloneConfirmation(ModelForm):
    long_description = CharField(widget=Textarea, required=False)
    cc = CharField(required=False)
    description = CharField(label='Short description', widget=Textarea, required=True)
    cstatus = CharField(widget=forms.HiddenInput, required=False)
    project = ModelChoiceField(queryset=TProject.objects.all(),required=True)
    provenance = CharField(required=True)
    phys_group = CharField(required=True, widget=forms.Select(choices=TRequest.PHYS_GROUPS))
    campaign = CharField(required=True)
    need_approve = NullBooleanField(required=False,initial=True)

    class Meta:
        model = TRequest
        exclude = ['reqid','is_error','jira_reference','info_fields','is_fast']

    def clean(self):
        cleaned_data = super(TRequestCreateCloneConfirmation, self).clean()
        if type(self) == TRequestCreateCloneConfirmation:
            project = cleaned_data.get('project')
            energy = cleaned_data.get('energy_gev')
            if project:
                if 'eV' in str(project):
                    if energy_to_str(energy) not in str(project):
                        msg = "Energy doesn't correspond project"
                        self._errors['project'] = self.error_class([msg])
                        self._errors['energy_gev'] = self.error_class([msg])
                        del cleaned_data['project']
                        del cleaned_data['energy_gev']

        return cleaned_data

class TRequestMCCreateCloneForm(TRequestCreateCloneConfirmation):

    excellink = CharField(required=False, label="Spreadsheet Link")
    excelfile = FileField(required=False, label="Spreadsheet File")
    manager = CharField(widget=forms.HiddenInput, required=False)
    project = ModelChoiceField(queryset=TProject.objects.all(),required=False)
    phys_group = CharField(required=False, widget=forms.Select(choices=TRequest.PHYS_GROUPS))
    campaign = CharField(required=False)
    provenance = CharField(widget=forms.HiddenInput, required=False)
    description = CharField(label='Short description', widget=Textarea, required=False)
    need_approve = NullBooleanField(widget=forms.HiddenInput,required=False,initial=True)

    class Meta:
        model = TRequest
        exclude = ['reqid','is_error','jira_reference','info_fields', 'is_fast']

class TRequestDPDCreateCloneForm(TRequestCreateCloneConfirmation):
    excellink = CharField(required=False, label="DPD link")
    excelfile = FileField(required=False, label="DPD file")
    provenance = CharField(widget=forms.HiddenInput, required=False)
    cstatus = CharField(widget=forms.HiddenInput, required=False)
    request_type = CharField(widget=forms.HiddenInput, required=False)
    project = ModelChoiceField(queryset=TProject.objects.all(),required=False)
    hidden_json_slices = CharField(widget=forms.HiddenInput, required=False, label="Will be hidden")
    phys_group = CharField(required=False, widget=forms.Select(choices=TRequest.PHYS_GROUPS))
    campaign = CharField(required=False)
    description = CharField(label='Short description', widget=Textarea, required=False)
    need_approve = NullBooleanField(widget=forms.HiddenInput,required=False,initial=True)

    class Meta:
        model = TRequest
        exclude = ['reqid','is_error','jira_reference','info_fields' ,'is_fast']

class TRequestHLTCreateCloneForm(TRequestCreateCloneConfirmation):
    excellink = CharField(required=False, label="HLT LIST link")
    excelfile = FileField(required=False, label="HLT LIST file")
    provenance = CharField(widget=forms.HiddenInput, required=False)
    cstatus = CharField(widget=forms.HiddenInput, required=False)
    request_type = CharField(widget=forms.HiddenInput, required=False)
    project = ModelChoiceField(queryset=TProject.objects.all(),required=False)
    phys_group = CharField(required=False, widget=forms.Select(choices=TRequest.PHYS_GROUPS), initial='THLT')
    campaign = CharField(required=False)
    hidden_json_slices = CharField(widget=forms.HiddenInput, required=False, label="Will be hidden")
    description = CharField(label='Short description', widget=Textarea, required=False)
    need_approve = NullBooleanField(widget=forms.HiddenInput,required=False,initial=True)

    class Meta:
        model = TRequest
        exclude = ['reqid','is_error','jira_reference','info_fields','is_fast']

class PatternTextInput(widgets.MultiWidget):
    def __init__(self, attrs={'0':None,'1':None}):

        _widgets = (
            widgets.TextInput(attrs=attrs['0'] ),
            widgets.TextInput(attrs=attrs['1']),
            widgets.NumberInput(attrs=attrs['2']),
        )
        super(PatternTextInput, self).__init__(_widgets, attrs)

    def decompress(self, value):
        if value:
            return [value[0],value[1],value[2]]
        return None

    def format_output(self, rendered_widgets):
        return ''.join(rendered_widgets)

    def value_from_datadict(self, data, files, name):
        datelist = [
            widget.value_from_datadict(data, files, name + '_%s' % i)
            for i, widget in enumerate(self.widgets)]
        try:
            D = [datelist[0], datelist[1], datelist[2]]
        except ValueError:
            return ''
        else:
            return D

class DoubleCharField(MultiValueField):
    def __init__(self, *args, **kwargs):

        fields = (
            CharField(),
            CharField(),
            DecimalField()
        )
        super(DoubleCharField, self).__init__(fields=fields, *args, **kwargs)

    def compress(self, data_list):
        return json.dumps([str(x) for x in data_list])

class TRequestReprocessingCreateCloneForm(TRequestCreateCloneConfirmation):
    excellink = CharField(required=False, label="First step LIST link")
    excelfile = FileField(required=False, label="First step LIST file")
    provenance = CharField(widget=forms.HiddenInput, required=False)
    cstatus = CharField(widget=forms.HiddenInput, required=False)
    request_type = CharField(widget=forms.HiddenInput, required=False)
    project = ModelChoiceField(queryset=TProject.objects.all(),required=False)
    phys_group = CharField(required=False, widget=forms.Select(choices=TRequest.PHYS_GROUPS), initial='REPR')
    campaign = CharField(required=False)
    hidden_json_slices = CharField(widget=forms.HiddenInput, required=False, label="Will be hidden")
    description = CharField(label='Short description', widget=Textarea, required=False)
    need_approve = NullBooleanField(widget=forms.HiddenInput,required=False,initial=True)

    class Meta:
        model = TRequest
        exclude = ['reqid','is_error','jira_reference','info_fields','is_fast']

class MCPatternForm(ModelForm):

    def __init__(self, *args, **kwargs):
        steps = kwargs.pop('steps')
        super(MCPatternForm, self).__init__(*args, **kwargs)
        for step, value in steps:
            #self.fields['custom_%s' % step] = CharField(label=step, required=False)
            self.fields['custom_%s' % step] = DoubleCharField(label=step,
                                                              required=False,
                                                              widget=PatternTextInput(attrs={'0':{'placeholder':'ami tag', 'value':value[0]},
                                                                                '1':{'placeholder':'project mode', 'value':value[1]},
                                                                                '2':{'placeholder':'nEventsPerJob', 'value':value[2]}}))
            # if value:
            #     self.data['custom_%s' % step] = ['a','n',1]

    def steps_dict(self):
        return_dict = {}
        for name, value in self.cleaned_data.items():
            if name.startswith('custom_'):
                return_dict.update({self.fields[name].label: value})
        return return_dict

    class Meta:
        model = MCPattern
        exclude = ['id','pattern_dict']

class MCPatternUpdateForm(MCPatternForm):

    class Meta:
        model = MCPattern
        exclude = ['id','pattern_dict','pattern_name']

class MCPriorityForm(ModelForm):

    def __init__(self, *args, **kwargs):
        steps = kwargs.pop('steps')
        super(MCPriorityForm, self).__init__(*args, **kwargs)
        for step, value in steps:
            self.fields['custom_%s' % step] = DecimalField(label=step, required=True)
            if value:
                self.data['custom_%s' % step] = value

    def steps_dict(self):
        return_dict = {}
        for name, value in self.cleaned_data.items():
            if name.startswith('custom_'):
                return_dict.update({self.fields[name].label: int(value)})
        return return_dict

    class Meta:
        model = MCPriority
        exclude = ['id','priority_dict']

class MCPriorityUpdateForm(MCPriorityForm):

    class Meta:
        model = MCPriority
        exclude = ['id','priority_dict','priority_key']

class RequestUpdateForm(ModelForm):
    class Meta:
        model = TRequest
        widgets = {
            'reqid': TextInput(attrs={'readonly': 'readonly'}),
            'info_fields': TextInput(attrs={'readonly': 'readonly'}),
        }

class StepExecutionForm(ModelForm):
    class Meta:
        model = StepExecution

class ProductionTaskForm(ModelForm):
    class Meta:
        model = ProductionTask

class ProductionTaskCreateCloneForm(ModelForm):
    class Meta:
        model = ProductionTask

class ProductionTaskUpdateForm(ModelForm):
    class Meta:
        model = ProductionTask
        widgets = {
            'id': TextInput(attrs={'readonly': 'readonly'}),
        }<|endoftext|>