<commit_before># ----------------------------------------------------------------------------
# Copyright (c) 2016-2022, QIIME 2 development team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file LICENSE, distributed with this software.
# ----------------------------------------------------------------------------

import click

from .type import QIIME2Type

# Sentinel to avoid the situation where `None` *is* the default value.
NoDefault = {}

class GeneratedOption(click.Option):
    def __init__(self, *, prefix, name, repr, ast, multiple, is_bool_flag,
                 metadata, metavar, default=NoDefault, description=None,
                 **attrs):
        import q2cli.util

        if metadata is not None:
            prefix = 'm'
        if multiple is not None:
            multiple = list if multiple == 'list' else set

        if is_bool_flag:
            yes = q2cli.util.to_cli_name(name)
            no = q2cli.util.to_cli_name('no_' + name)
            opt = f'--{prefix}-{yes}/--{prefix}-{no}'
        elif metadata is not None:
            cli_name = q2cli.util.to_cli_name(name)
            opt = f'--{prefix}-{cli_name}-file'
            if metadata == 'column':
                self.q2_extra_dest, self.q2_extra_opts, _ = \
                    self._parse_decls([f'--{prefix}-{cli_name}-column'], True)
        else:
            cli_name = q2cli.util.to_cli_name(name)
            opt = f'--{prefix}-{cli_name}'

        click_type = QIIME2Type(ast, repr, is_output=prefix == 'o')
        attrs['metavar'] = metavar
        attrs['multiple'] = multiple is not None
        attrs['param_decls'] = [opt]
        attrs['required'] = default is NoDefault
        attrs['help'] = self._add_default(description, default)
        if default is not NoDefault:
            attrs['default'] = default

        # This is to evade clicks __DEBUG__ check
        if not is_bool_flag:
            attrs['type'] = click_type
        else:
            attrs['type'] = None

        super().__init__(**attrs)
        # put things back the way they _should_ be after evading __DEBUG__
        self.is_bool_flag = is_bool_flag
        self.type = click_type

        # attrs we will use elsewhere
        self.q2_multiple = multiple
        self.q2_prefix = prefix
        self.q2_name = name
        self.q2_ast = ast
        self.q2_metadata = metadata

    @property
    def meta_help(self):
        if self.q2_metadata == 'file':
            return 'multiple arguments will be merged'

    def _add_default(self, desc, default):
        if desc is not None:
            desc += '  '
        else:
            desc = ''
        if default is not NoDefault:
            if default is None:
                desc += '[optional]'
            else:
                desc += '[default: %r]' % (default,)
        return desc

    def consume_value(self, ctx, opts):
        if self.q2_metadata == 'column':
            return self._consume_metadata(ctx, opts)
        else:
            return super().consume_value(ctx, opts)

    def _consume_metadata(self, ctx, opts):
        # double consume
        md_file, source = super().consume_value(ctx, opts)
        # consume uses self.name, so mutate but backup for after
        backup, self.name = self.name, self.q2_extra_dest
        md_col, _ = super().consume_value(ctx, opts)

        self.name = backup

        if (md_col is None) != (md_file is None):
            # missing one or the other
            if md_file is None:
                raise click.MissingParameter(ctx=ctx, param=self)
            else:
                raise click.MissingParameter(param_hint=self.q2_extra_opts,
                                             ctx=ctx, param=self)

        if md_col is None and md_file is None:
            return (None, source)
        else:
            return ((md_file, md_col), source)

    def get_help_record(self, ctx):
        record = super().get_help_record(ctx)
        if self.is_bool_flag:
            metavar = self.make_metavar()
            if metavar:
                record = (record[0] + ' ' + self.make_metavar(), record[1])
        elif self.q2_metadata == 'column':
            opts = (record[0], self.q2_extra_opts[0] + ' COLUMN ')
            record = (opts, record[1])
        return record

    # Override
    def add_to_parser(self, parser, ctx):
        shared = dict(dest=self.name, nargs=0, obj=self)
        if self.q2_metadata == 'column':
            parser.add_option(opts=self.opts, action='store', dest=self.name,
                              nargs=1, obj=self)
            parser.add_option(opts=self.q2_extra_opts, action='store',
                              dest=self.q2_extra_dest, nargs=1, obj=self)
        elif self.is_bool_flag:
            if self.multiple:
                action = 'append_maybe'
            else:
                action = 'store_maybe'
            parser.add_option(opts=self.opts, action=action, const=True,
                              **shared)
            parser.add_option(opts=self.secondary_opts, action=action,
                              const=False, **shared)
        elif self.multiple:
            action = 'append_greedy'
            parser.add_option(opts=self.opts, action='append_greedy', **shared)
        else:
            super().add_to_parser(parser, ctx)

    def get_default(self, ctx, call=True):
        if self.required and not ctx.resilient_parsing and not (
                self.q2_prefix == 'o' and ctx.params.get('output_dir', False)):
            raise click.MissingParameter(ctx=ctx, param=self)
        return super().get_default(ctx, call=call)

    def process_value(self, ctx, value):
        try:
            return super().process_value(ctx, value)
        except click.MissingParameter:
            if not (self.q2_prefix == 'o'
                    and ctx.params.get('output_dir', False)):
                raise

    def type_cast_value(self, ctx, value):
        import sys
        import q2cli.util
        import qiime2.sdk.util

        if self.multiple:
            if value == () or value is None:
                return None
            elif self.q2_prefix == 'i':
                value = super().type_cast_value(ctx, value)
                if self.q2_multiple is set:
                    self._check_length(value, ctx)
                value = self.q2_multiple(value)
                type_expr = qiime2.sdk.util.type_from_ast(self.q2_ast)
                args = ', '.join(map(repr, (x.type for x in value)))
                if value not in type_expr:
                    raise click.BadParameter(
                        'received <%s> as an argument, which is incompatible'
                        ' with parameter type: %r' % (args, type_expr),
                        ctx=ctx, param=self)
                return value
            elif self.q2_metadata == 'file':
                value = super().type_cast_value(ctx, value)
                if len(value) == 1:
                    return value[0]
                else:
                    try:
                        return value[0].merge(*value[1:])
                    except Exception as e:
                        header = ("There was an issue with merging "
                                  "QIIME 2 Metadata:")
                        tb = 'stderr' if '--verbose' in sys.argv else None
                        q2cli.util.exit_with_error(
                            e, header=header, traceback=tb)
            elif self.q2_prefix == 'p':
                try:
                    if self.q2_multiple is set:
                        self._check_length(value, ctx)
                    value = qiime2.sdk.util.parse_primitive(self.q2_ast, value)
                except ValueError:
                    args = ', '.join(map(repr, value))
                    expr = qiime2.sdk.util.type_from_ast(self.q2_ast)
                    raise click.BadParameter(
                        'received <%s> as an argument, which is incompatible'
                        ' with parameter type: %r' % (args, expr),
                        ctx=ctx, param=self)
                return value

        return super().type_cast_value(ctx, value)

    def _check_length(self, value, ctx):
        import collections

        counter = collections.Counter(value)
        dups = ', '.join(map(repr, (v for v, n in counter.items() if n > 1)))
        args = ', '.join(map(repr, value))
        if dups:
            raise click.BadParameter(
                'received <%s> as an argument, which contains duplicates'
                ' of the following: <%s>' % (args, dups), ctx=ctx, param=self)
<commit_msg>HACK: fix List[Bool] types (found by mystery-stew)<commit_after># ----------------------------------------------------------------------------
# Copyright (c) 2016-2022, QIIME 2 development team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file LICENSE, distributed with this software.
# ----------------------------------------------------------------------------

import click

from .type import QIIME2Type

# Sentinel to avoid the situation where `None` *is* the default value.
NoDefault = {}

class GeneratedOption(click.Option):
    def __init__(self, *, prefix, name, repr, ast, multiple, is_bool_flag,
                 metadata, metavar, default=NoDefault, description=None,
                 **attrs):
        import q2cli.util

        if metadata is not None:
            prefix = 'm'
        if multiple is not None:
            multiple = list if multiple == 'list' else set

        if is_bool_flag:
            yes = q2cli.util.to_cli_name(name)
            no = q2cli.util.to_cli_name('no_' + name)
            opt = f'--{prefix}-{yes}/--{prefix}-{no}'
        elif metadata is not None:
            cli_name = q2cli.util.to_cli_name(name)
            opt = f'--{prefix}-{cli_name}-file'
            if metadata == 'column':
                self.q2_extra_dest, self.q2_extra_opts, _ = \
                    self._parse_decls([f'--{prefix}-{cli_name}-column'], True)
        else:
            cli_name = q2cli.util.to_cli_name(name)
            opt = f'--{prefix}-{cli_name}'

        click_type = QIIME2Type(ast, repr, is_output=prefix == 'o')
        attrs['metavar'] = metavar
        attrs['multiple'] = multiple is not None
        attrs['param_decls'] = [opt]
        attrs['required'] = default is NoDefault
        attrs['help'] = self._add_default(description, default)
        if default is not NoDefault:
            attrs['default'] = default

        # This is to evade clicks __DEBUG__ check
        if not is_bool_flag:
            attrs['type'] = click_type
        else:
            attrs['type'] = None

        # This nonsense:
        # https://github.com/pallets/click/blob
        # /08f71b08e2b7ee9b1ea27daf6d3040999fc68551
        # /src/click/core.py#L2576-L2584
        if is_bool_flag and multiple is not None:
            to_add_multiple = attrs.pop('multiple')

        super().__init__(**attrs)

        if is_bool_flag and multiple is not None:
            self.multiple = to_add_multiple

        # put things back the way they _should_ be after evading __DEBUG__
        self.is_bool_flag = is_bool_flag
        self.type = click_type

        # attrs we will use elsewhere
        self.q2_multiple = multiple
        self.q2_prefix = prefix
        self.q2_name = name
        self.q2_ast = ast
        self.q2_metadata = metadata

    @property
    def meta_help(self):
        if self.q2_metadata == 'file':
            return 'multiple arguments will be merged'

    def _add_default(self, desc, default):
        if desc is not None:
            desc += '  '
        else:
            desc = ''
        if default is not NoDefault:
            if default is None:
                desc += '[optional]'
            else:
                desc += '[default: %r]' % (default,)
        return desc

    def consume_value(self, ctx, opts):
        if self.q2_metadata == 'column':
            return self._consume_metadata(ctx, opts)
        else:
            return super().consume_value(ctx, opts)

    def _consume_metadata(self, ctx, opts):
        # double consume
        md_file, source = super().consume_value(ctx, opts)
        # consume uses self.name, so mutate but backup for after
        backup, self.name = self.name, self.q2_extra_dest
        md_col, _ = super().consume_value(ctx, opts)

        self.name = backup

        if (md_col is None) != (md_file is None):
            # missing one or the other
            if md_file is None:
                raise click.MissingParameter(ctx=ctx, param=self)
            else:
                raise click.MissingParameter(param_hint=self.q2_extra_opts,
                                             ctx=ctx, param=self)

        if md_col is None and md_file is None:
            return (None, source)
        else:
            return ((md_file, md_col), source)

    def get_help_record(self, ctx):
        record = super().get_help_record(ctx)
        if self.is_bool_flag:
            metavar = self.make_metavar()
            if metavar:
                record = (record[0] + ' ' + self.make_metavar(), record[1])
        elif self.q2_metadata == 'column':
            opts = (record[0], self.q2_extra_opts[0] + ' COLUMN ')
            record = (opts, record[1])
        return record

    # Override
    def add_to_parser(self, parser, ctx):
        shared = dict(dest=self.name, nargs=0, obj=self)
        if self.q2_metadata == 'column':
            parser.add_option(opts=self.opts, action='store', dest=self.name,
                              nargs=1, obj=self)
            parser.add_option(opts=self.q2_extra_opts, action='store',
                              dest=self.q2_extra_dest, nargs=1, obj=self)
        elif self.is_bool_flag:
            if self.multiple:
                action = 'append_maybe'
            else:
                action = 'store_maybe'
            parser.add_option(opts=self.opts, action=action, const=True,
                              **shared)
            parser.add_option(opts=self.secondary_opts, action=action,
                              const=False, **shared)
        elif self.multiple:
            action = 'append_greedy'
            parser.add_option(opts=self.opts, action='append_greedy', **shared)
        else:
            super().add_to_parser(parser, ctx)

    def get_default(self, ctx, call=True):
        if self.required and not ctx.resilient_parsing and not (
                self.q2_prefix == 'o' and ctx.params.get('output_dir', False)):
            raise click.MissingParameter(ctx=ctx, param=self)
        return super().get_default(ctx, call=call)

    def process_value(self, ctx, value):
        try:
            return super().process_value(ctx, value)
        except click.MissingParameter:
            if not (self.q2_prefix == 'o'
                    and ctx.params.get('output_dir', False)):
                raise

    def type_cast_value(self, ctx, value):
        import sys
        import q2cli.util
        import qiime2.sdk.util

        if self.multiple:
            if value == () or value is None:
                return None
            elif self.q2_prefix == 'i':
                value = super().type_cast_value(ctx, value)
                if self.q2_multiple is set:
                    self._check_length(value, ctx)
                value = self.q2_multiple(value)
                type_expr = qiime2.sdk.util.type_from_ast(self.q2_ast)
                args = ', '.join(map(repr, (x.type for x in value)))
                if value not in type_expr:
                    raise click.BadParameter(
                        'received <%s> as an argument, which is incompatible'
                        ' with parameter type: %r' % (args, type_expr),
                        ctx=ctx, param=self)
                return value
            elif self.q2_metadata == 'file':
                value = super().type_cast_value(ctx, value)
                if len(value) == 1:
                    return value[0]
                else:
                    try:
                        return value[0].merge(*value[1:])
                    except Exception as e:
                        header = ("There was an issue with merging "
                                  "QIIME 2 Metadata:")
                        tb = 'stderr' if '--verbose' in sys.argv else None
                        q2cli.util.exit_with_error(
                            e, header=header, traceback=tb)
            elif self.q2_prefix == 'p':
                try:
                    if self.q2_multiple is set:
                        self._check_length(value, ctx)
                    value = qiime2.sdk.util.parse_primitive(self.q2_ast, value)
                except ValueError:
                    args = ', '.join(map(repr, value))
                    expr = qiime2.sdk.util.type_from_ast(self.q2_ast)
                    raise click.BadParameter(
                        'received <%s> as an argument, which is incompatible'
                        ' with parameter type: %r' % (args, expr),
                        ctx=ctx, param=self)
                return value

        return super().type_cast_value(ctx, value)

    def _check_length(self, value, ctx):
        import collections

        counter = collections.Counter(value)
        dups = ', '.join(map(repr, (v for v, n in counter.items() if n > 1)))
        args = ', '.join(map(repr, value))
        if dups:
            raise click.BadParameter(
                'received <%s> as an argument, which contains duplicates'
                ' of the following: <%s>' % (args, dups), ctx=ctx, param=self)
<|endoftext|>


<commit_before>import os

# Just to trick it into importing fir ReadTheDocs.
if os.environ.get('READTHEDOCS', None) == 'True':
    import sys.modules['qb'] = sys.modules[__name__]

from core import Executor, Future

_main = Executor()
submit = _main.submit
submit_ext = _main.submit_ext
map = _main.map
<commit_msg>Fix syntax error<commit_after>import os

# Just to trick it into importing fir ReadTheDocs.
if os.environ.get('READTHEDOCS', None) == 'True':
    import sys
    sys.modules['qb'] = sys.modules[__name__]

from core import Executor, Future

_main = Executor()
submit = _main.submit
submit_ext = _main.submit_ext
map = _main.map
<|endoftext|>


<commit_before>#!/usr/bin/env python

__author__ = "Greg Caporaso"
__copyright__ = "Copyright 2013, The QIIME Project"
__credits__ = ["Greg Caporaso", "Daniel McDonald", "Doug Wendel",
                       "Jai Ram Rideout"]
__license__ = "GPL"
__version__ = "0.1.0-dev"
__maintainer__ = "Greg Caporaso"
__email__ = "gregcaporaso@gmail.com"

from qcli.interface.core import Interface
from qcli.interface.factory import general_factory
from qcli.exception import IncompetentDeveloperError
from qcli.command.core import Parameter
from qcli.option_parsing import (OptionParser, OptionGroup, Option,
                                 OptionValueError, OptionError, make_option)

CLTypes = set(['float','int','string','existing_filepath', float, int, str, None])
CLActions = set(['store','store_true','store_false', 'append'])

class CLOption(Parameter):
    def __init__(self, Type, Help, Name, LongName, CLType, CLAction='store',
                 Required=False, Default=None, DefaultDescription=None,
                 ShortName=None):
        self.LongName = LongName
        self.CLType = CLType
        self.CLAction = CLAction
        self.ShortName = ShortName

        super(CLOption,self).__init__(Type=Type,Help=Help,Name=Name,
                                      Required=Required,Default=Default,
                                      DefaultDescription=DefaultDescription)

        if LongName != self.Name:
            self.DepWarn = "parameter %s will be renamed %s in QIIME 2.0.0" % \
                                                    (self.LongName, self.Name)
        else:
            self.DepWarn = ""

    def __str__(self):
        return '-%s/--%s' % (self.ShortName, self.LongName)

    @classmethod
    def fromParameter(cls, parameter, LongName, CLType, CLAction='store',
                      ShortName=None):
        result = cls(Type=parameter.Type,
                     Help=parameter.Help,
                     Name=parameter.Name,
                     Required=parameter.Required,
                     LongName=LongName,
                     CLType=CLType,
                     CLAction=CLAction,
                     Default=parameter.Default,
                     DefaultDescription=parameter.DefaultDescription,
                     ShortName=ShortName)
        return result

class UsageExample(object):
    def __init__(self, ShortDesc=None, LongDesc=None, Ex=None):
        if ShortDesc is None:
            raise IncompetentDeveloperError("No short description provided!")
        if LongDesc is None:
            raise IncompetentDeveloperError("No long description provided!")
        if Ex is None:
            raise IncompetentDeveloperError("No example provided!")

        self.ShortDesc = ShortDesc
        self.LongDesc = LongDesc
        self.Ex = Ex

    def to_tuple(self):
        """Returns (short, long, ex)"""
        return (self.ShortDesc, self.LongDesc, self.Ex)

class ParameterConversion(object):
    def __init__(self, ShortName=None, LongName=None, CLType=None,
                 CLAction=None):
        if LongName is None:
            raise IncompetentDeveloperError("No long name provided!")
        if CLType not in CLTypes:
            raise IncompetentDeveloperError("Invalid CLType specified: %s" % CLType)
        if CLAction is not None and CLAction not in CLActions:
            raise IncompetentDeveloperError("Invalid CLAction specified: %s" % CLAction)

        self.ShortName = ShortName
        self.LongName = LongName
        self.CLType = CLType
        self.CLAction = CLAction

    def to_dict(self):
        return {'short-name':self.ShortName,
                'long-name':self.LongName,
                'cl-type':self.CLType,
                'cl-action':self.CLAction}

class CLInterface(Interface):
    DisallowPositionalArguments = True
    HelpOnNoArguments = True
    OptionalInputLine = '[] indicates optional input (order unimportant)'
    RequiredInputLine = '{} indicates required input (order unimportant)'

    def __init__(self, **kwargs):
        self.UsageExamples = []
        self.UsageExamples.extend(self._get_usage_examples())

        if len(self.UsageExamples) < 1:
            raise IncompetentDeveloperError("How the fuck do I use this "
                                            "command?")

        self.ParameterConversionInfo = {
                'verbose':ParameterConversion(ShortName='v',
                                              LongName='verbose',
                                              CLType=None,
                                              CLAction='store_true')
        }

        self.ParameterConversionInfo.update(self._get_param_conv_info())

        super(CLInterface, self).__init__(**kwargs)

        self.Options.extend(self._get_additional_options())

    def _get_param_conv_info(self):
        raise NotImplementedError("Must define _get_param_conv_info")

    def _get_usage_examples(self):
        raise NotImplementedError("Must define _get_usage_examples")

    def _get_additional_options(self):
        raise NotImplementedError("Must define _get_additional_options")

    def _the_in_validator(self, in_):
        if not isinstance(in_, list):
            raise IncompetentDeveloperError("The in_ validator is very upset.")

    def _option_factory(self, parameter):
        name = parameter.Name
        if name not in self.ParameterConversionInfo:
            raise IncompetentDeveloperError("%s does not have parameter conversion info (parameter conversions are available for %s)" % (name, ' '.join(self.ParameterConversionInfo.keys())))

        return CLOption.fromParameter(parameter,
                     self.ParameterConversionInfo[name].LongName,
                     self.ParameterConversionInfo[name].CLType,
                     ShortName=self.ParameterConversionInfo[name].ShortName)

    def _input_handler(self, in_, *args, **kwargs):
        """ Constructs the OptionParser object and parses command line arguments

            parse_command_line_parameters takes a dict of objects via kwargs which
             it uses to build command line interfaces according to standards
             developed in the Knight Lab, and enforced in QIIME. The currently
             supported options are listed below with their default values. If no
             default is provided, the option is required.

            script_description
            script_usage = [("","","")]
            version
            required_options=None
            optional_options=None
            suppress_verbose=False
            disallow_positional_arguments=True
            help_on_no_arguments=True
            optional_input_line = '[] indicates optional input (order unimportant)'
            required_input_line = '{} indicates required input (order unimportant)'

           These values can either be passed directly, as:
            parse_command_line_parameters(script_description="My script",\
                                         script_usage=[('Print help','%prog -h','')],\
                                         version=1.0)

           or they can be passed via a pre-constructed dict, as:
            d = {'script_description':"My script",\
                 'script_usage':[('Print help','%prog -h','')],\
                 'version':1.0}
            parse_command_line_parameters(**d)

        """
        # command_line_text will usually be nothing, but can be passed for
        # testing purposes

        # Do we need this? Was used for testing
        #command_line_args = set_parameter('command_line_args',kwargs,None)

        required_opts = [opt for opt in self.Options if opt.Required]
        optional_opts = [opt for opt in self.Options if not opt.Required]

        # Build the usage and version strings
        usage = self._build_usage_lines(required_opts)
        version = 'Version: %prog ' + __version__

        # Instantiate the command line parser object
        parser = OptionParser(usage=usage, version=version)

        # What does this do?
        #parser.exit = set_parameter('exit_func',kwargs,parser.exit)

        # If no arguments were provided, print the help string (unless the
        # caller specified not to)

        # Need to figure out what to do with command_line_args
        #if self.HelpOnNoArguments and (not command_line_args) and len(argv) == 1:
        if self.HelpOnNoArguments and len(in_) == 0:
            parser.print_usage()
            return parser.exit(-1)

        # Process the required options
        if required_opts:
            # Define an option group so all required options are
            # grouped together, and under a common header
            required = OptionGroup(parser, "REQUIRED options",
                                   "The following options must be provided "
                                   "under all circumstances.")
            for ro in required_opts:
                # if the option doesn't already end with [REQUIRED], add it.
                if not ro.Help.strip().endswith('[REQUIRED]'):
                    ro.Help += ' [REQUIRED]'
                if ro.ShortName is None:
                    option = make_option('--' + ro.LongName,
                                         type=ro.CLType, help=ro.Help)
                else:
                    option = make_option('-' + ro.ShortName, '--' + ro.LongName,
                                         type=ro.CLType, help=ro.Help)
                required.add_option(option)
            parser.add_option_group(required)

        # Add the optional options
        for oo in optional_opts:
            help_text = '%s [default: %s]' % (oo.Help, oo.DefaultDescription)
            if oo.ShortName is None:
                option = make_option('--' + oo.LongName,
                     type=oo.CLType, help=help_text,
                     default=oo.Default)
            else:
                option = make_option('-' + oo.ShortName, '--' + oo.LongName,
                                     type=oo.CLType, help=help_text,
                                     default=oo.Default)
            parser.add_option(option)

        # Parse the command line
        # command_line_text will None except in test cases, in which
        # case sys.argv[1:] will be parsed

        # Need to figure out what to do with command_line_args
        #opts,args = parser.parse_args(command_line_args)
        opts, args = parser.parse_args(in_)

        # If positional arguments are not allowed, and any were provided, raise
        # an error.
        if self.DisallowPositionalArguments and len(args) != 0:
            parser.error("Positional argument detected: %s\n" % str(args[0]) +
             " Be sure all parameters are identified by their option name.\n" +
             " (e.g.: include the '-i' in '-i INPUT_DIR')")

        # Test that all required options were provided.
        if required_opts:
            required_option_ids = [o.dest for o in required.option_list]
            for required_option_id in required_option_ids:
                if getattr(opts,required_option_id) == None:
                    return parser.error('Required option --%s omitted.' %
                                        required_option_id)

        # Return the parser, the options, and the arguments. The parser is returned
        # so users have access to any additional functionality they may want at
        # this stage -- most commonly, it will be used for doing custom tests of
        # parameter values.
        hated_functionality = eval(str(opts))
        return hated_functionality

    def _build_usage_lines(self, required_options):
        """ Build the usage string from components """
        line1 = 'usage: %prog [options] ' + \
                '{%s}' % ' '.join(['%s %s' % (str(rp),rp.Name.upper())
                                   for rp in required_options])

        formatted_usage_examples = []
        for usage_example in self.UsageExamples:
            short_description = usage_example.ShortDesc.strip(':').strip()
            long_description = usage_example.LongDesc.strip(':').strip()
            example = usage_example.Ex.strip()
            if short_description:
                formatted_usage_examples.append('%s: %s\n %s' %
                                                (short_description,long_description,example))
            else:
                formatted_usage_examples.append('%s\n %s' %
                                                (long_description,example))

        formatted_usage_examples = '\n\n'.join(formatted_usage_examples)

        lines = (line1,
                 '', # Blank line
                 self.OptionalInputLine,
                 self.RequiredInputLine,
                 '', # Blank line
                 self.CmdInstance.LongDescription,
                 '', # Blank line
                 'Example usage: ',
                 'Print help message and exit',
                 ' %prog -h\n',
                 formatted_usage_examples)

        return '\n'.join(lines)

    def _output_handler(self, results):
        pass

    def getOutputFilepaths(results, **kwargs):
        mapping = {}

        for k,v in results.items():
            if isinstance(v, FilePath):
                output_fp = str(v)
            else:
                # figure out filepath
                pass

            mapping[k] = output_fp

        return mapping

def cli(command_constructor, usage_examples, param_conversions, added_options):
    """Command line interface factory

    command_constructor - a subclass of ``Command``
    usage_examples - usage examples for using ``command_constructor`` on via a
        command line interface.
    param_conversions - necessary conversion information to converting
        parameters to options.
    added_options - any additional options that are not defined by the
        ``command_constructor``.
    """
    return general_factory(command_constructor, usage_examples, param_conversions,
                           added_options, CLInterface)

def clmain(interface_object, local_argv):
    cli_cmd = interface_object()

    result = cli_cmd(local_argv[1:])

    return 0
<commit_msg>added cl types<commit_after>#!/usr/bin/env python

__author__ = "Greg Caporaso"
__copyright__ = "Copyright 2013, The QIIME Project"
__credits__ = ["Greg Caporaso", "Daniel McDonald", "Doug Wendel",
                       "Jai Ram Rideout"]
__license__ = "GPL"
__version__ = "0.1.0-dev"
__maintainer__ = "Greg Caporaso"
__email__ = "gregcaporaso@gmail.com"

from qcli.interface.core import Interface
from qcli.interface.factory import general_factory
from qcli.exception import IncompetentDeveloperError
from qcli.command.core import Parameter
from qcli.option_parsing import (OptionParser, OptionGroup, Option,
                                 OptionValueError, OptionError, make_option)

CLTypes = set(['float','int','string','existing_filepath', float, int, str, None,
               'new_filepath','new_dirpath','existing_dirpath'])
CLActions = set(['store','store_true','store_false', 'append'])

class CLOption(Parameter):
    def __init__(self, Type, Help, Name, LongName, CLType, CLAction='store',
                 Required=False, Default=None, DefaultDescription=None,
                 ShortName=None):
        self.LongName = LongName
        self.CLType = CLType
        self.CLAction = CLAction
        self.ShortName = ShortName

        super(CLOption,self).__init__(Type=Type,Help=Help,Name=Name,
                                      Required=Required,Default=Default,
                                      DefaultDescription=DefaultDescription)

        if LongName != self.Name:
            self.DepWarn = "parameter %s will be renamed %s in QIIME 2.0.0" % \
                                                    (self.LongName, self.Name)
        else:
            self.DepWarn = ""

    def __str__(self):
        return '-%s/--%s' % (self.ShortName, self.LongName)

    @classmethod
    def fromParameter(cls, parameter, LongName, CLType, CLAction='store',
                      ShortName=None):
        result = cls(Type=parameter.Type,
                     Help=parameter.Help,
                     Name=parameter.Name,
                     Required=parameter.Required,
                     LongName=LongName,
                     CLType=CLType,
                     CLAction=CLAction,
                     Default=parameter.Default,
                     DefaultDescription=parameter.DefaultDescription,
                     ShortName=ShortName)
        return result

class UsageExample(object):
    def __init__(self, ShortDesc=None, LongDesc=None, Ex=None):
        if ShortDesc is None:
            raise IncompetentDeveloperError("No short description provided!")
        if LongDesc is None:
            raise IncompetentDeveloperError("No long description provided!")
        if Ex is None:
            raise IncompetentDeveloperError("No example provided!")

        self.ShortDesc = ShortDesc
        self.LongDesc = LongDesc
        self.Ex = Ex

    def to_tuple(self):
        """Returns (short, long, ex)"""
        return (self.ShortDesc, self.LongDesc, self.Ex)

class ParameterConversion(object):
    def __init__(self, ShortName=None, LongName=None, CLType=None,
                 CLAction=None):
        if LongName is None:
            raise IncompetentDeveloperError("No long name provided!")
        if CLType not in CLTypes:
            raise IncompetentDeveloperError("Invalid CLType specified: %s" % CLType)
        if CLAction is not None and CLAction not in CLActions:
            raise IncompetentDeveloperError("Invalid CLAction specified: %s" % CLAction)

        self.ShortName = ShortName
        self.LongName = LongName
        self.CLType = CLType
        self.CLAction = CLAction

    def to_dict(self):
        return {'short-name':self.ShortName,
                'long-name':self.LongName,
                'cl-type':self.CLType,
                'cl-action':self.CLAction}

class CLInterface(Interface):
    DisallowPositionalArguments = True
    HelpOnNoArguments = True
    OptionalInputLine = '[] indicates optional input (order unimportant)'
    RequiredInputLine = '{} indicates required input (order unimportant)'

    def __init__(self, **kwargs):
        self.UsageExamples = []
        self.UsageExamples.extend(self._get_usage_examples())

        if len(self.UsageExamples) < 1:
            raise IncompetentDeveloperError("How the fuck do I use this "
                                            "command?")

        self.ParameterConversionInfo = {
                'verbose':ParameterConversion(ShortName='v',
                                              LongName='verbose',
                                              CLType=None,
                                              CLAction='store_true')
        }

        self.ParameterConversionInfo.update(self._get_param_conv_info())

        super(CLInterface, self).__init__(**kwargs)

        self.Options.extend(self._get_additional_options())

    def _get_param_conv_info(self):
        raise NotImplementedError("Must define _get_param_conv_info")

    def _get_usage_examples(self):
        raise NotImplementedError("Must define _get_usage_examples")

    def _get_additional_options(self):
        raise NotImplementedError("Must define _get_additional_options")

    def _the_in_validator(self, in_):
        if not isinstance(in_, list):
            raise IncompetentDeveloperError("The in_ validator is very upset.")

    def _option_factory(self, parameter):
        name = parameter.Name
        if name not in self.ParameterConversionInfo:
            raise IncompetentDeveloperError("%s does not have parameter conversion info (parameter conversions are available for %s)" % (name, ' '.join(self.ParameterConversionInfo.keys())))

        return CLOption.fromParameter(parameter,
                     self.ParameterConversionInfo[name].LongName,
                     self.ParameterConversionInfo[name].CLType,
                     ShortName=self.ParameterConversionInfo[name].ShortName)

    def _input_handler(self, in_, *args, **kwargs):
        """ Constructs the OptionParser object and parses command line arguments

            parse_command_line_parameters takes a dict of objects via kwargs which
             it uses to build command line interfaces according to standards
             developed in the Knight Lab, and enforced in QIIME. The currently
             supported options are listed below with their default values. If no
             default is provided, the option is required.

            script_description
            script_usage = [("","","")]
            version
            required_options=None
            optional_options=None
            suppress_verbose=False
            disallow_positional_arguments=True
            help_on_no_arguments=True
            optional_input_line = '[] indicates optional input (order unimportant)'
            required_input_line = '{} indicates required input (order unimportant)'

           These values can either be passed directly, as:
            parse_command_line_parameters(script_description="My script",\
                                         script_usage=[('Print help','%prog -h','')],\
                                         version=1.0)

           or they can be passed via a pre-constructed dict, as:
            d = {'script_description':"My script",\
                 'script_usage':[('Print help','%prog -h','')],\
                 'version':1.0}
            parse_command_line_parameters(**d)

        """
        # command_line_text will usually be nothing, but can be passed for
        # testing purposes

        # Do we need this? Was used for testing
        #command_line_args = set_parameter('command_line_args',kwargs,None)

        required_opts = [opt for opt in self.Options if opt.Required]
        optional_opts = [opt for opt in self.Options if not opt.Required]

        # Build the usage and version strings
        usage = self._build_usage_lines(required_opts)
        version = 'Version: %prog ' + __version__

        # Instantiate the command line parser object
        parser = OptionParser(usage=usage, version=version)

        # What does this do?
        #parser.exit = set_parameter('exit_func',kwargs,parser.exit)

        # If no arguments were provided, print the help string (unless the
        # caller specified not to)

        # Need to figure out what to do with command_line_args
        #if self.HelpOnNoArguments and (not command_line_args) and len(argv) == 1:
        if self.HelpOnNoArguments and len(in_) == 0:
            parser.print_usage()
            return parser.exit(-1)

        # Process the required options
        if required_opts:
            # Define an option group so all required options are
            # grouped together, and under a common header
            required = OptionGroup(parser, "REQUIRED options",
                                   "The following options must be provided "
                                   "under all circumstances.")
            for ro in required_opts:
                # if the option doesn't already end with [REQUIRED], add it.
                if not ro.Help.strip().endswith('[REQUIRED]'):
                    ro.Help += ' [REQUIRED]'
                if ro.ShortName is None:
                    option = make_option('--' + ro.LongName,
                                         type=ro.CLType, help=ro.Help)
                else:
                    option = make_option('-' + ro.ShortName, '--' + ro.LongName,
                                         type=ro.CLType, help=ro.Help)
                required.add_option(option)
            parser.add_option_group(required)

        # Add the optional options
        for oo in optional_opts:
            help_text = '%s [default: %s]' % (oo.Help, oo.DefaultDescription)
            if oo.ShortName is None:
                option = make_option('--' + oo.LongName,
                     type=oo.CLType, help=help_text,
                     default=oo.Default)
            else:
                option = make_option('-' + oo.ShortName, '--' + oo.LongName,
                                     type=oo.CLType, help=help_text,
                                     default=oo.Default)
            parser.add_option(option)

        # Parse the command line
        # command_line_text will None except in test cases, in which
        # case sys.argv[1:] will be parsed

        # Need to figure out what to do with command_line_args
        #opts,args = parser.parse_args(command_line_args)
        opts, args = parser.parse_args(in_)

        # If positional arguments are not allowed, and any were provided, raise
        # an error.
        if self.DisallowPositionalArguments and len(args) != 0:
            parser.error("Positional argument detected: %s\n" % str(args[0]) +
             " Be sure all parameters are identified by their option name.\n" +
             " (e.g.: include the '-i' in '-i INPUT_DIR')")

        # Test that all required options were provided.
        if required_opts:
            required_option_ids = [o.dest for o in required.option_list]
            for required_option_id in required_option_ids:
                if getattr(opts,required_option_id) == None:
                    return parser.error('Required option --%s omitted.' %
                                        required_option_id)

        # Return the parser, the options, and the arguments. The parser is returned
        # so users have access to any additional functionality they may want at
        # this stage -- most commonly, it will be used for doing custom tests of
        # parameter values.
        hated_functionality = eval(str(opts))
        return hated_functionality

    def _build_usage_lines(self, required_options):
        """ Build the usage string from components """
        line1 = 'usage: %prog [options] ' + \
                '{%s}' % ' '.join(['%s %s' % (str(rp),rp.Name.upper())
                                   for rp in required_options])

        formatted_usage_examples = []
        for usage_example in self.UsageExamples:
            short_description = usage_example.ShortDesc.strip(':').strip()
            long_description = usage_example.LongDesc.strip(':').strip()
            example = usage_example.Ex.strip()
            if short_description:
                formatted_usage_examples.append('%s: %s\n %s' %
                                                (short_description,long_description,example))
            else:
                formatted_usage_examples.append('%s\n %s' %
                                                (long_description,example))

        formatted_usage_examples = '\n\n'.join(formatted_usage_examples)

        lines = (line1,
                 '', # Blank line
                 self.OptionalInputLine,
                 self.RequiredInputLine,
                 '', # Blank line
                 self.CmdInstance.LongDescription,
                 '', # Blank line
                 'Example usage: ',
                 'Print help message and exit',
                 ' %prog -h\n',
                 formatted_usage_examples)

        return '\n'.join(lines)

    def _output_handler(self, results):
        pass

    def getOutputFilepaths(results, **kwargs):
        mapping = {}

        for k,v in results.items():
            if isinstance(v, FilePath):
                output_fp = str(v)
            else:
                # figure out filepath
                pass

            mapping[k] = output_fp

        return mapping

def cli(command_constructor, usage_examples, param_conversions, added_options):
    """Command line interface factory

    command_constructor - a subclass of ``Command``
    usage_examples - usage examples for using ``command_constructor`` on via a
        command line interface.
    param_conversions - necessary conversion information to converting
        parameters to options.
    added_options - any additional options that are not defined by the
        ``command_constructor``.
    """
    return general_factory(command_constructor, usage_examples, param_conversions,
                           added_options, CLInterface)

def clmain(interface_object, local_argv):
    cli_cmd = interface_object()

    result = cli_cmd(local_argv[1:])

    return 0
<|endoftext|>


<commit_before>from subprocess import Popen, PIPE

from IPython.parallel import Client

from qiita_ware.exceptions import ComputeError

def system_call(cmd):
    """Call cmd and return (stdout, stderr, return_value).

    cmd: can be either a string containing the command to be run, or a
     sequence of strings that are the tokens of the command.

    This function is ported from QIIME (http://www.qiime.org), previously
    named qiime_system_call. QIIME is a GPL project, but we obtained permission
    from the authors of this function to port it to pyqi (and keep it under
    pyqi's BSD license).
    """
    proc = Popen(cmd,
                 universal_newlines=True,
                 shell=True,
                 stdout=PIPE,
                 stderr=PIPE)
    # communicate pulls all stdout/stderr from the PIPEs to
    # avoid blocking -- don't remove this line!
    stdout, stderr = proc.communicate()
    return_value = proc.returncode

    if return_value != 0:
        raise ComputeError("Failed to execute: %s\nstdout: %s\nstderr: %s" %
                           (cmd, stdout, stderr))

    return stdout, stderr, return_value

class Dispatch(object):
    """Dispatch compute

    Attributes
    ----------
    reserved
    reserved_lview
    general
    general_lview
    demo
    demo_lview

    Methods
    -------
    submit_async
    submit_sync
    sync

    """
    def __init__(self):
        # self.reserved = Client(profile='qiita_reserved')
        # self.general = Client(profile='qiita_general')
        self.demo = Client(profile='qiita_demo')

        # self._stage_imports(self.reserved)
        # self._stage_imports(self.general)
        self._stage_imports(self.demo)

        # self.reserved_lview = self.reserved.load_balanced_view()
        # self.general_lview = self.general.load_balanced_view()
        self.demo_lview = self.demo.load_balanced_view()

    def _stage_imports(self, cluster):
        with cluster[:].sync_imports(quiet=True):
            from qiita_ware.context import system_call

    def sync(self, data):
        """Sync data to engines

        Parameters
        ----------
        data : dict
            dict of objects and to sync

        """
        # self.reserved[:].update(data)
        # self.general[:].update(data)
        self.demo[:].update(data)

    def submit_async(self, cmd, *args, **kwargs):
        """Submit an async command to execute

        Parameters
        ----------
        cmd : {function, str}
            A function to execute or a system call to execute
        args : list
            Arguments to pass to a function (if cmd is function)
        kwargs : dict
            Keyword arguments to pass to a function (if cmd is function)

        Returns
        -------
        IPython.parallel.client.asyncresult.AsyncResult

        """
        if isinstance(cmd, str):
            task = self.demo_lview.apply_async(system_call, cmd)
        else:
            task = self.demo_lview.apply_async(cmd, *args, **kwargs)

        return task

    def submit_sync(self, cmd, *args, **kwargs):
        """Submit an sync command to execute

        Parameters
        ----------
        cmd : {function, str}
            A function to execute or a system call to execute
        args : list
            Arguments to pass to a function (if cmd is function)
        kwargs : dict
            Keyword arguments to pass to a function (if cmd is function)

        Returns
        -------
        Dependent on cmd

        """
        if isinstance(cmd, str):
            result = self.demo_lview.apply_sync(system_call, cmd)
        else:
            result = self.demo_lview.apply_sync(cmd, *args, **kwargs)

        return result

# likely want this in qiita_ware.__init__
context = Dispatch()
<commit_msg>add missing function<commit_after>from subprocess import Popen, PIPE

from IPython.parallel import Client

from qiita_ware.exceptions import ComputeError

def system_call(cmd):
    """Call cmd and return (stdout, stderr, return_value).

    cmd: can be either a string containing the command to be run, or a
     sequence of strings that are the tokens of the command.

    This function is ported from QIIME (http://www.qiime.org), previously
    named qiime_system_call. QIIME is a GPL project, but we obtained permission
    from the authors of this function to port it to pyqi (and keep it under
    pyqi's BSD license).
    """
    proc = Popen(cmd,
                 universal_newlines=True,
                 shell=True,
                 stdout=PIPE,
                 stderr=PIPE)
    # communicate pulls all stdout/stderr from the PIPEs to
    # avoid blocking -- don't remove this line!
    stdout, stderr = proc.communicate()
    return_value = proc.returncode

    if return_value != 0:
        raise ComputeError("Failed to execute: %s\nstdout: %s\nstderr: %s" %
                           (cmd, stdout, stderr))

    return stdout, stderr, return_value

class Dispatch(object):
    """Dispatch compute

    Attributes
    ----------
    reserved
    reserved_lview
    general
    general_lview
    demo
    demo_lview

    Methods
    -------
    submit_async
    submit_async_deps
    submit_sync
    sync

    """
    def __init__(self):
        # self.reserved = Client(profile='qiita_reserved')
        # self.general = Client(profile='qiita_general')
        self.demo = Client(profile='qiita_demo')

        # self._stage_imports(self.reserved)
        # self._stage_imports(self.general)
        self._stage_imports(self.demo)

        # self.reserved_lview = self.reserved.load_balanced_view()
        # self.general_lview = self.general.load_balanced_view()
        self.demo_lview = self.demo.load_balanced_view()

    def _stage_imports(self, cluster):
        with cluster[:].sync_imports(quiet=True):
            from qiita_ware.context import system_call

    def sync(self, data):
        """Sync data to engines

        Parameters
        ----------
        data : dict
            dict of objects and to sync

        """
        # self.reserved[:].update(data)
        # self.general[:].update(data)
        self.demo[:].update(data)

    def submit_async(self, cmd, *args, **kwargs):
        """Submit an async command to execute

        Parameters
        ----------
        cmd : {function, str}
            A function to execute or a system call to execute
        args : list
            Arguments to pass to a function (if cmd is function)
        kwargs : dict
            Keyword arguments to pass to a function (if cmd is function)

        Returns
        -------
        IPython.parallel.client.asyncresult.AsyncResult

        """
        if isinstance(cmd, str):
            task = self.demo_lview.apply_async(system_call, cmd)
        else:
            task = self.demo_lview.apply_async(cmd, *args, **kwargs)

        return task

    def submit_async_deps(self, deps, cmd, *args, **kwargs):
        """Submit as async command to execute after all dependencies are done

        Parameters
        ----------
        deps : list of AsyncResult
            The list of job dependencies for cmd
        cmd : {function, str}
            A function to execute or a system call to execute
        args : list
            Arguments to pass to a function (if cmd is function)
        kwargs : dict
            Keyword arguments to pass to a function (if cmd is function)

        Returns
        -------
        IPython.parallel.client.asyncresult.AsyncResult
        """
        with self.demo_lview.temp_flags(after=deps, block=False):
            if isinstance(cmd, str):
                task = self.demo_lview.apply_async(system_call, cmd)
            else:
                task = self.demo_lview.apply_async(cmd, *args, **kwargs)

        return task

    def submit_sync(self, cmd, *args, **kwargs):
        """Submit an sync command to execute

        Parameters
        ----------
        cmd : {function, str}
            A function to execute or a system call to execute
        args : list
            Arguments to pass to a function (if cmd is function)
        kwargs : dict
            Keyword arguments to pass to a function (if cmd is function)

        Returns
        -------
        Dependent on cmd

        """
        if isinstance(cmd, str):
            result = self.demo_lview.apply_sync(system_call, cmd)
        else:
            result = self.demo_lview.apply_sync(cmd, *args, **kwargs)

        return result

# likely want this in qiita_ware.__init__
context = Dispatch()
<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-
#
# The Qubes OS Project, http://www.qubes-os.org
#
# Copyright (C) 2017 Marek Marczykowski-Górecki
#                               <marmarek@invisiblethingslab.com>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, see <https://www.gnu.org/licenses/>.

"""Extension responsible for qvm-service framework"""

import os
import qubes.ext
import qubes.config

class ServicesExtension(qubes.ext.Extension):
    """This extension export features with 'service.' prefix to QubesDB in
    /qubes-service/ tree.
    """

    @staticmethod
    def add_dom0_service(vm, service):
        try:
            os.makedirs(
                qubes.config.system_path['dom0_services_dir'], exist_ok=True)
            service = '{}/{}'.format(
                qubes.config.system_path['dom0_services_dir'], service)
            if not os.path.exists(service):
                os.mknod(service)
        except PermissionError:
            vm.log.warning("Cannot write to {}".format(
                qubes.config.system_path['dom0_services_dir']))

    @staticmethod
    def remove_dom0_service(vm, service):
        try:
            service = '{}/{}'.format(
                qubes.config.system_path['dom0_services_dir'], service)
            if os.path.exists(service):
                os.remove(service)
        except PermissionError:
            vm.log.warning("Cannot write to {}".format(
                qubes.config.system_path['dom0_services_dir']))

    # pylint: disable=no-self-use
    @qubes.ext.handler('domain-qdb-create')
    def on_domain_qdb_create(self, vm, event):
        """Actually export features"""
        # pylint: disable=unused-argument
        for feature, value in vm.features.items():
            if not feature.startswith('service.'):
                continue
            service = feature[len('service.'):]
            # forcefully convert to '0' or '1'
            vm.untrusted_qdb.write('/qubes-service/{}'.format(service),
                                   str(int(bool(value))))

        # always set meminfo-writer according to maxmem
        vm.untrusted_qdb.write('/qubes-service/meminfo-writer',
                               '1' if vm.maxmem > 0 else '0')

    @qubes.ext.handler('domain-feature-set:*')
    def on_domain_feature_set(self, vm, event, feature, value, oldvalue=None):
        """Update /qubes-service/ QubesDB tree in runtime"""
        # pylint: disable=unused-argument

        # TODO: remove this compatibility hack in Qubes 4.1
        if feature == 'service.meminfo-writer':
            # if someone try to enable meminfo-writer ...
            if value:
                # ... reset maxmem to default
                vm.maxmem = qubes.property.DEFAULT
            else:
                # otherwise, set to 0
                vm.maxmem = 0
            # in any case, remove the entry, as it does not indicate memory
            # balancing state anymore
            del vm.features['service.meminfo-writer']

        if not vm.is_running():
            return
        if not feature.startswith('service.'):
            return
        service = feature[len('service.'):]
        # forcefully convert to '0' or '1'
        vm.untrusted_qdb.write('/qubes-service/{}'.format(service),
                               str(int(bool(value))))

        if vm.name == "dom0":
            if str(int(bool(value))) == "1":
                self.add_dom0_service(vm, service)
            else:
                self.remove_dom0_service(vm, service)

    @qubes.ext.handler('domain-feature-delete:*')
    def on_domain_feature_delete(self, vm, event, feature):
        """Update /qubes-service/ QubesDB tree in runtime"""
        # pylint: disable=unused-argument
        if not vm.is_running():
            return
        if not feature.startswith('service.'):
            return
        service = feature[len('service.'):]
        # this one is excluded from user control
        if service == 'meminfo-writer':
            return
        vm.untrusted_qdb.rm('/qubes-service/{}'.format(service))

        if vm.name == "dom0":
            self.remove_dom0_service(vm, service)

    @qubes.ext.handler('domain-load')
    def on_domain_load(self, vm, event):
        """Migrate meminfo-writer service into maxmem"""
        # pylint: disable=no-self-use,unused-argument
        if 'service.meminfo-writer' in vm.features:
            # if was set to false, force maxmem=0
            # otherwise, simply ignore as the default is fine
            if not vm.features['service.meminfo-writer']:
                vm.maxmem = 0
            del vm.features['service.meminfo-writer']

        if vm.name == "dom0":
            for feature, value in vm.features.items():
                if not feature.startswith('service.'):
                    continue
                service = feature[len('service.'):]
                if str(int(bool(value))) == "1":
                    self.add_dom0_service(vm, service)
                else:
                    self.remove_dom0_service(vm, service)

    @qubes.ext.handler('features-request')
    def supported_services(self, vm, event, untrusted_features):
        """Handle advertisement of supported services"""
        # pylint: disable=no-self-use,unused-argument

        if getattr(vm, 'template', None):
            vm.log.warning(
                'Ignoring qubes.FeaturesRequest from template-based VM')
            return

        new_supported_services = set()
        for requested_service in untrusted_features:
            if not requested_service.startswith('supported-service.'):
                continue
            if untrusted_features[requested_service] == '1':
                # only allow to advertise service as supported, lack of entry
                #  means service is not supported
                new_supported_services.add(requested_service)
        del untrusted_features

        # if no service is supported, ignore the whole thing - do not clear
        # all services in case of empty request (manual or such)
        if not new_supported_services:
            return

        old_supported_services = set(
            feat for feat in vm.features
            if feat.startswith('supported-service.') and vm.features[feat])

        for feature in new_supported_services.difference(
                old_supported_services):
            vm.features[feature] = True

        for feature in old_supported_services.difference(
                new_supported_services):
            del vm.features[feature]
<commit_msg>Set apparmor feature when template advertises support for it<commit_after># -*- encoding: utf-8 -*-
#
# The Qubes OS Project, http://www.qubes-os.org
#
# Copyright (C) 2017 Marek Marczykowski-Górecki
#                               <marmarek@invisiblethingslab.com>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, see <https://www.gnu.org/licenses/>.

"""Extension responsible for qvm-service framework"""

import os
import qubes.ext
import qubes.config

class ServicesExtension(qubes.ext.Extension):
    """This extension export features with 'service.' prefix to QubesDB in
    /qubes-service/ tree.
    """

    @staticmethod
    def add_dom0_service(vm, service):
        try:
            os.makedirs(
                qubes.config.system_path['dom0_services_dir'], exist_ok=True)
            service = '{}/{}'.format(
                qubes.config.system_path['dom0_services_dir'], service)
            if not os.path.exists(service):
                os.mknod(service)
        except PermissionError:
            vm.log.warning("Cannot write to {}".format(
                qubes.config.system_path['dom0_services_dir']))

    @staticmethod
    def remove_dom0_service(vm, service):
        try:
            service = '{}/{}'.format(
                qubes.config.system_path['dom0_services_dir'], service)
            if os.path.exists(service):
                os.remove(service)
        except PermissionError:
            vm.log.warning("Cannot write to {}".format(
                qubes.config.system_path['dom0_services_dir']))

    # pylint: disable=no-self-use
    @qubes.ext.handler('domain-qdb-create')
    def on_domain_qdb_create(self, vm, event):
        """Actually export features"""
        # pylint: disable=unused-argument
        for feature, value in vm.features.items():
            if not feature.startswith('service.'):
                continue
            service = feature[len('service.'):]
            # forcefully convert to '0' or '1'
            vm.untrusted_qdb.write('/qubes-service/{}'.format(service),
                                   str(int(bool(value))))

        # always set meminfo-writer according to maxmem
        vm.untrusted_qdb.write('/qubes-service/meminfo-writer',
                               '1' if vm.maxmem > 0 else '0')

    @qubes.ext.handler('domain-feature-set:*')
    def on_domain_feature_set(self, vm, event, feature, value, oldvalue=None):
        """Update /qubes-service/ QubesDB tree in runtime"""
        # pylint: disable=unused-argument

        # TODO: remove this compatibility hack in Qubes 4.1
        if feature == 'service.meminfo-writer':
            # if someone try to enable meminfo-writer ...
            if value:
                # ... reset maxmem to default
                vm.maxmem = qubes.property.DEFAULT
            else:
                # otherwise, set to 0
                vm.maxmem = 0
            # in any case, remove the entry, as it does not indicate memory
            # balancing state anymore
            del vm.features['service.meminfo-writer']

        if not vm.is_running():
            return
        if not feature.startswith('service.'):
            return
        service = feature[len('service.'):]
        # forcefully convert to '0' or '1'
        vm.untrusted_qdb.write('/qubes-service/{}'.format(service),
                               str(int(bool(value))))

        if vm.name == "dom0":
            if str(int(bool(value))) == "1":
                self.add_dom0_service(vm, service)
            else:
                self.remove_dom0_service(vm, service)

    @qubes.ext.handler('domain-feature-delete:*')
    def on_domain_feature_delete(self, vm, event, feature):
        """Update /qubes-service/ QubesDB tree in runtime"""
        # pylint: disable=unused-argument
        if not vm.is_running():
            return
        if not feature.startswith('service.'):
            return
        service = feature[len('service.'):]
        # this one is excluded from user control
        if service == 'meminfo-writer':
            return
        vm.untrusted_qdb.rm('/qubes-service/{}'.format(service))

        if vm.name == "dom0":
            self.remove_dom0_service(vm, service)

    @qubes.ext.handler('domain-load')
    def on_domain_load(self, vm, event):
        """Migrate meminfo-writer service into maxmem"""
        # pylint: disable=no-self-use,unused-argument
        if 'service.meminfo-writer' in vm.features:
            # if was set to false, force maxmem=0
            # otherwise, simply ignore as the default is fine
            if not vm.features['service.meminfo-writer']:
                vm.maxmem = 0
            del vm.features['service.meminfo-writer']

        if vm.name == "dom0":
            for feature, value in vm.features.items():
                if not feature.startswith('service.'):
                    continue
                service = feature[len('service.'):]
                if str(int(bool(value))) == "1":
                    self.add_dom0_service(vm, service)
                else:
                    self.remove_dom0_service(vm, service)

    @qubes.ext.handler('features-request')
    def supported_services(self, vm, event, untrusted_features):
        """Handle advertisement of supported services"""
        # pylint: disable=no-self-use,unused-argument

        if getattr(vm, 'template', None):
            vm.log.warning(
                'Ignoring qubes.FeaturesRequest from template-based VM')
            return

        new_supported_services = set()
        for requested_service in untrusted_features:
            if not requested_service.startswith('supported-service.'):
                continue
            if untrusted_features[requested_service] == '1':
                # only allow to advertise service as supported, lack of entry
                #  means service is not supported
                new_supported_services.add(requested_service)
        del untrusted_features

        # if no service is supported, ignore the whole thing - do not clear
        # all services in case of empty request (manual or such)
        if not new_supported_services:
            return

        old_supported_services = set(
            feat for feat in vm.features
            if feat.startswith('supported-service.') and vm.features[feat])

        for feature in new_supported_services.difference(
                old_supported_services):
            vm.features[feature] = True
            if feature == 'supported-service.apparmor' and \
               not 'apparmor' in vm.features:
                vm.features['apparmor'] = True

        for feature in old_supported_services.difference(
                new_supported_services):
            del vm.features[feature]
            if feature == 'supported-service.apparmor' and \
               'apparmor' in vm.features and vm.features['apparmor'] == '1':
                del vm.features['apparmor']
<|endoftext|>


<commit_before>from autocontroller import AutoController

###### STATE MACHINE FOR NAVIGATIONAL STATES ########

class State:
    def run(self):
        assert 0, "run not implemented"
    def sensor_data_received(self, data, new_ir_left, new_ir_right):
        assert 0, "sensor_data_received not implemented"

class auto_control(State):

    auto_controller = AutoController()

    def is_at_left_turn(self, data):
        diff = data['ir_left'] - data['old_ir_left']
        return diff >= Navigator.DISCONTINUITY_DIST and data['side'] == Navigator.LEFT_SIDE

    def is_at_right_turn(self, data):
        right_diff = data['ir_right'] - data['old_ir_right']
        return right_diff >= Navigator.DISCONTINUITY_DIST and data['side'] == Navigator.RIGHT_SIDE

    def sensor_data_received(self, data, new_ir_left, new_ir_right):

        if self.is_at_left_turn(data) or self.is_at_right_turn(data):
            return

        right_speed, left_speed, regulation = auto_control.auto_controller.auto_control(new_ir_left, new_ir_right, data['side'])

        if abs(regulation) >= 15:
            return

        #Duration set to something quite high to mimic running forever (until next update)
        data['driver'].drive(left_speed, right_speed, 500)

    def run(self, data):
        left_diff = data['ir_left'] - data['old_ir_left']
        right_diff = data['ir_right'] - data['old_ir_right']

        print('ir right:' + str(data['ir_right']) + ' old ir right: ' + str(data['old_ir_right']) + ' right diff : ' + str(right_diff))
        #Outer turn, prioritize following right wall
        if self.is_at_right_turn(data):
            data['driver'].outer_turn_right()
            print('outer turn right')
            return turn()

        if self.is_at_left_turn(data):
            data['driver'].outer_turn_left()
            print('outer turn left')
            return turn()

        print('Left diff: ' + str(left_diff) + ' right diff ' + str(right_diff))
        #print('laser distance: ' + str(data['laser'].read_data()))

        #Inner turn
        #if data['laser'].read_data() <=  Navigator.FACING_WALL_DIST:
        #    if data['side'] == Navigator.LEFT_SIDE:
        #        data['driver'].inner_turn_right()
        #        print('inner turn right')
        #        return turn()
        #    if data['side'] == Navigator.RIGHT_SIDE:
        #        data['driver'].inner_turn_left()
        #        print('inner turn left')
        #        return turn()

        return auto_control()

class warmup(State):
    def sensor_data_received(self, data, new_ir_left, new_ir_right):
        return #Do nothing

    def run(self, data):
        if not data['driver'].driving():
            return auto_control()
        else:
            return warmup()

class turn(State):
    def sensor_data_received(self, data, new_ir_left, new_ir_right):
        return #Do nothing. Only auto control uses it

    def run(self, data):
        if not data['driver'].driving():
            print('changing to auto control')
            return auto_control()
        else:
            return turn()

###### NAVIGATOR CLASS #######
class Navigator:

    LEFT_SIDE = 0
    RIGHT_SIDE = 1

    DISCONTINUITY_DIST = 8.0 #mm
    FACING_WALL_DIST = 200 #mm

    def __init__(self, driver, laser):
        self.data = {'ir_left': 0,
                    'ir_right': 0,
                    'old_ir_right': 0,
                    'old_ir_left': 0,
                    'driver' : driver,
                    'laser' : laser,
                    'side' : Navigator.RIGHT_SIDE
                    }

       #Stand still waiting for sensors
        self.data['driver'].drive(0, 0, 2000)
        self.state = warmup()

    def sensor_data_received(self, new_ir_left, new_ir_right):
        self.data['old_ir_left'] = self.data['ir_left']
        self.data['old_ir_right'] = self.data['ir_right']
        self.data['ir_left'] = new_ir_left
        self.data['ir_right'] = new_ir_right
        self.state.sensor_data_received(self.data, new_ir_left, new_ir_right)

    #Runs the state. The states run method returns the next state
    def navigate(self):
        next_state = self.state.run(self.data)
        self.state = next_state<commit_msg>enabled laser<commit_after>from autocontroller import AutoController

###### STATE MACHINE FOR NAVIGATIONAL STATES ########

class State:
    def run(self):
        assert 0, "run not implemented"
    def sensor_data_received(self, data, new_ir_left, new_ir_right):
        assert 0, "sensor_data_received not implemented"

class auto_control(State):

    auto_controller = AutoController()

    def is_at_left_turn(self, data):
        diff = data['ir_left'] - data['old_ir_left']
        return diff >= Navigator.DISCONTINUITY_DIST and data['side'] == Navigator.LEFT_SIDE

    def is_at_right_turn(self, data):
        right_diff = data['ir_right'] - data['old_ir_right']
        return right_diff >= Navigator.DISCONTINUITY_DIST and data['side'] == Navigator.RIGHT_SIDE

    def sensor_data_received(self, data, new_ir_left, new_ir_right):

        if self.is_at_left_turn(data) or self.is_at_right_turn(data):
            return

        right_speed, left_speed, regulation = auto_control.auto_controller.auto_control(new_ir_left, new_ir_right, data['side'])

        if abs(regulation) >= 15:
            return

        #Duration set to something quite high to mimic running forever (until next update)
        data['driver'].drive(left_speed, right_speed, 500)

    def run(self, data):
        left_diff = data['ir_left'] - data['old_ir_left']
        right_diff = data['ir_right'] - data['old_ir_right']

        print('ir right:' + str(data['ir_right']) + ' old ir right: ' + str(data['old_ir_right']) + ' right diff : ' + str(right_diff))
        #Outer turn, prioritize following right wall
        if self.is_at_right_turn(data):
            data['driver'].outer_turn_right()
            print('outer turn right')
            return turn()

        if self.is_at_left_turn(data):
            data['driver'].outer_turn_left()
            print('outer turn left')
            return turn()

        print('Left diff: ' + str(left_diff) + ' right diff ' + str(right_diff))
        print('laser distance: ' + str(data['laser'].read_data()))

        #Inner turn
        if data['laser'].read_data() <=  Navigator.FACING_WALL_DIST && data['laser'] != -1:
            if data['side'] == Navigator.LEFT_SIDE:
                data['driver'].inner_turn_right()
                print('inner turn right')
                return turn()
            if data['side'] == Navigator.RIGHT_SIDE:
                data['driver'].inner_turn_left()
                print('inner turn left')
                return turn()

        return auto_control()

class warmup(State):
    def sensor_data_received(self, data, new_ir_left, new_ir_right):
        return #Do nothing

    def run(self, data):
        if not data['driver'].driving():
            return auto_control()
        else:
            return warmup()

class turn(State):
    def sensor_data_received(self, data, new_ir_left, new_ir_right):
        return #Do nothing. Only auto control uses it

    def run(self, data):
        if not data['driver'].driving():
            print('changing to auto control')
            return auto_control()
        else:
            return turn()

###### NAVIGATOR CLASS #######
class Navigator:

    LEFT_SIDE = 0
    RIGHT_SIDE = 1

    DISCONTINUITY_DIST = 8.0 #mm
    FACING_WALL_DIST = 200 #mm

    def __init__(self, driver, laser):
        self.data = {'ir_left': 0,
                    'ir_right': 0,
                    'old_ir_right': 0,
                    'old_ir_left': 0,
                    'driver' : driver,
                    'laser' : laser,
                    'side' : Navigator.RIGHT_SIDE
                    }

       #Stand still waiting for sensors
        self.data['driver'].drive(0, 0, 2000)
        self.state = warmup()

    def sensor_data_received(self, new_ir_left, new_ir_right):
        self.data['old_ir_left'] = self.data['ir_left']
        self.data['old_ir_right'] = self.data['ir_right']
        self.data['ir_left'] = new_ir_left
        self.data['ir_right'] = new_ir_right
        self.state.sensor_data_received(self.data, new_ir_left, new_ir_right)

    #Runs the state. The states run method returns the next state
    def navigate(self):
        next_state = self.state.run(self.data)
        self.state = next_state<|endoftext|>


<commit_before># coding: UTF-8
'''
Created on 2013-04-06

@author: tianwei

Desc: Celery Tasks
'''
import os
import sys
import uuid
import time
import datetime

from celery.decorators import task

from django.shortcuts import get_object_or_404
from django.shortcuts import render
from django.shortcuts import render_to_response
from django.conf import settings
from django.core.urlresolvers import reverse
from django.http import HttpResponseRedirect, HttpResponse
from django.http import HttpResponseForbidden, Http404
from django.template import RequestContext
from django.utils import simplejson
from django.views.decorators import csrf
from django.contrib.auth.decorators import login_required
from django.core.files.uploadedfile import UploadedFile
from django.core.cache import cache
from django.utils.hashcompat import md5_constructor as md5
from calcore.controllers.prediciton_model import PredictionModel
from backend.logging import logger, loginfo
from calcore.models import SingleTask, ProcessedFile, SuiteTask
from const.models import StatusCategory
from const import STATUS_WORKING, STATUS_SUCCESS, STATUS_FAILED, STATUS_UNDEFINED
from settings import MEDIA_ROOT, SETTINGS_ROOT

import pybel
from calcore.models import *
from const import ORIGIN_DRAW

LOCK_EXPIRE = 60 * 5 # Lock expires in 5 minutes

def get_ModelName(name):
    temp={
            "koa":"logKOA",
            "pl":"logRP",
            "kof":"logKOF"
            }
    # return temp.get(name)
    if temp.has_key(name):
        return temp.get(name)
    else:
        raise KeyError,"We don't have this model"

@task()
def add(x, y):
    print "sleep!"
    return x+y+x+y

@task()
def add_a(x, y):
    print "sleep! a"
    return x+y+x+y

def add_counter_core(suite_id):
    """
    core counter algorightm
    """
    print "add counter"
    suite = SuiteTask.objects.get(sid=suite_id)
    if suite.has_finished_tasks < suite.total_tasks-1:
        suite.has_finished_tasks = suite.has_finished_tasks + 1
        print "add:" + str(suite.has_finished_tasks)
    else:
        suite.has_finished_tasks = suite.total_tasks
        print "Finished:" + str(suite.has_finished_tasks)
        print suite.has_finished_tasks
        suite.status_id = StatusCategory.objects.get(category=STATUS_SUCCESS)
    suite.save()

@task()
def add_counter(suite_id):
    """
    use filter to get task numbers
    """
    finished_count = SingleTask.objects.filter(sid=suite_id)\
                                       .exclude(status=StatusCategory.objects.get(category=STATUS_WORKING))\
                                       .count()
    print finished_count
    suite = SuiteTask.objects.get(sid=suite_id)
    if finished_count == suite.total_tasks:
        suite.has_finished_tasks = suite.total_tasks
        print "Finished:" + str(suite.has_finished_tasks)
        suite.status_id = StatusCategory.objects.get(category=STATUS_SUCCESS)
    else:
        suite.has_finished_tasks = finished_count

    suite.save()

@task()
def add_counter_cache(suite_id):
    """
    Add counter by cache
    """
    id = md5(suite_id).hexdigest()
    lock_id = "%s-lock-%s" % ("add_counter", id)

    # cache.add fails if if the key already exists
    acquire_lock = lambda: cache.add(lock_id, 'true', LOCK_EXPIRE)
    release_lock = lambda: cache.delete(lock_id)

    print cache.get(lock_id)

    if acquire_lock():
        print cache.get(lock_id)
        try:
            add_counter_core(suite_id)
        finally:
            release_lock()

@task()
def send_email(task):
    """
    send result email to user
    """
    pass

@task
def generate_pdf(task):
    """
    generate result in pdf format
    """
    pass

def convert_smile_png(task):
    """
    convert mol into smile and png
    """
    abpath = task.file_obj.file_obj.url
    fullpath = os.path.join(settings.MEDIA_ROOT, abpath)

    mol = pybel.readfile("mol", fullpath).next()
    task.file_obj.smiles = ("%s" % mol).split(" ")[0]

    picname = uuid.uuid4() + ".png"
    picpath = os.path.join(settings.SEARCH_IMAGE_PATH, picname)
    mol.draw(show=False, filename=picpath)

    f = open(picpath, "r")
    task.file_obj.image = f
    task.save()
    f.close()
    os.remove(picpath)

@task
def generate_smile_image(task_id):
    """
    generate smile and image for task
    """
    task = SingleTask.objects.get(pid=task_id)
    filetype = task.file_obj.file_source.category

    if filetype == ORIGIN_SMILE:
        #this type has already have image and smiles in local search machine,
        #only copy them
        print "search engine test"
        task.image = SearchEngineModel.objects.get(smiles__contains=task.file_obj.smiles).image
        task.save()
    else:
        #other types only contains mol file
        print "other input method"
        try:
            convert_smile_png(task)
        except Exception, err:
            loginfo(p=err, label="convert error")

@task()
def calculateTask(task, model_name):
    """
    Calculate task
    """
    #Covert smiles, png
    generate_smile_image.delay(task.pid)

    para = dict.fromkeys(['smilestring', 'filename', 'cas'], "")

    fullpath = os.path.join(SETTINGS_ROOT, task.file_obj.file_obj.path)
    para['filename'] = os.path.basename(fullpath)
    filepath = os.path.dirname(fullpath)
    suite = task.sid
    result = 0

    print fullpath
    print para["filename"]
    print filepath

    try:
        pm = PredictionModel([get_ModelName(model_name)], para, filepath)
        result = pm.predict_results[para['filename'].split(".")[0]][get_ModelName(model_name)]
    except KeyError, err:
        print "We don't have this model"
        result = 0
        task.result_state = "We don't support this model now"
        task.status = StatusCategory.objects.get(category=STATUS_SUCCESS)
        suite.status_id = StatusCategory.objects.get(category=STATUS_WORKING)
    except Exception, err:
        print err
        result = -10000
        task.result_state = str(err)
        task.status = StatusCategory.objects.get(category=STATUS_FAILED)
        suite.status_id = StatusCategory.objects.get(category=STATUS_FAILED)
    else:
        print "calculate Successfully in celery queue!"
        task.result_state = "Calculate Successfully!"
        task.status = StatusCategory.objects.get(category=STATUS_SUCCESS)
        suite.status_id = StatusCategory.objects.get(category=STATUS_WORKING)
    suite.save()

    loginfo(p=result, label="calculate task result")
    task.end_time = datetime.datetime.now()
    task.results = result
    task.save()

    #Add single task counter
    add_counter.delay(suite.sid)

    return result
<commit_msg>finish draw type convert<commit_after># coding: UTF-8
'''
Created on 2013-04-06

@author: tianwei

Desc: Celery Tasks
'''
import os
import sys
import uuid
import time
import datetime

from celery.decorators import task

from django.shortcuts import get_object_or_404
from django.shortcuts import render
from django.shortcuts import render_to_response
from django.conf import settings
from django.core.urlresolvers import reverse
from django.http import HttpResponseRedirect, HttpResponse
from django.http import HttpResponseForbidden, Http404
from django.template import RequestContext
from django.utils import simplejson
from django.core.files import File
from django.views.decorators import csrf
from django.contrib.auth.decorators import login_required
from django.core.files.uploadedfile import UploadedFile
from django.core.cache import cache
from django.utils.hashcompat import md5_constructor as md5
from calcore.controllers.prediciton_model import PredictionModel
from backend.logging import logger, loginfo
from calcore.models import SingleTask, ProcessedFile, SuiteTask
from const.models import StatusCategory
from const import STATUS_WORKING, STATUS_SUCCESS, STATUS_FAILED, STATUS_UNDEFINED

import pybel
from calcore.models import *
from const import ORIGIN_DRAW

LOCK_EXPIRE = 60 * 5 # Lock expires in 5 minutes

def get_ModelName(name):
    temp={
            "koa":"logKOA",
            "pl":"logRP",
            "kof":"logKOF"
            }
    # return temp.get(name)
    if temp.has_key(name):
        return temp.get(name)
    else:
        raise KeyError,"We don't have this model"

@task()
def add(x, y):
    print "sleep!"
    return x+y+x+y

@task()
def add_a(x, y):
    print "sleep! a"
    return x+y+x+y

def add_counter_core(suite_id):
    """
    core counter algorightm
    """
    print "add counter"
    suite = SuiteTask.objects.get(sid=suite_id)
    if suite.has_finished_tasks < suite.total_tasks-1:
        suite.has_finished_tasks = suite.has_finished_tasks + 1
        print "add:" + str(suite.has_finished_tasks)
    else:
        suite.has_finished_tasks = suite.total_tasks
        print "Finished:" + str(suite.has_finished_tasks)
        print suite.has_finished_tasks
        suite.status_id = StatusCategory.objects.get(category=STATUS_SUCCESS)
    suite.save()

@task()
def add_counter(suite_id):
    """
    use filter to get task numbers
    """
    finished_count = SingleTask.objects.filter(sid=suite_id)\
                                       .exclude(status=StatusCategory.objects.get(category=STATUS_WORKING))\
                                       .count()
    print finished_count
    suite = SuiteTask.objects.get(sid=suite_id)
    if finished_count == suite.total_tasks:
        suite.has_finished_tasks = suite.total_tasks
        print "Finished:" + str(suite.has_finished_tasks)
        suite.status_id = StatusCategory.objects.get(category=STATUS_SUCCESS)
    else:
        suite.has_finished_tasks = finished_count

    suite.save()

@task()
def add_counter_cache(suite_id):
    """
    Add counter by cache
    """
    id = md5(suite_id).hexdigest()
    lock_id = "%s-lock-%s" % ("add_counter", id)

    # cache.add fails if if the key already exists
    acquire_lock = lambda: cache.add(lock_id, 'true', LOCK_EXPIRE)
    release_lock = lambda: cache.delete(lock_id)

    print cache.get(lock_id)

    if acquire_lock():
        print cache.get(lock_id)
        try:
            add_counter_core(suite_id)
        finally:
            release_lock()

@task()
def send_email(task):
    """
    send result email to user
    """
    pass

@task
def generate_pdf(task):
    """
    generate result in pdf format
    """
    pass

def convert_smile_png(singletask):
    """
    convert mol into smile and png
    """
    abpath = singletask.file_obj.file_obj.url
    fullpath = settings.SETTINGS_ROOT + abpath

    print "convert_smile_png"
    print fullpath
    mol = pybel.readfile("mol", fullpath).next()
    singletask.file_obj.smiles = ("%s" % mol).split("\t")[0]

    picname = str(uuid.uuid4()) + ".png"
    picpath = os.path.join(settings.SEARCH_IMAGE_PATH, picname)
    mol.draw(show=False, filename=picpath)

    f = File(open(picpath, "r"))
    singletask.file_obj.image = f
    singletask.file_obj.save()
    singletask.save()
    f.close()
    print singletask.file_obj.smiles
    print singletask.file_obj.image
    print singletask

def generate_smile_image(pid):
    """
    generate smile and image for task
    """
    singletask = SingleTask.objects.get(pid=pid)
    filetype = singletask.file_obj.file_source.category

    if filetype == ORIGIN_SMILE:
        #this type has already have image and smiles in local search machine,
        #only copy them
        print "search engine test"
        singletask.image = SearchEngineModel.objects.get(smiles__contains=task.file_obj.smiles).image
        singletask.save()
    else:
        #other types only contains mol file
        print "other input method"
        convert_smile_png(singletask)

@task()
def calculateTask(task, model_name):
    """
    Calculate task
    """
    #Covert smiles, png
    generate_smile_image(task.pid)

    para = dict.fromkeys(['smilestring', 'filename', 'cas'], "")

    fullpath = os.path.join(settings.SETTINGS_ROOT, task.file_obj.file_obj.path)
    para['filename'] = os.path.basename(fullpath)
    filepath = os.path.dirname(fullpath)
    suite = task.sid
    result = 0

    print fullpath
    print para["filename"]
    print filepath

    try:
        pm = PredictionModel([get_ModelName(model_name)], para, filepath)
        result = pm.predict_results[para['filename'].split(".")[0]][get_ModelName(model_name)]
    except KeyError, err:
        print "We don't have this model"
        result = 0
        task.result_state = "We don't support this model now"
        task.status = StatusCategory.objects.get(category=STATUS_SUCCESS)
        suite.status_id = StatusCategory.objects.get(category=STATUS_WORKING)
    except Exception, err:
        print err
        result = -10000
        task.result_state = str(err)
        task.status = StatusCategory.objects.get(category=STATUS_FAILED)
        suite.status_id = StatusCategory.objects.get(category=STATUS_FAILED)
    else:
        print "calculate Successfully in celery queue!"
        task.result_state = "Calculate Successfully!"
        task.status = StatusCategory.objects.get(category=STATUS_SUCCESS)
        suite.status_id = StatusCategory.objects.get(category=STATUS_WORKING)
    suite.save()

    loginfo(p=result, label="calculate task result")
    task.end_time = datetime.datetime.now()
    task.results = result
    task.save()

    #Add single task counter
    add_counter.delay(suite.sid)

    return result
<|endoftext|>