<commit_before># (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import pipes
import random

from ansible import constants as C
from ansible.template import Templar
from ansible.utils.boolean import boolean

__all__ = ['ConnectionInformation']

class ConnectionInformation:

    '''
    This class is used to consolidate the connection information for
    hosts in a play and child tasks, where the task may override some
    connection/authentication information.
    '''

    def __init__(self, play=None, options=None):
        # FIXME: implement the new methodology here for supporting
        #        various different auth escalation methods (becomes, etc.)

        self.connection  = C.DEFAULT_TRANSPORT
        self.remote_addr = None
        self.remote_user = 'root'
        self.password    = ''
        self.port        = 22
        self.private_key_file = None
        self.verbosity   = 0
        self.only_tags   = set()
        self.skip_tags   = set()

        # privilege escalation
        self.become        = False
        self.become_method = C.DEFAULT_BECOME_METHOD
        self.become_user   = ''
        self.become_pass   = ''

        self.no_log      = False
        self.check_mode  = False

        if play:
            self.set_play(play)

        if options:
            self.set_options(options)

    def __repr__(self):
        value = "CONNECTION INFO:\n"
        fields = self._get_fields()
        fields.sort()
        for field in fields:
            value += "%20s : %s\n" % (field, getattr(self, field))
        return value

    def set_play(self, play):
        '''
        Configures this connection information instance with data from
        the play class.
        '''

        if play.connection:
            self.connection = play.connection

        self.remote_user   = play.remote_user
        self.password      = ''
        self.port          = int(play.port) if play.port else 22
        self.become        = play.become
        self.become_method = play.become_method
        self.become_user   = play.become_user
        self.become_pass   = play.become_pass

        # non connection related
        self.no_log      = play.no_log
        self.environment = play.environment

    def set_options(self, options):
        '''
        Configures this connection information instance with data from
        options specified by the user on the command line. These have a
        higher precedence than those set on the play or host.
        '''

        # FIXME: set other values from options here?

        self.verbosity = options.verbosity
        if options.connection:
            self.connection = options.connection

        if options.check:
            self.check_mode = boolean(options.check)

        # get the tag info from options, converting a comma-separated list
        # of values into a proper list if need be. We check to see if the
        # options have the attribute, as it is not always added via the CLI
        if hasattr(options, 'tags'):
            if isinstance(options.tags, list):
                self.only_tags.update(options.tags)
            elif isinstance(options.tags, basestring):
                self.only_tags.update(options.tags.split(','))

        if len(self.only_tags) == 0:
            self.only_tags = set(['all'])

        if hasattr(options, 'skip_tags'):
            if isinstance(options.skip_tags, list):
                self.skip_tags.update(options.skip_tags)
            elif isinstance(options.skip_tags, basestring):
                self.skip_tags.update(options.skip_tags.split(','))

    def copy(self, ci):
        '''
        Copies the connection info from another connection info object, used
        when merging in data from task overrides.
        '''

        for field in self._get_fields():
            value = getattr(ci, field, None)
            if isinstance(value, dict):
                setattr(self, field, value.copy())
            elif isinstance(value, set):
                setattr(self, field, value.copy())
            elif isinstance(value, list):
                setattr(self, field, value[:])
            else:
                setattr(self, field, value)

    def set_task_override(self, task):
        '''
        Sets attributes from the task if they are set, which will override
        those from the play.
        '''

        new_info = ConnectionInformation()
        new_info.copy(self)

        for attr in ('connection', 'remote_user', 'become', 'become_user', 'become_pass', 'become_method', 'environment', 'no_log'):
            attr_val = None
            if hasattr(task, attr):
                attr_val = getattr(task, attr)
            if task._block and hasattr(task._block, attr) and not attr_val:
                attr_val = getattr(task._block, attr)
            if attr_val:
                setattr(new_info, attr, attr_val)

        return new_info

    def make_become_cmd(self, cmd, executable, become_settings=None):

        """
        helper function to create privilege escalation commands
        """

        # FIXME: become settings should probably be stored in the connection info itself
        if become_settings is None:
            become_settings = {}

        randbits    = ''.join(chr(random.randint(ord('a'), ord('z'))) for x in xrange(32))
        success_key = 'BECOME-SUCCESS-%s' % randbits
        prompt      = None
        becomecmd   = None

        executable = executable or '$SHELL'

        if self.become:
            if self.become_method == 'sudo':
                # Rather than detect if sudo wants a password this time, -k makes sudo always ask for
                # a password if one is required. Passing a quoted compound command to sudo (or sudo -s)
                # directly doesn't work, so we shellquote it with pipes.quote() and pass the quoted
                # string to the user's shell.  We loop reading output until we see the randomly-generated
                # sudo prompt set with the -p option.
                prompt = '[sudo via ansible, key=%s] password: ' % randbits
                exe = become_settings.get('sudo_exe', C.DEFAULT_SUDO_EXE)
                flags = become_settings.get('sudo_flags', C.DEFAULT_SUDO_FLAGS)
                becomecmd = '%s -k && %s %s -S -p "%s" -u %s %s -c "%s"' % \
                    (exe, exe, flags or C.DEFAULT_SUDO_FLAGS, prompt, self.become_user, executable, 'echo %s; %s' % (success_key, cmd))

            elif self.become_method == 'su':
                exe = become_settings.get('su_exe', C.DEFAULT_SU_EXE)
                flags = become_settings.get('su_flags', C.DEFAULT_SU_FLAGS)
                becomecmd = '%s %s %s -c "%s -c %s"' % (exe, flags, self.become_user, executable, pipes.quote('echo %s; %s' % (success_key, cmd)))

            elif self.become_method == 'pbrun':
                exe = become_settings.get('pbrun_exe', 'pbrun')
                flags = become_settings.get('pbrun_flags', '')
                becomecmd = '%s -b -l %s -u %s "%s"' % (exe, flags, self.become_user, 'echo %s; %s' % (success_key,cmd))

            elif self.become_method == 'pfexec':
                exe = become_settings.get('pfexec_exe', 'pbrun')
                flags = become_settings.get('pfexec_flags', '')
                # No user as it uses it's own exec_attr to figure it out
                becomecmd = '%s %s "%s"' % (exe, flags, 'echo %s; %s' % (success_key,cmd))

            else:
                raise errors.AnsibleError("Privilege escalation method not found: %s" % method)

            return (('%s -c ' % executable) + pipes.quote(becomecmd), prompt, success_key)

        return (cmd, "", "")

    def check_become_success(self, output, become_settings):
        #TODO: implement
        pass

    def _get_fields(self):
        return [i for i in self.__dict__.keys() if i[:1] != '_']

    def post_validate(self, variables, loader):
        '''
        Finalizes templated values which may be set on this objects fields.
        '''

        templar = Templar(loader=loader, variables=variables)
        for field in self._get_fields():
            value = templar.template(getattr(self, field))
            setattr(self, field, value)
<commit_msg>Changes to become cmd formatting, per a47c132<commit_after># (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import pipes
import random

from ansible import constants as C
from ansible.template import Templar
from ansible.utils.boolean import boolean

__all__ = ['ConnectionInformation']

class ConnectionInformation:

    '''
    This class is used to consolidate the connection information for
    hosts in a play and child tasks, where the task may override some
    connection/authentication information.
    '''

    def __init__(self, play=None, options=None):
        # FIXME: implement the new methodology here for supporting
        #        various different auth escalation methods (becomes, etc.)

        self.connection  = C.DEFAULT_TRANSPORT
        self.remote_addr = None
        self.remote_user = 'root'
        self.password    = ''
        self.port        = 22
        self.private_key_file = None
        self.verbosity   = 0
        self.only_tags   = set()
        self.skip_tags   = set()

        # privilege escalation
        self.become        = False
        self.become_method = C.DEFAULT_BECOME_METHOD
        self.become_user   = ''
        self.become_pass   = ''

        self.no_log      = False
        self.check_mode  = False

        if play:
            self.set_play(play)

        if options:
            self.set_options(options)

    def __repr__(self):
        value = "CONNECTION INFO:\n"
        fields = self._get_fields()
        fields.sort()
        for field in fields:
            value += "%20s : %s\n" % (field, getattr(self, field))
        return value

    def set_play(self, play):
        '''
        Configures this connection information instance with data from
        the play class.
        '''

        if play.connection:
            self.connection = play.connection

        self.remote_user   = play.remote_user
        self.password      = ''
        self.port          = int(play.port) if play.port else 22
        self.become        = play.become
        self.become_method = play.become_method
        self.become_user   = play.become_user
        self.become_pass   = play.become_pass

        # non connection related
        self.no_log      = play.no_log
        self.environment = play.environment

    def set_options(self, options):
        '''
        Configures this connection information instance with data from
        options specified by the user on the command line. These have a
        higher precedence than those set on the play or host.
        '''

        # FIXME: set other values from options here?

        self.verbosity = options.verbosity
        if options.connection:
            self.connection = options.connection

        if options.check:
            self.check_mode = boolean(options.check)

        # get the tag info from options, converting a comma-separated list
        # of values into a proper list if need be. We check to see if the
        # options have the attribute, as it is not always added via the CLI
        if hasattr(options, 'tags'):
            if isinstance(options.tags, list):
                self.only_tags.update(options.tags)
            elif isinstance(options.tags, basestring):
                self.only_tags.update(options.tags.split(','))

        if len(self.only_tags) == 0:
            self.only_tags = set(['all'])

        if hasattr(options, 'skip_tags'):
            if isinstance(options.skip_tags, list):
                self.skip_tags.update(options.skip_tags)
            elif isinstance(options.skip_tags, basestring):
                self.skip_tags.update(options.skip_tags.split(','))

    def copy(self, ci):
        '''
        Copies the connection info from another connection info object, used
        when merging in data from task overrides.
        '''

        for field in self._get_fields():
            value = getattr(ci, field, None)
            if isinstance(value, dict):
                setattr(self, field, value.copy())
            elif isinstance(value, set):
                setattr(self, field, value.copy())
            elif isinstance(value, list):
                setattr(self, field, value[:])
            else:
                setattr(self, field, value)

    def set_task_override(self, task):
        '''
        Sets attributes from the task if they are set, which will override
        those from the play.
        '''

        new_info = ConnectionInformation()
        new_info.copy(self)

        for attr in ('connection', 'remote_user', 'become', 'become_user', 'become_pass', 'become_method', 'environment', 'no_log'):
            attr_val = None
            if hasattr(task, attr):
                attr_val = getattr(task, attr)
            if task._block and hasattr(task._block, attr) and not attr_val:
                attr_val = getattr(task._block, attr)
            if attr_val:
                setattr(new_info, attr, attr_val)

        return new_info

    def make_become_cmd(self, cmd, executable, become_settings=None):

        """
        helper function to create privilege escalation commands
        """

        # FIXME: become settings should probably be stored in the connection info itself
        if become_settings is None:
            become_settings = {}

        randbits    = ''.join(chr(random.randint(ord('a'), ord('z'))) for x in xrange(32))
        success_key = 'BECOME-SUCCESS-%s' % randbits
        prompt      = None
        becomecmd   = None

        executable = executable or '$SHELL'

        if self.become:
            if self.become_method == 'sudo':
                # Rather than detect if sudo wants a password this time, -k makes sudo always ask for
                # a password if one is required. Passing a quoted compound command to sudo (or sudo -s)
                # directly doesn't work, so we shellquote it with pipes.quote() and pass the quoted
                # string to the user's shell.  We loop reading output until we see the randomly-generated
                # sudo prompt set with the -p option.
                prompt = '[sudo via ansible, key=%s] password: ' % randbits
                exe = become_settings.get('sudo_exe', C.DEFAULT_SUDO_EXE)
                flags = become_settings.get('sudo_flags', C.DEFAULT_SUDO_FLAGS)
                becomecmd = '%s -k && %s %s -S -p "%s" -u %s %s -c %s' % \
                    (exe, exe, flags or C.DEFAULT_SUDO_FLAGS, prompt, self.become_user, executable, pipes.quote('echo %s; %s' % (success_key, cmd)))

            elif self.become_method == 'su':
                exe = become_settings.get('su_exe', C.DEFAULT_SU_EXE)
                flags = become_settings.get('su_flags', C.DEFAULT_SU_FLAGS)
                becomecmd = '%s %s %s -c "%s -c %s"' % (exe, flags, self.become_user, executable, pipes.quote('echo %s; %s' % (success_key, cmd)))

            elif self.become_method == 'pbrun':
                exe = become_settings.get('pbrun_exe', 'pbrun')
                flags = become_settings.get('pbrun_flags', '')
                becomecmd = '%s -b -l %s -u %s "%s"' % (exe, flags, user, pipes.quote('echo %s; %s' % (success_key, cmd)))

            elif self.become_method == 'pfexec':
                exe = become_settings.get('pfexec_exe', 'pbrun')
                flags = become_settings.get('pfexec_flags', '')
                # No user as it uses it's own exec_attr to figure it out
                becomecmd = '%s %s "%s"' % (exe, flags, pipes.quote('echo %s; %s' % (success_key, cmd)))

            else:
                raise errors.AnsibleError("Privilege escalation method not found: %s" % method)

            return (('%s -c ' % executable) + pipes.quote(becomecmd), prompt, success_key)

        return (cmd, "", "")

    def check_become_success(self, output, become_settings):
        #TODO: implement
        pass

    def _get_fields(self):
        return [i for i in self.__dict__.keys() if i[:1] != '_']

    def post_validate(self, variables, loader):
        '''
        Finalizes templated values which may be set on this objects fields.
        '''

        templar = Templar(loader=loader, variables=variables)
        for field in self._get_fields():
            value = templar.template(getattr(self, field))
            setattr(self, field, value)
<|endoftext|>


<commit_before>#!/usr/bin/env python3

"""
    XML creator
    -----------

    This library provides custom XML tools
    for ruledxml using the lxml library.

    (C) 2015, meisterluk, BSD 3-clause license
"""

import lxml.etree

from . import exceptions

def read(xmlinfile: str):
    """Given a filepath or file descriptor to an XML file, read the XML.

    :param xmlinfile:   file descriptor to XML file
    :type xmlinfile:    str
    :return:            an object representing the document object model
    :rtype:             lxml.etree.Element
    """
    return lxml.etree.parse(xmlinfile).getroot()

def write(dom: lxml.etree.Element, fd, encoding='utf-8', **lxml_options):
    """Write a given DOM into open file descriptor `fd`.

    :param dom:             a DOM (ie. root element) to store in an XML file
    :type dom:              lxml.etree.Element
    :param fd:              the file descriptor to write to
    :type fd:               _io.TextIOWrapper
    :param encoding:        which encoding shall be used for the XML file?
    :type encoding:         str
    :param lxml_options:    options for the lxml.etree.tostring
    :type lxml_options:     dict
    """
    opts = {
        'xml_declaration': True,
        'pretty_print': True,
        'encoding': encoding
    }
    opts.update(lxml_options)
    fd.write(lxml.etree.tostring(dom, **opts))

def strip_last_element(path):
    """Strip the last element off an XPath `path`.
    Raises InvalidPathException, if path refers to an attribute.

    For example ``/root/child`` returns ``/root`` and ``child``.

    :param path:    the XPath
    :type path:     str
    """
    index = path.rfind('/')
    if index == -1:
        base, last = '', path
    else:
        base, last = path[0:index], path[index + 1:]

    if '@' in last:
        msg = 'Expected path {} to refer to an element; refers to attribute'
        raise exceptions.InvalidPathException(msg.format(path))
    else:
        return base, last

def traverse(dom, path, *,
    initial_element=lambda elem: lxml.etree.Element(elem),
    multiple_options=lambda opts: opts[0],
    no_options=lambda elem, current: None,
    finish=lambda elem, attribute='', attr_xmlns='': None) -> tuple:
    """Traverse an XPath `path` in `dom`.

    *initial_element(name)*
      Called to create the root element representing the tree
    *multiple_options(alternatives)*
      Called whenever `path` is ambiguous.
      Return value is one of `alternatives` to pick to continue traversal.
    *no_options(name, current)*
      Called whenever at element `current` in DOM,
      element `name` does not exist. Return value
      contains a new element to consider as new current
      element. If None is returned instead, traversal is aborted.
    *finish(element, attribute, attr_xmlns)*
      Called when traversal is about to finish at `element`.
      If `attribute` is non-empty, it is the attribute name requested
      in the original `path` and `attr_xmlns` attribute's namespace.
      Return value is second return value of `traverse` function.

    :param dom:                 a root node representing a DOM
    :type dom:                  lxml.etree.Element
    :param path:                an XPath to traverse
    :type path:                 str
    :param multiple_options:    see above
    :type multiple_options:     function
    :param no_options:      see above
    :type no_options:       function
    :param finish:      see above
    :type finish:       function
    :return:        A root node for the new XML DOM and the finish return value
    :rtype:         tuple([lxml.etree.Element, *])
    """
    path, *attrs = str(path).split('@')
    elements = path.lstrip('/').split('/')
    if elements[-1] == '':
        elements = elements[:-1]

    current = dom
    for i, pelement in enumerate(elements):
        if i == 0 and dom is None:
            current = dom = initial_element(pelement)
            continue
        elif i == 0 and (dom.tag == pelement or dom.tag.endswith("}" + pelement)):
            # <tag>.xpath("tag") returns []   => current = dom
            continue

        options = current.xpath(pelement)

        if len(options) == 0 or options is None:
            current = no_options(name=pelement, current=current)
            if current is None:
                return dom, None
        elif len(options) == 1:
            current = options[0]
        else:
            current = multiple_options(options)

    if attrs:
        if ':' in attrs[0]:
            xmlns, attr = attrs[0].split(':')
        else:
            xmlns, attr = None, attrs[0]

        return dom, finish(element=current, attribute=attr, attr_xmlns=xmlns)
    else:
        return dom, finish(element=current)

def xmlns_to_lxml(element, xmlmap={}):
    """Given an `element` and an optional `xmlmap`. Return lxml-element name.

    >>> xmlns_to_lxml('name')
    'name'
    >>> xmlns_to_lxml('text', xmlmap={None: "http://www.w3.org/2000/svg",
    ... "xhtml":"http://www.w3.org/1999/xhtml"})
    ...
    '{http://www.w3.org/2000/svg}text'
    >>> xmlns_to_lxml('xhtml:text', xmlmap={None: "http://www.w3.org/2000/svg",
    ... "xhtml":"http://www.w3.org/1999/xhtml"})
    ...
    '{http://www.w3.org/1999/xhtml}text'

    :param element:     An element/element name
    :type element:      str
    :param xmlmap:      associates namespace identifiers to its URIs
    :type xmlmap:       dict
    :return:            lxml-element name (eg. useful for lxml.etree.Element)
    :rtype:             str
    """
    if not xmlmap:
        return element

    try:
        if ':' in element:
            ns, tag = element.split(":")
        else:
            ns, tag = None, element
    except ValueError:
        msg = "Invalid element name: '{}'".format(element)
        raise exceptions.InvalidPathException(msg)

    try:
        uri = xmlmap[ns]
    except KeyError:
        if ns is None:
            ns = 'default namespace'
        msg = "Unknown XML namespace: {}".format(ns)
        raise exceptions.InvalidPathException(msg)

    return '{' + uri + '}' + tag

def write_base_destination(dom: lxml.etree.Element, path: str, value,
    bases: list, xmlmap=None) -> lxml.etree.Element:
    """Behaves very much like `write_destination`, but also accepts `bases`,
    which defines a set of elements which is considered if the path is ambiguous.

    `value` will be converted to a string.

    :param dom:     root element of an XML DOM
    :type dom:      lxml.etree.Element
    :param path:    XPath to apply
    :type path:     str
    :param value:   the value to be written as text content or attribute value
    :param bases:   a set of elements considered if path is ambiguous
    :type bases:    iterable
    :param xmlmap:  Create new elements with given xmlmap and
                    traverse `path` with given `xmlmap`
    :type xmlmap:   dict
    :return:        text content, attribute or ''
    :rtype:         str
    """
    def root(name):
        return lxml.etree.Element(xmlns_to_lxml(name, xmlmap))

    def base_or_first(alternatives):
        for alt in alternatives:
            if alt in bases:
                return alt
        return alternatives[0]

    def write(element, *, attribute='', attr_xmlns=None):
        if attribute and not attr_xmlns:
            element.attrib[attribute] = str(value)
        elif attribute and attr_xmlns:
            attrname = '{%s}%s' % (attr_xmlns, attribute)
            element.attrib[attrname] = str(value)
        else:
            element.text = str(value)

    def create_element(name, current):
        new_element = lxml.etree.Element(name, nsmap=xmlmap)
        current.append(new_element)
        return new_element

    return traverse(dom, path, initial_element=root,
        multiple_options=base_or_first, no_options=create_element,
        finish=write)[0]

def read_base_source(dom: lxml.etree.Element, path: str, bases: list) -> str:
    """Behaves very much like `read_source`, but also accepts `bases`, which
    defines a set of elements which is considered if the path is ambiguous.

    :param dom:     root element of an XML DOM
    :type dom:      lxml.etree.Element
    :param path:    XPath to apply
    :type path:     str
    :param bases:   a set of elements considered if path is ambiguous
    :type bases:    iterable
    :return:        text content, attribute or ''
    :rtype:         str
    """
    def base_or_first(alternatives):
        for alt in alternatives:
            if alt in bases:
                return alt
        return alternatives[0]

    def read(element, attribute='', attr_xmlns=None):
        # TODO: namespace support
        if attribute:
            return str(element.attrib[attribute])
        else:
            return str(element.text or '')

    def abort(name, current):
        return None

    return traverse(dom, path, multiple_options=base_or_first,
        no_options=abort, finish=read)[1] or ''

def write_new_ambiguous_element(dom: lxml.etree.Element, path: str,
    bases=None, xmlmap=None) -> lxml.etree.Element:
    """Given a `path`, traverse it in `path`, use `bases` on ambiguous elements
    and create a new element for the top-level element of `path`.

    If a base for decision is missing, the first match is taken.

    :param dom:     root element of a DOM tree
    :type dom:      lxml.etree.Element
    :param path:    XPath to apply
    :type path:     str
    :param bases:   bases (ie. elements) to use if ambiguous
    :type bases:    list
    :param xmlmap:  Create new elements with given xmlmap and
                    traverse `path` with given `xmlmap`
    :type xmlmap:   dict
    :return:        the new created element at `path`
    :rtype:         lxml.etree.Element
    """
    path, last = strip_last_element(path)

    if bases is None:
        bases = []
    if not last:
        msg = "Path '{}' does not specify an element to create"
        raise exceptions.InvalidPathException(msg.format(path))

    def base_or_first(alternatives):
        for alt in alternatives:
            if alt in bases:
                return alt
        return alternatives[0]

    def return_element(element, attribute='', attr_xmlns=None):
        if attribute:
            msg = "Expected reference to element, but attribute {} reference given"
            raise exceptions.InvalidPathException(msg.format(attribute))
        return element

    def create_element(name, current):
        new_element = lxml.etree.Element(name, nsmap=xmlmap)
        current.append(new_element)
        return new_element

    last_element = traverse(dom, path, multiple_options=base_or_first,
        no_options=create_element, finish=return_element)[1]
    new_element = create_element(last, last_element)

    return new_element

def read_ambiguous_element(dom: lxml.etree.Element, path: str, bases=None) -> list:
    """Given a `path`, traverse it in `path`, use `bases` on ambiguous elements
    and return all elements which exist at the most-nested level of `path`.

    If a base for decision is missing, the first match is taken.

    :param dom:     root element of a DOM tree
    :type dom:      lxml.etree.Element
    :param path:    XPath to apply
    :type path:     str
    :param bases:   bases (ie. elements) to use if ambiguous
    :type bases:    list
    :return:        a list of elements at `path`
    :rtype:         list([lxml.etree.Element])
    """
    path, last = strip_last_element(path)

    if bases is None:
        bases = []
    if not last:
        return dom.xpath(last)

    def base_or_first(alternatives):
        for alt in alternatives:
            if alt in bases:
                return alt
        return alternatives[0]

    def return_element(element, attribute='', attr_xmlns=None):
        if attribute:
            msg = "Expected reference to element, but attribute {} reference given"
            raise exceptions.InvalidPathException(msg.format(attribute))
        return element

    def cont(name, current):
        return None

    last_element = traverse(dom, path, multiple_options=base_or_first,
        no_options=cont, finish=return_element)[1]
    if last_element is None:
        return []

    return last_element.xpath(last) or []

def write_destination(dom: lxml.etree.Element, path: str, value,
    xmlmap=None) -> lxml.etree.Element:
    """Write a `value` to an XPath `path` in `dom`.
    If `path` points to element, set text node to `value`.
    If `path` points to attribute, set attribute content to `value`.
    `value` will be converted to a string before written.

    This corresponds to the behavior of @destination.

    :param dom:     root element of an XML DOM
    :type dom:      lxml.etree.Element
    :param path:    XPath to apply
    :type path:     str
    :param value:   a value to write, string representation is taken
    :param xmlmap:  Create new elements with given xmlmap and
                    traverse `path` with given `xmlmap`
    :type xmlmap:   dict
    :return:        the (potentially modified) `dom` element
    :rtype:         lxml.etree.Element
    """
    def root(name):
        element_id = xmlns_to_lxml(name, xmlmap=xmlmap)
        return lxml.etree.Element(element_id, nsmap=xmlmap)

    def first(alternatives):
        return alternatives[0]

    def write(element, *, attribute='', attr_xmlns=None):
        if attribute and not attr_xmlns:
            element.attrib[attribute] = str(value)
        elif attribute and attr_xmlns:
            try:
                attrname = '{%s}%s' % (xmlmap[attr_xmlns], attr)
                element.attrib[attrname] = str(value)
            except KeyError:
                raise KeyError("Unknown namespace: {}".format(attr_xmlns))
        else:
            element.text = str(value)

    def cont(name, current):
        new_element = lxml.etree.Element(name, nsmap=xmlmap)
        current.append(new_element)
        return new_element

    return traverse(dom, path, initial_element=root,
        multiple_options=first, no_options=cont, finish=write)[0]

def read_source(dom: lxml.etree.Element, path: str) -> str:
    """Apply a XPath `path` to `dom`. If path is ambiguous, take first option.
    If `path` points to element, return text node of it.
    If `path` points to attribute, return attribute content as string.
    If any error occurs, return an empty string.

    This corresponds to the behavior of @source.

    :param dom:     root element of an XML DOM
    :type dom:      lxml.etree.Element
    :param path:    XPath to apply
    :type path:     str
    :return:        text content, attribute or ''
    :rtype:         str
    """
    # TODO: namespace support
    if path == '':
        return ''
    elif '@' in path:
        val = dom.xpath(path)
        if val:
            return val[0] or ''
        else:
            return ''
    else:
        elements = dom.xpath(path)
        if elements:
            return elements[0].text or ''
        else:
            return ''
<commit_msg>Fix typo.<commit_after>#!/usr/bin/env python3

"""
    XML creator
    -----------

    This library provides custom XML tools
    for ruledxml using the lxml library.

    (C) 2015, meisterluk, BSD 3-clause license
"""

import lxml.etree

from . import exceptions

def read(xmlinfile: str):
    """Given a filepath or file descriptor to an XML file, read the XML.

    :param xmlinfile:   file descriptor to XML file
    :type xmlinfile:    str
    :return:            an object representing the document object model
    :rtype:             lxml.etree.Element
    """
    return lxml.etree.parse(xmlinfile).getroot()

def write(dom: lxml.etree.Element, fd, encoding='utf-8', **lxml_options):
    """Write a given DOM into open file descriptor `fd`.

    :param dom:             a DOM (ie. root element) to store in an XML file
    :type dom:              lxml.etree.Element
    :param fd:              the file descriptor to write to
    :type fd:               _io.TextIOWrapper
    :param encoding:        which encoding shall be used for the XML file?
    :type encoding:         str
    :param lxml_options:    options for the lxml.etree.tostring
    :type lxml_options:     dict
    """
    opts = {
        'xml_declaration': True,
        'pretty_print': True,
        'encoding': encoding
    }
    opts.update(lxml_options)
    fd.write(lxml.etree.tostring(dom, **opts))

def strip_last_element(path):
    """Strip the last element off an XPath `path`.
    Raises InvalidPathException, if path refers to an attribute.

    For example ``/root/child`` returns ``/root`` and ``child``.

    :param path:    the XPath
    :type path:     str
    """
    index = path.rfind('/')
    if index == -1:
        base, last = '', path
    else:
        base, last = path[0:index], path[index + 1:]

    if '@' in last:
        msg = 'Expected path {} to refer to an element; refers to attribute'
        raise exceptions.InvalidPathException(msg.format(path))
    else:
        return base, last

def traverse(dom, path, *,
    initial_element=lambda elem: lxml.etree.Element(elem),
    multiple_options=lambda opts: opts[0],
    no_options=lambda elem, current: None,
    finish=lambda elem, attribute='', attr_xmlns='': None) -> tuple:
    """Traverse an XPath `path` in `dom`.

    *initial_element(name)*
      Called to create the root element representing the tree
    *multiple_options(alternatives)*
      Called whenever `path` is ambiguous.
      Return value is one of `alternatives` to pick to continue traversal.
    *no_options(name, current)*
      Called whenever at element `current` in DOM,
      element `name` does not exist. Return value
      contains a new element to consider as new current
      element. If None is returned instead, traversal is aborted.
    *finish(element, attribute, attr_xmlns)*
      Called when traversal is about to finish at `element`.
      If `attribute` is non-empty, it is the attribute name requested
      in the original `path` and `attr_xmlns` attribute's namespace.
      Return value is second return value of `traverse` function.

    :param dom:                 a root node representing a DOM
    :type dom:                  lxml.etree.Element
    :param path:                an XPath to traverse
    :type path:                 str
    :param multiple_options:    see above
    :type multiple_options:     function
    :param no_options:      see above
    :type no_options:       function
    :param finish:      see above
    :type finish:       function
    :return:        A root node for the new XML DOM and the finish return value
    :rtype:         tuple([lxml.etree.Element, *])
    """
    path, *attrs = str(path).split('@')
    elements = path.lstrip('/').split('/')
    if elements[-1] == '':
        elements = elements[:-1]

    current = dom
    for i, pelement in enumerate(elements):
        if i == 0 and dom is None:
            current = dom = initial_element(pelement)
            continue
        elif i == 0 and (dom.tag == pelement or dom.tag.endswith("}" + pelement)):
            # <tag>.xpath("tag") returns []   => current = dom
            continue

        options = current.xpath(pelement)

        if len(options) == 0 or options is None:
            current = no_options(name=pelement, current=current)
            if current is None:
                return dom, None
        elif len(options) == 1:
            current = options[0]
        else:
            current = multiple_options(options)

    if attrs:
        if ':' in attrs[0]:
            xmlns, attr = attrs[0].split(':')
        else:
            xmlns, attr = None, attrs[0]

        return dom, finish(element=current, attribute=attr, attr_xmlns=xmlns)
    else:
        return dom, finish(element=current)

def xmlns_to_lxml(element, xmlmap={}):
    """Given an `element` and an optional `xmlmap`. Return lxml-element name.

    >>> xmlns_to_lxml('name')
    'name'
    >>> xmlns_to_lxml('text', xmlmap={None: "http://www.w3.org/2000/svg",
    ... "xhtml":"http://www.w3.org/1999/xhtml"})
    ...
    '{http://www.w3.org/2000/svg}text'
    >>> xmlns_to_lxml('xhtml:text', xmlmap={None: "http://www.w3.org/2000/svg",
    ... "xhtml":"http://www.w3.org/1999/xhtml"})
    ...
    '{http://www.w3.org/1999/xhtml}text'

    :param element:     An element/element name
    :type element:      str
    :param xmlmap:      associates namespace identifiers to its URIs
    :type xmlmap:       dict
    :return:            lxml-element name (eg. useful for lxml.etree.Element)
    :rtype:             str
    """
    if not xmlmap:
        return element

    try:
        if ':' in element:
            ns, tag = element.split(":")
        else:
            ns, tag = None, element
    except ValueError:
        msg = "Invalid element name: '{}'".format(element)
        raise exceptions.InvalidPathException(msg)

    try:
        uri = xmlmap[ns]
    except KeyError:
        if ns is None:
            ns = 'default namespace'
        msg = "Unknown XML namespace: {}".format(ns)
        raise exceptions.InvalidPathException(msg)

    return '{' + uri + '}' + tag

def write_base_destination(dom: lxml.etree.Element, path: str, value,
    bases: list, xmlmap=None) -> lxml.etree.Element:
    """Behaves very much like `write_destination`, but also accepts `bases`,
    which defines a set of elements which is considered if the path is ambiguous.

    `value` will be converted to a string.

    :param dom:     root element of an XML DOM
    :type dom:      lxml.etree.Element
    :param path:    XPath to apply
    :type path:     str
    :param value:   the value to be written as text content or attribute value
    :param bases:   a set of elements considered if path is ambiguous
    :type bases:    iterable
    :param xmlmap:  Create new elements with given xmlmap and
                    traverse `path` with given `xmlmap`
    :type xmlmap:   dict
    :return:        text content, attribute or ''
    :rtype:         str
    """
    def root(name):
        return lxml.etree.Element(xmlns_to_lxml(name, xmlmap))

    def base_or_first(alternatives):
        for alt in alternatives:
            if alt in bases:
                return alt
        return alternatives[0]

    def write(element, *, attribute='', attr_xmlns=None):
        if attribute and not attr_xmlns:
            element.attrib[attribute] = str(value)
        elif attribute and attr_xmlns:
            attrname = '{%s}%s' % (attr_xmlns, attribute)
            element.attrib[attrname] = str(value)
        else:
            element.text = str(value)

    def create_element(name, current):
        new_element = lxml.etree.Element(name, nsmap=xmlmap)
        current.append(new_element)
        return new_element

    return traverse(dom, path, initial_element=root,
        multiple_options=base_or_first, no_options=create_element,
        finish=write)[0]

def read_base_source(dom: lxml.etree.Element, path: str, bases: list) -> str:
    """Behaves very much like `read_source`, but also accepts `bases`, which
    defines a set of elements which is considered if the path is ambiguous.

    :param dom:     root element of an XML DOM
    :type dom:      lxml.etree.Element
    :param path:    XPath to apply
    :type path:     str
    :param bases:   a set of elements considered if path is ambiguous
    :type bases:    iterable
    :return:        text content, attribute or ''
    :rtype:         str
    """
    def base_or_first(alternatives):
        for alt in alternatives:
            if alt in bases:
                return alt
        return alternatives[0]

    def read(element, attribute='', attr_xmlns=None):
        # TODO: namespace support
        if attribute:
            return str(element.attrib[attribute])
        else:
            return str(element.text or '')

    def abort(name, current):
        return None

    return traverse(dom, path, multiple_options=base_or_first,
        no_options=abort, finish=read)[1] or ''

def write_new_ambiguous_element(dom: lxml.etree.Element, path: str,
    bases=None, xmlmap=None) -> lxml.etree.Element:
    """Given a `path`, traverse it in `path`, use `bases` on ambiguous elements
    and create a new element for the top-level element of `path`.

    If a base for decision is missing, the first match is taken.

    :param dom:     root element of a DOM tree
    :type dom:      lxml.etree.Element
    :param path:    XPath to apply
    :type path:     str
    :param bases:   bases (ie. elements) to use if ambiguous
    :type bases:    list
    :param xmlmap:  Create new elements with given xmlmap and
                    traverse `path` with given `xmlmap`
    :type xmlmap:   dict
    :return:        the new created element at `path`
    :rtype:         lxml.etree.Element
    """
    path, last = strip_last_element(path)

    if bases is None:
        bases = []
    if not last:
        msg = "Path '{}' does not specify an element to create"
        raise exceptions.InvalidPathException(msg.format(path))

    def base_or_first(alternatives):
        for alt in alternatives:
            if alt in bases:
                return alt
        return alternatives[0]

    def return_element(element, attribute='', attr_xmlns=None):
        if attribute:
            msg = "Expected reference to element, but attribute {} reference given"
            raise exceptions.InvalidPathException(msg.format(attribute))
        return element

    def create_element(name, current):
        new_element = lxml.etree.Element(name, nsmap=xmlmap)
        current.append(new_element)
        return new_element

    last_element = traverse(dom, path, multiple_options=base_or_first,
        no_options=create_element, finish=return_element)[1]
    new_element = create_element(last, last_element)

    return new_element

def read_ambiguous_element(dom: lxml.etree.Element, path: str, bases=None) -> list:
    """Given a `path`, traverse it in `path`, use `bases` on ambiguous elements
    and return all elements which exist at the most-nested level of `path`.

    If a base for decision is missing, the first match is taken.

    :param dom:     root element of a DOM tree
    :type dom:      lxml.etree.Element
    :param path:    XPath to apply
    :type path:     str
    :param bases:   bases (ie. elements) to use if ambiguous
    :type bases:    list
    :return:        a list of elements at `path`
    :rtype:         list([lxml.etree.Element])
    """
    path, last = strip_last_element(path)

    if bases is None:
        bases = []
    if not last:
        return dom.xpath(last)

    def base_or_first(alternatives):
        for alt in alternatives:
            if alt in bases:
                return alt
        return alternatives[0]

    def return_element(element, attribute='', attr_xmlns=None):
        if attribute:
            msg = "Expected reference to element, but attribute {} reference given"
            raise exceptions.InvalidPathException(msg.format(attribute))
        return element

    def cont(name, current):
        return None

    last_element = traverse(dom, path, multiple_options=base_or_first,
        no_options=cont, finish=return_element)[1]
    if last_element is None:
        return []

    return last_element.xpath(last) or []

def write_destination(dom: lxml.etree.Element, path: str, value,
    xmlmap=None) -> lxml.etree.Element:
    """Write a `value` to an XPath `path` in `dom`.
    If `path` points to element, set text node to `value`.
    If `path` points to attribute, set attribute content to `value`.
    `value` will be converted to a string before written.

    This corresponds to the behavior of @destination.

    :param dom:     root element of an XML DOM
    :type dom:      lxml.etree.Element
    :param path:    XPath to apply
    :type path:     str
    :param value:   a value to write, string representation is taken
    :param xmlmap:  Create new elements with given xmlmap and
                    traverse `path` with given `xmlmap`
    :type xmlmap:   dict
    :return:        the (potentially modified) `dom` element
    :rtype:         lxml.etree.Element
    """
    def root(name):
        element_id = xmlns_to_lxml(name, xmlmap=xmlmap)
        return lxml.etree.Element(element_id, nsmap=xmlmap)

    def first(alternatives):
        return alternatives[0]

    def write(element, *, attribute='', attr_xmlns=None):
        if attribute and not attr_xmlns:
            element.attrib[attribute] = str(value)
        elif attribute and attr_xmlns:
            try:
                attrname = '{%s}%s' % (xmlmap[attr_xmlns], attribute)
                element.attrib[attrname] = str(value)
            except KeyError:
                raise KeyError("Unknown namespace: {}".format(attr_xmlns))
        else:
            element.text = str(value)

    def cont(name, current):
        new_element = lxml.etree.Element(name, nsmap=xmlmap)
        current.append(new_element)
        return new_element

    return traverse(dom, path, initial_element=root,
        multiple_options=first, no_options=cont, finish=write)[0]

def read_source(dom: lxml.etree.Element, path: str) -> str:
    """Apply a XPath `path` to `dom`. If path is ambiguous, take first option.
    If `path` points to element, return text node of it.
    If `path` points to attribute, return attribute content as string.
    If any error occurs, return an empty string.

    This corresponds to the behavior of @source.

    :param dom:     root element of an XML DOM
    :type dom:      lxml.etree.Element
    :param path:    XPath to apply
    :type path:     str
    :return:        text content, attribute or ''
    :rtype:         str
    """
    # TODO: namespace support
    if path == '':
        return ''
    elif '@' in path:
        val = dom.xpath(path)
        if val:
            return val[0] or ''
        else:
            return ''
    else:
        elements = dom.xpath(path)
        if elements:
            return elements[0].text or ''
        else:
            return ''
<|endoftext|>


<commit_before>from django.conf.urls import url, include
from django.contrib import admin
from django.http import HttpResponse
from rvpsite.core import views as rvpsite_views
from rvpsite.contact.views import contact
from rvpsite.blog.views import blog, one_blog
from rvpsite.client.views import clients, clients_tmp

urlpatterns = [
    url(r'^admin/', admin.site.urls),
    url(r'^$', rvpsite_views.home, name='home'),
    url(r'^area/$', rvpsite_views.area, name='area'),
    url(r'^representadas/$', rvpsite_views.representations, name='representations'),
    url(r'^clientes/$', clients, name='clients'),
    url(r'^clientes_novo/$', clients_tmp, name='new_clients'), # desenvolvimento
    url(r'^noticias(?:/(?P<category>[\w-]*)(?:/(?P<page>[\w]*))?)?/$', blog, name='blog'),
    url(r'^noticia/(?P<slug>[\w-]*)/', one_blog, name='one_blog'),
    url(r'^s3direct/', include('s3direct.urls')),
    url(r'^contato/$', contact, name='contact'),
    url(r'^robots.txt', lambda x: HttpResponse("User-Agent: *\nDisallow: /clientes/\nDisallow: /clientes_novo/",
                                               content_type="text/plain"), name="robots_file"),
]
<commit_msg>robots.txt update<commit_after>from django.conf.urls import url, include
from django.contrib import admin
from django.http import HttpResponse
from rvpsite.core import views as rvpsite_views
from rvpsite.contact.views import contact
from rvpsite.blog.views import blog, one_blog
from rvpsite.client.views import clients, clients_tmp

urlpatterns = [
    url(r'^admin/', admin.site.urls),
    url(r'^$', rvpsite_views.home, name='home'),
    url(r'^area/$', rvpsite_views.area, name='area'),
    url(r'^representadas/$', rvpsite_views.representations, name='representations'),
    url(r'^clientes/$', clients, name='clients'),
    url(r'^clientes_novo/$', clients_tmp, name='new_clients'), # desenvolvimento
    url(r'^noticias(?:/(?P<category>[\w-]*)(?:/(?P<page>[\w]*))?)?/$', blog, name='blog'),
    url(r'^noticia/(?P<slug>[\w-]*)/', one_blog, name='one_blog'),
    url(r'^s3direct/', include('s3direct.urls')),
    url(r'^contato/$', contact, name='contact'),
    url(r'^robots.txt', lambda x: HttpResponse("User-Agent: *\nDisallow: /clientes/\nDisallow: /clientes_novo/\nDisallow: /noticia/",
                                               content_type="text/plain"), name="robots_file"),
]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import pytest
import time
import sys
from .as_status_codes import AerospikeStatus
from aerospike import exception as e
from aerospike import predicates as p

aerospike = pytest.importorskip("aerospike")
try:
    import aerospike
except:
    print("Please install aerospike python client.")
    sys.exit(1)

def add_indexes_to_client(client):
    try:
        client.index_integer_create('test', 'demo', 'age',
                                    'test_demo_age_idx')
    except e.IndexFoundError:
        pass

    try:
        client.index_integer_create('test', None, 'age',
                                    'test_null_age_idx')
    except e.IndexFoundError:
        pass

def add_test_udf(client):
    policy = {}
    client.udf_put(u"bin_lua.lua", 0, policy)

def drop_test_udf(client):
    client.udf_remove("bin_lua.lua")

def remove_indexes_from_client(client):
    client.index_remove('test', 'test_demo_age_idx')
    client.index_remove('test', 'test_null_age_idx')

class TestQueryApply(object):

    # These functions will run once for this test class, and do all of the
    # required setup and teardown
    connection_setup_functions = (add_test_udf, add_indexes_to_client)
    connection_teardown_functions = (drop_test_udf, remove_indexes_from_client)
    age_range_pred = p.between('age', 0, 5)  # Predicate for ages between [0,5)

    @pytest.fixture(autouse=True)
    def setup(self, request, connection_with_config_funcs):
        client = connection_with_config_funcs
        for i in range(1, 5):
            key = ('test', 'demo', i)
            rec = {'name': 'name%s' % (str(i)), 'age': i}
            client.put(key, rec)

        key = ('test', None, "no_set")
        self.no_set_key = key
        rec = {'name': 'no_set_name', 'age': 0}
        client.put(key, rec)

        def teardown():
            for i in range(1, 5):
                key = ('test', 'demo', i)
                client.remove(key)

            client.remove(self.no_set_key)

        request.addfinalizer(teardown)

    def test_query_apply_with_no_parameters(self):
        """
        Invoke query_apply() without any mandatory parameters.
        It should raise a type error as the wrong parameters are passed
        """
        with pytest.raises(TypeError) as typeError:
            self.as_connection.query_apply()

    def test_query_apply_with_correct_parameters(self):
        """
        Invoke query_apply() with correct parameters.
        It should apply the proper UDF, and
        """
        query_id = self.as_connection.query_apply(
            "test", "demo", self.age_range_pred, "bin_lua",
            "mytransform", ['age', 2])

        time.sleep(0.1)
        while True:
            response = self.as_connection.job_info(
                query_id, aerospike.JOB_QUERY)
            if response['status'] != aerospike.JOB_STATUS_INPROGRESS:
                break
            time.sleep(0.1)

        for i in range(1, 5):
            key = ('test', 'demo', i)
            _, _, bins = self.as_connection.get(key)

            assert bins['age'] == i + 2

        _, _, bins = self.as_connection.get(self.no_set_key)
        assert bins['age'] == 0

    def test_query_apply_with_correct_policy(self):
        """
        Invoke query_apply() with correct policy
        """
        policy = {'timeout': 1000}
        query_id = self.as_connection.query_apply(
            "test", "demo", self.age_range_pred, "bin_lua",
            "mytransform", ['age', 2], policy)

        time.sleep(0.1)
        while True:
            response = self.as_connection.job_info(
                query_id, aerospike.JOB_QUERY)
            if response['status'] != aerospike.JOB_STATUS_INPROGRESS:
                break
            time.sleep(0.1)

        for i in range(1, 5):
            key = ('test', 'demo', i)
            _, _, bins = self.as_connection.get(key)
            assert bins['age'] == i + 2

        _, _, bins = self.as_connection.get(self.no_set_key)
        assert bins['age'] == 0

    def test_query_apply_with_none_set(self):
        """
        Invoke query_apply() with correct policy,
        Should casuse no changes as the
        """
        policy = {'timeout': 1000}
        query_id = self.as_connection.query_apply(
            "test", None, self.age_range_pred, "bin_lua",
            "mytransform", ['age', 2], policy)

        time.sleep(0.1)
        while True:
            response = self.as_connection.job_info(
                query_id, aerospike.JOB_QUERY)
            if response['status'] != aerospike.JOB_STATUS_INPROGRESS:
                break
            time.sleep(0.1)

        # Since this query passes no set, the records stored in a set should
        # not be affected
        for i in range(1, 5):
            key = ('test', 'demo', i)
            _, _, bins = self.as_connection.get(key)
            assert bins['age'] == i

        # the setless record should have been changed
        _, _, bins = self.as_connection.get(self.no_set_key)
        assert bins['age'] == 2

    def test_query_apply_with_incorrect_policy(self):
        """
        Invoke query_apply() with incorrect policy
        """
        policy = {
            'timeout': 0.5
        }

        with pytest.raises(e.ParamError) as err_info:
            self.as_connection.query_apply(
                "test", "demo", self.age_range_pred, "bin_lua",
                "mytransform", ['age', 2], policy)

        err_code = err_info.value.code
        assert err_code == AerospikeStatus.AEROSPIKE_ERR_PARAM

    def test_query_apply_with_incorrect_ns_set(self):
        """
        Invoke query_apply() with incorrect ns and set
        """
        with pytest.raises(e.NamespaceNotFound) as err_info:
            self.as_connection.query_apply(
                "test1", "demo1", self.age_range_pred, "bin_lua",
                "mytransform", ['age', 2])

        err_code = err_info.value.code
        assert err_code == AerospikeStatus.AEROSPIKE_ERR_NAMESPACE_NOT_FOUND

    def test_query_apply_with_incorrect_module_name(self):
        """
        Invoke query_apply() with incorrect module name
        Test that passing an incorrect lua_module does
        not call a function
        """
        query_id = self.as_connection.query_apply(
            "test", "demo", self.age_range_pred, "bin_lua_incorrect",
            "mytransform", ['age', 2])

        time.sleep(0.1)
        while True:
            response = self.as_connection.job_info(
                query_id, aerospike.JOB_QUERY)
            if response['status'] != aerospike.JOB_STATUS_INPROGRESS:
                break
            time.sleep(0.1)

        for i in range(1, 5):
            key = ('test', 'demo', i)
            _, _, bins = self.as_connection.get(key)
            assert bins['age'] == i

    def test_query_apply_with_incorrect_function_name(self):
        """
        Invoke query_apply() with incorrect function name,
        does not invoke a different function
        """
        query_id = self.as_connection.query_apply(
            "test", "demo", self.age_range_pred, "bin_lua",
            "mytransform_incorrect", ['age', 2])

        time.sleep(0.1)
        while True:
            response = self.as_connection.job_info(
                query_id, aerospike.JOB_QUERY)
            if response['status'] != aerospike.JOB_STATUS_INPROGRESS:
                break
            time.sleep(0.1)

        for i in range(1, 5):
            key = ('test', 'demo', i)
            _, _, bins = self.as_connection.get(key)
            assert bins['age'] == i

    def test_query_apply_with_ns_set_none(self):
        """
        Invoke query_apply() with ns and set as None
        """
        with pytest.raises(TypeError) as typeError:
            self.as_connection.query_apply(None, None, self.age_range_pred,
                                           "bin_lua", "mytransform",
                                           ['age', 2])

        assert "query_apply() argument 1 must be str" in str(typeError.value)

    def test_query_apply_with_module_function_none(self):
        """
        Invoke query_apply() with None module function
        """

        with pytest.raises(e.ParamError) as err_info:
            self.as_connection.query_apply(
                "test", "demo", self.age_range_pred, None, None, ['age', 2])

        err_code = err_info.value.code
        assert err_code == AerospikeStatus.AEROSPIKE_ERR_PARAM

    def test_query_apply_with_extra_argument(self):
        """
        Invoke query_apply() with extra argument
        """
        policy = {'timeout': 1000}
        with pytest.raises(TypeError) as typeError:
            self.as_connection.query_apply(
                "test", "demo", self.age_range_pred, "bin_lua",
                "mytransform_incorrect", ['age', 2], policy, "")

        assert "query_apply() takes at most 7 arguments (8 given)" in str(
            typeError.value)

    def test_query_apply_with_argument_is_string(self):
        """
        Invoke query_apply() with arguments as string
        """
        with pytest.raises(e.ParamError) as err_info:
            self.as_connection.query_apply("test", "demo", self.age_range_pred,
                                           "bin_lua", "mytransform", "")

        err_code = err_info.value.code
        assert err_code == AerospikeStatus.AEROSPIKE_ERR_PARAM

    def test_query_apply_with_argument_is_none(self):
        """
        Invoke query_apply() with arguments as None
        """
        with pytest.raises(e.ParamError) as err_info:
            self.as_connection.query_apply(
                "test", "demo", self.age_range_pred,
                "bin_lua", "mytransform", None)

        err_code = err_info.value.code
        assert err_code == AerospikeStatus.AEROSPIKE_ERR_PARAM

    def test_query_apply_with_extra_call_to_lua(self):
        """
        Invoke query_apply() with extra call to lua
        test that passing an extra argument to a udf does
        not cause the function to fail
        """
        query_id = self.as_connection.query_apply(
            "test", "demo", self.age_range_pred,
            "bin_lua", "mytransform", ['age', 2, 3])

        # time.sleep(2)

        time.sleep(0.1)
        while True:
            response = self.as_connection.job_info(
                query_id, aerospike.JOB_QUERY)
            if response['status'] != aerospike.JOB_STATUS_INPROGRESS:
                break
            time.sleep(0.1)

        for i in range(1, 5):
            key = ('test', 'demo', i)
            _, _, bins = self.as_connection.get(key)
            assert bins['age'] == i + 2

    def test_query_apply_with_extra_parameter_in_lua(self):
        """
        Invoke query_apply() with a missing argument
        to a lua function does not cause an error
        """
        query_id = self.as_connection.query_apply(
            "test", "demo", self.age_range_pred,
            "bin_lua", "mytransformextra", ['age', 2])

        # time.sleep(2)

        time.sleep(0.1)
        while True:
            response = self.as_connection.job_info(
                query_id, aerospike.JOB_QUERY)
            if response['status'] != aerospike.JOB_STATUS_INPROGRESS:
                break
            time.sleep(0.1)

        for i in range(1, 5):
            key = ('test', 'demo', i)
            _, _, bins = self.as_connection.get(key)
            assert bins['age'] == i + 2

    def test_query_apply_with_less_parameter_in_lua(self):
        """
        Invoke query_apply() with less parameter in lua
        this verifies that passing 3 arguments to a lua
        function which expects 2, will cause an undefined int
        to be treated as 0, and not crash.
        """
        query_id = self.as_connection.query_apply(
            "test", "demo", self.age_range_pred,
            "bin_lua", "mytransformless", ['age', 2])

        # time.sleep(2)

        time.sleep(0.1)
        while True:
            response = self.as_connection.job_info(
                query_id, aerospike.JOB_QUERY)
            if response['status'] != aerospike.JOB_STATUS_INPROGRESS:
                break
            time.sleep(0.1)

        for i in range(1, 5):
            key = ('test', 'demo', i)
            _, _, bins = self.as_connection.get(key)
            if bins['age'] != i:
                assert True is False
            else:
                assert True is True

    def test_query_apply_unicode_input(self):
        """
        Invoke query_apply() with unicode udf
        """
        query_id = self.as_connection.query_apply(
            "test", "demo", self.age_range_pred, u"bin_lua",
            u"mytransform", ['age', 2])

        time.sleep(0.1)
        while True:
            response = self.as_connection.job_info(
                query_id, aerospike.JOB_QUERY)
            if response['status'] != aerospike.JOB_STATUS_INPROGRESS:
                break
            time.sleep(0.1)

        for i in range(1, 5):
            key = ('test', 'demo', i)
            _, _, bins = self.as_connection.get(key)
            assert bins['age'] == i + 2

    def test_query_apply_with_correct_parameters_without_connection(self):
        """
        Invoke query_apply() with correct parameters without connection
        """
        config = {'hosts': [('127.0.0.1', 3000)]}
        client1 = aerospike.client(config)

        with pytest.raises(e.ClusterError) as err_info:
            client1.query_apply("test", "demo", self.age_range_pred,
                                "bin_lua", "mytransform", ['age', 2])

        err_code = err_info.value.code

        assert err_code == AerospikeStatus.AEROSPIKE_CLUSTER_ERROR

    @pytest.mark.skip(reason="This isn't a test for query_apply," +
                             " but for job_info")
    def test_job_info_with_incorrect_module_type(self):
        """
        Invoke query_apply() with incorrect module type
        """
        query_id = self.as_connection.query_apply(
            "test", "demo", self.age_range_pred, "bin_lua",
            "mytransform", ['age', 2])

        with pytest.raises(e.ParamError) as err_info:
            time.sleep(0.1)
            self.as_connection.job_info(query_id, "aggregate")
        err_code = err_info.value.code

        assert err_code == AerospikeStatus.AEROSPIKE_ERR_PARAM

    # @pytest.mark.xfail(reason="Passing an invalid predicate currently works " +
    #                           " or raises a System Error")
    @pytest.mark.parametrize(
        "predicate",
        (
            (),  # Raises system error
            None,
            ('age'),
            ('age', 1),
            ('age', 1, 5),
            (1, 1, 'bin')  # start of a valid predicate
        )
    )
    def test_invalid_predicate(self, predicate):

        with pytest.raises(e.ParamError) as err_info:
            query_id = self.as_connection.query_apply(
                "test", "demo", predicate, "bin_lua",
                "mytransform", ['age', 2])

        err_code = err_info.value.code
        assert err_code == AerospikeStatus.AEROSPIKE_ERR_PARAM
<commit_msg>Fix parameter to use newer name<commit_after># -*- coding: utf-8 -*-
import pytest
import time
import sys
from .as_status_codes import AerospikeStatus
from aerospike import exception as e
from aerospike import predicates as p

aerospike = pytest.importorskip("aerospike")
try:
    import aerospike
except:
    print("Please install aerospike python client.")
    sys.exit(1)

def add_indexes_to_client(client):
    try:
        client.index_integer_create('test', 'demo', 'age',
                                    'test_demo_age_idx')
    except e.IndexFoundError:
        pass

    try:
        client.index_integer_create('test', None, 'age',
                                    'test_null_age_idx')
    except e.IndexFoundError:
        pass

def add_test_udf(client):
    policy = {}
    client.udf_put(u"bin_lua.lua", 0, policy)

def drop_test_udf(client):
    client.udf_remove("bin_lua.lua")

def remove_indexes_from_client(client):
    client.index_remove('test', 'test_demo_age_idx')
    client.index_remove('test', 'test_null_age_idx')

class TestQueryApply(object):

    # These functions will run once for this test class, and do all of the
    # required setup and teardown
    connection_setup_functions = (add_test_udf, add_indexes_to_client)
    connection_teardown_functions = (drop_test_udf, remove_indexes_from_client)
    age_range_pred = p.between('age', 0, 5)  # Predicate for ages between [0,5)

    @pytest.fixture(autouse=True)
    def setup(self, request, connection_with_config_funcs):
        client = connection_with_config_funcs
        for i in range(1, 5):
            key = ('test', 'demo', i)
            rec = {'name': 'name%s' % (str(i)), 'age': i}
            client.put(key, rec)

        key = ('test', None, "no_set")
        self.no_set_key = key
        rec = {'name': 'no_set_name', 'age': 0}
        client.put(key, rec)

        def teardown():
            for i in range(1, 5):
                key = ('test', 'demo', i)
                client.remove(key)

            client.remove(self.no_set_key)

        request.addfinalizer(teardown)

    def test_query_apply_with_no_parameters(self):
        """
        Invoke query_apply() without any mandatory parameters.
        It should raise a type error as the wrong parameters are passed
        """
        with pytest.raises(TypeError) as typeError:
            self.as_connection.query_apply()

    def test_query_apply_with_correct_parameters(self):
        """
        Invoke query_apply() with correct parameters.
        It should apply the proper UDF, and
        """
        query_id = self.as_connection.query_apply(
            "test", "demo", self.age_range_pred, "bin_lua",
            "mytransform", ['age', 2])

        time.sleep(0.1)
        while True:
            response = self.as_connection.job_info(
                query_id, aerospike.JOB_QUERY)
            if response['status'] != aerospike.JOB_STATUS_INPROGRESS:
                break
            time.sleep(0.1)

        for i in range(1, 5):
            key = ('test', 'demo', i)
            _, _, bins = self.as_connection.get(key)

            assert bins['age'] == i + 2

        _, _, bins = self.as_connection.get(self.no_set_key)
        assert bins['age'] == 0

    def test_query_apply_with_correct_policy(self):
        """
        Invoke query_apply() with correct policy
        """
        policy = {'total_timeout': 0}
        query_id = self.as_connection.query_apply(
            "test", "demo", self.age_range_pred, "bin_lua",
            "mytransform", ['age', 2], policy)

        time.sleep(0.1)
        while True:
            response = self.as_connection.job_info(
                query_id, aerospike.JOB_QUERY)
            if response['status'] != aerospike.JOB_STATUS_INPROGRESS:
                break
            time.sleep(0.1)

        for i in range(1, 5):
            key = ('test', 'demo', i)
            _, _, bins = self.as_connection.get(key)
            assert bins['age'] == i + 2

        _, _, bins = self.as_connection.get(self.no_set_key)
        assert bins['age'] == 0

    def test_query_apply_with_none_set(self):
        """
        Invoke query_apply() with correct policy,
        Should casuse no changes as the
        """
        policy = {'total_timeout': 0}
        query_id = self.as_connection.query_apply(
            "test", None, self.age_range_pred, "bin_lua",
            "mytransform", ['age', 2], policy)

        time.sleep(0.1)
        while True:
            response = self.as_connection.job_info(
                query_id, aerospike.JOB_QUERY)
            if response['status'] != aerospike.JOB_STATUS_INPROGRESS:
                break
            time.sleep(0.1)

        # Since this query passes no set, the records stored in a set should
        # not be affected
        for i in range(1, 5):
            key = ('test', 'demo', i)
            _, _, bins = self.as_connection.get(key)
            assert bins['age'] == i

        # the setless record should have been changed
        _, _, bins = self.as_connection.get(self.no_set_key)
        assert bins['age'] == 2

    def test_query_apply_with_incorrect_policy(self):
        """
        Invoke query_apply() with incorrect policy
        """
        policy = {
            'timeout': 0.5
        }

        with pytest.raises(e.ParamError) as err_info:
            self.as_connection.query_apply(
                "test", "demo", self.age_range_pred, "bin_lua",
                "mytransform", ['age', 2], policy)

        err_code = err_info.value.code
        assert err_code == AerospikeStatus.AEROSPIKE_ERR_PARAM

    def test_query_apply_with_incorrect_ns_set(self):
        """
        Invoke query_apply() with incorrect ns and set
        """
        with pytest.raises(e.NamespaceNotFound) as err_info:
            self.as_connection.query_apply(
                "test1", "demo1", self.age_range_pred, "bin_lua",
                "mytransform", ['age', 2])

        err_code = err_info.value.code
        assert err_code == AerospikeStatus.AEROSPIKE_ERR_NAMESPACE_NOT_FOUND

    def test_query_apply_with_incorrect_module_name(self):
        """
        Invoke query_apply() with incorrect module name
        Test that passing an incorrect lua_module does
        not call a function
        """
        query_id = self.as_connection.query_apply(
            "test", "demo", self.age_range_pred, "bin_lua_incorrect",
            "mytransform", ['age', 2])

        time.sleep(0.1)
        while True:
            response = self.as_connection.job_info(
                query_id, aerospike.JOB_QUERY)
            if response['status'] != aerospike.JOB_STATUS_INPROGRESS:
                break
            time.sleep(0.1)

        for i in range(1, 5):
            key = ('test', 'demo', i)
            _, _, bins = self.as_connection.get(key)
            assert bins['age'] == i

    def test_query_apply_with_incorrect_function_name(self):
        """
        Invoke query_apply() with incorrect function name,
        does not invoke a different function
        """
        query_id = self.as_connection.query_apply(
            "test", "demo", self.age_range_pred, "bin_lua",
            "mytransform_incorrect", ['age', 2])

        time.sleep(0.1)
        while True:
            response = self.as_connection.job_info(
                query_id, aerospike.JOB_QUERY)
            if response['status'] != aerospike.JOB_STATUS_INPROGRESS:
                break
            time.sleep(0.1)

        for i in range(1, 5):
            key = ('test', 'demo', i)
            _, _, bins = self.as_connection.get(key)
            assert bins['age'] == i

    def test_query_apply_with_ns_set_none(self):
        """
        Invoke query_apply() with ns and set as None
        """
        with pytest.raises(TypeError) as typeError:
            self.as_connection.query_apply(None, None, self.age_range_pred,
                                           "bin_lua", "mytransform",
                                           ['age', 2])

        assert "query_apply() argument 1 must be str" in str(typeError.value)

    def test_query_apply_with_module_function_none(self):
        """
        Invoke query_apply() with None module function
        """

        with pytest.raises(e.ParamError) as err_info:
            self.as_connection.query_apply(
                "test", "demo", self.age_range_pred, None, None, ['age', 2])

        err_code = err_info.value.code
        assert err_code == AerospikeStatus.AEROSPIKE_ERR_PARAM

    def test_query_apply_with_extra_argument(self):
        """
        Invoke query_apply() with extra argument
        """
        policy = {'timeout': 1000}
        with pytest.raises(TypeError) as typeError:
            self.as_connection.query_apply(
                "test", "demo", self.age_range_pred, "bin_lua",
                "mytransform_incorrect", ['age', 2], policy, "")

        assert "query_apply() takes at most 7 arguments (8 given)" in str(
            typeError.value)

    def test_query_apply_with_argument_is_string(self):
        """
        Invoke query_apply() with arguments as string
        """
        with pytest.raises(e.ParamError) as err_info:
            self.as_connection.query_apply("test", "demo", self.age_range_pred,
                                           "bin_lua", "mytransform", "")

        err_code = err_info.value.code
        assert err_code == AerospikeStatus.AEROSPIKE_ERR_PARAM

    def test_query_apply_with_argument_is_none(self):
        """
        Invoke query_apply() with arguments as None
        """
        with pytest.raises(e.ParamError) as err_info:
            self.as_connection.query_apply(
                "test", "demo", self.age_range_pred,
                "bin_lua", "mytransform", None)

        err_code = err_info.value.code
        assert err_code == AerospikeStatus.AEROSPIKE_ERR_PARAM

    def test_query_apply_with_extra_call_to_lua(self):
        """
        Invoke query_apply() with extra call to lua
        test that passing an extra argument to a udf does
        not cause the function to fail
        """
        query_id = self.as_connection.query_apply(
            "test", "demo", self.age_range_pred,
            "bin_lua", "mytransform", ['age', 2, 3])

        # time.sleep(2)

        time.sleep(0.1)
        while True:
            response = self.as_connection.job_info(
                query_id, aerospike.JOB_QUERY)
            if response['status'] != aerospike.JOB_STATUS_INPROGRESS:
                break
            time.sleep(0.1)

        for i in range(1, 5):
            key = ('test', 'demo', i)
            _, _, bins = self.as_connection.get(key)
            assert bins['age'] == i + 2

    def test_query_apply_with_extra_parameter_in_lua(self):
        """
        Invoke query_apply() with a missing argument
        to a lua function does not cause an error
        """
        query_id = self.as_connection.query_apply(
            "test", "demo", self.age_range_pred,
            "bin_lua", "mytransformextra", ['age', 2])

        # time.sleep(2)

        time.sleep(0.1)
        while True:
            response = self.as_connection.job_info(
                query_id, aerospike.JOB_QUERY)
            if response['status'] != aerospike.JOB_STATUS_INPROGRESS:
                break
            time.sleep(0.1)

        for i in range(1, 5):
            key = ('test', 'demo', i)
            _, _, bins = self.as_connection.get(key)
            assert bins['age'] == i + 2

    def test_query_apply_with_less_parameter_in_lua(self):
        """
        Invoke query_apply() with less parameter in lua
        this verifies that passing 3 arguments to a lua
        function which expects 2, will cause an undefined int
        to be treated as 0, and not crash.
        """
        query_id = self.as_connection.query_apply(
            "test", "demo", self.age_range_pred,
            "bin_lua", "mytransformless", ['age', 2])

        # time.sleep(2)

        time.sleep(0.1)
        while True:
            response = self.as_connection.job_info(
                query_id, aerospike.JOB_QUERY)
            if response['status'] != aerospike.JOB_STATUS_INPROGRESS:
                break
            time.sleep(0.1)

        for i in range(1, 5):
            key = ('test', 'demo', i)
            _, _, bins = self.as_connection.get(key)
            if bins['age'] != i:
                assert True is False
            else:
                assert True is True

    def test_query_apply_unicode_input(self):
        """
        Invoke query_apply() with unicode udf
        """
        query_id = self.as_connection.query_apply(
            "test", "demo", self.age_range_pred, u"bin_lua",
            u"mytransform", ['age', 2])

        time.sleep(0.1)
        while True:
            response = self.as_connection.job_info(
                query_id, aerospike.JOB_QUERY)
            if response['status'] != aerospike.JOB_STATUS_INPROGRESS:
                break
            time.sleep(0.1)

        for i in range(1, 5):
            key = ('test', 'demo', i)
            _, _, bins = self.as_connection.get(key)
            assert bins['age'] == i + 2

    def test_query_apply_with_correct_parameters_without_connection(self):
        """
        Invoke query_apply() with correct parameters without connection
        """
        config = {'hosts': [('127.0.0.1', 3000)]}
        client1 = aerospike.client(config)

        with pytest.raises(e.ClusterError) as err_info:
            client1.query_apply("test", "demo", self.age_range_pred,
                                "bin_lua", "mytransform", ['age', 2])

        err_code = err_info.value.code

        assert err_code == AerospikeStatus.AEROSPIKE_CLUSTER_ERROR

    @pytest.mark.skip(reason="This isn't a test for query_apply," +
                             " but for job_info")
    def test_job_info_with_incorrect_module_type(self):
        """
        Invoke query_apply() with incorrect module type
        """
        query_id = self.as_connection.query_apply(
            "test", "demo", self.age_range_pred, "bin_lua",
            "mytransform", ['age', 2])

        with pytest.raises(e.ParamError) as err_info:
            time.sleep(0.1)
            self.as_connection.job_info(query_id, "aggregate")
        err_code = err_info.value.code

        assert err_code == AerospikeStatus.AEROSPIKE_ERR_PARAM

    # @pytest.mark.xfail(reason="Passing an invalid predicate currently works " +
    #                           " or raises a System Error")
    @pytest.mark.parametrize(
        "predicate",
        (
            (),  # Raises system error
            None,
            ('age'),
            ('age', 1),
            ('age', 1, 5),
            (1, 1, 'bin')  # start of a valid predicate
        )
    )
    def test_invalid_predicate(self, predicate):

        with pytest.raises(e.ParamError) as err_info:
            query_id = self.as_connection.query_apply(
                "test", "demo", predicate, "bin_lua",
                "mytransform", ['age', 2])

        err_code = err_info.value.code
        assert err_code == AerospikeStatus.AEROSPIKE_ERR_PARAM
<|endoftext|>


<commit_before>import logging
import pkgutil
import importlib
from os.path import dirname, basename
from fnmatch import fnmatch

def load_image_providers(filter_pattern):
    """
    Loads image providing modules in submodules of `sections`.
    Said image providers are declared in an `IMAGES` list variable on the
    submodule of `sections`
    """

    logging.info('Loading packages')

    packages = pkgutil.walk_packages(
        [dirname(__file__)],
        prefix='saau.',
        onerror=logging.error
    )

    packages = [
        loader.find_module(name).load_module()
        for loader, name, _ in packages
        if name.startswith('saau.sections.')
    ]

    top_level = [
        package
        for package in packages
        if package.__name__.count('.') == 2
        and basename(package.__file__) == '__init__.py'
    ]

    if not filter_pattern:
        matcher = lambda _: True
    else:
        logging.info(
            "Filtering image providers by \"%s\"",
            filter_pattern
        )
        matcher = lambda prov: fnmatch(prov, filter_pattern)

    image_providers = [
        (package,) + tuple(image_provider.split('.'))
        for package in top_level
        for image_provider in package.IMAGES
        if matcher(image_provider)
    ]

    logging.info('Loading image providers')

    yield from load_classes(load_submodules(image_providers))

def load_classes(modules):
    for module, classname in modules:
        try:
            yield getattr(module, classname)
        except AttributeError:
            logging.error(
                "Couldn't load class \"%s\"",
                module.__name__ + '.' + classname
            )

def load_submodules(image_providers):
    for package, submodule, classname in image_providers:
        try:
            yield (
                importlib.import_module('.' + submodule, package.__package__),
                classname
            )
        except ImportError:
            logging.error(
                "Couldn't load module \"%s\"",
                package.__package__ + '.' + submodule
            )
<commit_msg>Generalize load_image_providers<commit_after>import logging
import pkgutil
import importlib
from os.path import dirname, basename
from fnmatch import fnmatch

def load_image_providers(filter_pattern):
    """
    Loads image providing modules in submodules of `sections`.
    Said image providers are declared in an `IMAGES` list variable on the
    submodule of `sections`
    """
    return load_providers(filter_pattern, 'IMAGES', 'saau.sections.')

def load_providers(filter_pattern, attr_name, package_name):
    logging.info('Loading packages')

    packages = pkgutil.walk_packages(
        [dirname(__file__)],
        prefix='saau.',
        onerror=logging.error
    )

    packages = [
        loader.find_module(name).load_module()
        for loader, name, _ in packages
        if name.startswith(package_name)
    ]

    top_level = [
        package
        for package in packages
        if package.__name__.count('.') == 2
        and basename(package.__file__) == '__init__.py'
    ]

    if not filter_pattern:
        matcher = lambda _: True
    else:
        logging.info(
            "Filtering providers by \"%s\"",
            filter_pattern
        )
        matcher = lambda prov: fnmatch(prov, filter_pattern)

    image_providers = [
        (package,) + tuple(image_provider.split('.'))
        for package in top_level
        for image_provider in getattr(package, attr_name)
        if matcher(image_provider)
    ]

    logging.info('Loading image providers')

    yield from load_classes(load_submodules(image_providers))

def load_classes(modules):
    for module, classname in modules:
        try:
            yield getattr(module, classname)
        except AttributeError:
            logging.error(
                "Couldn't load class \"%s\"",
                module.__name__ + '.' + classname
            )

def load_submodules(image_providers):
    for package, submodule, classname in image_providers:
        try:
            yield (
                importlib.import_module('.' + submodule, package.__package__),
                classname
            )
        except ImportError:
            logging.error(
                "Couldn't load module \"%s\"",
                package.__package__ + '.' + submodule
            )
<|endoftext|>