<commit_before>"""Tests Let's Encrypt plugins against different server configurations."""
import argparse
import filecmp
import functools
import logging
import os
import shutil
import tempfile
import time

import OpenSSL

from acme import challenges
from acme import crypto_util
from acme import messages
from letsencrypt import achallenges
from letsencrypt import errors as le_errors
from letsencrypt import validator
from letsencrypt.tests import acme_util

from letsencrypt_compatibility_test import errors
from letsencrypt_compatibility_test import util
from letsencrypt_compatibility_test.configurators.apache import apache24

DESCRIPTION = """
Tests Let's Encrypt plugins against different server configuratons. It is
assumed that Docker is already installed. If no test types is specified, all
tests that the plugin supports are performed.

"""

PLUGINS = {"apache" : apache24.Proxy}

logger = logging.getLogger(__name__)

def test_authenticator(plugin, config, temp_dir):
    """Tests authenticator, returning True if the tests are successful"""
    backup = _create_backup(config, temp_dir)

    achalls = _create_achalls(plugin)
    if not achalls:
        logger.error("The plugin and this program support no common "
                     "challenge types")
        return False

    try:
        responses = plugin.perform(achalls)
    except le_errors.Error as error:
        logger.error("Performing challenges on %s caused an error:", config)
        logger.exception(error)
        return False

    success = True
    for i in xrange(len(responses)):
        if not responses[i]:
            logger.error(
                "Plugin failed to complete %s for %s in %s",
                type(achalls[i]), achalls[i].domain, config)
            success = False
        elif isinstance(responses[i], challenges.DVSNIResponse):
            verify = functools.partial(responses[i].simple_verify, achalls[i],
                                       achalls[i].domain,
                                       util.JWK.public_key(),
                                       host="127.0.0.1",
                                       port=plugin.https_port)
            if _try_until_true(verify):
                logger.info(
                    "DVSNI verification for %s succeeded", achalls[i].domain)
            else:
                logger.error(
                    "DVSNI verification for %s in %s failed",
                    achalls[i].domain, config)
                success = False

    if success:
        try:
            plugin.cleanup(achalls)
        except le_errors.Error as error:
            logger.error("Challenge cleanup for %s caused an error:", config)
            logger.exception(error)
            success = False

        if _dirs_are_unequal(config, backup):
            logger.error("Challenge cleanup failed for %s", config)
            return False
        else:
            logger.info("Challenge cleanup succeeded")

    return success

def _create_achalls(plugin):
    """Returns a list of annotated challenges to test on plugin"""
    achalls = list()
    names = plugin.get_testable_domain_names()
    for domain in names:
        prefs = plugin.get_chall_pref(domain)
        for chall_type in prefs:
            if chall_type == challenges.DVSNI:
                chall = challenges.DVSNI(
                    token=os.urandom(challenges.DVSNI.TOKEN_SIZE))
                challb = acme_util.chall_to_challb(
                    chall, messages.STATUS_PENDING)
                achall = achallenges.DVSNI(
                    challb=challb, domain=domain, account_key=util.JWK)
                achalls.append(achall)

    return achalls

def test_installer(args, plugin, config, temp_dir):
    """Tests plugin as an installer"""
    backup = _create_backup(config, temp_dir)

    names_match = plugin.get_all_names() == plugin.get_all_names_answer()
    if names_match:
        logger.info("get_all_names test succeeded")
    else:
        logger.error("get_all_names test failed for config %s", config)

    domains = list(plugin.get_testable_domain_names())
    success = test_deploy_cert(plugin, temp_dir, domains)

    if success and args.enhance:
        success = test_enhancements(plugin, domains)

    good_rollback = test_rollback(plugin, config, backup)
    return names_match and success and good_rollback

def test_deploy_cert(plugin, temp_dir, domains):
    """Tests deploy_cert returning True if the tests are successful"""
    cert = crypto_util.gen_ss_cert(util.KEY, domains)
    cert_path = os.path.join(temp_dir, "cert.pem")
    with open(cert_path, "w") as f:
        f.write(OpenSSL.crypto.dump_certificate(
            OpenSSL.crypto.FILETYPE_PEM, cert))

    for domain in domains:
        try:
            plugin.deploy_cert(domain, cert_path, util.KEY_PATH)
        except le_errors.Error as error:
            logger.error("Plugin failed to deploy ceritificate for %s:", domain)
            logger.exception(error)
            return False

    if not _save_and_restart(plugin, "deployed"):
        return False

    success = True
    for domain in domains:
        verify = functools.partial(validator.Validator().certificate, cert,
                                   domain, "127.0.0.1", plugin.https_port)
        if not _try_until_true(verify):
            logger.error("Could not verify certificate for domain %s", domain)
            success = False

    if success:
        logger.info("HTTPS validation succeeded")

    return success

def test_enhancements(plugin, domains):
    """Tests supported enhancements returning True if successful"""
    supported = plugin.supported_enhancements()

    if "redirect" not in supported:
        logger.error("The plugin and this program support no common "
                     "enhancements")
        return False

    for domain in domains:
        try:
            plugin.enhance(domain, "redirect")
        except le_errors.PluginError as error:
            # Don't immediately fail because a redirect may already be enabled
            logger.warning("Plugin failed to enable redirect for %s:", domain)
            logger.warning("%s", error)
        except le_errors.Error as error:
            logger.error("An error occurred while enabling redirect for %s:",
                         domain)
            logger.exception(error)

    if not _save_and_restart(plugin, "enhanced"):
        return False

    success = True
    for domain in domains:
        verify = functools.partial(validator.Validator().redirect, "localhost",
                                   plugin.http_port, headers={"Host" : domain})
        if not _try_until_true(verify):
            logger.error("Improper redirect for domain %s", domain)
            success = False

    if success:
        logger.info("Enhancments test succeeded")

    return success

def _try_until_true(func, max_tries=5, sleep_time=0.5):
    """Calls func up to max_tries times until it returns True"""
    for _ in xrange(0, max_tries):
        if func():
            return True
        else:
            time.sleep(sleep_time)

    return False

def _save_and_restart(plugin, title=None):
    """Saves and restart the plugin, returning True if no errors occurred"""
    try:
        plugin.save(title)
        plugin.restart()
        return True
    except le_errors.Error as error:
        logger.error("Plugin failed to save and restart server:")
        logger.exception(error)
        return False

def test_rollback(plugin, config, backup):
    """Tests the rollback checkpoints function"""
    try:
        plugin.rollback_checkpoints(1337)
    except le_errors.Error as error:
        logger.error("Plugin raised an exception during rollback:")
        logger.exception(error)
        return False

    if _dirs_are_unequal(config, backup):
        logger.error("Rollback failed for config `%s`", config)
        return False
    else:
        logger.info("Rollback succeeded")
        return True

def _create_backup(config, temp_dir):
    """Creates a backup of config in temp_dir"""
    backup = os.path.join(temp_dir, "backup")
    shutil.rmtree(backup, ignore_errors=True)
    shutil.copytree(config, backup, symlinks=True)

    print backup
    return backup

def _dirs_are_unequal(dir1, dir2):
    """Returns True if dir1 and dir2 are unequal"""
    dircmps = [filecmp.dircmp(dir1, dir2)]
    while len(dircmps):
        dircmp = dircmps.pop()
        if dircmp.left_only or dircmp.right_only:
            logger.error("The following files and directories are only "
                         "present in one directory")
            if dircmp.left_only:
                logger.error(dircmp.left_only)
            else:
                logger.error(dircmp.right_only)
            return True
        elif dircmp.common_funny or dircmp.funny_files:
            logger.error("The following files and directories could not be "
                         "compared:")
            if dircmp.common_funny:
                logger.error(dircmp.common_funny)
            else:
                logger.error(dircmp.funny_files)
            return True
        elif dircmp.diff_files:
            logger.error("The following files differ:")
            logger.error(dircmp.diff_files)
            return True

        for subdir in dircmp.subdirs.itervalues():
            dircmps.append(subdir)

    return False

def get_args():
    """Returns parsed command line arguments."""
    parser = argparse.ArgumentParser(
        description=DESCRIPTION,
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    group = parser.add_argument_group("general")
    group.add_argument(
        "-c", "--configs", default="configs.tar.gz",
        help="a directory or tarball containing server configurations")
    group.add_argument(
        "-p", "--plugin", default="apache", help="the plugin to be tested")
    group.add_argument(
        "-v", "--verbose", dest="verbose_count", action="count",
        default=0, help="you know how to use this")
    group.add_argument(
        "-a", "--auth", action="store_true",
        help="tests the challenges the plugin supports")
    group.add_argument(
        "-i", "--install", action="store_true",
        help="tests the plugin as an installer")
    group.add_argument(
        "-e", "--enhance", action="store_true", help="tests the enhancements "
        "the plugin supports (implicitly includes installer tests)")

    for plugin in PLUGINS.itervalues():
        plugin.add_parser_arguments(parser)

    args = parser.parse_args()
    if args.enhance:
        args.install = True
    elif not (args.auth or args.install):
        args.auth = args.install = args.enhance = True

    return args

def setup_logging(args):
    """Prepares logging for the program"""
    handler = logging.StreamHandler()

    root_logger = logging.getLogger()
    root_logger.setLevel(logging.ERROR - args.verbose_count * 10)
    root_logger.addHandler(handler)

def main():
    """Main test script execution."""
    args = get_args()
    setup_logging(args)

    if args.plugin not in PLUGINS:
        raise errors.Error("Unknown plugin {0}".format(args.plugin))

    temp_dir = tempfile.mkdtemp()
    plugin = PLUGINS[args.plugin](args)
    try:
        plugin.execute_in_docker("mkdir -p /var/log/apache2")
        while plugin.has_more_configs():
            success = True

            try:
                config = plugin.load_config()
                logger.info("Loaded configuration: %s", config)
                if args.auth:
                    success = test_authenticator(plugin, config, temp_dir)
                if success and args.install:
                    success = test_installer(args, plugin, config, temp_dir)
            except errors.Error as error:
                logger.error("Tests on %s raised:", config)
                logger.exception(error)
                success = False

            if success:
                logger.info("All tests on %s succeeded", config)
            else:
                logger.error("Tests on %s failed", config)
    finally:
        plugin.cleanup_from_tests()

if __name__ == "__main__":
    main()
<commit_msg>Removed stray debugging statement<commit_after>"""Tests Let's Encrypt plugins against different server configurations."""
import argparse
import filecmp
import functools
import logging
import os
import shutil
import tempfile
import time

import OpenSSL

from acme import challenges
from acme import crypto_util
from acme import messages
from letsencrypt import achallenges
from letsencrypt import errors as le_errors
from letsencrypt import validator
from letsencrypt.tests import acme_util

from letsencrypt_compatibility_test import errors
from letsencrypt_compatibility_test import util
from letsencrypt_compatibility_test.configurators.apache import apache24

DESCRIPTION = """
Tests Let's Encrypt plugins against different server configuratons. It is
assumed that Docker is already installed. If no test types is specified, all
tests that the plugin supports are performed.

"""

PLUGINS = {"apache" : apache24.Proxy}

logger = logging.getLogger(__name__)

def test_authenticator(plugin, config, temp_dir):
    """Tests authenticator, returning True if the tests are successful"""
    backup = _create_backup(config, temp_dir)

    achalls = _create_achalls(plugin)
    if not achalls:
        logger.error("The plugin and this program support no common "
                     "challenge types")
        return False

    try:
        responses = plugin.perform(achalls)
    except le_errors.Error as error:
        logger.error("Performing challenges on %s caused an error:", config)
        logger.exception(error)
        return False

    success = True
    for i in xrange(len(responses)):
        if not responses[i]:
            logger.error(
                "Plugin failed to complete %s for %s in %s",
                type(achalls[i]), achalls[i].domain, config)
            success = False
        elif isinstance(responses[i], challenges.DVSNIResponse):
            verify = functools.partial(responses[i].simple_verify, achalls[i],
                                       achalls[i].domain,
                                       util.JWK.public_key(),
                                       host="127.0.0.1",
                                       port=plugin.https_port)
            if _try_until_true(verify):
                logger.info(
                    "DVSNI verification for %s succeeded", achalls[i].domain)
            else:
                logger.error(
                    "DVSNI verification for %s in %s failed",
                    achalls[i].domain, config)
                success = False

    if success:
        try:
            plugin.cleanup(achalls)
        except le_errors.Error as error:
            logger.error("Challenge cleanup for %s caused an error:", config)
            logger.exception(error)
            success = False

        if _dirs_are_unequal(config, backup):
            logger.error("Challenge cleanup failed for %s", config)
            return False
        else:
            logger.info("Challenge cleanup succeeded")

    return success

def _create_achalls(plugin):
    """Returns a list of annotated challenges to test on plugin"""
    achalls = list()
    names = plugin.get_testable_domain_names()
    for domain in names:
        prefs = plugin.get_chall_pref(domain)
        for chall_type in prefs:
            if chall_type == challenges.DVSNI:
                chall = challenges.DVSNI(
                    token=os.urandom(challenges.DVSNI.TOKEN_SIZE))
                challb = acme_util.chall_to_challb(
                    chall, messages.STATUS_PENDING)
                achall = achallenges.DVSNI(
                    challb=challb, domain=domain, account_key=util.JWK)
                achalls.append(achall)

    return achalls

def test_installer(args, plugin, config, temp_dir):
    """Tests plugin as an installer"""
    backup = _create_backup(config, temp_dir)

    names_match = plugin.get_all_names() == plugin.get_all_names_answer()
    if names_match:
        logger.info("get_all_names test succeeded")
    else:
        logger.error("get_all_names test failed for config %s", config)

    domains = list(plugin.get_testable_domain_names())
    success = test_deploy_cert(plugin, temp_dir, domains)

    if success and args.enhance:
        success = test_enhancements(plugin, domains)

    good_rollback = test_rollback(plugin, config, backup)
    return names_match and success and good_rollback

def test_deploy_cert(plugin, temp_dir, domains):
    """Tests deploy_cert returning True if the tests are successful"""
    cert = crypto_util.gen_ss_cert(util.KEY, domains)
    cert_path = os.path.join(temp_dir, "cert.pem")
    with open(cert_path, "w") as f:
        f.write(OpenSSL.crypto.dump_certificate(
            OpenSSL.crypto.FILETYPE_PEM, cert))

    for domain in domains:
        try:
            plugin.deploy_cert(domain, cert_path, util.KEY_PATH)
        except le_errors.Error as error:
            logger.error("Plugin failed to deploy ceritificate for %s:", domain)
            logger.exception(error)
            return False

    if not _save_and_restart(plugin, "deployed"):
        return False

    success = True
    for domain in domains:
        verify = functools.partial(validator.Validator().certificate, cert,
                                   domain, "127.0.0.1", plugin.https_port)
        if not _try_until_true(verify):
            logger.error("Could not verify certificate for domain %s", domain)
            success = False

    if success:
        logger.info("HTTPS validation succeeded")

    return success

def test_enhancements(plugin, domains):
    """Tests supported enhancements returning True if successful"""
    supported = plugin.supported_enhancements()

    if "redirect" not in supported:
        logger.error("The plugin and this program support no common "
                     "enhancements")
        return False

    for domain in domains:
        try:
            plugin.enhance(domain, "redirect")
        except le_errors.PluginError as error:
            # Don't immediately fail because a redirect may already be enabled
            logger.warning("Plugin failed to enable redirect for %s:", domain)
            logger.warning("%s", error)
        except le_errors.Error as error:
            logger.error("An error occurred while enabling redirect for %s:",
                         domain)
            logger.exception(error)

    if not _save_and_restart(plugin, "enhanced"):
        return False

    success = True
    for domain in domains:
        verify = functools.partial(validator.Validator().redirect, "localhost",
                                   plugin.http_port, headers={"Host" : domain})
        if not _try_until_true(verify):
            logger.error("Improper redirect for domain %s", domain)
            success = False

    if success:
        logger.info("Enhancments test succeeded")

    return success

def _try_until_true(func, max_tries=5, sleep_time=0.5):
    """Calls func up to max_tries times until it returns True"""
    for _ in xrange(0, max_tries):
        if func():
            return True
        else:
            time.sleep(sleep_time)

    return False

def _save_and_restart(plugin, title=None):
    """Saves and restart the plugin, returning True if no errors occurred"""
    try:
        plugin.save(title)
        plugin.restart()
        return True
    except le_errors.Error as error:
        logger.error("Plugin failed to save and restart server:")
        logger.exception(error)
        return False

def test_rollback(plugin, config, backup):
    """Tests the rollback checkpoints function"""
    try:
        plugin.rollback_checkpoints(1337)
    except le_errors.Error as error:
        logger.error("Plugin raised an exception during rollback:")
        logger.exception(error)
        return False

    if _dirs_are_unequal(config, backup):
        logger.error("Rollback failed for config `%s`", config)
        return False
    else:
        logger.info("Rollback succeeded")
        return True

def _create_backup(config, temp_dir):
    """Creates a backup of config in temp_dir"""
    backup = os.path.join(temp_dir, "backup")
    shutil.rmtree(backup, ignore_errors=True)
    shutil.copytree(config, backup, symlinks=True)

    return backup

def _dirs_are_unequal(dir1, dir2):
    """Returns True if dir1 and dir2 are unequal"""
    dircmps = [filecmp.dircmp(dir1, dir2)]
    while len(dircmps):
        dircmp = dircmps.pop()
        if dircmp.left_only or dircmp.right_only:
            logger.error("The following files and directories are only "
                         "present in one directory")
            if dircmp.left_only:
                logger.error(dircmp.left_only)
            else:
                logger.error(dircmp.right_only)
            return True
        elif dircmp.common_funny or dircmp.funny_files:
            logger.error("The following files and directories could not be "
                         "compared:")
            if dircmp.common_funny:
                logger.error(dircmp.common_funny)
            else:
                logger.error(dircmp.funny_files)
            return True
        elif dircmp.diff_files:
            logger.error("The following files differ:")
            logger.error(dircmp.diff_files)
            return True

        for subdir in dircmp.subdirs.itervalues():
            dircmps.append(subdir)

    return False

def get_args():
    """Returns parsed command line arguments."""
    parser = argparse.ArgumentParser(
        description=DESCRIPTION,
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    group = parser.add_argument_group("general")
    group.add_argument(
        "-c", "--configs", default="configs.tar.gz",
        help="a directory or tarball containing server configurations")
    group.add_argument(
        "-p", "--plugin", default="apache", help="the plugin to be tested")
    group.add_argument(
        "-v", "--verbose", dest="verbose_count", action="count",
        default=0, help="you know how to use this")
    group.add_argument(
        "-a", "--auth", action="store_true",
        help="tests the challenges the plugin supports")
    group.add_argument(
        "-i", "--install", action="store_true",
        help="tests the plugin as an installer")
    group.add_argument(
        "-e", "--enhance", action="store_true", help="tests the enhancements "
        "the plugin supports (implicitly includes installer tests)")

    for plugin in PLUGINS.itervalues():
        plugin.add_parser_arguments(parser)

    args = parser.parse_args()
    if args.enhance:
        args.install = True
    elif not (args.auth or args.install):
        args.auth = args.install = args.enhance = True

    return args

def setup_logging(args):
    """Prepares logging for the program"""
    handler = logging.StreamHandler()

    root_logger = logging.getLogger()
    root_logger.setLevel(logging.ERROR - args.verbose_count * 10)
    root_logger.addHandler(handler)

def main():
    """Main test script execution."""
    args = get_args()
    setup_logging(args)

    if args.plugin not in PLUGINS:
        raise errors.Error("Unknown plugin {0}".format(args.plugin))

    temp_dir = tempfile.mkdtemp()
    plugin = PLUGINS[args.plugin](args)
    try:
        plugin.execute_in_docker("mkdir -p /var/log/apache2")
        while plugin.has_more_configs():
            success = True

            try:
                config = plugin.load_config()
                logger.info("Loaded configuration: %s", config)
                if args.auth:
                    success = test_authenticator(plugin, config, temp_dir)
                if success and args.install:
                    success = test_installer(args, plugin, config, temp_dir)
            except errors.Error as error:
                logger.error("Tests on %s raised:", config)
                logger.exception(error)
                success = False

            if success:
                logger.info("All tests on %s succeeded", config)
            else:
                logger.error("Tests on %s failed", config)
    finally:
        plugin.cleanup_from_tests()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals, print_function
import sys
from parglare.termui import s_attention as _a

if sys.version < '3':
    text = unicode  # NOQA
else:
    text = str

class Location(object):
    """
    Represents a location (point or span) of the object in the source code.

    Args:
    context(Context): Parsing context used to populate this object.

    Attributes:
    input_str: The input string (from context) being parsed.
    file_name(str): The name (path) to the file this location refers to.
    start_position(int): The position of the span if applicable
    end_position(int): The end of the span if applicable.
    line, column (int): The line/column calculated from the position start and
        input_str.
    line_end, column_end (int): The line/column calculated from the position
        end and input_str.
    """

    __slots__ = ['context', 'file_name',
                 '_line', '_column',
                 '_line_end', '_column_end']

    def __init__(self, context=None, file_name=None):

        self.context = context
        self.file_name = file_name

        # Evaluate this only when string representation is needed.
        # E.g. during error reporting
        self._line = None
        self._column = None

        self._line_end = None
        self._column_end = None

    @property
    def line(self):
        if self._line is None:
            self.evaluate_line_col()
        return self._line

    @property
    def line_end(self):
        if self._line_end is None:
            self.evaluate_line_col_end()
        return self._line_end

    @property
    def column(self):
        if self._column is None:
            self.evaluate_line_col()
        return self._column

    @property
    def column_end(self):
        if self._column_end is None:
            self.evaluate_line_col_end()
        return self._column_end

    def evaluate_line_col(self):
        context = self.context
        self._line, self._column = pos_to_line_col(
            context.input_str, context.start_position)

    def evaluate_line_col_end(self):
        context = self.context
        if hasattr(context, 'end_position') \
                and context.end_position:
            self._line_end, self._column_end = \
                pos_to_line_col(context.input_str, context.end_position)

    def __getattr__(self, name):
        if self.context is not None:
            return getattr(self.context, name)
        else:
            raise AttributeError(name)

    def __str__(self):
        if self.context is None:
            line, column = None, None
        else:
            line, column = self.line, self.column
        context = self.context
        if line is not None:
            return ('{}{}:{}:"{}"'
                    .format("{}:".format(self.file_name)
                            if self.file_name else "",
                            line, column,
                            position_context(context.input_str,
                                             context.start_position)))
        elif self.file_name:
            return _a(self.file_name)
        else:
            return "<Unknown location>"

    def __repr__(self):
        return str(self)

def position_context(input_str, position):
    """
    Returns position context string.
    """
    start = max(position-10, 0)
    c = text(input_str[start:position]) + _a(" **> ") \
        + text(input_str[position:position+10])
    return replace_newlines(c)

def replace_newlines(in_str):
    try:
        return in_str.replace("\n", "\\n")
    except AttributeError:
        return in_str

def load_python_module(mod_name, mod_path):
    """
    Loads Python module from an arbitrary location.
    See https://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path  # noqa
    """
    if sys.version_info >= (3, 5):
        import importlib.util
        spec = importlib.util.spec_from_file_location(
            mod_name, mod_path)
        module = importlib.util.module_from_spec(spec)
        spec.loader.exec_module(module)
    elif sys.version_info >= (3, 3):
        from importlib.machinery import SourceFileLoader
        module = SourceFileLoader(
            mod_name, mod_path).load_module()
    else:
        import imp
        module = imp.load_source(mod_name, mod_path)

    return module

def get_collector():
    """
    Produces action/recognizers collector/decorator that will collect all
    decorated objects under dictionary attribute `all`.
    """
    all = {}

    class Collector(object):
        def __call__(self, name_or_f):
            """
            If called with action/recognizer name return decorator.
            If called over function apply decorator.
            """
            is_name = type(name_or_f) in [str, text]

            def decorator(f):
                if is_name:
                    name = name_or_f
                else:
                    name = f.__name__
                objects = all.get(name, None)
                if objects:
                    if type(objects) is list:
                        objects.append(f)
                    else:
                        all[name] = [objects, f]
                else:
                    all[name] = f
                return f
            if is_name:
                return decorator
            else:
                return decorator(name_or_f)

    objects = Collector()
    objects.all = all
    return objects

def pos_to_line_col(input_str, position):
    """
    Returns position in the (line,column) form.
    """

    if position is None:
        return None, None

    if type(input_str) is not text:
        # If we are not parsing string
        return 1, position

    line = 1
    old_pos = 0
    try:
        cur_pos = input_str.index("\n")
        while cur_pos < position:
            line += 1
            old_pos = cur_pos + 1
            cur_pos = input_str.index("\n", cur_pos + 1)
    except ValueError:
        pass

    return line, position - old_pos

class ErrorContext(object):
    """
    Context for errors.  Errors are constructed from parsing heads and are
    represented as location span.  Initially, the start and end of the span are
    set to the position where the error is found but end of the span can be
    moved forward during error recovery.
    """

    __slots__ = ['input_str', 'file_name', 'start_position', 'end_position']

    def __init__(self, context):
        self.start_position = self.end_position = context.position
        self.input_str = context.input_str
        self.file_name = context.file_name
<commit_msg>Fix file name in error context<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals, print_function
import sys
from parglare.termui import s_attention as _a

if sys.version < '3':
    text = unicode  # NOQA
else:
    text = str

class Location(object):
    """
    Represents a location (point or span) of the object in the source code.

    Args:
    context(Context): Parsing context used to populate this object.

    Attributes:
    input_str: The input string (from context) being parsed.
    file_name(str): The name (path) to the file this location refers to.
    start_position(int): The position of the span if applicable
    end_position(int): The end of the span if applicable.
    line, column (int): The line/column calculated from the position start and
        input_str.
    line_end, column_end (int): The line/column calculated from the position
        end and input_str.
    """

    __slots__ = ['context', 'file_name',
                 '_line', '_column',
                 '_line_end', '_column_end']

    def __init__(self, context=None, file_name=None):

        self.context = context
        self.file_name = file_name or context.file_name

        # Evaluate this only when string representation is needed.
        # E.g. during error reporting
        self._line = None
        self._column = None

        self._line_end = None
        self._column_end = None

    @property
    def line(self):
        if self._line is None:
            self.evaluate_line_col()
        return self._line

    @property
    def line_end(self):
        if self._line_end is None:
            self.evaluate_line_col_end()
        return self._line_end

    @property
    def column(self):
        if self._column is None:
            self.evaluate_line_col()
        return self._column

    @property
    def column_end(self):
        if self._column_end is None:
            self.evaluate_line_col_end()
        return self._column_end

    def evaluate_line_col(self):
        context = self.context
        self._line, self._column = pos_to_line_col(
            context.input_str, context.start_position)

    def evaluate_line_col_end(self):
        context = self.context
        if hasattr(context, 'end_position') \
                and context.end_position:
            self._line_end, self._column_end = \
                pos_to_line_col(context.input_str, context.end_position)

    def __getattr__(self, name):
        if self.context is not None:
            return getattr(self.context, name)
        else:
            raise AttributeError(name)

    def __str__(self):
        if self.context is None:
            line, column = None, None
        else:
            line, column = self.line, self.column
        context = self.context
        if line is not None:
            return ('{}{}:{}:"{}"'
                    .format("{}:".format(self.file_name)
                            if self.file_name else "",
                            line, column,
                            position_context(context.input_str,
                                             context.start_position)))
        elif self.file_name:
            return _a(self.file_name)
        else:
            return "<Unknown location>"

    def __repr__(self):
        return str(self)

def position_context(input_str, position):
    """
    Returns position context string.
    """
    start = max(position-10, 0)
    c = text(input_str[start:position]) + _a(" **> ") \
        + text(input_str[position:position+10])
    return replace_newlines(c)

def replace_newlines(in_str):
    try:
        return in_str.replace("\n", "\\n")
    except AttributeError:
        return in_str

def load_python_module(mod_name, mod_path):
    """
    Loads Python module from an arbitrary location.
    See https://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path  # noqa
    """
    if sys.version_info >= (3, 5):
        import importlib.util
        spec = importlib.util.spec_from_file_location(
            mod_name, mod_path)
        module = importlib.util.module_from_spec(spec)
        spec.loader.exec_module(module)
    elif sys.version_info >= (3, 3):
        from importlib.machinery import SourceFileLoader
        module = SourceFileLoader(
            mod_name, mod_path).load_module()
    else:
        import imp
        module = imp.load_source(mod_name, mod_path)

    return module

def get_collector():
    """
    Produces action/recognizers collector/decorator that will collect all
    decorated objects under dictionary attribute `all`.
    """
    all = {}

    class Collector(object):
        def __call__(self, name_or_f):
            """
            If called with action/recognizer name return decorator.
            If called over function apply decorator.
            """
            is_name = type(name_or_f) in [str, text]

            def decorator(f):
                if is_name:
                    name = name_or_f
                else:
                    name = f.__name__
                objects = all.get(name, None)
                if objects:
                    if type(objects) is list:
                        objects.append(f)
                    else:
                        all[name] = [objects, f]
                else:
                    all[name] = f
                return f
            if is_name:
                return decorator
            else:
                return decorator(name_or_f)

    objects = Collector()
    objects.all = all
    return objects

def pos_to_line_col(input_str, position):
    """
    Returns position in the (line,column) form.
    """

    if position is None:
        return None, None

    if type(input_str) is not text:
        # If we are not parsing string
        return 1, position

    line = 1
    old_pos = 0
    try:
        cur_pos = input_str.index("\n")
        while cur_pos < position:
            line += 1
            old_pos = cur_pos + 1
            cur_pos = input_str.index("\n", cur_pos + 1)
    except ValueError:
        pass

    return line, position - old_pos

class ErrorContext(object):
    """
    Context for errors.  Errors are constructed from parsing heads and are
    represented as location span.  Initially, the start and end of the span are
    set to the position where the error is found but end of the span can be
    moved forward during error recovery.
    """

    __slots__ = ['input_str', 'file_name', 'start_position', 'end_position']

    def __init__(self, context):
        self.start_position = self.end_position = context.position
        self.input_str = context.input_str
        self.file_name = context.file_name
<|endoftext|>


<commit_before>"""
XPath selectors based on lxml
"""

import sys

import six
from lxml import etree, html

from .utils import flatten, iflatten, extract_regex
from .csstranslator import HTMLTranslator, GenericTranslator

class SafeXMLParser(etree.XMLParser):
    def __init__(self, *args, **kwargs):
        kwargs.setdefault('resolve_entities', False)
        super(SafeXMLParser, self).__init__(*args, **kwargs)

_ctgroup = {
    'html': {'_parser': html.HTMLParser,
             '_csstranslator': HTMLTranslator(),
             '_tostring_method': 'html'},
    'xml': {'_parser': SafeXMLParser,
            '_csstranslator': GenericTranslator(),
            '_tostring_method': 'xml'},
}

def _st(st):
    if st is None:
        return 'html'
    elif st in _ctgroup:
        return st
    else:
        raise ValueError('Invalid type: %s' % st)

def create_root_node(text, parser_cls, base_url=None):
    """Create root node for text using given parser class.
    """
    body = text.strip().replace('\x00', '').encode('utf8') or b'<html/>'
    parser = parser_cls(recover=True, encoding='utf8')
    root = etree.fromstring(body, parser=parser, base_url=base_url)
    if root is None:
        root = etree.fromstring(b'<html/>', parser=parser, base_url=base_url)
    return root

class SelectorList(list):
    """
    The :class:`SelectorList` class is a subclass of the builtin ``list``
    class, which provides a few additional methods.
    """

    # __getslice__ is deprecated but `list` builtin implements it only in Py2
    def __getslice__(self, i, j):
        o = super(SelectorList, self).__getslice__(i, j)
        return self.__class__(o)

    def __getitem__(self, pos):
        o = super(SelectorList, self).__getitem__(pos)
        return self.__class__(o) if isinstance(pos, slice) else o

    def __getstate__(self):
        raise TypeError("can't pickle SelectorList objects")

    def xpath(self, xpath, namespaces=None, **kwargs):
        """
        Call the ``.xpath()`` method for each element in this list and return
        their results flattened as another :class:`SelectorList`.

        ``query`` is the same argument as the one in :meth:`Selector.xpath`

        ``namespaces`` is an optional ``prefix: namespace-uri`` mapping (dict)
        for additional prefixes to those registered with ``register_namespace(prefix, uri)``.
        Contrary to ``register_namespace()``, these prefixes are not
        saved for future calls.

        Any additional named arguments can be used to pass values for XPath
        variables in the XPath expression, e.g.::

            selector.xpath('//a[href=$url]', url="http://www.example.com")
        """
        return self.__class__(flatten([x.xpath(xpath, namespaces=namespaces, **kwargs) for x in self]))

    def css(self, query):
        """
        Call the ``.css()`` method for each element in this list and return
        their results flattened as another :class:`SelectorList`.

        ``query`` is the same argument as the one in :meth:`Selector.css`
        """
        return self.__class__(flatten([x.css(query) for x in self]))

    def re(self, regex, replace_entities=True):
        """
        Call the ``.re()`` method for each element in this list and return
        their results flattened, as a list of unicode strings.

        By default, character entity references are replaced by their
        corresponding character (except for ``&`` and ``<``.
        Passing ``replace_entities`` as ``False`` switches off these
        replacements.
        """
        return flatten([x.re(regex, replace_entities=replace_entities) for x in self])

    def re_first(self, regex, default=None, replace_entities=True):
        """
        Call the ``.re()`` method for the first element in this list and
        return the result in an unicode string. If the list is empty or the
        regex doesn't match anything, return the default value (``None`` if
        the argument is not provided).

        By default, character entity references are replaced by their
        corresponding character (except for ``&`` and ``<``.
        Passing ``replace_entities`` as ``False`` switches off these
        replacements.
        """
        for el in iflatten(x.re(regex, replace_entities=replace_entities) for x in self):
            return el
        else:
            return default

    def getall(self):
        """
        Call the ``.get()`` method for each element is this list and return
        their results flattened, as a list of unicode strings.
        """
        return [x.get() for x in self]
    extract = getall

    def get(self, default=None):
        """
        Return the result of ``.get()`` for the first element in this list.
        If the list is empty, return the default value.
        """
        for x in self:
            return x.get()
        else:
            return default
    extract_first = get

    @property
    def attrib(self):
        """Return the attributes dictionary for the first element.
        If the list is empty, return an empty dict.
        """
        for x in self:
            return x.attrib
        else:
            return {}

class Selector(object):
    """
    :class:`Selector` allows you to select parts of an XML or HTML text using CSS
    or XPath expressions and extract data from it.

    ``text`` is a ``unicode`` object in Python 2 or a ``str`` object in Python 3

    ``type`` defines the selector type, it can be ``"html"``, ``"xml"`` or ``None`` (default).
    If ``type`` is ``None``, the selector defaults to ``"html"``.
    """

    __slots__ = ['text', 'namespaces', 'type', '_expr', 'root',
                 '__weakref__', '_parser', '_csstranslator', '_tostring_method']

    _default_type = None
    _default_namespaces = {
        "re": "http://exslt.org/regular-expressions",

        # supported in libxslt:
        # set:difference
        # set:has-same-node
        # set:intersection
        # set:leading
        # set:trailing
        "set": "http://exslt.org/sets"
    }
    _lxml_smart_strings = False
    selectorlist_cls = SelectorList

    def __init__(self, text=None, type=None, namespaces=None, root=None,
                 base_url=None, _expr=None):
        self.type = st = _st(type or self._default_type)
        self._parser = _ctgroup[st]['_parser']
        self._csstranslator = _ctgroup[st]['_csstranslator']
        self._tostring_method = _ctgroup[st]['_tostring_method']

        if text is not None:
            if not isinstance(text, six.text_type):
                raise TypeError("text argument should be of type %s" % six.text_type)
            root = self._get_root(text, base_url)
        elif root is None:
            raise ValueError("Selector needs either text or root argument")

        self.namespaces = dict(self._default_namespaces)
        if namespaces is not None:
            self.namespaces.update(namespaces)
        self.root = root
        self._expr = _expr

    def __getstate__(self):
        raise TypeError("can't pickle Selector objects")

    def _get_root(self, text, base_url=None):
        return create_root_node(text, self._parser, base_url=base_url)

    def xpath(self, query, namespaces=None, **kwargs):
        """
        Find nodes matching the xpath ``query`` and return the result as a
        :class:`SelectorList` instance with all elements flattened. List
        elements implement :class:`Selector` interface too.

        ``query`` is a string containing the XPATH query to apply.

        ``namespaces`` is an optional ``prefix: namespace-uri`` mapping (dict)
        for additional prefixes to those registered with ``register_namespace(prefix, uri)``.
        Contrary to ``register_namespace()``, these prefixes are not
        saved for future calls.

        Any additional named arguments can be used to pass values for XPath
        variables in the XPath expression, e.g.::

            selector.xpath('//a[href=$url]', url="http://www.example.com")
        """
        try:
            xpathev = self.root.xpath
        except AttributeError:
            return self.selectorlist_cls([])

        nsp = dict(self.namespaces)
        if namespaces is not None:
            nsp.update(namespaces)
        try:
            result = xpathev(query, namespaces=nsp,
                             smart_strings=self._lxml_smart_strings,
                             **kwargs)
        except etree.XPathError as exc:
            msg = u"XPath error: %s in %s" % (exc, query)
            msg = msg if six.PY3 else msg.encode('unicode_escape')
            six.reraise(ValueError, ValueError(msg), sys.exc_info()[2])

        if type(result) is not list:
            result = [result]

        result = [self.__class__(root=x, _expr=query,
                                 namespaces=self.namespaces,
                                 type=self.type)
                  for x in result]
        return self.selectorlist_cls(result)

    def css(self, query):
        """
        Apply the given CSS selector and return a :class:`SelectorList` instance.

        ``query`` is a string containing the CSS selector to apply.

        In the background, CSS queries are translated into XPath queries using
        `cssselect`_ library and run ``.xpath()`` method.
        """
        return self.xpath(self._css2xpath(query))

    def _css2xpath(self, query):
        return self._csstranslator.css_to_xpath(query)

    def re(self, regex, replace_entities=True):
        """
        Apply the given regex and return a list of unicode strings with the
        matches.

        ``regex`` can be either a compiled regular expression or a string which
        will be compiled to a regular expression using ``re.compile(regex)``.

        By default, character entity references are replaced by their
        corresponding character (except for ``&`` and ``<``).
        Passing ``replace_entities`` as ``False`` switches off these
        replacements.
        """
        return extract_regex(regex, self.get(), replace_entities=replace_entities)

    def re_first(self, regex, default=None, replace_entities=True):
        """
        Apply the given regex and return the first unicode string which
        matches. If there is no match, return the default value (``None`` if
        the argument is not provided).

        By default, character entity references are replaced by their
        corresponding character (except for ``&`` and ``<``).
        Passing ``replace_entities`` as ``False`` switches off these
        replacements.
        """
        return next(iflatten(self.re(regex, replace_entities=replace_entities)), default)

    def get(self):
        """
        Serialize and return the matched nodes in a single unicode string.
        Percent encoded content is unquoted.
        """
        try:
            return etree.tostring(self.root,
                                  method=self._tostring_method,
                                  encoding='unicode',
                                  with_tail=False)
        except (AttributeError, TypeError):
            if self.root is True:
                return u'1'
            elif self.root is False:
                return u'0'
            else:
                return six.text_type(self.root)
    extract = get

    def getall(self):
        """
        Serialize and return the matched node in a 1-element list of unicode strings.
        """
        return [self.get()]

    def register_namespace(self, prefix, uri):
        """
        Register the given namespace to be used in this :class:`Selector`.
        Without registering namespaces you can't select or extract data from
        non-standard namespaces. See :ref:`selector-examples-xml`.
        """
        self.namespaces[prefix] = uri

    def remove_namespaces(self):
        """
        Remove all namespaces, allowing to traverse the document using
        namespace-less xpaths. See :ref:`removing-namespaces`.
        """
        for el in self.root.iter('*'):
            if el.tag.startswith('{'):
                el.tag = el.tag.split('}', 1)[1]
            # loop on element attributes also
            for an in el.attrib.keys():
                if an.startswith('{'):
                    el.attrib[an.split('}', 1)[1]] = el.attrib.pop(an)
            # remove namespace declarations
            etree.cleanup_namespaces(self.root)

    @property
    def attrib(self):
        """Return the attributes dictionary for underlying element.
        """
        return dict(self.root.attrib)

    def __bool__(self):
        """
        Return ``True`` if there is any real content selected or ``False``
        otherwise.  In other words, the boolean value of a :class:`Selector` is
        given by the contents it selects.
        """
        return bool(self.get())
    __nonzero__ = __bool__

    def __str__(self):
        data = repr(self.get()[:40])
        return "<%s xpath=%r data=%s>" % (type(self).__name__, self._expr, data)
    __repr__ = __str__
<commit_msg>Embed cssselect reference<commit_after>"""
XPath selectors based on lxml
"""

import sys

import six
from lxml import etree, html

from .utils import flatten, iflatten, extract_regex
from .csstranslator import HTMLTranslator, GenericTranslator

class SafeXMLParser(etree.XMLParser):
    def __init__(self, *args, **kwargs):
        kwargs.setdefault('resolve_entities', False)
        super(SafeXMLParser, self).__init__(*args, **kwargs)

_ctgroup = {
    'html': {'_parser': html.HTMLParser,
             '_csstranslator': HTMLTranslator(),
             '_tostring_method': 'html'},
    'xml': {'_parser': SafeXMLParser,
            '_csstranslator': GenericTranslator(),
            '_tostring_method': 'xml'},
}

def _st(st):
    if st is None:
        return 'html'
    elif st in _ctgroup:
        return st
    else:
        raise ValueError('Invalid type: %s' % st)

def create_root_node(text, parser_cls, base_url=None):
    """Create root node for text using given parser class.
    """
    body = text.strip().replace('\x00', '').encode('utf8') or b'<html/>'
    parser = parser_cls(recover=True, encoding='utf8')
    root = etree.fromstring(body, parser=parser, base_url=base_url)
    if root is None:
        root = etree.fromstring(b'<html/>', parser=parser, base_url=base_url)
    return root

class SelectorList(list):
    """
    The :class:`SelectorList` class is a subclass of the builtin ``list``
    class, which provides a few additional methods.
    """

    # __getslice__ is deprecated but `list` builtin implements it only in Py2
    def __getslice__(self, i, j):
        o = super(SelectorList, self).__getslice__(i, j)
        return self.__class__(o)

    def __getitem__(self, pos):
        o = super(SelectorList, self).__getitem__(pos)
        return self.__class__(o) if isinstance(pos, slice) else o

    def __getstate__(self):
        raise TypeError("can't pickle SelectorList objects")

    def xpath(self, xpath, namespaces=None, **kwargs):
        """
        Call the ``.xpath()`` method for each element in this list and return
        their results flattened as another :class:`SelectorList`.

        ``query`` is the same argument as the one in :meth:`Selector.xpath`

        ``namespaces`` is an optional ``prefix: namespace-uri`` mapping (dict)
        for additional prefixes to those registered with ``register_namespace(prefix, uri)``.
        Contrary to ``register_namespace()``, these prefixes are not
        saved for future calls.

        Any additional named arguments can be used to pass values for XPath
        variables in the XPath expression, e.g.::

            selector.xpath('//a[href=$url]', url="http://www.example.com")
        """
        return self.__class__(flatten([x.xpath(xpath, namespaces=namespaces, **kwargs) for x in self]))

    def css(self, query):
        """
        Call the ``.css()`` method for each element in this list and return
        their results flattened as another :class:`SelectorList`.

        ``query`` is the same argument as the one in :meth:`Selector.css`
        """
        return self.__class__(flatten([x.css(query) for x in self]))

    def re(self, regex, replace_entities=True):
        """
        Call the ``.re()`` method for each element in this list and return
        their results flattened, as a list of unicode strings.

        By default, character entity references are replaced by their
        corresponding character (except for ``&`` and ``<``.
        Passing ``replace_entities`` as ``False`` switches off these
        replacements.
        """
        return flatten([x.re(regex, replace_entities=replace_entities) for x in self])

    def re_first(self, regex, default=None, replace_entities=True):
        """
        Call the ``.re()`` method for the first element in this list and
        return the result in an unicode string. If the list is empty or the
        regex doesn't match anything, return the default value (``None`` if
        the argument is not provided).

        By default, character entity references are replaced by their
        corresponding character (except for ``&`` and ``<``.
        Passing ``replace_entities`` as ``False`` switches off these
        replacements.
        """
        for el in iflatten(x.re(regex, replace_entities=replace_entities) for x in self):
            return el
        else:
            return default

    def getall(self):
        """
        Call the ``.get()`` method for each element is this list and return
        their results flattened, as a list of unicode strings.
        """
        return [x.get() for x in self]
    extract = getall

    def get(self, default=None):
        """
        Return the result of ``.get()`` for the first element in this list.
        If the list is empty, return the default value.
        """
        for x in self:
            return x.get()
        else:
            return default
    extract_first = get

    @property
    def attrib(self):
        """Return the attributes dictionary for the first element.
        If the list is empty, return an empty dict.
        """
        for x in self:
            return x.attrib
        else:
            return {}

class Selector(object):
    """
    :class:`Selector` allows you to select parts of an XML or HTML text using CSS
    or XPath expressions and extract data from it.

    ``text`` is a ``unicode`` object in Python 2 or a ``str`` object in Python 3

    ``type`` defines the selector type, it can be ``"html"``, ``"xml"`` or ``None`` (default).
    If ``type`` is ``None``, the selector defaults to ``"html"``.
    """

    __slots__ = ['text', 'namespaces', 'type', '_expr', 'root',
                 '__weakref__', '_parser', '_csstranslator', '_tostring_method']

    _default_type = None
    _default_namespaces = {
        "re": "http://exslt.org/regular-expressions",

        # supported in libxslt:
        # set:difference
        # set:has-same-node
        # set:intersection
        # set:leading
        # set:trailing
        "set": "http://exslt.org/sets"
    }
    _lxml_smart_strings = False
    selectorlist_cls = SelectorList

    def __init__(self, text=None, type=None, namespaces=None, root=None,
                 base_url=None, _expr=None):
        self.type = st = _st(type or self._default_type)
        self._parser = _ctgroup[st]['_parser']
        self._csstranslator = _ctgroup[st]['_csstranslator']
        self._tostring_method = _ctgroup[st]['_tostring_method']

        if text is not None:
            if not isinstance(text, six.text_type):
                raise TypeError("text argument should be of type %s" % six.text_type)
            root = self._get_root(text, base_url)
        elif root is None:
            raise ValueError("Selector needs either text or root argument")

        self.namespaces = dict(self._default_namespaces)
        if namespaces is not None:
            self.namespaces.update(namespaces)
        self.root = root
        self._expr = _expr

    def __getstate__(self):
        raise TypeError("can't pickle Selector objects")

    def _get_root(self, text, base_url=None):
        return create_root_node(text, self._parser, base_url=base_url)

    def xpath(self, query, namespaces=None, **kwargs):
        """
        Find nodes matching the xpath ``query`` and return the result as a
        :class:`SelectorList` instance with all elements flattened. List
        elements implement :class:`Selector` interface too.

        ``query`` is a string containing the XPATH query to apply.

        ``namespaces`` is an optional ``prefix: namespace-uri`` mapping (dict)
        for additional prefixes to those registered with ``register_namespace(prefix, uri)``.
        Contrary to ``register_namespace()``, these prefixes are not
        saved for future calls.

        Any additional named arguments can be used to pass values for XPath
        variables in the XPath expression, e.g.::

            selector.xpath('//a[href=$url]', url="http://www.example.com")
        """
        try:
            xpathev = self.root.xpath
        except AttributeError:
            return self.selectorlist_cls([])

        nsp = dict(self.namespaces)
        if namespaces is not None:
            nsp.update(namespaces)
        try:
            result = xpathev(query, namespaces=nsp,
                             smart_strings=self._lxml_smart_strings,
                             **kwargs)
        except etree.XPathError as exc:
            msg = u"XPath error: %s in %s" % (exc, query)
            msg = msg if six.PY3 else msg.encode('unicode_escape')
            six.reraise(ValueError, ValueError(msg), sys.exc_info()[2])

        if type(result) is not list:
            result = [result]

        result = [self.__class__(root=x, _expr=query,
                                 namespaces=self.namespaces,
                                 type=self.type)
                  for x in result]
        return self.selectorlist_cls(result)

    def css(self, query):
        """
        Apply the given CSS selector and return a :class:`SelectorList` instance.

        ``query`` is a string containing the CSS selector to apply.

        In the background, CSS queries are translated into XPath queries using
        `cssselect`_ library and run ``.xpath()`` method.

        .. _cssselect: https://pypi.python.org/pypi/cssselect/
        """
        return self.xpath(self._css2xpath(query))

    def _css2xpath(self, query):
        return self._csstranslator.css_to_xpath(query)

    def re(self, regex, replace_entities=True):
        """
        Apply the given regex and return a list of unicode strings with the
        matches.

        ``regex`` can be either a compiled regular expression or a string which
        will be compiled to a regular expression using ``re.compile(regex)``.

        By default, character entity references are replaced by their
        corresponding character (except for ``&`` and ``<``).
        Passing ``replace_entities`` as ``False`` switches off these
        replacements.
        """
        return extract_regex(regex, self.get(), replace_entities=replace_entities)

    def re_first(self, regex, default=None, replace_entities=True):
        """
        Apply the given regex and return the first unicode string which
        matches. If there is no match, return the default value (``None`` if
        the argument is not provided).

        By default, character entity references are replaced by their
        corresponding character (except for ``&`` and ``<``).
        Passing ``replace_entities`` as ``False`` switches off these
        replacements.
        """
        return next(iflatten(self.re(regex, replace_entities=replace_entities)), default)

    def get(self):
        """
        Serialize and return the matched nodes in a single unicode string.
        Percent encoded content is unquoted.
        """
        try:
            return etree.tostring(self.root,
                                  method=self._tostring_method,
                                  encoding='unicode',
                                  with_tail=False)
        except (AttributeError, TypeError):
            if self.root is True:
                return u'1'
            elif self.root is False:
                return u'0'
            else:
                return six.text_type(self.root)
    extract = get

    def getall(self):
        """
        Serialize and return the matched node in a 1-element list of unicode strings.
        """
        return [self.get()]

    def register_namespace(self, prefix, uri):
        """
        Register the given namespace to be used in this :class:`Selector`.
        Without registering namespaces you can't select or extract data from
        non-standard namespaces. See :ref:`selector-examples-xml`.
        """
        self.namespaces[prefix] = uri

    def remove_namespaces(self):
        """
        Remove all namespaces, allowing to traverse the document using
        namespace-less xpaths. See :ref:`removing-namespaces`.
        """
        for el in self.root.iter('*'):
            if el.tag.startswith('{'):
                el.tag = el.tag.split('}', 1)[1]
            # loop on element attributes also
            for an in el.attrib.keys():
                if an.startswith('{'):
                    el.attrib[an.split('}', 1)[1]] = el.attrib.pop(an)
            # remove namespace declarations
            etree.cleanup_namespaces(self.root)

    @property
    def attrib(self):
        """Return the attributes dictionary for underlying element.
        """
        return dict(self.root.attrib)

    def __bool__(self):
        """
        Return ``True`` if there is any real content selected or ``False``
        otherwise.  In other words, the boolean value of a :class:`Selector` is
        given by the contents it selects.
        """
        return bool(self.get())
    __nonzero__ = __bool__

    def __str__(self):
        data = repr(self.get()[:40])
        return "<%s xpath=%r data=%s>" % (type(self).__name__, self._expr, data)
    __repr__ = __str__
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import importlib
import inspect
import re
import sys

import pytest

def pytest_addoption(parser):
    group = parser.getgroup('wish')
    group.addoption('--wish-modules', default=(), nargs='+',
                    help="Space separated list of module names.")
    group.addoption('--wish-includes', nargs='+',
                    help="Space separated list of regexs matching full object names to include.")
    group.addoption('--wish-excludes', default=(), nargs='+',
                    help="Space separated list of regexs matching full object names to exclude.")
    group.addoption('--wish-fail', action='store_true', help="Show wish failures.")

def generate_module_objects(module):
    for object_name, object_ in inspect.getmembers(module):
        obj_module = inspect.getmodule(object_)
        if obj_module is not module:
            continue
        yield object_name, object_

def index_modules(modules, include_patterns, exclude_patterns=()):
    include_res = [re.compile(pattern) for pattern in include_patterns]
    exclude_res = [re.compile(pattern) for pattern in exclude_patterns]
    object_index = {}
    for module_name, module in modules.items():
        for object_name, object_ in generate_module_objects(module):
            full_object_name = '{}:{}'.format(module_name, object_name)
            include_name = any(include_re.match(full_object_name) for include_re in include_res)
            exclude_name = any(exclude_re.match(full_object_name) for exclude_re in exclude_res)
            if include_name and not exclude_name:
                object_index[full_object_name] = object_
    return object_index

def pytest_generate_tests(metafunc):
    if 'wish' not in metafunc.fixturenames:
        return

    wish_modules = metafunc.config.getoption('wish_modules')
    for module_name in wish_modules:
        importlib.import_module(module_name)

    wish_includes = metafunc.config.getoption('wish_includes') or wish_modules
    wish_excludes = metafunc.config.getoption('wish_excludes')
    # NOTE: 'copy' is needed here because index_modules may unexpectedly trigger a module load
    object_index = index_modules(sys.modules.copy(), wish_includes, wish_excludes)
    object_items = sorted(object_index.items())
    ids, params = list(zip(*object_items)) or [[], []]
    metafunc.parametrize('wish', params, ids=ids, scope='module')

    wish_fail = metafunc.config.getoption('wish_fail')
    if not wish_fail:
        metafunc.function = pytest.mark.xfail(metafunc.function)
<commit_msg>Define valid_name as a helper function for easier testing.<commit_after># -*- coding: utf-8 -*-

import importlib
import inspect
import re
import sys

import pytest

def pytest_addoption(parser):
    group = parser.getgroup('wish')
    group.addoption('--wish-modules', default=(), nargs='+',
                    help="Space separated list of module names.")
    group.addoption('--wish-includes', nargs='+',
                    help="Space separated list of regexs matching full object names to include.")
    group.addoption('--wish-excludes', default=(), nargs='+',
                    help="Space separated list of regexs matching full object names to exclude.")
    group.addoption('--wish-fail', action='store_true', help="Show wish failures.")

def generate_module_objects(module):
    for object_name, object_ in inspect.getmembers(module):
        obj_module = inspect.getmodule(object_)
        if obj_module is not module:
            continue
        yield object_name, object_

def valid_name(name, include_res, exclude_res):
    include_name = any(include_re.match(name) for include_re in include_res)
    exclude_name = any(exclude_re.match(name) for exclude_re in exclude_res)
    return include_name and not exclude_name

def index_modules(modules, include_patterns, exclude_patterns=()):
    include_res = [re.compile(pattern) for pattern in include_patterns]
    exclude_res = [re.compile(pattern) for pattern in exclude_patterns]
    object_index = {}
    for module_name, module in modules.items():
        for object_name, object_ in generate_module_objects(module):
            full_object_name = '{}:{}'.format(module_name, object_name)
            if valid_name(full_object_name, include_res, exclude_res):
                object_index[full_object_name] = object_
    return object_index

def pytest_generate_tests(metafunc):
    if 'wish' not in metafunc.fixturenames:
        return

    wish_modules = metafunc.config.getoption('wish_modules')
    for module_name in wish_modules:
        importlib.import_module(module_name)

    wish_includes = metafunc.config.getoption('wish_includes') or wish_modules
    wish_excludes = metafunc.config.getoption('wish_excludes')
    # NOTE: 'copy' is needed here because index_modules may unexpectedly trigger a module load
    object_index = index_modules(sys.modules.copy(), wish_includes, wish_excludes)
    object_items = sorted(object_index.items())
    ids, params = list(zip(*object_items)) or [[], []]
    metafunc.parametrize('wish', params, ids=ids, scope='module')

    wish_fail = metafunc.config.getoption('wish_fail')
    if not wish_fail:
        metafunc.function = pytest.mark.xfail(metafunc.function)
<|endoftext|>


<commit_before>from sculpt.common.parameter_proxy import parameter_proxy
import collections

# Enumeration
#
# Django's support for enumerated types is... quirky. As in,
# you define a list of (value, label) pairs and pass it to an
# integer/char field, and in the admin it restricts the set.
# It does NOT create an actual ENUM field type, nor does it
# create a foreign-key type, nor does it enforce the choices
# on direct assignment. And referencing the values by label
# in your code isn't possible.
#
# What we would LIKE is to be able to reference
# classname.ENUM_LABEL directly AND have that automatically
# used for the choices on the field. We'd really like it to
# be enforced, but we'll settle for not.
#
# Use this way:
#
# class MyModel(model.Model):
#
#     MY_ENUMS = Enumeration(
#             (0, 'FOO'),
#             (1, 'BAR'),
#         )
#     enum_field = models.IntegerField(choices = MY_ENUMS.choices)
#
# print MyModel.MY_ENUMS.FOO
# >>> 0
#
# NOTE: if your list of enumerated values is longer than 254,
# you won't be able to pass it directly as parameters because
# Python function parameters are actually tuples and tuples are
# capped at 254 entries. Instead, pass choices = [ <tuples> ]
#
# You can create additional fields with each enumeration,
# essentially turning it into a miniature record set that has
# its values defined by code, rather than a database. This
# will let you do useful things like associate helper classes
# or functions with specific enumerated values.
#
# To accommodate this, internally all enumeration values are
# actually labeled; the defaults for 2-tuples are "value" and
# "id". If you pass labels = <tuple> then you can provide
# any number of columns. You can access the data as
#
#     MY_ENUMS.data.FOO['<column_name>']
#
# You can generate a tuple of 2-tuples from any two columns:
#
#     MY_ENUMS.tuples('<column_name1>','<column_name2>')
#
# and in fact the choices field is populated exactly this way.
# Also available is the notion of "display" labels, which can
# be used to create drop-downs without sacrificing code
# readability. Define a "label" column, and you can use this:
#
#     MY_ENUMS.labels
#
# which is a tuple of value,label tuples instead of the
# value,id tuples returned by MY_ENUMS.choices
#
# If you pass a list of column labels, you can use the value
# and id columns at any position, but their presence is
# required.
#
# The Enumeration class uses tuples because this stuff is
# supposed to be static. If you find yourself wanting to muck
# with the internal representation of an enumeration to modify
# its values at run-time, don't. Call it something else.
#
class Enumeration(object):

    _columns = None
    _columns_idx = None
    _data = None
    _data_dicts = None
    _idxs = None

    _choices = None
    _labels = None

    _values_are_strings = False

    def __init__(self, *args, **kwargs):
        # pluck data from parameters
        self._data = kwargs.get('choices', args)
        default_column_names = ( 'value', 'id' ) if len(self._data) and len(self._data[0]) == 2 else ( 'value', 'id', 'label' )
        self._columns = kwargs.get('labels', default_column_names)

        # create column-label to column-index lookup
        self._columns_idx = dict([ (self._columns[i],i) for i in range(len(self._columns)) ])

        # create indices for each column heading; any entries
        # which do not hold values in that column are ignored
        # (we also ignore entries that are not hashable)
        self._idxs = {}
        for i in range(len(self._columns)):
            column = self._columns[i]
            idx = {}
            self._idxs[column] = idx

            for j in range(len(self._data)):
                row = self._data[j]
                if i < len(row) and isinstance(row[i], collections.Hashable):
                    idx[row[i]] = j

        # go ahead and prep the data dicts for each row, so
        # that we don't constantly create new throw-away dicts
        # for this data; any fields missing from the list
        # (because the tuple is too short) are filled in as
        # None
        self._data_dicts = []
        for row in self._data:
            data_dict = {}
            self._data_dicts.append(data_dict)

            for i in range(len(self._columns)):
                column = self._columns[i]
                if i < len(row):
                    data_dict[column] = row[i]
                else:
                    data_dict[column] = None

        # we need to know if our values ever include strings
        # because we change how get_value() works if our values
        # include strings
        for d in self._data_dicts:
            if isinstance(d.get('value'), basestring):
                self._values_are_strings = True
                break

        # fun bit: we want .data to be a parameter proxy to
        # the get_data() method, but we need a reference to
        # ourselves to pass in as the enumeration, so we have
        # to do this at creation time
        #self.data = parameter_proxy('get_data', enumeration = self)

    # generate a tuple of 2-tuples given a pair of column names
    def tuples(self, column1, column2):
        return tuple( (d[column1],d[column2]) for d in self._data_dicts )

    #
    # cached access to common tuples-of-tuples extractions
    #

    @property
    def choices(self):
        if self._choices == None:
            # choices must be a list of 2-tuples to keep South happy
            self._choices = self.tuples('value', 'id')
        return self._choices

    @property
    def labels(self):
        if self._labels == None:
            self._labels = self.tuples('value', 'label')
        return self._labels

    # given a particular column name and value, return the row
    #
    # NOTE: this is more efficient than using the .data
    # property, and is the preferred implementation for code
    # rather than templates
    #
    def get_data(self, column, value):
        # if column is not in self._idxs, KeyError raised
        if value not in self._idxs[column]:
            # pretend we're a real attribute, and throw a similar exception,
            # instead of KeyError from a dict lookup
            raise AttributeError()
        row_number = self._idxs[column][value]
        return self._data_dicts[row_number]

    # given a particular ID, return the row
    # (a thin wrapper around get_data())
    #
    # NOTE: this is called _by_id because the
    # "value" that is passed is actually
    # checked to see if it's an ID value first
    #
    def get_data_by_id(self, value):
        return self.get_data('value', self.get_value(value))

    # look up an attribute, if it is not found
    # elsewhere (we look up the name in our set
    # of enumerations)
    def __getattr__(self, attr):
        return self.get_data('id', attr)['value']

    # and a parameter proxy to make this super-easy
    # for templates; since we're creating the proxy
    # at class definition time, we don't have an
    # enumeration for parameter_proxy to work with,
    # so we handle that ourselves in the wrapper
    data = property(parameter_proxy('get_data_by_id'))

    # sometimes (e.g. JSON parsing) we want to accept either
    # a numerical value directly or its label; this method
    # accepts either and always returns the value
    #
    # NOTE: does NOT validate values but may validate IDs;
    # numbers not part of the enumeration will not be
    # rejected, but names not part of the enumeration will
    # raise AttributeError if none of the values are strings
    #
    def get_value(self, value):
        if isinstance(value, basestring):
            if self._values_are_strings:
                # this might be an ID or it might be a value;
                # attempt to translate it, but don't throw an
                # error if that fails
                return self._idxs['id'].get(value, value)

            else:
                # assume it's an ID, convert to value and
                # raise AttributeError if it's wrong
                return self.get_data('id', value)['value']

        else:
            # already a value, return as-is
            return value

    # methods to make the class iterable
    def __len__(self):
        return len(self._data)

    def __iter__(self):
        return self._data.__iter__()

    def __getitem__(self, key):
        return self._data[key]

    def __setitem__(self, key, value):
        raise TypeError("'Enumeration' object does not support item assignment")

    def __delitem__(self, key):
        raise TypeError("'Enumeration' object does not support item assignment")

    # without this, 'in' tests by iterating, which isn't useful;
    # we want to test if an id is in the enumeration
    def __contains__(self, key):
        return key in self._idxs['id']

    # and because it's useful, provide a method to
    # iterate over the enumeration as dicts
    def iter_dicts(self):
        return self._data_dicts.__iter__()

# EnumerationData
#
# Often we want to use an Enumeration to set acceptable choices
# on a field, but still have access to all the other data
# associated with the current enumerated choice. For example:
#
#   class Foo(models.Model):
#       BARS = Enumeration(
#           labels = ('value','id','label','helper'),
#           choices = (
#               (0,'BAZ','baz',BazHelper),
#               (1,'BOOYAH','booyah',BooyahHelper),
#           )
#       )
#       bar = models.IntegerField(choices = BARS.choices)
#
# If we have an instance of Foo named foo, we can access the
# numerical value of bar as foo.bar directly, and compare it
# to enumerated values like this:
#
#   if foo.bar == Foo.BARS.BAZ:
#       ...
#
# But if we want to access any of the corresponding data that
# goes along with the actual value, we have to use the far
# more cumbersome construct:
#
#   helper = Foo.BARS.get_data_by_id(foo.bar)['helper']
#
# Which requires us to know what the correct enumeration to
# go with .bar happens to be, a potential source of errors.
#
# This function allows us to create a property on Foo that
# automatically fetches the correct data based on the value
# of the variable we specify. So, we include this in the
# class definition:
#
#       bar_data = property(EnumerationData('BARS', 'bar'))
#
# And then the cumbersome construct becomes this:
#
#   helper = foo.bar_data['helper']
#
# Which is much simpler. It does hide that bar_data is
# derived from bar, though, so it's best to use a clear
# naming convention.
#
# NOTE: the enumeration is specified via attribute name
# rather than directly passed so that the lookup is done
# at the moment the data is fetched; in this way, the
# data property can be added to a base class even if the
# derived class overrides the enumeration. If you must
# reference an enumeration that isn't a member of the class
# (possibly because you're sharing it among several
# classes, which is good) then you can specify an actual
# Enumeration object as well, but you will then lose the
# polymorphic capability.
#
def EnumerationData(enumeration, fieldname):
    def inner(self):
        if not isinstance(enumeration, Enumeration):
            e = getattr(self, enumeration)
        else:
            e = enumeration
        return e.get_data_by_id(getattr(self, fieldname))
    return inner

<commit_msg>Allow a callable to be included in the choices list which generates (and inserts) tuples into that list.<commit_after>from sculpt.common.parameter_proxy import parameter_proxy
import collections

# Enumeration
#
# Django's support for enumerated types is... quirky. As in,
# you define a list of (value, label) pairs and pass it to an
# integer/char field, and in the admin it restricts the set.
# It does NOT create an actual ENUM field type, nor does it
# create a foreign-key type, nor does it enforce the choices
# on direct assignment. And referencing the values by label
# in your code isn't possible.
#
# What we would LIKE is to be able to reference
# classname.ENUM_LABEL directly AND have that automatically
# used for the choices on the field. We'd really like it to
# be enforced, but we'll settle for not.
#
# Use this way:
#
# class MyModel(model.Model):
#
#     MY_ENUMS = Enumeration(
#             (0, 'FOO'),
#             (1, 'BAR'),
#         )
#     enum_field = models.IntegerField(choices = MY_ENUMS.choices)
#
# print MyModel.MY_ENUMS.FOO
# >>> 0
#
# NOTE: if your list of enumerated values is longer than 254,
# you won't be able to pass it directly as parameters because
# Python function parameters are actually tuples and tuples are
# capped at 254 entries. Instead, pass choices = [ <tuples> ]
#
# You can create additional fields with each enumeration,
# essentially turning it into a miniature record set that has
# its values defined by code, rather than a database. This
# will let you do useful things like associate helper classes
# or functions with specific enumerated values.
#
# To accommodate this, internally all enumeration values are
# actually labeled; the defaults for 2-tuples are "value" and
# "id". If you pass labels = <tuple> then you can provide
# any number of columns. You can access the data as
#
#     MY_ENUMS.data.FOO['<column_name>']
#
# You can generate a tuple of 2-tuples from any two columns:
#
#     MY_ENUMS.tuples('<column_name1>','<column_name2>')
#
# and in fact the choices field is populated exactly this way.
# Also available is the notion of "display" labels, which can
# be used to create drop-downs without sacrificing code
# readability. Define a "label" column, and you can use this:
#
#     MY_ENUMS.labels
#
# which is a tuple of value,label tuples instead of the
# value,id tuples returned by MY_ENUMS.choices
#
# If you pass a list of column labels, you can use the value
# and id columns at any position, but their presence is
# required.
#
# The Enumeration class uses tuples because this stuff is
# supposed to be static. If you find yourself wanting to muck
# with the internal representation of an enumeration to modify
# its values at run-time, don't. Call it something else.
#
class Enumeration(object):

    _columns = None
    _columns_idx = None
    _data = None
    _data_dicts = None
    _idxs = None

    _choices = None
    _labels = None

    _values_are_strings = False

    def __init__(self, *args, **kwargs):
        # before we begin processing enumeration entries, scan
        # it for callables which might yield generated entries;
        # this can be used to generate additional enumerations
        # that are partially derived from other enumerations
        self._data = []
        choices = kwargs.get('choices', args)
        for t in choices:
            if callable(t):
                self._data.extend(t())
            else:
                self._data.append(t)

        # pluck data from parameters now that the choices are
        # fully generated and flattened
        default_column_names = ( 'value', 'id' ) if len(self._data) and len(self._data[0]) == 2 else ( 'value', 'id', 'label' )
        self._columns = kwargs.get('labels', default_column_names)

        # create column-label to column-index lookup
        self._columns_idx = dict([ (self._columns[i],i) for i in range(len(self._columns)) ])

        # create indices for each column heading; any entries
        # which do not hold values in that column are ignored
        # (we also ignore entries that are not hashable)
        self._idxs = {}
        for i in range(len(self._columns)):
            column = self._columns[i]
            idx = {}
            self._idxs[column] = idx

            for j in range(len(self._data)):
                row = self._data[j]
                if i < len(row) and isinstance(row[i], collections.Hashable):
                    idx[row[i]] = j

        # go ahead and prep the data dicts for each row, so
        # that we don't constantly create new throw-away dicts
        # for this data; any fields missing from the list
        # (because the tuple is too short) are filled in as
        # None
        self._data_dicts = []
        for row in self._data:
            data_dict = {}
            self._data_dicts.append(data_dict)

            for i in range(len(self._columns)):
                column = self._columns[i]
                if i < len(row):
                    data_dict[column] = row[i]
                else:
                    data_dict[column] = None

        # we need to know if our values ever include strings
        # because we change how get_value() works if our values
        # include strings
        for d in self._data_dicts:
            if isinstance(d.get('value'), basestring):
                self._values_are_strings = True
                break

        # fun bit: we want .data to be a parameter proxy to
        # the get_data() method, but we need a reference to
        # ourselves to pass in as the enumeration, so we have
        # to do this at creation time
        #self.data = parameter_proxy('get_data', enumeration = self)

    # generate a tuple of 2-tuples given a pair of column names
    def tuples(self, column1, column2):
        return tuple( (d[column1],d[column2]) for d in self._data_dicts )

    #
    # cached access to common tuples-of-tuples extractions
    #

    @property
    def choices(self):
        if self._choices == None:
            # choices must be a list of 2-tuples to keep South happy
            self._choices = self.tuples('value', 'id')
        return self._choices

    @property
    def labels(self):
        if self._labels == None:
            self._labels = self.tuples('value', 'label')
        return self._labels

    # given a particular column name and value, return the row
    #
    # NOTE: this is more efficient than using the .data
    # property, and is the preferred implementation for code
    # rather than templates
    #
    def get_data(self, column, value):
        # if column is not in self._idxs, KeyError raised
        if value not in self._idxs[column]:
            # pretend we're a real attribute, and throw a similar exception,
            # instead of KeyError from a dict lookup
            raise AttributeError()
        row_number = self._idxs[column][value]
        return self._data_dicts[row_number]

    # given a particular ID, return the row
    # (a thin wrapper around get_data())
    #
    # NOTE: this is called _by_id because the
    # "value" that is passed is actually
    # checked to see if it's an ID value first
    #
    def get_data_by_id(self, value):
        return self.get_data('value', self.get_value(value))

    # look up an attribute, if it is not found
    # elsewhere (we look up the name in our set
    # of enumerations)
    def __getattr__(self, attr):
        return self.get_data('id', attr)['value']

    # and a parameter proxy to make this super-easy
    # for templates; since we're creating the proxy
    # at class definition time, we don't have an
    # enumeration for parameter_proxy to work with,
    # so we handle that ourselves in the wrapper
    data = property(parameter_proxy('get_data_by_id'))

    # sometimes (e.g. JSON parsing) we want to accept either
    # a numerical value directly or its label; this method
    # accepts either and always returns the value
    #
    # NOTE: does NOT validate values but may validate IDs;
    # numbers not part of the enumeration will not be
    # rejected, but names not part of the enumeration will
    # raise AttributeError if none of the values are strings
    #
    def get_value(self, value):
        if isinstance(value, basestring):
            if self._values_are_strings:
                # this might be an ID or it might be a value;
                # attempt to translate it, but don't throw an
                # error if that fails
                return self._idxs['id'].get(value, value)

            else:
                # assume it's an ID, convert to value and
                # raise AttributeError if it's wrong
                return self.get_data('id', value)['value']

        else:
            # already a value, return as-is
            return value

    # methods to make the class iterable
    def __len__(self):
        return len(self._data)

    def __iter__(self):
        return self._data.__iter__()

    def __getitem__(self, key):
        return self._data[key]

    def __setitem__(self, key, value):
        raise TypeError("'Enumeration' object does not support item assignment")

    def __delitem__(self, key):
        raise TypeError("'Enumeration' object does not support item assignment")

    # without this, 'in' tests by iterating, which isn't useful;
    # we want to test if an id is in the enumeration
    def __contains__(self, key):
        return key in self._idxs['id']

    # and because it's useful, provide a method to
    # iterate over the enumeration as dicts
    def iter_dicts(self):
        return self._data_dicts.__iter__()

# EnumerationData
#
# Often we want to use an Enumeration to set acceptable choices
# on a field, but still have access to all the other data
# associated with the current enumerated choice. For example:
#
#   class Foo(models.Model):
#       BARS = Enumeration(
#           labels = ('value','id','label','helper'),
#           choices = (
#               (0,'BAZ','baz',BazHelper),
#               (1,'BOOYAH','booyah',BooyahHelper),
#           )
#       )
#       bar = models.IntegerField(choices = BARS.choices)
#
# If we have an instance of Foo named foo, we can access the
# numerical value of bar as foo.bar directly, and compare it
# to enumerated values like this:
#
#   if foo.bar == Foo.BARS.BAZ:
#       ...
#
# But if we want to access any of the corresponding data that
# goes along with the actual value, we have to use the far
# more cumbersome construct:
#
#   helper = Foo.BARS.get_data_by_id(foo.bar)['helper']
#
# Which requires us to know what the correct enumeration to
# go with .bar happens to be, a potential source of errors.
#
# This function allows us to create a property on Foo that
# automatically fetches the correct data based on the value
# of the variable we specify. So, we include this in the
# class definition:
#
#       bar_data = property(EnumerationData('BARS', 'bar'))
#
# And then the cumbersome construct becomes this:
#
#   helper = foo.bar_data['helper']
#
# Which is much simpler. It does hide that bar_data is
# derived from bar, though, so it's best to use a clear
# naming convention.
#
# NOTE: the enumeration is specified via attribute name
# rather than directly passed so that the lookup is done
# at the moment the data is fetched; in this way, the
# data property can be added to a base class even if the
# derived class overrides the enumeration. If you must
# reference an enumeration that isn't a member of the class
# (possibly because you're sharing it among several
# classes, which is good) then you can specify an actual
# Enumeration object as well, but you will then lose the
# polymorphic capability.
#
def EnumerationData(enumeration, fieldname):
    def inner(self):
        if not isinstance(enumeration, Enumeration):
            e = getattr(self, enumeration)
        else:
            e = enumeration
        return e.get_data_by_id(getattr(self, fieldname))
    return inner

# Sometimes we need an enumeration to be derived from,
# or include elements of, another enumeration. To handle
# this we define a simple method that walks through the
# enumeration and a template, fills out any strings with
# placeholders, and inserts the resulting list directly
# into the new enumeration.
#
# NOTE: this actually returns a callable suitable for
# use in the Enumeration choices list.
#
def expand_enumeration(template, source):

    def inner():
        expanded = []
        for d in source._data_dicts:
            expanded.append(tuple([ v % d if isinstance(v, basestring) else v for v in template ]))
        return expanded

    return inner
<|endoftext|>