<commit_before>"""This module contains class for libclang based completions

Attributes:
    cindex_dict (dict): dict of cindex entries for each version of clang
    log (logging.Logger): logger for this module
"""
import importlib
import sublime
import time
import logging

from os import path

from .. import error_vis
from .. import tools
from .base_complete import BaseCompleter

log = logging.getLogger(__name__)
log.debug(" reloading module")

cindex_dict = {
    '3.2': tools.PKG_NAME + ".clang.cindex32",
    '3.3': tools.PKG_NAME + ".clang.cindex33",
    '3.4': tools.PKG_NAME + ".clang.cindex34",
    '3.5': tools.PKG_NAME + ".clang.cindex35",
    '3.6': tools.PKG_NAME + ".clang.cindex36",
    '3.7': tools.PKG_NAME + ".clang.cindex37",
    '3.8': tools.PKG_NAME + ".clang.cindex38",
}

class Completer(BaseCompleter):

    """Encapsulates completions based on libclang

    Attributes:
        tu_module (cindex.TranslationUnit): module for proper cindex
        translation_units (dict): dictionary of tu's for each view id
    """

    tu_module = None

    translation_units = {}

    def __init__(self, clang_binary):
        """Initialize the Completer from clang binary, reading its version.
        Picks an according cindex for the found version.

        Args:
            clang_binary (str): string for clang binary e.g. 'clang-3.6++'

        """
        super(Completer, self).__init__(clang_binary)

        # initialize cindex
        if self.version_str in cindex_dict:
            try:
                # should work if python bindings are installed
                cindex = importlib.import_module("clang.cindex")
            except Exception as e:
                # should work for other cases
                log.warning(" cannot get default cindex with error: %s", e)
                log.warning(" using bundled one: %s",
                            cindex_dict[self.version_str])
                cindex = importlib.import_module(
                    cindex_dict[self.version_str])
            Completer.tu_module = cindex.TranslationUnit
            # check if we can build an index. If not, set valid to false
            try:
                cindex.Index.create()
                self.valid = True
            except Exception as e:
                log.error(" error: %s", e)
                self.valid = False

    def remove(self, view_id):
        """Remove tu for this view. Happens when we don't need it anymore.

        Args:
            view_id (int): view id

        """
        if view_id not in self.translation_units:
            log.error(" no tu for view id: %s, so not removing", view_id)
            return
        log.debug(" removing translation unit for view id: %s", view_id)
        del self.translation_units[view_id]

    def exists_for_view(self, view_id):
        """find if there is a completer for the view

        Args:
            view_id (int): current view id

        Returns:
            bool: has completer
        """
        if view_id in self.translation_units:
            return True
        return False

    def init(self, view, includes, settings):
        """Initialize the completer. Builds the view.

        Args:
            view (sublime.View): current view
            includes (list): includes from settings
            settings (Settings): plugin settings

        """
        file_name = view.file_name()
        file_body = view.substr(sublime.Region(0, view.size()))
        file_folder = path.dirname(file_name)

        # initialize unsaved files
        files = [(file_name, file_body)]

        # init needed variables from settings
        clang_flags = []

        # if we use project-specific settings we ignore everything else
        if settings.project_specific_settings:
            log.debug(" overriding all flags by project ones")
            project_flags = settings.get_project_clang_flags()
            if project_flags:
                clang_flags.append(settings.std_flag)
                clang_flags += project_flags
            else:
                log.error(" there are no project-specific settings.")
                log.info(" falling back to using plugin settings.")
        if len(clang_flags) < 2:
            # add std flag to all flags
            clang_flags.append(settings.std_flag)
            # this means that project specific settings are either not used or
            # invalid, so we still need to initialize from settings
            for include in includes:
                clang_flags.append('-I' + include)
            # support .clang_complete file with -I<indlude> entries
            if settings.search_clang_complete:
                log.debug(" searching for .clang_complete in %s up to %s",
                          file_folder, settings.project_base_folder)
                clang_complete_file = BaseCompleter._search_clang_complete_file(
                    file_folder, settings.project_base_folder)
                if clang_complete_file:
                    log.debug(" found .clang_complete: %s", clang_complete_file)
                    flags = BaseCompleter._parse_clang_complete_file(
                        clang_complete_file)
                    clang_flags += flags
        # now we have the flags and can continue initializing the TU
        log.debug(" clang flags are: %s", clang_flags)
        try:
            TU = Completer.tu_module
            start = time.time()
            log.debug(" compilation started for view id: %s", view.id())
            self.translation_units[view.id()] = TU.from_source(
                filename=file_name,
                args=clang_flags,
                unsaved_files=files,
                options=TU.PARSE_PRECOMPILED_PREAMBLE |
                TU.PARSE_CACHE_COMPLETION_RESULTS)
            end = time.time()
            log.debug(" compilation done in %s seconds", end - start)
        except Exception as e:
            log.error(" error while compiling: %s", e)
        if settings.errors_on_save:
            self.error_vis.generate(
                view, self.translation_units[view.id()].diagnostics,
                error_vis.FORMAT_LIBCLANG)
            self.error_vis.show_regions(view)

    def complete(self, view, cursor_pos, show_errors):
        """This function is called asynchronously to create a list of
        autocompletions. Using the current translation unit it queries libclang
        for the possible completions. It also shows compile errors if needed.

        Args:
            view (sublime.View): current view
            cursor_pos (int): sublime provided poistion of the cursor
            show_errors (bool): controls if we need to show errors

        """
        file_body = view.substr(sublime.Region(0, view.size()))
        (row, col) = view.rowcol(cursor_pos)
        row += 1
        col += 1

        # unsaved files
        files = [(view.file_name(), file_body)]

        # do nothing if there in no translation_unit present
        if not view.id() in self.translation_units:
            log.debug(" cannot complete. No translation unit for view %s",
                      view.id())
            return None
        # execute clang code completion
        start = time.time()
        log.debug(" started code complete for view %s", view.id())
        complete_obj = self.translation_units[view.id()].codeComplete(
            view.file_name(),
            row, col,
            unsaved_files=files)
        end = time.time()
        if complete_obj is None or len(complete_obj.results) == 0:
            log.debug(" no completions")
            return None
        log.debug(" code complete done in %s seconds", end - start)

        self.completions = Completer._parse_completions(complete_obj)
        log.debug(self.completions)
        self.async_completions_ready = True
        Completer._reload_completions(view)
        if show_errors:
            self.error_vis.generate(
                view, self.translation_units[view.id()].diagnostics,
                error_vis.FORMAT_LIBCLANG)
            self.error_vis.show_regions(view)

    def update(self, view, show_errors):
        """Reparse the translation unit. This speeds up completions
        significantly, so we perform this upon file save.

        Args:
            view (sublime.View): current view
            show_errors (bool): if true - highlight compile errors

        Returns:
            bool: reparsed successfully

        """
        if view.id() in self.translation_units:
            log.debug(" reparsing translation_unit for view %s", view.id())
            start = time.time()
            self.translation_units[view.id()].reparse()
            log.debug(" reparsed translation unit in %s seconds",
                      time.time() - start)
            if show_errors:
                self.error_vis.generate(
                    view, self.translation_units[view.id()].diagnostics,
                    error_vis.FORMAT_LIBCLANG)
                self.error_vis.show_regions(view)
            return True
        log.error(" no translation unit for view id %s")
        return False

    @staticmethod
    def _parse_completions(complete_results):
        """Create snippet-like structures from a list of completions

        Args:
            complete_results (list): raw completions list

        Returns:
            list: updated completions
        """
        completions = []
        for c in complete_results.results:
            hint = ''
            contents = ''
            place_holders = 1
            for chunk in c.string:
                hint += chunk.spelling
                if chunk.isKindTypedText():
                    trigger = chunk.spelling
                if chunk.isKindResultType():
                    hint += ' '
                    continue
                if chunk.isKindOptional():
                    continue
                if chunk.isKindInformative():
                    continue
                if chunk.isKindPlaceHolder():
                    contents += ('${' + str(place_holders) + ':' +
                                 chunk.spelling + '}')
                    place_holders += 1
                else:
                    contents += chunk.spelling
            completions.append([trigger + "\t" + hint, contents])
        return completions
<commit_msg>OS X: Locate correct libclang.dylib.<commit_after>"""This module contains class for libclang based completions

Attributes:
    cindex_dict (dict): dict of cindex entries for each version of clang
    log (logging.Logger): logger for this module
"""
import importlib
import os
import platform
import sublime
import subprocess
import time
import logging

from os import path

from .. import error_vis
from .. import tools
from .base_complete import BaseCompleter

log = logging.getLogger(__name__)
log.debug(" reloading module")

cindex_dict = {
    '3.2': tools.PKG_NAME + ".clang.cindex32",
    '3.3': tools.PKG_NAME + ".clang.cindex33",
    '3.4': tools.PKG_NAME + ".clang.cindex34",
    '3.5': tools.PKG_NAME + ".clang.cindex35",
    '3.6': tools.PKG_NAME + ".clang.cindex36",
    '3.7': tools.PKG_NAME + ".clang.cindex37",
    '3.8': tools.PKG_NAME + ".clang.cindex38",
}

class Completer(BaseCompleter):

    """Encapsulates completions based on libclang

    Attributes:
        tu_module (cindex.TranslationUnit): module for proper cindex
        translation_units (dict): dictionary of tu's for each view id
    """

    tu_module = None

    translation_units = {}

    def __init__(self, clang_binary):
        """Initialize the Completer from clang binary, reading its version.
        Picks an according cindex for the found version.

        Args:
            clang_binary (str): string for clang binary e.g. 'clang-3.6++'

        """
        super(Completer, self).__init__(clang_binary)

        # initialize cindex
        if self.version_str in cindex_dict:
            try:
                # should work if python bindings are installed
                cindex = importlib.import_module("clang.cindex")
            except Exception as e:
                # should work for other cases
                log.warning(" cannot get default cindex with error: %s", e)
                log.warning(" using bundled one: %s",
                            cindex_dict[self.version_str])
                cindex = importlib.import_module(
                    cindex_dict[self.version_str])

            if platform.system() == "Darwin":
                # Try to figure out the base path for this installation of clang.
                # This will return something like /.../lib/clang/3.x.0
                get_library_path_cmd = [clang_binary, "-print-file-name="]
                output = subprocess.check_output(get_library_path_cmd).decode('utf8').strip()
                if output:
                    # libclang.dylib can be found in the lib folder of the path
                    # returned above, so we need to go two levels up.
                    libclang_dir = os.path.join(output, "..", "..")
                    if os.path.isdir(libclang_dir):
                        log.info(" setting libclang library dir to %s" % libclang_dir)
                        cindex.Config.set_library_path(libclang_dir)

            Completer.tu_module = cindex.TranslationUnit
            # check if we can build an index. If not, set valid to false
            try:
                cindex.Index.create()
                self.valid = True
            except Exception as e:
                log.error(" error: %s", e)
                self.valid = False

    def remove(self, view_id):
        """Remove tu for this view. Happens when we don't need it anymore.

        Args:
            view_id (int): view id

        """
        if view_id not in self.translation_units:
            log.error(" no tu for view id: %s, so not removing", view_id)
            return
        log.debug(" removing translation unit for view id: %s", view_id)
        del self.translation_units[view_id]

    def exists_for_view(self, view_id):
        """find if there is a completer for the view

        Args:
            view_id (int): current view id

        Returns:
            bool: has completer
        """
        if view_id in self.translation_units:
            return True
        return False

    def init(self, view, includes, settings):
        """Initialize the completer. Builds the view.

        Args:
            view (sublime.View): current view
            includes (list): includes from settings
            settings (Settings): plugin settings

        """
        file_name = view.file_name()
        file_body = view.substr(sublime.Region(0, view.size()))
        file_folder = path.dirname(file_name)

        # initialize unsaved files
        files = [(file_name, file_body)]

        # init needed variables from settings
        clang_flags = []

        # if we use project-specific settings we ignore everything else
        if settings.project_specific_settings:
            log.debug(" overriding all flags by project ones")
            project_flags = settings.get_project_clang_flags()
            if project_flags:
                clang_flags.append(settings.std_flag)
                clang_flags += project_flags
            else:
                log.error(" there are no project-specific settings.")
                log.info(" falling back to using plugin settings.")
        if len(clang_flags) < 2:
            # add std flag to all flags
            clang_flags.append(settings.std_flag)
            # this means that project specific settings are either not used or
            # invalid, so we still need to initialize from settings
            for include in includes:
                clang_flags.append('-I' + include)
            # support .clang_complete file with -I<indlude> entries
            if settings.search_clang_complete:
                log.debug(" searching for .clang_complete in %s up to %s",
                          file_folder, settings.project_base_folder)
                clang_complete_file = BaseCompleter._search_clang_complete_file(
                    file_folder, settings.project_base_folder)
                if clang_complete_file:
                    log.debug(" found .clang_complete: %s", clang_complete_file)
                    flags = BaseCompleter._parse_clang_complete_file(
                        clang_complete_file)
                    clang_flags += flags
        # now we have the flags and can continue initializing the TU
        log.debug(" clang flags are: %s", clang_flags)
        try:
            TU = Completer.tu_module
            start = time.time()
            log.debug(" compilation started for view id: %s", view.id())
            self.translation_units[view.id()] = TU.from_source(
                filename=file_name,
                args=clang_flags,
                unsaved_files=files,
                options=TU.PARSE_PRECOMPILED_PREAMBLE |
                TU.PARSE_CACHE_COMPLETION_RESULTS)
            end = time.time()
            log.debug(" compilation done in %s seconds", end - start)
        except Exception as e:
            log.error(" error while compiling: %s", e)
        if settings.errors_on_save:
            self.error_vis.generate(
                view, self.translation_units[view.id()].diagnostics,
                error_vis.FORMAT_LIBCLANG)
            self.error_vis.show_regions(view)

    def complete(self, view, cursor_pos, show_errors):
        """This function is called asynchronously to create a list of
        autocompletions. Using the current translation unit it queries libclang
        for the possible completions. It also shows compile errors if needed.

        Args:
            view (sublime.View): current view
            cursor_pos (int): sublime provided poistion of the cursor
            show_errors (bool): controls if we need to show errors

        """
        file_body = view.substr(sublime.Region(0, view.size()))
        (row, col) = view.rowcol(cursor_pos)
        row += 1
        col += 1

        # unsaved files
        files = [(view.file_name(), file_body)]

        # do nothing if there in no translation_unit present
        if not view.id() in self.translation_units:
            log.debug(" cannot complete. No translation unit for view %s",
                      view.id())
            return None
        # execute clang code completion
        start = time.time()
        log.debug(" started code complete for view %s", view.id())
        complete_obj = self.translation_units[view.id()].codeComplete(
            view.file_name(),
            row, col,
            unsaved_files=files)
        end = time.time()
        if complete_obj is None or len(complete_obj.results) == 0:
            log.debug(" no completions")
            return None
        log.debug(" code complete done in %s seconds", end - start)

        self.completions = Completer._parse_completions(complete_obj)
        log.debug(self.completions)
        self.async_completions_ready = True
        Completer._reload_completions(view)
        if show_errors:
            self.error_vis.generate(
                view, self.translation_units[view.id()].diagnostics,
                error_vis.FORMAT_LIBCLANG)
            self.error_vis.show_regions(view)

    def update(self, view, show_errors):
        """Reparse the translation unit. This speeds up completions
        significantly, so we perform this upon file save.

        Args:
            view (sublime.View): current view
            show_errors (bool): if true - highlight compile errors

        Returns:
            bool: reparsed successfully

        """
        if view.id() in self.translation_units:
            log.debug(" reparsing translation_unit for view %s", view.id())
            start = time.time()
            self.translation_units[view.id()].reparse()
            log.debug(" reparsed translation unit in %s seconds",
                      time.time() - start)
            if show_errors:
                self.error_vis.generate(
                    view, self.translation_units[view.id()].diagnostics,
                    error_vis.FORMAT_LIBCLANG)
                self.error_vis.show_regions(view)
            return True
        log.error(" no translation unit for view id %s")
        return False

    @staticmethod
    def _parse_completions(complete_results):
        """Create snippet-like structures from a list of completions

        Args:
            complete_results (list): raw completions list

        Returns:
            list: updated completions
        """
        completions = []
        for c in complete_results.results:
            hint = ''
            contents = ''
            place_holders = 1
            for chunk in c.string:
                hint += chunk.spelling
                if chunk.isKindTypedText():
                    trigger = chunk.spelling
                if chunk.isKindResultType():
                    hint += ' '
                    continue
                if chunk.isKindOptional():
                    continue
                if chunk.isKindInformative():
                    continue
                if chunk.isKindPlaceHolder():
                    contents += ('${' + str(place_holders) + ':' +
                                 chunk.spelling + '}')
                    place_holders += 1
                else:
                    contents += chunk.spelling
            completions.append([trigger + "\t" + hint, contents])
        return completions
<|endoftext|>


<commit_before>from corehq import toggles
from corehq.apps.app_manager import id_strings
from corehq.apps.app_manager.suite_xml.contributors import (
    SuiteContributorByModule,
)
from corehq.apps.app_manager.suite_xml.post_process.instances import (
    get_all_instances_referenced_in_xpaths,
)
from corehq.apps.app_manager.suite_xml.sections.details import DetailsHelper
from corehq.apps.app_manager.suite_xml.xml_models import (
    Command,
    Display,
    Hint,
    Instance,
    Itemset,
    PushFrame,
    QueryData,
    QueryPrompt,
    RemoteRequest,
    RemoteRequestPost,
    RemoteRequestQuery,
    RemoteRequestSession,
    SessionDatum,
    Stack,
    Text,
)
from corehq.apps.app_manager.util import module_offers_search
from corehq.apps.app_manager.xpath import CaseTypeXpath, InstanceXpath, interpolate_xpath
from corehq.apps.case_search.models import CASE_SEARCH_BLACKLISTED_OWNER_ID_KEY
from corehq.util.timer import time_method
from corehq.util.view_utils import absolute_reverse

RESULTS_INSTANCE = 'results'  # The name of the instance where search results are stored
SESSION_INSTANCE = 'commcaresession'

class QuerySessionXPath(InstanceXpath):
    id = SESSION_INSTANCE

    @property
    def path(self):
        return 'session/data/{}'.format(self)

class RemoteRequestFactory(object):
    def __init__(self, domain, app, module, detail_section_elements):
        self.domain = domain
        self.app = app
        self.module = module
        self.detail_section_elements = detail_section_elements

    def build_remote_request(self):
        return RemoteRequest(
            post=self._build_remote_request_post(),
            command=self._build_command(),
            instances=self._build_instances(),
            session=self._build_session(),
            stack=self._build_stack(),
        )

    def _build_remote_request_post(self):
        kwargs = {
            "url": absolute_reverse('claim_case', args=[self.domain]),
            "data": [
                QueryData(
                    key='case_id',
                    ref=QuerySessionXPath(self.module.search_config.case_session_var).instance(),
                ),
            ],
        }
        relevant = self.module.search_config.get_relevant()
        if relevant:
            kwargs["relevant"] = relevant
        return RemoteRequestPost(**kwargs)

    def _build_command(self):
        return Command(
            id=id_strings.search_command(self.module),
            display=Display(
                text=Text(locale_id=id_strings.case_search_locale(self.module)),
            ),
        )

    def _build_instances(self):
        prompt_select_instances = [
            Instance(id=prop.itemset.instance_id, src=prop.itemset.instance_uri)
            for prop in self.module.search_config.properties
            if prop.itemset.instance_id
        ]

        query_xpaths = [QuerySessionXPath(self.module.search_config.case_session_var).instance()]
        query_xpaths.extend([datum.ref for datum in self._get_remote_request_query_datums()])
        query_xpaths.extend([self.module.search_config.get_relevant(), self.module.search_config.search_filter])
        query_xpaths.extend([prop.default_value for prop in self.module.search_config.properties])
        instances, unknown_instances = get_all_instances_referenced_in_xpaths(self.app, query_xpaths)
        # we use the module's case list/details view to select the datum so also
        # need these instances to be available
        instances |= self._get_instances_for_module(self.app, self.module, self.detail_section_elements)

        # sorted list to prevent intermittent test failures
        return sorted(set(list(instances) + prompt_select_instances), key=lambda i: i.id)

    def _get_instances_for_module(self, app, module, detail_section_elements):
        helper = DetailsHelper(app)
        details = detail_section_elements
        detail_mapping = {detail.id: detail for detail in details}
        details_by_id = detail_mapping
        detail_ids = [helper.get_detail_id_safe(module, detail_type)
                    for detail_type, detail, enabled in module.get_details()
                    if enabled]
        detail_ids = [_f for _f in detail_ids if _f]
        xpaths = set()

        for detail_id in detail_ids:
            xpaths.update(details_by_id[detail_id].get_all_xpaths())

        instances, _ = get_all_instances_referenced_in_xpaths(app, xpaths)
        return instances

    def _build_session(self):
        return RemoteRequestSession(
            queries=self._build_remote_request_queries(),
            data=self._build_remote_request_datums(),
        )

    def _build_remote_request_queries(self):
        return [
            RemoteRequestQuery(
                url=absolute_reverse('app_aware_remote_search', args=[self.app.domain, self.app._id]),
                storage_instance=RESULTS_INSTANCE,
                template='case',
                data=self._get_remote_request_query_datums(),
                prompts=self._build_query_prompts(),
                default_search=self.module.search_config.default_search,
            )
        ]

    def _build_remote_request_datums(self):
        details_helper = DetailsHelper(self.app)
        if self.module.case_details.short.custom_xml:
            short_detail_id = 'case_short'
            long_detail_id = 'case_long'
        else:
            short_detail_id = 'search_short'
            long_detail_id = 'search_long'

        nodeset = CaseTypeXpath(self.module.case_type).case(instance_name=RESULTS_INSTANCE)
        if toggles.USH_CASE_CLAIM_UPDATES.enabled(self.app.domain):
            additional_types = list(set(self.module.search_config.additional_case_types) - {self.module.case_type})
            if additional_types:
                nodeset = CaseTypeXpath(self.module.case_type).cases(
                    additional_types, instance_name=RESULTS_INSTANCE)
            if self.module.search_config.search_filter:
                nodeset = f"{nodeset}[{interpolate_xpath(self.module.search_config.search_filter)}]"

        return [SessionDatum(
            id=self.module.search_config.case_session_var,
            nodeset=nodeset,
            value='./@case_id',
            detail_select=details_helper.get_detail_id_safe(self.module, short_detail_id),
            detail_confirm=details_helper.get_detail_id_safe(self.module, long_detail_id),
        )]

    def _get_remote_request_query_datums(self):
        default_query_datums = [
            QueryData(
                key='case_type',
                ref="'{}'".format(self.module.case_type)
            ),
        ]
        additional_types = list(set(self.module.search_config.additional_case_types) - {self.module.case_type})
        for type in additional_types:
            default_query_datums.append(
                QueryData(
                    key='case_type',
                    ref="'{}'".format(type)
                )
            )
        extra_query_datums = [
            QueryData(key="{}".format(c.property), ref="{}".format(c.defaultValue))
            for c in self.module.search_config.default_properties
        ]
        if self.module.search_config.blacklisted_owner_ids_expression:
            extra_query_datums.append(
                QueryData(
                    key=CASE_SEARCH_BLACKLISTED_OWNER_ID_KEY,
                    ref="{}".format(self.module.search_config.blacklisted_owner_ids_expression)
                )
            )
        return default_query_datums + extra_query_datums

    def _build_query_prompts(self):
        prompts = []
        for prop in self.module.search_config.properties:
            text = Text(locale_id=id_strings.search_property_locale(self.module, prop.name))
            if prop.hint:
                display = Display(
                    text=text,
                    hint=Hint(text=Text(locale_id=id_strings.search_property_hint_locale(self.module, prop.name)))
                )
            else:
                display = Display(text=text)

            kwargs = {
                'key': prop.name,
                'display': display
            }
            if not prop.appearance or prop.itemset.nodeset:
                kwargs['receive'] = prop.receiver_expression
            if prop.hidden:
                kwargs['hidden'] = prop.hidden
            if prop.appearance and self.app.enable_search_prompt_appearance:
                if prop.appearance == 'address':
                    kwargs['input_'] = prop.appearance
                else:
                    kwargs['appearance'] = prop.appearance
            if prop.input_:
                kwargs['input_'] = prop.input_
            if prop.default_value and self.app.enable_default_value_expression:
                kwargs['default_value'] = prop.default_value
            if prop.itemset.nodeset:
                kwargs['itemset'] = Itemset(
                    nodeset=prop.itemset.nodeset,
                    label_ref=prop.itemset.label,
                    value_ref=prop.itemset.value,
                    sort_ref=prop.itemset.sort,
                )
            prompts.append(QueryPrompt(**kwargs))
        return prompts

    def _build_stack(self):
        stack = Stack()
        frame = PushFrame()
        frame.add_rewind(QuerySessionXPath(self.module.search_config.case_session_var).instance())
        stack.add_frame(frame)
        return stack

class RemoteRequestContributor(SuiteContributorByModule):
    """
    Adds a remote-request node, which sets the URL and query details for
    synchronous searching and case claiming.

    Search is available from the module's case list.

    See "remote-request" in the `CommCare 2.0 Suite Definition`_ for details.

    .. _CommCare 2.0 Suite Definition: https://github.com/dimagi/commcare/wiki/Suite20#remote-request

    """

    @time_method()
    def get_module_contributions(self, module, detail_section_elements):
        if module_offers_search(module):
            return [RemoteRequestFactory(
                self.app.domain, self.app, module, detail_section_elements).build_remote_request()]
        return []
<commit_msg>share DetailsHelper instance<commit_after>from django.utils.functional import cached_property

from corehq import toggles
from corehq.apps.app_manager import id_strings
from corehq.apps.app_manager.suite_xml.contributors import (
    SuiteContributorByModule,
)
from corehq.apps.app_manager.suite_xml.post_process.instances import (
    get_all_instances_referenced_in_xpaths,
)
from corehq.apps.app_manager.suite_xml.sections.details import DetailsHelper
from corehq.apps.app_manager.suite_xml.xml_models import (
    Command,
    Display,
    Hint,
    Instance,
    Itemset,
    PushFrame,
    QueryData,
    QueryPrompt,
    RemoteRequest,
    RemoteRequestPost,
    RemoteRequestQuery,
    RemoteRequestSession,
    SessionDatum,
    Stack,
    Text,
)
from corehq.apps.app_manager.util import module_offers_search
from corehq.apps.app_manager.xpath import CaseTypeXpath, InstanceXpath, interpolate_xpath
from corehq.apps.case_search.models import CASE_SEARCH_BLACKLISTED_OWNER_ID_KEY
from corehq.util.timer import time_method
from corehq.util.view_utils import absolute_reverse

RESULTS_INSTANCE = 'results'  # The name of the instance where search results are stored
SESSION_INSTANCE = 'commcaresession'

class QuerySessionXPath(InstanceXpath):
    id = SESSION_INSTANCE

    @property
    def path(self):
        return 'session/data/{}'.format(self)

class RemoteRequestFactory(object):
    def __init__(self, domain, app, module, detail_section_elements):
        self.domain = domain
        self.app = app
        self.module = module
        self.detail_section_elements = detail_section_elements

    def build_remote_request(self):
        return RemoteRequest(
            post=self._build_remote_request_post(),
            command=self._build_command(),
            instances=self._build_instances(),
            session=self._build_session(),
            stack=self._build_stack(),
        )

    def _build_remote_request_post(self):
        kwargs = {
            "url": absolute_reverse('claim_case', args=[self.domain]),
            "data": [
                QueryData(
                    key='case_id',
                    ref=QuerySessionXPath(self.module.search_config.case_session_var).instance(),
                ),
            ],
        }
        relevant = self.module.search_config.get_relevant()
        if relevant:
            kwargs["relevant"] = relevant
        return RemoteRequestPost(**kwargs)

    def _build_command(self):
        return Command(
            id=id_strings.search_command(self.module),
            display=Display(
                text=Text(locale_id=id_strings.case_search_locale(self.module)),
            ),
        )

    def _build_instances(self):
        prompt_select_instances = [
            Instance(id=prop.itemset.instance_id, src=prop.itemset.instance_uri)
            for prop in self.module.search_config.properties
            if prop.itemset.instance_id
        ]

        query_xpaths = [QuerySessionXPath(self.module.search_config.case_session_var).instance()]
        query_xpaths.extend([datum.ref for datum in self._get_remote_request_query_datums()])
        query_xpaths.extend([self.module.search_config.get_relevant(), self.module.search_config.search_filter])
        query_xpaths.extend([prop.default_value for prop in self.module.search_config.properties])
        instances, unknown_instances = get_all_instances_referenced_in_xpaths(self.app, query_xpaths)
        # we use the module's case list/details view to select the datum so also
        # need these instances to be available
        instances |= self._get_instances_for_module()

        # sorted list to prevent intermittent test failures
        return sorted(set(list(instances) + prompt_select_instances), key=lambda i: i.id)

    @cached_property
    def _details_helper(self):
        return DetailsHelper(self.app)

    def _get_instances_for_module(self):
        details = self.detail_section_elements
        detail_mapping = {detail.id: detail for detail in details}
        details_by_id = detail_mapping
        detail_ids = [self._details_helper.get_detail_id_safe(self.module, detail_type)
                      for detail_type, detail, enabled in self.module.get_details()
                      if enabled]
        detail_ids = [_f for _f in detail_ids if _f]
        xpaths = set()

        for detail_id in detail_ids:
            xpaths.update(details_by_id[detail_id].get_all_xpaths())

        instances, _ = get_all_instances_referenced_in_xpaths(self.app, xpaths)
        return instances

    def _build_session(self):
        return RemoteRequestSession(
            queries=self._build_remote_request_queries(),
            data=self._build_remote_request_datums(),
        )

    def _build_remote_request_queries(self):
        return [
            RemoteRequestQuery(
                url=absolute_reverse('app_aware_remote_search', args=[self.app.domain, self.app._id]),
                storage_instance=RESULTS_INSTANCE,
                template='case',
                data=self._get_remote_request_query_datums(),
                prompts=self._build_query_prompts(),
                default_search=self.module.search_config.default_search,
            )
        ]

    def _build_remote_request_datums(self):
        if self.module.case_details.short.custom_xml:
            short_detail_id = 'case_short'
            long_detail_id = 'case_long'
        else:
            short_detail_id = 'search_short'
            long_detail_id = 'search_long'

        nodeset = CaseTypeXpath(self.module.case_type).case(instance_name=RESULTS_INSTANCE)
        if toggles.USH_CASE_CLAIM_UPDATES.enabled(self.app.domain):
            additional_types = list(set(self.module.search_config.additional_case_types) - {self.module.case_type})
            if additional_types:
                nodeset = CaseTypeXpath(self.module.case_type).cases(
                    additional_types, instance_name=RESULTS_INSTANCE)
            if self.module.search_config.search_filter:
                nodeset = f"{nodeset}[{interpolate_xpath(self.module.search_config.search_filter)}]"

        return [SessionDatum(
            id=self.module.search_config.case_session_var,
            nodeset=nodeset,
            value='./@case_id',
            detail_select=self._details_helper.get_detail_id_safe(self.module, short_detail_id),
            detail_confirm=self._details_helper.get_detail_id_safe(self.module, long_detail_id),
        )]

    def _get_remote_request_query_datums(self):
        default_query_datums = [
            QueryData(
                key='case_type',
                ref="'{}'".format(self.module.case_type)
            ),
        ]
        additional_types = list(set(self.module.search_config.additional_case_types) - {self.module.case_type})
        for type in additional_types:
            default_query_datums.append(
                QueryData(
                    key='case_type',
                    ref="'{}'".format(type)
                )
            )
        extra_query_datums = [
            QueryData(key="{}".format(c.property), ref="{}".format(c.defaultValue))
            for c in self.module.search_config.default_properties
        ]
        if self.module.search_config.blacklisted_owner_ids_expression:
            extra_query_datums.append(
                QueryData(
                    key=CASE_SEARCH_BLACKLISTED_OWNER_ID_KEY,
                    ref="{}".format(self.module.search_config.blacklisted_owner_ids_expression)
                )
            )
        return default_query_datums + extra_query_datums

    def _build_query_prompts(self):
        prompts = []
        for prop in self.module.search_config.properties:
            text = Text(locale_id=id_strings.search_property_locale(self.module, prop.name))
            if prop.hint:
                display = Display(
                    text=text,
                    hint=Hint(text=Text(locale_id=id_strings.search_property_hint_locale(self.module, prop.name)))
                )
            else:
                display = Display(text=text)

            kwargs = {
                'key': prop.name,
                'display': display
            }
            if not prop.appearance or prop.itemset.nodeset:
                kwargs['receive'] = prop.receiver_expression
            if prop.hidden:
                kwargs['hidden'] = prop.hidden
            if prop.appearance and self.app.enable_search_prompt_appearance:
                if prop.appearance == 'address':
                    kwargs['input_'] = prop.appearance
                else:
                    kwargs['appearance'] = prop.appearance
            if prop.input_:
                kwargs['input_'] = prop.input_
            if prop.default_value and self.app.enable_default_value_expression:
                kwargs['default_value'] = prop.default_value
            if prop.itemset.nodeset:
                kwargs['itemset'] = Itemset(
                    nodeset=prop.itemset.nodeset,
                    label_ref=prop.itemset.label,
                    value_ref=prop.itemset.value,
                    sort_ref=prop.itemset.sort,
                )
            prompts.append(QueryPrompt(**kwargs))
        return prompts

    def _build_stack(self):
        stack = Stack()
        frame = PushFrame()
        frame.add_rewind(QuerySessionXPath(self.module.search_config.case_session_var).instance())
        stack.add_frame(frame)
        return stack

class RemoteRequestContributor(SuiteContributorByModule):
    """
    Adds a remote-request node, which sets the URL and query details for
    synchronous searching and case claiming.

    Search is available from the module's case list.

    See "remote-request" in the `CommCare 2.0 Suite Definition`_ for details.

    .. _CommCare 2.0 Suite Definition: https://github.com/dimagi/commcare/wiki/Suite20#remote-request

    """

    @time_method()
    def get_module_contributions(self, module, detail_section_elements):
        if module_offers_search(module):
            return [RemoteRequestFactory(
                self.app.domain, self.app, module, detail_section_elements).build_remote_request()]
        return []
<|endoftext|>


<commit_before>import boto3, botocore, datetime, hashlib, json, os, re, time
from helpers import *

def uploadCloudFrontCertificates(domain_objects, certificate_path):
    logMessage("Starting CloudFront upload process")
    for primary_domain, config in domain_objects.iteritems():
        logMessage(primary_domain + ": Starting upload check")
        is_uploaded = False
        if config['distribution_id']:
            logMessage(primary_domain + ": Distribution_id set")
            certificate_hash = generateCloudFrontHash(primary_domain, certificate_path)
            uploaded_certificates = listCertificates(primary_domain)

            if uploaded_certificates:
                logMessage(primary_domain + ": Checking if SSL has been uploaded")
                for uploaded_certificate in uploaded_certificates:
                    uploaded = re.search(certificate_hash, uploaded_certificate['ServerCertificateName'])

                if not uploaded:
                    logMessage(primary_domain + ": Uploading SSL")
                    upload_result = uploadCertificate(primary_domain, certificate_path)
                else:
                    logMessage(primary_domain + ": SSL already exists in IAM")
            else:
                logMessage(primary_domain + ": Uploading SSL")
                upload_result = uploadCertificate(primary_domain, certificate_path)
        logMessage(primary_domain + ": Finished upload process")

    return True

def updateCloudFrontDistributions(domain_objects, certificate_path):
    for primary_domain, config in domain_objects.iteritems():
        latest_certificate = getLastestCertificate(primary_domain)
        active_certificate = getActiveCertficateID(config['distribution_id'])

        if latest_certificate['id'] != active_certificate:
            updated = updateDistributionCertificate(config['distribution_id'], latest_certificate['name'])
            if not updated:
                logError("Unable to update certificate for" + primary_domain)
    return True

def listCertificates(primary_domain):
    iam_client = createAWSClient('iam')
    response = iam_client.list_server_certificates(
        PathPrefix='/cloudfront/' + primary_domain + '/'
    )

    if response['ServerCertificateMetadataList']:
        return response['ServerCertificateMetadataList']
    else:
        return False

def uploadCertificate(primary_domain, certificate_path):
    iam_client = createAWSClient('iam')
    primary_path = certificate_path + '/' + primary_domain

    if os.path.isfile(primary_path + '/cert.pem'):
        with open(certificate_path + '/' + primary_domain + '/cert.pem') as cert_file:
            cert = cert_file.read()
            cert_hash = hashlib.md5(cert).hexdigest()
    else:
        logError('Cert file does not exist.')
        return False

    if os.path.isfile(primary_path + '/privkey.pem'):
        with open(certificate_path + '/' + primary_domain + '/privkey.pem') as cert_file:
            privkey = cert_file.read()
    else:
        logError('Private key file does not exist.')
        return False

    if os.path.isfile(primary_path + '/chain.pem'):
        with open(certificate_path + '/' + primary_domain + '/chain.pem') as cert_file:
            chain = cert_file.read()
    else:
        logError('Chain file does not exist.')
        return False

    try:
        response = iam_client.upload_server_certificate(
            Path='/cloudfront/' + primary_domain + '/',
            ServerCertificateName=primary_domain + '-' + cert_hash,
            CertificateBody=cert,
            PrivateKey=privkey,
            CertificateChain=chain
        )
        return response['ServerCertificateId']
    except botocore.exceptions.ClientError as e:
        if e.response['Error']['Code'] == 'EntityAlreadyExists':
            logError("Certificate already exists")
        return False

def deleteCertificate(server_certificate_id):
    iam_client = createAWSClient('iam')

    try:
        response = iam_client.delete_server_certificate(
            ServerCertificateName=server_certificate_id
        )
        return True
    except botocore.exceptions.ClientError as e:
        return False

def updateDistributionCertificate(distribution_id, server_certificate_name):
    cloudfront_client = createAWSClient('cloudfront')
    iam_client = createAWSClient('iam')

    try:
        certificate = iam_client.get_server_certificate(ServerCertificateName=server_certificate_name)
        distribution = cloudfront_client.get_distribution(Id=distribution_id)
    except botocore.exceptions.ClientError as e:
        return False

    certificate_id = certificate['ServerCertificate']['ServerCertificateMetadata']['ServerCertificateId']

    if distribution['Distribution']:
        distribution_config = distribution['Distribution']['DistributionConfig']
        if distribution_config['ViewerCertificate']['IAMCertificateId']:
            if distribution_config['ViewerCertificate']['IAMCertificateId'] == certificate_id:
                return True
        distribution_config['ViewerCertificate']['IAMCertificateId'] = certificate_id
        distribution_config['ViewerCertificate']['Certificate'] = certificate_id
        distribution_config['ViewerCertificate']['CertificateSource'] = 'iam'
        distribution_config['ViewerCertificate']['MinimumProtocolVersion'] = 'TLSv1'
        distribution_config['ViewerCertificate']['SSLSupportMethod'] = 'sni-only'

    try:
        response = cloudfront_client.update_distribution(
            DistributionConfig=distribution_config,
            Id=distribution_id,
            IfMatch=distribution['ETag'],
        )
        return True
    except botocore.exceptions.ClientError as e:
        return False

    #TODO: verify that the domains match before enabling

def getActiveCertficateID(distribution_id):
    cloudfront_client = createAWSClient('cloudfront')

    try:
        distribution = cloudfront_client.get_distribution(Id=distribution_id)
    except botocore.exceptions.ClientError as e:
        return False

    distribution_config = distribution['Distribution']['DistributionConfig']
    if distribution_config['ViewerCertificate']['IAMCertificateId']:
        certificate_id = distribution_config['ViewerCertificate']['IAMCertificateId']
        return certificate_id
    else:
        return False

def getCertificateName(primary_domain, certificate_id):
    certificates = listCertificates(primary_domain)
    certificate_name = False

    if certificates:
        for certificate in certificates:
            if certificate['ServerCertificateId'] == certificate_id:
                certificate_name = certificate['ServerCertificateName']
                break
    return certificate_name

def getLastestCertificate(primary_domain):
    latest_time = 0
    iam_client = createAWSClient('iam')

    response = iam_client.list_server_certificates(
        PathPrefix='/cloudfront/' + primary_domain + '/'
    )

    if response['ServerCertificateMetadataList']:
        for certificate in response['ServerCertificateMetadataList']:
            certificate_time = certificate['Expiration'].strftime("%s")
            if certificate_time > latest_time:
                latest_time = certificate_time
                latest_id = certificate['ServerCertificateId']
                latest_name = certificate['ServerCertificateName']
        certificate = {}
        certificate['id'] = latest_id
        certificate['name'] = latest_name
        return certificate
    else:
        return False

#def addWellKnownOrigin(distribution_id):
<commit_msg>add missing ServerCertificateMetadata key on upload response<commit_after>import boto3, botocore, datetime, hashlib, json, os, re, time
from helpers import *

def uploadCloudFrontCertificates(domain_objects, certificate_path):
    logMessage("Starting CloudFront upload process")
    for primary_domain, config in domain_objects.iteritems():
        logMessage(primary_domain + ": Starting upload check")
        is_uploaded = False
        if config['distribution_id']:
            logMessage(primary_domain + ": Distribution_id set")
            certificate_hash = generateCloudFrontHash(primary_domain, certificate_path)
            uploaded_certificates = listCertificates(primary_domain)

            if uploaded_certificates:
                logMessage(primary_domain + ": Checking if SSL has been uploaded")
                for uploaded_certificate in uploaded_certificates:
                    uploaded = re.search(certificate_hash, uploaded_certificate['ServerCertificateName'])

                if not uploaded:
                    logMessage(primary_domain + ": Uploading SSL")
                    upload_result = uploadCertificate(primary_domain, certificate_path)
                else:
                    logMessage(primary_domain + ": SSL already exists in IAM")
            else:
                logMessage(primary_domain + ": Uploading SSL")
                upload_result = uploadCertificate(primary_domain, certificate_path)
        logMessage(primary_domain + ": Finished upload process")

    return True

def updateCloudFrontDistributions(domain_objects, certificate_path):
    for primary_domain, config in domain_objects.iteritems():
        latest_certificate = getLastestCertificate(primary_domain)
        active_certificate = getActiveCertficateID(config['distribution_id'])

        if latest_certificate['id'] != active_certificate:
            updated = updateDistributionCertificate(config['distribution_id'], latest_certificate['name'])
            if not updated:
                logError("Unable to update certificate for" + primary_domain)
    return True

def listCertificates(primary_domain):
    iam_client = createAWSClient('iam')
    response = iam_client.list_server_certificates(
        PathPrefix='/cloudfront/' + primary_domain + '/'
    )

    if response['ServerCertificateMetadataList']:
        return response['ServerCertificateMetadataList']
    else:
        return False

def uploadCertificate(primary_domain, certificate_path):
    iam_client = createAWSClient('iam')
    primary_path = certificate_path + '/' + primary_domain

    if os.path.isfile(primary_path + '/cert.pem'):
        with open(certificate_path + '/' + primary_domain + '/cert.pem') as cert_file:
            cert = cert_file.read()
            cert_hash = hashlib.md5(cert).hexdigest()
    else:
        logError('Cert file does not exist.')
        return False

    if os.path.isfile(primary_path + '/privkey.pem'):
        with open(certificate_path + '/' + primary_domain + '/privkey.pem') as cert_file:
            privkey = cert_file.read()
    else:
        logError('Private key file does not exist.')
        return False

    if os.path.isfile(primary_path + '/chain.pem'):
        with open(certificate_path + '/' + primary_domain + '/chain.pem') as cert_file:
            chain = cert_file.read()
    else:
        logError('Chain file does not exist.')
        return False

    try:
        response = iam_client.upload_server_certificate(
            Path='/cloudfront/' + primary_domain + '/',
            ServerCertificateName=primary_domain + '-' + cert_hash,
            CertificateBody=cert,
            PrivateKey=privkey,
            CertificateChain=chain
        )
        return response['ServerCertificateMetadata']['ServerCertificateId']
    except botocore.exceptions.ClientError as e:
        if e.response['Error']['Code'] == 'EntityAlreadyExists':
            logError("Certificate already exists")
        return False

def deleteCertificate(server_certificate_id):
    iam_client = createAWSClient('iam')

    try:
        response = iam_client.delete_server_certificate(
            ServerCertificateName=server_certificate_id
        )
        return True
    except botocore.exceptions.ClientError as e:
        return False

def updateDistributionCertificate(distribution_id, server_certificate_name):
    cloudfront_client = createAWSClient('cloudfront')
    iam_client = createAWSClient('iam')

    try:
        certificate = iam_client.get_server_certificate(ServerCertificateName=server_certificate_name)
        distribution = cloudfront_client.get_distribution(Id=distribution_id)
    except botocore.exceptions.ClientError as e:
        return False

    certificate_id = certificate['ServerCertificate']['ServerCertificateMetadata']['ServerCertificateId']

    if distribution['Distribution']:
        distribution_config = distribution['Distribution']['DistributionConfig']
        if distribution_config['ViewerCertificate']['IAMCertificateId']:
            if distribution_config['ViewerCertificate']['IAMCertificateId'] == certificate_id:
                return True
        distribution_config['ViewerCertificate']['IAMCertificateId'] = certificate_id
        distribution_config['ViewerCertificate']['Certificate'] = certificate_id
        distribution_config['ViewerCertificate']['CertificateSource'] = 'iam'
        distribution_config['ViewerCertificate']['MinimumProtocolVersion'] = 'TLSv1'
        distribution_config['ViewerCertificate']['SSLSupportMethod'] = 'sni-only'

    try:
        response = cloudfront_client.update_distribution(
            DistributionConfig=distribution_config,
            Id=distribution_id,
            IfMatch=distribution['ETag'],
        )
        return True
    except botocore.exceptions.ClientError as e:
        return False

    #TODO: verify that the domains match before enabling

def getActiveCertficateID(distribution_id):
    cloudfront_client = createAWSClient('cloudfront')

    try:
        distribution = cloudfront_client.get_distribution(Id=distribution_id)
    except botocore.exceptions.ClientError as e:
        return False

    distribution_config = distribution['Distribution']['DistributionConfig']
    if distribution_config['ViewerCertificate']['IAMCertificateId']:
        certificate_id = distribution_config['ViewerCertificate']['IAMCertificateId']
        return certificate_id
    else:
        return False

def getCertificateName(primary_domain, certificate_id):
    certificates = listCertificates(primary_domain)
    certificate_name = False

    if certificates:
        for certificate in certificates:
            if certificate['ServerCertificateId'] == certificate_id:
                certificate_name = certificate['ServerCertificateName']
                break
    return certificate_name

def getLastestCertificate(primary_domain):
    latest_time = 0
    iam_client = createAWSClient('iam')

    response = iam_client.list_server_certificates(
        PathPrefix='/cloudfront/' + primary_domain + '/'
    )

    if response['ServerCertificateMetadataList']:
        for certificate in response['ServerCertificateMetadataList']:
            certificate_time = certificate['Expiration'].strftime("%s")
            if certificate_time > latest_time:
                latest_time = certificate_time
                latest_id = certificate['ServerCertificateId']
                latest_name = certificate['ServerCertificateName']
        certificate = {}
        certificate['id'] = latest_id
        certificate['name'] = latest_name
        return certificate
    else:
        return False

#def addWellKnownOrigin(distribution_id):
<|endoftext|>


<commit_before># coding: UTF-8
u"""Module for a URI query parser.

It has dictionary like access. While this class will accept more than one
value per key (a key can be repeated) the dictionary methods will tend to
return only the first of those values. Care has been taken, however, to always
maintain the order of the multiple values.

NOTE that I do NOT treat '+' as a space! It will stay a '+'. Spaces MUST be
percent encoded to be picked up.

TODO: make sure that is what jquery does.

Instantiate with a string:

    >>> query = Query('key1=value1&key2=value2')
    >>> query.items()
    [(u'key1', u'value1'), (u'key2', u'value2')]

Instantiate with a dict:

    >>> query = Query({u'a': 1, u'b': 2})
    >>> sorted(query.allitems())
    [(u'a', u'1'), (u'b', u'2')]

Instantiate with a list of pairs:

    >>> query = Query([('one', u'1'), ('two', u'2')])
    >>> query.allitems()
    [(u'one', u'1'), (u'two', u'2')]

Cast back to a string:
    >>> str(query)
    'one=1&two=2'

It can deal with multiple values per key (although by all the normal dict methods it does not appear this way).:

    >>> query = Query('key=value1&key=value2')
    >>> query['key']
    u'value1'

    >>> query.list('key')
    [u'value1', u'value2']

    >>> query.allitems()
    [(u'key', u'value1'), (u'key', u'value2')]

Order is maintained very precisely, even with multiple values per key:

    >>> query = Query('a=1&b=2&a=3')
    >>> query.allitems()
    [(u'a', u'1'), (u'b', u'2'), (u'a', u'3')]

Setting is a little more difficult. Assume that unless something that extends a tuple or list (ie passes isinstance(input, (tuple, list))) it is supposed to be the ONLY string value for that key.

    >>> # Notice that we are still using the query with multiple values for a.
    >>> query.list(u'a')
    [u'1', u'3']
    >>> query[u'a'] = 'value'
    >>> query.list(u'a')
    [u'value']

Setting a list:

    >>> query['key'] = 'a b c'.split()
    >>> query.list('key')
    [u'a', u'b', u'c']

You can provide a sequence that is not a tuple or list by using the setlist method. This will remove all existing pairs by key, and append the new ones on the end of the query.

    >>> def g():
    ... 	for x in [1, 2, 3]:
    ... 		yield x
    >>> query.setlist('key', g())
    >>> query.list('key')
    [u'1', u'2', u'3']

The query can be sorted via list.sort (notice that the key function is passed a key/value tuple):

    >>> query = Query('a=1&c=2&b=3')
    >>> query.sort()
    >>> query.allitems()
    [(u'a', u'1'), (u'b', u'3'), (u'c', u'2')]
    >>> query.sort(key=lambda x: -ord(x[0]))
    >>> query.allitems()
    [(u'c', u'2'), (u'b', u'3'), (u'a', u'1')]

Pairs can be appended with list.append and list.insert and list.extend. (We will just cast the values to tuples and assert they have length 2.)

    >>> query = Query()
    >>> query.append((u'a', u'1'))
    >>> query.allitems()
    [(u'a', u'1')]

    >>> query.extend(sorted({u'b': 2, u'c': 3}.items()))
    >>> query.allitems()
    [(u'a', u'1'), (u'b', u'2'), (u'c', u'3')]

    >>> query.insert(0, ('z', '-1'))
    >>> query.allitems()
    [(u'z', u'-1'), (u'a', u'1'), (u'b', u'2'), (u'c', u'3')]

You can update the query via dict.update:

    >>> query = Query()
    >>> query.update({u'a': 1, u'b': [2, 3], u'c': u'4'})
    >>> query.sort() # Because the dictionary does not come in order.
    >>> query.allitems()
    [(u'a', u'1'), (u'b', u'2'), (u'b', u'3'), (u'c', u'4')]

    >>> query.update({u'b': '2/3'})
    >>> query.sort() # Dicts are not ordered!
    >>> query.allitems()
    [(u'a', u'1'), (u'b', u'2/3'), (u'c', u'4')]

Empty queries test as false:
    >>> bool(Query())
    False
    >>> bool(Query('a=b'))
    True

Empty queries are empty:
    >>> query = Query('')
    >>> query
    <uri.Query:[]>
    >>> print query
    <BLANKLINE>

Parse and unparses properly

    >>> parse('key=value')
    [(u'key', u'value')]
    >>> parse('key=')
    [(u'key', u'')]
    >>> parse('key')
    [(u'key', None)]

    >>> query = Query('key')
    >>> query['a'] = None
    >>> query
    <uri.Query:[(u'key', None), (u'a', None)]>
    >>> str(query)
    'key&a'

    >>> query = Query('key=value/with/slashes.and.dots=and=equals')
    >>> query
    <uri.Query:[(u'key', u'value/with/slashes.and.dots=and=equals')]>
    >>> str(query)
    'key=value/with/slashes.and.dots=and=equals'

Unicode does work properly.
    >>> query = Query('k%C3%A9y=%C2%A1%E2%84%A2%C2%A3%C2%A2%E2%88%9E%C2%A7%C2%B6%E2%80%A2%C2%AA%C2%BA')
    >>> print query.keys()[0]
    kéy
    >>> print query[u'kéy']
    ¡™£¢∞§¶•ªº

    >>> query = Query()
    >>> query[u'kéy'] = u'¡™£¢∞§¶•ªº'
    >>> print query
    k%C3%A9y=%C2%A1%E2%84%A2%C2%A3%C2%A2%E2%88%9E%C2%A7%C2%B6%E2%80%A2%C2%AA%C2%BA

Easy signatures!

# >>> query = Query('v=value')
# >>> # query.add_nonce(u'n', length=8)
# >>> query[u'n'] = '12345'
# >>> query.sign_with_md5(hmac_key='this is the key', sig_key=u's')
# >>> str(query)
# [(u'v', 'value'), (u'n', '12345'), (u's', 'md5signgoesinhere')]
#
# >>> query.sign('this is the key')
# True
# >>> query.verify_signature('this is not the key')
# False

"""

import collections
from transcode import *

def parse(query):
    ret = []
    if not query:
        return ret
    for pair in query.split(u'&'):
        pair = [decode(x) for x in pair.split(u'=', 1)]
        if isinstance(pair[0], str):
            pair = [unicoder(x) for x in pair]
        if len(pair) == 1:
            pair.append(None)
        ret.append(tuple(pair))
    return ret

def unparse(pairs):
    ret = []
    for pair in pairs:
        if pair[1] is None:
            ret.append(encode(pair[0], u'/'))
        else:
            ret.append(encode(pair[0], u'/') + u'=' + encode(pair[1], '/='))
    return u'&'.join(ret)

class Query(object):

    def __init__(self, input=None):
        self._pairs = []
        if isinstance(input, basestring):
            self._pairs = parse(input)
        elif isinstance(input, collections.Mapping):
            self.update(input)
        elif input is not None:
            self.extend(input)

    def __str__(self):
        return unparse(self._pairs)

    def __repr__(self):
        return '<uri.Query:%r>' % self._pairs

    def __nonzero__(self):
        return len(self._pairs)

    def __getitem__(self, key):
        key = unicoder(key)
        for x in self._pairs:
            if x[0] == key:
                return x[1]

    def list(self, key):
        key = unicoder(key)
        return [x[1] for x in self._pairs if x[0] == key]

    def iteritems(self):
        keys_yielded = set()
        for k, v in self._pairs:
            if k not in keys_yielded:
                keys_yielded.add(k)
                yield k, v

    def __iter__(self):
        return (x[0] for x in self.iteritems())

    def keys(self):
        return list(self)

    def iterkeys(self):
        return iter(self)

    def items(self):
        return list(self.iteritems())

    def itervalues(self):
        return (x[1] for x in self.iteritems())

    def values(self):
        return list(self.itervalues())

    def iterallitems(self):
        return iter(self._pairs)

    def allitems(self):
        return self._pairs[:]

    def __delitem__(self, key):
        key = unicoder(key)
        self._pairs = [x for x in self._pairs if x[0] != key]

    def __setitem__(self, key, value):
        key = unicoder(key)
        if isinstance(value, (tuple, list)):
            self.setlist(key, value)
        else:
            del self[key]
            self._pairs.append((unicoder(key), self._conform_value(value)))

    def setlist(self, key, value):
        key = unicoder(key)
        del self[key]
        for v in value:
            self._pairs.append((key, self._conform_value(v)))

    def sort(self, *args, **kwargs):
        self._pairs.sort(*args, **kwargs)

    def _conform_pair(self, pair):
        pair = tuple(pair)
        if len(pair) != 2:
            raise ValueError('Pair must be length 2.')
        return (unicoder(pair[0]), self._conform_value(pair[1]))

    def _conform_value(self, value):
        if value is None:
            return None
        return unicoder(value)

    def append(self, pair):
        self._pairs.append(self._conform_pair(pair))

    def extend(self, pairs):
        self._pairs.extend(self._conform_pair(x) for x in pairs)

    def insert(self, index, pair):
        self._pairs.insert(index, self._conform_pair(pair))

    def update(self, mapping):
        for k, v in mapping.iteritems():
            del self[k]
            self[k] = v

if __name__ == '__main__':
    import doctest
    print 'Testing', __file__
    doctest.testmod()
    print 'Done.'<commit_msg>Query now parses + as a space.<commit_after># coding: UTF-8
u"""Module for a URI query parser.

It has dictionary like access. While this class will accept more than one
value per key (a key can be repeated) the dictionary methods will tend to
return only the first of those values. Care has been taken, however, to always
maintain the order of the multiple values.

I need to decode + as a space, because that is what firefox does to me. I
won't, however, re-encode it as a space. It feels wrong to me.

TODO: make sure that is what jquery does.

Instantiate with a string:

    >>> query = Query('key1=value1&key2=value2')
    >>> query.items()
    [(u'key1', u'value1'), (u'key2', u'value2')]

Instantiate with a dict:

    >>> query = Query({u'a': 1, u'b': 2})
    >>> sorted(query.allitems())
    [(u'a', u'1'), (u'b', u'2')]

Instantiate with a list of pairs:

    >>> query = Query([('one', u'1'), ('two', u'2')])
    >>> query.allitems()
    [(u'one', u'1'), (u'two', u'2')]

Cast back to a string:
    >>> str(query)
    'one=1&two=2'

It can deal with multiple values per key (although by all the normal dict methods it does not appear this way).:

    >>> query = Query('key=value1&key=value2')
    >>> query['key']
    u'value1'

    >>> query.list('key')
    [u'value1', u'value2']

    >>> query.allitems()
    [(u'key', u'value1'), (u'key', u'value2')]

Order is maintained very precisely, even with multiple values per key:

    >>> query = Query('a=1&b=2&a=3')
    >>> query.allitems()
    [(u'a', u'1'), (u'b', u'2'), (u'a', u'3')]

Setting is a little more difficult. Assume that unless something that extends a tuple or list (ie passes isinstance(input, (tuple, list))) it is supposed to be the ONLY string value for that key.

    >>> # Notice that we are still using the query with multiple values for a.
    >>> query.list(u'a')
    [u'1', u'3']
    >>> query[u'a'] = 'value'
    >>> query.list(u'a')
    [u'value']

Setting a list:

    >>> query['key'] = 'a b c'.split()
    >>> query.list('key')
    [u'a', u'b', u'c']

You can provide a sequence that is not a tuple or list by using the setlist method. This will remove all existing pairs by key, and append the new ones on the end of the query.

    >>> def g():
    ... 	for x in [1, 2, 3]:
    ... 		yield x
    >>> query.setlist('key', g())
    >>> query.list('key')
    [u'1', u'2', u'3']

The query can be sorted via list.sort (notice that the key function is passed a key/value tuple):

    >>> query = Query('a=1&c=2&b=3')
    >>> query.sort()
    >>> query.allitems()
    [(u'a', u'1'), (u'b', u'3'), (u'c', u'2')]
    >>> query.sort(key=lambda x: -ord(x[0]))
    >>> query.allitems()
    [(u'c', u'2'), (u'b', u'3'), (u'a', u'1')]

Pairs can be appended with list.append and list.insert and list.extend. (We will just cast the values to tuples and assert they have length 2.)

    >>> query = Query()
    >>> query.append((u'a', u'1'))
    >>> query.allitems()
    [(u'a', u'1')]

    >>> query.extend(sorted({u'b': 2, u'c': 3}.items()))
    >>> query.allitems()
    [(u'a', u'1'), (u'b', u'2'), (u'c', u'3')]

    >>> query.insert(0, ('z', '-1'))
    >>> query.allitems()
    [(u'z', u'-1'), (u'a', u'1'), (u'b', u'2'), (u'c', u'3')]

You can update the query via dict.update:

    >>> query = Query()
    >>> query.update({u'a': 1, u'b': [2, 3], u'c': u'4'})
    >>> query.sort() # Because the dictionary does not come in order.
    >>> query.allitems()
    [(u'a', u'1'), (u'b', u'2'), (u'b', u'3'), (u'c', u'4')]

    >>> query.update({u'b': '2/3'})
    >>> query.sort() # Dicts are not ordered!
    >>> query.allitems()
    [(u'a', u'1'), (u'b', u'2/3'), (u'c', u'4')]

Empty queries test as false:
    >>> bool(Query())
    False
    >>> bool(Query('a=b'))
    True

Empty queries are empty:
    >>> query = Query('')
    >>> query
    <uri.Query:[]>
    >>> print query
    <BLANKLINE>

Parse and unparses properly

    >>> parse('key=value')
    [(u'key', u'value')]
    >>> parse('key=')
    [(u'key', u'')]
    >>> parse('key')
    [(u'key', None)]

    >>> query = Query('key')
    >>> query['a'] = None
    >>> query
    <uri.Query:[(u'key', None), (u'a', None)]>
    >>> str(query)
    'key&a'

    >>> query = Query('key=value/with/slashes.and.dots=and=equals')
    >>> query
    <uri.Query:[(u'key', u'value/with/slashes.and.dots=and=equals')]>
    >>> str(query)
    'key=value/with/slashes.and.dots=and=equals'

Unicode does work properly.
    >>> query = Query('k%C3%A9y=%C2%A1%E2%84%A2%C2%A3%C2%A2%E2%88%9E%C2%A7%C2%B6%E2%80%A2%C2%AA%C2%BA')
    >>> print query.keys()[0]
    kéy
    >>> print query[u'kéy']
    ¡™£¢∞§¶•ªº

    >>> query = Query()
    >>> query[u'kéy'] = u'¡™£¢∞§¶•ªº'
    >>> print query
    k%C3%A9y=%C2%A1%E2%84%A2%C2%A3%C2%A2%E2%88%9E%C2%A7%C2%B6%E2%80%A2%C2%AA%C2%BA

Spaces as pluses:
    >>> query = Query('key+with+spaces=value+with+spaces')
    >>> query
    <uri.Query:[(u'key with spaces', u'value with spaces')]>
    >>> str(query)
    'key%20with%20spaces=value%20with%20spaces'

Easy signatures!

# >>> query = Query('v=value')
# >>> # query.add_nonce(u'n', length=8)
# >>> query[u'n'] = '12345'
# >>> query.sign_with_md5(hmac_key='this is the key', sig_key=u's')
# >>> str(query)
# [(u'v', 'value'), (u'n', '12345'), (u's', 'md5signgoesinhere')]
#
# >>> query.sign('this is the key')
# True
# >>> query.verify_signature('this is not the key')
# False

"""

import collections
from transcode import *

def parse(query):
    ret = []
    if not query:
        return ret
    for pair in query.split(u'&'):
        pair = [decode(x.replace('+', ' ')) for x in pair.split(u'=', 1)]
        if isinstance(pair[0], str):
            pair = [unicoder(x) for x in pair]
        if len(pair) == 1:
            pair.append(None)
        ret.append(tuple(pair))
    return ret

def unparse(pairs):
    ret = []
    for pair in pairs:
        if pair[1] is None:
            ret.append(encode(pair[0], u'/'))
        else:
            ret.append(encode(pair[0], u'/') + u'=' + encode(pair[1], '/='))
    return u'&'.join(ret)

class Query(object):

    def __init__(self, input=None):
        self._pairs = []
        if isinstance(input, basestring):
            self._pairs = parse(input)
        elif isinstance(input, collections.Mapping):
            self.update(input)
        elif input is not None:
            self.extend(input)

    def __str__(self):
        return unparse(self._pairs)

    def __repr__(self):
        return '<uri.Query:%r>' % self._pairs

    def __nonzero__(self):
        return len(self._pairs)

    def __getitem__(self, key):
        key = unicoder(key)
        for x in self._pairs:
            if x[0] == key:
                return x[1]

    def list(self, key):
        key = unicoder(key)
        return [x[1] for x in self._pairs if x[0] == key]

    def iteritems(self):
        keys_yielded = set()
        for k, v in self._pairs:
            if k not in keys_yielded:
                keys_yielded.add(k)
                yield k, v

    def __iter__(self):
        return (x[0] for x in self.iteritems())

    def keys(self):
        return list(self)

    def iterkeys(self):
        return iter(self)

    def items(self):
        return list(self.iteritems())

    def itervalues(self):
        return (x[1] for x in self.iteritems())

    def values(self):
        return list(self.itervalues())

    def iterallitems(self):
        return iter(self._pairs)

    def allitems(self):
        return self._pairs[:]

    def __delitem__(self, key):
        key = unicoder(key)
        self._pairs = [x for x in self._pairs if x[0] != key]

    def __setitem__(self, key, value):
        key = unicoder(key)
        if isinstance(value, (tuple, list)):
            self.setlist(key, value)
        else:
            del self[key]
            self._pairs.append((unicoder(key), self._conform_value(value)))

    def setlist(self, key, value):
        key = unicoder(key)
        del self[key]
        for v in value:
            self._pairs.append((key, self._conform_value(v)))

    def sort(self, *args, **kwargs):
        self._pairs.sort(*args, **kwargs)

    def _conform_pair(self, pair):
        pair = tuple(pair)
        if len(pair) != 2:
            raise ValueError('Pair must be length 2.')
        return (unicoder(pair[0]), self._conform_value(pair[1]))

    def _conform_value(self, value):
        if value is None:
            return None
        return unicoder(value)

    def append(self, pair):
        self._pairs.append(self._conform_pair(pair))

    def extend(self, pairs):
        self._pairs.extend(self._conform_pair(x) for x in pairs)

    def insert(self, index, pair):
        self._pairs.insert(index, self._conform_pair(pair))

    def update(self, mapping):
        for k, v in mapping.iteritems():
            del self[k]
            self[k] = v

if __name__ == '__main__':
    import doctest
    print 'Testing', __file__
    doctest.testmod()
    print 'Done.'<|endoftext|>