<commit_before>import os
import re
import sys
import logging

from coalib.collecting.Collectors import (
    collect_all_bears_from_sections, filter_section_bears_by_languages)
from coalib.misc import Constants
from coalib.output.ConfWriter import ConfWriter
from coalib.output.printers.LOG_LEVEL import LOG_LEVEL
from coalib.parsing.CliParsing import parse_cli, check_conflicts
from coalib.parsing.ConfParser import ConfParser
from coalib.settings.Section import Section, extract_aspects_from_section
from coalib.settings.SectionFilling import fill_settings
from coalib.settings.Setting import Setting, path
from string import Template

COAFILE_OUTPUT = Template('$type \'$file\' $found!\n'
                          'Here\'s what you can do:\n'
                          '* add `--save` to generate a config file with '
                          'your current options\n'
                          '* add `-I` to suppress any use of config files\n')

def aspectize_sections(sections):
    """
    Search for aspects related setting in a section, initialize it, and then
    embed the aspects information as AspectList object into the section itself.

    :param sections:  List of section that potentially contain aspects setting.
    :return:          The new sections.
    """
    for section_name, section in sections.items():
        section.aspects = extract_aspects_from_section(section)
        if section.aspects is not None and len(section.get('bears')):
            logging.warning("'aspects' and 'bears' configuration is detected "
                            "in section '{}'. Aspect-based configuration will "
                            'takes priority and will overwrite any '
                            'explicitly listed bears'.format(section_name))
    return sections

def merge_section_dicts(lower, higher):
    """
    Merges the section dictionaries. The values of higher will take
    precedence over the ones of lower. Lower will hold the modified dict in
    the end.

    :param lower:  A section.
    :param higher: A section which values will take precedence over the ones
                   from the other.
    :return:       The merged dict.
    """
    for name in higher:
        if name in lower:
            lower[name].update(higher[name], ignore_defaults=True)
        else:
            # no deep copy needed
            lower[name] = higher[name]

    return lower

def load_config_file(filename, log_printer, silent=False):
    """
    Loads sections from a config file. Prints an appropriate warning if
    it doesn't exist and returns a section dict containing an empty
    default section in that case.

    It assumes that the cli_sections are available.

    :param filename:    The file to load settings from.
    :param log_printer: The log printer to log the warning/error to (in case).
    :param silent:      Whether or not to warn the user/exit if the file
                        doesn't exist.
    :raises SystemExit: Exits when the given filename is invalid and is not the
                        default coafile. Only raised when ``silent`` is
                        ``False``.
    """
    filename = os.path.abspath(filename)

    try:
        return ConfParser().parse(filename)
    except FileNotFoundError:
        if not silent:
            if os.path.basename(filename) == Constants.default_coafile:
                log_printer.warn(COAFILE_OUTPUT
                                 .substitute(type='Default coafile',
                                             file=Constants.default_coafile,
                                             found='not found'))
            else:
                log_printer.err(COAFILE_OUTPUT
                                .substitute(type='Requested coafile',
                                            file=filename,
                                            found='does not exist'))
                sys.exit(2)

        return {'default': Section('default')}

def save_sections(sections):
    """
    Saves the given sections if they are to be saved.

    :param sections: A section dict.
    """
    default_section = sections['cli']
    try:
        if bool(default_section.get('save', 'false')):
            conf_writer = ConfWriter(
                str(default_section.get('config', Constants.default_coafile)))
        else:
            return
    except ValueError:
        conf_writer = ConfWriter(str(default_section.get('save', '.coafile')))

    conf_writer.write_sections(sections)
    conf_writer.close()

def warn_nonexistent_targets(targets, sections, log_printer):
    """
    Prints out a warning on the given log printer for all targets that are
    not existent within the given sections.

    :param targets:     The targets to check.
    :param sections:    The sections to search. (Dict.)
    :param log_printer: The log printer to warn to.
    """
    for target in targets:
        if target not in sections:
            log_printer.warn(
                "The requested section '{section}' is not existent. "
                'Thus it cannot be executed.'.format(section=target))

    # Can't be summarized as python will evaluate conditions lazily, those
    # functions have intended side effects though.
    files_config_absent = warn_config_absent(sections, 'files', log_printer)
    bears_config_absent = warn_config_absent(sections, 'bears', log_printer)
    if files_config_absent or bears_config_absent:
        raise SystemExit(2)  # Invalid CLI options provided

def warn_config_absent(sections, argument, log_printer):
    """
    Checks if the given argument is present somewhere in the sections and emits
    a warning that code analysis can not be run without it.

    :param sections:    A dictionary of sections.
    :param argument:    The argument to check for, e.g. "files".
    :param log_printer: A log printer to emit the warning to.
    :return:            Returns a boolean True if the given argument
                        is present in the sections, else returns False.
    """
    if all(argument not in section for section in sections.values()):
        log_printer.warn('coala will not run any analysis. Did you forget '
                         'to give the `--{}` argument?'.format(argument))
        return True

    return False

def load_configuration(arg_list,
                       log_printer,
                       arg_parser=None,
                       args=None,
                       silent=False):
    """
    Parses the CLI args and loads the config file accordingly, taking
    default_coafile and the users .coarc into account.

    :param arg_list:    The list of CLI arguments.
    :param log_printer: The LogPrinter object for logging.
    :param arg_parser:  An ``argparse.ArgumentParser`` instance used for
                        parsing the CLI arguments.
    :param args:        Alternative pre-parsed CLI arguments.
    :param silent:      Whether or not to display warnings, ignored if ``save``
                        is enabled.
    :return:            A tuple holding (log_printer: LogPrinter, sections:
                        dict(str, Section), targets: list(str)). (Types
                        indicated after colon.)
    """
    cli_sections = parse_cli(arg_list=arg_list, arg_parser=arg_parser,
                             args=args)
    check_conflicts(cli_sections)

    if (
            bool(cli_sections['cli'].get('find_config', 'False')) and
            str(cli_sections['cli'].get('config')) == ''):
        cli_sections['cli'].add_or_create_setting(
            Setting('config', re.escape(find_user_config(os.getcwd()))))

    targets = []
    # We don't want to store targets argument back to file, thus remove it
    for item in list(cli_sections['cli'].contents.pop('targets', '')):
        targets.append(item.lower())

    if bool(cli_sections['cli'].get('no_config', 'False')):
        sections = cli_sections
    else:
        base_sections = load_config_file(Constants.system_coafile,
                                         log_printer,
                                         silent=silent)
        user_sections = load_config_file(
            Constants.user_coafile,
            log_printer,
            silent=True)

        default_config = str(base_sections['default'].get('config', '.coafile'))
        user_config = str(user_sections['default'].get(
            'config', default_config))
        config = os.path.abspath(
            str(cli_sections['cli'].get('config', user_config)))

        try:
            save = bool(cli_sections['cli'].get('save', 'False'))
        except ValueError:
            # A file is deposited for the save parameter, means we want to save
            # but to a specific file.
            save = True

        coafile_sections = load_config_file(config,
                                            log_printer,
                                            silent=save or silent)

        sections = merge_section_dicts(base_sections, user_sections)

        sections = merge_section_dicts(sections, coafile_sections)

        if 'cli' in sections:
            logging.warning('\'cli\' is an internally reserved section name. '
                            'It may have been generated into your coafile '
                            'while running coala with `--save`. The settings '
                            'in that section will inherit implicitly to all '
                            'sections as defaults just like CLI args do.'
                            'Please change the name of that section in your '
                            'coafile to avoid any unexpected behavior.')

        sections = merge_section_dicts(sections, cli_sections)

    for name, section in list(sections.items()):
        section.set_default_section(sections)
        if name == 'default':
            if section.contents:
                logging.warning('Implicit \'Default\' section inheritance is '
                                'deprecated. It will be removed soon. To '
                                'silence this warning remove settings in the '
                                '\'Default\' section from your coafile. You '
                                'can use dots to specify inheritance: the '
                                'section \'all.python\' will inherit all '
                                'settings from \'all\'.')
                sections['default'].update(sections['cli'])
                sections['default'].name = 'cli'
                sections['cli'] = sections['default']
            del sections['default']

    str_log_level = str(sections['cli'].get('log_level', '')).upper()
    log_printer.log_level = LOG_LEVEL.str_dict.get(str_log_level,
                                                   LOG_LEVEL.INFO)

    return sections, targets

def find_user_config(file_path, max_trials=10):
    """
    Uses the filepath to find the most suitable user config file for the file
    by going down one directory at a time and finding config files there.

    :param file_path:  The path of the file whose user config needs to be found
    :param max_trials: The maximum number of directories to go down to.
    :return:           The config file's path, empty string if none was found
    """
    file_path = os.path.normpath(os.path.abspath(os.path.expanduser(
        file_path)))
    old_dir = None
    base_dir = (file_path if os.path.isdir(file_path)
                else os.path.dirname(file_path))
    home_dir = os.path.expanduser('~')

    while base_dir != old_dir and old_dir != home_dir and max_trials != 0:
        config_file = os.path.join(base_dir, '.coafile')
        if os.path.isfile(config_file):
            return config_file

        old_dir = base_dir
        base_dir = os.path.dirname(old_dir)
        max_trials = max_trials - 1

    return ''

def get_config_directory(section):
    """
    Retrieves the configuration directory for the given section.

    Given an empty section:

    >>> section = Section("name")

    The configuration directory is not defined and will therefore fallback to
    the current directory:

    >>> get_config_directory(section) == os.path.abspath(".")
    True

    If the ``files`` setting is given with an originating coafile, the directory
    of the coafile will be assumed the configuration directory:

    >>> section.append(Setting("files", "**", origin="/tmp/.coafile"))
    >>> get_config_directory(section) == os.path.abspath('/tmp/')
    True

    However if its origin is already a directory this will be preserved:

    >>> files = section['files']
    >>> files.origin = os.path.abspath('/tmp/dir/')
    >>> section.append(files)
    >>> os.makedirs(section['files'].origin, exist_ok=True)
    >>> get_config_directory(section) == section['files'].origin
    True

    The user can manually set a project directory with the ``project_dir``
    setting:

    >>> section.append(Setting('project_dir', os.path.abspath('/tmp'), '/'))
    >>> get_config_directory(section) == os.path.abspath('/tmp')
    True

    If no section is given, the current directory is returned:

    >>> get_config_directory(None) == os.path.abspath(".")
    True

    To summarize, the config directory will be chosen by the following
    priorities if possible in that order:

    - the ``project_dir`` setting
    - the origin of the ``files`` setting, if it's a directory
    - the directory of the origin of the ``files`` setting
    - the current directory

    :param section: The section to inspect.
    :return: The directory where the project is lying.
    """
    if section is None:
        return os.getcwd()

    if 'project_dir' in section:
        return path(section.get('project_dir'))

    config = os.path.abspath(section.get('files', '').origin)
    return config if os.path.isdir(config) else os.path.dirname(config)

def get_all_bears(log_printer, arg_parser=None, silent=True):
    """
    :param log_printer: The log_printer to handle logging.
    :param arg_parser:  An ``ArgParser`` object.
    :param silent:      Whether or not to display warnings.
    :return:            Tuple containing dictionaries of local bears
                        and global bears.
    """
    sections, _ = load_configuration(arg_list=None,
                                     log_printer=log_printer,
                                     arg_parser=arg_parser,
                                     args=None,
                                     silent=silent)
    local_bears, global_bears = collect_all_bears_from_sections(
        sections, log_printer)
    return local_bears, global_bears

def get_filtered_bears(languages,
                       log_printer,
                       arg_parser=None,
                       silent=True):
    """
    :param languages:   List of languages.
    :param log_printer: The log_printer to handle logging.
    :param arg_parser:  An ``ArgParser`` object.
    :param silent:      Whether or not to display warnings.
    :return:            Tuple containing dictionaries of local bears
                        and global bears.
    """
    local_bears, global_bears = get_all_bears(log_printer,
                                              arg_parser,
                                              silent=silent)
    if languages:
        local_bears = filter_section_bears_by_languages(
            local_bears, languages)
        global_bears = filter_section_bears_by_languages(
            global_bears, languages)
    return local_bears, global_bears

def gather_configuration(acquire_settings,
                         log_printer,
                         arg_list=None,
                         arg_parser=None,
                         args=None):
    """
    Loads all configuration files, retrieves bears and all needed
    settings, saves back if needed and warns about non-existent targets.

    This function:

    -  Reads and merges all settings in sections from

       -  Default config
       -  User config
       -  Configuration file
       -  CLI

    -  Collects all the bears
    -  Fills up all needed settings
    -  Writes back the new sections to the configuration file if needed
    -  Gives all information back to caller

    :param acquire_settings: The method to use for requesting settings. It will
                             get a parameter which is a dictionary with the
                             settings name as key and a list containing a
                             description in [0] and the names of the bears
                             who need this setting in all following indexes.
    :param log_printer:      The log printer to use for logging. The log level
                             will be adjusted to the one given by the section.
    :param arg_list:         CLI args to use
    :param arg_parser:       Instance of ArgParser that is used to parse
                             none-setting arguments.
    :param args:             Alernative pre-parsed CLI arguments.
    :return:                 A tuple with the following contents:

                             -  A dictionary with the sections
                             -  Dictionary of list of local bears for each
                                section
                             -  Dictionary of list of global bears for each
                                section
                             -  The targets list
    """
    if args is None:
        # Note: arg_list can also be []. Hence we cannot use
        # `arg_list = arg_list or default_list`
        arg_list = sys.argv[1:] if arg_list is None else arg_list
    sections, targets = load_configuration(arg_list, log_printer, arg_parser,
                                           args=args)
    aspectize_sections(sections)
    local_bears, global_bears = fill_settings(sections,
                                              acquire_settings,
                                              log_printer)
    save_sections(sections)
    warn_nonexistent_targets(targets, sections, log_printer)

    return (sections,
            local_bears,
            global_bears,
            targets)
<commit_msg>ConfigurationGathering: Fix typo in docstring<commit_after>import os
import re
import sys
import logging

from coalib.collecting.Collectors import (
    collect_all_bears_from_sections, filter_section_bears_by_languages)
from coalib.misc import Constants
from coalib.output.ConfWriter import ConfWriter
from coalib.output.printers.LOG_LEVEL import LOG_LEVEL
from coalib.parsing.CliParsing import parse_cli, check_conflicts
from coalib.parsing.ConfParser import ConfParser
from coalib.settings.Section import Section, extract_aspects_from_section
from coalib.settings.SectionFilling import fill_settings
from coalib.settings.Setting import Setting, path
from string import Template

COAFILE_OUTPUT = Template('$type \'$file\' $found!\n'
                          'Here\'s what you can do:\n'
                          '* add `--save` to generate a config file with '
                          'your current options\n'
                          '* add `-I` to suppress any use of config files\n')

def aspectize_sections(sections):
    """
    Search for aspects related setting in a section, initialize it, and then
    embed the aspects information as AspectList object into the section itself.

    :param sections:  List of section that potentially contain aspects setting.
    :return:          The new sections.
    """
    for section_name, section in sections.items():
        section.aspects = extract_aspects_from_section(section)
        if section.aspects is not None and len(section.get('bears')):
            logging.warning("'aspects' and 'bears' configuration is detected "
                            "in section '{}'. Aspect-based configuration will "
                            'takes priority and will overwrite any '
                            'explicitly listed bears'.format(section_name))
    return sections

def merge_section_dicts(lower, higher):
    """
    Merges the section dictionaries. The values of higher will take
    precedence over the ones of lower. Lower will hold the modified dict in
    the end.

    :param lower:  A section.
    :param higher: A section which values will take precedence over the ones
                   from the other.
    :return:       The merged dict.
    """
    for name in higher:
        if name in lower:
            lower[name].update(higher[name], ignore_defaults=True)
        else:
            # no deep copy needed
            lower[name] = higher[name]

    return lower

def load_config_file(filename, log_printer, silent=False):
    """
    Loads sections from a config file. Prints an appropriate warning if
    it doesn't exist and returns a section dict containing an empty
    default section in that case.

    It assumes that the cli_sections are available.

    :param filename:    The file to load settings from.
    :param log_printer: The log printer to log the warning/error to (in case).
    :param silent:      Whether or not to warn the user/exit if the file
                        doesn't exist.
    :raises SystemExit: Exits when the given filename is invalid and is not the
                        default coafile. Only raised when ``silent`` is
                        ``False``.
    """
    filename = os.path.abspath(filename)

    try:
        return ConfParser().parse(filename)
    except FileNotFoundError:
        if not silent:
            if os.path.basename(filename) == Constants.default_coafile:
                log_printer.warn(COAFILE_OUTPUT
                                 .substitute(type='Default coafile',
                                             file=Constants.default_coafile,
                                             found='not found'))
            else:
                log_printer.err(COAFILE_OUTPUT
                                .substitute(type='Requested coafile',
                                            file=filename,
                                            found='does not exist'))
                sys.exit(2)

        return {'default': Section('default')}

def save_sections(sections):
    """
    Saves the given sections if they are to be saved.

    :param sections: A section dict.
    """
    default_section = sections['cli']
    try:
        if bool(default_section.get('save', 'false')):
            conf_writer = ConfWriter(
                str(default_section.get('config', Constants.default_coafile)))
        else:
            return
    except ValueError:
        conf_writer = ConfWriter(str(default_section.get('save', '.coafile')))

    conf_writer.write_sections(sections)
    conf_writer.close()

def warn_nonexistent_targets(targets, sections, log_printer):
    """
    Prints out a warning on the given log printer for all targets that are
    not existent within the given sections.

    :param targets:     The targets to check.
    :param sections:    The sections to search. (Dict.)
    :param log_printer: The log printer to warn to.
    """
    for target in targets:
        if target not in sections:
            log_printer.warn(
                "The requested section '{section}' is not existent. "
                'Thus it cannot be executed.'.format(section=target))

    # Can't be summarized as python will evaluate conditions lazily, those
    # functions have intended side effects though.
    files_config_absent = warn_config_absent(sections, 'files', log_printer)
    bears_config_absent = warn_config_absent(sections, 'bears', log_printer)
    if files_config_absent or bears_config_absent:
        raise SystemExit(2)  # Invalid CLI options provided

def warn_config_absent(sections, argument, log_printer):
    """
    Checks if the given argument is present somewhere in the sections and emits
    a warning that code analysis can not be run without it.

    :param sections:    A dictionary of sections.
    :param argument:    The argument to check for, e.g. "files".
    :param log_printer: A log printer to emit the warning to.
    :return:            Returns a boolean True if the given argument
                        is present in the sections, else returns False.
    """
    if all(argument not in section for section in sections.values()):
        log_printer.warn('coala will not run any analysis. Did you forget '
                         'to give the `--{}` argument?'.format(argument))
        return True

    return False

def load_configuration(arg_list,
                       log_printer,
                       arg_parser=None,
                       args=None,
                       silent=False):
    """
    Parses the CLI args and loads the config file accordingly, taking
    default_coafile and the users .coarc into account.

    :param arg_list:    The list of CLI arguments.
    :param log_printer: The LogPrinter object for logging.
    :param arg_parser:  An ``argparse.ArgumentParser`` instance used for
                        parsing the CLI arguments.
    :param args:        Alternative pre-parsed CLI arguments.
    :param silent:      Whether or not to display warnings, ignored if ``save``
                        is enabled.
    :return:            A tuple holding (log_printer: LogPrinter, sections:
                        dict(str, Section), targets: list(str)). (Types
                        indicated after colon.)
    """
    cli_sections = parse_cli(arg_list=arg_list, arg_parser=arg_parser,
                             args=args)
    check_conflicts(cli_sections)

    if (
            bool(cli_sections['cli'].get('find_config', 'False')) and
            str(cli_sections['cli'].get('config')) == ''):
        cli_sections['cli'].add_or_create_setting(
            Setting('config', re.escape(find_user_config(os.getcwd()))))

    targets = []
    # We don't want to store targets argument back to file, thus remove it
    for item in list(cli_sections['cli'].contents.pop('targets', '')):
        targets.append(item.lower())

    if bool(cli_sections['cli'].get('no_config', 'False')):
        sections = cli_sections
    else:
        base_sections = load_config_file(Constants.system_coafile,
                                         log_printer,
                                         silent=silent)
        user_sections = load_config_file(
            Constants.user_coafile,
            log_printer,
            silent=True)

        default_config = str(base_sections['default'].get('config', '.coafile'))
        user_config = str(user_sections['default'].get(
            'config', default_config))
        config = os.path.abspath(
            str(cli_sections['cli'].get('config', user_config)))

        try:
            save = bool(cli_sections['cli'].get('save', 'False'))
        except ValueError:
            # A file is deposited for the save parameter, means we want to save
            # but to a specific file.
            save = True

        coafile_sections = load_config_file(config,
                                            log_printer,
                                            silent=save or silent)

        sections = merge_section_dicts(base_sections, user_sections)

        sections = merge_section_dicts(sections, coafile_sections)

        if 'cli' in sections:
            logging.warning('\'cli\' is an internally reserved section name. '
                            'It may have been generated into your coafile '
                            'while running coala with `--save`. The settings '
                            'in that section will inherit implicitly to all '
                            'sections as defaults just like CLI args do.'
                            'Please change the name of that section in your '
                            'coafile to avoid any unexpected behavior.')

        sections = merge_section_dicts(sections, cli_sections)

    for name, section in list(sections.items()):
        section.set_default_section(sections)
        if name == 'default':
            if section.contents:
                logging.warning('Implicit \'Default\' section inheritance is '
                                'deprecated. It will be removed soon. To '
                                'silence this warning remove settings in the '
                                '\'Default\' section from your coafile. You '
                                'can use dots to specify inheritance: the '
                                'section \'all.python\' will inherit all '
                                'settings from \'all\'.')
                sections['default'].update(sections['cli'])
                sections['default'].name = 'cli'
                sections['cli'] = sections['default']
            del sections['default']

    str_log_level = str(sections['cli'].get('log_level', '')).upper()
    log_printer.log_level = LOG_LEVEL.str_dict.get(str_log_level,
                                                   LOG_LEVEL.INFO)

    return sections, targets

def find_user_config(file_path, max_trials=10):
    """
    Uses the filepath to find the most suitable user config file for the file
    by going down one directory at a time and finding config files there.

    :param file_path:  The path of the file whose user config needs to be found
    :param max_trials: The maximum number of directories to go down to.
    :return:           The config file's path, empty string if none was found
    """
    file_path = os.path.normpath(os.path.abspath(os.path.expanduser(
        file_path)))
    old_dir = None
    base_dir = (file_path if os.path.isdir(file_path)
                else os.path.dirname(file_path))
    home_dir = os.path.expanduser('~')

    while base_dir != old_dir and old_dir != home_dir and max_trials != 0:
        config_file = os.path.join(base_dir, '.coafile')
        if os.path.isfile(config_file):
            return config_file

        old_dir = base_dir
        base_dir = os.path.dirname(old_dir)
        max_trials = max_trials - 1

    return ''

def get_config_directory(section):
    """
    Retrieves the configuration directory for the given section.

    Given an empty section:

    >>> section = Section("name")

    The configuration directory is not defined and will therefore fallback to
    the current directory:

    >>> get_config_directory(section) == os.path.abspath(".")
    True

    If the ``files`` setting is given with an originating coafile, the directory
    of the coafile will be assumed the configuration directory:

    >>> section.append(Setting("files", "**", origin="/tmp/.coafile"))
    >>> get_config_directory(section) == os.path.abspath('/tmp/')
    True

    However if its origin is already a directory this will be preserved:

    >>> files = section['files']
    >>> files.origin = os.path.abspath('/tmp/dir/')
    >>> section.append(files)
    >>> os.makedirs(section['files'].origin, exist_ok=True)
    >>> get_config_directory(section) == section['files'].origin
    True

    The user can manually set a project directory with the ``project_dir``
    setting:

    >>> section.append(Setting('project_dir', os.path.abspath('/tmp'), '/'))
    >>> get_config_directory(section) == os.path.abspath('/tmp')
    True

    If no section is given, the current directory is returned:

    >>> get_config_directory(None) == os.path.abspath(".")
    True

    To summarize, the config directory will be chosen by the following
    priorities if possible in that order:

    - the ``project_dir`` setting
    - the origin of the ``files`` setting, if it's a directory
    - the directory of the origin of the ``files`` setting
    - the current directory

    :param section: The section to inspect.
    :return: The directory where the project is lying.
    """
    if section is None:
        return os.getcwd()

    if 'project_dir' in section:
        return path(section.get('project_dir'))

    config = os.path.abspath(section.get('files', '').origin)
    return config if os.path.isdir(config) else os.path.dirname(config)

def get_all_bears(log_printer, arg_parser=None, silent=True):
    """
    :param log_printer: The log_printer to handle logging.
    :param arg_parser:  An ``ArgParser`` object.
    :param silent:      Whether or not to display warnings.
    :return:            Tuple containing dictionaries of local bears
                        and global bears.
    """
    sections, _ = load_configuration(arg_list=None,
                                     log_printer=log_printer,
                                     arg_parser=arg_parser,
                                     args=None,
                                     silent=silent)
    local_bears, global_bears = collect_all_bears_from_sections(
        sections, log_printer)
    return local_bears, global_bears

def get_filtered_bears(languages,
                       log_printer,
                       arg_parser=None,
                       silent=True):
    """
    :param languages:   List of languages.
    :param log_printer: The log_printer to handle logging.
    :param arg_parser:  An ``ArgParser`` object.
    :param silent:      Whether or not to display warnings.
    :return:            Tuple containing dictionaries of local bears
                        and global bears.
    """
    local_bears, global_bears = get_all_bears(log_printer,
                                              arg_parser,
                                              silent=silent)
    if languages:
        local_bears = filter_section_bears_by_languages(
            local_bears, languages)
        global_bears = filter_section_bears_by_languages(
            global_bears, languages)
    return local_bears, global_bears

def gather_configuration(acquire_settings,
                         log_printer,
                         arg_list=None,
                         arg_parser=None,
                         args=None):
    """
    Loads all configuration files, retrieves bears and all needed
    settings, saves back if needed and warns about non-existent targets.

    This function:

    -  Reads and merges all settings in sections from

       -  Default config
       -  User config
       -  Configuration file
       -  CLI

    -  Collects all the bears
    -  Fills up all needed settings
    -  Writes back the new sections to the configuration file if needed
    -  Gives all information back to caller

    :param acquire_settings: The method to use for requesting settings. It will
                             get a parameter which is a dictionary with the
                             settings name as key and a list containing a
                             description in [0] and the names of the bears
                             who need this setting in all following indexes.
    :param log_printer:      The log printer to use for logging. The log level
                             will be adjusted to the one given by the section.
    :param arg_list:         CLI args to use
    :param arg_parser:       Instance of ArgParser that is used to parse
                             none-setting arguments.
    :param args:             Alternative pre-parsed CLI arguments.
    :return:                 A tuple with the following contents:

                             -  A dictionary with the sections
                             -  Dictionary of list of local bears for each
                                section
                             -  Dictionary of list of global bears for each
                                section
                             -  The targets list
    """
    if args is None:
        # Note: arg_list can also be []. Hence we cannot use
        # `arg_list = arg_list or default_list`
        arg_list = sys.argv[1:] if arg_list is None else arg_list
    sections, targets = load_configuration(arg_list, log_printer, arg_parser,
                                           args=args)
    aspectize_sections(sections)
    local_bears, global_bears = fill_settings(sections,
                                              acquire_settings,
                                              log_printer)
    save_sections(sections)
    warn_nonexistent_targets(targets, sections, log_printer)

    return (sections,
            local_bears,
            global_bears,
            targets)
<|endoftext|>


<commit_before>#!/usr/bin/env python

__author__ = 'Thomas R. Lennan, Michael Meisinger'
__license__ = 'Apache 2.0'

from pyon.core.bootstrap import sys_name
from pyon.core.exception import NotFound, Inconsistent
from pyon.datastore.couchdb.couchdb_datastore import CouchDB_DataStore
from pyon.datastore.mockdb.mockdb_datastore import MockDB_DataStore
from pyon.public import LCS
from pyon.util.containers import current_time_millis

from interface.services.coi.iresource_registry_service import BaseResourceRegistryService

class ResourceRegistryService(BaseResourceRegistryService):

    def on_init(self):
        resource_registry_name = sys_name + "_resources"
        resource_registry_name = resource_registry_name.lower()
        persistent = False
        force_clean = False
        if 'resource_registry' in self.CFG:
            resource_registry_cfg = self.CFG['resource_registry']
            if 'persistent' in resource_registry_cfg:
                if resource_registry_cfg['persistent'] == True:
                    persistent = True
            if 'force_clean' in resource_registry_cfg:
                if resource_registry_cfg['force_clean'] == True:
                    force_clean = True
        if persistent:
            self.resource_registry = CouchDB_DataStore(datastore_name=resource_registry_name)
        else:
            self.resource_registry = MockDB_DataStore(datastore_name=resource_registry_name)
        if force_clean:
            try:
                self.resource_registry.delete_datastore(resource_registry_name)
            except NotFound:
                pass
        if not self.resource_registry.datastore_exists(resource_registry_name):
            self.resource_registry.create_datastore(resource_registry_name)

    def create(self, object={}):
        cur_time = str(current_time_millis())
        object.ts_created = cur_time
        object.ts_updated = cur_time
        return self.resource_registry.create(object)

    def read(self, object_id='', rev_id=''):
        return self.resource_registry.read(object_id, rev_id)

    def update(self, object={}):
        # Do an check whether LCS has been modified
        res_obj = self.read(object._id, object._rev)
        self.assert_condition(res_obj.lcstate == object.lcstate, "Cannot modify life cycle state in update!")
        object.ts_updated = str(current_time_millis())
        return self.resource_registry.update(object)

    def delete(self, object={}):
        return self.resource_registry.delete(object)

    def execute_lifecycle_transition(self, resource_id='', lcstate=''):
        self.assert_condition(lcstate in LCS, "Unknown life-cycle state %s" % lcstate)
        res_obj = self.read(resource_id)
        res_obj.lcstate = lcstate
        res_obj.ts_updated = str(current_time_millis())
        return self.update(res_obj)

    def create_association(self, subject=None, predicate=None, object=None):
        return self.resource_registry.create_association(subject, predicate, object)

    def delete_association(self, association=''):
        return self.resource_registry.delete_association(association, datastore_name)

    def find(self, **kwargs):
        raise NotImplementedError("Do not use find. Use a specific find operation instead.")

    def find_objects(self, subject=None, predicate=None, object_type=None, id_only=False):
        return self.resource_registry.find_objects(subject, predicate, object_type, id_only=id_only)

    def find_subjects(self, subject_type="", predicate="", object="", id_only=False):
        return self.resource_registry.find_subjects(subject_type, predicate, object, id_only=id_only)

    def find_associations(self, subject="", predicate="", object="", id_only=False):
        return self.resource_registry.find_associations(subject, predicate, object, id_only=id_only)

    def get_association(self, subject="", predicate="", object=""):
        assoc = self.resource_registry.find_associations(subject, predicate, object, id_only=True)
        if not assoc:
            raise NotFound("Association for subject/predicate/object %s/%s/%s not found" % (str(subject),str(predicate),str(object)))
        elif len(assoc) > 1:
            raise Inconsistent("Duplicate associations found for subject/predicate/object %s/%s/%s" % (str(subject),str(predicate),str(object)))
        return assoc[0]

    def find_resources(self, restype="", lcstate="", name="", id_only=False):

        return self.resource_registry.find_resources(restype, lcstate, id_only=id_only)
<commit_msg>Fixed another default value discrepency<commit_after>#!/usr/bin/env python

__author__ = 'Thomas R. Lennan, Michael Meisinger'
__license__ = 'Apache 2.0'

from pyon.core.bootstrap import sys_name
from pyon.core.exception import NotFound, Inconsistent
from pyon.datastore.couchdb.couchdb_datastore import CouchDB_DataStore
from pyon.datastore.mockdb.mockdb_datastore import MockDB_DataStore
from pyon.public import LCS
from pyon.util.containers import current_time_millis

from interface.services.coi.iresource_registry_service import BaseResourceRegistryService

class ResourceRegistryService(BaseResourceRegistryService):

    def on_init(self):
        resource_registry_name = sys_name + "_resources"
        resource_registry_name = resource_registry_name.lower()
        persistent = False
        force_clean = False
        if 'resource_registry' in self.CFG:
            resource_registry_cfg = self.CFG['resource_registry']
            if 'persistent' in resource_registry_cfg:
                if resource_registry_cfg['persistent'] == True:
                    persistent = True
            if 'force_clean' in resource_registry_cfg:
                if resource_registry_cfg['force_clean'] == True:
                    force_clean = True
        if persistent:
            self.resource_registry = CouchDB_DataStore(datastore_name=resource_registry_name)
        else:
            self.resource_registry = MockDB_DataStore(datastore_name=resource_registry_name)
        if force_clean:
            try:
                self.resource_registry.delete_datastore(resource_registry_name)
            except NotFound:
                pass
        if not self.resource_registry.datastore_exists(resource_registry_name):
            self.resource_registry.create_datastore(resource_registry_name)

    def create(self, object={}):
        cur_time = str(current_time_millis())
        object.ts_created = cur_time
        object.ts_updated = cur_time
        return self.resource_registry.create(object)

    def read(self, object_id='', rev_id=''):
        return self.resource_registry.read(object_id, rev_id)

    def update(self, object={}):
        # Do an check whether LCS has been modified
        res_obj = self.read(object._id, object._rev)
        self.assert_condition(res_obj.lcstate == object.lcstate, "Cannot modify life cycle state in update!")
        object.ts_updated = str(current_time_millis())
        return self.resource_registry.update(object)

    def delete(self, object={}):
        return self.resource_registry.delete(object)

    def execute_lifecycle_transition(self, resource_id='', lcstate=''):
        self.assert_condition(lcstate in LCS, "Unknown life-cycle state %s" % lcstate)
        res_obj = self.read(resource_id)
        res_obj.lcstate = lcstate
        res_obj.ts_updated = str(current_time_millis())
        return self.update(res_obj)

    def create_association(self, subject=None, predicate=None, object=None):
        return self.resource_registry.create_association(subject, predicate, object)

    def delete_association(self, association=''):
        return self.resource_registry.delete_association(association, datastore_name)

    def find(self, **kwargs):
        raise NotImplementedError("Do not use find. Use a specific find operation instead.")

    def find_objects(self, subject="", predicate="", object_type="", id_only=False):
        return self.resource_registry.find_objects(subject, predicate, object_type, id_only=id_only)

    def find_subjects(self, subject_type="", predicate="", object="", id_only=False):
        return self.resource_registry.find_subjects(subject_type, predicate, object, id_only=id_only)

    def find_associations(self, subject="", predicate="", object="", id_only=False):
        return self.resource_registry.find_associations(subject, predicate, object, id_only=id_only)

    def get_association(self, subject="", predicate="", object=""):
        assoc = self.resource_registry.find_associations(subject, predicate, object, id_only=True)
        if not assoc:
            raise NotFound("Association for subject/predicate/object %s/%s/%s not found" % (str(subject),str(predicate),str(object)))
        elif len(assoc) > 1:
            raise Inconsistent("Duplicate associations found for subject/predicate/object %s/%s/%s" % (str(subject),str(predicate),str(object)))
        return assoc[0]

    def find_resources(self, restype="", lcstate="", name="", id_only=False):

        return self.resource_registry.find_resources(restype, lcstate, id_only=id_only)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import unittest

from you_get import *
from you_get.extractors import *
from you_get.common import *

class YouGetTests(unittest.TestCase):

    def test_freesound(self):
        freesound.download("http://www.freesound.org/people/Corsica_S/sounds/184419/", info_only=True)

    def test_magisto(self):
        magisto.download("http://www.magisto.com/album/video/f3x9AAQORAkfDnIFDA", info_only=True)

    def test_mixcloud(self):
        mixcloud.download("http://www.mixcloud.com/DJVadim/north-america-are-you-ready/", info_only=True)

    def test_vimeo(self):
        vimeo.download("http://vimeo.com/56810854", info_only=True)

    def test_youtube(self):
        youtube.download("http://www.youtube.com/watch?v=pzKerr0JIPA", info_only=True)
        youtube.download("http://youtu.be/pzKerr0JIPA", info_only=True)
        youtube.download("http://www.youtube.com/attribution_link?u=/watch?v%3DldAKIzq7bvs%26feature%3Dshare", info_only=True)
<commit_msg>[imgur] add test<commit_after>#!/usr/bin/env python

import unittest

from you_get import *
from you_get.extractors import *
from you_get.common import *

class YouGetTests(unittest.TestCase):

    def test_freesound(self):
        freesound.download("http://www.freesound.org/people/Corsica_S/sounds/184419/", info_only=True)

    def test_imgur(self):
        imgur.download("http://imgur.com/WVLk5nD", info_only=True)
        imgur.download("http://imgur.com/gallery/WVLk5nD", info_only=True)

    def test_magisto(self):
        magisto.download("http://www.magisto.com/album/video/f3x9AAQORAkfDnIFDA", info_only=True)

    def test_mixcloud(self):
        mixcloud.download("http://www.mixcloud.com/DJVadim/north-america-are-you-ready/", info_only=True)

    def test_vimeo(self):
        vimeo.download("http://vimeo.com/56810854", info_only=True)

    def test_youtube(self):
        youtube.download("http://www.youtube.com/watch?v=pzKerr0JIPA", info_only=True)
        youtube.download("http://youtu.be/pzKerr0JIPA", info_only=True)
        youtube.download("http://www.youtube.com/attribution_link?u=/watch?v%3DldAKIzq7bvs%26feature%3Dshare", info_only=True)
<|endoftext|>


<commit_before>"""This is a basic script for plotting an analysis file.

    This script is just the first step in visualizing the data set. This file
    comprises of the method creating an html bokeh plot and a command at the
    end to implement the method.

"""
from collections import OrderedDict

import numpy as np
import pandas as pd
import pdb

from bokeh.plotting import figure, show, output_file
from bokeh.models import (BoxZoomTool, ResetTool, PreviewSaveTool,
                          ResizeTool, PanTool, PolySelectTool,
                          WheelZoomTool, HoverTool, BoxSelectTool,
                          ColumnDataSource)

import data_processing as dp

def make_plot(dataframe,  top=100, minvalues=0.01, stacked=True, lgaxis=True):
    """Basic method to plot sensitivity anlaysis.

    This is the method to generate a bokeh plot followung the burtin example
    template at the bokeh website. For clarification, parameters refer to an
    output being measured (Tmax, C, k2, etc.) and stats refer to the 1st or
    total order sensitivity index.

    Parameters:
    --------------
    dataframe : Dataframe containing sensitivity analysis results to be plotted

    top: Integer indicating the number of parameters to display (highest
                 sensitivity values)

    minvalues: Cutoff minimum for which parameters should be plotted (float)

    stacked: Boolean indicating in bars should be stacked for each parameter.

    lgaxis: Boolean indicating if log axis should be used (True) or if a
            linear axis should be used (False).

    Returns:
    ---------------
    p: Figure of the data to be plotted
    """
    # Read in csv file as panda dataframe.
    # tdf = pd.read_csv(Dataframe, delimiter=' ', skipinitialspace=True,
    #                  engine='python')
    # df = tdf
    df = dataframe

    # Remove rows which have values less than cutoff values
    df = df[df['S1'] > minvalues]
    df = df[df['ST'] > minvalues]
    df = df.dropna()

    # Only keep top values indicated by variable top
    df = df.sort_values('ST', ascending=False)
    df = df.head(top)
    df = df.reset_index(drop=True)

    # Create arrays of colors and order labels for plotting
    colors = ["#a1d99b", "#31a354"]
    s1color = np.array(["#31a354"]*df.S1.size)
    sTcolor = np.array(["#a1d99b"]*df.ST.size)
    errs1color = np.array(["#a8cfeb"]*df.S1.size)
    errsTcolor = np.array(["#546775"]*df.ST.size)
    firstorder = np.array(["1st (S1)"]*df.S1.size)
    totalorder = np.array(["Total (ST)"]*df.S1.size)

    # Create Dictionary of colors
    stat_color = OrderedDict()
    for i in range(0, 2):
        stat_color[i] = colors[i]
    # Reset index of dataframe.

    # Sizing parameters
    width = 800
    height = 800
    inner_radius = 90
    outer_radius = 300 - 10

    # Determine wedge size based off number of parameters
    big_angle = 2.0 * np.pi / (len(df)+1)
    # Determine division of wedges for plotting bars based on # stats plotted
    # for stacked or unstacked bars
    if stacked is False:
        small_angle = big_angle / (5)
    else:
        small_angle = big_angle / (3)
    # pdb.set_trace()
    # params = df['Parameter']
    # S1vals = df['S1']
    # S1vals = df['ST']

    # plottools = [BoxZoomTool(), ResetTool(), PreviewSaveTool(),
    #             ResizeTool(), PanTool(), PolySelectTool(),
    #             WheelZoomTool(), HoverTool(), BoxSelectTool()]
    plottools = "hover, wheel_zoom, save, reset, resize"  # , tap"
    # Initialize figure with tools, coloring, etc.
    p = figure(plot_width=width, plot_height=height, title="",
               x_axis_type=None, y_axis_type=None,
               x_range=(-420, 420), y_range=(-420, 420),
               min_border=0, outline_line_color="black",
               background_fill_color="#f2f2f2", border_fill_color="#f2f2f2",
               tools=plottools)
    # Specify labels for hover tool
    hover = p.select(dict(type=HoverTool))
    hover.tooltips = [("Order", "@Order"), ("Parameter", "@Param"),
                      ("Sensitivity", "@Sens"), ("Confidence", "@Conf")]
    hover.point_policy = "follow_mouse"

    p.xgrid.grid_line_color = None
    p.ygrid.grid_line_color = None

    # annular wedges divided into smaller sections for bars
    # Angles for axial line placement
    num_lines = np.arange(0, len(df)+1, 1)
    line_angles = np.pi/2 - big_angle/2 - num_lines*big_angle

    # Angles for data placement
    angles = np.pi/2 - big_angle/2 - df.index.to_series()*big_angle

    # circular axes and labels
    labels = np.power(10.0, np.arange(0, -3, -1))

    # Set max radial line to correspond to 1.1 * maximum value + error
    maxvalST = max(df.ST+df.ST_conf)
    maxvalS1 = max(df.S1+df.S1_conf)
    maxval = max(maxvalST, maxvalS1)
    labels = np.append(labels, 0.0)
    labels[0] = round(1.1*maxval, 1)

    # Determine if radial axis are log or linearly scaled
    if lgaxis is True:
            radii = (((np.log10(labels / labels[0])) +
                     labels.size) * (outer_radius - inner_radius) /
                     labels.size + inner_radius)
            radii[-1] = inner_radius
    else:
        labels = np.delete(labels, -2)
        radii = (outer_radius - inner_radius)*labels/labels[0] + inner_radius

    # Adding axis lines and labels
    p.circle(0, 0, radius=radii, fill_color=None, line_color="white")
    p.text(0, radii[:], [str(r) for r in labels[:]],
           text_font_size="8pt", text_align="center", text_baseline="middle")

    # Convert sensitivity values to the plotted values
    # Same conversion as for the labels above
    # Also calculate the angle to which the bars are placed
    # Add values to the dataframe for future reference
    Cols = np.array(['S1', 'ST'])
    for statistic in range(0, 2):
        if lgaxis is True:
            radius_of_stat = (((np.log10(df[Cols[statistic]] / labels[0])) +
                              labels.size) * (outer_radius - inner_radius) /
                              labels.size + inner_radius)
            lower_of_stat = (((np.log10((df[Cols[statistic]] -
                               df[Cols[statistic]+'_conf']) / labels[0])) +
                              labels.size) * (outer_radius - inner_radius) /
                             labels.size + inner_radius)
            higher_of_stat = (((np.log10((df[Cols[statistic]] +
                               df[Cols[statistic]+'_conf']) / labels[0])) +
                              labels.size) * (outer_radius - inner_radius) /
                              labels.size + inner_radius)
        else:

            radius_of_stat = ((outer_radius - inner_radius) *
                              df[Cols[statistic]]/labels[0] + inner_radius)
            lower_of_stat = ((outer_radius - inner_radius) *
                             (df[Cols[statistic]] -
                             df[Cols[statistic]+'_conf'])/labels[0] +
                             inner_radius)
            higher_of_stat = ((outer_radius - inner_radius) *
                              ((df[Cols[statistic]] +
                               df[Cols[statistic]+'_conf'])/labels[0]) +
                              inner_radius)

        if stacked is False:
            startA = -big_angle + angles + (2*statistic + 1)*small_angle
            stopA = -big_angle + angles + (2*statistic + 2)*small_angle
            df[Cols[statistic]+'_err_angle'] = pd.Series((startA+stopA)/2,
                                                         index=df.index)
        else:
            startA = -big_angle + angles + (1)*small_angle
            stopA = -big_angle + angles + (2)*small_angle
            if statistic == 0:
                df[Cols[statistic]+'_err_angle'] = pd.Series((startA*2 +
                                                              stopA)/3,
                                                             index=df.index)
            if statistic == 1:
                df[Cols[statistic]+'_err_angle'] = pd.Series((startA +
                                                              stopA*2)/3,
                                                             index=df.index)
        df[Cols[statistic]+'radial'] = pd.Series(radius_of_stat,
                                                 index=df.index)
        df[Cols[statistic]+'upper'] = pd.Series(higher_of_stat,
                                                index=df.index)
        df[Cols[statistic]+'lower'] = pd.Series(lower_of_stat,
                                                index=df.index)
        df[Cols[statistic]+'_start_angle'] = pd.Series(startA,
                                                       index=df.index)
        df[Cols[statistic]+'_stop_angle'] = pd.Series(stopA,
                                                      index=df.index)

        inner_rad = np.ones_like(angles)*inner_radius
        df[Cols[statistic]+'lower'] = df[Cols[statistic]+'lower'].fillna(90)
    # Store plotted values into dictionary to be add glyphs
    pdata = pd.DataFrame({
                         'x': np.append(np.zeros_like(inner_rad),
                                        np.zeros_like(inner_rad)),
                         'y': np.append(np.zeros_like(inner_rad),
                                        np.zeros_like(inner_rad)),
                         'ymin': np.append(inner_rad, inner_rad),
                         'ymax': pd.Series.append(df[Cols[1]+'radial'],
                                                  df[Cols[0]+'radial']
                                                  ).reset_index(drop=True),
                         'starts': pd.Series.append(df[Cols[1] +
                                                    '_start_angle'],
                                                    df[Cols[0] +
                                                    '_start_angle']
                                                    ).reset_index(drop=True),
                         'stops': pd.Series.append(df[Cols[1] +
                                                      '_stop_angle'],
                                                   df[Cols[0] +
                                                      '_stop_angle']
                                                   ).reset_index(drop=True),
                         'Param': pd.Series.append(df.Parameter,
                                                       df.Parameter
                                                   ).reset_index(drop=True),
                         'Colors': np.append(sTcolor, s1color),
                         'Error Colors': np.append(errsTcolor, errs1color),
                         'Conf': pd.Series.append(df.ST_conf,
                                                        df.S1_conf
                                                  ).reset_index(drop=True),
                         'Order': np.append(totalorder, firstorder),
                         'Sens': pd.Series.append(df.ST, df.S1
                                                  ).reset_index(drop=True),
                         'Lower': pd.Series.append(df.STlower,
                                                   df.S1lower
                                                   ).reset_index(drop=True),
                         'Upper': pd.Series.append(df.STupper,
                                                   df.S1upper,
                                                   ).reset_index(drop=True),
                         'Err_Angle': pd.Series.append(df.ST_err_angle,
                                                       df.S1_err_angle,
                                                       ).reset_index(drop=True)
                         })

    pdata_s = ColumnDataSource(pdata)
    # Specify that the plotted bars are the only thing to activate hovertool
    hoverable = p.annular_wedge(x='x', y='y', inner_radius='ymin',
                                outer_radius='ymax',
                                start_angle='starts',
                                end_angle='stops',
                                color='Colors',
                                source=pdata_s
                                )
    hover.renderers = [hoverable]

    # dictdata = df.set_index('Parameter').to_dict()

    # plot lines to separate parameters
    p.annular_wedge(0, 0, inner_radius-10, outer_radius+10,
                    -big_angle+line_angles, -big_angle+line_angles,
                    color="black")
    # p.annular_wedge(0, 0, pdata['Lower'], pdata['Upper'],
    #                 pdata['Err_Angle'],
    #                 pdata['Err_Angle'],
    #                 color=pdata['Error Colors'])

    # Placement of parameter labels
    xr = (radii[0]*1.1)*np.cos(np.array(-big_angle/2 + angles))
    yr = (radii[0]*1.1)*np.sin(np.array(-big_angle/2 + angles))

    label_angle = np.array(-big_angle/2+angles)
    label_angle[label_angle < -np.pi/2] += np.pi

    # Placing Labels and Legend
    p.text(xr, yr, df.Parameter, angle=label_angle,
           text_font_size="9pt", text_align="center", text_baseline="middle")

    p.rect([-40, -40, -40], [18, 0, -18], width=30, height=13,
           color=list(stat_color.values()))
    p.text([-15, -15, -15], [18, 0, -18], text=[Cols[1], Cols[0]],
           text_font_size="9pt", text_align="left", text_baseline="middle")
    # output_file('HoverTrial.html', title="HoverTrial.py example")
    # show(p)
    return p
# sa = dp.get_sa_data()

# make_plot(sa['totaltars'][0], 20, 0, False, False)
<commit_msg>Added in error bars and updated legend<commit_after>"""This is a basic script for plotting an analysis file.

    This script is just the first step in visualizing the data set. This file
    comprises of the method creating an html bokeh plot and a command at the
    end to implement the method.

"""
from collections import OrderedDict

import numpy as np
import pandas as pd
import pdb

from bokeh.plotting import figure, show, output_file
from bokeh.models import (BoxZoomTool, ResetTool, PreviewSaveTool,
                          ResizeTool, PanTool, PolySelectTool,
                          WheelZoomTool, HoverTool, BoxSelectTool,
                          ColumnDataSource)

import data_processing as dp

def make_plot(dataframe,  top=100, minvalues=0.01, stacked=True, lgaxis=True):
    """Basic method to plot sensitivity anlaysis.

    This is the method to generate a bokeh plot followung the burtin example
    template at the bokeh website. For clarification, parameters refer to an
    output being measured (Tmax, C, k2, etc.) and stats refer to the 1st or
    total order sensitivity index.

    Parameters:
    --------------
    dataframe : Dataframe containing sensitivity analysis results to be plotted

    top: Integer indicating the number of parameters to display (highest
                 sensitivity values)

    minvalues: Cutoff minimum for which parameters should be plotted (float)

    stacked: Boolean indicating in bars should be stacked for each parameter.

    lgaxis: Boolean indicating if log axis should be used (True) or if a
            linear axis should be used (False).

    Returns:
    ---------------
    p: Figure of the data to be plotted
    """
    # Read in csv file as panda dataframe.
    # tdf = pd.read_csv(Dataframe, delimiter=' ', skipinitialspace=True,
    #                  engine='python')
    # df = tdf
    df = dataframe

    # Remove rows which have values less than cutoff values
    df = df[df['S1'] > minvalues]
    df = df[df['ST'] > minvalues]
    df = df.dropna()

    # Only keep top values indicated by variable top
    df = df.sort_values('ST', ascending=False)
    df = df.head(top)
    df = df.reset_index(drop=True)

    # Create arrays of colors and order labels for plotting
    colors = ["#a1d99b", "#31a354", "#546775", "#a8cfeb"]
    s1color = np.array(["#31a354"]*df.S1.size)
    sTcolor = np.array(["#a1d99b"]*df.ST.size)
    errs1color = np.array(["#a8cfeb"]*df.S1.size)
    errsTcolor = np.array(["#546775"]*df.ST.size)
    firstorder = np.array(["1st (S1)"]*df.S1.size)
    totalorder = np.array(["Total (ST)"]*df.S1.size)

    # Create Dictionary of colors
    stat_color = OrderedDict()
    error_color = OrderedDict()
    for i in range(0, 2):
        stat_color[i] = colors[i]
    # Reset index of dataframe.
    for i in range(2, 4):
        error_color[i] = colors[i]
    # Sizing parameters
    width = 800
    height = 800
    inner_radius = 90
    outer_radius = 300 - 10

    # Determine wedge size based off number of parameters
    big_angle = 2.0 * np.pi / (len(df)+1)
    # Determine division of wedges for plotting bars based on # stats plotted
    # for stacked or unstacked bars
    if stacked is False:
        small_angle = big_angle / (5)
    else:
        small_angle = big_angle / (3)
    # pdb.set_trace()
    # params = df['Parameter']
    # S1vals = df['S1']
    # S1vals = df['ST']

    # plottools = [BoxZoomTool(), ResetTool(), PreviewSaveTool(),
    #             ResizeTool(), PanTool(), PolySelectTool(),
    #             WheelZoomTool(), HoverTool(), BoxSelectTool()]
    plottools = "hover, wheel_zoom, save, reset, resize"  # , tap"
    # Initialize figure with tools, coloring, etc.
    p = figure(plot_width=width, plot_height=height, title="",
               x_axis_type=None, y_axis_type=None,
               x_range=(-420, 420), y_range=(-420, 420),
               min_border=0, outline_line_color="black",
               background_fill_color="#f2f2f2", border_fill_color="#f2f2f2",
               tools=plottools)
    # Specify labels for hover tool
    hover = p.select(dict(type=HoverTool))
    hover.tooltips = [("Order", "@Order"), ("Parameter", "@Param"),
                      ("Sensitivity", "@Sens"), ("Confidence", "@Conf")]
    hover.point_policy = "follow_mouse"

    p.xgrid.grid_line_color = None
    p.ygrid.grid_line_color = None

    # annular wedges divided into smaller sections for bars
    # Angles for axial line placement
    num_lines = np.arange(0, len(df)+1, 1)
    line_angles = np.pi/2 - big_angle/2 - num_lines*big_angle

    # Angles for data placement
    angles = np.pi/2 - big_angle/2 - df.index.to_series()*big_angle

    # circular axes and labels
    labels = np.power(10.0, np.arange(0, -3, -1))

    # Set max radial line to correspond to 1.1 * maximum value + error
    maxvalST = max(df.ST+df.ST_conf)
    maxvalS1 = max(df.S1+df.S1_conf)
    maxval = max(maxvalST, maxvalS1)
    labels = np.append(labels, 0.0)
    labels[0] = round(1.1*maxval, 1)

    # Determine if radial axis are log or linearly scaled
    if lgaxis is True:
            radii = (((np.log10(labels / labels[0])) +
                     labels.size) * (outer_radius - inner_radius) /
                     labels.size + inner_radius)
            radii[-1] = inner_radius
    else:
        labels = np.delete(labels, -2)
        radii = (outer_radius - inner_radius)*labels/labels[0] + inner_radius

    # Adding axis lines and labels
    p.circle(0, 0, radius=radii, fill_color=None, line_color="white")
    p.text(0, radii[:], [str(r) for r in labels[:]],
           text_font_size="8pt", text_align="center", text_baseline="middle")

    # Convert sensitivity values to the plotted values
    # Same conversion as for the labels above
    # Also calculate the angle to which the bars are placed
    # Add values to the dataframe for future reference
    Cols = np.array(['S1', 'ST'])
    for statistic in range(0, 2):
        if lgaxis is True:
            radius_of_stat = (((np.log10(df[Cols[statistic]] / labels[0])) +
                              labels.size) * (outer_radius - inner_radius) /
                              labels.size + inner_radius)
            lower_of_stat = (((np.log10((df[Cols[statistic]] -
                               df[Cols[statistic]+'_conf']) / labels[0])) +
                              labels.size) * (outer_radius - inner_radius) /
                             labels.size + inner_radius)
            higher_of_stat = (((np.log10((df[Cols[statistic]] +
                               df[Cols[statistic]+'_conf']) / labels[0])) +
                              labels.size) * (outer_radius - inner_radius) /
                              labels.size + inner_radius)
        else:

            radius_of_stat = ((outer_radius - inner_radius) *
                              df[Cols[statistic]]/labels[0] + inner_radius)
            lower_of_stat = ((outer_radius - inner_radius) *
                             (df[Cols[statistic]] -
                             df[Cols[statistic]+'_conf'])/labels[0] +
                             inner_radius)
            higher_of_stat = ((outer_radius - inner_radius) *
                              ((df[Cols[statistic]] +
                               df[Cols[statistic]+'_conf'])/labels[0]) +
                              inner_radius)

        if stacked is False:
            startA = -big_angle + angles + (2*statistic + 1)*small_angle
            stopA = -big_angle + angles + (2*statistic + 2)*small_angle
            df[Cols[statistic]+'_err_angle'] = pd.Series((startA+stopA)/2,
                                                         index=df.index)
        else:
            startA = -big_angle + angles + (1)*small_angle
            stopA = -big_angle + angles + (2)*small_angle
            if statistic == 0:
                df[Cols[statistic]+'_err_angle'] = pd.Series((startA*2 +
                                                              stopA)/3,
                                                             index=df.index)
            if statistic == 1:
                df[Cols[statistic]+'_err_angle'] = pd.Series((startA +
                                                              stopA*2)/3,
                                                             index=df.index)
        df[Cols[statistic]+'radial'] = pd.Series(radius_of_stat,
                                                 index=df.index)
        df[Cols[statistic]+'upper'] = pd.Series(higher_of_stat,
                                                index=df.index)
        df[Cols[statistic]+'lower'] = pd.Series(lower_of_stat,
                                                index=df.index)
        df[Cols[statistic]+'_start_angle'] = pd.Series(startA,
                                                       index=df.index)
        df[Cols[statistic]+'_stop_angle'] = pd.Series(stopA,
                                                      index=df.index)

        # df[Cols[statistic]+'_err_angle'] = pd.Series((startA+stopA)/2,
        #                                              index=df.index)
        inner_rad = np.ones_like(angles)*inner_radius
        df[Cols[statistic]+'lower'] = df[Cols[statistic]+'lower'].fillna(90)
    # Store plotted values into dictionary to be add glyphs
    pdata = pd.DataFrame({
                         'x': np.append(np.zeros_like(inner_rad),
                                        np.zeros_like(inner_rad)),
                         'y': np.append(np.zeros_like(inner_rad),
                                        np.zeros_like(inner_rad)),
                         'ymin': np.append(inner_rad, inner_rad),
                         'ymax': pd.Series.append(df[Cols[1]+'radial'],
                                                  df[Cols[0]+'radial']
                                                  ).reset_index(drop=True),
                         'starts': pd.Series.append(df[Cols[1] +
                                                    '_start_angle'],
                                                    df[Cols[0] +
                                                    '_start_angle']
                                                    ).reset_index(drop=True),
                         'stops': pd.Series.append(df[Cols[1] +
                                                      '_stop_angle'],
                                                   df[Cols[0] +
                                                      '_stop_angle']
                                                   ).reset_index(drop=True),
                         'Param': pd.Series.append(df.Parameter,
                                                       df.Parameter
                                                   ).reset_index(drop=True),
                         'Colors': np.append(sTcolor, s1color),
                         'Error Colors': np.append(errsTcolor, errs1color),
                         'Conf': pd.Series.append(df.ST_conf,
                                                        df.S1_conf
                                                  ).reset_index(drop=True),
                         'Order': np.append(totalorder, firstorder),
                         'Sens': pd.Series.append(df.ST, df.S1
                                                  ).reset_index(drop=True),
                         'Lower': pd.Series.append(df.STlower,
                                                   df.S1lower
                                                   ).reset_index(drop=True),
                         'Upper': pd.Series.append(df.STupper,
                                                   df.S1upper,
                                                   ).reset_index(drop=True),
                         'Err_Angle': pd.Series.append(df.ST_err_angle,
                                                       df.S1_err_angle,
                                                       ).reset_index(drop=True)
                         })

    pdata_s = ColumnDataSource(pdata)
    # Specify that the plotted bars are the only thing to activate hovertool
    hoverable = p.annular_wedge(x='x', y='y', inner_radius='ymin',
                                outer_radius='ymax',
                                start_angle='starts',
                                end_angle='stops',
                                color='Colors',
                                source=pdata_s
                                )
    hover.renderers = [hoverable]

    # dictdata = df.set_index('Parameter').to_dict()

    # plot lines to separate parameters
    p.annular_wedge(0, 0, inner_radius-10, outer_radius+10,
                    -big_angle+line_angles, -big_angle+line_angles,
                    color="black")
    p.annular_wedge(0, 0, pdata['Lower'], pdata['Upper'],
                    pdata['Err_Angle'],
                    pdata['Err_Angle'],
                    color=pdata['Error Colors'])

    p.annular_wedge(0, 0, pdata['Lower'], pdata['Lower'],
                    pdata['starts'],
                    pdata['stops'],
                    color=pdata['Error Colors'])

    p.annular_wedge(0, 0, pdata['Upper'], pdata['Upper'],
                    pdata['starts'],
                    pdata['stops'],
                    color=pdata['Error Colors'])
    # Placement of parameter labels
    xr = (radii[0]*1.1)*np.cos(np.array(-big_angle/2 + angles))
    yr = (radii[0]*1.1)*np.sin(np.array(-big_angle/2 + angles))

    label_angle = np.array(-big_angle/2+angles)
    label_angle[label_angle < -np.pi/2] += np.pi

    # Placing Labels and Legend
    legend_text = ['ST', 'ST Conf', 'S1', 'S1 Conf']
    p.text(xr, yr, df.Parameter, angle=label_angle,
           text_font_size="9pt", text_align="center", text_baseline="middle")

    p.rect([-40, -40], [30, -10], width=30, height=13,
           color=list(stat_color.values()))
    p.rect([-40, -40], [10, -30], width=30, height=1,
           color=list(error_color.values()))
    p.text([-15, -15, -15, -15], [30, 10, -10, -30], text=legend_text,
           text_font_size="9pt", text_align="left", text_baseline="middle")
    # output_file('HoverTrial.html', title="HoverTrial.py example")
    # show(p)
    return p
# sa = dp.get_sa_data()

# make_plot(sa['totaltars'][0], 20, 0, True, True)
<|endoftext|>


<commit_before>from sqlalchemy.exc import DataError
from sqlalchemy.orm.exc import NoResultFound
from sqlalchemy.orm import load_only
from sqlalchemy import desc

from pulsar import PermissionDenied, MethodNotAllowed, Http404
from pulsar.apps.wsgi import Json

import odm

from lux import route
from lux.extensions import rest

class RestRouter(rest.RestRouter):
    '''A REST Router based on database models
    '''
    def query(self, request, session):
        """
        Returns a query object for the model.

        The loading of columns the user does not have read
        access to is deferred. This is only a performance enhancement.

        :param request:     request object
        :param session:     SQLAlchemy session
        :return:            query object
        """
        entities = self.columns_with_permission(request, rest.READ)
        if not entities:
            raise PermissionDenied
        model = self.model(request.app)
        db_model = model.db_model()
        db_columns = model.db_columns(self.column_fields(entities))
        return session.query(db_model).options(load_only(*db_columns))

    # RestView implementation
    def get_model(self, request, **args):
        odm = request.app.odm()
        args = args or request.urlargs
        if not args:  # pragma    nocover
            raise Http404
        with odm.begin() as session:
            query = self.query(request, session)
            try:
                return query.filter_by(**args).one()
            except (DataError, NoResultFound):
                raise Http404

    def create_model(self, request, data):
        odm = request.app.odm()
        model = self.model(request.app).db_model()
        with odm.begin() as session:
            instance = model()
            session.add(instance)
            for name, value in data.items():
                self.set_model_attribute(instance, name, value)
        return instance

    def update_model(self, request, instance, data):
        odm = request.app.odm()
        with odm.begin() as session:
            session.add(instance)
            for name, value in data.items():
                self.set_model_attribute(instance, name, value)
        return instance

    def delete_model(self, request, instance):
        with request.app.odm().begin() as session:
            session.delete(instance)

    def set_model_attribute(self, instance, name, value):
        '''Set the the attribute ``name`` to ``value`` in a model ``instance``
        '''
        current_value = getattr(instance, name, None)
        if isinstance(current_value, list):
            if not isinstance(value, (list, tuple)):
                raise TypeError('list or tuple required')
            current_value[:] = value
        else:
            setattr(instance, name, value)

    def serialise_model(self, request, data, **kw):
        """
        Makes a model instance JSON-friendly. Removes fields that the
        user does not have read access to.

        :param request:     request object
        :param data:        data
        :param kw:          not used
        :return:            dict
        """
        exclude = self.columns_without_permission(request, rest.READ)
        exclude = self.column_fields(exclude, 'name')
        model = self.model(request.app)
        return model.tojson(request, data, exclude=exclude)

    def meta(self, request):
        meta = super().meta(request)
        odm = request.app.odm()
        with odm.begin() as session:
            query = self.query(request, session)
            meta['total'] = query.count()
        return meta

    def _do_filter(self, request, model, query, field, op, value):
        if value == '':
            value = None
        odm = request.app.odm()
        field = getattr(odm[model.name], field)
        if op == 'eq':
            query = query.filter(field == value)
        elif op == 'gt':
            query = query.filter(field > value)
        elif op == 'ge':
            query = query.filter(field >= value)
        elif op == 'lt':
            query = query.filter(field < value)
        elif op == 'le':
            query = query.filter(field <= value)
        return query

    def _do_sortby(self, request, query, entry, direction):
        if direction == 'desc':
            entry = desc(entry)
        return query.order_by(entry)

class CRUD(RestRouter):
    '''A Router for handling CRUD JSON requests for a database model

    This class adds routes to the :class:`.RestRouter`
    '''
    def get(self, request):
        '''Get a list of models
        '''
        self.check_model_permission(request, rest.READ)
        # Columns the user doesn't have access to are dropped by
        # serialise_model
        return self.collection_response(request)

    def post(self, request):
        '''Create a new model
        '''
        model = self.model(request.app)
        assert model.form

        self.check_model_permission(request, rest.CREATE)
        columns = self.columns_with_permission(request, rest.CREATE)
        columns = self.column_fields(columns)

        data, files = request.data_and_files()
        form = model.form(request, data=data, files=files)
        if form.is_valid():
            # At the moment, we silently drop any data
            # for columns the user doesn't have update access to,
            # like they don't exist
            filtered_data = {k: v for k, v in form.cleaned_data.items() if
                             k in columns}
            try:
                instance = self.create_model(request, filtered_data)
            except DataError as exc:
                odm.logger.exception('Could not create model')
                form.add_error_message(str(exc))
                data = form.tojson()
            else:
                data = self.serialise(request, instance)
                request.response.status_code = 201
        else:
            data = form.tojson()
        return Json(data).http_response(request)

    # Additional Routes

    @route(method=('get', 'options'))
    def metadata(self, request):
        '''Model metadata
        '''
        if request.method == 'OPTIONS':
            request.app.fire('on_preflight', request)
            return request.response

        backend = request.cache.auth_backend
        model = self.model(request.app)

        if backend.has_permission(request, model.name, rest.READ):
            meta = self.meta(request)
            return Json(meta).http_response(request)
        raise PermissionDenied

    @route('<id>', method=('get', 'post', 'put', 'delete', 'head', 'options'))
    def read_update_delete(self, request):
        model = self.model(request.app)
        args = {model.id_field: request.urlargs['id']}
        instance = self.get_model(request, **args)

        if request.method == 'OPTIONS':
            request.app.fire('on_preflight', request)
            return request.response

        elif request.method == 'GET':
            self.check_model_permission(request, rest.READ)
            # url = request.absolute_uri()
            # Columns the user doesn't have access to are dropped by
            # serialise_model
            data = self.serialise(request, instance)
            return Json(data).http_response(request)

        elif request.method == 'HEAD':
            self.check_model_permission(request, rest.READ)
            return request.response

        elif request.method in ('POST', 'PUT'):
            form_class = model.updateform

            self.check_model_permission(request, rest.UPDATE)
            columns = self.columns_with_permission(request, rest.UPDATE)
            columns = self.column_fields(columns)

            if not form_class:
                raise MethodNotAllowed

            data, files = request.data_and_files()
            form = form_class(request, data=data, files=files,
                              previous_state=instance)
            if form.is_valid(exclude_missing=True):
                # At the moment, we silently drop any data
                # for columns the user doesn't have update access to,
                # like they don't exist
                filtered_data = {k: v for k, v in form.cleaned_data.items()
                                 if k in columns}

                instance = self.update_model(request, instance,
                                             filtered_data)
                data = self.serialise(request, instance)
            else:
                data = form.tojson()
            return Json(data).http_response(request)

        elif request.method == 'DELETE':

            self.check_model_permission(request, rest.DELETE)
            self.delete_model(request, instance)
            request.response.status_code = 204
            return request.response

        raise MethodNotAllowed
<commit_msg>get_model to get_instance<commit_after>from sqlalchemy.exc import DataError
from sqlalchemy.orm.exc import NoResultFound
from sqlalchemy.orm import load_only
from sqlalchemy import desc

from pulsar import PermissionDenied, MethodNotAllowed, Http404
from pulsar.apps.wsgi import Json

import odm

from lux import route
from lux.extensions import rest

class RestRouter(rest.RestRouter):
    '''A REST Router based on database models
    '''
    def query(self, request, session):
        """
        Returns a query object for the model.

        The loading of columns the user does not have read
        access to is deferred. This is only a performance enhancement.

        :param request:     request object
        :param session:     SQLAlchemy session
        :return:            query object
        """
        entities = self.columns_with_permission(request, rest.READ)
        if not entities:
            raise PermissionDenied
        model = self.model(request.app)
        db_model = model.db_model()
        db_columns = model.db_columns(self.column_fields(entities))
        return session.query(db_model).options(load_only(*db_columns))

    # RestView implementation
    def get_instance(self, request, **args):
        odm = request.app.odm()
        args = args or request.urlargs
        if not args:  # pragma    nocover
            raise Http404
        with odm.begin() as session:
            query = self.query(request, session)
            try:
                return query.filter_by(**args).one()
            except (DataError, NoResultFound):
                raise Http404

    def create_model(self, request, data):
        odm = request.app.odm()
        model = self.model(request.app).db_model()
        with odm.begin() as session:
            instance = model()
            session.add(instance)
            for name, value in data.items():
                self.set_model_attribute(instance, name, value)
        return instance

    def update_model(self, request, instance, data):
        odm = request.app.odm()
        with odm.begin() as session:
            session.add(instance)
            for name, value in data.items():
                self.set_model_attribute(instance, name, value)
        return instance

    def delete_model(self, request, instance):
        with request.app.odm().begin() as session:
            session.delete(instance)

    def set_model_attribute(self, instance, name, value):
        '''Set the the attribute ``name`` to ``value`` in a model ``instance``
        '''
        current_value = getattr(instance, name, None)
        if isinstance(current_value, list):
            if not isinstance(value, (list, tuple)):
                raise TypeError('list or tuple required')
            current_value[:] = value
        else:
            setattr(instance, name, value)

    def serialise_model(self, request, data, **kw):
        """
        Makes a model instance JSON-friendly. Removes fields that the
        user does not have read access to.

        :param request:     request object
        :param data:        data
        :param kw:          not used
        :return:            dict
        """
        exclude = self.columns_without_permission(request, rest.READ)
        exclude = self.column_fields(exclude, 'name')
        model = self.model(request.app)
        return model.tojson(request, data, exclude=exclude)

    def meta(self, request):
        meta = super().meta(request)
        odm = request.app.odm()
        with odm.begin() as session:
            query = self.query(request, session)
            meta['total'] = query.count()
        return meta

    def _do_filter(self, request, model, query, field, op, value):
        if value == '':
            value = None
        odm = request.app.odm()
        field = getattr(odm[model.name], field)
        if op == 'eq':
            query = query.filter(field == value)
        elif op == 'gt':
            query = query.filter(field > value)
        elif op == 'ge':
            query = query.filter(field >= value)
        elif op == 'lt':
            query = query.filter(field < value)
        elif op == 'le':
            query = query.filter(field <= value)
        return query

    def _do_sortby(self, request, query, entry, direction):
        if direction == 'desc':
            entry = desc(entry)
        return query.order_by(entry)

class CRUD(RestRouter):
    '''A Router for handling CRUD JSON requests for a database model

    This class adds routes to the :class:`.RestRouter`
    '''
    def get(self, request):
        '''Get a list of models
        '''
        self.check_model_permission(request, rest.READ)
        # Columns the user doesn't have access to are dropped by
        # serialise_model
        return self.collection_response(request)

    def post(self, request):
        '''Create a new model
        '''
        model = self.model(request.app)
        assert model.form

        self.check_model_permission(request, rest.CREATE)
        columns = self.columns_with_permission(request, rest.CREATE)
        columns = self.column_fields(columns)

        data, files = request.data_and_files()
        form = model.form(request, data=data, files=files)
        if form.is_valid():
            # At the moment, we silently drop any data
            # for columns the user doesn't have update access to,
            # like they don't exist
            filtered_data = {k: v for k, v in form.cleaned_data.items() if
                             k in columns}
            try:
                instance = self.create_model(request, filtered_data)
            except DataError as exc:
                odm.logger.exception('Could not create model')
                form.add_error_message(str(exc))
                data = form.tojson()
            else:
                data = self.serialise(request, instance)
                request.response.status_code = 201
        else:
            data = form.tojson()
        return Json(data).http_response(request)

    # Additional Routes

    @route(method=('get', 'options'))
    def metadata(self, request):
        '''Model metadata
        '''
        if request.method == 'OPTIONS':
            request.app.fire('on_preflight', request)
            return request.response

        backend = request.cache.auth_backend
        model = self.model(request.app)

        if backend.has_permission(request, model.name, rest.READ):
            meta = self.meta(request)
            return Json(meta).http_response(request)
        raise PermissionDenied

    @route('<id>', method=('get', 'post', 'put', 'delete', 'head', 'options'))
    def read_update_delete(self, request):
        model = self.model(request.app)
        args = {model.id_field: request.urlargs['id']}
        instance = self.get_instance(request, **args)

        if request.method == 'OPTIONS':
            request.app.fire('on_preflight', request)
            return request.response

        elif request.method == 'GET':
            self.check_model_permission(request, rest.READ)
            # url = request.absolute_uri()
            # Columns the user doesn't have access to are dropped by
            # serialise_model
            data = self.serialise(request, instance)
            return Json(data).http_response(request)

        elif request.method == 'HEAD':
            self.check_model_permission(request, rest.READ)
            return request.response

        elif request.method in ('POST', 'PUT'):
            form_class = model.updateform

            self.check_model_permission(request, rest.UPDATE)
            columns = self.columns_with_permission(request, rest.UPDATE)
            columns = self.column_fields(columns)

            if not form_class:
                raise MethodNotAllowed

            data, files = request.data_and_files()
            form = form_class(request, data=data, files=files,
                              previous_state=instance)
            if form.is_valid(exclude_missing=True):
                # At the moment, we silently drop any data
                # for columns the user doesn't have update access to,
                # like they don't exist
                filtered_data = {k: v for k, v in form.cleaned_data.items()
                                 if k in columns}

                instance = self.update_model(request, instance,
                                             filtered_data)
                data = self.serialise(request, instance)
            else:
                data = form.tojson()
            return Json(data).http_response(request)

        elif request.method == 'DELETE':

            self.check_model_permission(request, rest.DELETE)
            self.delete_model(request, instance)
            request.response.status_code = 204
            return request.response

        raise MethodNotAllowed
<|endoftext|>