<commit_before># -*- coding: utf-8 -*-
"""
Windows aggregator resize each item
in a batch output and save as an image.
"""
from __future__ import absolute_import, print_function, division

import os

import tensorflow as tf

import niftynet.io.misc_io as misc_io
import numpy as np
from niftynet.engine.sampler_resize_v2 import zoom_3d
from niftynet.engine.windows_aggregator_base import ImageWindowsAggregator
from niftynet.layer.discrete_label_normalisation import \
    DiscreteLabelNormalisationLayer
from niftynet.layer.pad import PadLayer

class ResizeSamplesAggregator(ImageWindowsAggregator):
    """
    This class decodes each item in a batch by resizing each image
    window and save as a new image volume. Multiple output image can be
    proposed and csv output can be performed as well
    """
    def __init__(self,
                 image_reader,
                 name='image',
                 output_path=os.path.join('.', 'output'),
                 window_border=(),
                 interp_order=0,
                 postfix='niftynet_out'):
        ImageWindowsAggregator.__init__(
            self, image_reader=image_reader, output_path=output_path)
        self.name = name
        self.image_out = None
        self.csv_out = None
        self.window_border = window_border
        self.output_interp_order = interp_order
        self.postfix = postfix
        self.current_out = {}

    def decode_batch(self, window, location):
        """
        Resizing each output image window in the batch as an image volume
        location specifies the original input image (so that the
        interpolation order, original shape information retained in the

        generated outputs).For the fields that have the keyword 'window' in the
        dictionary key, it will be saved as image. The rest will be saved as
        csv. CSV files will contain at saving a first line of 0 (to be
        changed into the header by the user), the first column being the
        index of the window, followed by the list of output.

        """
        n_samples = location.shape[0]
        location_init = np.copy(location)
        test = None
        for i in window:
            if 'window' in i:

                test = np.ones_like(window[i])
                window[i], _ = self.crop_batch(window[i], location_init,
                                               self.window_border)
                location_init = np.copy(location)
                print(i, np.sum(window[i]), np.max(window[i]))
        _, location = self.crop_batch(test, location_init, self.window_border)

        for batch_id in range(n_samples):
            if self._is_stopping_signal(location[batch_id]):
                return False
            self.image_id = location[batch_id, 0]
            if self._is_stopping_signal(location[batch_id]):
                return False
            self.image_out = {}
            self.csv_out = {}

            for i in window:
                if 'window' in i:

                    while window[i].ndim < 5:
                        window[i] = window[i][..., np.newaxis, :]
                    self.image_out[i] = window[i][batch_id, ...]
                else:
                    if not isinstance(window[i], (list, tuple, np.ndarray)):
                        window_loc = np.reshape(window[i], [1, 1])
                        self.csv_out[i] = self._initialise_empty_csv(1)
                    else:
                        window[i] = np.asarray(window[i])
                        if n_samples > 1 and window[i].ndim < 2:
                            window[i] = np.expand_dims(window[i], 1)
                        elif n_samples == 1 and window[i].shape[0] != n_samples:
                            window[i] = np.expand_dims(window[i], 0)
                        window_save = np.asarray(np.squeeze(
                            window[i][batch_id, ...]))
                        try:
                            assert window_save.ndim <= 2
                        except (TypeError, AssertionError):
                            tf.logging.error(
                                "The output you are trying to "
                                "save as csv is more than "
                                "bidimensional. Did you want "
                                "to save an image instead? "
                                "Put the keyword window "
                                "in the output dictionary"
                                " in your application file")
                        while window_save.ndim < 2:
                            window_save = np.expand_dims(window_save, 0)
                        window_loc = window_save
                        self.csv_out[i] = self._initialise_empty_csv(
                            n_channel=window_save.shape[-1])

                    self.csv_out[i] = np.concatenate([self.csv_out[i],
                                                      window_loc], 0)

            self._save_current_image()
            self._save_current_csv()

        return True

    def _initialise_image_shape(self, image_id, n_channels):
        '''
        Return the shape of the empty image to be saved
        :param image_id: index to find the appropriate input image from the
        reader
        :param n_channels: number of channels of the image
        :return:  shape of the empty image
        '''
        self.image_id = image_id
        spatial_shape = self.input_image[self.name].shape[:3]
        output_image_shape = spatial_shape + (1, n_channels,)
        empty_image = np.zeros(output_image_shape, dtype=np.bool)
        for layer in self.reader.preprocessors:
            if isinstance(layer, PadLayer):
                empty_image, _ = layer(empty_image)
        return empty_image.shape

    def _save_current_image(self):
        '''
        Loop through the dictionary of images output and resize and reverse
        the preprocessing prior to saving
        :return:
        '''
        if self.input_image is None:
            return

        self.current_out = {}
        for i in self.image_out:
            resize_to_shape = self._initialise_image_shape(
                image_id=self.image_id,
                n_channels=self.image_out[i].shape[-1])
            window_shape = resize_to_shape
            current_out = self.image_out[i]
            while current_out.ndim < 5:
                current_out = current_out[..., np.newaxis, :]
            if self.window_border and any([b > 0 for b in self.window_border]):
                np_border = self.window_border
                while len(np_border) < 5:
                    np_border = np_border + (0,)
                np_border = [(b,) for b in np_border]
                current_out = np.pad(current_out, np_border, mode='edge')
            image_shape = current_out.shape
            zoom_ratio = \
                [float(p) / float(d) for p, d in zip(window_shape, image_shape)]
            image_shape = list(image_shape[:3]) + [1, image_shape[-1]]
            print(np.sum(self.image_out[i]), " is sum of image out %s before"
                  % i)
            current_out = np.reshape(current_out, image_shape)
            current_out = zoom_3d(image=current_out,
                                  ratio=zoom_ratio,
                                  interp_order=self.output_interp_order)
            self.current_out[i] = current_out

        for layer in reversed(self.reader.preprocessors):
            if isinstance(layer, PadLayer):
                for i in self.image_out:
                    self.current_out[i], _ = layer.inverse_op(
                        self.current_out[i])
            if isinstance(layer, DiscreteLabelNormalisationLayer):
                for i in self.image_out:
                    self.image_out[i], _ = layer.inverse_op(self.image_out[i])
        subject_name = self.reader.get_subject_id(self.image_id)
        for i in self.image_out:
            print(np.sum(self.image_out[i]), " is sum of image out %s after"
                  % i)
        for i in self.image_out:
            filename = "{}_{}_{}.nii.gz".format(i, subject_name, self.postfix)
            source_image_obj = self.input_image[self.name]
            misc_io.save_data_array(self.output_path,
                                    filename,
                                    self.current_out[i],
                                    source_image_obj,
                                    self.output_interp_order)
            self.log_inferred(subject_name, filename)
        return

    def _save_current_csv(self):
        '''
        Save all csv output present in the dictionary of csv_output.
        :return:
        '''
        if self.input_image is None:
            return
        subject_name = self.reader.get_subject_id(self.image_id)
        for i in self.csv_out:
            filename = "{}_{}_{}.csv".format(i, subject_name, self.postfix)
            misc_io.save_csv_array(self.output_path,
                                   filename,
                                   self.csv_out[i])
            self.log_inferred(subject_name, filename)
        return

    def _initialise_empty_csv(self, n_channel):
        '''
        Initialise the array to be saved as csv as a line of zeros according
        to the number of elements to be saved
        :param n_channel:
        :return:
        '''
        return np.zeros([1, n_channel])
    <commit_msg>fixes pylint<commit_after># -*- coding: utf-8 -*-
"""
Windows aggregator resize each item
in a batch output and save as an image.
"""
from __future__ import absolute_import, division, print_function

import os

import numpy as np
import tensorflow as tf

import niftynet.io.misc_io as misc_io
from niftynet.engine.sampler_resize_v2 import zoom_3d
from niftynet.engine.windows_aggregator_base import ImageWindowsAggregator
from niftynet.layer.discrete_label_normalisation import \
    DiscreteLabelNormalisationLayer
from niftynet.layer.pad import PadLayer

class ResizeSamplesAggregator(ImageWindowsAggregator):
    """
    This class decodes each item in a batch by resizing each image
    window and save as a new image volume. Multiple output image can be
    proposed and csv output can be performed as well
    """

    def __init__(self,
                 image_reader,
                 name='image',
                 output_path=os.path.join('.', 'output'),
                 window_border=(),
                 interp_order=0,
                 postfix='niftynet_out'):
        ImageWindowsAggregator.__init__(
            self, image_reader=image_reader, output_path=output_path)
        self.name = name
        self.image_out = None
        self.csv_out = None
        self.window_border = window_border
        self.output_interp_order = interp_order
        self.postfix = postfix
        self.current_out = {}

    def decode_batch(self, window, location):
        """
        Resizing each output image window in the batch as an image volume
        location specifies the original input image (so that the
        interpolation order, original shape information retained in the

        generated outputs).For the fields that have the keyword 'window' in the
        dictionary key, it will be saved as image. The rest will be saved as
        csv. CSV files will contain at saving a first line of 0 (to be
        changed into the header by the user), the first column being the
        index of the window, followed by the list of output.

        """
        n_samples = location.shape[0]
        location_init = np.copy(location)
        test = None
        for i in window:
            if 'window' in i:

                test = np.ones_like(window[i])
                window[i], _ = self.crop_batch(window[i], location_init,
                                               self.window_border)
                location_init = np.copy(location)
                print(i, np.sum(window[i]), np.max(window[i]))
        _, location = self.crop_batch(test, location_init, self.window_border)

        for batch_id in range(n_samples):
            if self._is_stopping_signal(location[batch_id]):
                return False
            self.image_id = location[batch_id, 0]
            if self._is_stopping_signal(location[batch_id]):
                return False
            self.image_out = {}
            self.csv_out = {}

            for i in window:
                if 'window' in i:

                    while window[i].ndim < 5:
                        window[i] = window[i][..., np.newaxis, :]
                    self.image_out[i] = window[i][batch_id, ...]
                else:
                    if not isinstance(window[i], (list, tuple, np.ndarray)):
                        window_loc = np.reshape(window[i], [1, 1])
                        self.csv_out[i] = self._initialise_empty_csv(1)
                    else:
                        window[i] = np.asarray(window[i])
                        if n_samples > 1 and window[i].ndim < 2:
                            window[i] = np.expand_dims(window[i], 1)
                        elif n_samples == 1 and window[i].shape[0] != n_samples:
                            window[i] = np.expand_dims(window[i], 0)
                        window_save = np.asarray(
                            np.squeeze(window[i][batch_id, ...]))
                        try:
                            assert window_save.ndim <= 2
                        except (TypeError, AssertionError):
                            tf.logging.error("The output you are trying to "
                                             "save as csv is more than "
                                             "bidimensional. Did you want "
                                             "to save an image instead? "
                                             "Put the keyword window "
                                             "in the output dictionary"
                                             " in your application file")
                        while window_save.ndim < 2:
                            window_save = np.expand_dims(window_save, 0)
                        window_loc = window_save
                        self.csv_out[i] = self._initialise_empty_csv(
                            n_channel=window_save.shape[-1])

                    self.csv_out[i] = np.concatenate(
                        [self.csv_out[i], window_loc], 0)

            self._save_current_image()
            self._save_current_csv()

        return True

    def _initialise_image_shape(self, image_id, n_channels):
        """
        Return the shape of the empty image to be saved
        :param image_id: index to find the appropriate input image from the
        reader
        :param n_channels: number of channels of the image
        :return:  shape of the empty image
        """
        self.image_id = image_id
        spatial_shape = self.input_image[self.name].shape[:3]
        output_image_shape = spatial_shape + (1, n_channels)
        empty_image = np.zeros(output_image_shape, dtype=np.bool)
        for layer in self.reader.preprocessors:
            if isinstance(layer, PadLayer):
                empty_image, _ = layer(empty_image)
        return empty_image.shape

    def _save_current_image(self):
        """
        Loop through the dictionary of images output and resize and reverse
        the preprocessing prior to saving
        :return:
        """
        if self.input_image is None:
            return

        self.current_out = {}
        for i in self.image_out:
            resize_to_shape = self._initialise_image_shape(
                image_id=self.image_id, n_channels=self.image_out[i].shape[-1])
            window_shape = resize_to_shape
            current_out = self.image_out[i]
            while current_out.ndim < 5:
                current_out = current_out[..., np.newaxis, :]
            if self.window_border and any([b > 0 for b in self.window_border]):
                np_border = self.window_border
                while len(np_border) < 5:
                    np_border = np_border + (0, )
                np_border = [(b, ) for b in np_border]
                current_out = np.pad(current_out, np_border, mode='edge')
            image_shape = current_out.shape
            zoom_ratio = \
                [float(p) / float(d) for p, d in zip(window_shape, image_shape)]
            image_shape = list(image_shape[:3]) + [1, image_shape[-1]]
            print(
                np.sum(self.image_out[i]),
                " is sum of image out %s before" % i)
            current_out = np.reshape(current_out, image_shape)
            current_out = zoom_3d(
                image=current_out,
                ratio=zoom_ratio,
                interp_order=self.output_interp_order)
            self.current_out[i] = current_out

        for layer in reversed(self.reader.preprocessors):
            if isinstance(layer, PadLayer):
                for i in self.image_out:
                    self.current_out[i], _ = layer.inverse_op(
                        self.current_out[i])
            if isinstance(layer, DiscreteLabelNormalisationLayer):
                for i in self.image_out:
                    self.image_out[i], _ = layer.inverse_op(self.image_out[i])
        subject_name = self.reader.get_subject_id(self.image_id)
        for i in self.image_out:
            print(
                np.sum(self.image_out[i]), " is sum of image out %s after" % i)
        for i in self.image_out:
            filename = "{}_{}_{}.nii.gz".format(i, subject_name, self.postfix)
            source_image_obj = self.input_image[self.name]
            misc_io.save_data_array(self.output_path, filename,
                                    self.current_out[i], source_image_obj,
                                    self.output_interp_order)
            self.log_inferred(subject_name, filename)
        return

    def _save_current_csv(self):
        """
        Save all csv output present in the dictionary of csv_output.
        :return:
        """
        if self.input_image is None:
            return
        subject_name = self.reader.get_subject_id(self.image_id)
        for i in self.csv_out:
            filename = "{}_{}_{}.csv".format(i, subject_name, self.postfix)
            misc_io.save_csv_array(self.output_path, filename, self.csv_out[i])
            self.log_inferred(subject_name, filename)
        return

    def _initialise_empty_csv(self, n_channel):
        """
        Initialise the array to be saved as csv as a line of zeros according
        to the number of elements to be saved
        :param n_channel:
        :return:
        """
        return np.zeros([1, n_channel])
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright © 2012-2016 Roberto Alsina and others.

# Permission is hereby granted, free of charge, to any
# person obtaining a copy of this software and associated
# documentation files (the "Software"), to deal in the
# Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the
# Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice
# shall be included in all copies or substantial portions of
# the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

"""Render the taxonomy overviews, classification pages and feeds."""

from __future__ import unicode_literals
import blinker
import functools
import natsort
import os
import sys

from collections import defaultdict

from nikola.plugin_categories import SignalHandler
from nikola import utils

class TaxonomiesClassifier(SignalHandler):
    """Render the tag/category pages and feeds."""

    name = "render_taxonomies"

    def _do_classification(self, site):
        # Needed to avoid strange errors during tests
        if site is not self.site:
            return

        # Get list of enabled taxonomy plugins and initialize data structures
        taxonomies = site.taxonomy_plugins.values()
        site.posts_per_classification = {}
        for taxonomy in taxonomies:
            site.posts_per_classification[taxonomy.classification_name] = {
                lang: defaultdict(set) for lang in site.config['TRANSLATIONS'].keys()
            }

        # Classify posts
        for post in site.timeline:
            if not post.use_in_feeds:
                continue
            for taxonomy in taxonomies:
                if taxonomy.apply_to_posts if post.is_post else taxonomy.apply_to_pages:
                    classifications = {}
                    for lang in site.config['TRANSLATIONS'].keys():
                        # Extract classifications for this language
                        classifications[lang] = taxonomy.classify(post, lang)
                        if not taxonomy.more_than_one_classifications_per_post and len(classifications[lang]) > 1:
                            raise ValueError("Too many {0} classifications for post {1}".format(taxonomy.classification_name, post.source_path))
                        # Add post to sets
                        for classification in classifications[lang]:
                            while True:
                                site.posts_per_classification[taxonomy.classification_name][lang][classification].add(post)
                                if not taxonomy.include_posts_from_subhierarchies or not taxonomy.has_hierarchy:
                                    break
                                classification_path = taxonomy.extract_hierarchy(classification)
                                if len(classification_path) <= 1:
                                    if len(classification_path) == 0 or not taxonomy.include_posts_into_hierarchy_root:
                                        break
                                classification = taxonomy.recombine_classification_from_hierarchy(classification_path[:-1])

        # Sort everything.
        site.page_count_per_classification = {}
        site.hierarchy_per_classification = {}
        site.flat_hierarchy_per_classification = {}
        site.hierarchy_lookup_per_classification = {}
        for taxonomy in taxonomies:
            site.page_count_per_classification[taxonomy.classification_name] = {}
            # Sort post lists
            for lang, posts_per_classification in site.posts_per_classification[taxonomy.classification_name].items():
                # Ensure implicit classifications are inserted
                for classification in taxonomy.get_implicit_classifications(lang):
                    if classification not in posts_per_classification:
                        posts_per_classification[classification] = []
                site.page_count_per_classification[taxonomy.classification_name][lang] = {}
                # Convert sets to lists and sort them
                for classification in list(posts_per_classification.keys()):
                    posts = list(posts_per_classification[classification])
                    posts.sort(key=lambda p:
                               (int(p.meta('priority')) if p.meta('priority') else 0,
                                p.date, p.source_path))
                    posts.reverse()
                    taxonomy.sort_posts(posts, classification, lang)
                    posts_per_classification[classification] = posts
            # Create hierarchy information
            if taxonomy.has_hierarchy:
                site.hierarchy_per_classification[taxonomy.classification_name] = {}
                site.flat_hierarchy_per_classification[taxonomy.classification_name] = {}
                site.hierarchy_lookup_per_classification[taxonomy.classification_name] = {}
                for lang, posts_per_classification in site.posts_per_classification[taxonomy.classification_name].items():
                    # Compose hierarchy
                    hierarchy = {}
                    for classification in posts_per_classification.keys():
                        hier = taxonomy.extract_hierarchy(classification)
                        node = hierarchy
                        for he in hier:
                            if he not in node:
                                node[he] = {}
                            node = node[he]
                    hierarchy_lookup = {}

                    def create_hierarchy(hierarchy, parent=None, level=0):
                        """Create hierarchy."""
                        result = {}
                        for name, children in hierarchy.items():
                            node = utils.TreeNode(name, parent)
                            node.children = create_hierarchy(children, node, level + 1)
                            node.classification_path = [pn.name for pn in node.get_path()]
                            node.classification_name = taxonomy.recombine_classification_from_hierarchy(node.classification_path)
                            hierarchy_lookup[node.classification_name] = node
                            result[node.name] = node
                        classifications = natsort.natsorted(result.keys(), alg=natsort.ns.F | natsort.ns.IC)
                        taxonomy.sort_classifications(classifications, lang, level=level)
                        return [result[classification] for classification in classifications]

                    root_list = create_hierarchy(hierarchy)
                    if '' in posts_per_classification:
                        node = utils.TreeNode('', parent=None)
                        node.children = root_list
                        node.classification_path = []
                        node.classification_name = ''
                        hierarchy_lookup[node.name] = node
                        root_list = [node]
                    flat_hierarchy = utils.flatten_tree_structure(root_list)
                    # Store result
                    site.hierarchy_per_classification[taxonomy.classification_name][lang] = root_list
                    site.flat_hierarchy_per_classification[taxonomy.classification_name][lang] = flat_hierarchy
                    site.hierarchy_lookup_per_classification[taxonomy.classification_name][lang] = hierarchy_lookup
                taxonomy.postprocess_posts_per_classification(site.posts_per_classification[taxonomy.classification_name],
                                                              site.flat_hierarchy_per_classification[taxonomy.classification_name],
                                                              site.hierarchy_lookup_per_classification[taxonomy.classification_name])
            else:
                taxonomy.postprocess_posts_per_classification(site.posts_per_classification[taxonomy.classification_name])

        # Check for valid paths and for collisions
        taxonomy_outputs = {lang: dict() for lang in site.config['TRANSLATIONS'].keys()}
        quit = False
        for taxonomy in taxonomies:
            # Check for collisions (per language)
            for lang in site.config['TRANSLATIONS'].keys():
                if not taxonomy.is_enabled(lang):
                    continue
                for tlang in site.config['TRANSLATIONS'].keys():
                    if lang != tlang and not taxonomy.also_create_classifications_from_other_languages:
                        continue
                    for classification, posts in site.posts_per_classification[taxonomy.classification_name][tlang].items():
                        # Obtain path as tuple
                        path = site.path_handlers[taxonomy.classification_name](classification, lang)
                        # Check that path is OK
                        for path_element in path:
                            if len(path_element) == 0:
                                utils.LOGGER.error("{0} {1} yields invalid path '{2}'!".format(taxonomy.classification_name.title(), classification, '/'.join(path)))
                                quit = True
                        # Combine path
                        path = os.path.join(*[os.path.normpath(p) for p in path if p != '.'])
                        # Determine collisions
                        if path in taxonomy_outputs[lang]:
                            other_classification_name, other_classification, other_posts = taxonomy_outputs[lang][path]
                            if other_classification_name == taxonomy.classification_name and other_classification == classification:
                                taxonomy_outputs[lang][path][2].extend(posts)
                            else:
                                utils.LOGGER.error('You have classifications that are too similar: {0} "{1}" and {2} "{3}" both result in output path {4} for language {5}.'.format(
                                    taxonomy.classification_name, classification, other_classification_name, other_classification, path, lang))
                                utils.LOGGER.error('{0} {1} is used in: {2}'.format(
                                    taxonomy.classification_name.title(), classification, ', '.join(sorted([p.source_path for p in posts]))))
                                utils.LOGGER.error('{0} {1} is used in: {2}'.format(
                                    other_classification_name.title(), other_classification, ', '.join(sorted([p.source_path for p in other_posts]))))
                                quit = True
                        else:
                            taxonomy_outputs[lang][path] = (taxonomy.classification_name, classification, list(posts))
        if quit:
            sys.exit(1)

    def _get_filtered_list(self, taxonomy, classification, lang):
        """Return the filtered list of posts for this classification and language."""
        post_list = self.site.posts_per_classification[taxonomy.classification_name][lang].get(classification, [])
        if self.site.config["SHOW_UNTRANSLATED_POSTS"]:
            return post_list
        else:
            return [x for x in post_list if x.is_translation_available(lang)]

    @staticmethod
    def _compute_number_of_pages(filtered_posts, posts_count):
        """Given a list of posts and the maximal number of posts per page, computes the number of pages needed."""
        return min(1, (len(filtered_posts) + posts_count - 1) // posts_count)

    def _postprocess_path(self, path, lang, append_index='auto', dest_type='page', page_info=None, alternative_path=False):
        """Postprocess a generated path.

        Takes the path `path` for language `lang`, and postprocesses it.

        It appends `site.config['INDEX_FILE']` depending on `append_index`
        (which can have the values `'always'`, `'never'` and `'auto'`) and
        `site.config['PRETTY_URLS']`.

        It also modifies/adds the extension of the last path element resp.
        `site.config['INDEX_FILE']` depending on `dest_type`, which can be
        `'feed'`, `'rss'` or `'page'`.

        If `dest_type` is `'page'`, `page_info` can be `None` or a tuple
        of two integers: the page number and the number of pages. This will
        be used to append the correct page number by calling
        `utils.adjust_name_for_index_path_list` and
        `utils.get_displayed_page_number`.

        If `alternative_path` is set to `True`, `utils.adjust_name_for_index_path_list`
        is called with `force_addition=True`, resulting in an alternative path for the
        first page of an index or Atom feed by including the page number into the path.
        """
        # Forcing extension for Atom feeds and RSS feeds
        force_extension = None
        if dest_type == 'feed':
            force_extension = '.atom'
        elif dest_type == 'rss':
            force_extension = '.xml'
        # Determine how to extend path
        path = [_f for _f in path if _f]
        if force_extension is not None:
            if len(path) == 0 and dest_type == 'rss':
                path = ['rss']
            elif len(path) == 0 or append_index == 'always':
                path = path + [os.path.splitext(self.site.config['INDEX_FILE'])[0]]
            path[-1] += force_extension
        elif (self.site.config['PRETTY_URLS'] and append_index != 'never') or len(path) == 0 or append_index == 'always':
            path = path + [self.site.config['INDEX_FILE']]
        else:
            path[-1] += '.html'
        # Create path
        result = [_f for _f in [self.site.config['TRANSLATIONS'][lang]] + path if _f]
        if page_info is not None and dest_type in ('page', 'feed'):
            result = utils.adjust_name_for_index_path_list(result,
                                                           page_info[0],
                                                           utils.get_displayed_page_number(page_info[0], page_info[1], self.site),
                                                           lang,
                                                           self.site, force_addition=alternative_path, extension=force_extension)
        return result

    @staticmethod
    def _parse_path_result(result):
        """Interpret the return values of taxonomy.get_path() and taxonomy.get_overview_path() as if all three return values were given."""
        if not isinstance(result[0], (list, tuple)):
            # The result must be a list or tuple of strings. Wrap into a tuple
            result = (result, )
        path = result[0]
        append_index = result[1] if len(result) > 1 else 'auto'
        page_info = result[2] if len(result) > 2 else None
        return path, append_index, page_info

    def _taxonomy_index_path(self, name, lang, taxonomy):
        """Return path to the classification overview."""
        result = taxonomy.get_overview_path(lang)
        path, append_index, _ = self._parse_path_result(result)
        return self._postprocess_path(path, lang, append_index=append_index, dest_type='list')

    def _taxonomy_path(self, name, lang, taxonomy, dest_type='page', page=None, alternative_path=False):
        """Return path to a classification."""
        if taxonomy.has_hierarchy:
            result = taxonomy.get_path(taxonomy.extract_hierarchy(name), lang, dest_type=dest_type)
        else:
            result = taxonomy.get_path(name, lang, dest_type=dest_type)
        path, append_index, page_ = self._parse_path_result(result)

        if page is not None:
            page = int(page)
        else:
            page = page_

        page_info = None
        if taxonomy.show_list_as_index and page is not None:
            number_of_pages = self.site.page_count_per_classification[taxonomy.classification_name][lang].get(name)
            if number_of_pages is None:
                number_of_pages = self._compute_number_of_pages(self._get_filtered_list(taxonomy, name, lang), self.site.config['INDEX_DISPLAY_POST_COUNT'])
                self.site.page_count_per_classification[taxonomy.classification_name][lang][name] = number_of_pages
            page_info = (page, number_of_pages)
        return self._postprocess_path(path, lang, append_index=append_index, dest_type=dest_type, page_info=page_info)

    def _taxonomy_atom_path(self, name, lang, taxonomy, page=None, alternative_path=False):
        """Return path to a classification Atom feed."""
        return self._taxonomy_path(name, lang, taxonomy, dest_type='feed', page=page, alternative_path=alternative_path)

    def _taxonomy_rss_path(self, name, lang, taxonomy):
        """Return path to a classification RSS feed."""
        return self._taxonomy_path(name, lang, taxonomy, dest_type='rss')

    def _register_path_handlers(self, taxonomy):
        self.site.register_path_handler('{0}_index'.format(taxonomy.classification_name), functools.partial(self._taxonomy_index_path, taxonomy=taxonomy))
        self.site.register_path_handler('{0}'.format(taxonomy.classification_name), functools.partial(self._taxonomy_path, taxonomy=taxonomy))
        self.site.register_path_handler('{0}_atom'.format(taxonomy.classification_name), functools.partial(self._taxonomy_atom_path, taxonomy=taxonomy))
        self.site.register_path_handler('{0}_rss'.format(taxonomy.classification_name), functools.partial(self._taxonomy_rss_path, taxonomy=taxonomy))

    def set_site(self, site):
        """Set site, which is a Nikola instance."""
        super(TaxonomiesClassifier, self).set_site(site)
        # Add hook for after post scanning
        blinker.signal("scanned").connect(self._do_classification)
        # Register path handlers
        for taxonomy in site.taxonomy_plugins.values():
            self._register_path_handlers(taxonomy)
<commit_msg>Improving documentation and plugin metadata.<commit_after># -*- coding: utf-8 -*-

# Copyright © 2012-2016 Roberto Alsina and others.

# Permission is hereby granted, free of charge, to any
# person obtaining a copy of this software and associated
# documentation files (the "Software"), to deal in the
# Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the
# Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice
# shall be included in all copies or substantial portions of
# the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

"""Render the taxonomy overviews, classification pages and feeds."""

from __future__ import unicode_literals
import blinker
import functools
import natsort
import os
import sys

from collections import defaultdict

from nikola.plugin_categories import SignalHandler
from nikola import utils

class TaxonomiesClassifier(SignalHandler):
    """Classify posts and pages by taxonomies."""

    name = "classify_taxonomies"

    def _do_classification(self, site):
        # Needed to avoid strange errors during tests
        if site is not self.site:
            return

        # Get list of enabled taxonomy plugins and initialize data structures
        taxonomies = site.taxonomy_plugins.values()
        site.posts_per_classification = {}
        for taxonomy in taxonomies:
            site.posts_per_classification[taxonomy.classification_name] = {
                lang: defaultdict(set) for lang in site.config['TRANSLATIONS'].keys()
            }

        # Classify posts
        for post in site.timeline:
            if not post.use_in_feeds:
                continue
            for taxonomy in taxonomies:
                if taxonomy.apply_to_posts if post.is_post else taxonomy.apply_to_pages:
                    classifications = {}
                    for lang in site.config['TRANSLATIONS'].keys():
                        # Extract classifications for this language
                        classifications[lang] = taxonomy.classify(post, lang)
                        if not taxonomy.more_than_one_classifications_per_post and len(classifications[lang]) > 1:
                            raise ValueError("Too many {0} classifications for post {1}".format(taxonomy.classification_name, post.source_path))
                        # Add post to sets
                        for classification in classifications[lang]:
                            while True:
                                site.posts_per_classification[taxonomy.classification_name][lang][classification].add(post)
                                if not taxonomy.include_posts_from_subhierarchies or not taxonomy.has_hierarchy:
                                    break
                                classification_path = taxonomy.extract_hierarchy(classification)
                                if len(classification_path) <= 1:
                                    if len(classification_path) == 0 or not taxonomy.include_posts_into_hierarchy_root:
                                        break
                                classification = taxonomy.recombine_classification_from_hierarchy(classification_path[:-1])

        # Sort everything.
        site.page_count_per_classification = {}
        site.hierarchy_per_classification = {}
        site.flat_hierarchy_per_classification = {}
        site.hierarchy_lookup_per_classification = {}
        for taxonomy in taxonomies:
            site.page_count_per_classification[taxonomy.classification_name] = {}
            # Sort post lists
            for lang, posts_per_classification in site.posts_per_classification[taxonomy.classification_name].items():
                # Ensure implicit classifications are inserted
                for classification in taxonomy.get_implicit_classifications(lang):
                    if classification not in posts_per_classification:
                        posts_per_classification[classification] = []
                site.page_count_per_classification[taxonomy.classification_name][lang] = {}
                # Convert sets to lists and sort them
                for classification in list(posts_per_classification.keys()):
                    posts = list(posts_per_classification[classification])
                    posts.sort(key=lambda p:
                               (int(p.meta('priority')) if p.meta('priority') else 0,
                                p.date, p.source_path))
                    posts.reverse()
                    taxonomy.sort_posts(posts, classification, lang)
                    posts_per_classification[classification] = posts
            # Create hierarchy information
            if taxonomy.has_hierarchy:
                site.hierarchy_per_classification[taxonomy.classification_name] = {}
                site.flat_hierarchy_per_classification[taxonomy.classification_name] = {}
                site.hierarchy_lookup_per_classification[taxonomy.classification_name] = {}
                for lang, posts_per_classification in site.posts_per_classification[taxonomy.classification_name].items():
                    # Compose hierarchy
                    hierarchy = {}
                    for classification in posts_per_classification.keys():
                        hier = taxonomy.extract_hierarchy(classification)
                        node = hierarchy
                        for he in hier:
                            if he not in node:
                                node[he] = {}
                            node = node[he]
                    hierarchy_lookup = {}

                    def create_hierarchy(hierarchy, parent=None, level=0):
                        """Create hierarchy."""
                        result = {}
                        for name, children in hierarchy.items():
                            node = utils.TreeNode(name, parent)
                            node.children = create_hierarchy(children, node, level + 1)
                            node.classification_path = [pn.name for pn in node.get_path()]
                            node.classification_name = taxonomy.recombine_classification_from_hierarchy(node.classification_path)
                            hierarchy_lookup[node.classification_name] = node
                            result[node.name] = node
                        classifications = natsort.natsorted(result.keys(), alg=natsort.ns.F | natsort.ns.IC)
                        taxonomy.sort_classifications(classifications, lang, level=level)
                        return [result[classification] for classification in classifications]

                    root_list = create_hierarchy(hierarchy)
                    if '' in posts_per_classification:
                        node = utils.TreeNode('', parent=None)
                        node.children = root_list
                        node.classification_path = []
                        node.classification_name = ''
                        hierarchy_lookup[node.name] = node
                        root_list = [node]
                    flat_hierarchy = utils.flatten_tree_structure(root_list)
                    # Store result
                    site.hierarchy_per_classification[taxonomy.classification_name][lang] = root_list
                    site.flat_hierarchy_per_classification[taxonomy.classification_name][lang] = flat_hierarchy
                    site.hierarchy_lookup_per_classification[taxonomy.classification_name][lang] = hierarchy_lookup
                taxonomy.postprocess_posts_per_classification(site.posts_per_classification[taxonomy.classification_name],
                                                              site.flat_hierarchy_per_classification[taxonomy.classification_name],
                                                              site.hierarchy_lookup_per_classification[taxonomy.classification_name])
            else:
                taxonomy.postprocess_posts_per_classification(site.posts_per_classification[taxonomy.classification_name])

        # Check for valid paths and for collisions
        taxonomy_outputs = {lang: dict() for lang in site.config['TRANSLATIONS'].keys()}
        quit = False
        for taxonomy in taxonomies:
            # Check for collisions (per language)
            for lang in site.config['TRANSLATIONS'].keys():
                if not taxonomy.is_enabled(lang):
                    continue
                for tlang in site.config['TRANSLATIONS'].keys():
                    if lang != tlang and not taxonomy.also_create_classifications_from_other_languages:
                        continue
                    for classification, posts in site.posts_per_classification[taxonomy.classification_name][tlang].items():
                        # Obtain path as tuple
                        path = site.path_handlers[taxonomy.classification_name](classification, lang)
                        # Check that path is OK
                        for path_element in path:
                            if len(path_element) == 0:
                                utils.LOGGER.error("{0} {1} yields invalid path '{2}'!".format(taxonomy.classification_name.title(), classification, '/'.join(path)))
                                quit = True
                        # Combine path
                        path = os.path.join(*[os.path.normpath(p) for p in path if p != '.'])
                        # Determine collisions
                        if path in taxonomy_outputs[lang]:
                            other_classification_name, other_classification, other_posts = taxonomy_outputs[lang][path]
                            if other_classification_name == taxonomy.classification_name and other_classification == classification:
                                taxonomy_outputs[lang][path][2].extend(posts)
                            else:
                                utils.LOGGER.error('You have classifications that are too similar: {0} "{1}" and {2} "{3}" both result in output path {4} for language {5}.'.format(
                                    taxonomy.classification_name, classification, other_classification_name, other_classification, path, lang))
                                utils.LOGGER.error('{0} {1} is used in: {2}'.format(
                                    taxonomy.classification_name.title(), classification, ', '.join(sorted([p.source_path for p in posts]))))
                                utils.LOGGER.error('{0} {1} is used in: {2}'.format(
                                    other_classification_name.title(), other_classification, ', '.join(sorted([p.source_path for p in other_posts]))))
                                quit = True
                        else:
                            taxonomy_outputs[lang][path] = (taxonomy.classification_name, classification, list(posts))
        if quit:
            sys.exit(1)

    def _get_filtered_list(self, taxonomy, classification, lang):
        """Return the filtered list of posts for this classification and language."""
        post_list = self.site.posts_per_classification[taxonomy.classification_name][lang].get(classification, [])
        if self.site.config["SHOW_UNTRANSLATED_POSTS"]:
            return post_list
        else:
            return [x for x in post_list if x.is_translation_available(lang)]

    @staticmethod
    def _compute_number_of_pages(filtered_posts, posts_count):
        """Given a list of posts and the maximal number of posts per page, computes the number of pages needed."""
        return min(1, (len(filtered_posts) + posts_count - 1) // posts_count)

    def _postprocess_path(self, path, lang, append_index='auto', dest_type='page', page_info=None, alternative_path=False):
        """Postprocess a generated path.

        Takes the path `path` for language `lang`, and postprocesses it.

        It appends `site.config['INDEX_FILE']` depending on `append_index`
        (which can have the values `'always'`, `'never'` and `'auto'`) and
        `site.config['PRETTY_URLS']`.

        It also modifies/adds the extension of the last path element resp.
        `site.config['INDEX_FILE']` depending on `dest_type`, which can be
        `'feed'`, `'rss'` or `'page'`.

        If `dest_type` is `'page'`, `page_info` can be `None` or a tuple
        of two integers: the page number and the number of pages. This will
        be used to append the correct page number by calling
        `utils.adjust_name_for_index_path_list` and
        `utils.get_displayed_page_number`.

        If `alternative_path` is set to `True`, `utils.adjust_name_for_index_path_list`
        is called with `force_addition=True`, resulting in an alternative path for the
        first page of an index or Atom feed by including the page number into the path.
        """
        # Forcing extension for Atom feeds and RSS feeds
        force_extension = None
        if dest_type == 'feed':
            force_extension = '.atom'
        elif dest_type == 'rss':
            force_extension = '.xml'
        # Determine how to extend path
        path = [_f for _f in path if _f]
        if force_extension is not None:
            if len(path) == 0 and dest_type == 'rss':
                path = ['rss']
            elif len(path) == 0 or append_index == 'always':
                path = path + [os.path.splitext(self.site.config['INDEX_FILE'])[0]]
            path[-1] += force_extension
        elif (self.site.config['PRETTY_URLS'] and append_index != 'never') or len(path) == 0 or append_index == 'always':
            path = path + [self.site.config['INDEX_FILE']]
        else:
            path[-1] += '.html'
        # Create path
        result = [_f for _f in [self.site.config['TRANSLATIONS'][lang]] + path if _f]
        if page_info is not None and dest_type in ('page', 'feed'):
            result = utils.adjust_name_for_index_path_list(result,
                                                           page_info[0],
                                                           utils.get_displayed_page_number(page_info[0], page_info[1], self.site),
                                                           lang,
                                                           self.site, force_addition=alternative_path, extension=force_extension)
        return result

    @staticmethod
    def _parse_path_result(result):
        """Interpret the return values of taxonomy.get_path() and taxonomy.get_overview_path() as if all three return values were given."""
        if not isinstance(result[0], (list, tuple)):
            # The result must be a list or tuple of strings. Wrap into a tuple
            result = (result, )
        path = result[0]
        append_index = result[1] if len(result) > 1 else 'auto'
        page_info = result[2] if len(result) > 2 else None
        return path, append_index, page_info

    def _taxonomy_index_path(self, name, lang, taxonomy):
        """Return path to the classification overview."""
        result = taxonomy.get_overview_path(lang)
        path, append_index, _ = self._parse_path_result(result)
        return self._postprocess_path(path, lang, append_index=append_index, dest_type='list')

    def _taxonomy_path(self, name, lang, taxonomy, dest_type='page', page=None, alternative_path=False):
        """Return path to a classification."""
        if taxonomy.has_hierarchy:
            result = taxonomy.get_path(taxonomy.extract_hierarchy(name), lang, dest_type=dest_type)
        else:
            result = taxonomy.get_path(name, lang, dest_type=dest_type)
        path, append_index, page_ = self._parse_path_result(result)

        if page is not None:
            page = int(page)
        else:
            page = page_

        page_info = None
        if taxonomy.show_list_as_index and page is not None:
            number_of_pages = self.site.page_count_per_classification[taxonomy.classification_name][lang].get(name)
            if number_of_pages is None:
                number_of_pages = self._compute_number_of_pages(self._get_filtered_list(taxonomy, name, lang), self.site.config['INDEX_DISPLAY_POST_COUNT'])
                self.site.page_count_per_classification[taxonomy.classification_name][lang][name] = number_of_pages
            page_info = (page, number_of_pages)
        return self._postprocess_path(path, lang, append_index=append_index, dest_type=dest_type, page_info=page_info)

    def _taxonomy_atom_path(self, name, lang, taxonomy, page=None, alternative_path=False):
        """Return path to a classification Atom feed."""
        return self._taxonomy_path(name, lang, taxonomy, dest_type='feed', page=page, alternative_path=alternative_path)

    def _taxonomy_rss_path(self, name, lang, taxonomy):
        """Return path to a classification RSS feed."""
        return self._taxonomy_path(name, lang, taxonomy, dest_type='rss')

    def _register_path_handlers(self, taxonomy):
        self.site.register_path_handler('{0}_index'.format(taxonomy.classification_name), functools.partial(self._taxonomy_index_path, taxonomy=taxonomy))
        self.site.register_path_handler('{0}'.format(taxonomy.classification_name), functools.partial(self._taxonomy_path, taxonomy=taxonomy))
        self.site.register_path_handler('{0}_atom'.format(taxonomy.classification_name), functools.partial(self._taxonomy_atom_path, taxonomy=taxonomy))
        self.site.register_path_handler('{0}_rss'.format(taxonomy.classification_name), functools.partial(self._taxonomy_rss_path, taxonomy=taxonomy))

    def set_site(self, site):
        """Set site, which is a Nikola instance."""
        super(TaxonomiesClassifier, self).set_site(site)
        # Add hook for after post scanning
        blinker.signal("scanned").connect(self._do_classification)
        # Register path handlers
        for taxonomy in site.taxonomy_plugins.values():
            self._register_path_handlers(taxonomy)
<|endoftext|>


<commit_before># emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
import numpy as np

from ..coord_tools import coord_transform, find_cut_coords

def test_coord_transform_trivial():
    sform = np.eye(4)
    x = np.random.random((10,))
    y = np.random.random((10,))
    z = np.random.random((10,))

    x_, y_, z_ = coord_transform(x, y, z, sform)
    np.testing.assert_array_equal(x, x_)
    np.testing.assert_array_equal(y, y_)
    np.testing.assert_array_equal(z, z_)

    sform[:, -1] = 1
    x_, y_, z_ = coord_transform(x, y, z, sform)
    np.testing.assert_array_equal(x+1, x_)
    np.testing.assert_array_equal(y+1, y_)
    np.testing.assert_array_equal(z+1, z_)

def test_find_cut_coords():
    map = np.zeros((100, 100, 100))
    x_map, y_map, z_map = 50, 10, 40
    map[x_map-30:x_map+30, y_map-3:y_map+3, z_map-10:z_map+10] = 1
    x, y, z = find_cut_coords(map, mask=np.ones(map.shape, np.bool))
    np.testing.assert_array_equal(
                        (int(round(x)), int(round(y)), int(round(z))),
                                (x_map, y_map, z_map))

<commit_msg>TST: add basic tests for find_maxsep_cut_coords<commit_after># emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
import numpy as np

from ..coord_tools import (coord_transform, find_cut_coords,
                           find_maxsep_cut_coords)

from numpy.testing import assert_array_equal

def test_coord_transform_trivial():
    sform = np.eye(4)
    x = np.random.random((10,))
    y = np.random.random((10,))
    z = np.random.random((10,))

    x_, y_, z_ = coord_transform(x, y, z, sform)
    np.testing.assert_array_equal(x, x_)
    np.testing.assert_array_equal(y, y_)
    np.testing.assert_array_equal(z, z_)

    sform[:, -1] = 1
    x_, y_, z_ = coord_transform(x, y, z, sform)
    np.testing.assert_array_equal(x+1, x_)
    np.testing.assert_array_equal(y+1, y_)
    np.testing.assert_array_equal(z+1, z_)

def test_find_cut_coords():
    map = np.zeros((100, 100, 100))
    x_map, y_map, z_map = 50, 10, 40
    map[x_map-30:x_map+30, y_map-3:y_map+3, z_map-10:z_map+10] = 1
    x, y, z = find_cut_coords(map, mask=np.ones(map.shape, np.bool))
    np.testing.assert_array_equal(
                        (int(round(x)), int(round(y)), int(round(z))),
                                (x_map, y_map, z_map))

def test_find_maxsep_cut_coords():
    # Test find_maxsep_cut_coords function
    assert_array_equal(
        find_maxsep_cut_coords(np.ones((2, 3, 5)), np.eye(4)), range(5))
    assert_array_equal(
        find_maxsep_cut_coords(np.ones((2, 3, 5)), np.eye(4), threshold=1),
        range(5))
    assert_array_equal(
        find_maxsep_cut_coords(np.ones((2, 3, 4)), np.eye(4), n_cuts=4),
        range(4))
    map_3d = np.ones((2, 3, 5))
    map_3d[:, :, 1] = 0
    assert_array_equal(
        find_maxsep_cut_coords(map_3d, np.eye(4), n_cuts=4), [0, 2, 3, 4])
    map_3d[:, :, 1] = 0.5
    assert_array_equal(
        find_maxsep_cut_coords(map_3d, np.eye(4), n_cuts=4, threshold=0.6),
        [0, 2, 3, 4])
<|endoftext|>


<commit_before># Copyright 2014-2017 The ODL contributors
#
# This file is part of ODL.
#
# This Source Code Form is subject to the terms of the Mozilla Public License,
# v. 2.0. If a copy of the MPL was not distributed with this file, You can
# obtain one at https://mozilla.org/MPL/2.0/.

"""Test for parameter optimization."""

import pytest
import odl
import odl.contrib.fom
import odl.contrib.param_opt
from odl.util.testutils import simple_fixture

space = simple_fixture('space',
                       [odl.rn(3),
                        odl.uniform_discr([0, 0], [1, 1], [9, 11]),
                        odl.uniform_discr(0, 1, 10)])

fom = simple_fixture('fom',
                     [odl.contrib.fom.mean_squared_error,
                      odl.contrib.fom.mean_absolute_error])

def test_optimal_parameters_one_parameter(space, fom):
    """Tests if optimal_parameters works for some simple examples."""
#    fom = odl.contrib.fom.mean_squared_error
    mynoise = odl.phantom.white_noise(space)
    phantoms = [mynoise]
    data = [mynoise]

    def reconstruction(data, lam):
        """Perturbs the data by adding lam to it."""
        return data + lam

    result = odl.contrib.param_opt.optimal_parameters(reconstruction, fom,
                                                      phantoms, data, 1)
    assert result == pytest.approx(0, abs=1e-4)

def test_optimal_parameters_two_parameters(space, fom):
    """Tests if optimal_parameters works for some simple examples."""
#    fom = odl.contrib.fom.mean_squared_error
    mynoise = odl.phantom.white_noise(space)
    phantoms = [mynoise]
    data = [mynoise]

    def reconstruction1(data, params):
        """Perturbs the data by adding the sum of squares of params to it."""
        return data + sum([param ** 2 for param in params])

    result1 = odl.contrib.param_opt.optimal_parameters(reconstruction1, fom,
                                                       phantoms, data, [1, 2])
    assert list(result1) == pytest.approx([0, 0], abs=1e-4)

    def reconstruction2(data, params):
        """Perturbs the data by adding the sum of params to it."""
        return data + sum(params)

    result2 = odl.contrib.param_opt.optimal_parameters(reconstruction2, fom,
                                                       phantoms, data, [1, 2])
    assert sum(result2) == pytest.approx(0, abs=1e-4)

if __name__ == '__main__':
    odl.util.test_file(__file__)
<commit_msg>TST: Extend the parameter optimization tests to two data-phantom pairs.<commit_after># Copyright 2014-2017 The ODL contributors
#
# This file is part of ODL.
#
# This Source Code Form is subject to the terms of the Mozilla Public License,
# v. 2.0. If a copy of the MPL was not distributed with this file, You can
# obtain one at https://mozilla.org/MPL/2.0/.

"""Test for parameter optimization."""

import pytest
import odl
import odl.contrib.fom
import odl.contrib.param_opt
from odl.util.testutils import simple_fixture

space = simple_fixture('space',
                       [odl.rn(3),
                        odl.uniform_discr([0, 0], [1, 1], [9, 11]),
                        odl.uniform_discr(0, 1, 10)])

fom = simple_fixture('fom',
                     [odl.contrib.fom.mean_squared_error,
                      odl.contrib.fom.mean_absolute_error])

def test_optimal_parameters_one_parameter(space, fom):
    """Tests if optimal_parameters works for some simple examples."""
    mynoise = [odl.phantom.white_noise(space) for _ in range(2)]
    phantoms = mynoise.copy()
    data = mynoise.copy()

    def reconstruction(data, lam):
        """Perturbs the data by adding lam to it."""
        return data + lam

    result = odl.contrib.param_opt.optimal_parameters(reconstruction, fom,
                                                      phantoms, data, 1)
    assert result == pytest.approx(0, abs=1e-4)

def test_optimal_parameters_two_parameters(space, fom):
    """Tests if optimal_parameters works for some simple examples."""
    mynoise = [odl.phantom.white_noise(space) for _ in range(2)]
    phantoms = mynoise.copy()
    data = mynoise.copy()

    def reconstruction1(data, params):
        """Perturbs the data by adding the sum of squares of params to it."""
        return data + sum([param ** 2 for param in params])

    result1 = odl.contrib.param_opt.optimal_parameters(reconstruction1, fom,
                                                       phantoms, data, [1, 2])
    assert list(result1) == pytest.approx([0, 0], abs=1e-4)

    def reconstruction2(data, params):
        """Perturbs the data by adding the sum of params to it."""
        return data + sum(params)

    result2 = odl.contrib.param_opt.optimal_parameters(reconstruction2, fom,
                                                       phantoms, data, [1, 2])
    assert sum(result2) == pytest.approx(0, abs=1e-4)

if __name__ == '__main__':
    odl.util.test_file(__file__)
<|endoftext|>


<commit_before># Copyright 2014-2017 The ODL contributors
#
# This file is part of ODL.
#
# This Source Code Form is subject to the terms of the Mozilla Public License,
# v. 2.0. If a copy of the MPL was not distributed with this file, You can
# obtain one at https://mozilla.org/MPL/2.0/.

"""Test for the alternating dual updates solver."""

from __future__ import division
import odl
from odl.solvers.nonsmooth import adupdates_method
from odl.solvers.nonsmooth.adupdates import adupdates_method_simple

from odl.util.testutils import all_almost_equal

# Places for the accepted error when comparing results
HIGH_ACCURACY = 8
LOW_ACCURACY = 4

def test_adupdates():
    """Test if the adupdates solver handles the following problem correctly:

    ( 1  1/2 1/3 1/4) (x_1)   (a)
    (1/2 1/3 1/4 1/5) (x_2)   (b)
    (1/3 1/4 1/5 1/6) (x_3) = (c)
    (1/4 1/5 1/6 1/7) (x_4)   (d).

    The matrix is the ill-conditined Hilbert matrix, the inverse of which
    can be given in closed form. If we set

    (a)   (25/12)                    (x_1) = (1)
    (b)   (77/60)                    (x_2) = (1)
    (c) = (19/20)         then       (x_3) = (1)
    (d)   (319/420),                 (x_4) = (1).

    We therefore solve the problem

    min ||Ax - b||^2 + TV(x) s.t. x >= 0

    for the matrix A, the r.h.s. b as above and the total variation TV.
    The solution of this problem is clearly x = (1, 1, 1, 1).
    """

    mat1 = [[1, 1 / 2, 1 / 3, 1 / 4],
            [1 / 2, 1 / 3, 1 / 4, 1 / 5]]
    mat2 = [[1 / 3, 1 / 4, 1 / 5, 1 / 6],
            [1 / 4, 1 / 5, 1 / 6, 1 / 7]]

    # Create the linear operators
    mat1op = odl.MatrixOperator(mat1)
    mat2op = odl.MatrixOperator(mat2)
    domain = mat1op.domain
    tv1 = odl.MatrixOperator([[1.0, -1.0, 0.0, 0.0],
                              [0.0, 0.0, 1.0, -1.0]])
    tv2 = odl.MatrixOperator([[0.0, 1.0, -1.0, 0.0]])
    nneg = odl.IdentityOperator(domain)
    ops = [mat1op, mat2op, tv1, tv2, nneg]

    # Create inner stepsizes for linear operators
    mat1s = 1 / mat1op(mat1op.adjoint(mat1op.range.one()))
    mat2s = 1 / mat2op(mat2op.adjoint(mat2op.range.one()))
    tv1s = tv1.range.element([0.5, 0.5])
    tv2s = 0.5
    nnegs = nneg.range.element([1.0, 1.0, 1.0, 1.0])
    inner_stepsizes = [mat1s, mat2s, tv1s, tv2s, nnegs]

    expected_solution = domain.element([1, 1, 1, 1])
    # Create right-hand-sides of the equation
    rhs1 = mat1op(expected_solution)
    rhs2 = mat2op(expected_solution)

    # Create the functionals
    fid1 = odl.solvers.L2NormSquared(mat1op.range).translated(rhs1)
    fid2 = odl.solvers.L2NormSquared(mat2op.range).translated(rhs2)
    reg1 = odl.solvers.L1Norm(tv1.range)
    reg2 = odl.solvers.L1Norm(tv2.range)
    ind = odl.solvers.IndicatorNonnegativity(nneg.range)
    funcs = [fid1, fid2, reg1, reg2, ind]

    # Start from zero
    x = tv1.domain.zero()
    x_simple = tv1.domain.zero()

    stepsize = 1
    niter = 100

    adupdates_method(x, funcs, ops, stepsize, inner_stepsizes, niter)
    adupdates_method_simple(x_simple, funcs, ops, stepsize,
                            inner_stepsizes, niter)
    assert all_almost_equal(x, x_simple)
    assert domain.dist(x, expected_solution) < 1e-3

if __name__ == '__main__':
    odl.util.test_file(__file__)
<commit_msg>DOC/TST: Add clarifying remarks to the docstring of the adupdates test.<commit_after># Copyright 2014-2017 The ODL contributors
#
# This file is part of ODL.
#
# This Source Code Form is subject to the terms of the Mozilla Public License,
# v. 2.0. If a copy of the MPL was not distributed with this file, You can
# obtain one at https://mozilla.org/MPL/2.0/.

"""Test for the alternating dual updates solver."""

from __future__ import division
import odl
from odl.solvers.nonsmooth import adupdates_method
from odl.solvers.nonsmooth.adupdates import adupdates_method_simple

from odl.util.testutils import all_almost_equal

# Places for the accepted error when comparing results
HIGH_ACCURACY = 8
LOW_ACCURACY = 4

def test_adupdates():
    """Test if the adupdates solver handles the following problem correctly:

    ( 1  1/2 1/3 1/4) (x_1)   (a)
    (1/2 1/3 1/4 1/5) (x_2)   (b)
    (1/3 1/4 1/5 1/6) (x_3) = (c)
    (1/4 1/5 1/6 1/7) (x_4)   (d).

    The matrix is the ill-conditined Hilbert matrix, the inverse of which
    can be given in closed form. If we set

    (a)   (25/12)                    (x_1) = (1)
    (b)   (77/60)                    (x_2) = (1)
    (c) = (19/20)         then       (x_3) = (1)
    (d)   (319/420),                 (x_4) = (1).

    We solve the problem

    min ||Ax - b||^2 + TV(x) s.t. x >= 0

    for the matrix A, the r.h.s. b as above and the total variation TV, which
    is given as the sum of the absoulute values of consecutive entries of the
    solution. The solution of this problem is clearly x = (1, 1, 1, 1), since
    it satisfies the additional constraint and minimizes both terms of the
    objective function.
    """

    mat1 = [[1, 1 / 2, 1 / 3, 1 / 4],
            [1 / 2, 1 / 3, 1 / 4, 1 / 5]]
    mat2 = [[1 / 3, 1 / 4, 1 / 5, 1 / 6],
            [1 / 4, 1 / 5, 1 / 6, 1 / 7]]

    # Create the linear operators
    mat1op = odl.MatrixOperator(mat1)
    mat2op = odl.MatrixOperator(mat2)
    domain = mat1op.domain
    tv1 = odl.MatrixOperator([[1.0, -1.0, 0.0, 0.0],
                              [0.0, 0.0, 1.0, -1.0]])
    tv2 = odl.MatrixOperator([[0.0, 1.0, -1.0, 0.0]])
    nneg = odl.IdentityOperator(domain)
    ops = [mat1op, mat2op, tv1, tv2, nneg]

    # Create inner stepsizes for linear operators
    mat1s = 1 / mat1op(mat1op.adjoint(mat1op.range.one()))
    mat2s = 1 / mat2op(mat2op.adjoint(mat2op.range.one()))
    tv1s = tv1.range.element([0.5, 0.5])
    tv2s = 0.5
    nnegs = nneg.range.element([1.0, 1.0, 1.0, 1.0])
    inner_stepsizes = [mat1s, mat2s, tv1s, tv2s, nnegs]

    expected_solution = domain.element([1, 1, 1, 1])
    # Create right-hand-sides of the equation
    rhs1 = mat1op(expected_solution)
    rhs2 = mat2op(expected_solution)

    # Create the functionals
    fid1 = odl.solvers.L2NormSquared(mat1op.range).translated(rhs1)
    fid2 = odl.solvers.L2NormSquared(mat2op.range).translated(rhs2)
    reg1 = odl.solvers.L1Norm(tv1.range)
    reg2 = odl.solvers.L1Norm(tv2.range)
    ind = odl.solvers.IndicatorNonnegativity(nneg.range)
    funcs = [fid1, fid2, reg1, reg2, ind]

    # Start from zero
    x = tv1.domain.zero()
    x_simple = tv1.domain.zero()

    stepsize = 1
    niter = 100

    adupdates_method(x, funcs, ops, stepsize, inner_stepsizes, niter)
    adupdates_method_simple(x_simple, funcs, ops, stepsize,
                            inner_stepsizes, niter)
    assert all_almost_equal(x, x_simple)
    assert domain.dist(x, expected_solution) < 1e-3

if __name__ == '__main__':
    odl.util.test_file(__file__)
<|endoftext|>


<commit_before>import os
import unittest
from mock import patch

from onadata.apps.main.tests.test_base import TestBase
from onadata.libs.utils.chart_tools import build_chart_data_for_field,\
    build_chart_data, utc_time_string_for_javascript, calculate_ranges
from pyxform.question import InputQuestion

def find_field_by_name(dd, field_name):
        return filter(
            lambda f: f.name == field_name, [e for e in dd.survey_elements])[0]

class TestChartTools(TestBase):

    def setUp(self):
        super(TestChartTools, self).setUp()
        # create an xform
        path = os.path.join(os.path.dirname(__file__), "..", "..", "..",
                            "apps", "api", "tests", "fixtures", "forms",
                            "tutorial", "tutorial.xls")
        self._publish_xls_file_and_set_xform(path)
        # make a couple of submissions
        for i in range(1, 3):
            path = os.path.join(os.path.dirname(__file__), "..", "..", "..",
                                "apps", "api", "tests", "fixtures", "forms",
                                "tutorial", "instances", "{}.xml".format(i))
            self._make_submission(path)

    def test_build_chart_data_for_field_on_submission_time(self):
        data = build_chart_data_for_field(self.xform, '_submission_time')
        self.assertEqual(data['field_name'], '_submission_time')
        self.assertEqual(data['field_type'], 'datetime')
        self.assertEqual(data['data_type'], 'time_based')

    def test_build_chart_data_for_field_on_select_one(self):
        dd = self.xform.data_dictionary()
        field = find_field_by_name(dd, 'gender')
        data = build_chart_data_for_field(self.xform, field)
        self.assertEqual(data['field_name'], 'gender')
        self.assertEqual(data['field_type'], 'select one')
        self.assertEqual(data['data_type'], 'categorized')
        # map the list to a dict
        values = dict([(d['gender'], d['count'], ) for d in data['data']])
        self.assertEqual(values, {
            'Male': 1,
            'Female': 1
        })

    def test_build_chart_data_for_field_on_grouped_field(self):
        dd = self.xform.data_dictionary()
        field = find_field_by_name(dd, 'a_text')
        data = build_chart_data_for_field(self.xform, field)
        self.assertEqual(data['field_name'], 'a_group-a_text')
        self.assertEqual(data['field_xpath'], 'a_text')
        self.assertEqual(data['field_type'], 'text')
        self.assertEqual(data['data_type'], 'categorized')

    def test_build_chart_data_output(self):
        data = build_chart_data(self.xform)
        self.assertIsInstance(data, list)
        # check expected fields
        expected_fields = sorted(['_submission_time', 'pizza_type', 'age',
                                  'gender', 'date', 'pizza_fan', 'net_worth',
                                  'start_time', 'end_time', 'today'])
        data_field_names = sorted([f['field_name'] for f in data])
        self.assertEqual(expected_fields, data_field_names)

    def test_build_chart_data_strips_none_from_dates(self):
        # make the 3rd submission that doesnt have a date
        path = os.path.join(os.path.dirname(__file__), "..", "..", "..",
                            "apps", "api", "tests", "fixtures", "forms",
                            "tutorial", "instances", "3.xml")
        self._make_submission(path)
        dd = self.xform.data_dictionary()
        field = find_field_by_name(dd, 'date')
        data = build_chart_data_for_field(self.xform, field)
        # create a list with comparisons to the dict values
        values = [d['date'] is not None for d in data['data']]
        self.assertTrue(all(values))

    def test_build_chart_data_for_field_with_language(self):
        path = os.path.join(os.path.dirname(__file__), "..", "..", "..",
                            "apps", "main", "tests", "fixtures",
                            "good_eats_multilang", "good_eats_multilang.xls")
        self._publish_xls_file_and_set_xform(path)
        path = os.path.join(os.path.dirname(__file__), "..", "..", "..",
                            "apps", "main", "tests", "fixtures",
                            "good_eats_multilang", "1.xml")
        self._make_submission(path)
        dd = self.xform.data_dictionary()
        field = find_field_by_name(dd, 'food_type')
        data = build_chart_data_for_field(self.xform, field, language_index=1)
        self.assertEqual(data['field_label'], u"Type of Eat")

    def test_build_chart_data_for_field_with_language_on_non_lang_field(self):
        path = os.path.join(os.path.dirname(__file__), "..", "..", "..",
                            "apps", "main", "tests", "fixtures",
                            "good_eats_multilang", "good_eats_multilang.xls")
        self._publish_xls_file_and_set_xform(path)
        path = os.path.join(os.path.dirname(__file__), "..", "..", "..",
                            "apps", "main", "tests", "fixtures",
                            "good_eats_multilang", "1.xml")
        self._make_submission(path)
        dd = self.xform.data_dictionary()
        field = find_field_by_name(dd, 'submit_date')
        data = build_chart_data_for_field(self.xform, field, language_index=1)
        self.assertEqual(data['field_label'], 'submit_date')

    def test_build_chart_data_with_nonexisting_field_xpath(self):
        # make the 3rd submission that doesnt have a date
        path = os.path.join(os.path.dirname(__file__), "..", "..", "..",
                            "apps", "api", "tests", "fixtures", "forms",
                            "tutorial", "instances", "3.xml")
        self._make_submission(path)
        dd = self.xform.data_dictionary()
        field = find_field_by_name(dd, 'date')
        field.name = 'informed_consent/pas_denfants_elig/q7b'

        data = build_chart_data_for_field(self.xform, field)
        # create a list with comparisons to the dict values
        values = [d['date'] is not None for d in data['data']]
        self.assertTrue(all(values))

    def test_build_chart_data_with_field_name_with_lengh_65(self):
        # make the 3rd submission that doesnt have a date
        path = os.path.join(os.path.dirname(__file__), "..", "..", "..",
                            "apps", "api", "tests", "fixtures", "forms",
                            "tutorial", "instances", "3.xml")
        self._make_submission(path)
        dd = self.xform.data_dictionary()
        field = find_field_by_name(dd, 'date')
        field.name = 'a' * 65

        data = build_chart_data_for_field(self.xform, field)
        self.assertEqual(data['field_name'], field.name)

    def mock_get_abbreviated_xpath(self):
        return 'informed_consent/pas_denfants_elig/date'

    @patch.object(InputQuestion, 'get_abbreviated_xpath',
                  mock_get_abbreviated_xpath)
    def test_build_chart_data_with_datetime_field(self):
        dd = self.xform.data_dictionary()
        field = find_field_by_name(dd, 'date')

        data = build_chart_data_for_field(self.xform, field)
        data = build_chart_data_for_field(self.xform, field)

        values = [d['date'] is not None for d in data['data']]
        self.assertTrue(all(values))

class TestChartUtilFunctions(unittest.TestCase):

    def test_utc_time_string_for_javascript(self):
        time_str = '2014-01-16T12:07:23.322+03'
        expected_time_str = '2014-01-16T12:07:23.322+0300'
        result = utc_time_string_for_javascript(time_str)
        self.assertEqual(result, expected_time_str)

    def test_raise_value_error_if_no_match(self):
        time_str = '2014-01-16T12:07:23.322'
        self.assertRaises(ValueError, utc_time_string_for_javascript, time_str)

    def test_raise_value_error_if_bad_time_zone(self):
        time_str = '2014-01-16T12:07:23.322+033'
        self.assertRaises(ValueError, utc_time_string_for_javascript, time_str)

    def test_calculate_range_on_normal_values(self):
        requested_page = 0
        items_per_page = 2
        num_items = 8
        ranges = calculate_ranges(requested_page, items_per_page, num_items)
        self.assertEqual(ranges, (0, 2))

    def test_calculate_range_when_page_is_beyond_limit(self):
        requested_page = 10
        items_per_page = 5
        num_items = 41
        ranges = calculate_ranges(requested_page, items_per_page, num_items)
        self.assertEqual(ranges, (41, 41))
<commit_msg>GM: Removed duplicated function call<commit_after>import os
import unittest
from mock import patch

from onadata.apps.main.tests.test_base import TestBase
from onadata.libs.utils.chart_tools import build_chart_data_for_field,\
    build_chart_data, utc_time_string_for_javascript, calculate_ranges
from pyxform.question import InputQuestion

def find_field_by_name(dd, field_name):
        return filter(
            lambda f: f.name == field_name, [e for e in dd.survey_elements])[0]

class TestChartTools(TestBase):

    def setUp(self):
        super(TestChartTools, self).setUp()
        # create an xform
        path = os.path.join(os.path.dirname(__file__), "..", "..", "..",
                            "apps", "api", "tests", "fixtures", "forms",
                            "tutorial", "tutorial.xls")
        self._publish_xls_file_and_set_xform(path)
        # make a couple of submissions
        for i in range(1, 3):
            path = os.path.join(os.path.dirname(__file__), "..", "..", "..",
                                "apps", "api", "tests", "fixtures", "forms",
                                "tutorial", "instances", "{}.xml".format(i))
            self._make_submission(path)

    def test_build_chart_data_for_field_on_submission_time(self):
        data = build_chart_data_for_field(self.xform, '_submission_time')
        self.assertEqual(data['field_name'], '_submission_time')
        self.assertEqual(data['field_type'], 'datetime')
        self.assertEqual(data['data_type'], 'time_based')

    def test_build_chart_data_for_field_on_select_one(self):
        dd = self.xform.data_dictionary()
        field = find_field_by_name(dd, 'gender')
        data = build_chart_data_for_field(self.xform, field)
        self.assertEqual(data['field_name'], 'gender')
        self.assertEqual(data['field_type'], 'select one')
        self.assertEqual(data['data_type'], 'categorized')
        # map the list to a dict
        values = dict([(d['gender'], d['count'], ) for d in data['data']])
        self.assertEqual(values, {
            'Male': 1,
            'Female': 1
        })

    def test_build_chart_data_for_field_on_grouped_field(self):
        dd = self.xform.data_dictionary()
        field = find_field_by_name(dd, 'a_text')
        data = build_chart_data_for_field(self.xform, field)
        self.assertEqual(data['field_name'], 'a_group-a_text')
        self.assertEqual(data['field_xpath'], 'a_text')
        self.assertEqual(data['field_type'], 'text')
        self.assertEqual(data['data_type'], 'categorized')

    def test_build_chart_data_output(self):
        data = build_chart_data(self.xform)
        self.assertIsInstance(data, list)
        # check expected fields
        expected_fields = sorted(['_submission_time', 'pizza_type', 'age',
                                  'gender', 'date', 'pizza_fan', 'net_worth',
                                  'start_time', 'end_time', 'today'])
        data_field_names = sorted([f['field_name'] for f in data])
        self.assertEqual(expected_fields, data_field_names)

    def test_build_chart_data_strips_none_from_dates(self):
        # make the 3rd submission that doesnt have a date
        path = os.path.join(os.path.dirname(__file__), "..", "..", "..",
                            "apps", "api", "tests", "fixtures", "forms",
                            "tutorial", "instances", "3.xml")
        self._make_submission(path)
        dd = self.xform.data_dictionary()
        field = find_field_by_name(dd, 'date')
        data = build_chart_data_for_field(self.xform, field)
        # create a list with comparisons to the dict values
        values = [d['date'] is not None for d in data['data']]
        self.assertTrue(all(values))

    def test_build_chart_data_for_field_with_language(self):
        path = os.path.join(os.path.dirname(__file__), "..", "..", "..",
                            "apps", "main", "tests", "fixtures",
                            "good_eats_multilang", "good_eats_multilang.xls")
        self._publish_xls_file_and_set_xform(path)
        path = os.path.join(os.path.dirname(__file__), "..", "..", "..",
                            "apps", "main", "tests", "fixtures",
                            "good_eats_multilang", "1.xml")
        self._make_submission(path)
        dd = self.xform.data_dictionary()
        field = find_field_by_name(dd, 'food_type')
        data = build_chart_data_for_field(self.xform, field, language_index=1)
        self.assertEqual(data['field_label'], u"Type of Eat")

    def test_build_chart_data_for_field_with_language_on_non_lang_field(self):
        path = os.path.join(os.path.dirname(__file__), "..", "..", "..",
                            "apps", "main", "tests", "fixtures",
                            "good_eats_multilang", "good_eats_multilang.xls")
        self._publish_xls_file_and_set_xform(path)
        path = os.path.join(os.path.dirname(__file__), "..", "..", "..",
                            "apps", "main", "tests", "fixtures",
                            "good_eats_multilang", "1.xml")
        self._make_submission(path)
        dd = self.xform.data_dictionary()
        field = find_field_by_name(dd, 'submit_date')
        data = build_chart_data_for_field(self.xform, field, language_index=1)
        self.assertEqual(data['field_label'], 'submit_date')

    def test_build_chart_data_with_nonexisting_field_xpath(self):
        # make the 3rd submission that doesnt have a date
        path = os.path.join(os.path.dirname(__file__), "..", "..", "..",
                            "apps", "api", "tests", "fixtures", "forms",
                            "tutorial", "instances", "3.xml")
        self._make_submission(path)
        dd = self.xform.data_dictionary()
        field = find_field_by_name(dd, 'date')
        field.name = 'informed_consent/pas_denfants_elig/q7b'

        data = build_chart_data_for_field(self.xform, field)
        # create a list with comparisons to the dict values
        values = [d['date'] is not None for d in data['data']]
        self.assertTrue(all(values))

    def test_build_chart_data_with_field_name_with_lengh_65(self):
        # make the 3rd submission that doesnt have a date
        path = os.path.join(os.path.dirname(__file__), "..", "..", "..",
                            "apps", "api", "tests", "fixtures", "forms",
                            "tutorial", "instances", "3.xml")
        self._make_submission(path)
        dd = self.xform.data_dictionary()
        field = find_field_by_name(dd, 'date')
        field.name = 'a' * 65

        data = build_chart_data_for_field(self.xform, field)
        self.assertEqual(data['field_name'], field.name)

    def mock_get_abbreviated_xpath(self):
        return 'informed_consent/pas_denfants_elig/date'

    @patch.object(InputQuestion, 'get_abbreviated_xpath',
                  mock_get_abbreviated_xpath)
    def test_build_chart_data_with_datetime_field(self):
        dd = self.xform.data_dictionary()
        field = find_field_by_name(dd, 'date')

        data = build_chart_data_for_field(self.xform, field)

        values = [d['date'] is not None for d in data['data']]
        self.assertTrue(all(values))

class TestChartUtilFunctions(unittest.TestCase):

    def test_utc_time_string_for_javascript(self):
        time_str = '2014-01-16T12:07:23.322+03'
        expected_time_str = '2014-01-16T12:07:23.322+0300'
        result = utc_time_string_for_javascript(time_str)
        self.assertEqual(result, expected_time_str)

    def test_raise_value_error_if_no_match(self):
        time_str = '2014-01-16T12:07:23.322'
        self.assertRaises(ValueError, utc_time_string_for_javascript, time_str)

    def test_raise_value_error_if_bad_time_zone(self):
        time_str = '2014-01-16T12:07:23.322+033'
        self.assertRaises(ValueError, utc_time_string_for_javascript, time_str)

    def test_calculate_range_on_normal_values(self):
        requested_page = 0
        items_per_page = 2
        num_items = 8
        ranges = calculate_ranges(requested_page, items_per_page, num_items)
        self.assertEqual(ranges, (0, 2))

    def test_calculate_range_when_page_is_beyond_limit(self):
        requested_page = 10
        items_per_page = 5
        num_items = 41
        ranges = calculate_ranges(requested_page, items_per_page, num_items)
        self.assertEqual(ranges, (41, 41))
<|endoftext|>