<commit_before>import os
import imghdr
from collections import namedtuple

import numpy as np
from skimage import io
from skimage import img_as_ubyte
from skimage.transform import resize
from skimage.color import color_dict
from skimage._shared import six

# Convert colors from `skimage.color` to uint8 and allow access through
# dict or a named tuple.
color_dict = dict((name, tuple(int(255 * c + 0.5) for c in rgb))
                  for name, rgb in color_dict.iteritems())
colors = namedtuple('colors', color_dict.keys())(**color_dict)

def open(path):
    """Return Picture object from the given image path."""
    return Picture(path=os.path.abspath(path))

class Pixel(object):
    """A single pixel in a Picture.

    Attributes
    ----------
    pic : Picture
        The Picture object that this pixel references.
    array : array_like
        Byte array with raw image data (RGB).
    x : int
        Horizontal coordinate of this pixel (left = 0).
    y : int
        Vertical coordinate of this pixel (bottom = 0).
    rgb : tuple
        RGB tuple with red, green, and blue components (0-255)

    """
    def __init__(self, pic, array, x, y, rgb):
        self._picture = pic
        self._array = array
        self._x = x
        self._y = y
        self._red = self._validate(rgb[0])
        self._green = self._validate(rgb[1])
        self._blue = self._validate(rgb[2])

    @property
    def x(self):
        """Horizontal location of this pixel in the parent image(left = 0)."""
        return self._x

    @property
    def y(self):
        """Vertical location of this pixel in the parent image (bottom = 0)."""
        return self._y

    @property
    def red(self):
        """The red component of the pixel (0-255)."""
        return self._red

    @red.setter
    def red(self, value):
        self._red = self._validate(value)
        self._setpixel()

    @property
    def green(self):
        """The green component of the pixel (0-255)."""
        return self._green

    @green.setter
    def green(self, value):
        self._green = self._validate(value)
        self._setpixel()

    @property
    def blue(self):
        """The blue component of the pixel (0-255)."""
        return self._blue

    @blue.setter
    def blue(self, value):
        self._blue = self._validate(value)
        self._setpixel()

    @property
    def rgb(self):
        """The RGB color components of the pixel (3 values 0-255)."""
        return (self.red, self.green, self.blue)

    @rgb.setter
    def rgb(self, value):
        self._red, self._green, self._blue = (self._validate(v) for v in value)
        self._setpixel()

    def _validate(self, value):
        """Verifies that the pixel value is in [0, 255]."""
        try:
            value = int(value)
            if (value < 0) or (value > 255):
                raise ValueError()
        except ValueError:
            msg = "Expected an integer between 0 and 255, but got {0} instead!"
            raise ValueError(msg.format(value))

        return value

    def _setpixel(self):
        """Sets the actual pixel value in the picture.

        NOTE: Using Cartesian coordinate system!

        """
        row = self._picture.height - self._y - 1
        self._picture._array[row, self._x] = self.rgb
        self._picture._array_modified()

    def __repr__(self):
        args = self.red, self.green, self.blue
        return "Pixel (red: {0}, green: {1}, blue: {2})".format(*args)

class PixelGroup(object):
    """A group of Pixel objects that can be manipulated together.

    Attributes
    ----------
    pic : Picture
        The Picture object that this pixel group references.
    key : tuple
        tuple with x and y slices or ints for extracting part of raw image data.

    """
    def __init__(self, pic, key):
        self._pic = pic

        # Use a slice so that the _getdim and _setdim functions can index
        # consistently.
        if isinstance(key[0], int):
            key = (slice(key[0], key[0] + 1), key[1])

        if isinstance(key[1], int):
            key = (key[0], slice(key[1], key[1] + 1))

        for dim_slice in key:
            if dim_slice.start is not None and dim_slice.start < 0:
                raise IndexError("Negative slicing not supported")

            if dim_slice.stop is not None and dim_slice.stop < 0:
                raise IndexError("Negative slicing not supported")

            if dim_slice.step is not None and dim_slice.step != 1:
                raise IndexError("Only a step size of 1 is supported")

        # Flip y axis
        y_slice = key[1]
        start = y_slice.start if y_slice.start is not None else 0
        stop = y_slice.stop if y_slice.stop is not None else pic.height

        start = pic.height - start - 1
        stop = pic.height - stop

        key = (key[0], slice(stop, start + 1, y_slice.step))

        # array dimensions are row, column (i.e. y, x)
        self._key = (key[1], key[0])
        self._array = pic._array

        # Save slice for _getdim operations.
        # This allows you to swap parts of an array.
        self._slice = self._array[self._key[0], self._key[1]]

        shape = self._getdim(0).shape
        self.size = (shape[1], shape[0])

    def _getdim(self, dim):
        """Gets a specific dimension out of the raw image data slice.

        """
        return self._slice[:, :, dim]

    def _setdim(self, dim, value):
        """ Sets a specific dimension in the raw image data slice.

        """
        self._array[self._key[0], self._key[1], dim] = value

    @property
    def red(self):
        """The red component of the pixel (0-255)."""
        return self._getdim(0).ravel()

    @red.setter
    def red(self, value):
        self._setdim(0, value)

    @property
    def green(self):
        """The green component of the pixel (0-255)."""
        return self._getdim(1).ravel()

    @green.setter
    def green(self, value):
        self._setdim(1, value)

    @property
    def blue(self):
        """The blue component of the pixel (0-255)."""
        return self._getdim(2).ravel()

    @blue.setter
    def blue(self, value):
        self._setdim(2, value)

    @property
    def rgb(self):
        """The RGB color components of the pixel (3 values 0-255)."""
        return self._getdim(None)

    @rgb.setter
    def rgb(self, value):
        self._setdim(None, value)

    def __iter__(self):
        """Iterates through all pixels in the pixel group.

        """
        x_idx = range(self._pic.width)[self._key[0]]
        y_idx = range(self._pic.height)[self._key[1]]

        for x in x_idx:
            for y in y_idx:
                yield self._pic._makepixel((x, y))

    def __repr__(self):
        return "PixelGroup ({0} pixels)".format(self.size[0] * self.size[1])

class Picture(object):
    """A 2-D picture made up of pixels.

    Attributes
    ----------
    path : str
        Path to an image file to load.
    array : array
        Raw RGB image data [0-255]
    size : tuple
        Size of the empty array to create (width, height).
    color : tuple
        Color to fill empty array if size is given (red, green, blue) [0-255].

    Notes
    -----
    Cannot provide more than one of 'path' and 'size' and 'array'.
    Can only provide 'color' if 'size' provided.

    Examples
    --------
    Load an image from a file
    >>> from skimage import novice
    >>> from skimage import data
    >>> picture = novice.open(data.data_dir + '/elephant.png')

    Create a blank 100 pixel wide, 200 pixel tall white image
    >>> pic = Picture(size=(100, 200), color=(255, 255, 255))

    Use numpy to make an RGB byte array (shape is height x width x 3)
    >>> import numpy as np
    >>> data = np.zeros(shape=(200, 100, 3), dtype=np.uint8)
    >>> data[:, :, 0] = 255  # Set red component to maximum
    >>> pic = Picture(array=data)

    Get the bottom-left pixel
    >>> pic[0, 0]
    Pixel (red: 255, green: 0, blue: 0)

    Get the top row of the picture
    >>> pic[:, pic.height-1]
    PixelGroup (100 pixels)

    Set the bottom-left pixel to black
    >>> pic[0, 0] = (0, 0, 0)

    Set the top row to red
    >>> pic[:, pic.height-1] = (255, 0, 0)

    """
    def __init__(self, path=None, array=None):
        if path is not None and array is not None:
            ValueError("Only provide path or array not both.")
        elif path is not None:
            self._array = img_as_ubyte(io.imread(path))
            self._path = path
            self._format = imghdr.what(path)
        elif array is not None:
            self._array = array
            self._path = None
            self._format = None
        else:
            ValueError("Must provide path or array.")

        # Common setup.
        self._modified = False
        self.scale = 1

    @staticmethod
    def from_path(path):
        """Creates a Picture from an image file.

        Parameters
        ----------
        path : str
            Path to the image file.

        Returns
        -------
        pic : Picture
            A Picture with the image file data loaded.

        """
        return Picture(path=path)

    @staticmethod
    def from_size(size, color='black'):
        """Return a Picture of the specified size and a uniform color.

        Parameters
        ----------
        size : tuple
            Width and height of the picture in pixels.
        color : tuple or str
            RGB tuple with the fill color for the picture [0-255] or a valid
            key in `color_dict`.
        """
        if isinstance(color, six.string_types):
            color = color_dict[color]
        rgb_size = tuple(size) + (3,)
        array = np.ones(rgb_size, dtype=np.uint8) * color
        return Picture(array=array)

    @staticmethod
    def from_array(array):
        """Creates a single color Picture.

        Parameters
        ----------
        color : tuple
            RGB tuple with the fill color for the picture [0-255]

        size : tuple
            Width and height of the picture in pixels.

        Returns
        -------
        pic : Picture
            A Picture with the given color and size.

        """
        return Picture(array=array)

    def save(self, path):
        """Saves the picture to the given path.

        Parameters
        ----------
        path : str
            Path to save the picture (with file extension).

        """
        io.imsave(path, self._rescale(self._array))
        self._modified = False
        self._path = os.path.abspath(path)
        self._format = imghdr.what(path)

    @property
    def path(self):
        """The path to the picture."""
        return self._path

    @property
    def modified(self):
        """True if the picture has changed."""
        return self._modified

    def _array_modified(self):
        self._modified = True
        self._path = None

    @property
    def format(self):
        """The image format of the picture."""
        return self._format

    @property
    def size(self):
        """The size (width, height) of the picture."""
        # array dimensions are flipped: y, x
        return (self._array.shape[1], self._array.shape[0])

    @size.setter
    def size(self, value):
        try:
            # Don't resize if no change in size
            if (value[0] != self.width) or (value[1] != self.height):
                # skimage dimensions are flipped: y, x
                self._array = img_as_ubyte(resize(self._array,
                    (int(value[1]), int(value[0])), order=0))

                self._array_modified()
        except TypeError:
            msg = "Expected (width, height), but got {0} instead!"
            raise TypeError(msg.format(value))

    @property
    def width(self):
        """The width of the picture."""
        return self.size[0]

    @width.setter
    def width(self, value):
        self.size = (value, self.height)

    @property
    def height(self):
        """The height of the picture."""
        return self.size[1]

    @height.setter
    def height(self, value):
        self.size = (self.width, value)

    def _repr_html_(self):
        return io.Image(self._rescale(self._array))

    def _repr_png_(self):
        return io.Image(self._rescale(self._array))

    def show(self):
        """Displays the image in a separate window.

        """
        io.imshow(self._rescale(self._array))

    def _makepixel(self, xy):
        """ Creates a Pixel object for a given x, y location.

        Parameters
        ----------
        xy : tuple
            Cartesian coordinates to create a Pixel object for.

        """
        # skimage dimensions are flipped: y, x
        rgb = self._array[self.height - xy[1] - 1, xy[0]]
        return Pixel(self, self._array, xy[0], xy[1], rgb)

    def _rescale(self, array):
        """Inflates image according to scale factor.

        Parameters
        ----------
        array : array
            Raw RGB image data to rescale using nearest neighbor algorithm.

        """
        if self.scale == 1:
            return array
        new_size = (self.height * self.scale, self.width * self.scale)
        return img_as_ubyte(resize(array, new_size, order=0))

    def __iter__(self):
        """Iterates over all pixels in the image."""
        for x in xrange(self.width):
            for y in xrange(self.height):
                yield self._makepixel((x, y))

    def __getitem__(self, key):
        if isinstance(key, tuple) and len(key) == 2:
            if isinstance(key[0], int) and isinstance(key[1], int):
                if key[0] < 0 or key[1] < 0:
                    raise IndexError("Negative indices not supported")

                return self._makepixel((key[0], key[1]))
            else:
                return PixelGroup(self, key)
        else:
            raise TypeError("Invalid key type")

    def __setitem__(self, key, value):
        if isinstance(key, tuple) and len(key) == 2:
            if isinstance(value, tuple):
                self[key[0], key[1]].rgb = value
            elif isinstance(value, PixelGroup):
                src_key = self[key[0], key[1]]._key
                self._array[src_key] = value._array[value._key]
            else:
                raise TypeError("Invalid value type")
        else:
            raise TypeError("Invalid key type")
        self._array_modified()

    def __repr__(self):
        args = self.format, self.path, self.modified
        return "Picture (format: {0}, path: {1}, modified: {2})".format(*args)
<commit_msg>Remove `from_path` and `from_array` methods<commit_after>import os
import imghdr
from collections import namedtuple

import numpy as np
from skimage import io
from skimage import img_as_ubyte
from skimage.transform import resize
from skimage.color import color_dict
from skimage._shared import six

# Convert colors from `skimage.color` to uint8 and allow access through
# dict or a named tuple.
color_dict = dict((name, tuple(int(255 * c + 0.5) for c in rgb))
                  for name, rgb in color_dict.iteritems())
colors = namedtuple('colors', color_dict.keys())(**color_dict)

def open(path):
    """Return Picture object from the given image path."""
    return Picture(path=os.path.abspath(path))

class Pixel(object):
    """A single pixel in a Picture.

    Attributes
    ----------
    pic : Picture
        The Picture object that this pixel references.
    array : array_like
        Byte array with raw image data (RGB).
    x : int
        Horizontal coordinate of this pixel (left = 0).
    y : int
        Vertical coordinate of this pixel (bottom = 0).
    rgb : tuple
        RGB tuple with red, green, and blue components (0-255)

    """
    def __init__(self, pic, array, x, y, rgb):
        self._picture = pic
        self._array = array
        self._x = x
        self._y = y
        self._red = self._validate(rgb[0])
        self._green = self._validate(rgb[1])
        self._blue = self._validate(rgb[2])

    @property
    def x(self):
        """Horizontal location of this pixel in the parent image(left = 0)."""
        return self._x

    @property
    def y(self):
        """Vertical location of this pixel in the parent image (bottom = 0)."""
        return self._y

    @property
    def red(self):
        """The red component of the pixel (0-255)."""
        return self._red

    @red.setter
    def red(self, value):
        self._red = self._validate(value)
        self._setpixel()

    @property
    def green(self):
        """The green component of the pixel (0-255)."""
        return self._green

    @green.setter
    def green(self, value):
        self._green = self._validate(value)
        self._setpixel()

    @property
    def blue(self):
        """The blue component of the pixel (0-255)."""
        return self._blue

    @blue.setter
    def blue(self, value):
        self._blue = self._validate(value)
        self._setpixel()

    @property
    def rgb(self):
        """The RGB color components of the pixel (3 values 0-255)."""
        return (self.red, self.green, self.blue)

    @rgb.setter
    def rgb(self, value):
        self._red, self._green, self._blue = (self._validate(v) for v in value)
        self._setpixel()

    def _validate(self, value):
        """Verifies that the pixel value is in [0, 255]."""
        try:
            value = int(value)
            if (value < 0) or (value > 255):
                raise ValueError()
        except ValueError:
            msg = "Expected an integer between 0 and 255, but got {0} instead!"
            raise ValueError(msg.format(value))

        return value

    def _setpixel(self):
        """Sets the actual pixel value in the picture.

        NOTE: Using Cartesian coordinate system!

        """
        row = self._picture.height - self._y - 1
        self._picture._array[row, self._x] = self.rgb
        self._picture._array_modified()

    def __repr__(self):
        args = self.red, self.green, self.blue
        return "Pixel (red: {0}, green: {1}, blue: {2})".format(*args)

class PixelGroup(object):
    """A group of Pixel objects that can be manipulated together.

    Attributes
    ----------
    pic : Picture
        The Picture object that this pixel group references.
    key : tuple
        tuple with x and y slices or ints for extracting part of raw image data.

    """
    def __init__(self, pic, key):
        self._pic = pic

        # Use a slice so that the _getdim and _setdim functions can index
        # consistently.
        if isinstance(key[0], int):
            key = (slice(key[0], key[0] + 1), key[1])

        if isinstance(key[1], int):
            key = (key[0], slice(key[1], key[1] + 1))

        for dim_slice in key:
            if dim_slice.start is not None and dim_slice.start < 0:
                raise IndexError("Negative slicing not supported")

            if dim_slice.stop is not None and dim_slice.stop < 0:
                raise IndexError("Negative slicing not supported")

            if dim_slice.step is not None and dim_slice.step != 1:
                raise IndexError("Only a step size of 1 is supported")

        # Flip y axis
        y_slice = key[1]
        start = y_slice.start if y_slice.start is not None else 0
        stop = y_slice.stop if y_slice.stop is not None else pic.height

        start = pic.height - start - 1
        stop = pic.height - stop

        key = (key[0], slice(stop, start + 1, y_slice.step))

        # array dimensions are row, column (i.e. y, x)
        self._key = (key[1], key[0])
        self._array = pic._array

        # Save slice for _getdim operations.
        # This allows you to swap parts of an array.
        self._slice = self._array[self._key[0], self._key[1]]

        shape = self._getdim(0).shape
        self.size = (shape[1], shape[0])

    def _getdim(self, dim):
        """Gets a specific dimension out of the raw image data slice.

        """
        return self._slice[:, :, dim]

    def _setdim(self, dim, value):
        """ Sets a specific dimension in the raw image data slice.

        """
        self._array[self._key[0], self._key[1], dim] = value

    @property
    def red(self):
        """The red component of the pixel (0-255)."""
        return self._getdim(0).ravel()

    @red.setter
    def red(self, value):
        self._setdim(0, value)

    @property
    def green(self):
        """The green component of the pixel (0-255)."""
        return self._getdim(1).ravel()

    @green.setter
    def green(self, value):
        self._setdim(1, value)

    @property
    def blue(self):
        """The blue component of the pixel (0-255)."""
        return self._getdim(2).ravel()

    @blue.setter
    def blue(self, value):
        self._setdim(2, value)

    @property
    def rgb(self):
        """The RGB color components of the pixel (3 values 0-255)."""
        return self._getdim(None)

    @rgb.setter
    def rgb(self, value):
        self._setdim(None, value)

    def __iter__(self):
        """Iterates through all pixels in the pixel group.

        """
        x_idx = range(self._pic.width)[self._key[0]]
        y_idx = range(self._pic.height)[self._key[1]]

        for x in x_idx:
            for y in y_idx:
                yield self._pic._makepixel((x, y))

    def __repr__(self):
        return "PixelGroup ({0} pixels)".format(self.size[0] * self.size[1])

class Picture(object):
    """A 2-D picture made up of pixels.

    Attributes
    ----------
    path : str
        Path to an image file to load.
    array : array
        Raw RGB image data [0-255]
    size : tuple
        Size of the empty array to create (width, height).
    color : tuple
        Color to fill empty array if size is given (red, green, blue) [0-255].

    Notes
    -----
    Cannot provide more than one of 'path' and 'size' and 'array'.
    Can only provide 'color' if 'size' provided.

    Examples
    --------
    Load an image from a file
    >>> from skimage import novice
    >>> from skimage import data
    >>> picture = novice.open(data.data_dir + '/elephant.png')

    Create a blank 100 pixel wide, 200 pixel tall white image
    >>> pic = Picture(size=(100, 200), color=(255, 255, 255))

    Use numpy to make an RGB byte array (shape is height x width x 3)
    >>> import numpy as np
    >>> data = np.zeros(shape=(200, 100, 3), dtype=np.uint8)
    >>> data[:, :, 0] = 255  # Set red component to maximum
    >>> pic = Picture(array=data)

    Get the bottom-left pixel
    >>> pic[0, 0]
    Pixel (red: 255, green: 0, blue: 0)

    Get the top row of the picture
    >>> pic[:, pic.height-1]
    PixelGroup (100 pixels)

    Set the bottom-left pixel to black
    >>> pic[0, 0] = (0, 0, 0)

    Set the top row to red
    >>> pic[:, pic.height-1] = (255, 0, 0)

    """
    def __init__(self, path=None, array=None):
        if path is not None and array is not None:
            ValueError("Only provide path or array not both.")
        elif path is not None:
            self._array = img_as_ubyte(io.imread(path))
            self._path = path
            self._format = imghdr.what(path)
        elif array is not None:
            self._array = array
            self._path = None
            self._format = None
        else:
            ValueError("Must provide path or array.")

        # Common setup.
        self._modified = False
        self.scale = 1

    @staticmethod
    def from_size(size, color='black'):
        """Return a Picture of the specified size and a uniform color.

        Parameters
        ----------
        size : tuple
            Width and height of the picture in pixels.
        color : tuple or str
            RGB tuple with the fill color for the picture [0-255] or a valid
            key in `color_dict`.
        """
        if isinstance(color, six.string_types):
            color = color_dict[color]
        rgb_size = tuple(size) + (3,)
        array = np.ones(rgb_size, dtype=np.uint8) * color
        return Picture(array=array)

    def save(self, path):
        """Saves the picture to the given path.

        Parameters
        ----------
        path : str
            Path to save the picture (with file extension).

        """
        io.imsave(path, self._rescale(self._array))
        self._modified = False
        self._path = os.path.abspath(path)
        self._format = imghdr.what(path)

    @property
    def path(self):
        """The path to the picture."""
        return self._path

    @property
    def modified(self):
        """True if the picture has changed."""
        return self._modified

    def _array_modified(self):
        self._modified = True
        self._path = None

    @property
    def format(self):
        """The image format of the picture."""
        return self._format

    @property
    def size(self):
        """The size (width, height) of the picture."""
        # array dimensions are flipped: y, x
        return (self._array.shape[1], self._array.shape[0])

    @size.setter
    def size(self, value):
        try:
            # Don't resize if no change in size
            if (value[0] != self.width) or (value[1] != self.height):
                # skimage dimensions are flipped: y, x
                self._array = img_as_ubyte(resize(self._array,
                    (int(value[1]), int(value[0])), order=0))

                self._array_modified()
        except TypeError:
            msg = "Expected (width, height), but got {0} instead!"
            raise TypeError(msg.format(value))

    @property
    def width(self):
        """The width of the picture."""
        return self.size[0]

    @width.setter
    def width(self, value):
        self.size = (value, self.height)

    @property
    def height(self):
        """The height of the picture."""
        return self.size[1]

    @height.setter
    def height(self, value):
        self.size = (self.width, value)

    def _repr_html_(self):
        return io.Image(self._rescale(self._array))

    def _repr_png_(self):
        return io.Image(self._rescale(self._array))

    def show(self):
        """Displays the image in a separate window.

        """
        io.imshow(self._rescale(self._array))

    def _makepixel(self, xy):
        """ Creates a Pixel object for a given x, y location.

        Parameters
        ----------
        xy : tuple
            Cartesian coordinates to create a Pixel object for.

        """
        # skimage dimensions are flipped: y, x
        rgb = self._array[self.height - xy[1] - 1, xy[0]]
        return Pixel(self, self._array, xy[0], xy[1], rgb)

    def _rescale(self, array):
        """Inflates image according to scale factor.

        Parameters
        ----------
        array : array
            Raw RGB image data to rescale using nearest neighbor algorithm.

        """
        if self.scale == 1:
            return array
        new_size = (self.height * self.scale, self.width * self.scale)
        return img_as_ubyte(resize(array, new_size, order=0))

    def __iter__(self):
        """Iterates over all pixels in the image."""
        for x in xrange(self.width):
            for y in xrange(self.height):
                yield self._makepixel((x, y))

    def __getitem__(self, key):
        if isinstance(key, tuple) and len(key) == 2:
            if isinstance(key[0], int) and isinstance(key[1], int):
                if key[0] < 0 or key[1] < 0:
                    raise IndexError("Negative indices not supported")

                return self._makepixel((key[0], key[1]))
            else:
                return PixelGroup(self, key)
        else:
            raise TypeError("Invalid key type")

    def __setitem__(self, key, value):
        if isinstance(key, tuple) and len(key) == 2:
            if isinstance(value, tuple):
                self[key[0], key[1]].rgb = value
            elif isinstance(value, PixelGroup):
                src_key = self[key[0], key[1]]._key
                self._array[src_key] = value._array[value._key]
            else:
                raise TypeError("Invalid value type")
        else:
            raise TypeError("Invalid key type")
        self._array_modified()

    def __repr__(self):
        args = self.format, self.path, self.modified
        return "Picture (format: {0}, path: {1}, modified: {2})".format(*args)
<|endoftext|>


<commit_before>__version__ = "0.1.0b"
<commit_msg>Version 0.1.0b2<commit_after>__version__ = "0.1.0b2"
<|endoftext|>


<commit_before>from __future__ import absolute_import

import os
from copy import copy
from decimal import Decimal

from django.contrib.gis.gdal import DataSource
from django.contrib.gis.tests.utils import mysql
from django.contrib.gis.utils.layermapping import (LayerMapping, LayerMapError,
    InvalidDecimal, MissingForeignKey)
from django.db import router
from django.test import TestCase

from .models import (
    City, County, CountyFeat, Interstate, ICity1, ICity2, Invalid, State,
    city_mapping, co_mapping, cofeat_mapping, inter_mapping)

shp_path = os.path.realpath(os.path.join(os.path.dirname(__file__), os.pardir, 'data'))
city_shp = os.path.join(shp_path, 'cities', 'cities.shp')
co_shp = os.path.join(shp_path, 'counties', 'counties.shp')
inter_shp = os.path.join(shp_path, 'interstates', 'interstates.shp')
invalid_shp = os.path.join(shp_path, 'invalid', 'emptypoints.shp')

# Dictionaries to hold what's expected in the county shapefile.
NAMES  = ['Bexar', 'Galveston', 'Harris', 'Honolulu', 'Pueblo']
NUMS   = [1, 2, 1, 19, 1] # Number of polygons for each.
STATES = ['Texas', 'Texas', 'Texas', 'Hawaii', 'Colorado']

class LayerMapTest(TestCase):

    def test_init(self):
        "Testing LayerMapping initialization."

        # Model field that does not exist.
        bad1 = copy(city_mapping)
        bad1['foobar'] = 'FooField'

        # Shapefile field that does not exist.
        bad2 = copy(city_mapping)
        bad2['name'] = 'Nombre'

        # Nonexistent geographic field type.
        bad3 = copy(city_mapping)
        bad3['point'] = 'CURVE'

        # Incrementing through the bad mapping dictionaries and
        # ensuring that a LayerMapError is raised.
        for bad_map in (bad1, bad2, bad3):
            with self.assertRaises(LayerMapError):
                lm = LayerMapping(City, city_shp, bad_map)

        # A LookupError should be thrown for bogus encodings.
        with self.assertRaises(LookupError):
            lm = LayerMapping(City, city_shp, city_mapping, encoding='foobar')

    def test_simple_layermap(self):
        "Test LayerMapping import of a simple point shapefile."
        # Setting up for the LayerMapping.
        lm = LayerMapping(City, city_shp, city_mapping)
        lm.save()

        # There should be three cities in the shape file.
        self.assertEqual(3, City.objects.count())

        # Opening up the shapefile, and verifying the values in each
        # of the features made it to the model.
        ds = DataSource(city_shp)
        layer = ds[0]
        for feat in layer:
            city = City.objects.get(name=feat['Name'].value)
            self.assertEqual(feat['Population'].value, city.population)
            self.assertEqual(Decimal(str(feat['Density'])), city.density)
            self.assertEqual(feat['Created'].value, city.dt)

            # Comparing the geometries.
            pnt1, pnt2 = feat.geom, city.point
            self.assertAlmostEqual(pnt1.x, pnt2.x, 5)
            self.assertAlmostEqual(pnt1.y, pnt2.y, 5)

    def test_layermap_strict(self):
        "Testing the `strict` keyword, and import of a LineString shapefile."
        # When the `strict` keyword is set an error encountered will force
        # the importation to stop.
        with self.assertRaises(InvalidDecimal):
            lm = LayerMapping(Interstate, inter_shp, inter_mapping)
            lm.save(silent=True, strict=True)
        Interstate.objects.all().delete()

        # This LayerMapping should work b/c `strict` is not set.
        lm = LayerMapping(Interstate, inter_shp, inter_mapping)
        lm.save(silent=True)

        # Two interstate should have imported correctly.
        self.assertEqual(2, Interstate.objects.count())

        # Verifying the values in the layer w/the model.
        ds = DataSource(inter_shp)

        # Only the first two features of this shapefile are valid.
        valid_feats = ds[0][:2]
        for feat in valid_feats:
            istate = Interstate.objects.get(name=feat['Name'].value)

            if feat.fid == 0:
                self.assertEqual(Decimal(str(feat['Length'])), istate.length)
            elif feat.fid == 1:
                # Everything but the first two decimal digits were truncated,
                # because the Interstate model's `length` field has decimal_places=2.
                self.assertAlmostEqual(feat.get('Length'), float(istate.length), 2)

            for p1, p2 in zip(feat.geom, istate.path):
                self.assertAlmostEqual(p1[0], p2[0], 6)
                self.assertAlmostEqual(p1[1], p2[1], 6)

    def county_helper(self, county_feat=True):
        "Helper function for ensuring the integrity of the mapped County models."
        for name, n, st in zip(NAMES, NUMS, STATES):
            # Should only be one record b/c of `unique` keyword.
            c = County.objects.get(name=name)
            self.assertEqual(n, len(c.mpoly))
            self.assertEqual(st, c.state.name) # Checking ForeignKey mapping.

            # Multiple records because `unique` was not set.
            if county_feat:
                qs = CountyFeat.objects.filter(name=name)
                self.assertEqual(n, qs.count())

    def test_layermap_unique_multigeometry_fk(self):
        "Testing the `unique`, and `transform`, geometry collection conversion, and ForeignKey mappings."
        # All the following should work.
        try:
            # Telling LayerMapping that we want no transformations performed on the data.
            lm = LayerMapping(County, co_shp, co_mapping, transform=False)

            # Specifying the source spatial reference system via the `source_srs` keyword.
            lm = LayerMapping(County, co_shp, co_mapping, source_srs=4269)
            lm = LayerMapping(County, co_shp, co_mapping, source_srs='NAD83')

            # Unique may take tuple or string parameters.
            for arg in ('name', ('name', 'mpoly')):
                lm = LayerMapping(County, co_shp, co_mapping, transform=False, unique=arg)
        except:
            self.fail('No exception should be raised for proper use of keywords.')

        # Testing invalid params for the `unique` keyword.
        for e, arg in ((TypeError, 5.0), (ValueError, 'foobar'), (ValueError, ('name', 'mpolygon'))):
            self.assertRaises(e, LayerMapping, County, co_shp, co_mapping, transform=False, unique=arg)

        # No source reference system defined in the shapefile, should raise an error.
        if not mysql:
            self.assertRaises(LayerMapError, LayerMapping, County, co_shp, co_mapping)

        # Passing in invalid ForeignKey mapping parameters -- must be a dictionary
        # mapping for the model the ForeignKey points to.
        bad_fk_map1 = copy(co_mapping); bad_fk_map1['state'] = 'name'
        bad_fk_map2 = copy(co_mapping); bad_fk_map2['state'] = {'nombre' : 'State'}
        self.assertRaises(TypeError, LayerMapping, County, co_shp, bad_fk_map1, transform=False)
        self.assertRaises(LayerMapError, LayerMapping, County, co_shp, bad_fk_map2, transform=False)

        # There exist no State models for the ForeignKey mapping to work -- should raise
        # a MissingForeignKey exception (this error would be ignored if the `strict`
        # keyword is not set).
        lm = LayerMapping(County, co_shp, co_mapping, transform=False, unique='name')
        self.assertRaises(MissingForeignKey, lm.save, silent=True, strict=True)

        # Now creating the state models so the ForeignKey mapping may work.
        State.objects.bulk_create([
            State(name='Colorado'), State(name='Hawaii'), State(name='Texas')
        ])

        # If a mapping is specified as a collection, all OGR fields that
        # are not collections will be converted into them.  For example,
        # a Point column would be converted to MultiPoint. Other things being done
        # w/the keyword args:
        #  `transform=False`: Specifies that no transform is to be done; this
        #    has the effect of ignoring the spatial reference check (because the
        #    county shapefile does not have implicit spatial reference info).
        #
        #  `unique='name'`: Creates models on the condition that they have
        #    unique county names; geometries from each feature however will be
        #    appended to the geometry collection of the unique model.  Thus,
        #    all of the various islands in Honolulu county will be in in one
        #    database record with a MULTIPOLYGON type.
        lm = LayerMapping(County, co_shp, co_mapping, transform=False, unique='name')
        lm.save(silent=True, strict=True)

        # A reference that doesn't use the unique keyword; a new database record will
        # created for each polygon.
        lm = LayerMapping(CountyFeat, co_shp, cofeat_mapping, transform=False)
        lm.save(silent=True, strict=True)

        # The county helper is called to ensure integrity of County models.
        self.county_helper()

    def test_test_fid_range_step(self):
        "Tests the `fid_range` keyword and the `step` keyword of .save()."
        # Function for clearing out all the counties before testing.
        def clear_counties(): County.objects.all().delete()

        State.objects.bulk_create([
            State(name='Colorado'), State(name='Hawaii'), State(name='Texas')
        ])

        # Initializing the LayerMapping object to use in these tests.
        lm = LayerMapping(County, co_shp, co_mapping, transform=False, unique='name')

        # Bad feature id ranges should raise a type error.
        bad_ranges = (5.0, 'foo', co_shp)
        for bad in bad_ranges:
            self.assertRaises(TypeError, lm.save, fid_range=bad)

        # Step keyword should not be allowed w/`fid_range`.
        fr = (3, 5) # layer[3:5]
        self.assertRaises(LayerMapError, lm.save, fid_range=fr, step=10)
        lm.save(fid_range=fr)

        # Features IDs 3 & 4 are for Galveston County, Texas -- only
        # one model is returned because the `unique` keyword was set.
        qs = County.objects.all()
        self.assertEqual(1, qs.count())
        self.assertEqual('Galveston', qs[0].name)

        # Features IDs 5 and beyond for Honolulu County, Hawaii, and
        # FID 0 is for Pueblo County, Colorado.
        clear_counties()
        lm.save(fid_range=slice(5, None), silent=True, strict=True) # layer[5:]
        lm.save(fid_range=slice(None, 1), silent=True, strict=True) # layer[:1]

        # Only Pueblo & Honolulu counties should be present because of
        # the `unique` keyword.  Have to set `order_by` on this QuerySet
        # or else MySQL will return a different ordering than the other dbs.
        qs = County.objects.order_by('name')
        self.assertEqual(2, qs.count())
        hi, co = tuple(qs)
        hi_idx, co_idx = tuple(map(NAMES.index, ('Honolulu', 'Pueblo')))
        self.assertEqual('Pueblo', co.name)
        self.assertEqual(NUMS[co_idx], len(co.mpoly))
        self.assertEqual('Honolulu', hi.name)
        self.assertEqual(NUMS[hi_idx], len(hi.mpoly))

        # Testing the `step` keyword -- should get the same counties
        # regardless of we use a step that divides equally, that is odd,
        # or that is larger than the dataset.
        for st in (4,7,1000):
            clear_counties()
            lm.save(step=st, strict=True)
            self.county_helper(county_feat=False)

    def test_model_inheritance(self):
        "Tests LayerMapping on inherited models.  See #12093."
        icity_mapping = {'name' : 'Name',
                         'population' : 'Population',
                         'density' : 'Density',
                         'point' : 'POINT',
                         'dt' : 'Created',
                         }

        # Parent model has geometry field.
        lm1 = LayerMapping(ICity1, city_shp, icity_mapping)
        lm1.save()

        # Grandparent has geometry field.
        lm2 = LayerMapping(ICity2, city_shp, icity_mapping)
        lm2.save()

        self.assertEqual(6, ICity1.objects.count())
        self.assertEqual(3, ICity2.objects.count())

    def test_invalid_layer(self):
        "Tests LayerMapping on invalid geometries.  See #15378."
        invalid_mapping = {'point': 'POINT'}
        lm = LayerMapping(Invalid, invalid_shp, invalid_mapping,
                          source_srs=4326)
        lm.save(silent=True)

    def test_textfield(self):
        "Tests that String content fits also in a TextField"
        mapping = copy(city_mapping)
        mapping['name_txt'] = 'Name'
        lm = LayerMapping(City, city_shp, mapping)
        lm.save(silent=True, strict=True)
        self.assertEqual(City.objects.count(), 3)
        self.assertEqual(City.objects.all().order_by('name_txt')[0].name_txt, "Houston")

class OtherRouter(object):
    def db_for_read(self, model, **hints):
        return 'other'

    def db_for_write(self, model, **hints):
        return self.db_for_read(model, **hints)

    def allow_relation(self, obj1, obj2, **hints):
        return None

    def allow_syncdb(self, db, model):
        return True

class LayerMapRouterTest(TestCase):

    def setUp(self):
        self.old_routers = router.routers
        router.routers = [OtherRouter()]

    def tearDown(self):
        router.routers = self.old_routers

    def test_layermapping_default_db(self):
        lm = LayerMapping(City, city_shp, city_mapping)
        self.assertEqual(lm.using, 'other')
<commit_msg>Skip `LayerMapRouterTest` if there are not multiple databases.<commit_after>from __future__ import absolute_import

import os
import unittest
from copy import copy
from decimal import Decimal

from django.contrib.gis.gdal import DataSource
from django.contrib.gis.tests.utils import mysql
from django.contrib.gis.utils.layermapping import (LayerMapping, LayerMapError,
    InvalidDecimal, MissingForeignKey)
from django.db import router
from django.conf import settings
from django.test import TestCase

from .models import (
    City, County, CountyFeat, Interstate, ICity1, ICity2, Invalid, State,
    city_mapping, co_mapping, cofeat_mapping, inter_mapping)

shp_path = os.path.realpath(os.path.join(os.path.dirname(__file__), os.pardir, 'data'))
city_shp = os.path.join(shp_path, 'cities', 'cities.shp')
co_shp = os.path.join(shp_path, 'counties', 'counties.shp')
inter_shp = os.path.join(shp_path, 'interstates', 'interstates.shp')
invalid_shp = os.path.join(shp_path, 'invalid', 'emptypoints.shp')

# Dictionaries to hold what's expected in the county shapefile.
NAMES  = ['Bexar', 'Galveston', 'Harris', 'Honolulu', 'Pueblo']
NUMS   = [1, 2, 1, 19, 1] # Number of polygons for each.
STATES = ['Texas', 'Texas', 'Texas', 'Hawaii', 'Colorado']

class LayerMapTest(TestCase):

    def test_init(self):
        "Testing LayerMapping initialization."

        # Model field that does not exist.
        bad1 = copy(city_mapping)
        bad1['foobar'] = 'FooField'

        # Shapefile field that does not exist.
        bad2 = copy(city_mapping)
        bad2['name'] = 'Nombre'

        # Nonexistent geographic field type.
        bad3 = copy(city_mapping)
        bad3['point'] = 'CURVE'

        # Incrementing through the bad mapping dictionaries and
        # ensuring that a LayerMapError is raised.
        for bad_map in (bad1, bad2, bad3):
            with self.assertRaises(LayerMapError):
                lm = LayerMapping(City, city_shp, bad_map)

        # A LookupError should be thrown for bogus encodings.
        with self.assertRaises(LookupError):
            lm = LayerMapping(City, city_shp, city_mapping, encoding='foobar')

    def test_simple_layermap(self):
        "Test LayerMapping import of a simple point shapefile."
        # Setting up for the LayerMapping.
        lm = LayerMapping(City, city_shp, city_mapping)
        lm.save()

        # There should be three cities in the shape file.
        self.assertEqual(3, City.objects.count())

        # Opening up the shapefile, and verifying the values in each
        # of the features made it to the model.
        ds = DataSource(city_shp)
        layer = ds[0]
        for feat in layer:
            city = City.objects.get(name=feat['Name'].value)
            self.assertEqual(feat['Population'].value, city.population)
            self.assertEqual(Decimal(str(feat['Density'])), city.density)
            self.assertEqual(feat['Created'].value, city.dt)

            # Comparing the geometries.
            pnt1, pnt2 = feat.geom, city.point
            self.assertAlmostEqual(pnt1.x, pnt2.x, 5)
            self.assertAlmostEqual(pnt1.y, pnt2.y, 5)

    def test_layermap_strict(self):
        "Testing the `strict` keyword, and import of a LineString shapefile."
        # When the `strict` keyword is set an error encountered will force
        # the importation to stop.
        with self.assertRaises(InvalidDecimal):
            lm = LayerMapping(Interstate, inter_shp, inter_mapping)
            lm.save(silent=True, strict=True)
        Interstate.objects.all().delete()

        # This LayerMapping should work b/c `strict` is not set.
        lm = LayerMapping(Interstate, inter_shp, inter_mapping)
        lm.save(silent=True)

        # Two interstate should have imported correctly.
        self.assertEqual(2, Interstate.objects.count())

        # Verifying the values in the layer w/the model.
        ds = DataSource(inter_shp)

        # Only the first two features of this shapefile are valid.
        valid_feats = ds[0][:2]
        for feat in valid_feats:
            istate = Interstate.objects.get(name=feat['Name'].value)

            if feat.fid == 0:
                self.assertEqual(Decimal(str(feat['Length'])), istate.length)
            elif feat.fid == 1:
                # Everything but the first two decimal digits were truncated,
                # because the Interstate model's `length` field has decimal_places=2.
                self.assertAlmostEqual(feat.get('Length'), float(istate.length), 2)

            for p1, p2 in zip(feat.geom, istate.path):
                self.assertAlmostEqual(p1[0], p2[0], 6)
                self.assertAlmostEqual(p1[1], p2[1], 6)

    def county_helper(self, county_feat=True):
        "Helper function for ensuring the integrity of the mapped County models."
        for name, n, st in zip(NAMES, NUMS, STATES):
            # Should only be one record b/c of `unique` keyword.
            c = County.objects.get(name=name)
            self.assertEqual(n, len(c.mpoly))
            self.assertEqual(st, c.state.name) # Checking ForeignKey mapping.

            # Multiple records because `unique` was not set.
            if county_feat:
                qs = CountyFeat.objects.filter(name=name)
                self.assertEqual(n, qs.count())

    def test_layermap_unique_multigeometry_fk(self):
        "Testing the `unique`, and `transform`, geometry collection conversion, and ForeignKey mappings."
        # All the following should work.
        try:
            # Telling LayerMapping that we want no transformations performed on the data.
            lm = LayerMapping(County, co_shp, co_mapping, transform=False)

            # Specifying the source spatial reference system via the `source_srs` keyword.
            lm = LayerMapping(County, co_shp, co_mapping, source_srs=4269)
            lm = LayerMapping(County, co_shp, co_mapping, source_srs='NAD83')

            # Unique may take tuple or string parameters.
            for arg in ('name', ('name', 'mpoly')):
                lm = LayerMapping(County, co_shp, co_mapping, transform=False, unique=arg)
        except:
            self.fail('No exception should be raised for proper use of keywords.')

        # Testing invalid params for the `unique` keyword.
        for e, arg in ((TypeError, 5.0), (ValueError, 'foobar'), (ValueError, ('name', 'mpolygon'))):
            self.assertRaises(e, LayerMapping, County, co_shp, co_mapping, transform=False, unique=arg)

        # No source reference system defined in the shapefile, should raise an error.
        if not mysql:
            self.assertRaises(LayerMapError, LayerMapping, County, co_shp, co_mapping)

        # Passing in invalid ForeignKey mapping parameters -- must be a dictionary
        # mapping for the model the ForeignKey points to.
        bad_fk_map1 = copy(co_mapping); bad_fk_map1['state'] = 'name'
        bad_fk_map2 = copy(co_mapping); bad_fk_map2['state'] = {'nombre' : 'State'}
        self.assertRaises(TypeError, LayerMapping, County, co_shp, bad_fk_map1, transform=False)
        self.assertRaises(LayerMapError, LayerMapping, County, co_shp, bad_fk_map2, transform=False)

        # There exist no State models for the ForeignKey mapping to work -- should raise
        # a MissingForeignKey exception (this error would be ignored if the `strict`
        # keyword is not set).
        lm = LayerMapping(County, co_shp, co_mapping, transform=False, unique='name')
        self.assertRaises(MissingForeignKey, lm.save, silent=True, strict=True)

        # Now creating the state models so the ForeignKey mapping may work.
        State.objects.bulk_create([
            State(name='Colorado'), State(name='Hawaii'), State(name='Texas')
        ])

        # If a mapping is specified as a collection, all OGR fields that
        # are not collections will be converted into them.  For example,
        # a Point column would be converted to MultiPoint. Other things being done
        # w/the keyword args:
        #  `transform=False`: Specifies that no transform is to be done; this
        #    has the effect of ignoring the spatial reference check (because the
        #    county shapefile does not have implicit spatial reference info).
        #
        #  `unique='name'`: Creates models on the condition that they have
        #    unique county names; geometries from each feature however will be
        #    appended to the geometry collection of the unique model.  Thus,
        #    all of the various islands in Honolulu county will be in in one
        #    database record with a MULTIPOLYGON type.
        lm = LayerMapping(County, co_shp, co_mapping, transform=False, unique='name')
        lm.save(silent=True, strict=True)

        # A reference that doesn't use the unique keyword; a new database record will
        # created for each polygon.
        lm = LayerMapping(CountyFeat, co_shp, cofeat_mapping, transform=False)
        lm.save(silent=True, strict=True)

        # The county helper is called to ensure integrity of County models.
        self.county_helper()

    def test_test_fid_range_step(self):
        "Tests the `fid_range` keyword and the `step` keyword of .save()."
        # Function for clearing out all the counties before testing.
        def clear_counties(): County.objects.all().delete()

        State.objects.bulk_create([
            State(name='Colorado'), State(name='Hawaii'), State(name='Texas')
        ])

        # Initializing the LayerMapping object to use in these tests.
        lm = LayerMapping(County, co_shp, co_mapping, transform=False, unique='name')

        # Bad feature id ranges should raise a type error.
        bad_ranges = (5.0, 'foo', co_shp)
        for bad in bad_ranges:
            self.assertRaises(TypeError, lm.save, fid_range=bad)

        # Step keyword should not be allowed w/`fid_range`.
        fr = (3, 5) # layer[3:5]
        self.assertRaises(LayerMapError, lm.save, fid_range=fr, step=10)
        lm.save(fid_range=fr)

        # Features IDs 3 & 4 are for Galveston County, Texas -- only
        # one model is returned because the `unique` keyword was set.
        qs = County.objects.all()
        self.assertEqual(1, qs.count())
        self.assertEqual('Galveston', qs[0].name)

        # Features IDs 5 and beyond for Honolulu County, Hawaii, and
        # FID 0 is for Pueblo County, Colorado.
        clear_counties()
        lm.save(fid_range=slice(5, None), silent=True, strict=True) # layer[5:]
        lm.save(fid_range=slice(None, 1), silent=True, strict=True) # layer[:1]

        # Only Pueblo & Honolulu counties should be present because of
        # the `unique` keyword.  Have to set `order_by` on this QuerySet
        # or else MySQL will return a different ordering than the other dbs.
        qs = County.objects.order_by('name')
        self.assertEqual(2, qs.count())
        hi, co = tuple(qs)
        hi_idx, co_idx = tuple(map(NAMES.index, ('Honolulu', 'Pueblo')))
        self.assertEqual('Pueblo', co.name)
        self.assertEqual(NUMS[co_idx], len(co.mpoly))
        self.assertEqual('Honolulu', hi.name)
        self.assertEqual(NUMS[hi_idx], len(hi.mpoly))

        # Testing the `step` keyword -- should get the same counties
        # regardless of we use a step that divides equally, that is odd,
        # or that is larger than the dataset.
        for st in (4,7,1000):
            clear_counties()
            lm.save(step=st, strict=True)
            self.county_helper(county_feat=False)

    def test_model_inheritance(self):
        "Tests LayerMapping on inherited models.  See #12093."
        icity_mapping = {'name' : 'Name',
                         'population' : 'Population',
                         'density' : 'Density',
                         'point' : 'POINT',
                         'dt' : 'Created',
                         }

        # Parent model has geometry field.
        lm1 = LayerMapping(ICity1, city_shp, icity_mapping)
        lm1.save()

        # Grandparent has geometry field.
        lm2 = LayerMapping(ICity2, city_shp, icity_mapping)
        lm2.save()

        self.assertEqual(6, ICity1.objects.count())
        self.assertEqual(3, ICity2.objects.count())

    def test_invalid_layer(self):
        "Tests LayerMapping on invalid geometries.  See #15378."
        invalid_mapping = {'point': 'POINT'}
        lm = LayerMapping(Invalid, invalid_shp, invalid_mapping,
                          source_srs=4326)
        lm.save(silent=True)

    def test_textfield(self):
        "Tests that String content fits also in a TextField"
        mapping = copy(city_mapping)
        mapping['name_txt'] = 'Name'
        lm = LayerMapping(City, city_shp, mapping)
        lm.save(silent=True, strict=True)
        self.assertEqual(City.objects.count(), 3)
        self.assertEqual(City.objects.all().order_by('name_txt')[0].name_txt, "Houston")

class OtherRouter(object):
    def db_for_read(self, model, **hints):
        return 'other'

    def db_for_write(self, model, **hints):
        return self.db_for_read(model, **hints)

    def allow_relation(self, obj1, obj2, **hints):
        return None

    def allow_syncdb(self, db, model):
        return True

class LayerMapRouterTest(TestCase):

    def setUp(self):
        self.old_routers = router.routers
        router.routers = [OtherRouter()]

    def tearDown(self):
        router.routers = self.old_routers

    @unittest.skipUnless(len(settings.DATABASES) > 1, 'multiple databases required')
    def test_layermapping_default_db(self):
        lm = LayerMapping(City, city_shp, city_mapping)
        self.assertEqual(lm.using, 'other')
<|endoftext|>


<commit_before>import datetime
from django.core.management.base import NoArgsCommand

class Command(NoArgsCommand):
    help = "Can be run as a cronjob or directly to clean out old data from the database (only expired sessions at the moment)."

    def handle_noargs(self, **options):
        from django.db import transaction
        from django.contrib.sessions.models import Session
        Session.objects.filter(expire_date__lt=datetime.datetime.now()).delete()
        transaction.commit_unless_managed()
<commit_msg>Upgraded manage.py cleanup to be compatible with time zone support. See also r17121.<commit_after>from django.core.management.base import NoArgsCommand
from django.utils import timezone

class Command(NoArgsCommand):
    help = "Can be run as a cronjob or directly to clean out old data from the database (only expired sessions at the moment)."

    def handle_noargs(self, **options):
        from django.db import transaction
        from django.contrib.sessions.models import Session
        Session.objects.filter(expire_date__lt=timezone.now()).delete()
        transaction.commit_unless_managed()
<|endoftext|>


<commit_before>class TeachingHierarchy:
	def __init__(self, year):
		self.year = year
	def get_teaching_years(self):
		return self.year

class SchoolYear:
	def __init__(self, name, classes):
		self.name = name
		self.classes =  classes
	def get_teaching_classes(self):
		return self.classes
	def get_name(self):
		return self.name

class TeachingClass:
	def __init__(self, name, id=-1, assignments=None,students_data=None):
		self.name = name
		self.students_data = students_data
		self.id = id
		self.assignments = assignments
	def get_assignments(self):
		return self.assignments
	def get_id(self):
		return self.id
	def get_name(self):
		return self.name
	def get_students_data(self):
		return self.students_data<commit_msg>removed some random non required code<commit_after>class TeachingHierarchy:
	def __init__(self, year):
		self.year = year
	def get_teaching_years(self):
		return self.year

class SchoolYear:
	def __init__(self, name, classes):
		self.name = name
		self.classes =  classes
	def get_teaching_classes(self):
		return self.classes
	def get_name(self):
		return self.name

class TeachingClass:
	def __init__(self, name, id=-1, assignments=None):
		self.name = name
		self.id = id
		self.assignments = assignments
	def get_assignments(self):
		return self.assignments
	def get_id(self):
		return self.id
	def get_name(self):
		return self.name<|endoftext|>


<commit_before>import logging

class  SlackUser(object):
    """Represents a Slack user"""
    def __init__(self, id, sc=None):
        self.id = id
        self._sc = sc
        self._username = None
        self.logger = logging.getLogger(type(self).__name__)
        self.logger.setLevel(logging.DEBUG)

    @property
    def username(self):
        if self._username is None and self._sc is not None:
            resp = self._sc.server.users.find(self.id)
            if resp is not None:
                self._username = resp.name
        return self._username

    @property
    def userid(self):
        self.logger.warn('Use of userid is deprecated, use id instead')
        return self.id

    @staticmethod
    def get_user(sc, username):
        resp = sc.server.users.find(username)
        if resp is None:
            return None
        user = SlackUser(resp.id, sc)
        return user

    def __str__(self):
        return '<@%s|%s>' % (self.id, self.username)

    def __repr__(self):
        return self.id
<commit_msg>Review comments<commit_after>import logging

class SlackUser(object):
    """Represents a Slack user"""
    def __init__(self, id, sc=None):
        self.id = id
        self._sc = sc
        self._username = None
        self.logger = logging.getLogger(type(self).__name__)
        self.logger.setLevel(logging.DEBUG)

    @property
    def username(self):
        if self._username is None and self._sc is not None:
            resp = self._sc.server.users.find(self.id)
            if resp is not None:
                self._username = resp.name
        return self._username

    @property
    def userid(self):
        self.logger.warn('Use of userid is deprecated, use id instead')
        return self.id

    @staticmethod
    def get_user(sc, username):
        resp = sc.server.users.find(username)
        if resp is None:
            return None
        user = SlackUser(resp.id, sc)
        return user

    def __str__(self):
        return '<@%s|%s>' % (self.id, self.username)

    def __repr__(self):
        return self.id
<|endoftext|>


<commit_before>#
# File:   courseware/capa/capa_problem.py
#
'''
Main module which shows problems (of "capa" type).

This is used by capa_module.
'''

import copy
import logging
import math
import numpy
import os
import random
import re
import scipy
import struct

from lxml import etree
from lxml.etree import Element
from xml.sax.saxutils import escape, unescape

from mako.template import Template

from util import contextualize_text
import inputtypes

from responsetypes import NumericalResponse, FormulaResponse, CustomResponse, SchematicResponse, MultipleChoiceResponse,  StudentInputError, TrueFalseResponse, ExternalResponse,ImageResponse,OptionResponse

import calc
import eia

log = logging.getLogger("mitx.courseware")

response_types = {'numericalresponse':NumericalResponse,
                  'formularesponse':FormulaResponse,
                  'customresponse':CustomResponse,
                  'schematicresponse':SchematicResponse,
                  'externalresponse':ExternalResponse,
                  'multiplechoiceresponse':MultipleChoiceResponse,
                  'truefalseresponse':TrueFalseResponse,
                  'imageresponse':ImageResponse,
                  'optionresponse':OptionResponse,
                  }
entry_types = ['textline', 'schematic', 'choicegroup','textbox','imageinput','optioninput']
solution_types = ['solution']	# extra things displayed after "show answers" is pressed
response_properties = ["responseparam", "answer"]	# these get captured as student responses

# How to convert from original XML to HTML
# We should do this with xlst later
html_transforms = {'problem': {'tag':'div'},
                   "numericalresponse": {'tag':'span'},
                   "customresponse": {'tag':'span'},
                   "externalresponse": {'tag':'span'},
                   "schematicresponse": {'tag':'span'},
                   "formularesponse": {'tag':'span'},
                   "multiplechoiceresponse": {'tag':'span'},
                   "text": {'tag':'span'},
                   "math": {'tag':'span'},
                   }

global_context={'random':random,
                'numpy':numpy,
                'math':math,
                'scipy':scipy,
                'calc':calc,
                'eia':eia}

# These should be removed from HTML output, including all subelements
html_problem_semantics = ["responseparam", "answer", "script"]
# These should be removed from HTML output, but keeping subelements
html_skip = ["numericalresponse", "customresponse", "schematicresponse", "formularesponse", "text","externalresponse"]

# removed in MC
## These should be transformed
#html_special_response = {"textline":inputtypes.textline.render,
#                         "schematic":inputtypes.schematic.render,
#                         "textbox":inputtypes.textbox.render,
#                         "formulainput":inputtypes.jstextline.render,
#                         "solution":inputtypes.solution.render,
#                         }

class LoncapaProblem(object):
    def __init__(self, fileobject, id, state=None, seed=None, system=None):
        ## Initialize class variables from state
        self.seed = None
        self.student_answers = dict()
        self.correct_map = dict()
        self.done = False
        self.problem_id = id
        self.system = system

        if seed != None:
            self.seed = seed

        if state:
            if 'seed' in state:
                self.seed = state['seed']
            if 'student_answers' in state:
                self.student_answers = state['student_answers']
            if 'correct_map' in state:
                self.correct_map = state['correct_map']
            if 'done' in state:
                self.done = state['done']

        # TODO: Does this deplete the Linux entropy pool? Is this fast enough?
        if not self.seed:
            self.seed=struct.unpack('i', os.urandom(4))[0]

        ## Parse XML file
        file_text = fileobject.read()
        self.fileobject = fileobject	# save it, so we can use for debugging information later
        # Convert startouttext and endouttext to proper <text></text>
        # TODO: Do with XML operations
        file_text = re.sub("startouttext\s*/","text",file_text)
        file_text = re.sub("endouttext\s*/","/text",file_text)
        self.tree = etree.XML(file_text)

        self.preprocess_problem(self.tree, correct_map=self.correct_map, answer_map = self.student_answers)
        self.context = self.extract_context(self.tree, seed=self.seed)
        for response in self.tree.xpath('//'+"|//".join(response_types)):
            responder = response_types[response.tag](response, self.context, self.system)
            responder.preprocess_response()

    def get_state(self):
        ''' Stored per-user session data neeeded to:
            1) Recreate the problem
            2) Populate any student answers. '''
        return {'seed':self.seed,
                'student_answers':self.student_answers,
                'correct_map':self.correct_map,
                'done':self.done}

    def get_max_score(self):
        '''
        TODO: multiple points for programming problems.
        '''
        sum = 0
        for et in entry_types:
            sum = sum + self.tree.xpath('count(//'+et+')')
        return int(sum)

    def get_score(self):
        correct=0
        for key in self.correct_map:
            if self.correct_map[key] == u'correct':
                correct += 1
        if (not self.student_answers) or len(self.student_answers)==0:
            return {'score':0,
                    'total':self.get_max_score()}
        else:
            return {'score':correct,
                    'total':self.get_max_score()}

    def grade_answers(self, answers):
        '''
        Grade student responses.  Called by capa_module.check_problem.
        answers is a dict of all the entries from request.POST, but with the first part
        of each key removed (the string before the first "_").

        Thus, for example, input_ID123 -> ID123, and input_fromjs_ID123 -> fromjs_ID123
        '''
        self.student_answers = answers
        context=self.extract_context(self.tree)
        self.correct_map = dict()
        problems_simple = self.extract_problems(self.tree)
        for response in problems_simple:
            grader = response_types[response.tag](response, self.context, self.system)
            results = grader.get_score(answers)		# call the responsetype instance to do the actual grading
            self.correct_map.update(results)
        return self.correct_map

    def get_question_answers(self):
        '''
        Make a dict of (id,correct_answer) entries, for all the problems.
        Called by "show answers" button JSON request (see capa_module)
        '''
        context=self.extract_context(self.tree)
        answer_map = dict()
        problems_simple = self.extract_problems(self.tree)	# purified (flat) XML tree of just response queries
        for response in problems_simple:
            responder = response_types[response.tag](response, self.context, self.system)	# instance of numericalresponse, customresponse,...
            results = responder.get_answers()
            answer_map.update(results)				# dict of (id,correct_answer)

        # example for the following: <textline size="5" correct_answer="saturated" />
        for entry in problems_simple.xpath("//"+"|//".join(response_properties+entry_types)):
            answer = entry.get('correct_answer')		# correct answer, when specified elsewhere, eg in a textline
            if answer:
                answer_map[entry.get('id')] = contextualize_text(answer, self.context)

        # include solutions from <solution>...</solution> stanzas
        # Tentative merge; we should figure out how we want to handle hints and solutions
        for entry in self.tree.xpath("//"+"|//".join(solution_types)):
            answer = etree.tostring(entry)
            if answer:
                answer_map[entry.get('id')] = answer

        return answer_map

    # ======= Private ========

    def extract_context(self, tree, seed = struct.unpack('i', os.urandom(4))[0]):  # private
        '''
        Extract content of <script>...</script> from the problem.xml file, and exec it in the
        context of this problem.  Provides ability to randomize problems, and also set
        variables for problem answer checking.

        Problem XML goes to Python execution context. Runs everything in script tags
        '''
        random.seed(self.seed)
        context = {'global_context':global_context}	# save global context in here also
        context.update(global_context)			# initialize context to have stuff in global_context
        context['__builtins__'] = globals()['__builtins__']	# put globals there also
        context['the_lcp'] = self				# pass instance of LoncapaProblem in

        #for script in tree.xpath('/problem/script'):
        for script in tree.findall('.//script'):
            if 'javascript' in script.get('type'): continue	# skip javascript
            if 'perl' in script.get('type'): continue		# skip perl
            # TODO: evaluate only python
            code = script.text
            XMLESC = {"'": "'", """: '"'}
            code = unescape(code,XMLESC)
            try:
                exec code in context, context		# use "context" for global context; thus defs in code are global within code
            except Exception,err:
                print "[courseware.capa.capa_problem.extract_context] error %s" % err
                print "in doing exec of this code:",code
        return context

    def get_html(self):
        return contextualize_text(etree.tostring(self.extract_html(self.tree)[0]), self.context)

    def extract_html(self, problemtree):  # private
        ''' Helper function for get_html. Recursively converts XML tree to HTML
        '''
        if problemtree.tag in html_problem_semantics:
            return

        problemid = problemtree.get('id')	# my ID

        # used to be
        # if problemtree.tag in html_special_response:

        if problemtree.tag in inputtypes.get_input_xml_tags():
            # status is currently the answer for the problem ID for the input element,
            # but it will turn into a dict containing both the answer and any associated message
            # for the problem ID for the input element.
            status = "unsubmitted"
            if problemid in self.correct_map:
                status = self.correct_map[problemtree.get('id')]

            value = ""
            if self.student_answers and problemid in self.student_answers:
                value = self.student_answers[problemid]

            #### This code is a hack. It was merged to help bring two branches
            #### in sync, but should be replaced. msg should be passed in a
            #### response_type
            # prepare the response message, if it exists in correct_map
            if 'msg' in self.correct_map:
                msg = self.correct_map['msg']
            elif ('msg_%s' % problemid) in self.correct_map:
                msg = self.correct_map['msg_%s' % problemid]
            else:
                msg = ''

            #if settings.DEBUG:
            #    print "[courseware.capa.capa_problem.extract_html] msg = ",msg

            # do the rendering
            # This should be broken out into a helper function
            # that handles all input objects
            render_object = inputtypes.SimpleInput(system = self.system,
                                                   xml = problemtree,
                                                   state = {'value':value,
                                                            'status': status,
                                                            'id':problemtree.get('id'),
                                                            'feedback':{'message':msg}
                                                            },
                                                   use = 'capa_input')
            return render_object.get_html() #function(problemtree, value, status, msg) # render the special response (textline, schematic,...)

        tree=Element(problemtree.tag)
        for item in problemtree:
            subitems = self.extract_html(item)
            if subitems is not None:
                for subitem in subitems:
                    tree.append(subitem)
        for (key,value) in problemtree.items():
            tree.set(key, value)

        tree.text=problemtree.text
        tree.tail=problemtree.tail

        if problemtree.tag in html_transforms:
            tree.tag=html_transforms[problemtree.tag]['tag']
            # Reset attributes. Otherwise, we get metadata in HTML
            # (e.g. answers)
            # TODO: We should remove and not zero them.
            # I'm not sure how to do that quickly with lxml
            for k in tree.keys():
                tree.set(k,"")

        # TODO: Fix. This loses Element().tail
        #if problemtree.tag in html_skip:
        #    return tree
        return [tree]

    def preprocess_problem(self, tree, correct_map=dict(), answer_map=dict()): # private
        '''
        Assign IDs to all the responses
        Assign sub-IDs to all entries (textline, schematic, etc.)
        Annoted correctness and value
        In-place transformation
        '''
        response_id = 1
        for response in tree.xpath('//'+"|//".join(response_types)):
            response_id_str=self.problem_id+"_"+str(response_id)
            response.attrib['id']=response_id_str
            if response_id not in correct_map:
                correct = 'unsubmitted'
            response.attrib['state'] = correct
            response_id = response_id + 1
            answer_id = 1
            for entry in tree.xpath("|".join(['//'+response.tag+'[@id=$id]//'+x for x in (entry_types + solution_types)]),
                                    id=response_id_str):
                # assign one answer_id for each entry_type or solution_type
                entry.attrib['response_id'] = str(response_id)
                entry.attrib['answer_id'] = str(answer_id)
                entry.attrib['id'] = "%s_%i_%i"%(self.problem_id, response_id, answer_id)
                answer_id=answer_id+1

        # <solution>...</solution> may not be associated with any specific response; give IDs for those separately
		# TODO: We should make the namespaces consistent and unique (e.g. %s_problem_%i).
        solution_id = 1
        for solution in tree.findall('.//solution'):
            solution.attrib['id'] =  "%s_solution_%i"%(self.problem_id, solution_id)
            solution_id += 1

    def extract_problems(self, problem_tree):
        ''' Remove layout from the problem, and give a purified XML tree of just the problems '''
        problem_tree=copy.deepcopy(problem_tree)
        tree=Element('problem')
        for response in problem_tree.xpath("//"+"|//".join(response_types)):
            newresponse = copy.copy(response)
            for e in newresponse:
                newresponse.remove(e)
            # copy.copy is needed to make xpath work right. Otherwise, it starts at the root
            # of the tree. We should figure out if there's some work-around
            for e in copy.copy(response).xpath("//"+"|//".join(response_properties+entry_types)):
                newresponse.append(e)

            tree.append(newresponse)
        return tree

if __name__=='__main__':
    problem_id='simpleFormula'
    filename = 'simpleFormula.xml'

    problem_id='resistor'
    filename = 'resistor.xml'

    lcp = LoncapaProblem(filename, problem_id)

    context = lcp.extract_context(lcp.tree)
    problem = lcp.extract_problems(lcp.tree)
    print lcp.grade_problems({'resistor_2_1':'1.0','resistor_3_1':'2.0'})
    #print lcp.grade_problems({'simpleFormula_2_1':'3*x^3'})
#numericalresponse(problem, context)

#print etree.tostring((lcp.tree))
    print '============'
    print
#print etree.tostring(lcp.extract_problems(lcp.tree))
    print lcp.get_html()
#print extract_context(tree)

    # def handle_fr(self, element):
    #     problem={"answer":self.contextualize_text(answer),
    #              "type":"formularesponse",
    #              "tolerance":evaluator({},{},self.contextualize_text(tolerance)),
    #              "sample_range":dict(zip(variables, sranges)),
    #              "samples_count": numsamples,
    #              "id":id,
    #     self.questions[self.lid]=problem
<commit_msg>capa_problem: convert print of errors into log.*<commit_after>#
# File:   courseware/capa/capa_problem.py
#
'''
Main module which shows problems (of "capa" type).

This is used by capa_module.
'''

import copy
import logging
import math
import numpy
import os
import random
import re
import scipy
import struct

from lxml import etree
from lxml.etree import Element
from xml.sax.saxutils import escape, unescape

from mako.template import Template

from util import contextualize_text
import inputtypes

from responsetypes import NumericalResponse, FormulaResponse, CustomResponse, SchematicResponse, MultipleChoiceResponse,  StudentInputError, TrueFalseResponse, ExternalResponse,ImageResponse,OptionResponse

import calc
import eia

log = logging.getLogger("mitx.courseware")

response_types = {'numericalresponse':NumericalResponse,
                  'formularesponse':FormulaResponse,
                  'customresponse':CustomResponse,
                  'schematicresponse':SchematicResponse,
                  'externalresponse':ExternalResponse,
                  'multiplechoiceresponse':MultipleChoiceResponse,
                  'truefalseresponse':TrueFalseResponse,
                  'imageresponse':ImageResponse,
                  'optionresponse':OptionResponse,
                  }
entry_types = ['textline', 'schematic', 'choicegroup','textbox','imageinput','optioninput']
solution_types = ['solution']	# extra things displayed after "show answers" is pressed
response_properties = ["responseparam", "answer"]	# these get captured as student responses

# How to convert from original XML to HTML
# We should do this with xlst later
html_transforms = {'problem': {'tag':'div'},
                   "numericalresponse": {'tag':'span'},
                   "customresponse": {'tag':'span'},
                   "externalresponse": {'tag':'span'},
                   "schematicresponse": {'tag':'span'},
                   "formularesponse": {'tag':'span'},
                   "multiplechoiceresponse": {'tag':'span'},
                   "text": {'tag':'span'},
                   "math": {'tag':'span'},
                   }

global_context={'random':random,
                'numpy':numpy,
                'math':math,
                'scipy':scipy,
                'calc':calc,
                'eia':eia}

# These should be removed from HTML output, including all subelements
html_problem_semantics = ["responseparam", "answer", "script"]
# These should be removed from HTML output, but keeping subelements
html_skip = ["numericalresponse", "customresponse", "schematicresponse", "formularesponse", "text","externalresponse"]

# removed in MC
## These should be transformed
#html_special_response = {"textline":inputtypes.textline.render,
#                         "schematic":inputtypes.schematic.render,
#                         "textbox":inputtypes.textbox.render,
#                         "formulainput":inputtypes.jstextline.render,
#                         "solution":inputtypes.solution.render,
#                         }

class LoncapaProblem(object):
    def __init__(self, fileobject, id, state=None, seed=None, system=None):
        ## Initialize class variables from state
        self.seed = None
        self.student_answers = dict()
        self.correct_map = dict()
        self.done = False
        self.problem_id = id
        self.system = system

        if seed != None:
            self.seed = seed

        if state:
            if 'seed' in state:
                self.seed = state['seed']
            if 'student_answers' in state:
                self.student_answers = state['student_answers']
            if 'correct_map' in state:
                self.correct_map = state['correct_map']
            if 'done' in state:
                self.done = state['done']

        # TODO: Does this deplete the Linux entropy pool? Is this fast enough?
        if not self.seed:
            self.seed=struct.unpack('i', os.urandom(4))[0]

        ## Parse XML file
        log.info("[courseware.capa.capa_problem.lcp.init]  fileobject = %s" % fileobject)
        file_text = fileobject.read()
        self.fileobject = fileobject	# save it, so we can use for debugging information later
        # Convert startouttext and endouttext to proper <text></text>
        # TODO: Do with XML operations
        file_text = re.sub("startouttext\s*/","text",file_text)
        file_text = re.sub("endouttext\s*/","/text",file_text)
        self.tree = etree.XML(file_text)

        self.preprocess_problem(self.tree, correct_map=self.correct_map, answer_map = self.student_answers)
        self.context = self.extract_context(self.tree, seed=self.seed)
        for response in self.tree.xpath('//'+"|//".join(response_types)):
            responder = response_types[response.tag](response, self.context, self.system)
            responder.preprocess_response()

    def get_state(self):
        ''' Stored per-user session data neeeded to:
            1) Recreate the problem
            2) Populate any student answers. '''
        return {'seed':self.seed,
                'student_answers':self.student_answers,
                'correct_map':self.correct_map,
                'done':self.done}

    def get_max_score(self):
        '''
        TODO: multiple points for programming problems.
        '''
        sum = 0
        for et in entry_types:
            sum = sum + self.tree.xpath('count(//'+et+')')
        return int(sum)

    def get_score(self):
        correct=0
        for key in self.correct_map:
            if self.correct_map[key] == u'correct':
                correct += 1
        if (not self.student_answers) or len(self.student_answers)==0:
            return {'score':0,
                    'total':self.get_max_score()}
        else:
            return {'score':correct,
                    'total':self.get_max_score()}

    def grade_answers(self, answers):
        '''
        Grade student responses.  Called by capa_module.check_problem.
        answers is a dict of all the entries from request.POST, but with the first part
        of each key removed (the string before the first "_").

        Thus, for example, input_ID123 -> ID123, and input_fromjs_ID123 -> fromjs_ID123
        '''
        self.student_answers = answers
        context=self.extract_context(self.tree)
        self.correct_map = dict()
        problems_simple = self.extract_problems(self.tree)
        for response in problems_simple:
            grader = response_types[response.tag](response, self.context, self.system)
            results = grader.get_score(answers)		# call the responsetype instance to do the actual grading
            self.correct_map.update(results)
        return self.correct_map

    def get_question_answers(self):
        '''
        Make a dict of (id,correct_answer) entries, for all the problems.
        Called by "show answers" button JSON request (see capa_module)
        '''
        context=self.extract_context(self.tree)
        answer_map = dict()
        problems_simple = self.extract_problems(self.tree)	# purified (flat) XML tree of just response queries
        for response in problems_simple:
            responder = response_types[response.tag](response, self.context, self.system)	# instance of numericalresponse, customresponse,...
            results = responder.get_answers()
            answer_map.update(results)				# dict of (id,correct_answer)

        # example for the following: <textline size="5" correct_answer="saturated" />
        for entry in problems_simple.xpath("//"+"|//".join(response_properties+entry_types)):
            answer = entry.get('correct_answer')		# correct answer, when specified elsewhere, eg in a textline
            if answer:
                answer_map[entry.get('id')] = contextualize_text(answer, self.context)

        # include solutions from <solution>...</solution> stanzas
        # Tentative merge; we should figure out how we want to handle hints and solutions
        for entry in self.tree.xpath("//"+"|//".join(solution_types)):
            answer = etree.tostring(entry)
            if answer:
                answer_map[entry.get('id')] = answer

        return answer_map

    # ======= Private ========

    def extract_context(self, tree, seed = struct.unpack('i', os.urandom(4))[0]):  # private
        '''
        Extract content of <script>...</script> from the problem.xml file, and exec it in the
        context of this problem.  Provides ability to randomize problems, and also set
        variables for problem answer checking.

        Problem XML goes to Python execution context. Runs everything in script tags
        '''
        random.seed(self.seed)
        context = {'global_context':global_context}	# save global context in here also
        context.update(global_context)			# initialize context to have stuff in global_context
        context['__builtins__'] = globals()['__builtins__']	# put globals there also
        context['the_lcp'] = self				# pass instance of LoncapaProblem in

        #for script in tree.xpath('/problem/script'):
        for script in tree.findall('.//script'):
            if 'javascript' in script.get('type'): continue	# skip javascript
            if 'perl' in script.get('type'): continue		# skip perl
            # TODO: evaluate only python
            code = script.text
            XMLESC = {"'": "'", """: '"'}
            code = unescape(code,XMLESC)
            try:
                exec code in context, context		# use "context" for global context; thus defs in code are global within code
            except Exception,err:
                log.exception("[courseware.capa.capa_problem.extract_context] error %s" % err)
                log.exception("in doing exec of this code: %s" % code)
        return context

    def get_html(self):
        return contextualize_text(etree.tostring(self.extract_html(self.tree)[0]), self.context)

    def extract_html(self, problemtree):  # private
        ''' Helper function for get_html. Recursively converts XML tree to HTML
        '''
        if problemtree.tag in html_problem_semantics:
            return

        problemid = problemtree.get('id')	# my ID

        # used to be
        # if problemtree.tag in html_special_response:

        if problemtree.tag in inputtypes.get_input_xml_tags():
            # status is currently the answer for the problem ID for the input element,
            # but it will turn into a dict containing both the answer and any associated message
            # for the problem ID for the input element.
            status = "unsubmitted"
            if problemid in self.correct_map:
                status = self.correct_map[problemtree.get('id')]

            value = ""
            if self.student_answers and problemid in self.student_answers:
                value = self.student_answers[problemid]

            #### This code is a hack. It was merged to help bring two branches
            #### in sync, but should be replaced. msg should be passed in a
            #### response_type
            # prepare the response message, if it exists in correct_map
            if 'msg' in self.correct_map:
                msg = self.correct_map['msg']
            elif ('msg_%s' % problemid) in self.correct_map:
                msg = self.correct_map['msg_%s' % problemid]
            else:
                msg = ''

            #if settings.DEBUG:
            #    print "[courseware.capa.capa_problem.extract_html] msg = ",msg

            # do the rendering
            # This should be broken out into a helper function
            # that handles all input objects
            render_object = inputtypes.SimpleInput(system = self.system,
                                                   xml = problemtree,
                                                   state = {'value':value,
                                                            'status': status,
                                                            'id':problemtree.get('id'),
                                                            'feedback':{'message':msg}
                                                            },
                                                   use = 'capa_input')
            return render_object.get_html() #function(problemtree, value, status, msg) # render the special response (textline, schematic,...)

        tree=Element(problemtree.tag)
        for item in problemtree:
            subitems = self.extract_html(item)
            if subitems is not None:
                for subitem in subitems:
                    tree.append(subitem)
        for (key,value) in problemtree.items():
            tree.set(key, value)

        tree.text=problemtree.text
        tree.tail=problemtree.tail

        if problemtree.tag in html_transforms:
            tree.tag=html_transforms[problemtree.tag]['tag']
            # Reset attributes. Otherwise, we get metadata in HTML
            # (e.g. answers)
            # TODO: We should remove and not zero them.
            # I'm not sure how to do that quickly with lxml
            for k in tree.keys():
                tree.set(k,"")

        # TODO: Fix. This loses Element().tail
        #if problemtree.tag in html_skip:
        #    return tree
        return [tree]

    def preprocess_problem(self, tree, correct_map=dict(), answer_map=dict()): # private
        '''
        Assign IDs to all the responses
        Assign sub-IDs to all entries (textline, schematic, etc.)
        Annoted correctness and value
        In-place transformation
        '''
        response_id = 1
        for response in tree.xpath('//'+"|//".join(response_types)):
            response_id_str=self.problem_id+"_"+str(response_id)
            response.attrib['id']=response_id_str
            if response_id not in correct_map:
                correct = 'unsubmitted'
            response.attrib['state'] = correct
            response_id = response_id + 1
            answer_id = 1
            for entry in tree.xpath("|".join(['//'+response.tag+'[@id=$id]//'+x for x in (entry_types + solution_types)]),
                                    id=response_id_str):
                # assign one answer_id for each entry_type or solution_type
                entry.attrib['response_id'] = str(response_id)
                entry.attrib['answer_id'] = str(answer_id)
                entry.attrib['id'] = "%s_%i_%i"%(self.problem_id, response_id, answer_id)
                answer_id=answer_id+1

        # <solution>...</solution> may not be associated with any specific response; give IDs for those separately
		# TODO: We should make the namespaces consistent and unique (e.g. %s_problem_%i).
        solution_id = 1
        for solution in tree.findall('.//solution'):
            solution.attrib['id'] =  "%s_solution_%i"%(self.problem_id, solution_id)
            solution_id += 1

    def extract_problems(self, problem_tree):
        ''' Remove layout from the problem, and give a purified XML tree of just the problems '''
        problem_tree=copy.deepcopy(problem_tree)
        tree=Element('problem')
        for response in problem_tree.xpath("//"+"|//".join(response_types)):
            newresponse = copy.copy(response)
            for e in newresponse:
                newresponse.remove(e)
            # copy.copy is needed to make xpath work right. Otherwise, it starts at the root
            # of the tree. We should figure out if there's some work-around
            for e in copy.copy(response).xpath("//"+"|//".join(response_properties+entry_types)):
                newresponse.append(e)

            tree.append(newresponse)
        return tree

if __name__=='__main__':
    problem_id='simpleFormula'
    filename = 'simpleFormula.xml'

    problem_id='resistor'
    filename = 'resistor.xml'

    lcp = LoncapaProblem(filename, problem_id)

    context = lcp.extract_context(lcp.tree)
    problem = lcp.extract_problems(lcp.tree)
    print lcp.grade_problems({'resistor_2_1':'1.0','resistor_3_1':'2.0'})
    #print lcp.grade_problems({'simpleFormula_2_1':'3*x^3'})
#numericalresponse(problem, context)

#print etree.tostring((lcp.tree))
    print '============'
    print
#print etree.tostring(lcp.extract_problems(lcp.tree))
    print lcp.get_html()
#print extract_context(tree)

    # def handle_fr(self, element):
    #     problem={"answer":self.contextualize_text(answer),
    #              "type":"formularesponse",
    #              "tolerance":evaluator({},{},self.contextualize_text(tolerance)),
    #              "sample_range":dict(zip(variables, sranges)),
    #              "samples_count": numsamples,
    #              "id":id,
    #     self.questions[self.lid]=problem
<|endoftext|>