<commit_before># Licensed under a 3-clause BSD style license - see LICENSE.rst

import math

from astropy.coordinates import SkyCoord

from .core import PixelAperture, SkyAperture
from .bounding_box import BoundingBox
from .mask import ApertureMask
from ..geometry import circular_overlap_grid
from ..utils.wcs_helpers import assert_angle_or_pixel

__all__ = ['CircularMaskMixin', 'CircularAperture', 'CircularAnnulus',
           'SkyCircularAperture', 'SkyCircularAnnulus']

class CircularMaskMixin:
    """
    Mixin class to create masks for circular and circular-annulus
    aperture objects.
    """

    def to_mask(self, method='exact', subpixels=5):
        """
        Return a list of `~photutils.ApertureMask` objects, one for each
        aperture position.

        Parameters
        ----------
        method : {'exact', 'center', 'subpixel'}, optional
            The method used to determine the overlap of the aperture on
            the pixel grid.  Not all options are available for all
            aperture types.  Note that the more precise methods are
            generally slower.  The following methods are available:

                * ``'exact'`` (default):
                  The the exact fractional overlap of the aperture and
                  each pixel is calculated.  The returned mask will
                  contain values between 0 and 1.

                * ``'center'``:
                  A pixel is considered to be entirely in or out of the
                  aperture depending on whether its center is in or out
                  of the aperture.  The returned mask will contain
                  values only of 0 (out) and 1 (in).

                * ``'subpixel'``:
                  A pixel is divided into subpixels (see the
                  ``subpixels`` keyword), each of which are considered
                  to be entirely in or out of the aperture depending on
                  whether its center is in or out of the aperture.  If
                  ``subpixels=1``, this method is equivalent to
                  ``'center'``.  The returned mask will contain values
                  between 0 and 1.

        subpixels : int, optional
            For the ``'subpixel'`` method, resample pixels by this factor
            in each dimension.  That is, each pixel is divided into
            ``subpixels ** 2`` subpixels.

        Returns
        -------
        mask : list of `~photutils.ApertureMask`
            A list of aperture mask objects.
        """

        use_exact, subpixels = self._translate_mask_mode(method, subpixels)

        if hasattr(self, 'r'):
            radius = self.r
        elif hasattr(self, 'r_out'):    # annulus
            radius = self.r_out
        else:
            raise ValueError('Cannot determine the aperture radius.')

        masks = []
        for bbox, edges in zip(self.bounding_boxes, self._centered_edges):
            ny, nx = bbox.shape
            mask = circular_overlap_grid(edges[0], edges[1], edges[2],
                                         edges[3], nx, ny, radius, use_exact,
                                         subpixels)

            # subtract the inner circle for an annulus
            if hasattr(self, 'r_in'):
                mask -= circular_overlap_grid(edges[0], edges[1], edges[2],
                                              edges[3], nx, ny, self.r_in,
                                              use_exact, subpixels)

            masks.append(ApertureMask(mask, bbox))

        return masks

class CircularAperture(CircularMaskMixin, PixelAperture):
    """
    A circular aperture defined in pixel coordinates.

    The aperture has a single fixed size/shape, but it can have multiple
    positions (see the ``positions`` input).

    Parameters
    ----------
    positions : array_like or `~astropy.units.Quantity`
        The pixel coordinates of the aperture center(s) in one of the
        following formats:

            * single ``(x, y)`` tuple
            * list of ``(x, y)`` tuples
            * ``Nx2`` or ``2xN`` `~numpy.ndarray`
            * ``Nx2`` or ``2xN`` `~astropy.units.Quantity` in pixel units

        Note that a ``2x2`` `~numpy.ndarray` or
        `~astropy.units.Quantity` is interpreted as ``Nx2``, i.e. two
        rows of (x, y) coordinates.

    r : float
        The radius of the circle in pixels.

    Raises
    ------
    ValueError : `ValueError`
        If the input radius, ``r``, is negative.
    """

    def __init__(self, positions, r):
        if r < 0:
            raise ValueError('r must be non-negative')

        self.positions = self._sanitize_positions(positions)
        self.r = float(r)
        self._params = ['r']

    # TODO: make lazyproperty?, but update if positions or radius change
    @property
    def bounding_boxes(self):
        xmin = self.positions[:, 0] - self.r
        xmax = self.positions[:, 0] + self.r
        ymin = self.positions[:, 1] - self.r
        ymax = self.positions[:, 1] + self.r

        return [BoundingBox._from_float(x0, x1, y0, y1)
                for x0, x1, y0, y1 in zip(xmin, xmax, ymin, ymax)]

    # TODO: make lazyproperty?, but update if positions or radius change
    def area(self):
        return math.pi * self.r ** 2

    def plot(self, origin=(0, 0), indices=None, ax=None, fill=False,
             **kwargs):
        import matplotlib.patches as mpatches

        plot_positions, ax, kwargs = self._prepare_plot(
            origin, indices, ax, fill, **kwargs)

        for position in plot_positions:
            patch = mpatches.Circle(position, self.r, **kwargs)
            ax.add_patch(patch)

    def to_sky(self, wcs, mode='all'):
        """
        Convert the aperture to a `SkyCircularAperture` object defined
        in celestial coordinates.

        Parameters
        ----------
        wcs : `~astropy.wcs.WCS`
            The world coordinate system (WCS) transformation to use.

        mode : {'all', 'wcs'}, optional
            Whether to do the transformation including distortions
            (``'all'``; default) or only including only the core WCS
            transformation (``'wcs'``).

        Returns
        -------
        aperture : `SkyCircularAperture` object
            A `SkyCircularAperture` object.
        """

        sky_params = self._to_sky_params(wcs, mode=mode)
        return SkyCircularAperture(**sky_params)

class CircularAnnulus(CircularMaskMixin, PixelAperture):
    """
    A circular annulus aperture defined in pixel coordinates.

    The aperture has a single fixed size/shape, but it can have multiple
    positions (see the ``positions`` input).

    Parameters
    ----------
    positions : array_like or `~astropy.units.Quantity`
        The pixel coordinates of the aperture center(s) in one of the
        following formats:

            * single ``(x, y)`` tuple
            * list of ``(x, y)`` tuples
            * ``Nx2`` or ``2xN`` `~numpy.ndarray`
            * ``Nx2`` or ``2xN`` `~astropy.units.Quantity` in pixel units

        Note that a ``2x2`` `~numpy.ndarray` or
        `~astropy.units.Quantity` is interpreted as ``Nx2``, i.e. two
        rows of (x, y) coordinates.

    r_in : float
        The inner radius of the circular annulus in pixels.

    r_out : float
        The outer radius of the circular annulus in pixels.

    Raises
    ------
    ValueError : `ValueError`
        If inner radius (``r_in``) is greater than outer radius (``r_out``).

    ValueError : `ValueError`
        If inner radius (``r_in``) is negative.
    """

    def __init__(self, positions, r_in, r_out):
        if not (r_out > r_in):
            raise ValueError('r_out must be greater than r_in')
        if r_in < 0:
            raise ValueError('r_in must be non-negative')

        self.positions = self._sanitize_positions(positions)
        self.r_in = float(r_in)
        self.r_out = float(r_out)
        self._params = ['r_in', 'r_out']

    @property
    def bounding_boxes(self):
        xmin = self.positions[:, 0] - self.r_out
        xmax = self.positions[:, 0] + self.r_out
        ymin = self.positions[:, 1] - self.r_out
        ymax = self.positions[:, 1] + self.r_out

        return [BoundingBox._from_float(x0, x1, y0, y1)
                for x0, x1, y0, y1 in zip(xmin, xmax, ymin, ymax)]

    def area(self):
        return math.pi * (self.r_out ** 2 - self.r_in ** 2)

    def plot(self, origin=(0, 0), indices=None, ax=None, fill=False,
             **kwargs):
        import matplotlib.patches as mpatches

        plot_positions, ax, kwargs = self._prepare_plot(
            origin, indices, ax, fill, **kwargs)

        for position in plot_positions:
            patch_inner = mpatches.Circle(position, self.r_in)
            patch_outer = mpatches.Circle(position, self.r_out)
            path = self._make_annulus_path(patch_inner, patch_outer)
            patch = mpatches.PathPatch(path, **kwargs)
            ax.add_patch(patch)

    def to_sky(self, wcs, mode='all'):
        """
        Convert the aperture to a `SkyCircularAnnulus` object defined
        in celestial coordinates.

        Parameters
        ----------
        wcs : `~astropy.wcs.WCS`
            The world coordinate system (WCS) transformation to use.

        mode : {'all', 'wcs'}, optional
            Whether to do the transformation including distortions
            (``'all'``; default) or only including only the core WCS
            transformation (``'wcs'``).

        Returns
        -------
        aperture : `SkyCircularAnnulus` object
            A `SkyCircularAnnulus` object.
        """

        sky_params = self._to_sky_params(wcs, mode=mode)
        return SkyCircularAnnulus(**sky_params)

class SkyCircularAperture(SkyAperture):
    """
    A circular aperture defined in sky coordinates.

    The aperture has a single fixed size/shape, but it can have multiple
    positions (see the ``positions`` input).

    Parameters
    ----------
    positions : `~astropy.coordinates.SkyCoord`
        The celestial coordinates of the aperture center(s). This can be
        either scalar coordinates or an array of coordinates.

    r : scalar `~astropy.units.Quantity`
        The radius of the circle, either in angular or pixel units.
    """

    def __init__(self, positions, r):
        if isinstance(positions, SkyCoord):
            self.positions = positions
        else:
            raise TypeError('positions must be a SkyCoord object')

        assert_angle_or_pixel('r', r)
        self.r = r
        self._params = ['r']

    def to_pixel(self, wcs, mode='all'):
        """
        Convert the aperture to a `CircularAperture` object defined in
        pixel coordinates.

        Parameters
        ----------
        wcs : `~astropy.wcs.WCS`
            The world coordinate system (WCS) transformation to use.

        mode : {'all', 'wcs'}, optional
            Whether to do the transformation including distortions
            (``'all'``; default) or only including only the core WCS
            transformation (``'wcs'``).

        Returns
        -------
        aperture : `CircularAperture` object
            A `CircularAperture` object.
        """

        pixel_params = self._to_pixel_params(wcs, mode=mode)
        return CircularAperture(**pixel_params)

class SkyCircularAnnulus(SkyAperture):
    """
    A circular annulus aperture defined in sky coordinates.

    The aperture has a single fixed size/shape, but it can have multiple
    positions (see the ``positions`` input).

    Parameters
    ----------
    positions : `~astropy.coordinates.SkyCoord`
        The celestial coordinates of the aperture center(s). This can be
        either scalar coordinates or an array of coordinates.

    r_in : scalar `~astropy.units.Quantity`
        The inner radius of the circular annulus, either in angular or
        pixel units.

    r_out : scalar `~astropy.units.Quantity`
        The outer radius of the circular annulus, either in angular or
        pixel units.
    """

    def __init__(self, positions, r_in, r_out):
        if isinstance(positions, SkyCoord):
            self.positions = positions
        else:
            raise TypeError('positions must be a SkyCoord object')

        assert_angle_or_pixel('r_in', r_in)
        assert_angle_or_pixel('r_out', r_out)

        if r_in.unit.physical_type != r_out.unit.physical_type:
            raise ValueError("r_in and r_out should either both be angles "
                             "or in pixels.")

        self.r_in = r_in
        self.r_out = r_out
        self._params = ['r_in', 'r_out']

    def to_pixel(self, wcs, mode='all'):
        """
        Convert the aperture to a `CircularAnnulus` object defined in
        pixel coordinates.

        Parameters
        ----------
        wcs : `~astropy.wcs.WCS`
            The world coordinate system (WCS) transformation to use.

        mode : {'all', 'wcs'}, optional
            Whether to do the transformation including distortions
            (``'all'``; default) or only including only the core WCS
            transformation (``'wcs'``).

        Returns
        -------
        aperture : `CircularAnnulus` object
            A `CircularAnnulus` object.
        """

        pixel_params = self._to_pixel_params(wcs, mode=mode)
        return CircularAnnulus(**pixel_params)
<commit_msg>Validate circle aperture parameters<commit_after># Licensed under a 3-clause BSD style license - see LICENSE.rst

import math

from astropy.coordinates import SkyCoord

from .attributes import (PositiveScalar, AngleOrPixelScalarQuantity,
                         SkyCoordPosition)
from .core import PixelAperture, SkyAperture
from .bounding_box import BoundingBox
from .mask import ApertureMask
from ..geometry import circular_overlap_grid

__all__ = ['CircularMaskMixin', 'CircularAperture', 'CircularAnnulus',
           'SkyCircularAperture', 'SkyCircularAnnulus']

class CircularMaskMixin:
    """
    Mixin class to create masks for circular and circular-annulus
    aperture objects.
    """

    def to_mask(self, method='exact', subpixels=5):
        """
        Return a list of `~photutils.ApertureMask` objects, one for each
        aperture position.

        Parameters
        ----------
        method : {'exact', 'center', 'subpixel'}, optional
            The method used to determine the overlap of the aperture on
            the pixel grid.  Not all options are available for all
            aperture types.  Note that the more precise methods are
            generally slower.  The following methods are available:

                * ``'exact'`` (default):
                  The the exact fractional overlap of the aperture and
                  each pixel is calculated.  The returned mask will
                  contain values between 0 and 1.

                * ``'center'``:
                  A pixel is considered to be entirely in or out of the
                  aperture depending on whether its center is in or out
                  of the aperture.  The returned mask will contain
                  values only of 0 (out) and 1 (in).

                * ``'subpixel'``:
                  A pixel is divided into subpixels (see the
                  ``subpixels`` keyword), each of which are considered
                  to be entirely in or out of the aperture depending on
                  whether its center is in or out of the aperture.  If
                  ``subpixels=1``, this method is equivalent to
                  ``'center'``.  The returned mask will contain values
                  between 0 and 1.

        subpixels : int, optional
            For the ``'subpixel'`` method, resample pixels by this factor
            in each dimension.  That is, each pixel is divided into
            ``subpixels ** 2`` subpixels.

        Returns
        -------
        mask : list of `~photutils.ApertureMask`
            A list of aperture mask objects.
        """

        use_exact, subpixels = self._translate_mask_mode(method, subpixels)

        if hasattr(self, 'r'):
            radius = self.r
        elif hasattr(self, 'r_out'):    # annulus
            radius = self.r_out
        else:
            raise ValueError('Cannot determine the aperture radius.')

        masks = []
        for bbox, edges in zip(self.bounding_boxes, self._centered_edges):
            ny, nx = bbox.shape
            mask = circular_overlap_grid(edges[0], edges[1], edges[2],
                                         edges[3], nx, ny, radius, use_exact,
                                         subpixels)

            # subtract the inner circle for an annulus
            if hasattr(self, 'r_in'):
                mask -= circular_overlap_grid(edges[0], edges[1], edges[2],
                                              edges[3], nx, ny, self.r_in,
                                              use_exact, subpixels)

            masks.append(ApertureMask(mask, bbox))

        return masks

class CircularAperture(CircularMaskMixin, PixelAperture):
    """
    A circular aperture defined in pixel coordinates.

    The aperture has a single fixed size/shape, but it can have multiple
    positions (see the ``positions`` input).

    Parameters
    ----------
    positions : array_like or `~astropy.units.Quantity`
        The pixel coordinates of the aperture center(s) in one of the
        following formats:

            * single ``(x, y)`` tuple
            * list of ``(x, y)`` tuples
            * ``Nx2`` or ``2xN`` `~numpy.ndarray`
            * ``Nx2`` or ``2xN`` `~astropy.units.Quantity` in pixel units

        Note that a ``2x2`` `~numpy.ndarray` or
        `~astropy.units.Quantity` is interpreted as ``Nx2``, i.e. two
        rows of (x, y) coordinates.

    r : float
        The radius of the circle in pixels.

    Raises
    ------
    ValueError : `ValueError`
        If the input radius, ``r``, is negative.
    """

    r = PositiveScalar('r')

    def __init__(self, positions, r):
        self.positions = self._sanitize_positions(positions)
        self.r = r
        self._params = ['r']

    # TODO: make lazyproperty?, but update if positions or radius change
    @property
    def bounding_boxes(self):
        xmin = self.positions[:, 0] - self.r
        xmax = self.positions[:, 0] + self.r
        ymin = self.positions[:, 1] - self.r
        ymax = self.positions[:, 1] + self.r

        return [BoundingBox._from_float(x0, x1, y0, y1)
                for x0, x1, y0, y1 in zip(xmin, xmax, ymin, ymax)]

    # TODO: make lazyproperty?, but update if positions or radius change
    def area(self):
        return math.pi * self.r ** 2

    def plot(self, origin=(0, 0), indices=None, ax=None, fill=False,
             **kwargs):
        import matplotlib.patches as mpatches

        plot_positions, ax, kwargs = self._prepare_plot(
            origin, indices, ax, fill, **kwargs)

        for position in plot_positions:
            patch = mpatches.Circle(position, self.r, **kwargs)
            ax.add_patch(patch)

    def to_sky(self, wcs, mode='all'):
        """
        Convert the aperture to a `SkyCircularAperture` object defined
        in celestial coordinates.

        Parameters
        ----------
        wcs : `~astropy.wcs.WCS`
            The world coordinate system (WCS) transformation to use.

        mode : {'all', 'wcs'}, optional
            Whether to do the transformation including distortions
            (``'all'``; default) or only including only the core WCS
            transformation (``'wcs'``).

        Returns
        -------
        aperture : `SkyCircularAperture` object
            A `SkyCircularAperture` object.
        """

        sky_params = self._to_sky_params(wcs, mode=mode)
        return SkyCircularAperture(**sky_params)

class CircularAnnulus(CircularMaskMixin, PixelAperture):
    """
    A circular annulus aperture defined in pixel coordinates.

    The aperture has a single fixed size/shape, but it can have multiple
    positions (see the ``positions`` input).

    Parameters
    ----------
    positions : array_like or `~astropy.units.Quantity`
        The pixel coordinates of the aperture center(s) in one of the
        following formats:

            * single ``(x, y)`` tuple
            * list of ``(x, y)`` tuples
            * ``Nx2`` or ``2xN`` `~numpy.ndarray`
            * ``Nx2`` or ``2xN`` `~astropy.units.Quantity` in pixel units

        Note that a ``2x2`` `~numpy.ndarray` or
        `~astropy.units.Quantity` is interpreted as ``Nx2``, i.e. two
        rows of (x, y) coordinates.

    r_in : float
        The inner radius of the circular annulus in pixels.

    r_out : float
        The outer radius of the circular annulus in pixels.

    Raises
    ------
    ValueError : `ValueError`
        If inner radius (``r_in``) is greater than outer radius (``r_out``).

    ValueError : `ValueError`
        If inner radius (``r_in``) is negative.
    """

    r_in = PositiveScalar('r_in')
    r_out = PositiveScalar('r_out')

    def __init__(self, positions, r_in, r_out):
        if not (r_out > r_in):
            raise ValueError('r_out must be greater than r_in')

        self.positions = self._sanitize_positions(positions)
        self.r_in = r_in
        self.r_out = r_out
        self._params = ['r_in', 'r_out']

    @property
    def bounding_boxes(self):
        xmin = self.positions[:, 0] - self.r_out
        xmax = self.positions[:, 0] + self.r_out
        ymin = self.positions[:, 1] - self.r_out
        ymax = self.positions[:, 1] + self.r_out

        return [BoundingBox._from_float(x0, x1, y0, y1)
                for x0, x1, y0, y1 in zip(xmin, xmax, ymin, ymax)]

    def area(self):
        return math.pi * (self.r_out ** 2 - self.r_in ** 2)

    def plot(self, origin=(0, 0), indices=None, ax=None, fill=False,
             **kwargs):
        import matplotlib.patches as mpatches

        plot_positions, ax, kwargs = self._prepare_plot(
            origin, indices, ax, fill, **kwargs)

        for position in plot_positions:
            patch_inner = mpatches.Circle(position, self.r_in)
            patch_outer = mpatches.Circle(position, self.r_out)
            path = self._make_annulus_path(patch_inner, patch_outer)
            patch = mpatches.PathPatch(path, **kwargs)
            ax.add_patch(patch)

    def to_sky(self, wcs, mode='all'):
        """
        Convert the aperture to a `SkyCircularAnnulus` object defined
        in celestial coordinates.

        Parameters
        ----------
        wcs : `~astropy.wcs.WCS`
            The world coordinate system (WCS) transformation to use.

        mode : {'all', 'wcs'}, optional
            Whether to do the transformation including distortions
            (``'all'``; default) or only including only the core WCS
            transformation (``'wcs'``).

        Returns
        -------
        aperture : `SkyCircularAnnulus` object
            A `SkyCircularAnnulus` object.
        """

        sky_params = self._to_sky_params(wcs, mode=mode)
        return SkyCircularAnnulus(**sky_params)

class SkyCircularAperture(SkyAperture):
    """
    A circular aperture defined in sky coordinates.

    The aperture has a single fixed size/shape, but it can have multiple
    positions (see the ``positions`` input).

    Parameters
    ----------
    positions : `~astropy.coordinates.SkyCoord`
        The celestial coordinates of the aperture center(s). This can be
        either scalar coordinates or an array of coordinates.

    r : scalar `~astropy.units.Quantity`
        The radius of the circle, either in angular or pixel units.
    """

    positions = SkyCoordPosition('positions')
    r = AngleOrPixelScalarQuantity('r')

    def __init__(self, positions, r):
        self.positions = positions
        self.r = r
        self._params = ['r']

    def to_pixel(self, wcs, mode='all'):
        """
        Convert the aperture to a `CircularAperture` object defined in
        pixel coordinates.

        Parameters
        ----------
        wcs : `~astropy.wcs.WCS`
            The world coordinate system (WCS) transformation to use.

        mode : {'all', 'wcs'}, optional
            Whether to do the transformation including distortions
            (``'all'``; default) or only including only the core WCS
            transformation (``'wcs'``).

        Returns
        -------
        aperture : `CircularAperture` object
            A `CircularAperture` object.
        """

        pixel_params = self._to_pixel_params(wcs, mode=mode)
        return CircularAperture(**pixel_params)

class SkyCircularAnnulus(SkyAperture):
    """
    A circular annulus aperture defined in sky coordinates.

    The aperture has a single fixed size/shape, but it can have multiple
    positions (see the ``positions`` input).

    Parameters
    ----------
    positions : `~astropy.coordinates.SkyCoord`
        The celestial coordinates of the aperture center(s). This can be
        either scalar coordinates or an array of coordinates.

    r_in : scalar `~astropy.units.Quantity`
        The inner radius of the circular annulus, either in angular or
        pixel units.

    r_out : scalar `~astropy.units.Quantity`
        The outer radius of the circular annulus, either in angular or
        pixel units.
    """

    positions = SkyCoordPosition('positions')
    r_in = AngleOrPixelScalarQuantity('r_in')
    r_out = AngleOrPixelScalarQuantity('r_out')

    def __init__(self, positions, r_in, r_out):
        if r_in.unit.physical_type != r_out.unit.physical_type:
            raise ValueError("r_in and r_out should either both be angles "
                             "or in pixels.")

        self.positions = positions
        self.r_in = r_in
        self.r_out = r_out
        self._params = ['r_in', 'r_out']

    def to_pixel(self, wcs, mode='all'):
        """
        Convert the aperture to a `CircularAnnulus` object defined in
        pixel coordinates.

        Parameters
        ----------
        wcs : `~astropy.wcs.WCS`
            The world coordinate system (WCS) transformation to use.

        mode : {'all', 'wcs'}, optional
            Whether to do the transformation including distortions
            (``'all'``; default) or only including only the core WCS
            transformation (``'wcs'``).

        Returns
        -------
        aperture : `CircularAnnulus` object
            A `CircularAnnulus` object.
        """

        pixel_params = self._to_pixel_params(wcs, mode=mode)
        return CircularAnnulus(**pixel_params)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
This module contains the individual fixes that the main fix_text function
can perform.
"""

from __future__ import unicode_literals
from ftfy.chardata import (possible_encoding,
                           CHARMAP_ENCODINGS, CONTROL_CHARS)
from ftfy.badness import text_cost
from ftfy.compatibility import (htmlentitydefs, unichr, UNSAFE_PRIVATE_USE_RE)
import re
import sys

BYTES_ERROR_TEXT = """Hey wait, this isn't Unicode.

ftfy is designed to fix problems that were introduced by handling Unicode
incorrectly. It might be able to fix the bytes you just handed it, but the
fact that you just gave a pile of bytes to a function that fixes text means
that your code is *also* handling Unicode incorrectly.

ftfy takes Unicode text as input. You should take these bytes and decode
them from the encoding you think they are in. If you're not sure what encoding
they're in:

- First, try to find out. 'utf-8' is a good assumption.
- If the encoding is simply unknowable, try running your bytes through
  ftfy.guess_bytes. As the name implies, this may not always be accurate.

If you're confused by this, please read the Python Unicode HOWTO:

    http://docs.python.org/%d/howto/unicode.html
""" % sys.version_info[0]

def fix_text_encoding(text):
    r"""
    Fix text with incorrectly-decoded garbage ("mojibake") whenever possible.

    Something you will find all over the place, in real-world text, is text
    that's mistakenly encoded as utf-8, decoded in some ugly format like
    latin-1 or even Windows codepage 1252, and encoded as utf-8 again.

    This causes your perfectly good Unicode-aware code to end up with garbage
    text because someone else (or maybe "someone else") made a mistake.

    This function looks for the evidence of that having happened and fixes it.
    It determines whether it should replace nonsense sequences of single-byte
    characters that were really meant to be UTF-8 characters, and if so, turns
    them into the correctly-encoded Unicode character that they were meant to
    represent.

    The input to the function must be Unicode. If you don't have Unicode text,
    you're not using the right tool to solve your problem.

    .. note::
        The following examples are written using unmarked literal strings,
        but they are Unicode text. In Python 2 we have "unicode_literals"
        turned on, and in Python 3 this is always the case.

    ftfy decodes text that looks like it was decoded incorrectly. It leaves
    alone text that doesn't.

        >>> print(fix_text_encoding('Ãºnico'))
        único

        >>> print(fix_text_encoding('This text is fine already :þ'))
        This text is fine already :þ

    Because these characters often come from Microsoft products, we allow
    for the possibility that we get not just Unicode characters 128-255, but
    also Windows's conflicting idea of what characters 128-160 are.

        >>> print(fix_text_encoding('This â€” should be an em dash'))
        This — should be an em dash

    We might have to deal with both Windows characters and raw control
    characters at the same time, especially when dealing with characters like
    0x81 that have no mapping in Windows. This is a string that Python's
    standard `.encode` and `.decode` methods cannot correct.

        >>> print(fix_text_encoding('This text is sad .â\x81”.'))
        This text is sad .⁔.

    However, it has safeguards against fixing sequences of letters and
    punctuation that can occur in valid text:

        >>> print(fix_text_encoding('not such a fan of Charlotte Brontë…”'))
        not such a fan of Charlotte Brontë…”

    Cases of genuine ambiguity can sometimes be addressed by finding other
    characters that are not double-encoded, and expecting the encoding to
    be consistent:

        >>> print(fix_text_encoding('AHÅ™, the new sofa from IKEA®'))
        AHÅ™, the new sofa from IKEA®

    Finally, we handle the case where the text is in a single-byte encoding
    that was intended as Windows-1252 all along but read as Latin-1:

        >>> print(fix_text_encoding('This text was never UTF-8 at all\x85'))
        This text was never UTF-8 at all…

    The best version of the text is found using
    :func:`ftfy.badness.text_cost`.
    """
    text, plan = fix_encoding_and_explain(text)
    return text

def fix_encoding_and_explain(text):
    """
    Re-decodes text that has been decoded incorrectly, and also return a
    "plan" indicating all the steps required to fix it.

    To fix similar text in the same way, without having to detect anything,
    you can use the ``apply_plan`` function.
    """
    best_version = text
    best_cost = text_cost(text)
    best_plan = []
    plan_so_far = []
    while True:
        prevtext = text
        text, plan = fix_one_step_and_explain(text)
        plan_so_far.extend(plan)
        cost = text_cost(text)

        # Add a penalty if we used a particularly obsolete encoding. The result
        # is that we won't use these encodings unless they can successfully
        # replace multiple characters.
        if ('encode', 'macroman') in plan_so_far or\
           ('encode', 'cp437') in plan_so_far:
            cost += 2

        # We need pretty solid evidence to decode from Windows-1251 (Cyrillic).
        if ('encode', 'sloppy-windows-1251') in plan_so_far:
            cost += 5

        if cost < best_cost:
            best_cost = cost
            best_version = text
            best_plan = list(plan_so_far)
        if text == prevtext:
            return best_version, best_plan

def fix_one_step_and_explain(text):
    """
    Performs a single step of re-decoding text that's been decoded incorrectly.

    Returns the decoded text, plus a "plan" for how to reproduce what it
    did.
    """
    if isinstance(text, bytes):
        raise UnicodeError(BYTES_ERROR_TEXT)
    if len(text) == 0:
        return text, []

    # The first plan is to return ASCII text unchanged.
    if possible_encoding(text, 'ascii'):
        return text, []

    # As we go through the next step, remember the possible encodings
    # that we encounter but don't successfully fix yet. We may need them
    # later.
    possible_1byte_encodings = []

    # Suppose the text was supposed to be UTF-8, but it was decoded using
    # a single-byte encoding instead. When these cases can be fixed, they
    # are usually the correct thing to do, so try them next.
    for encoding in CHARMAP_ENCODINGS:
        if possible_encoding(text, encoding):
            encoded_bytes = text.encode(encoding)

            # Now, find out if it's UTF-8 (or close enough). Otherwise,
            # remember the encoding for later.
            try:
                decoding = 'utf-8'
                if b'\xed' in encoded_bytes or b'\xc0' in encoded_bytes:
                    decoding = 'utf-8-variants'
                fixed = encoded_bytes.decode(decoding)
                steps = [('encode', encoding), ('decode', decoding)]
                return fixed, steps
            except UnicodeDecodeError:
                possible_1byte_encodings.append(encoding)

    # The next most likely case is that this is Latin-1 that was intended to
    # be read as Windows-1252, because those two encodings in particular are
    # easily confused.
    if 'latin-1' in possible_1byte_encodings:
        if 'windows-1252' in possible_1byte_encodings:
            # This text is in the intersection of Latin-1 and
            # Windows-1252, so it's probably legit.
            return text, []
        else:
            # Otherwise, it means we have characters that are in Latin-1 but
            # not in Windows-1252. Those are C1 control characters. Nobody
            # wants those. Assume they were meant to be Windows-1252. Don't
            # use the sloppy codec, because bad Windows-1252 characters are
            # a bad sign.
            encoded = text.encode('latin-1')
            try:
                fixed = encoded.decode('windows-1252')
                steps = []
                if fixed != text:
                    steps = [('encode', 'latin-1'), ('decode', 'windows-1252')]
                return fixed, steps
            except UnicodeDecodeError:
                # This text contained characters that don't even make sense
                # if you assume they were supposed to be Windows-1252. In
                # that case, let's not assume anything.
                pass

    # The cases that remain are mixups between two different single-byte
    # encodings, and not the common case of Latin-1 vs. Windows-1252.
    #
    # Those cases are somewhat rare, and impossible to solve without false
    # positives. If you're in one of these situations, you should try using
    # the `ftfy.guess_bytes` function.

    # Return the text unchanged; the plan is empty.
    return text, []

def apply_plan(text, plan):
    """
    Apply a plan for fixing the encoding of text.

    The plan is a list of tuples of the form (operation, encoding), where
    `operation` is either 'encode' or 'decode', and `encoding` is an encoding
    name such as 'utf-8' or 'latin-1'.

    Because only text can be encoded, and only bytes can be decoded, the plan
    should alternate 'encode' and 'decode' steps, or else this function will
    encounter an error.
    """
    obj = text
    for operation, encoding in plan:
        if operation == 'encode':
            obj = obj.encode(encoding)
        elif operation == 'decode':
            obj = obj.decode(encoding)
        else:
            raise ValueError("Unknown plan step: %s" % operation)

    return obj

HTML_ENTITY_RE = re.compile(r"&#?\w{0,8};")

def unescape_html(text):
    """
    Decode all three types of HTML entities/character references.

    Code by Fredrik Lundh of effbot.org. Robyn Speer made a slight change
    to it for efficiency: it won't match entities longer than 8 characters,
    because there are no valid entities like that.

        >>> print(unescape_html('<tag>'))
        <tag>
    """
    def fixup(match):
        """
        Replace one matched HTML entity with the character it represents,
        if possible.
        """
        text = match.group(0)
        if text[:2] == "&#":
            # character reference
            try:
                if text[:3] == "&#x":
                    return unichr(int(text[3:-1], 16))
                else:
                    return unichr(int(text[2:-1]))
            except ValueError:
                pass
        else:
            # named entity
            try:
                text = unichr(htmlentitydefs.name2codepoint[text[1:-1]])
            except KeyError:
                pass
        return text  # leave as is
    return HTML_ENTITY_RE.sub(fixup, text)

ANSI_RE = re.compile('\033\\[((?:\\d|;)*)([a-zA-Z])')

def remove_terminal_escapes(text):
    r"""
    Strip out "ANSI" terminal escape sequences, such as those that produce
    colored text on Unix.

        >>> print(remove_terminal_escapes(
        ...     "\033[36;44mI'm blue, da ba dee da ba doo...\033[0m"
        ... ))
        I'm blue, da ba dee da ba doo...
    """
    return ANSI_RE.sub('', text)

SINGLE_QUOTE_RE = re.compile('[\u2018-\u201b]')
DOUBLE_QUOTE_RE = re.compile('[\u201c-\u201f]')

def uncurl_quotes(text):
    r"""
    Replace curly quotation marks with straight equivalents.

        >>> print(uncurl_quotes('\u201chere\u2019s a test\u201d'))
        "here's a test"
    """
    return SINGLE_QUOTE_RE.sub("'", DOUBLE_QUOTE_RE.sub('"', text))

def fix_line_breaks(text):
    r"""
    Convert line breaks to Unix style.

    In particular, this replaces CRLF (\\r\\n) with LF (\\n), then
    additionally replaces CR (\\r) with LF (\\n).
    """
    return text.replace('\r\n', '\n').replace('\r', '\n')

def remove_control_chars(text):
    """
    Remove all control characters except for the important ones.

    This removes characters in these ranges:

    - U+0000 to U+0008
    - U+000B
    - U+000E to U+001F
    - U+007F

    It leaves alone these characters that are commonly used for formatting:

    - TAB (U+0009)
    - LF (U+000A)
    - FF (U+000C)
    - CR (U+000D)
    """
    return text.translate(CONTROL_CHARS)

def remove_bom(text):
    r"""
    Remove a left-over byte-order mark.

    >>> print(remove_bom(unichr(0xfeff) + "Where do you want to go today?"))
    Where do you want to go today?
    """
    return text.lstrip(unichr(0xfeff))

def remove_unsafe_private_use(text):
    r"""
    Python 3.3's Unicode support isn't perfect, and in fact there are certain
    string operations that will crash some versions of it with a SystemError:
    http://bugs.python.org/issue18183

    You can trigger the bug by running `` '\U00010000\U00100000'.lower() ``.

    The best solution is to remove all characters from Supplementary Private
    Use Area B, using a regex that is known not to crash given those
    characters.

    These are the characters from U+100000 to U+10FFFF. It's sad to lose an
    entire plane of Unicode, but on the other hand, these characters are not
    assigned and never will be. If you get one of these characters and don't
    know what its purpose is, its purpose is probably to crash your code.

    If you were using these for actual private use, this might be inconvenient.
    You can turn off this fixer, of course, but I kind of encourage using
    Supplementary Private Use Area A instead.

        >>> print(remove_unsafe_private_use('\U0001F4A9\U00100000'))
        💩

    This fixer is off by default in Python 3.4 or later. (The bug is actually
    fixed in 3.3.3 and 2.7.6, but I don't want the default behavior to change
    based on a micro version upgrade of Python.)
    """
    return UNSAFE_PRIVATE_USE_RE.sub('', text)
<commit_msg>Fix three more kinds of purported line breaks<commit_after># -*- coding: utf-8 -*-
"""
This module contains the individual fixes that the main fix_text function
can perform.
"""

from __future__ import unicode_literals
from ftfy.chardata import (possible_encoding,
                           CHARMAP_ENCODINGS, CONTROL_CHARS)
from ftfy.badness import text_cost
from ftfy.compatibility import (htmlentitydefs, unichr, UNSAFE_PRIVATE_USE_RE)
import re
import sys

BYTES_ERROR_TEXT = """Hey wait, this isn't Unicode.

ftfy is designed to fix problems that were introduced by handling Unicode
incorrectly. It might be able to fix the bytes you just handed it, but the
fact that you just gave a pile of bytes to a function that fixes text means
that your code is *also* handling Unicode incorrectly.

ftfy takes Unicode text as input. You should take these bytes and decode
them from the encoding you think they are in. If you're not sure what encoding
they're in:

- First, try to find out. 'utf-8' is a good assumption.
- If the encoding is simply unknowable, try running your bytes through
  ftfy.guess_bytes. As the name implies, this may not always be accurate.

If you're confused by this, please read the Python Unicode HOWTO:

    http://docs.python.org/%d/howto/unicode.html
""" % sys.version_info[0]

def fix_text_encoding(text):
    r"""
    Fix text with incorrectly-decoded garbage ("mojibake") whenever possible.

    Something you will find all over the place, in real-world text, is text
    that's mistakenly encoded as utf-8, decoded in some ugly format like
    latin-1 or even Windows codepage 1252, and encoded as utf-8 again.

    This causes your perfectly good Unicode-aware code to end up with garbage
    text because someone else (or maybe "someone else") made a mistake.

    This function looks for the evidence of that having happened and fixes it.
    It determines whether it should replace nonsense sequences of single-byte
    characters that were really meant to be UTF-8 characters, and if so, turns
    them into the correctly-encoded Unicode character that they were meant to
    represent.

    The input to the function must be Unicode. If you don't have Unicode text,
    you're not using the right tool to solve your problem.

    .. note::
        The following examples are written using unmarked literal strings,
        but they are Unicode text. In Python 2 we have "unicode_literals"
        turned on, and in Python 3 this is always the case.

    ftfy decodes text that looks like it was decoded incorrectly. It leaves
    alone text that doesn't.

        >>> print(fix_text_encoding('Ãºnico'))
        único

        >>> print(fix_text_encoding('This text is fine already :þ'))
        This text is fine already :þ

    Because these characters often come from Microsoft products, we allow
    for the possibility that we get not just Unicode characters 128-255, but
    also Windows's conflicting idea of what characters 128-160 are.

        >>> print(fix_text_encoding('This â€” should be an em dash'))
        This — should be an em dash

    We might have to deal with both Windows characters and raw control
    characters at the same time, especially when dealing with characters like
    0x81 that have no mapping in Windows. This is a string that Python's
    standard `.encode` and `.decode` methods cannot correct.

        >>> print(fix_text_encoding('This text is sad .â\x81”.'))
        This text is sad .⁔.

    However, it has safeguards against fixing sequences of letters and
    punctuation that can occur in valid text:

        >>> print(fix_text_encoding('not such a fan of Charlotte Brontë…”'))
        not such a fan of Charlotte Brontë…”

    Cases of genuine ambiguity can sometimes be addressed by finding other
    characters that are not double-encoded, and expecting the encoding to
    be consistent:

        >>> print(fix_text_encoding('AHÅ™, the new sofa from IKEA®'))
        AHÅ™, the new sofa from IKEA®

    Finally, we handle the case where the text is in a single-byte encoding
    that was intended as Windows-1252 all along but read as Latin-1:

        >>> print(fix_text_encoding('This text was never UTF-8 at all\x85'))
        This text was never UTF-8 at all…

    The best version of the text is found using
    :func:`ftfy.badness.text_cost`.
    """
    text, plan = fix_encoding_and_explain(text)
    return text

def fix_encoding_and_explain(text):
    """
    Re-decodes text that has been decoded incorrectly, and also return a
    "plan" indicating all the steps required to fix it.

    To fix similar text in the same way, without having to detect anything,
    you can use the ``apply_plan`` function.
    """
    best_version = text
    best_cost = text_cost(text)
    best_plan = []
    plan_so_far = []
    while True:
        prevtext = text
        text, plan = fix_one_step_and_explain(text)
        plan_so_far.extend(plan)
        cost = text_cost(text)

        # Add a penalty if we used a particularly obsolete encoding. The result
        # is that we won't use these encodings unless they can successfully
        # replace multiple characters.
        if ('encode', 'macroman') in plan_so_far or\
           ('encode', 'cp437') in plan_so_far:
            cost += 2

        # We need pretty solid evidence to decode from Windows-1251 (Cyrillic).
        if ('encode', 'sloppy-windows-1251') in plan_so_far:
            cost += 5

        if cost < best_cost:
            best_cost = cost
            best_version = text
            best_plan = list(plan_so_far)
        if text == prevtext:
            return best_version, best_plan

def fix_one_step_and_explain(text):
    """
    Performs a single step of re-decoding text that's been decoded incorrectly.

    Returns the decoded text, plus a "plan" for how to reproduce what it
    did.
    """
    if isinstance(text, bytes):
        raise UnicodeError(BYTES_ERROR_TEXT)
    if len(text) == 0:
        return text, []

    # The first plan is to return ASCII text unchanged.
    if possible_encoding(text, 'ascii'):
        return text, []

    # As we go through the next step, remember the possible encodings
    # that we encounter but don't successfully fix yet. We may need them
    # later.
    possible_1byte_encodings = []

    # Suppose the text was supposed to be UTF-8, but it was decoded using
    # a single-byte encoding instead. When these cases can be fixed, they
    # are usually the correct thing to do, so try them next.
    for encoding in CHARMAP_ENCODINGS:
        if possible_encoding(text, encoding):
            encoded_bytes = text.encode(encoding)

            # Now, find out if it's UTF-8 (or close enough). Otherwise,
            # remember the encoding for later.
            try:
                decoding = 'utf-8'
                if b'\xed' in encoded_bytes or b'\xc0' in encoded_bytes:
                    decoding = 'utf-8-variants'
                fixed = encoded_bytes.decode(decoding)
                steps = [('encode', encoding), ('decode', decoding)]
                return fixed, steps
            except UnicodeDecodeError:
                possible_1byte_encodings.append(encoding)

    # The next most likely case is that this is Latin-1 that was intended to
    # be read as Windows-1252, because those two encodings in particular are
    # easily confused.
    if 'latin-1' in possible_1byte_encodings:
        if 'windows-1252' in possible_1byte_encodings:
            # This text is in the intersection of Latin-1 and
            # Windows-1252, so it's probably legit.
            return text, []
        else:
            # Otherwise, it means we have characters that are in Latin-1 but
            # not in Windows-1252. Those are C1 control characters. Nobody
            # wants those. Assume they were meant to be Windows-1252. Don't
            # use the sloppy codec, because bad Windows-1252 characters are
            # a bad sign.
            encoded = text.encode('latin-1')
            try:
                fixed = encoded.decode('windows-1252')
                steps = []
                if fixed != text:
                    steps = [('encode', 'latin-1'), ('decode', 'windows-1252')]
                return fixed, steps
            except UnicodeDecodeError:
                # This text contained characters that don't even make sense
                # if you assume they were supposed to be Windows-1252. In
                # that case, let's not assume anything.
                pass

    # The cases that remain are mixups between two different single-byte
    # encodings, and not the common case of Latin-1 vs. Windows-1252.
    #
    # Those cases are somewhat rare, and impossible to solve without false
    # positives. If you're in one of these situations, you should try using
    # the `ftfy.guess_bytes` function.

    # Return the text unchanged; the plan is empty.
    return text, []

def apply_plan(text, plan):
    """
    Apply a plan for fixing the encoding of text.

    The plan is a list of tuples of the form (operation, encoding), where
    `operation` is either 'encode' or 'decode', and `encoding` is an encoding
    name such as 'utf-8' or 'latin-1'.

    Because only text can be encoded, and only bytes can be decoded, the plan
    should alternate 'encode' and 'decode' steps, or else this function will
    encounter an error.
    """
    obj = text
    for operation, encoding in plan:
        if operation == 'encode':
            obj = obj.encode(encoding)
        elif operation == 'decode':
            obj = obj.decode(encoding)
        else:
            raise ValueError("Unknown plan step: %s" % operation)

    return obj

HTML_ENTITY_RE = re.compile(r"&#?\w{0,8};")

def unescape_html(text):
    """
    Decode all three types of HTML entities/character references.

    Code by Fredrik Lundh of effbot.org. Robyn Speer made a slight change
    to it for efficiency: it won't match entities longer than 8 characters,
    because there are no valid entities like that.

        >>> print(unescape_html('<tag>'))
        <tag>
    """
    def fixup(match):
        """
        Replace one matched HTML entity with the character it represents,
        if possible.
        """
        text = match.group(0)
        if text[:2] == "&#":
            # character reference
            try:
                if text[:3] == "&#x":
                    return unichr(int(text[3:-1], 16))
                else:
                    return unichr(int(text[2:-1]))
            except ValueError:
                pass
        else:
            # named entity
            try:
                text = unichr(htmlentitydefs.name2codepoint[text[1:-1]])
            except KeyError:
                pass
        return text  # leave as is
    return HTML_ENTITY_RE.sub(fixup, text)

ANSI_RE = re.compile('\033\\[((?:\\d|;)*)([a-zA-Z])')

def remove_terminal_escapes(text):
    r"""
    Strip out "ANSI" terminal escape sequences, such as those that produce
    colored text on Unix.

        >>> print(remove_terminal_escapes(
        ...     "\033[36;44mI'm blue, da ba dee da ba doo...\033[0m"
        ... ))
        I'm blue, da ba dee da ba doo...
    """
    return ANSI_RE.sub('', text)

SINGLE_QUOTE_RE = re.compile('[\u2018-\u201b]')
DOUBLE_QUOTE_RE = re.compile('[\u201c-\u201f]')

def uncurl_quotes(text):
    r"""
    Replace curly quotation marks with straight equivalents.

        >>> print(uncurl_quotes('\u201chere\u2019s a test\u201d'))
        "here's a test"
    """
    return SINGLE_QUOTE_RE.sub("'", DOUBLE_QUOTE_RE.sub('"', text))

def fix_line_breaks(text):
    r"""
    Convert all line breaks to Unix style.

    This will convert the following sequences into the standard \\n
    line break:

        - CRLF (\\r\\n), used on Windows and in some communication
          protocols
        - CR (\\r), once used on Mac OS Classic, and now kept alive
          by misguided software such as Microsoft Office for Mac
        - LINE SEPARATOR (\\u2028) and PARAGRAPH SEPARATOR (\\u2029),
          defined by Unicode and used to sow confusion and discord
        - NEXT LINE (\\x85), a C1 control character that is certainly
          not what you meant

        The NEXT LINE character is a bit of an odd case, because it
        usually won't show up if `fix_encoding` is also being run.
        \\x85 is very common mojibake for \\u2026, HORIZONTAL ELLIPSIS.

        >>> fix_line_breaks("Content-type: text/plain\r\n\r\nHi.")
        'Content-type: text/plain\n\nHi.'

        >>> fix_line_breaks("This is how Microsoft \r trolls Mac users")
        'This is how Microsoft \n trolls Mac users'

        >>> fix_line_breaks("What is this \x85 I don't even")
        "What is this \n I don't even"

        >>> print(fix_line_breaks(
        ...     "This string is made of two things:\u2029"
        ...     "1. Unicode\u2028"
        ...     "2. Spite"
        ... ))
        This string is made of two things:
        1. Unicode
        2. Spite
    """
    return text.replace('\r\n', '\n').replace('\r', '\n')\
               .replace('\u2028', '\n').replace('\u2029', '\n')\
               .replace('\u0085', '\n')

def remove_control_chars(text):
    """
    Remove all control characters except for the important ones.

    This removes characters in these ranges:

    - U+0000 to U+0008
    - U+000B
    - U+000E to U+001F
    - U+007F

    It leaves alone these characters that are commonly used for formatting:

    - TAB (U+0009)
    - LF (U+000A)
    - FF (U+000C)
    - CR (U+000D)
    """
    return text.translate(CONTROL_CHARS)

def remove_bom(text):
    r"""
    Remove a left-over byte-order mark.

    >>> print(remove_bom(unichr(0xfeff) + "Where do you want to go today?"))
    Where do you want to go today?
    """
    return text.lstrip(unichr(0xfeff))

def remove_unsafe_private_use(text):
    r"""
    Python 3.3's Unicode support isn't perfect, and in fact there are certain
    string operations that will crash some versions of it with a SystemError:
    http://bugs.python.org/issue18183

    You can trigger the bug by running `` '\U00010000\U00100000'.lower() ``.

    The best solution is to remove all characters from Supplementary Private
    Use Area B, using a regex that is known not to crash given those
    characters.

    These are the characters from U+100000 to U+10FFFF. It's sad to lose an
    entire plane of Unicode, but on the other hand, these characters are not
    assigned and never will be. If you get one of these characters and don't
    know what its purpose is, its purpose is probably to crash your code.

    If you were using these for actual private use, this might be inconvenient.
    You can turn off this fixer, of course, but I kind of encourage using
    Supplementary Private Use Area A instead.

        >>> print(remove_unsafe_private_use('\U0001F4A9\U00100000'))
        💩

    This fixer is off by default in Python 3.4 or later. (The bug is actually
    fixed in 3.3.3 and 2.7.6, but I don't want the default behavior to change
    based on a micro version upgrade of Python.)
    """
    return UNSAFE_PRIVATE_USE_RE.sub('', text)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# OpenCraft -- tools to aid developing and hosting free software projects
# Copyright (C) 2015 OpenCraft <xavier@opencraft.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
"""
GitHub Service API - Helper functions
"""

# Imports #####################################################################

import re
import requests

from django.conf import settings

# Constants ###################################################################

GH_HEADERS = {
    'Authorization': 'token {}'.format(settings.GITHUB_ACCESS_TOKEN),
    'Time-Zone': 'UTC',
}

# Functions ###################################################################

def fork_name2tuple(fork_name):
    """
    Converts a `fork_name` (eg. `'open-craft/edx-platform'`)
    to a `fork_tuple` (eg. `['open-craft', 'edx-platform']`)
    """
    return fork_name.split('/')

def get_commit_id_from_ref(fork_name, ref_name, ref_type='heads'):
    """
    Get the `commit_id` currently attached to a git reference
    """
    url = 'https://api.github.com/repos/{fork_name}/git/refs/{ref_type}/{ref_name}'.format(
        fork_name=fork_name,
        ref_type=ref_type,
        ref_name=ref_name,
    )
    r = requests.get(url, headers=GH_HEADERS)
    r.raise_for_status()
    return r.json()['object']['sha']

def get_settings_from_pr_body(pr_body):
    """
    Extract a settings string from a PR description body
    """
    m = re.search("..Settings..\r?\n+```[a-z]*\r?\n((?:.+\r?\n)*)```", pr_body)
    if m:
        return m.groups()[0]
    else:
        return ''

def get_pr_by_number(fork_name, pr_number):
    """
    Returns a PR() namedtuple based on the reponse
    """
    url = 'https://api.github.com/repos/{fork_name}/pulls/{pr_number}'.format(
        fork_name=fork_name,
        pr_number=pr_number,
    )
    r = requests.get(url, headers=GH_HEADERS)
    r.raise_for_status()

    r_pr = r.json()
    pr_fork_name = r_pr['head']['repo']['full_name']
    pr_branch_name = r_pr['head']['ref']
    pr = PR(
        pr_number,
        pr_fork_name,
        pr_branch_name,
        r_pr['title'],
        r_pr['user']['login'],
        body=r_pr['body'],
    )
    return pr

def get_pr_list_from_username(user_name, fork_name):
    """
    Retreive the current active PRs for a given user
    """
    q = 'is:open is:pr author:{author} repo:{repo}'.format(author=user_name, repo=fork_name)
    url = 'https://api.github.com/search/issues?sort=created&q={}'.format(q)
    r = requests.get(url, headers=GH_HEADERS)

    pr_list = []
    for pr_dict in r.json()['items']:
        pr = get_pr_by_number(fork_name, pr_dict['number'])
        pr_list.append(pr)
    return pr_list

def get_team_from_organization(organization_name, team_name='Owners'):
    """
    Retreive a team by organization & team name
    """
    url = 'https://api.github.com/orgs/{org}/teams'.format(org=organization_name)
    r = requests.get(url, headers=GH_HEADERS)
    r.raise_for_status()

    for team_dict in r.json():
        if team_dict['name'] == team_name:
            return team_dict
    raise KeyError(team_name)

def get_username_list_from_team(organization_name, team_name='Owners'):
    """
    Retreive the usernames of a given team's members
    """
    team = get_team_from_organization(organization_name, team_name)
    url = 'https://api.github.com/teams/{team_id}/members'.format(team_id=team['id'])
    r = requests.get(url, headers=GH_HEADERS)
    r.raise_for_status()
    return [user_dict['login'] for user_dict in r.json()]

# Classes #####################################################################

class PR:
    """
    Representation of a GitHub Pull Request
    """
    def __init__(self, number, fork_name, branch_name, title, username, body=''):
        self.number = number
        self.fork_name = fork_name
        self.branch_name = branch_name
        self.title = title
        self.username = username
        self.body = body

    @property
    def extra_settings(self):
        """
        Extra settings contained in the PR body
        """
        return get_settings_from_pr_body(self.body)
<commit_msg>Add logging to list of PRs received from github API call<commit_after># -*- coding: utf-8 -*-
#
# OpenCraft -- tools to aid developing and hosting free software projects
# Copyright (C) 2015 OpenCraft <xavier@opencraft.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
"""
GitHub Service API - Helper functions
"""

# Imports #####################################################################

import re
import requests

from django.conf import settings

# Logging #####################################################################

import logging
logger = logging.getLogger(__name__)

# Constants ###################################################################

GH_HEADERS = {
    'Authorization': 'token {}'.format(settings.GITHUB_ACCESS_TOKEN),
    'Time-Zone': 'UTC',
}

# Functions ###################################################################

def fork_name2tuple(fork_name):
    """
    Converts a `fork_name` (eg. `'open-craft/edx-platform'`)
    to a `fork_tuple` (eg. `['open-craft', 'edx-platform']`)
    """
    return fork_name.split('/')

def get_commit_id_from_ref(fork_name, ref_name, ref_type='heads'):
    """
    Get the `commit_id` currently attached to a git reference
    """
    url = 'https://api.github.com/repos/{fork_name}/git/refs/{ref_type}/{ref_name}'.format(
        fork_name=fork_name,
        ref_type=ref_type,
        ref_name=ref_name,
    )
    r = requests.get(url, headers=GH_HEADERS)
    r.raise_for_status()
    return r.json()['object']['sha']

def get_settings_from_pr_body(pr_body):
    """
    Extract a settings string from a PR description body
    """
    m = re.search("..Settings..\r?\n+```[a-z]*\r?\n((?:.+\r?\n)*)```", pr_body)
    if m:
        return m.groups()[0]
    else:
        return ''

def get_pr_by_number(fork_name, pr_number):
    """
    Returns a PR() namedtuple based on the reponse
    """
    url = 'https://api.github.com/repos/{fork_name}/pulls/{pr_number}'.format(
        fork_name=fork_name,
        pr_number=pr_number,
    )
    r = requests.get(url, headers=GH_HEADERS)
    r.raise_for_status()

    r_pr = r.json()
    pr_fork_name = r_pr['head']['repo']['full_name']
    pr_branch_name = r_pr['head']['ref']
    pr = PR(
        pr_number,
        pr_fork_name,
        pr_branch_name,
        r_pr['title'],
        r_pr['user']['login'],
        body=r_pr['body'],
    )
    return pr

def get_pr_list_from_username(user_name, fork_name):
    """
    Retreive the current active PRs for a given user
    """
    q = 'is:open is:pr author:{author} repo:{repo}'.format(author=user_name, repo=fork_name)
    url = 'https://api.github.com/search/issues?sort=created&q={}'.format(q)
    r = requests.get(url, headers=GH_HEADERS)
    r.raise_for_status()

    r_pr_list = r.json()
    logger.debug('List of PRs received for user %s: %s', user_name, r_pr_list)

    pr_list = []
    for pr_dict in r_pr_list['items']:
        pr = get_pr_by_number(fork_name, pr_dict['number'])
        pr_list.append(pr)
    return pr_list

def get_team_from_organization(organization_name, team_name='Owners'):
    """
    Retreive a team by organization & team name
    """
    url = 'https://api.github.com/orgs/{org}/teams'.format(org=organization_name)
    r = requests.get(url, headers=GH_HEADERS)
    r.raise_for_status()

    for team_dict in r.json():
        if team_dict['name'] == team_name:
            return team_dict
    raise KeyError(team_name)

def get_username_list_from_team(organization_name, team_name='Owners'):
    """
    Retreive the usernames of a given team's members
    """
    team = get_team_from_organization(organization_name, team_name)
    url = 'https://api.github.com/teams/{team_id}/members'.format(team_id=team['id'])
    r = requests.get(url, headers=GH_HEADERS)
    r.raise_for_status()
    return [user_dict['login'] for user_dict in r.json()]

# Classes #####################################################################

class PR:
    """
    Representation of a GitHub Pull Request
    """
    def __init__(self, number, fork_name, branch_name, title, username, body=''):
        self.number = number
        self.fork_name = fork_name
        self.branch_name = branch_name
        self.title = title
        self.username = username
        self.body = body

    @property
    def extra_settings(self):
        """
        Extra settings contained in the PR body
        """
        return get_settings_from_pr_body(self.body)
<|endoftext|>


<commit_before>
import subprocess as sp
import os
import shlex

class TestPrerequisites():

    def test_experiments_exist(self):
        """
        Test that the payu experiment repo has been downloaded.

        To get this test to pass:

        git clone https://github.com/nicholash/payu-experiments.git
        """

        assert(os.path.exists('./payu-experiments/.git'))

    def test_payu_exists(self):
        """
        Test that there is a payu installation.

        To get this test to pass:
        git clone https://github.com/marshallward/payu.git
        Then follow installation instructions.
        """

        cmd = 'which payu'
        ret = sp.call(shlex.split(cmd))
        assert(ret == 0)

    def test_ocean_ice_inputs(self):
        """
        Test that all the necessary ocean and ice inputs exist.

        To get this test to pass:

        Type payu init --laboratory ./lab --model access
        Then populate lab/input with the correct model inputs.
        You may find these at /short/v45/nah599/access/input
        """

        inputs = ['cice', 'core2_nyf_matm', 'mom_om_1440x1080',
                  'cice_cm_1440x1080', 'mom', 'oasis_cm', 'oasis_om',
                  'cice_om_1440x1080', 'mom_cm_1440x1080',
                  'oasis_cm_1440x1080', 'oasis_om_1440x1080']

        for i in inputs:
            assert(os.path.exists(os.path.join('lab/input', i)))

    def test_atm_inputs(self):
        """
        Test that the necessary atm inputs exists.

        To get this test to pass:

        See previous test.
        """

        assert(os.path.exists('lab/input/um'))

    def test_um_executable_exists(self):
        """
        Since payu cannot build the UM, check that executable
        is here.

        To get this test to pass:
        Talk to Hailin.Yan@csiro.au about how to get a UM executable for
        ACCESS-CM.
        """

        assert(os.path.exists('lab/bin/um7.3x'))

<commit_msg>Whitespace and UM exe name.<commit_after>
import subprocess as sp
import os
import shlex

class TestPrerequisites():

    def test_experiments_exist(self):
        """
        Test that the payu experiment repo has been downloaded.

        To get this test to pass:

        git clone https://github.com/nicholash/payu-experiments.git
        """

        assert(os.path.exists('./payu-experiments/.git'))

    def test_payu_exists(self):
        """
        Test that there is a payu installation.

        To get this test to pass.
        git clone https://github.com/marshallward/payu.git
        Then follow installation instructions.
        """

        cmd = 'which payu'
        ret = sp.call(shlex.split(cmd))
        assert(ret == 0)

    def test_ocean_ice_inputs(self):
        """
        Test that all the necessary ocean and ice inputs exist.

        To get this test to pass:

        Type payu init --laboratory ./lab --model access
        Then populate lab/input with the correct model inputs.
        You may find these at /short/v45/nah599/access/input
        """

        inputs = ['cice', 'core2_nyf_matm', 'mom_om_1440x1080',
                  'cice_cm_1440x1080', 'mom', 'oasis_cm', 'oasis_om',
                  'cice_om_1440x1080', 'mom_cm_1440x1080',
                  'oasis_cm_1440x1080', 'oasis_om_1440x1080']

        for i in inputs:
            assert(os.path.exists(os.path.join('lab/input', i)))

    def test_atm_inputs(self):
        """
        Test that the necessary atm inputs exists.

        To get this test to pass:

        See previous test.
        """

        assert(os.path.exists('lab/input/um'))

    def test_um_executable_exists(self):
        """
        Since payu cannot build the UM, check that executable
        is here.

        To get this test to pass:
        Talk to Hailin.Yan@csiro.au about how to get a UM executable for
        ACCESS-CM.
        """

        assert(os.path.exists('lab/bin/um7.3.exe'))

<|endoftext|>


<commit_before>__version__ = '2.1.11'
<commit_msg>Fixed too many open files bug.<commit_after>__version__ = '2.1.12'
<|endoftext|>


<commit_before># Copyright 2016 Matthias Gazzari
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

'''Module to retrieve coordinates of a given city'''

import csv
import os, os.path
import math
from pkg_resources import resource_filename
from whoosh.fields import Schema, STORED, ID, IDLIST
from whoosh import index
from whoosh.qparser import MultifieldParser
from whoosh.query import Term
from . import countries

_ix = None
_parser = None

def init(procs, multisegment, ixPath, force=False):
	'''
	Initialise the index and global variables

	Create the index if it does not exist or force is set to True and
	load it along with the parser required for the coordinate search.

	:param procs: maximum number of processes for index creation
	:param multisegment: split index into at least #procs segments
	:param ixPath: path to the index
	:param force: recreate the index
	'''
	# create the geoindex if it does not exist or force is enabled
	if not os.path.exists(ixPath):
		os.mkdir(ixPath)
		force = True
	if force:
		print('creating the geoindex - this may take some time')
		schema = Schema(
			name=ID(stored=True),
			asci=ID,
			alias=IDLIST(expression=r"[^,]+"),
			lat=STORED,
			lon=STORED,
			alpha2=ID(stored=True),
			country=ID(stored=True)
		)
		ix = index.create_in(ixPath, schema)
		writer = ix.writer(procs=procs, multisegment=multisegment)
		cities = resource_filename(__name__, 'data/cities1000.txt')
		with open(cities) as f:
			reader = csv.reader(f, delimiter='\t', quoting=csv.QUOTE_NONE)
			for row in reader:
				population = int(row[14])
				boost = math.log(max(math.e, population))
				writer.add_document(
					name=row[1],
					asci=row[2],
					alias=row[3],
					lat=row[4],
					lon=row[5],
					alpha2=row[8],
					country=countries.get(alpha2=row[8]).name,
					_boost=boost,
				)
		writer.commit()

	# load the index and create the city/alpha2 query parser
	global _ix, _parser
	_ix = index.open_dir(ixPath)
	_parser = MultifieldParser(['name', 'asci', 'alias',], _ix.schema)

def geocode(city, alpha2, **ignore):
	'''
	Search for geographical coordinates of a given city name, optionally
	restricting search results to a specified country.

	:param city: the city name to search for
	:param alpha2: the country to restrict search results to
	:param ignore: ignored keyword arguments
	'''
	if not city:
		return
	text = 'name:"{key}" OR asci:"{key}" OR alias:({key})'.format(key=city)
	query = _parser.parse(text)
	filterTerm = Term('alpha2', alpha2) if alpha2 else None
	with _ix.searcher() as searcher:
		results = searcher.search(query, limit=None, filter=filterTerm)
		for hit in results:
			yield {
				'lat': float(hit['lat']),
				'lon': float(hit['lon']),
				'alpha2': hit['alpha2'],
				'country': hit['country'],
			}
<commit_msg>fix typo<commit_after># Copyright 2016 Matthias Gazzari
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

'''Module to retrieve coordinates of a given city'''

import csv
import os, os.path
import math
from pkg_resources import resource_filename
from whoosh.fields import Schema, STORED, ID, IDLIST
from whoosh import index
from whoosh.qparser import MultifieldParser
from whoosh.query import Term
from . import countries

_ix = None
_parser = None

def init(procs, multisegment, ixPath, force=False):
	'''
	Initialise the index and global variables

	Create the index if it does not exist or force is set to True and
	load it along with the parser required for the coordinate search.

	:param procs: maximum number of processes for the index creation
	:param multisegment: split index into at least #procs segments
	:param ixPath: path to the index
	:param force: recreate the index
	'''
	# create the geoindex if it does not exist or force is enabled
	if not os.path.exists(ixPath):
		os.mkdir(ixPath)
		force = True
	if force:
		print('creating the geoindex - this may take some time')
		schema = Schema(
			name=ID(stored=True),
			asci=ID,
			alias=IDLIST(expression=r"[^,]+"),
			lat=STORED,
			lon=STORED,
			alpha2=ID(stored=True),
			country=ID(stored=True)
		)
		ix = index.create_in(ixPath, schema)
		writer = ix.writer(procs=procs, multisegment=multisegment)
		cities = resource_filename(__name__, 'data/cities1000.txt')
		with open(cities) as f:
			reader = csv.reader(f, delimiter='\t', quoting=csv.QUOTE_NONE)
			for row in reader:
				population = int(row[14])
				boost = math.log(max(math.e, population))
				writer.add_document(
					name=row[1],
					asci=row[2],
					alias=row[3],
					lat=row[4],
					lon=row[5],
					alpha2=row[8],
					country=countries.get(alpha2=row[8]).name,
					_boost=boost,
				)
		writer.commit()

	# load the index and create the city/alpha2 query parser
	global _ix, _parser
	_ix = index.open_dir(ixPath)
	_parser = MultifieldParser(['name', 'asci', 'alias',], _ix.schema)

def geocode(city, alpha2, **ignore):
	'''
	Search for geographical coordinates of a given city name, optionally
	restricting search results to a specified country.

	:param city: the city name to search for
	:param alpha2: the country to restrict search results to
	:param ignore: ignored keyword arguments
	'''
	if not city:
		return
	text = 'name:"{key}" OR asci:"{key}" OR alias:({key})'.format(key=city)
	query = _parser.parse(text)
	filterTerm = Term('alpha2', alpha2) if alpha2 else None
	with _ix.searcher() as searcher:
		results = searcher.search(query, limit=None, filter=filterTerm)
		for hit in results:
			yield {
				'lat': float(hit['lat']),
				'lon': float(hit['lon']),
				'alpha2': hit['alpha2'],
				'country': hit['country'],
			}
<|endoftext|>