<commit_before>import numpy
import six

import chainer
from chainer import cuda
from chainer import function
from chainer.functions.activation import log_softmax
from chainer.utils import type_check
from chainer import variable

def _broadcast_to(array, shape):
    if hasattr(numpy, "broadcast_to"):
        return numpy.broadcast_to(array, shape)
    dummy = numpy.empty(shape, array.dtype)
    return numpy.broadcast_arrays(array, dummy)[0]

def _check_class_weight_option(class_weight):
    if class_weight is not None:
        if class_weight.ndim != 1:
            raise ValueError('class_weight.ndim should be 1')
        if class_weight.dtype.kind != 'f':
            raise ValueError('The dtype of class_weight should be \'f\'')
        if isinstance(class_weight, variable.Variable):
            raise ValueError('class_weight should be a numpy.ndarray or '
                             'cupy.ndarray, not a chainer.Variable')

def _check_reduce_option(reduce):
    if reduce not in ('mean', 'no'):
        raise ValueError(
            "only 'mean' and 'no' are valid for 'reduce', but '%s' is "
            'given' % reduce)

def _check_input_values(x, t, ignore_label):
    # Extract the raw ndarray as Variable.__ge__ is not implemented.
    # We assume that t is already an ndarray.
    if isinstance(x, variable.Variable):
        x = x.data

    if not (((0 <= t) &
             (t < x.shape[1])) |
            (t == ignore_label)).all():
        msg = ('Each label `t` need to satisfy '
               '`0 <= t < x.shape[1] or t == %d`' % ignore_label)
        raise ValueError(msg)

class SoftmaxCrossEntropy(function.Function):

    """Softmax activation followed by a cross entropy loss."""

    normalize = True

    def __init__(self, normalize=True, cache_score=True, class_weight=None,
                 ignore_label=-1, reduce='mean'):
        self.normalize = normalize
        self.cache_score = cache_score
        _check_class_weight_option(class_weight)
        self.class_weight = class_weight
        self.ignore_label = ignore_label
        _check_reduce_option(reduce)
        self.reduce = reduce

    def check_type_forward(self, in_types):
        type_check.expect(in_types.size() == 2)
        x_type, t_type = in_types

        type_check.expect(
            x_type.dtype.kind == 'f',
            t_type.dtype == numpy.int32,
            t_type.ndim == x_type.ndim - 1,

            x_type.shape[0] == t_type.shape[0],
            x_type.shape[2:] == t_type.shape[1:],
        )

    def forward_cpu(self, inputs):
        x, t = inputs
        if chainer.is_debug():
            _check_input_values(x, t, self.ignore_label)

        log_y = log_softmax._log_softmax(x)
        if self.cache_score:
            self.y = numpy.exp(log_y)
        if self.class_weight is not None:
            shape = [1 if d != 1 else -1 for d in six.moves.range(x.ndim)]
            log_y *= _broadcast_to(self.class_weight.reshape(shape), x.shape)
        log_yd = numpy.rollaxis(log_y, 1)
        log_yd = log_yd.reshape(len(log_yd), -1)
        log_p = log_yd[numpy.maximum(t.ravel(), 0), numpy.arange(t.size)]

        log_p *= (t.ravel() != self.ignore_label)
        if self.reduce == 'mean':
            # deal with the case where the SoftmaxCrossEntropy is
            # unpickled from the old version
            if self.normalize:
                count = (t != self.ignore_label).sum()
            else:
                count = len(x)
            self._coeff = 1.0 / max(count, 1)

            y = log_p.sum(keepdims=True) * (-self._coeff)
            return y.reshape(()),
        else:
            return -log_p.reshape(t.shape),

    def forward_gpu(self, inputs):
        cupy = cuda.cupy
        x, t = inputs
        if chainer.is_debug():
            _check_input_values(x, t, self.ignore_label)

        log_y = log_softmax._log_softmax(x)
        if self.cache_score:
            self.y = cupy.exp(log_y)
        if self.class_weight is not None:
            shape = [1 if d != 1 else -1 for d in six.moves.range(x.ndim)]
            log_y *= cupy.broadcast_to(
                self.class_weight.reshape(shape), x.shape)
        if self.normalize:
            coeff = cupy.maximum(1, (t != self.ignore_label).sum())
        else:
            coeff = max(1, len(t))
        self._coeff = cupy.divide(1.0, coeff, dtype=x.dtype)

        log_y = cupy.rollaxis(log_y, 1, log_y.ndim)
        if self.reduce == 'mean':
            ret = cuda.reduce(
                'S t, raw T log_y, int32 n_channel, raw T coeff, '
                'S ignore_label',
                'T out',
                't == ignore_label ? T(0) : log_y[_j * n_channel + t]',
                'a + b', 'out = a * -coeff[0]', '0', 'crossent_fwd'
            )(t, log_y.reduced_view(), log_y.shape[-1],
              self._coeff, self.ignore_label)
        else:
            ret = cuda.elementwise(
                'S t, raw T log_y, int32 n_channel, T ignore', 'T out',
                '''
                if (t == ignore) {
                  out = 0;
                } else {
                  out = -log_y[i * n_channel + t];
                }
                ''',
                'softmax_crossent_no_reduce_fwd'
            )(t, log_y.reduced_view(), log_y.shape[-1], self.ignore_label)
            ret = ret.reshape(t.shape)
        return ret,

    def backward_cpu(self, inputs, grad_outputs):
        x, t = inputs
        gloss = grad_outputs[0]
        if hasattr(self, 'y'):
            y = self.y.copy()
        else:
            y = log_softmax._log_softmax(x)
            numpy.exp(y, out=y)
        if y.ndim == 2:
            gx = y
            gx[numpy.arange(len(t)), numpy.maximum(t, 0)] -= 1
            if self.class_weight is not None:
                shape = [1 if d != 1 else -1 for d in six.moves.range(x.ndim)]
                c = _broadcast_to(self.class_weight.reshape(shape), x.shape)
                c = c[numpy.arange(len(t)), numpy.maximum(t, 0)]
                gx *= _broadcast_to(numpy.expand_dims(c, 1), gx.shape)
            gx *= (t != self.ignore_label).reshape((len(t), 1))
        else:
            # in the case where y.ndim is higher than 2,
            # we think that a current implementation is inefficient
            # because it yields two provisional arrays for indexing.
            n_unit = t.size // len(t)
            gx = y.reshape(y.shape[0], y.shape[1], -1)
            fst_index = numpy.arange(t.size) // n_unit
            trd_index = numpy.arange(t.size) % n_unit
            gx[fst_index, numpy.maximum(t.ravel(), 0), trd_index] -= 1
            if self.class_weight is not None:
                shape = [1 if d != 1 else -1 for d in six.moves.range(x.ndim)]
                c = _broadcast_to(self.class_weight.reshape(shape), x.shape)
                c = c.reshape(gx.shape)
                c = c[fst_index, numpy.maximum(t.ravel(), 0), trd_index]
                c = c.reshape(y.shape[0], 1, -1)
                gx *= _broadcast_to(c, gx.shape)
            gx *= (t != self.ignore_label).reshape((len(t), 1, -1))
            gx = gx.reshape(y.shape)
        if self.reduce == 'mean':
            gx *= gloss * self._coeff
        else:
            gx *= gloss[:, None]
        return gx, None

    def backward_gpu(self, inputs, grad_outputs):
        cupy = cuda.cupy
        x, t = inputs
        if hasattr(self, 'y'):
            y = self.y
        else:
            y = log_softmax._log_softmax(x)
            cupy.exp(y, out=y)
        gloss = grad_outputs[0]
        n_unit = t.size // len(t)
        if self.reduce == 'mean':
            coeff = gloss * self._coeff
        else:
            coeff = gloss[:, None, ...]

        if self.class_weight is None:
            gx = cuda.elementwise(
                'T y, S t, T coeff, S n_channel, S n_unit, S ignore_label',
                'T gx',
                '''
                    const int c = (i / n_unit % n_channel);
                    gx = t == ignore_label ? 0 : coeff * (y - (c == t));
                ''',
                'softmax_crossent_bwd')(
                    y, cupy.expand_dims(t, 1), coeff, x.shape[1],
                    n_unit, self.ignore_label)
        else:
            gx = cuda.elementwise(
                'T y, raw T w, S t, T coeff, S n_channel, S n_unit, '
                'S ignore_label',
                'T gx',
                '''
                    const int c = (i / n_unit % n_channel);
                    gx = t == ignore_label ? 0 : coeff * (y - (c == t)) * w[t];
                ''',
                'softmax_crossent_weight_bwd')(
                    y, self.class_weight, cupy.expand_dims(t, 1), coeff,
                    x.shape[1], n_unit, self.ignore_label)

        return gx, None

def _double_backward_softmax_cross_entropy(x, t, normalize, class_weight,
                                           ignore_label, reduce):
    if isinstance(t, variable.Variable):
        t = t.data

    _check_class_weight_option(class_weight)
    _check_reduce_option(reduce)
    if chainer.is_debug():
        _check_input_values(x, t, ignore_label)

    loss = -chainer.functions.log_softmax(x)

    if class_weight is not None:
        shape = [1 if d != 1 else -1 for d in six.moves.range(x.ndim)]
        class_weight = chainer.functions.broadcast_to(
            class_weight.reshape(shape), x.shape)
        loss = loss * class_weight

    in_use = (t != ignore_label).astype(x.dtype)

    loss = chainer.functions.rollaxis(loss, 1, loss.ndim)
    loss = chainer.functions.reshape(loss, (-1, loss.shape[-1]))

    # As F.select_item does not allow the elements of t to be negative,
    # we substitute ignore_label with 0, which is valid.
    t = t.copy()
    t[t == ignore_label] = 0

    loss = chainer.functions.select_item(loss, t.ravel())
    loss = chainer.functions.reshape(loss, t.shape)

    loss = loss * in_use

    if reduce == 'mean':
        if normalize:
            count = in_use.sum()
        else:
            count = len(x)
        count = max(count, 1.)
        loss = loss / count
        return chainer.functions.sum(loss)
    else:
        return loss

def softmax_cross_entropy(
        x, t, normalize=True, cache_score=True, class_weight=None,
        ignore_label=-1, reduce='mean', enable_double_backprop=True):
    """Computes cross entropy loss for pre-softmax activations.

    Args:
        x (~chainer.Variable): Variable holding a multidimensional array whose
            element indicates unnormalized log probability: the first axis of
            the variable represents the number of samples, and the second axis
            represents the number of classes. While this function computes
            a usual softmax cross entropy if the number of dimensions is equal
            to 2, it computes a cross entropy of the replicated softmax if the
            number of dimensions is greater than 2.
        t (~chainer.Variable): Variable holding an int32 vector of ground truth
            labels. If ``t[i] == ignore_label``, corresponding ``x[i]`` is
            ignored.
        normalize (bool): If ``True``, this function normalizes the cross
            entropy loss across all instances. If ``False``, it only
            normalizes along a batch size.
        cache_score (bool): When it is ``True``, the function stores result
            of forward computation to use it on backward computation. It
            reduces computational cost though consumes more memory.
            If ``enable_double_backprop`` option is ``True``, this option
            is forcibly turned off and the function does not cache
            the intermediate value.
        class_weight (~numpy.ndarray or ~chainer.cuda.cupy.ndarray): An array
            that contains constant weights that will be multiplied with the
            loss values along with the second dimension. The shape of this
            array should be ``(x.shape[1],)``. If this is not ``None``, each
            class weight ``class_weight[i]`` is actually multiplied to
            ``y[:, i]`` that is the corresponding log-softmax output of ``x``
            and has the same shape as ``x`` before calculating the actual loss
            value.
        ignore_label (int): Label value you want to ignore. Its default value
            is ``-1``. See description of the argument `t`.
        reduce (str): A string that determines whether to reduce the loss
            values. If it is ``'mean'``, it computes the sum of the individual
            cross entropy and normalize it according to ``normalize`` option.
            If it is ``'no'``, this function computes cross entropy for each
            instance and does not normalize it (``normalize`` option is
            ignored). In this case, the loss value of the ignored instance,
            which has ``ignore_label`` as its target value, is set to ``0``.
        enable_double_backprop (bool): If ``True``, this function uses
            implementation that supports higher order differentiation.
            Otherwise, it uses single-backprop implementation.
            We expect the former is slower. So, if you need faster computation
            and does not need second or higher derivatives, you can turn off
            the option.

    Returns:
        Variable: A variable holding a scalar array of the cross entropy loss.
        If ``reduce`` is ``'mean'``, it is a scalar array.
        If ``reduce`` is ``'no'``, the shape is same as that of ``x``.

    .. note::

       This function is differentiable only by ``x``.

    """

    if enable_double_backprop:
        return _double_backward_softmax_cross_entropy(
            x, t, normalize, class_weight, ignore_label, reduce)
    else:
        return SoftmaxCrossEntropy(
            normalize, cache_score, class_weight, ignore_label, reduce)(x, t)
<commit_msg>Turn off enable_double_backprop by default<commit_after>import numpy
import six

import chainer
from chainer import cuda
from chainer import function
from chainer.functions.activation import log_softmax
from chainer.utils import type_check
from chainer import variable

def _broadcast_to(array, shape):
    if hasattr(numpy, "broadcast_to"):
        return numpy.broadcast_to(array, shape)
    dummy = numpy.empty(shape, array.dtype)
    return numpy.broadcast_arrays(array, dummy)[0]

def _check_class_weight_option(class_weight):
    if class_weight is not None:
        if class_weight.ndim != 1:
            raise ValueError('class_weight.ndim should be 1')
        if class_weight.dtype.kind != 'f':
            raise ValueError('The dtype of class_weight should be \'f\'')
        if isinstance(class_weight, variable.Variable):
            raise ValueError('class_weight should be a numpy.ndarray or '
                             'cupy.ndarray, not a chainer.Variable')

def _check_reduce_option(reduce):
    if reduce not in ('mean', 'no'):
        raise ValueError(
            "only 'mean' and 'no' are valid for 'reduce', but '%s' is "
            'given' % reduce)

def _check_input_values(x, t, ignore_label):
    # Extract the raw ndarray as Variable.__ge__ is not implemented.
    # We assume that t is already an ndarray.
    if isinstance(x, variable.Variable):
        x = x.data

    if not (((0 <= t) &
             (t < x.shape[1])) |
            (t == ignore_label)).all():
        msg = ('Each label `t` need to satisfy '
               '`0 <= t < x.shape[1] or t == %d`' % ignore_label)
        raise ValueError(msg)

class SoftmaxCrossEntropy(function.Function):

    """Softmax activation followed by a cross entropy loss."""

    normalize = True

    def __init__(self, normalize=True, cache_score=True, class_weight=None,
                 ignore_label=-1, reduce='mean'):
        self.normalize = normalize
        self.cache_score = cache_score
        _check_class_weight_option(class_weight)
        self.class_weight = class_weight
        self.ignore_label = ignore_label
        _check_reduce_option(reduce)
        self.reduce = reduce

    def check_type_forward(self, in_types):
        type_check.expect(in_types.size() == 2)
        x_type, t_type = in_types

        type_check.expect(
            x_type.dtype.kind == 'f',
            t_type.dtype == numpy.int32,
            t_type.ndim == x_type.ndim - 1,

            x_type.shape[0] == t_type.shape[0],
            x_type.shape[2:] == t_type.shape[1:],
        )

    def forward_cpu(self, inputs):
        x, t = inputs
        if chainer.is_debug():
            _check_input_values(x, t, self.ignore_label)

        log_y = log_softmax._log_softmax(x)
        if self.cache_score:
            self.y = numpy.exp(log_y)
        if self.class_weight is not None:
            shape = [1 if d != 1 else -1 for d in six.moves.range(x.ndim)]
            log_y *= _broadcast_to(self.class_weight.reshape(shape), x.shape)
        log_yd = numpy.rollaxis(log_y, 1)
        log_yd = log_yd.reshape(len(log_yd), -1)
        log_p = log_yd[numpy.maximum(t.ravel(), 0), numpy.arange(t.size)]

        log_p *= (t.ravel() != self.ignore_label)
        if self.reduce == 'mean':
            # deal with the case where the SoftmaxCrossEntropy is
            # unpickled from the old version
            if self.normalize:
                count = (t != self.ignore_label).sum()
            else:
                count = len(x)
            self._coeff = 1.0 / max(count, 1)

            y = log_p.sum(keepdims=True) * (-self._coeff)
            return y.reshape(()),
        else:
            return -log_p.reshape(t.shape),

    def forward_gpu(self, inputs):
        cupy = cuda.cupy
        x, t = inputs
        if chainer.is_debug():
            _check_input_values(x, t, self.ignore_label)

        log_y = log_softmax._log_softmax(x)
        if self.cache_score:
            self.y = cupy.exp(log_y)
        if self.class_weight is not None:
            shape = [1 if d != 1 else -1 for d in six.moves.range(x.ndim)]
            log_y *= cupy.broadcast_to(
                self.class_weight.reshape(shape), x.shape)
        if self.normalize:
            coeff = cupy.maximum(1, (t != self.ignore_label).sum())
        else:
            coeff = max(1, len(t))
        self._coeff = cupy.divide(1.0, coeff, dtype=x.dtype)

        log_y = cupy.rollaxis(log_y, 1, log_y.ndim)
        if self.reduce == 'mean':
            ret = cuda.reduce(
                'S t, raw T log_y, int32 n_channel, raw T coeff, '
                'S ignore_label',
                'T out',
                't == ignore_label ? T(0) : log_y[_j * n_channel + t]',
                'a + b', 'out = a * -coeff[0]', '0', 'crossent_fwd'
            )(t, log_y.reduced_view(), log_y.shape[-1],
              self._coeff, self.ignore_label)
        else:
            ret = cuda.elementwise(
                'S t, raw T log_y, int32 n_channel, T ignore', 'T out',
                '''
                if (t == ignore) {
                  out = 0;
                } else {
                  out = -log_y[i * n_channel + t];
                }
                ''',
                'softmax_crossent_no_reduce_fwd'
            )(t, log_y.reduced_view(), log_y.shape[-1], self.ignore_label)
            ret = ret.reshape(t.shape)
        return ret,

    def backward_cpu(self, inputs, grad_outputs):
        x, t = inputs
        gloss = grad_outputs[0]
        if hasattr(self, 'y'):
            y = self.y.copy()
        else:
            y = log_softmax._log_softmax(x)
            numpy.exp(y, out=y)
        if y.ndim == 2:
            gx = y
            gx[numpy.arange(len(t)), numpy.maximum(t, 0)] -= 1
            if self.class_weight is not None:
                shape = [1 if d != 1 else -1 for d in six.moves.range(x.ndim)]
                c = _broadcast_to(self.class_weight.reshape(shape), x.shape)
                c = c[numpy.arange(len(t)), numpy.maximum(t, 0)]
                gx *= _broadcast_to(numpy.expand_dims(c, 1), gx.shape)
            gx *= (t != self.ignore_label).reshape((len(t), 1))
        else:
            # in the case where y.ndim is higher than 2,
            # we think that a current implementation is inefficient
            # because it yields two provisional arrays for indexing.
            n_unit = t.size // len(t)
            gx = y.reshape(y.shape[0], y.shape[1], -1)
            fst_index = numpy.arange(t.size) // n_unit
            trd_index = numpy.arange(t.size) % n_unit
            gx[fst_index, numpy.maximum(t.ravel(), 0), trd_index] -= 1
            if self.class_weight is not None:
                shape = [1 if d != 1 else -1 for d in six.moves.range(x.ndim)]
                c = _broadcast_to(self.class_weight.reshape(shape), x.shape)
                c = c.reshape(gx.shape)
                c = c[fst_index, numpy.maximum(t.ravel(), 0), trd_index]
                c = c.reshape(y.shape[0], 1, -1)
                gx *= _broadcast_to(c, gx.shape)
            gx *= (t != self.ignore_label).reshape((len(t), 1, -1))
            gx = gx.reshape(y.shape)
        if self.reduce == 'mean':
            gx *= gloss * self._coeff
        else:
            gx *= gloss[:, None]
        return gx, None

    def backward_gpu(self, inputs, grad_outputs):
        cupy = cuda.cupy
        x, t = inputs
        if hasattr(self, 'y'):
            y = self.y
        else:
            y = log_softmax._log_softmax(x)
            cupy.exp(y, out=y)
        gloss = grad_outputs[0]
        n_unit = t.size // len(t)
        if self.reduce == 'mean':
            coeff = gloss * self._coeff
        else:
            coeff = gloss[:, None, ...]

        if self.class_weight is None:
            gx = cuda.elementwise(
                'T y, S t, T coeff, S n_channel, S n_unit, S ignore_label',
                'T gx',
                '''
                    const int c = (i / n_unit % n_channel);
                    gx = t == ignore_label ? 0 : coeff * (y - (c == t));
                ''',
                'softmax_crossent_bwd')(
                    y, cupy.expand_dims(t, 1), coeff, x.shape[1],
                    n_unit, self.ignore_label)
        else:
            gx = cuda.elementwise(
                'T y, raw T w, S t, T coeff, S n_channel, S n_unit, '
                'S ignore_label',
                'T gx',
                '''
                    const int c = (i / n_unit % n_channel);
                    gx = t == ignore_label ? 0 : coeff * (y - (c == t)) * w[t];
                ''',
                'softmax_crossent_weight_bwd')(
                    y, self.class_weight, cupy.expand_dims(t, 1), coeff,
                    x.shape[1], n_unit, self.ignore_label)

        return gx, None

def _double_backward_softmax_cross_entropy(x, t, normalize, class_weight,
                                           ignore_label, reduce):
    if isinstance(t, variable.Variable):
        t = t.data

    _check_class_weight_option(class_weight)
    _check_reduce_option(reduce)
    if chainer.is_debug():
        _check_input_values(x, t, ignore_label)

    loss = -chainer.functions.log_softmax(x)

    if class_weight is not None:
        shape = [1 if d != 1 else -1 for d in six.moves.range(x.ndim)]
        class_weight = chainer.functions.broadcast_to(
            class_weight.reshape(shape), x.shape)
        loss = loss * class_weight

    in_use = (t != ignore_label).astype(x.dtype)

    loss = chainer.functions.rollaxis(loss, 1, loss.ndim)
    loss = chainer.functions.reshape(loss, (-1, loss.shape[-1]))

    # As F.select_item does not allow the elements of t to be negative,
    # we substitute ignore_label with 0, which is valid.
    t = t.copy()
    t[t == ignore_label] = 0

    loss = chainer.functions.select_item(loss, t.ravel())
    loss = chainer.functions.reshape(loss, t.shape)

    loss = loss * in_use

    if reduce == 'mean':
        if normalize:
            count = in_use.sum()
        else:
            count = len(x)
        count = max(count, 1.)
        loss = loss / count
        return chainer.functions.sum(loss)
    else:
        return loss

def softmax_cross_entropy(
        x, t, normalize=True, cache_score=True, class_weight=None,
        ignore_label=-1, reduce='mean', enable_double_backprop=False):
    """Computes cross entropy loss for pre-softmax activations.

    Args:
        x (~chainer.Variable): Variable holding a multidimensional array whose
            element indicates unnormalized log probability: the first axis of
            the variable represents the number of samples, and the second axis
            represents the number of classes. While this function computes
            a usual softmax cross entropy if the number of dimensions is equal
            to 2, it computes a cross entropy of the replicated softmax if the
            number of dimensions is greater than 2.
        t (~chainer.Variable): Variable holding an int32 vector of ground truth
            labels. If ``t[i] == ignore_label``, corresponding ``x[i]`` is
            ignored.
        normalize (bool): If ``True``, this function normalizes the cross
            entropy loss across all instances. If ``False``, it only
            normalizes along a batch size.
        cache_score (bool): When it is ``True``, the function stores result
            of forward computation to use it on backward computation. It
            reduces computational cost though consumes more memory.
            If ``enable_double_backprop`` option is ``True``, this option
            is forcibly turned off and the function does not cache
            the intermediate value.
        class_weight (~numpy.ndarray or ~chainer.cuda.cupy.ndarray): An array
            that contains constant weights that will be multiplied with the
            loss values along with the second dimension. The shape of this
            array should be ``(x.shape[1],)``. If this is not ``None``, each
            class weight ``class_weight[i]`` is actually multiplied to
            ``y[:, i]`` that is the corresponding log-softmax output of ``x``
            and has the same shape as ``x`` before calculating the actual loss
            value.
        ignore_label (int): Label value you want to ignore. Its default value
            is ``-1``. See description of the argument `t`.
        reduce (str): A string that determines whether to reduce the loss
            values. If it is ``'mean'``, it computes the sum of the individual
            cross entropy and normalize it according to ``normalize`` option.
            If it is ``'no'``, this function computes cross entropy for each
            instance and does not normalize it (``normalize`` option is
            ignored). In this case, the loss value of the ignored instance,
            which has ``ignore_label`` as its target value, is set to ``0``.
        enable_double_backprop (bool): If ``True``, this function uses
            implementation that supports higher order differentiation.
            If ``False``, it uses single-backprop implementation.
            This function use the single-backprop version because we expect
            it is faster. So, if you need second or higher derivatives,
            you need to turn it on explicitly.

    Returns:
        Variable: A variable holding a scalar array of the cross entropy loss.
        If ``reduce`` is ``'mean'``, it is a scalar array.
        If ``reduce`` is ``'no'``, the shape is same as that of ``x``.

    .. note::

       This function is differentiable only by ``x``.

    """

    if enable_double_backprop:
        return _double_backward_softmax_cross_entropy(
            x, t, normalize, class_weight, ignore_label, reduce)
    else:
        return SoftmaxCrossEntropy(
            normalize, cache_score, class_weight, ignore_label, reduce)(x, t)
<|endoftext|>


<commit_before># -------------------------------------------------------------------------------
# Name:        test_georef
# Purpose:     testing file for the wradlib.georef module
#
# Authors:     Kai Muehlbauer
#
# Created:     11.02.2015
# Copyright:   (c) Kai Muehlbauer
# Licence:     The MIT License
# -------------------------------------------------------------------------------
# !/usr/bin/env python

import unittest
import wradlib.georef as georef
import numpy as np
from osgeo import osr

class GetGridsTest(unittest.TestCase):
    def setUp(self):
        # calculate xy and lonlat grids with georef function
        self.radolan_grid_xy = georef.get_radolan_grid(900, 900, trig=True)
        self.radolan_grid_ll = georef.get_radolan_grid(900, 900, trig=True, wgs84=True)

    def test_get_radolan_grid_equality(self):
        # create radolan projection osr object
        scale = (1. + np.sin(np.radians(60.))) / (1. + np.sin(np.radians(90.)))
        dwd_string = ('+proj=stere +lat_0=90 +lat_ts=90 +lon_0=10 +k={0:10.8f} '
                      '+x_0=0 +y_0=0 +a=6370040 +b=6370040 +to_meter=1000 +no_defs').format(scale)
        proj_stereo = georef.proj4_to_osr(dwd_string)

        # create wgs84 projection osr object
        proj_wgs = osr.SpatialReference()
        proj_wgs.ImportFromEPSG(4326)

        # transform radolan polar stereographic projection to wgs84 and wgs84 to polar stereographic
        # using osr transformation routines
        radolan_grid_ll = georef.reproject(self.radolan_grid_xy, projection_source=proj_stereo,
                                           projection_target=proj_wgs)
        radolan_grid_xy = georef.reproject(self.radolan_grid_ll, projection_source=proj_wgs,
                                           projection_target=proj_stereo)

        # check source and target arrays for equality
        self.assertTrue(np.allclose(radolan_grid_ll, self.radolan_grid_ll))
        self.assertTrue(np.allclose(radolan_grid_xy, self.radolan_grid_xy))

    def test_get_radolan_grid_raises(self):
        self.assertRaises(TypeError, lambda: georef.get_radolan_grid('900', '900'))
        self.assertRaises(ValueError, lambda: georef.get_radolan_grid(2000, 2000))

    def test_get_radolan_grid_shape(self):
        radolan_grid_xy = georef.get_radolan_grid()
        self.assertEqual((900, 900, 2), radolan_grid_xy.shape)

if __name__ == '__main__':
    unittest.main()
<commit_msg>WIP: add test stubs in test_georef<commit_after># -------------------------------------------------------------------------------
# Name:        test_georef
# Purpose:     testing file for the wradlib.georef module
#
# Authors:     Kai Muehlbauer
#
# Created:     11.02.2015
# Copyright:   (c) Kai Muehlbauer
# Licence:     The MIT License
# -------------------------------------------------------------------------------
# !/usr/bin/env python

import unittest
import wradlib.georef as georef
import numpy as np
from osgeo import osr

class CoordinateTransformTest(unittest.TestCase):
    def test_hor2aeq(self):
        pass
    def test_aeq2hor(self):
        pass
    def test_polar2lonlat(self):
        pass
    def test___pol2lonlat(self):
        pass
    def test_polar2lonlatalt(self):
        pass
    def test__latscale(self):
        pass
    def test__lonscale(self):
        pass
    def test_polar2lonlatalt_n(self):
        pass

class GetGridsTest(unittest.TestCase):
    def setUp(self):
        # calculate xy and lonlat grids with georef function
        self.radolan_grid_xy = georef.get_radolan_grid(900, 900, trig=True)
        self.radolan_grid_ll = georef.get_radolan_grid(900, 900, trig=True, wgs84=True)

    def test_get_radolan_grid_equality(self):
        # create radolan projection osr object
        scale = (1. + np.sin(np.radians(60.))) / (1. + np.sin(np.radians(90.)))
        dwd_string = ('+proj=stere +lat_0=90 +lat_ts=90 +lon_0=10 +k={0:10.8f} '
                      '+x_0=0 +y_0=0 +a=6370040 +b=6370040 +to_meter=1000 +no_defs').format(scale)
        proj_stereo = georef.proj4_to_osr(dwd_string)

        # create wgs84 projection osr object
        proj_wgs = osr.SpatialReference()
        proj_wgs.ImportFromEPSG(4326)

        # transform radolan polar stereographic projection to wgs84 and wgs84 to polar stereographic
        # using osr transformation routines
        radolan_grid_ll = georef.reproject(self.radolan_grid_xy, projection_source=proj_stereo,
                                           projection_target=proj_wgs)
        radolan_grid_xy = georef.reproject(self.radolan_grid_ll, projection_source=proj_wgs,
                                           projection_target=proj_stereo)

        # check source and target arrays for equality
        self.assertTrue(np.allclose(radolan_grid_ll, self.radolan_grid_ll))
        self.assertTrue(np.allclose(radolan_grid_xy, self.radolan_grid_xy))

    def test_get_radolan_grid_raises(self):
        self.assertRaises(TypeError, lambda: georef.get_radolan_grid('900', '900'))
        self.assertRaises(ValueError, lambda: georef.get_radolan_grid(2000, 2000))

    def test_get_radolan_grid_shape(self):
        radolan_grid_xy = georef.get_radolan_grid()
        self.assertEqual((900, 900, 2), radolan_grid_xy.shape)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

'''
Generate gold profile
Latest version can be found at https://github.com/letuananh/intsem.fx

References:
    Python documentation:
        https://docs.python.org/
    argparse module:
        https://docs.python.org/3/howto/argparse.html
    PEP 257 - Python Docstring Conventions:
        https://www.python.org/dev/peps/pep-0257/

@author: Le Tuan Anh <tuananh.ke@gmail.com>
@license: MIT
'''

# Copyright (c) 2015, Le Tuan Anh <tuananh.ke@gmail.com>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

########################################################################

import os
import datetime
import gzip
import logging
from lxml import etree

from chirptext.leutile import FileHelper
from chirptext import texttaglib as ttl
from chirptext import chio
from lelesk import LeLeskWSD
from lelesk import LeskCache  # WSDResources

from coolisf.data import read_ccby30
from coolisf.dao import read_tsdb
from coolisf.model import Sentence
from coolisf.lexsem import tag_gold, Lexsem
from coolisf.mappings import PredSense
from coolisf.common import write_file
from coolisf.config import read_config

# -----------------------------------------------------------------------
# CONFIGURATION
# -----------------------------------------------------------------------

cfg = read_config()
if cfg and 'data_root' in cfg:
    DATA_DIR = FileHelper.abspath(cfg['data_root'])
else:
    DATA_DIR = FileHelper.abspath('./data')
GOLD_PATH = os.path.join(DATA_DIR, 'gold')
GOLD_TTL_PATH = os.path.join(DATA_DIR, 'gold')
OUTPUT_ISF = os.path.join(DATA_DIR, 'speckled_tsdb_imi.xml.gz')

def getLogger():
    return logging.getLogger(__name__)

# -----------------------------------------------------------------------

def match_sents(isf_doc, ttl_doc):
    ''' Match TSDB profile sentences with TTL sentences '''
    if len(isf_doc) != len(ttl_doc):
        print(len(isf_doc))
        print(len(ttl_doc))
        for sent in isf_doc:
            if not ttl_doc.get(sent.ident, default=None):
                getLogger().warning("Sentence ID(s) in ISF doc and TTL doc are different.")
                return None
        # it's possible to match all
        for sent in isf_doc:
            sent.shallow = ttl_doc.get(sent.ident)
        return isf_doc
    # else
    for isf_sent, ttl_sent in zip(isf_doc, ttl_doc):
        if isf_sent.text != ttl_sent.text:
            # try to match all available sentences
            getLogger().warning("Sentence(s) in ISF doc and TTL doc are different. E.g. ISF/#{}: {} vs TTL/#{}: {}".format(isf_sent.ident, isf_sent.text, ttl_sent.ID, ttl_sent.text))
            return None
    # only import when everything could be matched
    for isf_sent, ttl_sent in zip(isf_doc, ttl_doc):
        isf_sent.shallow = ttl_sent
    return isf_doc

def tag_doc(isf_doc, ttl_doc, use_ttl_sid=True, wsd_method=None, wsd=None, taggold=True, ctx=None, **kwargs):
    ''' Tag an ISF document using a TTL '''
    isf_doc = match_sents(isf_doc, ttl_doc)
    if isf_doc is None:
        raise Exception("isf_doc and ttl_doc could not be matched")
    not_matched = set()
    wsd = LeLeskWSD(dbcache=LeskCache())
    ctx = PredSense.wn.ctx()
    for sent in isf_doc:
        if use_ttl_sid and sent.shallow and sent.shallow.ID:
            sent.ident = sent.shallow.ID
        for reading in sent:
            if wsd_method:
                sent.tag(method=wsd_method, wsd=wsd, ctx=ctx)
            if taggold and sent.shallow:
                m, n, ignored = tag_gold(reading.dmrs(), sent.shallow, sent.text, **kwargs)
                # getLogger().debug("Matched: {}".format(m))
                if n:
                    not_matched.add(sent.ident)
                    sent.flag = Sentence.ERROR
            # update XML
            reading.dmrs().tag_xml(method=None, update_back=True, wsd=wsd, ctx=ctx)
        sent.tag_xml()
    if not_matched:
        getLogger().warning("TSDB/IMI mismatched sentences: {} - IDs={}".format(len(not_matched), not_matched))
    return isf_doc

def read_tsdb_ttl(tsdb_path, ttl_path=None, name=None, title=None, *args, **kwargs):
    ''' Combine TSDB profile and TTL profile to create ISF document (shallow + deep)
    This function return an instance of coolisf.model.Document
    '''
    tsdb_doc = read_tsdb(tsdb_path, name=name, title=title)
    if ttl_path is None:
        ttl_path = tsdb_path
    ttl_doc = ttl.Document.read_ttl(ttl_path)
    isf_doc = tag_doc(tsdb_doc, ttl_doc, *args, **kwargs)
    getLogger().debug("TTL doc {} contains {} sentence(s).".format(ttl_path, len(ttl_doc)))
    getLogger().debug("isf_doc size: {}".format(len(isf_doc)))
    return isf_doc

def read_gold_mrs():
    ''' Read gold MRS only (without TTL) '''
    return read_tsdb(GOLD_PATH, name="speckled", title="The Adventure of the Speckled Band")

def read_gold_sents(perform_wsd=False):
    ''' Read gold sentences (TSDB+TTL) '''
    return read_tsdb_ttl(GOLD_PATH, name="speckled", title="The Adventure of the Speckled Band")

def build_root_node(filename='spec-isf.xml'):
    isf_node = etree.Element('rootisf')
    isf_node.set('version', '0.1')
    isf_node.set('lang', 'eng')
    # Add license information
    header_node = etree.SubElement(isf_node, 'headerisf')
    filedesc_node = etree.SubElement(header_node, 'description')
    filedesc_node.set("title", "The Adventure of the Speckled Band")
    filedesc_node.set("author", "Arthur Conan Doyle")
    filedesc_node.set("filename", filename)
    filedesc_node.set("creationtime", datetime.datetime.now().isoformat())
    # License text
    license_node = etree.SubElement(filedesc_node, "license")
    license_node.text = etree.CDATA(read_ccby30())
    # CoolISF
    procs_node = etree.SubElement(header_node, "linguisticProcessors")
    proc_node = etree.SubElement(procs_node, "linguisticProcessor")
    proc_node.set("name", "coolisf")
    proc_node.set("description", "Python implementation of Integrated Semantic Framework")
    proc_node.set("version", "pre 0.1")
    proc_node.set("url", "https://github.com/letuananh/intsem.fx")
    proc_node.set("timestamp", datetime.datetime.now().isoformat())
    # ACE
    proc_node = etree.SubElement(procs_node, "linguisticProcessor")
    proc_node.set("name", "ACE")
    proc_node.set("description", "the Answer Constraint Engine (Delph-in)")
    proc_node.set("version", "0.9.17")
    proc_node.set("url", "http://moin.delph-in.net/AceTop")
    proc_node.set("timestamp", datetime.datetime.now().isoformat())
    # NLTK
    proc_node = etree.SubElement(procs_node, "linguisticProcessor")
    proc_node.set("name", "NLTK")
    proc_node.set("description", "Natural Language Toolkit for Python")
    proc_node.set("version", "3.0.4")
    proc_node.set("url", "http://www.nltk.org/")
    proc_node.set("timestamp", datetime.datetime.now().isoformat())
    # pyDelphin
    proc_node = etree.SubElement(procs_node, "linguisticProcessor")
    proc_node.set("name", "pyDelphin")
    proc_node.set("description", "Python libraries for DELPH-IN")
    proc_node.set("version", "0.3")
    proc_node.set("url", "https://github.com/delph-in/pydelphin")
    proc_node.set("timestamp", datetime.datetime.now().isoformat())
    # Contributors
    contributors_node = etree.SubElement(header_node, "contributors")
    contributor_node = etree.SubElement(contributors_node, "contributor")
    contributor_node.set("name", "Le Tuan Anh")
    contributor_node.set("email", "tuananh.ke@gmail.com")
    contributor_node = etree.SubElement(contributors_node, "contributor")
    contributor_node.set("name", "Francis Bond")
    contributor_node.set("email", "fcbond@gmail.com")
    contributor_node = etree.SubElement(contributors_node, "contributor")
    contributor_node.set("name", "Dan Flickinger")
    contributor_node.set("email", "danf@stanford.edu")
    return isf_node

def generate_gold_profile():
    # doc = read_gold_sents()
    print("Reading TSDB/TTL data")
    doc = read_tsdb_ttl(GOLD_PATH, ttl_path=GOLD_TTL_PATH)
    # Process data
    print("Creating XML file ...")
    # build root XML node for data file
    isf_node = build_root_node(filename=FileHelper.getfullfilename(OUTPUT_ISF))
    # Add document nodes
    doc.to_xml_node(isf_node)
    # write to file
    print("Making it beautiful ...")
    xml_string = etree.tostring(isf_node, encoding='utf-8', pretty_print=True)
    print("Saving XML data to file ...")
    write_file(xml_string, OUTPUT_ISF)
    print("ISF gold profile has been written to %s" % (OUTPUT_ISF,))
    print("All done!")

GOLD_WRONG = {
    '02108026-v': [10109, 10114, 10178, 10593],  # => {'have'}
    '00066781-r': [10405, 10498, 10501],  # => {'in front'}
    '01554230-a': [10468, 10582],  # => {'such'}
    '01712704-v': [10061, 10265, 10358, 10384],  # => {'do'}
    '01188342-v': [10292],  # => {'be full'}
}

def filter_wrong_senses(doc):
    for sent in doc:
        to_remove = []
        for c in sent.concepts:
            if c.tag[0] in '=!':
                c.tag = c.tag[1:]
            if c.tag in GOLD_WRONG and int(sent.ID) in GOLD_WRONG[c.tag]:
                to_remove.append(c)
        for c in to_remove:
            getLogger().debug("popping {} in sent #{}".format(repr(c.cidx), sent.ID))
            sent.pop_concept(c.cidx)

def export_to_visko(sents, doc_path, pretty_print=True, separate=True):
    ''' Export sentences to XML files '''
    print("Exporting %s sentences to Visko" % (len(sents),))
    print("Visko doc path: {}".format(doc_path))
    if separate:
        if not os.path.exists(doc_path):
            os.makedirs(doc_path)
        for sent in sents:
            sentpath = os.path.join(doc_path, str(sent.ident) + '.xml.gz')
            with gzip.open(sentpath, 'w') as f:
                f.write(etree.tostring(sent.to_xml_node(), encoding='utf-8', pretty_print=pretty_print))
    else:
        # write to file
        chio.write_file(doc_path, sents.to_xml_str(pretty_print=pretty_print))
    print("Done!")
<commit_msg>Allow to continue when some sentences cannot be processed<commit_after># -*- coding: utf-8 -*-

'''
Generate gold profile
Latest version can be found at https://github.com/letuananh/intsem.fx

References:
    Python documentation:
        https://docs.python.org/
    argparse module:
        https://docs.python.org/3/howto/argparse.html
    PEP 257 - Python Docstring Conventions:
        https://www.python.org/dev/peps/pep-0257/

@author: Le Tuan Anh <tuananh.ke@gmail.com>
@license: MIT
'''

# Copyright (c) 2015, Le Tuan Anh <tuananh.ke@gmail.com>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

########################################################################

import os
import datetime
import gzip
import logging
from lxml import etree

from chirptext.leutile import FileHelper
from chirptext import texttaglib as ttl
from chirptext import chio
from lelesk import LeLeskWSD
from lelesk import LeskCache  # WSDResources

from coolisf.data import read_ccby30
from coolisf.dao import read_tsdb
from coolisf.model import Sentence
from coolisf.lexsem import tag_gold, Lexsem
from coolisf.mappings import PredSense
from coolisf.common import write_file
from coolisf.config import read_config

# -----------------------------------------------------------------------
# CONFIGURATION
# -----------------------------------------------------------------------

cfg = read_config()
if cfg and 'data_root' in cfg:
    DATA_DIR = FileHelper.abspath(cfg['data_root'])
else:
    DATA_DIR = FileHelper.abspath('./data')
GOLD_PATH = os.path.join(DATA_DIR, 'gold')
GOLD_TTL_PATH = os.path.join(DATA_DIR, 'gold')
OUTPUT_ISF = os.path.join(DATA_DIR, 'speckled_tsdb_imi.xml.gz')

def getLogger():
    return logging.getLogger(__name__)

# -----------------------------------------------------------------------

def match_sents(isf_doc, ttl_doc):
    ''' Match TSDB profile sentences with TTL sentences '''
    if len(isf_doc) != len(ttl_doc):
        getLogger().info("ISF doc size: {}".format(len(isf_doc)))
        getLogger().info("TTL doc size: {}".format(len(ttl_doc)))
        for sent in isf_doc:
            if not ttl_doc.get(sent.ident, default=None):
                getLogger().warning("Sentence ID(s) in ISF doc and TTL doc are different.")
                return None
        # it's possible to match all
        for sent in isf_doc:
            sent.shallow = ttl_doc.get(sent.ident)
        return isf_doc
    # else
    for isf_sent, ttl_sent in zip(isf_doc, ttl_doc):
        if isf_sent.text != ttl_sent.text:
            # try to match all available sentences
            getLogger().warning("Sentence(s) in ISF doc and TTL doc are different. E.g. ISF/#{}: {} vs TTL/#{}: {}".format(isf_sent.ident, isf_sent.text, ttl_sent.ID, ttl_sent.text))
            return None
    # only import when everything could be matched
    for isf_sent, ttl_sent in zip(isf_doc, ttl_doc):
        isf_sent.shallow = ttl_sent
    return isf_doc

def tag_doc(isf_doc, ttl_doc, use_ttl_sid=True, wsd_method=None, wsd=None, taggold=True, on_error='raise', ctx=None, **kwargs):
    ''' Tag an ISF document using a TTL '''
    isf_doc = match_sents(isf_doc, ttl_doc)
    if isf_doc is None:
        raise Exception("isf_doc and ttl_doc could not be matched")
    not_matched = set()
    wsd = LeLeskWSD(dbcache=LeskCache())
    ctx = PredSense.wn.ctx()
    to_remove = set()
    for sent in isf_doc:
        if use_ttl_sid and sent.shallow and sent.shallow.ID:
            sent.ident = sent.shallow.ID
        for reading in sent:
            if wsd_method:
                sent.tag(method=wsd_method, wsd=wsd, ctx=ctx)
            if taggold and sent.shallow:
                try:
                    m, n, ignored = tag_gold(reading.dmrs(), sent.shallow, sent.text, **kwargs)
                except:
                    getLogger().exception("Could not process sentence #{}: {}".format(sent.ident, sent.text))
                    if on_error == 'ignore':
                        continue
                    elif on_error == 'remove':
                        to_remove.add(sent)
                        break
                    else:
                        raise
                # getLogger().debug("Matched: {}".format(m))
                if n:
                    not_matched.add(sent.ident)
                    sent.flag = Sentence.ERROR
            # update XML
            reading.dmrs().tag_xml(method=None, update_back=True, wsd=wsd, ctx=ctx)
        sent.tag_xml()
    for sent in to_remove:
        isf_doc.remove(sent)
    if not_matched:
        getLogger().warning("TSDB/IMI mismatched sentences: {} - IDs={}".format(len(not_matched), not_matched))
    return isf_doc

def read_tsdb_ttl(tsdb_path, ttl_path=None, name=None, title=None, *args, **kwargs):
    ''' Combine TSDB profile and TTL profile to create ISF document (shallow + deep)
    This function return an instance of coolisf.model.Document
    '''
    tsdb_doc = read_tsdb(tsdb_path, name=name, title=title)
    if ttl_path is None:
        ttl_path = tsdb_path
    ttl_doc = ttl.Document.read_ttl(ttl_path)
    isf_doc = tag_doc(tsdb_doc, ttl_doc, *args, **kwargs)
    getLogger().debug("TTL doc {} contains {} sentence(s).".format(ttl_path, len(ttl_doc)))
    getLogger().debug("isf_doc size: {}".format(len(isf_doc)))
    return isf_doc

def read_gold_mrs():
    ''' Read gold MRS only (without TTL) '''
    return read_tsdb(GOLD_PATH, name="speckled", title="The Adventure of the Speckled Band")

def read_gold_sents(perform_wsd=False):
    ''' Read gold sentences (TSDB+TTL) '''
    return read_tsdb_ttl(GOLD_PATH, name="speckled", title="The Adventure of the Speckled Band")

def build_root_node(filename='spec-isf.xml'):
    isf_node = etree.Element('rootisf')
    isf_node.set('version', '0.1')
    isf_node.set('lang', 'eng')
    # Add license information
    header_node = etree.SubElement(isf_node, 'headerisf')
    filedesc_node = etree.SubElement(header_node, 'description')
    filedesc_node.set("title", "The Adventure of the Speckled Band")
    filedesc_node.set("author", "Arthur Conan Doyle")
    filedesc_node.set("filename", filename)
    filedesc_node.set("creationtime", datetime.datetime.now().isoformat())
    # License text
    license_node = etree.SubElement(filedesc_node, "license")
    license_node.text = etree.CDATA(read_ccby30())
    # CoolISF
    procs_node = etree.SubElement(header_node, "linguisticProcessors")
    proc_node = etree.SubElement(procs_node, "linguisticProcessor")
    proc_node.set("name", "coolisf")
    proc_node.set("description", "Python implementation of Integrated Semantic Framework")
    proc_node.set("version", "pre 0.1")
    proc_node.set("url", "https://github.com/letuananh/intsem.fx")
    proc_node.set("timestamp", datetime.datetime.now().isoformat())
    # ACE
    proc_node = etree.SubElement(procs_node, "linguisticProcessor")
    proc_node.set("name", "ACE")
    proc_node.set("description", "the Answer Constraint Engine (Delph-in)")
    proc_node.set("version", "0.9.17")
    proc_node.set("url", "http://moin.delph-in.net/AceTop")
    proc_node.set("timestamp", datetime.datetime.now().isoformat())
    # NLTK
    proc_node = etree.SubElement(procs_node, "linguisticProcessor")
    proc_node.set("name", "NLTK")
    proc_node.set("description", "Natural Language Toolkit for Python")
    proc_node.set("version", "3.0.4")
    proc_node.set("url", "http://www.nltk.org/")
    proc_node.set("timestamp", datetime.datetime.now().isoformat())
    # pyDelphin
    proc_node = etree.SubElement(procs_node, "linguisticProcessor")
    proc_node.set("name", "pyDelphin")
    proc_node.set("description", "Python libraries for DELPH-IN")
    proc_node.set("version", "0.3")
    proc_node.set("url", "https://github.com/delph-in/pydelphin")
    proc_node.set("timestamp", datetime.datetime.now().isoformat())
    # Contributors
    contributors_node = etree.SubElement(header_node, "contributors")
    contributor_node = etree.SubElement(contributors_node, "contributor")
    contributor_node.set("name", "Le Tuan Anh")
    contributor_node.set("email", "tuananh.ke@gmail.com")
    contributor_node = etree.SubElement(contributors_node, "contributor")
    contributor_node.set("name", "Francis Bond")
    contributor_node.set("email", "fcbond@gmail.com")
    contributor_node = etree.SubElement(contributors_node, "contributor")
    contributor_node.set("name", "Dan Flickinger")
    contributor_node.set("email", "danf@stanford.edu")
    return isf_node

def generate_gold_profile():
    # doc = read_gold_sents()
    print("Reading TSDB/TTL data")
    doc = read_tsdb_ttl(GOLD_PATH, ttl_path=GOLD_TTL_PATH)
    # Process data
    print("Creating XML file ...")
    # build root XML node for data file
    isf_node = build_root_node(filename=FileHelper.getfullfilename(OUTPUT_ISF))
    # Add document nodes
    doc.to_xml_node(isf_node)
    # write to file
    print("Making it beautiful ...")
    xml_string = etree.tostring(isf_node, encoding='utf-8', pretty_print=True)
    print("Saving XML data to file ...")
    write_file(xml_string, OUTPUT_ISF)
    print("ISF gold profile has been written to %s" % (OUTPUT_ISF,))
    print("All done!")

GOLD_WRONG = {
    '02108026-v': [10109, 10114, 10178, 10593],  # => {'have'}
    '00066781-r': [10405, 10498, 10501],  # => {'in front'}
    '01554230-a': [10468, 10582],  # => {'such'}
    '01712704-v': [10061, 10265, 10358, 10384],  # => {'do'}
    '01188342-v': [10292],  # => {'be full'}
}

def filter_wrong_senses(doc):
    for sent in doc:
        to_remove = []
        for c in sent.concepts:
            if c.tag[0] in '=!':
                c.tag = c.tag[1:]
            if c.tag in GOLD_WRONG and int(sent.ID) in GOLD_WRONG[c.tag]:
                to_remove.append(c)
        for c in to_remove:
            getLogger().debug("popping {} in sent #{}".format(repr(c.cidx), sent.ID))
            sent.pop_concept(c.cidx)

def export_to_visko(sents, doc_path, pretty_print=True, separate=True):
    ''' Export sentences to XML files '''
    print("Exporting %s sentences to Visko" % (len(sents),))
    print("Visko doc path: {}".format(doc_path))
    if separate:
        if not os.path.exists(doc_path):
            os.makedirs(doc_path)
        for sent in sents:
            sentpath = os.path.join(doc_path, str(sent.ident) + '.xml.gz')
            with gzip.open(sentpath, 'w') as f:
                f.write(etree.tostring(sent.to_xml_node(), encoding='utf-8', pretty_print=pretty_print))
    else:
        # write to file
        chio.write_file(doc_path, sents.to_xml_str(pretty_print=pretty_print))
    print("Done!")
<|endoftext|>


<commit_before>"""
Eigenvector centrality.

"""
#    Copyright (C) 2004-2010 by
#    Aric Hagberg <hagberg@lanl.gov>
#    Dan Schult <dschult@colgate.edu>
#    Pieter Swart <swart@lanl.gov>
#    All rights reserved.
#    BSD license.
__author__ = "\n".join(['Aric Hagberg (hagberg@lanl.gov)',
                        'Pieter Swart (swart@lanl.gov)',
                        'Sasha Gutfraind (ag362@cornell.edu)'])

__all__ = ['eigenvector_centrality',
           'eigenvector_centrality_numpy']

import networkx

def eigenvector_centrality(G,max_iter=100,tol=1.0e-6,nstart=None):
    """Compute the eigenvector centrality for the graph G.

    Uses the power method to find the eigenvector for the
    largest eigenvalue of the adjacency matrix of G.

    Parameters
    ----------
    G : graph
      A networkx graph

    max_iter : interger, optional
      Maximum number of iterations in power method.

    tol : float, optional
      Error tolerance used to check convergence in power method iteration.

    nstart : dictionary, optional
      Starting value of eigenvector iteration for each node.

    Returns
    -------
    nodes : dictionary
       Dictionary of nodes with eigenvector centrality as the value.

    Examples
    --------
    >>> G=nx.path_graph(4)
    >>> centrality=nx.eigenvector_centrality(G)
    >>> print(['%s %0.2f'%(node,centrality[node]) for node in centrality])
    ['0 0.19', '1 0.31', '2 0.31', '3 0.19']

    Notes
    ------
    The eigenvector calculation is done by the power iteration method
    and has no guarantee of convergence.  The iteration will stop
    after max_iter iterations or an error tolerance of
    number_of_nodes(G)*tol has been reached.

    For directed graphs this is "right" eigevector centrality.  For
    "left" eigenvector centrality, first reverse the graph with
    G.reverse().

    See Also
    --------
    pagerank, hits
    """
    from math import sqrt
    if type(G) == networkx.MultiGraph or type(G) == networkx.MultiDiGraph:
        raise Exception(\
            "eigenvector_centrality() not defined for multigraphs.")

    if nstart is None:
        # choose starting vector with entries of 1/len(G)
        x=dict([(n,1.0/len(G)) for n in G])
    else:
        x=nstart
    # normalize starting vector
    s=1.0/sum(x.values())
    for k in x: x[k]*=s
    nnodes=G.number_of_nodes()
    # make up to max_iter iterations
    for i in range(max_iter):
        xlast=x
        x=dict.fromkeys(xlast.keys(),0)
        # do the multiplication y=Ax
        for n in x:
            for nbr in G[n]:
                x[n]+=xlast[nbr]*G[n][nbr].get('weight',1)
        # normalize vector
        try:
            s=1.0/sqrt(sum(v**2 for v in x.values()))
        except ZeroDivisionError:
            s=1.0
        for n in x: x[n]*=s
        # check convergence
        err=sum([abs(x[n]-xlast[n]) for n in x])
        if err < nnodes*tol:
            return x

    raise networkx.NetworkXError("""eigenvector_centrality():
power iteration failed to converge in %d iterations."%(i+1))""")

def eigenvector_centrality_numpy(G):
    """Compute the eigenvector centrality for the graph G.

    Parameters
    ----------
    G : graph
      A networkx graph

    Returns
    -------
    nodes : dictionary
       Dictionary of nodes with eigenvector centrality as the value.

    Examples
    --------
    >>> G=nx.path_graph(4)
    >>> centrality=nx.eigenvector_centrality_numpy(G)
    >>> print(['%s %0.2f'%(node,centrality[node]) for node in centrality])
    ['0 0.19', '1 0.31', '2 0.31', '3 0.19']

    Notes
    ------
    This algorithm uses the NumPy eigenvalue solver.

    For directed graphs this is "right" eigevector centrality.  For
    "left" eigenvector centrality, first reverse the graph with
    G.reverse().

    Handles disconnected graphs by computing centrality for each component.

    See Also
    --------
    pagerank, hits
    """
    try:
        import numpy as np
    except ImportError:
        raise ImportError(\
            "eigenvector_centrality_numpy() requires NumPy: http://scipy.org/")

    if type(G) == networkx.MultiGraph or type(G) == networkx.MultiDiGraph:
        raise Exception(\
            "eigenvector_centrality_numpy() not defined for multigraphs.")

    centrality=dict.fromkeys(G.nodes(),0)
    # handle multiple connected components by computing centrality
    # of each component individually
    for nodes in networkx.connected_components(G):
        A=networkx.adj_matrix(G,nodelist=nodes)
        eigenvalues,eigenvectors=np.linalg.eig(A)
        # eigenvalue indices in reverse sorted order
        ind=eigenvalues.argsort()[::-1]
        # eigenvector of largest eigenvalue at ind[0], normalized
        largest=np.array(eigenvectors[:,ind[0]])
        centrality.update(zip(nodes,largest*np.sign(largest.max())))
    norm=np.linalg.norm(centrality.values())
    return dict((n,v/norm) for n,v in centrality.items())

<commit_msg>Adjust eigenvector doctest for l2 normalization<commit_after>"""
Eigenvector centrality.

"""
#    Copyright (C) 2004-2010 by
#    Aric Hagberg <hagberg@lanl.gov>
#    Dan Schult <dschult@colgate.edu>
#    Pieter Swart <swart@lanl.gov>
#    All rights reserved.
#    BSD license.
__author__ = "\n".join(['Aric Hagberg (hagberg@lanl.gov)',
                        'Pieter Swart (swart@lanl.gov)',
                        'Sasha Gutfraind (ag362@cornell.edu)'])

__all__ = ['eigenvector_centrality',
           'eigenvector_centrality_numpy']

import networkx

def eigenvector_centrality(G,max_iter=100,tol=1.0e-6,nstart=None):
    """Compute the eigenvector centrality for the graph G.

    Uses the power method to find the eigenvector for the
    largest eigenvalue of the adjacency matrix of G.

    Parameters
    ----------
    G : graph
      A networkx graph

    max_iter : interger, optional
      Maximum number of iterations in power method.

    tol : float, optional
      Error tolerance used to check convergence in power method iteration.

    nstart : dictionary, optional
      Starting value of eigenvector iteration for each node.

    Returns
    -------
    nodes : dictionary
       Dictionary of nodes with eigenvector centrality as the value.

    Examples
    --------
    >>> G=nx.path_graph(4)
    >>> centrality=nx.eigenvector_centrality(G)
    >>> print(['%s %0.2f'%(node,centrality[node]) for node in centrality])
    ['0 0.37', '1 0.60', '2 0.60', '3 0.37']

    Notes
    ------
    The eigenvector calculation is done by the power iteration method
    and has no guarantee of convergence.  The iteration will stop
    after max_iter iterations or an error tolerance of
    number_of_nodes(G)*tol has been reached.

    For directed graphs this is "right" eigevector centrality.  For
    "left" eigenvector centrality, first reverse the graph with
    G.reverse().

    See Also
    --------
    pagerank, hits
    """
    from math import sqrt
    if type(G) == networkx.MultiGraph or type(G) == networkx.MultiDiGraph:
        raise Exception(\
            "eigenvector_centrality() not defined for multigraphs.")

    if nstart is None:
        # choose starting vector with entries of 1/len(G)
        x=dict([(n,1.0/len(G)) for n in G])
    else:
        x=nstart
    # normalize starting vector
    s=1.0/sum(x.values())
    for k in x: x[k]*=s
    nnodes=G.number_of_nodes()
    # make up to max_iter iterations
    for i in range(max_iter):
        xlast=x
        x=dict.fromkeys(xlast.keys(),0)
        # do the multiplication y=Ax
        for n in x:
            for nbr in G[n]:
                x[n]+=xlast[nbr]*G[n][nbr].get('weight',1)
        # normalize vector
        try:
            s=1.0/sqrt(sum(v**2 for v in x.values()))
        except ZeroDivisionError:
            s=1.0
        for n in x: x[n]*=s
        # check convergence
        err=sum([abs(x[n]-xlast[n]) for n in x])
        if err < nnodes*tol:
            return x

    raise networkx.NetworkXError("""eigenvector_centrality():
power iteration failed to converge in %d iterations."%(i+1))""")

def eigenvector_centrality_numpy(G):
    """Compute the eigenvector centrality for the graph G.

    Parameters
    ----------
    G : graph
      A networkx graph

    Returns
    -------
    nodes : dictionary
       Dictionary of nodes with eigenvector centrality as the value.

    Examples
    --------
    >>> G=nx.path_graph(4)
    >>> centrality=nx.eigenvector_centrality_numpy(G)
    >>> print(['%s %0.2f'%(node,centrality[node]) for node in centrality])
    ['0 0.37', '1 0.60', '2 0.60', '3 0.37']

    Notes
    ------
    This algorithm uses the NumPy eigenvalue solver.

    For directed graphs this is "right" eigevector centrality.  For
    "left" eigenvector centrality, first reverse the graph with
    G.reverse().

    Handles disconnected graphs by computing centrality for each component.

    See Also
    --------
    pagerank, hits
    """
    try:
        import numpy as np
    except ImportError:
        raise ImportError(\
            "eigenvector_centrality_numpy() requires NumPy: http://scipy.org/")

    if type(G) == networkx.MultiGraph or type(G) == networkx.MultiDiGraph:
        raise Exception(\
            "eigenvector_centrality_numpy() not defined for multigraphs.")

    centrality=dict.fromkeys(G.nodes(),0)
    # handle multiple connected components by computing centrality
    # of each component individually
    for nodes in networkx.connected_components(G):
        A=networkx.adj_matrix(G,nodelist=nodes)
        eigenvalues,eigenvectors=np.linalg.eig(A)
        # eigenvalue indices in reverse sorted order
        ind=eigenvalues.argsort()[::-1]
        # eigenvector of largest eigenvalue at ind[0], normalized
        largest=np.array(eigenvectors[:,ind[0]])
        centrality.update(zip(nodes,largest*np.sign(largest.max())))
    norm=np.linalg.norm(centrality.values())
    return dict((n,v/norm) for n,v in centrality.items())

if __name__=='__main__':
    import networkx
    G=networkx.path_graph(4)
    print eigenvector_centrality(G)
    print eigenvector_centrality_numpy(G)
<|endoftext|>


<commit_before>"""Command line script for making sky maps.
"""

import click

import numpy as np

@click.group()
@click.option('--nside', help='Set the map resolution (default: 256)', metavar='NSIDE', default=256)
@click.option('--freq', help='Define the frequency channels (in MHz). Default is for CHIME: FSTART=800.0, FSTOP=400.0, FNUM=1025',
              metavar='FSTART FSTOP FNUM', type=(float, float, int), default=(800.0, 400.0, 1025))
@click.option('--channels', help='Select a range of frequency channels',
              type=(int, int), metavar='CSTART CSTOP', default=(None, None))
@click.option('--channel-bin', help='If set, average over BIN channels', metavar='BIN', type=int, default=1)
@click.option('--freq-mode', type=click.Choice(['centre', 'chime', 'edge']), multiple=True, default='centre',
              help=('Choose whether FSTART and FSTOP are the centre frequencies, the frequencies for ' +
                    'the CHIME telescope or the very edges of the band, (default: centre).'))
@click.option('--pol', type=click.Choice(['full', 'zero', 'none']), default='full',
              help='Pick polarisation mode. Full output, zero polarisation, or only return Stokes I (default: full).')
@click.option('--filename', help='Output file [default=map.h5]', metavar='FILENAME', default='map.h5')
@click.pass_context
def cli(ctx, nside, freq, channels, channel_bin, freq_mode, pol, filename):
    """Generate a map of the low frequency radio sky.

    To produce frequency bins compatible with the output of a CASPER based
    channelisation, for example in CHIME, set --freq-mode=centre and set the
    --freq parameter to the start and (missing) end frequencies, and then add
    one to the number of channels (for the missing Nyquist frequency).
    """

    # Generate the set of frequency channels given the parameters
    if freq_mode == 'centre':
        df = abs((freq[1] - freq[0]) / (freq[2] - 1))
        frequencies = np.linspace(freq[0], freq[1], freq[2], endpoint=True)
    if freq_mode == 'chime':
        df = (freq[1] - freq[0]) / freq[2]
        frequencies = np.linspace(freq[0], freq[1], freq[2], endpoint=False)
    else:
        df = (freq[1] - freq[0]) / freq[2]
        frequencies = freq[0] + df * (np.arange(freq[2]) + 0.5)

    # Select a subset of channels if required
    if channels != (None, None):
        frequencies = frequencies[channels[0]:channels[1]]

    # Rebin frequencies if needed
    if channel_bin > 1:
        frequencies = frequencies.reshape(-1, channel_bin).mean(axis=1)
        df = df * channel_bin

    class t(object):
        pass

    ctx.obj = t()

    ctx.obj.freq = frequencies
    ctx.obj.freq_width = np.abs(df)

    ctx.obj.nside = nside
    ctx.obj.full_pol = (pol == 'full')
    ctx.obj.include_pol = (pol != 'none')
    ctx.obj.filename = filename

@cli.command()
@click.option('--maxflux', default=1e6, type=float, help='Maximum flux of point included point source (in Jy). Default is 1 MJy.')
@click.pass_context
def foreground(ctx, maxflux):
    """Generate a full foreground sky map."""

    from cora.foreground import galaxy, pointsource

    # Read in arguments.
    gal = galaxy.ConstrainedGalaxy()
    gal.nside = ctx.obj.nside
    gal.frequencies = ctx.obj.freq

    # Fetch galactic sky
    cs = gal.getpolsky() if ctx.obj.full_pol else gal.getsky()

    # Fetch point source maps
    ps = pointsource.CombinedPointSources.like_map(gal)
    ps.flux_max = maxflux

    cs = cs + (ps.getpolsky() if ctx.obj.full_pol else ps.getsky())

    # Save map
    write_map(ctx.obj.filename, cs, gal.frequencies, ctx.obj.freq_width, ctx.obj.include_pol)

@cli.command()
@click.option('--spectral-index', default='md', type=click.Choice(['md', 'gsm', 'gd']))
@click.pass_context
def galaxy(ctx, spectral_index):
    """Generate a Milky way only foreground map.

    Use Haslam (extrapolated with a spatially varying spectral index) as a base,
    and then generate random spatial, spectral and polarisation fluctuations for
    unconstrained modes.
    """

    from cora.foreground import galaxy

    # Read in arguments.
    gal = galaxy.ConstrainedGalaxy()
    gal.nside = ctx.obj.nside
    gal.frequencies = ctx.obj.freq
    gal.spectral_map = spectral_index

    # Fetch galactic sky
    cs = gal.getpolsky() if ctx.obj.full_pol else gal.getsky()

    # Save map
    write_map(ctx.obj.filename, cs, gal.frequencies, ctx.obj.freq_width, ctx.obj.include_pol)

@cli.command()
@click.option('--maxflux', default=1e6, type=float, help='Maximum flux of point included point source (in Jy). Default is 1 MJy.')
@click.pass_context
def pointsource(ctx, maxflux):
    """Generate a point source only foreground map.

    For S > 4 Jy (at 600 MHz) use real point sources, for dimmer sources (but S >
    0.1 Jy at 151 MHz) generate a synthetic population, for even dimmer sources
    use a Gaussian realisation of unresolved sources.
    """

    from cora.foreground import pointsource

    # Fetch point source maps
    ps = pointsource.CombinedPointSources()
    ps.nside = ctx.obj.nside
    ps.frequencies = ctx.obj.freq
    ps.flux_max = maxflux

    cs = ps.getpolsky() if ctx.obj.full_pol else ps.getsky()

    # Save map
    write_map(ctx.obj.filename, cs, ps.frequencies, ctx.obj.freq_width, ctx.obj.include_pol)

@cli.command('21cm')
@click.option('--eor', is_flag=True, help='Use parameters more suitable for reionisation epoch (rather than intensity mapping).')
@click.option('--oversample', type=int, help='Oversample in redshift by 2**oversample_z + 1 to capute finite width bins.')
@click.pass_context
def _21cm(ctx, eor, oversample):
    """Generate a Gaussian simulation of the unresolved 21cm background.
    """

    from cora.signal import corr21cm

    # Read in arguments.
    if eor:
        cr = corr21cm.EoR21cm()
    else:
        cr = corr21cm.Corr21cm()

    cr.nside = ctx.obj.nside
    cr.frequencies = ctx.obj.freq
    cr.oversample = oversample if oversample is not None else 3

    # Generate signal realisation and save.
    sg_map = cr.getpolsky() if ctx.obj.full_pol else cr.getsky()

    # Save map
    write_map(ctx.obj.filename, sg_map, cr.frequencies, ctx.obj.freq_width, ctx.obj.include_pol)

@cli.command()
@click.pass_context
def gaussianfg(ctx):
    """Generate a full-sky Gaussian random field for synchrotron emission.
    """

    import numpy as np

    from cora.core import skysim
    from cora.util import hputil
    from cora.foreground import galaxy

    fsyn = galaxy.FullSkySynchrotron()
    fpol = galaxy.FullSkyPolarisedSynchrotron()

    # Set frequency parameters
    fsyn.frequencies = ctx.obj.freq
    nfreq = len(fsyn.frequencies)

    nside = ctx.obj.nside
    lmax = 3 * nside
    npol = 4 if ctx.obj.full_pol else 1

    cv_fg = np.zeros((lmax+1, npol, nfreq, npol, nfreq))

    cv_fg[:, 0, :, 0, :] = skysim.clarray(fsyn.angular_powerspectrum, lmax, fsyn.nu_pixels)

    if ctx.obj.full_pol:
        cv_fg[:, 1, :, 1, :] = skysim.clarray(fpol.angular_powerspectrum, lmax, fsyn.nu_pixels)
        cv_fg[:, 2, :, 2, :] = skysim.clarray(fpol.angular_powerspectrum, lmax, fsyn.nu_pixels)

    cv_fg = cv_fg.reshape(lmax+1, npol*nfreq, npol*nfreq)

    alms = skysim.mkfullsky(cv_fg, nside, alms=True).reshape(npol, nfreq, lmax+1, lmax+1)
    alms = alms.transpose((1, 0, 2, 3))

    maps = hputil.sphtrans_inv_sky(alms, nside)
    write_map(ctx.obj.filename, maps, fsyn.frequencies, ctx.obj.freq_width, ctx.obj.include_pol)

def write_map(filename, data, freq, fwidth=None, include_pol=True):
    # Write out the map into an HDF5 file.

    import h5py
    import numpy as np

    # Make into 3D array
    if data.ndim == 3:
        polmap = np.array(['I', 'Q', 'U', 'V'])
    else:
        if include_pol:
            data2 = np.zeros((data.shape[0], 4, data.shape[1]), dtype=data.dtype)
            data2[:, 0] = data
            data = data2
            polmap = np.array(['I', 'Q', 'U', 'V'])
        else:
            data = data[:, np.newaxis, :]
            polmap = np.array(['I'])

    # Construct frequency index map
    freqmap = np.zeros(len(freq), dtype=[('centre', np.float64), ('width', np.float64)])
    freqmap['centre'][:] = freq
    freqmap['width'][:] = fwidth if fwidth is not None else np.abs(np.diff(freq)[0])

    # Open up file for writing
    with h5py.File(filename) as f:
        f.attrs['__memh5_distributed_file'] = True

        dset = f.create_dataset('map', data=data)
        dset.attrs['axis'] = np.array(['freq', 'pol', 'pixel'])
        dset.attrs['__memh5_distributed_dset'] = True

        dset = f.create_dataset('index_map/freq', data=freqmap)
        dset.attrs['__memh5_distributed_dset'] = False
        dset = f.create_dataset('index_map/pol', data=polmap)
        dset.attrs['__memh5_distributed_dset'] = False
        dset = f.create_dataset('index_map/pixel', data=np.arange(data.shape[2]))
        dset.attrs['__memh5_distributed_dset'] = False
<commit_msg>reverted freq-mode to previous version<commit_after>"""Command line script for making sky maps.
"""

import click

import numpy as np

@click.group()
@click.option('--nside', help='Set the map resolution (default: 256)', metavar='NSIDE', default=256)
@click.option('--freq', help='Define the frequency channels (in MHz). Default is for CHIME: FSTART=800.0, FSTOP=400.0, FNUM=1025',
              metavar='FSTART FSTOP FNUM', type=(float, float, int), default=(800.0, 400.0, 1025))
@click.option('--channels', help='Select a range of frequency channels',
              type=(int, int), metavar='CSTART CSTOP', default=(None, None))
@click.option('--channel-bin', help='If set, average over BIN channels', metavar='BIN', type=int, default=1)
@click.option('--freq-mode', type=click.Choice(['centre', 'edge']), default='centre',
              help=('Choose whether FSTART and FSTOP are the very edges of the band, '+
                    'or whether they are the centre frequencies (default: centre).'))
@click.option('--pol', type=click.Choice(['full', 'zero', 'none']), default='full',
              help='Pick polarisation mode. Full output, zero polarisation, or only return Stokes I (default: full).')
@click.option('--filename', help='Output file [default=map.h5]', metavar='FILENAME', default='map.h5')
@click.pass_context
def cli(ctx, nside, freq, channels, channel_bin, freq_mode, pol, filename):
    """Generate a map of the low frequency radio sky.

    To produce frequency bins compatible with the output of a CASPER based
    channelisation, for example in CHIME, set --freq-mode=centre and set the
    --freq parameter to the start and (missing) end frequencies, and then add
    one to the number of channels (for the missing Nyquist frequency).
    """

    # Generate the set of frequency channels given the parameters
    if freq_mode == 'centre':
        df = abs((freq[1] - freq[0]) / (freq[2] - 1))
        frequencies = np.linspace(freq[0], freq[1], freq[2], endpoint=True)
    else:
        df = (freq[1] - freq[0]) / freq[2]
        frequencies = freq[0] + df * (np.arange(freq[2]) + 0.5)

    # Select a subset of channels if required
    if channels != (None, None):
        frequencies = frequencies[channels[0]:channels[1]]

    # Rebin frequencies if needed
    if channel_bin > 1:
        frequencies = frequencies.reshape(-1, channel_bin).mean(axis=1)
        df = df * channel_bin

    class t(object):
        pass

    ctx.obj = t()

    ctx.obj.freq = frequencies
    ctx.obj.freq_width = np.abs(df)

    ctx.obj.nside = nside
    ctx.obj.full_pol = (pol == 'full')
    ctx.obj.include_pol = (pol != 'none')
    ctx.obj.filename = filename

@cli.command()
@click.option('--maxflux', default=1e6, type=float, help='Maximum flux of point included point source (in Jy). Default is 1 MJy.')
@click.pass_context
def foreground(ctx, maxflux):
    """Generate a full foreground sky map."""

    from cora.foreground import galaxy, pointsource

    # Read in arguments.
    gal = galaxy.ConstrainedGalaxy()
    gal.nside = ctx.obj.nside
    gal.frequencies = ctx.obj.freq

    # Fetch galactic sky
    cs = gal.getpolsky() if ctx.obj.full_pol else gal.getsky()

    # Fetch point source maps
    ps = pointsource.CombinedPointSources.like_map(gal)
    ps.flux_max = maxflux

    cs = cs + (ps.getpolsky() if ctx.obj.full_pol else ps.getsky())

    # Save map
    write_map(ctx.obj.filename, cs, gal.frequencies, ctx.obj.freq_width, ctx.obj.include_pol)

@cli.command()
@click.option('--spectral-index', default='md', type=click.Choice(['md', 'gsm', 'gd']))
@click.pass_context
def galaxy(ctx, spectral_index):
    """Generate a Milky way only foreground map.

    Use Haslam (extrapolated with a spatially varying spectral index) as a base,
    and then generate random spatial, spectral and polarisation fluctuations for
    unconstrained modes.
    """

    from cora.foreground import galaxy

    # Read in arguments.
    gal = galaxy.ConstrainedGalaxy()
    gal.nside = ctx.obj.nside
    gal.frequencies = ctx.obj.freq
    gal.spectral_map = spectral_index

    # Fetch galactic sky
    cs = gal.getpolsky() if ctx.obj.full_pol else gal.getsky()

    # Save map
    write_map(ctx.obj.filename, cs, gal.frequencies, ctx.obj.freq_width, ctx.obj.include_pol)

@cli.command()
@click.option('--maxflux', default=1e6, type=float, help='Maximum flux of point included point source (in Jy). Default is 1 MJy.')
@click.pass_context
def pointsource(ctx, maxflux):
    """Generate a point source only foreground map.

    For S > 4 Jy (at 600 MHz) use real point sources, for dimmer sources (but S >
    0.1 Jy at 151 MHz) generate a synthetic population, for even dimmer sources
    use a Gaussian realisation of unresolved sources.
    """

    from cora.foreground import pointsource

    # Fetch point source maps
    ps = pointsource.CombinedPointSources()
    ps.nside = ctx.obj.nside
    ps.frequencies = ctx.obj.freq
    ps.flux_max = maxflux

    cs = ps.getpolsky() if ctx.obj.full_pol else ps.getsky()

    # Save map
    write_map(ctx.obj.filename, cs, ps.frequencies, ctx.obj.freq_width, ctx.obj.include_pol)

@cli.command('21cm')
@click.option('--eor', is_flag=True, help='Use parameters more suitable for reionisation epoch (rather than intensity mapping).')
@click.option('--oversample', type=int, help='Oversample in redshift by 2**oversample_z + 1 to capute finite width bins.')
@click.pass_context
def _21cm(ctx, eor, oversample):
    """Generate a Gaussian simulation of the unresolved 21cm background.
    """

    from cora.signal import corr21cm

    # Read in arguments.
    if eor:
        cr = corr21cm.EoR21cm()
    else:
        cr = corr21cm.Corr21cm()

    cr.nside = ctx.obj.nside
    cr.frequencies = ctx.obj.freq
    cr.oversample = oversample if oversample is not None else 3

    # Generate signal realisation and save.
    sg_map = cr.getpolsky() if ctx.obj.full_pol else cr.getsky()

    # Save map
    write_map(ctx.obj.filename, sg_map, cr.frequencies, ctx.obj.freq_width, ctx.obj.include_pol)

@cli.command()
@click.pass_context
def gaussianfg(ctx):
    """Generate a full-sky Gaussian random field for synchrotron emission.
    """

    import numpy as np

    from cora.core import skysim
    from cora.util import hputil
    from cora.foreground import galaxy

    fsyn = galaxy.FullSkySynchrotron()
    fpol = galaxy.FullSkyPolarisedSynchrotron()

    # Set frequency parameters
    fsyn.frequencies = ctx.obj.freq
    nfreq = len(fsyn.frequencies)

    nside = ctx.obj.nside
    lmax = 3 * nside
    npol = 4 if ctx.obj.full_pol else 1

    cv_fg = np.zeros((lmax+1, npol, nfreq, npol, nfreq))

    cv_fg[:, 0, :, 0, :] = skysim.clarray(fsyn.angular_powerspectrum, lmax, fsyn.nu_pixels)

    if ctx.obj.full_pol:
        cv_fg[:, 1, :, 1, :] = skysim.clarray(fpol.angular_powerspectrum, lmax, fsyn.nu_pixels)
        cv_fg[:, 2, :, 2, :] = skysim.clarray(fpol.angular_powerspectrum, lmax, fsyn.nu_pixels)

    cv_fg = cv_fg.reshape(lmax+1, npol*nfreq, npol*nfreq)

    alms = skysim.mkfullsky(cv_fg, nside, alms=True).reshape(npol, nfreq, lmax+1, lmax+1)
    alms = alms.transpose((1, 0, 2, 3))

    maps = hputil.sphtrans_inv_sky(alms, nside)
    write_map(ctx.obj.filename, maps, fsyn.frequencies, ctx.obj.freq_width, ctx.obj.include_pol)

def write_map(filename, data, freq, fwidth=None, include_pol=True):
    # Write out the map into an HDF5 file.

    import h5py
    import numpy as np

    # Make into 3D array
    if data.ndim == 3:
        polmap = np.array(['I', 'Q', 'U', 'V'])
    else:
        if include_pol:
            data2 = np.zeros((data.shape[0], 4, data.shape[1]), dtype=data.dtype)
            data2[:, 0] = data
            data = data2
            polmap = np.array(['I', 'Q', 'U', 'V'])
        else:
            data = data[:, np.newaxis, :]
            polmap = np.array(['I'])

    # Construct frequency index map
    freqmap = np.zeros(len(freq), dtype=[('centre', np.float64), ('width', np.float64)])
    freqmap['centre'][:] = freq
    freqmap['width'][:] = fwidth if fwidth is not None else np.abs(np.diff(freq)[0])

    # Open up file for writing
    with h5py.File(filename) as f:
        f.attrs['__memh5_distributed_file'] = True

        dset = f.create_dataset('map', data=data)
        dset.attrs['axis'] = np.array(['freq', 'pol', 'pixel'])
        dset.attrs['__memh5_distributed_dset'] = True

        dset = f.create_dataset('index_map/freq', data=freqmap)
        dset.attrs['__memh5_distributed_dset'] = False
        dset = f.create_dataset('index_map/pol', data=polmap)
        dset.attrs['__memh5_distributed_dset'] = False
        dset = f.create_dataset('index_map/pixel', data=np.arange(data.shape[2]))
        dset.attrs['__memh5_distributed_dset'] = False
<|endoftext|>