<commit_before>from data import anagram_index, crossword, warehouse
from spec.data.fixtures import tries

def _get_unigram_anagram_index():
  return anagram_index.AnagramIndex(warehouse.get('/words/unigram/trie'))

def _get_unigram_trie():
  return tries.kitchen_sink()

def _get_crossword():
  connection = crossword.init(':memory:')
  cursor = connection.cursor()
  crossword.add(cursor, 'query', 1, {'ask': 1, 'question': 1})
  return connection, cursor

def _get_crossword_connection():
  connection, cursor = warehouse.get('/phrases/crossword')
  del cursor
  return connection

def _get_crossword_cursor():
  connection, cursor = warehouse.get('/phrases/crossword')
  del connection
  return cursor

warehouse.init()
warehouse.register('/phrases/crossword', _get_crossword)
warehouse.register('/phrases/crossword/connection', _get_crossword_connection)
warehouse.register('/phrases/crossword/cursor', _get_crossword_cursor)
warehouse.register('/words/unigram/anagram_index', _get_unigram_anagram_index)
warehouse.register('/words/unigram/trie', _get_unigram_trie)
<commit_msg>Introduce unigram data to warehouse module.<commit_after>import collections

from data import anagram_index, crossword, warehouse
from spec.data.fixtures import tries

def _get_unigram():
  return collections.OrderedDict(tries.kitchen_sink_data())

def _get_unigram_anagram_index():
  return anagram_index.AnagramIndex(warehouse.get('/words/unigram'))

def _get_unigram_trie():
  return tries.kitchen_sink()

def _get_crossword():
  connection = crossword.init(':memory:')
  cursor = connection.cursor()
  crossword.add(cursor, 'query', 1, {'ask': 1, 'question': 1})
  return connection, cursor

def _get_crossword_connection():
  connection, cursor = warehouse.get('/phrases/crossword')
  del cursor
  return connection

def _get_crossword_cursor():
  connection, cursor = warehouse.get('/phrases/crossword')
  del connection
  return cursor

warehouse.init()
warehouse.register('/phrases/crossword', _get_crossword)
warehouse.register('/phrases/crossword/connection', _get_crossword_connection)
warehouse.register('/phrases/crossword/cursor', _get_crossword_cursor)
warehouse.register('/words/unigram', _get_unigram)
warehouse.register('/words/unigram/anagram_index', _get_unigram_anagram_index)
warehouse.register('/words/unigram/trie', _get_unigram_trie)
<|endoftext|>


<commit_before>import numpy

import cupy
from cupy_backends.cuda.libs import cublas
from cupy_backends.cuda.libs import cusolver
from cupy.cuda import device
from cupy.cuda import runtime
from cupy.linalg import _util
from cupy._core import _dtype

_cuda_runtime_version = -1

def _syevd(a, UPLO, with_eigen_vector, overwrite_a=False):
    if UPLO not in ('L', 'U'):
        raise ValueError('UPLO argument must be \'L\' or \'U\'')

    # reject_float16=False for backward compatibility
    dtype, v_dtype = _util.linalg_common_type(a, reject_float16=False)
    real_dtype = dtype.char.lower()
    w_dtype = v_dtype.char.lower()

    # Note that cuSolver assumes fortran array
    v = a.astype(dtype, order='F', copy=not overwrite_a)

    m, lda = a.shape
    w = cupy.empty(m, real_dtype)
    dev_info = cupy.empty((), numpy.int32)
    handle = device.Device().cusolver_handle

    if with_eigen_vector:
        jobz = cusolver.CUSOLVER_EIG_MODE_VECTOR
    else:
        jobz = cusolver.CUSOLVER_EIG_MODE_NOVECTOR

    if UPLO == 'L':
        uplo = cublas.CUBLAS_FILL_MODE_LOWER
    else:  # UPLO == 'U'
        uplo = cublas.CUBLAS_FILL_MODE_UPPER

    global _cuda_runtime_version
    if _cuda_runtime_version < 0:
        _cuda_runtime_version = runtime.runtimeGetVersion()

    if (not runtime.is_hip and _cuda_runtime_version >= 11020):
        if dtype.char not in 'fdFD':
            raise RuntimeError('Only float32, float64, complex64, and '
                               'complex128 are supported')
        type_v = _dtype.to_cuda_dtype(dtype)
        type_w = _dtype.to_cuda_dtype(real_dtype)
        params = cusolver.createParams()
        try:
            work_device_size, work_host_sizse = cusolver.xsyevd_bufferSize(
                handle, params, jobz, uplo, m, type_v, v.data.ptr, lda,
                type_w, w.data.ptr, type_v)
            work_device = cupy.empty(work_device_size, 'b')
            work_host = numpy.empty(work_host_sizse, 'b')
            cusolver.xsyevd(
                handle, params, jobz, uplo, m, type_v, v.data.ptr, lda,
                type_w, w.data.ptr, type_v,
                work_device.data.ptr, work_device_size,
                work_host.ctypes.data, work_host_sizse, dev_info.data.ptr)
        finally:
            cusolver.destroyParams(params)
        cupy.linalg._util._check_cusolver_dev_info_if_synchronization_allowed(
            cusolver.xsyevd, dev_info)
    else:
        if dtype == 'f':
            buffer_size = cupy.cuda.cusolver.ssyevd_bufferSize
            syevd = cupy.cuda.cusolver.ssyevd
        elif dtype == 'd':
            buffer_size = cupy.cuda.cusolver.dsyevd_bufferSize
            syevd = cupy.cuda.cusolver.dsyevd
        elif dtype == 'F':
            buffer_size = cupy.cuda.cusolver.cheevd_bufferSize
            syevd = cupy.cuda.cusolver.cheevd
        elif dtype == 'D':
            buffer_size = cupy.cuda.cusolver.zheevd_bufferSize
            syevd = cupy.cuda.cusolver.zheevd
        else:
            raise RuntimeError('Only float and double and cuComplex and '
                               + 'cuDoubleComplex are supported')

        work_size = buffer_size(
            handle, jobz, uplo, m, v.data.ptr, lda, w.data.ptr)
        work = cupy.empty(work_size, dtype)
        syevd(
            handle, jobz, uplo, m, v.data.ptr, lda,
            w.data.ptr, work.data.ptr, work_size, dev_info.data.ptr)
        cupy.linalg._util._check_cusolver_dev_info_if_synchronization_allowed(
            syevd, dev_info)

    return w.astype(w_dtype, copy=False), v.astype(v_dtype, copy=False)

# TODO(okuta): Implement eig

def eigh(a, UPLO='L'):
    """
    Return the eigenvalues and eigenvectors of a complex Hermitian
    (conjugate symmetric) or a real symmetric matrix.

    Returns two objects, a 1-D array containing the eigenvalues of `a`, and
    a 2-D square array or matrix (depending on the input type) of the
    corresponding eigenvectors (in columns).

    Args:
        a (cupy.ndarray): A symmetric 2-D square matrix ``(M, M)`` or a batch
            of symmetric 2-D square matrices ``(..., M, M)``.
        UPLO (str): Select from ``'L'`` or ``'U'``. It specifies which
            part of ``a`` is used. ``'L'`` uses the lower triangular part of
            ``a``, and ``'U'`` uses the upper triangular part of ``a``.
    Returns:
        tuple of :class:`~cupy.ndarray`:
            Returns a tuple ``(w, v)``. ``w`` contains eigenvalues and
            ``v`` contains eigenvectors. ``v[:, i]`` is an eigenvector
            corresponding to an eigenvalue ``w[i]``. For batch input,
            ``v[k, :, i]`` is an eigenvector corresponding to an eigenvalue
            ``w[k, i]`` of ``a[k]``.

    .. warning::
        This function calls one or more cuSOLVER routine(s) which may yield
        invalid results if input conditions are not met.
        To detect these invalid results, you can set the `linalg`
        configuration to a value that is not `ignore` in
        :func:`cupyx.errstate` or :func:`cupyx.seterr`.

    .. seealso:: :func:`numpy.linalg.eigh`
    """
    if a.ndim < 2:
        raise ValueError('Array must be at least two-dimensional')

    m, n = a.shape[-2:]
    if m != n:
        raise ValueError('Last 2 dimensions of the array must be square')

    if a.ndim > 2 or runtime.is_hip:
        w, v = cupy.cusolver.syevj(a, UPLO, True)
        return w, v
    else:
        return _syevd(a, UPLO, True)

# TODO(okuta): Implement eigvals

def eigvalsh(a, UPLO='L'):
    """
    Compute the eigenvalues of a complex Hermitian or real symmetric matrix.

    Main difference from eigh: the eigenvectors are not computed.

    Args:
        a (cupy.ndarray): A symmetric 2-D square matrix ``(M, M)`` or a batch
            of symmetric 2-D square matrices ``(..., M, M)``.
        UPLO (str): Select from ``'L'`` or ``'U'``. It specifies which
            part of ``a`` is used. ``'L'`` uses the lower triangular part of
            ``a``, and ``'U'`` uses the upper triangular part of ``a``.
    Returns:
        cupy.ndarray:
            Returns eigenvalues as a vector ``w``. For batch input,
            ``w[k]`` is a vector of eigenvalues of matrix ``a[k]``.

    .. warning::
        This function calls one or more cuSOLVER routine(s) which may yield
        invalid results if input conditions are not met.
        To detect these invalid results, you can set the `linalg`
        configuration to a value that is not `ignore` in
        :func:`cupyx.errstate` or :func:`cupyx.seterr`.

    .. seealso:: :func:`numpy.linalg.eigvalsh`
    """
    _util._assert_stacked_2d(a)
    _util._assert_stacked_square(a)

    if a.ndim > 2 or runtime.is_hip:
        return cupy.cusolver.syevj(a, UPLO, False)
    else:
        return _syevd(a, UPLO, False)[0]
<commit_msg>Update cupy/linalg/_eigenvalue.py<commit_after>import numpy

import cupy
from cupy_backends.cuda.libs import cublas
from cupy_backends.cuda.libs import cusolver
from cupy.cuda import device
from cupy.cuda import runtime
from cupy.linalg import _util
from cupy._core import _dtype

_cuda_runtime_version = -1

def _syevd(a, UPLO, with_eigen_vector, overwrite_a=False):
    if UPLO not in ('L', 'U'):
        raise ValueError('UPLO argument must be \'L\' or \'U\'')

    # reject_float16=False for backward compatibility
    dtype, v_dtype = _util.linalg_common_type(a, reject_float16=False)
    real_dtype = dtype.char.lower()
    w_dtype = v_dtype.char.lower()

    # Note that cuSolver assumes fortran array
    v = a.astype(dtype, order='F', copy=not overwrite_a)

    m, lda = a.shape
    w = cupy.empty(m, real_dtype)
    dev_info = cupy.empty((), numpy.int32)
    handle = device.Device().cusolver_handle

    if with_eigen_vector:
        jobz = cusolver.CUSOLVER_EIG_MODE_VECTOR
    else:
        jobz = cusolver.CUSOLVER_EIG_MODE_NOVECTOR

    if UPLO == 'L':
        uplo = cublas.CUBLAS_FILL_MODE_LOWER
    else:  # UPLO == 'U'
        uplo = cublas.CUBLAS_FILL_MODE_UPPER

    global _cuda_runtime_version
    if _cuda_runtime_version < 0:
        _cuda_runtime_version = runtime.runtimeGetVersion()

    if (not runtime.is_hip and _cuda_runtime_version >= 11020):
        if dtype.char not in 'fdFD':
            raise RuntimeError('Only float32, float64, complex64, and '
                               'complex128 are supported')
        type_v = _dtype.to_cuda_dtype(dtype)
        type_w = _dtype.to_cuda_dtype(real_dtype)
        params = cusolver.createParams()
        try:
            work_device_size, work_host_sizse = cusolver.xsyevd_bufferSize(
                handle, params, jobz, uplo, m, type_v, v.data.ptr, lda,
                type_w, w.data.ptr, type_v)
            work_device = cupy.empty(work_device_size, 'b')
            work_host = numpy.empty(work_host_sizse, 'b')
            cusolver.xsyevd(
                handle, params, jobz, uplo, m, type_v, v.data.ptr, lda,
                type_w, w.data.ptr, type_v,
                work_device.data.ptr, work_device_size,
                work_host.ctypes.data, work_host_sizse, dev_info.data.ptr)
        finally:
            cusolver.destroyParams(params)
        cupy.linalg._util._check_cusolver_dev_info_if_synchronization_allowed(
            cusolver.xsyevd, dev_info)
    else:
        if dtype == 'f':
            buffer_size = cupy.cuda.cusolver.ssyevd_bufferSize
            syevd = cupy.cuda.cusolver.ssyevd
        elif dtype == 'd':
            buffer_size = cupy.cuda.cusolver.dsyevd_bufferSize
            syevd = cupy.cuda.cusolver.dsyevd
        elif dtype == 'F':
            buffer_size = cupy.cuda.cusolver.cheevd_bufferSize
            syevd = cupy.cuda.cusolver.cheevd
        elif dtype == 'D':
            buffer_size = cupy.cuda.cusolver.zheevd_bufferSize
            syevd = cupy.cuda.cusolver.zheevd
        else:
            raise RuntimeError('Only float32, float64, complex64, and '
                               'complex128 are supported')

        work_size = buffer_size(
            handle, jobz, uplo, m, v.data.ptr, lda, w.data.ptr)
        work = cupy.empty(work_size, dtype)
        syevd(
            handle, jobz, uplo, m, v.data.ptr, lda,
            w.data.ptr, work.data.ptr, work_size, dev_info.data.ptr)
        cupy.linalg._util._check_cusolver_dev_info_if_synchronization_allowed(
            syevd, dev_info)

    return w.astype(w_dtype, copy=False), v.astype(v_dtype, copy=False)

# TODO(okuta): Implement eig

def eigh(a, UPLO='L'):
    """
    Return the eigenvalues and eigenvectors of a complex Hermitian
    (conjugate symmetric) or a real symmetric matrix.

    Returns two objects, a 1-D array containing the eigenvalues of `a`, and
    a 2-D square array or matrix (depending on the input type) of the
    corresponding eigenvectors (in columns).

    Args:
        a (cupy.ndarray): A symmetric 2-D square matrix ``(M, M)`` or a batch
            of symmetric 2-D square matrices ``(..., M, M)``.
        UPLO (str): Select from ``'L'`` or ``'U'``. It specifies which
            part of ``a`` is used. ``'L'`` uses the lower triangular part of
            ``a``, and ``'U'`` uses the upper triangular part of ``a``.
    Returns:
        tuple of :class:`~cupy.ndarray`:
            Returns a tuple ``(w, v)``. ``w`` contains eigenvalues and
            ``v`` contains eigenvectors. ``v[:, i]`` is an eigenvector
            corresponding to an eigenvalue ``w[i]``. For batch input,
            ``v[k, :, i]`` is an eigenvector corresponding to an eigenvalue
            ``w[k, i]`` of ``a[k]``.

    .. warning::
        This function calls one or more cuSOLVER routine(s) which may yield
        invalid results if input conditions are not met.
        To detect these invalid results, you can set the `linalg`
        configuration to a value that is not `ignore` in
        :func:`cupyx.errstate` or :func:`cupyx.seterr`.

    .. seealso:: :func:`numpy.linalg.eigh`
    """
    if a.ndim < 2:
        raise ValueError('Array must be at least two-dimensional')

    m, n = a.shape[-2:]
    if m != n:
        raise ValueError('Last 2 dimensions of the array must be square')

    if a.ndim > 2 or runtime.is_hip:
        w, v = cupy.cusolver.syevj(a, UPLO, True)
        return w, v
    else:
        return _syevd(a, UPLO, True)

# TODO(okuta): Implement eigvals

def eigvalsh(a, UPLO='L'):
    """
    Compute the eigenvalues of a complex Hermitian or real symmetric matrix.

    Main difference from eigh: the eigenvectors are not computed.

    Args:
        a (cupy.ndarray): A symmetric 2-D square matrix ``(M, M)`` or a batch
            of symmetric 2-D square matrices ``(..., M, M)``.
        UPLO (str): Select from ``'L'`` or ``'U'``. It specifies which
            part of ``a`` is used. ``'L'`` uses the lower triangular part of
            ``a``, and ``'U'`` uses the upper triangular part of ``a``.
    Returns:
        cupy.ndarray:
            Returns eigenvalues as a vector ``w``. For batch input,
            ``w[k]`` is a vector of eigenvalues of matrix ``a[k]``.

    .. warning::
        This function calls one or more cuSOLVER routine(s) which may yield
        invalid results if input conditions are not met.
        To detect these invalid results, you can set the `linalg`
        configuration to a value that is not `ignore` in
        :func:`cupyx.errstate` or :func:`cupyx.seterr`.

    .. seealso:: :func:`numpy.linalg.eigvalsh`
    """
    _util._assert_stacked_2d(a)
    _util._assert_stacked_square(a)

    if a.ndim > 2 or runtime.is_hip:
        return cupy.cusolver.syevj(a, UPLO, False)
    else:
        return _syevd(a, UPLO, False)[0]
<|endoftext|>


<commit_before>import numpy as np
import scipy.optimize as op

def fit_single_patch(data, mask , psf, theta, floor, gain):

    """
       Inputs:
       data = patch,
       mask = True for healthy pixels, False for flagged pixels
       psf = psf model rendered at the data grid
       theta = [old_flux, old_bkg],
       where:
              old_flux = current flux estimate for a patch
              old_back = current bkg estimate for the patch
       floor = floor variance of the noise model
       gain = gain of the noise model
       Outputs:
       (non-regularized) NLL of the patch, and derivative
       of (non-regularized) NLL w.r.t flux and bkg of the patch.
       Note that the regularization is independent of F, B.
    """

    var = floor + gain * np.abs(theta[1] * psf[mask] + theta[0])
    A = np.ones((var.size, 2))
    A[:, 1] = psf[mask]
    model = np.dot(A, theta) #masked model
    res  = data[mask] - model
    func = 0.5*np.sum(((res)**2.)/var) + 0.5*np.sum(np.log(var))
    Grad = 0.5*gain*(1./var - res*res/(var*var)) - (res*res)/var
    grad = np.sum(Grad[:, None]*A , axis = 1)

    return func , grad

# different version of the function defined above. We'll see which one is faster:

def v2_fit_single_patch(theta, masked_data, masked_psf, floor, gain):

    """
       Inputs:

       theta = [old_bkg, old_flux],
       where:
              old_flux = current flux estimate for a patch
              old_back = current bkg estimate for the patch

       masked_data = patch with flagged pixels masked out
       masked_psf = psf model rendered at the data grid
                    masked out
                    where pixels are flagged

       floor = floor variance of the noise model
       gain  = gain of the noise model

       Outputs:

       (non-regularized) NLL of the patch, and derivative
       of (non-regularized) NLL w.r.t flux and bkg of the patch.
       Note that the regularization is independent of F, B.
    """

    var = floor + gain * np.abs(theta[1]* masked_psf + theta[0])
    A = np.ones((var.size, 2))
    A[:, 1] = masked_psf
    model = np.dot(A , theta) #masked model
    res  = masked_data - model
    func = 0.5*np.sum(((res)**2.)/var) + 0.5*np.sum(np.log(var))
    Grad = 0.5*gain*(1./var - res*res/(var*var)) - (res*res)/var
    grad = np.sum(Grad[:, None]*A , axis = 0)  #this could unnecessarily slow down the code!!
    print func.shape , grad.shape
    return func , grad

##### this is probably the fastest version!

def v3_fit_single_patch(theta, masked_data, masked_psf, floor, gain):

    """
       Inputs:

       theta = [old_flux, old_bkg],
       where:
              old_flux = current flux estimate for a patch
              old_back = current bkg estimate for the patch

       masked_data = patch with flagged pixels masked out
       masked_psf = psf model rendered at the data grid
                    masked out
                    where pixels are flagged

       floor = floor variance of the noise model
       gain  = gain of the noise model

       Outputs:

       (non-regularized) NLL of the patch, and derivative
       of (non-regularized) NLL w.r.t flux and bkg of the patch.
       Note that the regularization is independent of F, B.
    """
    grad = np.zeros((2))

    var = floor + gain * np.abs(theta[1] * masked_psf + theta[0])
    model = theta[0] + theta[1]*masked_psf #masked model
    res  = masked_data - model
    func = 0.5*np.sum(((res)**2.)/var) + 0.5*np.sum(np.log(var))

    gain_term_b = - (gain/2.)*(res**2./var**2.) + (gain/2.)*(var**-1.)
    #gainp[modelp<0] *= -1.   #var=f+g|model| to account for numerical artifacts when sr model is sampled at the data grid
    grad[0] = -1.*np.sum(res/var) + np.sum(gain_term_b)

    gain_term_f = (gain/2.)*masked_psf*(var**-1. - res**2./var**2.)
    #gainp[modelp<0] *= -1.   #var=f+g|model| to account for numerical artifacts when sr model is sampled at the data grid
    grad[1] = np.sum(-1.*res*masked_psf/var) + np.sum(gain_term_f)

    return func, grad
<commit_msg>modified bkg, amp updating<commit_after>import numpy as np
import scipy.optimize as op

def fit_single_patch(data, mask , psf, theta, floor, gain):
    """
       Inputs:
       data  = patch,
       mask  = True for healthy pixels, False for flagged pixels
       psf   = psf model rendered at the data grid
       theta = [old_flux, old_bkg],
       where:
               old_flux = current flux estimate for a patch
               old_back = current bkg estimate for the patch
       floor = floor variance of the noise model
       gain  = gain of the noise model
       Outputs:
       (non-regularized) NLL of the patch, and derivative
       of (non-regularized) NLL w.r.t flux and bkg of the patch.
       Note that the regularization is independent of F, B.
    """

    var = floor + gain * np.abs(theta[1] * psf[mask] + theta[0])
    A = np.ones((var.size, 2))
    A[:, 1] = psf[mask]
    model = np.dot(A, theta) #masked model
    res  = data[mask] - model
    func = 0.5*np.sum(((res)**2.)/var) + 0.5*np.sum(np.log(var))
    Grad = 0.5*gain*(1./var - res*res/(var*var)) - (res*res)/var
    grad = np.sum(Grad[:, None]*A , axis = 1)

    return func , grad

# different version of the function defined above. We'll see which one is faster:

def v2_fit_single_patch(theta, masked_data, masked_psf, floor, gain):

    """
       Inputs:

       theta = [old_bkg, old_flux],
       where:
              old_flux = current flux estimate for a patch
              old_back = current bkg estimate for the patch

       masked_data = patch with flagged pixels masked out
       masked_psf = psf model rendered at the data grid
                    masked out
                    where pixels are flagged

       floor = floor variance of the noise model
       gain  = gain of the noise model

       Outputs:

       (non-regularized) NLL of the patch, and derivative
       of (non-regularized) NLL w.r.t flux and bkg of the patch.
       Note that the regularization is independent of F, B.
    """

    var = floor + gain * np.abs(theta[1]* masked_psf + theta[0])
    A = np.ones((var.size, 2))
    A[:, 1] = masked_psf
    model = np.dot(A , theta) #masked model
    res  = masked_data - model
    func = 0.5*np.sum(((res)**2.)/var) + 0.5*np.sum(np.log(var))
    Grad = 0.5*gain*(1./var - res*res/(var*var)) - (res*res)/var
    grad = np.sum(Grad[:, None]*A , axis = 0)  #this could unnecessarily slow down the code!!
    print func.shape , grad.shape
    return func , grad

##### this is probably the fastest version!

def v3_fit_single_patch(theta, masked_data, masked_psf, floor, gain):

    """
       Inputs:

       theta = [old_flux, old_bkg],
       where:
              old_flux = current flux estimate for a patch
              old_back = current bkg estimate for the patch

       masked_data = patch with flagged pixels masked out
       masked_psf = psf model rendered at the data grid
                    masked out
                    where pixels are flagged

       floor = floor variance of the noise model
       gain  = gain of the noise model

       Outputs:

       (non-regularized) NLL of the patch, and derivative
       of (non-regularized) NLL w.r.t flux and bkg of the patch.
       Note that the regularization is independent of F, B.
    """
    grad = np.zeros((2))

    var = floor + gain * np.abs(theta[1] * masked_psf + theta[0])
    model = theta[0] + theta[1]*masked_psf #masked model
    res  = masked_data - model
    func = 0.5*np.sum(((res)**2.)/var) + 0.5*np.sum(np.log(var))

    gain_term_b = - (gain/2.)*(res**2./var**2.) + (gain/2.)*(var**-1.)
    #gainp[modelp<0] *= -1.   #var=f+g|model| to account for numerical artifacts when sr model is sampled at the data grid
    grad[0] = -1.*np.sum(res/var) + np.sum(gain_term_b)

    gain_term_f = (gain/2.)*masked_psf*(var**-1. - res**2./var**2.)
    #gainp[modelp<0] *= -1.   #var=f+g|model| to account for numerical artifacts when sr model is sampled at the data grid
    grad[1] = np.sum(-1.*res*masked_psf/var) + np.sum(gain_term_f)

    return func, grad
def v4_fit_single_patch(theta, data, mask, flux, bkg, \
                        alpha, mean, floor, gain):

    """
       Inputs:

       theta = [old_cx, old_cy],
       where:
              old_cx = current cx estimate for a patch
              old_cy = current cy estimate for the patch

       masked_data = patch with flagged pixels masked out
       masked_psf = psf model rendered at the data grid
                    masked out
                    where pixels are flagged

       floor = floor variance of the noise model
       gain  = gain of the noise model

       Outputs:

       (non-regularized) NLL of the patch,
       Note that the regularization is independent of F, B.
    """
    Dx , Dy = theta[0], theta[1]
    h = 1./25
    x2 = np.arange(0, 25)*h + 0.5*h - Dx*h
    y2 = np.arange(0, 25)*h + 0.5*h - Dy*h
    x2, y2 = np.meshgrid(x2, y2, indexing = "ij")
    hh = 1./101
    x = np.arange(0, 101)*hh + 0.5*hh
    y = np.arange(0, 101)*hh + 0.5*hh
    x, y = np.meshgrid(x, y, indexing = "ij")
    samples2 = np.vstack((x2.flatten(), y2.flatten())).T #data_grid
    samples = np.vstack((x.flatten(), y.flatten())).T    #sr grid
    Kxsx = self.kernel.value(samples2, samples)
    model = flux*(np.dot(Kxsx, alpha) + mean) + bkg

    data = data[mask!=0]
    model = model[mask!=0]
    res = data - model
    var = floor + gain*np.abs(model)
    func = np.sum(res**2. / var + np.log(var))

    return func
<|endoftext|>


<commit_before>"""
The HTTP module for Curious.

This code is 100% portable - it will work anywhere that h11 and multidict are also installed.
"""

import logging
import mimetypes
import random
import string
from functools import partial
import json as py_json
import curio
import cgi

import multidict
import yarl
import h11
from curio import io
from h11._events import _EventBundle
from multidict import MultiDict

__version__ = "0.1.0-curious"

logger = logging.getLogger(__name__)

class HTTPError(Exception):
    def __init__(self, *args, **kwargs):
        self.response = kwargs.pop('response')
        super().__init__(*args, **kwargs)

_BOUNDARY_CHARS = string.digits + string.ascii_letters

def encode_multipart(fields, files, boundary=None):
    r"""Encode dict of form fields and dict of files as multipart/form-data.
    Return tuple of (body_string, headers_dict). Each value in files is a dict
    with required keys 'filename' and 'content', and optional 'mimetype' (if
    not specified, tries to guess mime type or uses 'application/octet-stream').

    >>> body, headers = encode_multipart({'FIELD': 'VALUE'},
    ...                                  {'FILE': {'filename': 'F.TXT', 'content': 'CONTENT'}},
    ...                                  boundary='BOUNDARY')
    >>> print('\n'.join(repr(l) for l in body.split('\r\n')))
    '--BOUNDARY'
    'Content-Disposition: form-data; name="FIELD"'
    ''
    'VALUE'
    '--BOUNDARY'
    'Content-Disposition: form-data; name="FILE"; filename="F.TXT"'
    'Content-Type: text/plain'
    ''
    'CONTENT'
    '--BOUNDARY--'
    ''
    >>> print(sorted(headers.items()))
    [('Content-Length', '193'), ('Content-Type', 'multipart/form-data; boundary=BOUNDARY')]
    >>> len(body)
    193

    Copied from: https://code.activestate.com/recipes/578668-encode-multipart-form-data-for-uploading-files-via/
    """

    def escape_quote(s):
        return s.replace(b'"', b'\\"')

    if boundary is None:
        boundary = b''.join(random.choice(_BOUNDARY_CHARS).encode() for i in range(30))
    lines = []

    for name, value in fields.items():
        if isinstance(name, str):
            name = name.encode()
        else:
            name = str(name).encode()

        if isinstance(value, str):
            value = value.encode()
        else:
            value = str(value).encode()

        lines.extend((
            b'--%s' % boundary,
            b'Content-Disposition: form-data; name="%s"' % escape_quote(name),
            b'',
            value,
        ))

    for name, value in files.items():
        filename = value['filename']
        if 'mimetype' in value:
            mimetype = value['mimetype']
        else:
            mimetype = mimetypes.guess_type(filename)[0] or 'application/octet-stream'
        lines.extend((
            b'--%s' % boundary,
            b'Content-Disposition: form-data; name="%s"; filename="%s"' % (
                escape_quote(name.encode()), escape_quote(filename.encode())),
            b'Content-Type: %s' % mimetype.encode(),
            b'',
            value['content'],
        ))

    lines.extend((
        b'--%s--' % boundary,
        b'',
    ))
    body = b'\r\n'.join(lines)

    headers = {
        'Content-Type': 'multipart/form-data; boundary=%s' % boundary.decode(),
        'Content-Length': str(len(body)),
    }

    return body, headers

def get_encoding_from_headers(headers: MultiDict):
    """
    Attempts to get the encoding of the request based on the headers.
    """
    content_type = headers.get('content-type')

    if not content_type:
        return None

    content_type, params = cgi.parse_header(content_type)

    if 'charset' in params:
        return params['charset'].strip("'\"")

    if 'text' in content_type:
        return 'ISO-8859-1'

class _EventIterator:
    """
    An iterator that allows you to receive events from a connection.
    """

    def __init__(self, event_source):
        self.event_source = event_source

    async def __aiter__(self):
        return self

    async def __anext__(self):
        event = await self.event_source()

        if type(event) is h11.Data:
            return event.data
        elif type(event) is h11.EndOfMessage:
            raise StopAsyncIteration
        else:
            raise ValueError('Unknown h11 event: %r', event)

class Response:
    """
    A wrapper around the Response from the server.
    """

    def __init__(self, raw_response: h11.Response, h11_request: h11.Request, conn: 'HTTPConnection'):
        self.status_code = raw_response.status_code
        self.http_version = raw_response.http_version.decode('utf-8')

        self.headers = multidict.CIMultiDict(
            (key.decode("utf-8"), value.decode("utf-8")) for key, value in raw_response.headers
        )

        self.h11_request = h11_request
        self.conn = conn

        self.history = None

    def __repr__(self):
        return '<Response [%s]>' % (self.status_code)

    @property
    def url(self):
        """The final URL that was requested."""
        return '{host}:{port}{target}'.format(
            host=self.conn.host,
            port=self.conn.port,
            target=self.h11_request.target.decode('utf-8'),
        )

    async def close(self):
        """
        Closes the current request.

        This should always be called.
        """
        await self.conn.close()

    def raise_for_status(self):
        """Raises HTTPError, if one occurred."""
        http_error_msg = ''

        if 400 <= self.status_code < 500:
            http_error_msg = '%s Client Error for url: %s' % (
                self.status_code, self.url)

        elif 500 <= self.status_code < 600:
            http_error_msg = '%s Server Error for url: %s' % (
                self.status_code, self.url)

        if http_error_msg:
            raise HTTPError(http_error_msg, response=self)

    @property
    def is_redirect(self):
        """Whether the response is a well-formed redirect."""
        return 'location' in self.headers and 301 <= self.status_code < 400

    def iter_chunked(self, maxsize=None):
        """Stream raw response body, maxsize bytes at a time."""
        return _EventIterator(partial(self.conn._next_event, maxsize))

    async def binary(self) -> bytes:
        """Return the full response body as a bytearray."""
        data = b""
        async for chunk in self.iter_chunked():
            data += chunk

        return data

    async def text(self):
        """Return the full response body as a string."""
        data = await self.binary()
        if data is None:
            return None

        # encoding = get_encoding_from_headers(self.headers)

        return data.decode("utf-8")

    async def json(self):
        """Return the full response body as parsed JSON."""
        data = await self.binary()
        if data is None:
            return None

        return py_json.loads(data.decode('utf-8'))

class HTTPConnection:
    """Maries an async socket with an HTTP handler."""

    def __init__(self, host: str, port: int, ssl):
        """
        :param host: The host to connect to.
        :param port: The port to connect to.
        :param ssl: Any SSL context to use.
        """
        self.host = host
        self.port = port
        self.ssl = ssl

        #: The current socket connection.
        self.sock = None  # type: io.Socket
        #: The H11 connection state.
        self.state = None  # type: h11.Connection

    def __repr__(self):
        return '%s(host=%r, port=%r)' % (
            self.__class__.__name__, self.host, self.port)

    async def open(self):
        """
        Opens a connection to the server.
        """
        sock_args = dict(
            host=self.host,
            port=self.port,
        )

        if self.ssl:
            sock_args.update(dict(
                ssl=self.ssl,
                server_hostname=self.host
            ))

        self.sock = await curio.open_connection(**sock_args)
        self.state = h11.Connection(our_role=h11.CLIENT)

        logger.debug('Opened %r', self)

    async def close(self):
        """
        Closes a connection to the server.
        """
        await self.sock.close()

        self.sock = None
        self.state = None

        logger.debug('Closed %r', self)

    async def _send(self, event: _EventBundle):
        """
        Sends an event to the server.

        :param event: The event to send.
        """
        # logger.debug("Sending event: %s", event)

        data = self.state.send(event)
        await self.sock.sendall(data)

    async def _next_event(self, maxsize: int=None) -> _EventBundle:
        """
        Gets the next event from the connection.

        This will automatically read data off of the connection.

        :param maxsize: The maximum size to read off the socket.
            If not specified, 2048 will be used.
        :return:
        """

        if not maxsize:
            maxsize = 2048

        while True:
            event = self.state.next_event()

            # logger.debug("Received event: %s", event)

            if event is h11.NEED_DATA:
                data = await self.sock.recv(maxsize)
                self.state.receive_data(data)
                continue

            return event

    async def request(self, h11_request: h11.Request, data=None) -> h11.Response:
        """
        Makes a request.

        :param h11_request: The :class:`h11.Request` object to use.
        :param data: Any request data to send to the server.
        """
        # First, send the base request.
        await self._send(h11_request)

        # Send data chunks.
        if data is not None:
            await self._send(h11.Data(data=data))

        # Finally, send the EOF.
        await self._send(h11.EndOfMessage())

        # Read the response off of the server now.
        event = await self._next_event()

        assert isinstance(event, h11.Response)

        return event

def _prepare_request(method: str, url: yarl.URL, *,
                     params=None, headers=None,
                     body=None,
                     json: dict = None, files: dict = None):
    """
    Prepares a new request, creating a :class:`h11.Request` object to be sent.
    Additionally, this also serializes the body into a bytestring ready to be sent.

    :param method: The method of the request.
    :param url: The URL to request.
    :param headers: Optional. The headers of the request.
        This should be a dict or a MultiDict.
    :param body: Optional. Any body data to send.
        If this is a dict, it is encoded as multipart/form-data.
    :param json: Optional. A dict that should be JSON encoded and sent as the body.
    """
    url = yarl.URL(url)

    if params:
        query_vars = list(url.query.items()) + list(params.items())
        url = url.with_query(query_vars)

    target = str(url.relative())

    if headers is None:
        headers = MultiDict()
    elif not isinstance(headers, MultiDict):
        headers = MultiDict(headers)

    headers.setdefault('Host', url.raw_host)

    if json is not None:
        body = py_json.dumps(json).encode()
    else:
        # Check if the body is a dict.
        # If so, send it as `multipart/form-data`.
        if isinstance(body, dict):
            body, _h = encode_multipart(body, files)
            headers.update(_h)

    if body is not None:
        if "Content-Length" not in headers:
            headers["Content-Length"] = str(len(body)).encode('utf-8')

    if json is not None:
        headers["Content-Type"] = b"application/json"

    if "Content-Length" not in headers:
        headers["Content-Length"] = b"0"

    if "User-Agent" not in headers:
        headers["User-Agent"] = "curio_http/{} curio/{}".format(__version__, curio.__version__)

    h11_request = h11.Request(
        method=method,
        target=target,
        headers=list(headers.items())
    )

    return h11_request, body

class ClientSession:
    """
    The session class that is used to send data to the server.
    """
    def __init__(self):
        #: A dictionary of all open connections.
        self.open_connections = []

        #: Any headers that are to be sent on every request.
        self.headers = MultiDict()

    async def __aenter__(self):
        return self

    async def __aexit__(self, exc_type, exc, tb):
        for conn in self.open_connections:
            await conn.close()

    async def _request(self, method, url, *args, **kwargs):
        """
        Makes a request to the server.

        This method is internal - use `request` instead.
        """
        headers = MultiDict(self.headers.copy())
        headers.extend(kwargs.get("headers", {}))
        kwargs["headers"] = headers

        h11_request, body = _prepare_request(
            method, url, *args, **kwargs)

        conn = HTTPConnection(
            host=url.raw_host,
            port=url.port,
            ssl=url.scheme == 'https',
        )

        await conn.open()

        self.open_connections.append(conn)

        raw_response = await conn.request(h11_request, body)

        return Response(raw_response, h11_request, conn)

    async def request(self, method: str, url: str, *args,
                      allow_redirects=False, **kwargs):
        """
        Performs a HTTP request.

        :param method: The HTTP method of the request.
        :param url: The URL of the request.
        :param allow_redirects: Should redirects be automatically followed?
        """
        url = yarl.URL(url)

        response = await self._request(method, url, *args, **kwargs)

        if allow_redirects:
            history = []

            while response.is_redirect:
                history.append(response)

                # Redirects can be relative.
                new_url = url.join(yarl.URL(response.headers['location']))

                response = await self._request(method, new_url)

            response.history = history

        return response

    def get(self, *args, allow_redirects=True, **kwargs):
        """Perform HTTP GET request."""
        return self.request(
            'GET', *args, allow_redirects=allow_redirects, **kwargs)

    def post(self, *args, data=None, **kwargs):
        """Perform HTTP POST request."""
        return self.request('POST', *args, data=data, **kwargs)

    def put(self, *args, **kwargs):
        return self.request("PUT", *args, **kwargs)

    def patch(self, *args, **kwargs):
        return self.request("PATCH", *args, **kwargs)

    def delete(self, *args, **kwargs):
        return self.request("DELETE", *args, **kwargs)
<commit_msg>HTTPClient now uses `ujson` if possible.<commit_after>"""
The HTTP module for Curious.

This code is 100% portable - it will work anywhere that h11 and multidict are also installed.
"""

import logging
import mimetypes
import random
import string
from functools import partial
try:
    import ujson as py_json
except ImportError:
    import json as py_json
import cgi

import multidict
import yarl
import h11
from curio import io
from h11._events import _EventBundle
from multidict import MultiDict
import curio

__version__ = "0.1.0-curious"

logger = logging.getLogger(__name__)

class HTTPError(Exception):
    def __init__(self, *args, **kwargs):
        self.response = kwargs.pop('response')
        super().__init__(*args, **kwargs)

_BOUNDARY_CHARS = string.digits + string.ascii_letters

def encode_multipart(fields, files, boundary=None):
    r"""Encode dict of form fields and dict of files as multipart/form-data.
    Return tuple of (body_string, headers_dict). Each value in files is a dict
    with required keys 'filename' and 'content', and optional 'mimetype' (if
    not specified, tries to guess mime type or uses 'application/octet-stream').

    >>> body, headers = encode_multipart({'FIELD': 'VALUE'},
    ...                                  {'FILE': {'filename': 'F.TXT', 'content': 'CONTENT'}},
    ...                                  boundary='BOUNDARY')
    >>> print('\n'.join(repr(l) for l in body.split('\r\n')))
    '--BOUNDARY'
    'Content-Disposition: form-data; name="FIELD"'
    ''
    'VALUE'
    '--BOUNDARY'
    'Content-Disposition: form-data; name="FILE"; filename="F.TXT"'
    'Content-Type: text/plain'
    ''
    'CONTENT'
    '--BOUNDARY--'
    ''
    >>> print(sorted(headers.items()))
    [('Content-Length', '193'), ('Content-Type', 'multipart/form-data; boundary=BOUNDARY')]
    >>> len(body)
    193

    Copied from: https://code.activestate.com/recipes/578668-encode-multipart-form-data-for-uploading-files-via/
    """

    def escape_quote(s):
        return s.replace(b'"', b'\\"')

    if boundary is None:
        boundary = b''.join(random.choice(_BOUNDARY_CHARS).encode() for i in range(30))
    lines = []

    for name, value in fields.items():
        if isinstance(name, str):
            name = name.encode()
        else:
            name = str(name).encode()

        if isinstance(value, str):
            value = value.encode()
        else:
            value = str(value).encode()

        lines.extend((
            b'--%s' % boundary,
            b'Content-Disposition: form-data; name="%s"' % escape_quote(name),
            b'',
            value,
        ))

    for name, value in files.items():
        filename = value['filename']
        if 'mimetype' in value:
            mimetype = value['mimetype']
        else:
            mimetype = mimetypes.guess_type(filename)[0] or 'application/octet-stream'
        lines.extend((
            b'--%s' % boundary,
            b'Content-Disposition: form-data; name="%s"; filename="%s"' % (
                escape_quote(name.encode()), escape_quote(filename.encode())),
            b'Content-Type: %s' % mimetype.encode(),
            b'',
            value['content'],
        ))

    lines.extend((
        b'--%s--' % boundary,
        b'',
    ))
    body = b'\r\n'.join(lines)

    headers = {
        'Content-Type': 'multipart/form-data; boundary=%s' % boundary.decode(),
        'Content-Length': str(len(body)),
    }

    return body, headers

def get_encoding_from_headers(headers: MultiDict):
    """
    Attempts to get the encoding of the request based on the headers.
    """
    content_type = headers.get('content-type')

    if not content_type:
        return None

    content_type, params = cgi.parse_header(content_type)

    if 'charset' in params:
        return params['charset'].strip("'\"")

    if 'text' in content_type:
        return 'ISO-8859-1'

class _EventIterator:
    """
    An iterator that allows you to receive events from a connection.
    """

    def __init__(self, event_source):
        self.event_source = event_source

    async def __aiter__(self):
        return self

    async def __anext__(self):
        event = await self.event_source()

        if type(event) is h11.Data:
            return event.data
        elif type(event) is h11.EndOfMessage:
            raise StopAsyncIteration
        else:
            raise ValueError('Unknown h11 event: %r', event)

class Response:
    """
    A wrapper around the Response from the server.
    """

    def __init__(self, raw_response: h11.Response, h11_request: h11.Request, conn: 'HTTPConnection'):
        self.status_code = raw_response.status_code
        self.http_version = raw_response.http_version.decode('utf-8')

        self.headers = multidict.CIMultiDict(
            (key.decode("utf-8"), value.decode("utf-8")) for key, value in raw_response.headers
        )

        self.h11_request = h11_request
        self.conn = conn

        self.history = None

    def __repr__(self):
        return '<Response [%s]>' % (self.status_code)

    @property
    def url(self):
        """The final URL that was requested."""
        return '{host}:{port}{target}'.format(
            host=self.conn.host,
            port=self.conn.port,
            target=self.h11_request.target.decode('utf-8'),
        )

    async def close(self):
        """
        Closes the current request.

        This should always be called.
        """
        await self.conn.close()

    def raise_for_status(self):
        """Raises HTTPError, if one occurred."""
        http_error_msg = ''

        if 400 <= self.status_code < 500:
            http_error_msg = '%s Client Error for url: %s' % (
                self.status_code, self.url)

        elif 500 <= self.status_code < 600:
            http_error_msg = '%s Server Error for url: %s' % (
                self.status_code, self.url)

        if http_error_msg:
            raise HTTPError(http_error_msg, response=self)

    @property
    def is_redirect(self):
        """Whether the response is a well-formed redirect."""
        return 'location' in self.headers and 301 <= self.status_code < 400

    def iter_chunked(self, maxsize=None):
        """Stream raw response body, maxsize bytes at a time."""
        return _EventIterator(partial(self.conn._next_event, maxsize))

    async def binary(self) -> bytes:
        """Return the full response body as a bytearray."""
        data = b""
        async for chunk in self.iter_chunked():
            data += chunk

        return data

    async def text(self):
        """Return the full response body as a string."""
        data = await self.binary()
        if data is None:
            return None

        # encoding = get_encoding_from_headers(self.headers)

        return data.decode("utf-8")

    async def json(self):
        """Return the full response body as parsed JSON."""
        data = await self.binary()
        if data is None:
            return None

        return py_json.loads(data.decode('utf-8'))

class HTTPConnection:
    """Maries an async socket with an HTTP handler."""

    def __init__(self, host: str, port: int, ssl):
        """
        :param host: The host to connect to.
        :param port: The port to connect to.
        :param ssl: Any SSL context to use.
        """
        self.host = host
        self.port = port
        self.ssl = ssl

        #: The current socket connection.
        self.sock = None  # type: io.Socket
        #: The H11 connection state.
        self.state = None  # type: h11.Connection

    def __repr__(self):
        return '%s(host=%r, port=%r)' % (
            self.__class__.__name__, self.host, self.port)

    async def open(self):
        """
        Opens a connection to the server.
        """
        sock_args = dict(
            host=self.host,
            port=self.port,
        )

        if self.ssl:
            sock_args.update(dict(
                ssl=self.ssl,
                server_hostname=self.host
            ))

        self.sock = await curio.open_connection(**sock_args)
        self.state = h11.Connection(our_role=h11.CLIENT)

        logger.debug('Opened %r', self)

    async def close(self):
        """
        Closes a connection to the server.
        """
        await self.sock.close()

        self.sock = None
        self.state = None

        logger.debug('Closed %r', self)

    async def _send(self, event: _EventBundle):
        """
        Sends an event to the server.

        :param event: The event to send.
        """
        # logger.debug("Sending event: %s", event)

        data = self.state.send(event)
        await self.sock.sendall(data)

    async def _next_event(self, maxsize: int=None) -> _EventBundle:
        """
        Gets the next event from the connection.

        This will automatically read data off of the connection.

        :param maxsize: The maximum size to read off the socket.
            If not specified, 2048 will be used.
        :return:
        """

        if not maxsize:
            maxsize = 2048

        while True:
            event = self.state.next_event()

            # logger.debug("Received event: %s", event)

            if event is h11.NEED_DATA:
                data = await self.sock.recv(maxsize)
                self.state.receive_data(data)
                continue

            return event

    async def request(self, h11_request: h11.Request, data=None) -> h11.Response:
        """
        Makes a request.

        :param h11_request: The :class:`h11.Request` object to use.
        :param data: Any request data to send to the server.
        """
        # First, send the base request.
        await self._send(h11_request)

        # Send data chunks.
        if data is not None:
            await self._send(h11.Data(data=data))

        # Finally, send the EOF.
        await self._send(h11.EndOfMessage())

        # Read the response off of the server now.
        event = await self._next_event()

        assert isinstance(event, h11.Response)

        return event

def _prepare_request(method: str, url: yarl.URL, *,
                     params=None, headers=None,
                     body=None,
                     json: dict = None, files: dict = None):
    """
    Prepares a new request, creating a :class:`h11.Request` object to be sent.
    Additionally, this also serializes the body into a bytestring ready to be sent.

    :param method: The method of the request.
    :param url: The URL to request.
    :param headers: Optional. The headers of the request.
        This should be a dict or a MultiDict.
    :param body: Optional. Any body data to send.
        If this is a dict, it is encoded as multipart/form-data.
    :param json: Optional. A dict that should be JSON encoded and sent as the body.
    """
    url = yarl.URL(url)

    if params:
        query_vars = list(url.query.items()) + list(params.items())
        url = url.with_query(query_vars)

    target = str(url.relative())

    if headers is None:
        headers = MultiDict()
    elif not isinstance(headers, MultiDict):
        headers = MultiDict(headers)

    headers.setdefault('Host', url.raw_host)

    if json is not None:
        body = py_json.dumps(json).encode()
    else:
        # Check if the body is a dict.
        # If so, send it as `multipart/form-data`.
        if isinstance(body, dict):
            body, _h = encode_multipart(body, files)
            headers.update(_h)

    if body is not None:
        if "Content-Length" not in headers:
            headers["Content-Length"] = str(len(body)).encode('utf-8')

    if json is not None:
        headers["Content-Type"] = b"application/json"

    if "Content-Length" not in headers:
        headers["Content-Length"] = b"0"

    if "User-Agent" not in headers:
        headers["User-Agent"] = "curio_http/{} curio/{}".format(__version__, curio.__version__)

    h11_request = h11.Request(
        method=method,
        target=target,
        headers=list(headers.items())
    )

    return h11_request, body

class ClientSession:
    """
    The session class that is used to send data to the server.
    """
    def __init__(self):
        #: A dictionary of all open connections.
        self.open_connections = []

        #: Any headers that are to be sent on every request.
        self.headers = MultiDict()

    async def __aenter__(self):
        return self

    async def __aexit__(self, exc_type, exc, tb):
        for conn in self.open_connections:
            await conn.close()

    async def _request(self, method, url, *args, **kwargs):
        """
        Makes a request to the server.

        This method is internal - use `request` instead.
        """
        headers = MultiDict(self.headers.copy())
        headers.extend(kwargs.get("headers", {}))
        kwargs["headers"] = headers

        h11_request, body = _prepare_request(
            method, url, *args, **kwargs)

        conn = HTTPConnection(
            host=url.raw_host,
            port=url.port,
            ssl=url.scheme == 'https',
        )

        await conn.open()

        self.open_connections.append(conn)

        raw_response = await conn.request(h11_request, body)

        return Response(raw_response, h11_request, conn)

    async def request(self, method: str, url: str, *args,
                      allow_redirects=False, **kwargs):
        """
        Performs a HTTP request.

        :param method: The HTTP method of the request.
        :param url: The URL of the request.
        :param allow_redirects: Should redirects be automatically followed?
        """
        url = yarl.URL(url)

        response = await self._request(method, url, *args, **kwargs)

        if allow_redirects:
            history = []

            while response.is_redirect:
                history.append(response)

                # Redirects can be relative.
                new_url = url.join(yarl.URL(response.headers['location']))

                response = await self._request(method, new_url)

            response.history = history

        return response

    def get(self, *args, allow_redirects=True, **kwargs):
        """Perform HTTP GET request."""
        return self.request(
            'GET', *args, allow_redirects=allow_redirects, **kwargs)

    def post(self, *args, **kwargs):
        """Perform HTTP POST request."""
        return self.request('POST', *args, **kwargs)

    def put(self, *args, **kwargs):
        return self.request("PUT", *args, **kwargs)

    def patch(self, *args, **kwargs):
        return self.request("PATCH", *args, **kwargs)

    def delete(self, *args, **kwargs):
        return self.request("DELETE", *args, **kwargs)
<|endoftext|>


<commit_before>from dtype import *

<commit_msg>Fix Python 3 import error<commit_after>from .dtype import *

<|endoftext|>


<commit_before>from protocol import shout, proto_page, query_page
from reputation import Reputation
from orders import Orders
import protocol
import sys
import json
import lookup
from pymongo import MongoClient
import logging
import pyelliptic
import pycountry
from ecdsa import SigningKey,SECP256k1
import random
from obelisk import bitcoin

class Market(object):

    def __init__(self, transport):

        self._log = logging.getLogger(self.__class__.__name__)
        self._log.info("Initializing")

        # for now we have the id in the transport
        self._myself = transport._myself
        self._peers = transport._peers
        self._transport = transport
        self.query_ident = None

        self.reputation = Reputation(self._transport)
        self.orders = Orders(self._transport)
        self.order_entries = self.orders._orders

        # TODO: Persistent storage of nicknames and pages
        self.nicks = {}
        self.pages = {}

        # Connect to database
        MONGODB_URI = 'mongodb://localhost:27017'
        _dbclient = MongoClient()
        self._db = _dbclient.openbazaar

        self.settings = self._db.settings.find_one()

        welcome = True

        if self.settings:
            if  'welcome' in self.settings.keys() and self.settings['welcome']:
                welcome = False

        # Register callbacks for incoming events
        transport.add_callback('query_myorders', self.on_query_myorders)
        transport.add_callback('peer', self.on_peer)
        transport.add_callback('query_page', self.on_query_page)
        transport.add_callback('page', self.on_page)
        transport.add_callback('negotiate_pubkey', self.on_negotiate_pubkey)
        transport.add_callback('proto_response_pubkey', self.on_response_pubkey)

        self.load_page(welcome)

    def generate_new_secret(self):

        key = bitcoin.EllipticCurveKey()
        key.new_key_pair()
        hexkey = key.secret.encode('hex')

        print 'Pubkey generate: ', key._public_key.pubkey

        self._db.settings.update({}, {"$set": {"secret":hexkey, "pubkey":bitcoin.GetPubKey(key._public_key.pubkey, False).encode('hex')}})

    def lookup(self, msg):

        if self.query_ident is None:
            self._log.info("Initializing identity query")
            self.query_ident = lookup.QueryIdent()

        nickname = str(msg["text"])
        key = self.query_ident.lookup(nickname)
        if key is None:
            self._log.info("Key not found for this nickname")
            return ("Key not found for this nickname", None)

        self._log.info("Found key: %s " % key.encode("hex"))
        if nickname in self._transport.nick_mapping:
            self._log.info("Already have a cached mapping, just adding key there.")
            response = {'nickname': nickname,
                        'pubkey': self._transport.nick_mapping[nickname][1].encode('hex'),
                        'signature': self._transport.nick_mapping[nickname][0].encode('hex'),
                        'type': 'response_pubkey',
                        'signature': 'unknown'}
            self._transport.nick_mapping[nickname][0] = key
            return (None, response)

        self._transport.nick_mapping[nickname] = [key, None]

        self._transport.send(protocol.negotiate_pubkey(nickname, key))

	# Load default information for your market from your file
    def load_page(self, welcome):

        #self._log.info("Loading market config from %s." % self.store_file)

        #with open(self.store_file) as f:
        #    data = json.loads(f.read())

        #self._log.info("Configuration data: " + json.dumps(data))

        #assert "desc" in data
        #nickname = data["nickname"]
        #desc = data["desc"]

        nickname = self.settings['nickname'] if self.settings.has_key("nickname") else ""
        storeDescription = self.settings['storeDescription'] if self.settings.has_key("storeDescription") else ""

        tagline = "%s: %s" % (nickname, storeDescription)
        self.mypage = tagline
        self.nickname = nickname
        self.signature = self._transport._myself.sign(tagline)

        if welcome:
            self._db.settings.update({}, {"$set":{"welcome":"noshow"}})
        else:
            self.welcome = False

        #self._log.info("Tagline signature: " + self.signature.encode("hex"))

    # SETTINGS

    def save_settings(self, msg):
        print "Settings to save",msg
        self._db.settings.update({}, msg, True)

    def get_settings(self):

        settings = self._db.settings.find_one()

        if settings:
            return { "bitmessage": settings['bitmessage'] if settings.has_key("bitmessage") else "",
                "email": settings['email'] if settings.has_key("email") else "",
                "PGPPubKey": settings['PGPPubKey'] if settings.has_key("PGPPubKey") else "",
                "pubkey": settings['pubkey'] if settings.has_key("pubkey") else "",
                "nickname": settings['nickname'] if settings.has_key("nickname") else "",
                "secret": settings['secret'] if settings.has_key("secret") else "",
                "welcome": settings['welcome'] if settings.has_key("welcome") else "",
                "escrowAddresses": settings['escrowAddresses'] if settings.has_key("escrowAddresses") else "",
                "storeDescription": settings['storeDescription'] if settings.has_key("storeDescription") else "",
                }

    # PAGE QUERYING

    def query_page(self, pubkey):
        self._transport.send(query_page(pubkey))

    def on_page(self, page):

        self._log.info("Page returned: " + str(page))

        pubkey = page.get('pubkey')
        page = page.get('text')

        if pubkey and page:
            self.pages[pubkey] = page

    # Return your page info if someone requests it on the network
    def on_query_page(self, peer):
        self._log.info("Someone is querying for your page")
        self._transport.send(proto_page(self._transport._myself.get_pubkey().encode('hex'),
                                        self.mypage, self.signature,
                                        self.nickname))

    def on_query_myorders(self, peer):
        self._log.info("Someone is querying for your page")
        self._transport.send(proto_page(self._transport._myself.get_pubkey(),
                                        self.mypage, self.signature,
                                        self.nickname))

    def on_peer(self, peer):
        pass

    def on_negotiate_pubkey(self, ident_pubkey):
        self._log.info("Someone is asking for your real pubKey")
        assert "nickname" in ident_pubkey
        assert "ident_pubkey" in ident_pubkey
        nickname = ident_pubkey['nickname']
        ident_pubkey = ident_pubkey['ident_pubkey'].decode("hex")
        self._transport.respond_pubkey_if_mine(nickname, ident_pubkey)

    def on_response_pubkey(self, response):
        self._log.info("got a pubkey!")
        assert "pubkey" in response
        assert "nickname" in response
        assert "signature" in response
        pubkey = response["pubkey"].decode("hex")
        signature = response["signature"].decode("hex")
        nickname = response["nickname"]
        # Cache mapping for later.
        if nickname not in self._transport.nick_mapping:
            self._transport.nick_mapping[nickname] = [None, pubkey]
        # Verify signature here...
        # Add to our dict.
        self._transport.nick_mapping[nickname][1] = pubkey
        self._log.info("[market] mappings: ###############")
        for k, v in self._transport.nick_mapping.iteritems():
            self._log.info("'%s' -> '%s' (%s)" % (
                k, v[1].encode("hex") if v[1] is not None else v[1],
                v[0].encode("hex") if v[0] is not None else v[0]))
        self._log.info("##################################")
<commit_msg>Return updated store info on demand<commit_after>from protocol import shout, proto_page, query_page
from reputation import Reputation
from orders import Orders
import protocol
import sys
import json
import lookup
from pymongo import MongoClient
import logging
import pyelliptic
import pycountry
from ecdsa import SigningKey,SECP256k1
import random
from obelisk import bitcoin

class Market(object):

    def __init__(self, transport):

        self._log = logging.getLogger(self.__class__.__name__)
        self._log.info("Initializing")

        # for now we have the id in the transport
        self._myself = transport._myself
        self._peers = transport._peers
        self._transport = transport
        self.query_ident = None

        self.reputation = Reputation(self._transport)
        self.orders = Orders(self._transport)
        self.order_entries = self.orders._orders

        # TODO: Persistent storage of nicknames and pages
        self.nicks = {}
        self.pages = {}

        # Connect to database
        MONGODB_URI = 'mongodb://localhost:27017'
        _dbclient = MongoClient()
        self._db = _dbclient.openbazaar

        self.settings = self._db.settings.find_one()

        welcome = True

        if self.settings:
            if  'welcome' in self.settings.keys() and self.settings['welcome']:
                welcome = False

        # Register callbacks for incoming events
        transport.add_callback('query_myorders', self.on_query_myorders)
        transport.add_callback('peer', self.on_peer)
        transport.add_callback('query_page', self.on_query_page)
        transport.add_callback('page', self.on_page)
        transport.add_callback('negotiate_pubkey', self.on_negotiate_pubkey)
        transport.add_callback('proto_response_pubkey', self.on_response_pubkey)

        self.load_page(welcome)

    def generate_new_secret(self):

        key = bitcoin.EllipticCurveKey()
        key.new_key_pair()
        hexkey = key.secret.encode('hex')

        print 'Pubkey generate: ', key._public_key.pubkey

        self._db.settings.update({}, {"$set": {"secret":hexkey, "pubkey":bitcoin.GetPubKey(key._public_key.pubkey, False).encode('hex')}})

    def lookup(self, msg):

        if self.query_ident is None:
            self._log.info("Initializing identity query")
            self.query_ident = lookup.QueryIdent()

        nickname = str(msg["text"])
        key = self.query_ident.lookup(nickname)
        if key is None:
            self._log.info("Key not found for this nickname")
            return ("Key not found for this nickname", None)

        self._log.info("Found key: %s " % key.encode("hex"))
        if nickname in self._transport.nick_mapping:
            self._log.info("Already have a cached mapping, just adding key there.")
            response = {'nickname': nickname,
                        'pubkey': self._transport.nick_mapping[nickname][1].encode('hex'),
                        'signature': self._transport.nick_mapping[nickname][0].encode('hex'),
                        'type': 'response_pubkey',
                        'signature': 'unknown'}
            self._transport.nick_mapping[nickname][0] = key
            return (None, response)

        self._transport.nick_mapping[nickname] = [key, None]

        self._transport.send(protocol.negotiate_pubkey(nickname, key))

	# Load default information for your market from your file
    def load_page(self, welcome):

        #self._log.info("Loading market config from %s." % self.store_file)

        #with open(self.store_file) as f:
        #    data = json.loads(f.read())

        #self._log.info("Configuration data: " + json.dumps(data))

        #assert "desc" in data
        #nickname = data["nickname"]
        #desc = data["desc"]

        nickname = self.settings['nickname'] if self.settings.has_key("nickname") else ""
        storeDescription = self.settings['storeDescription'] if self.settings.has_key("storeDescription") else ""

        tagline = "%s: %s" % (nickname, storeDescription)
        self.mypage = tagline
        self.nickname = nickname
        self.signature = self._transport._myself.sign(tagline)

        if welcome:
            self._db.settings.update({}, {"$set":{"welcome":"noshow"}})
        else:
            self.welcome = False

        #self._log.info("Tagline signature: " + self.signature.encode("hex"))

    # SETTINGS

    def save_settings(self, msg):
        print "Settings to save",msg
        self._db.settings.update({}, msg, True)

    def get_settings(self):

        settings = self._db.settings.find_one()

        if settings:
            return { "bitmessage": settings['bitmessage'] if settings.has_key("bitmessage") else "",
                "email": settings['email'] if settings.has_key("email") else "",
                "PGPPubKey": settings['PGPPubKey'] if settings.has_key("PGPPubKey") else "",
                "pubkey": settings['pubkey'] if settings.has_key("pubkey") else "",
                "nickname": settings['nickname'] if settings.has_key("nickname") else "",
                "secret": settings['secret'] if settings.has_key("secret") else "",
                "welcome": settings['welcome'] if settings.has_key("welcome") else "",
                "escrowAddresses": settings['escrowAddresses'] if settings.has_key("escrowAddresses") else "",
                "storeDescription": settings['storeDescription'] if settings.has_key("storeDescription") else "",
                }

    # PAGE QUERYING

    def query_page(self, pubkey):
        self._transport.send(query_page(pubkey))

    def on_page(self, page):

        self._log.info("Page returned: " + str(page))

        pubkey = page.get('pubkey')
        page = page.get('text')

        if pubkey and page:
            print page
            self.pages[pubkey] = page

    # Return your page info if someone requests it on the network
    def on_query_page(self, peer):
        self._log.info("Someone is querying for your page")
        self._transport.send(proto_page(self._transport._myself.get_pubkey().encode('hex'),
                                        self.settings['storeDescription'], self.signature,
                                        self.settings['nickname']))

    def on_query_myorders(self, peer):
        self._log.info("Someone is querying for your page")
        self._transport.send(proto_page(self._transport._myself.get_pubkey(),
                                        self.mypage, self.signature,
                                        self.nickname))

    def on_peer(self, peer):
        pass

    def on_negotiate_pubkey(self, ident_pubkey):
        self._log.info("Someone is asking for your real pubKey")
        assert "nickname" in ident_pubkey
        assert "ident_pubkey" in ident_pubkey
        nickname = ident_pubkey['nickname']
        ident_pubkey = ident_pubkey['ident_pubkey'].decode("hex")
        self._transport.respond_pubkey_if_mine(nickname, ident_pubkey)

    def on_response_pubkey(self, response):
        self._log.info("got a pubkey!")
        assert "pubkey" in response
        assert "nickname" in response
        assert "signature" in response
        pubkey = response["pubkey"].decode("hex")
        signature = response["signature"].decode("hex")
        nickname = response["nickname"]
        # Cache mapping for later.
        if nickname not in self._transport.nick_mapping:
            self._transport.nick_mapping[nickname] = [None, pubkey]
        # Verify signature here...
        # Add to our dict.
        self._transport.nick_mapping[nickname][1] = pubkey
        self._log.info("[market] mappings: ###############")
        for k, v in self._transport.nick_mapping.iteritems():
            self._log.info("'%s' -> '%s' (%s)" % (
                k, v[1].encode("hex") if v[1] is not None else v[1],
                v[0].encode("hex") if v[0] is not None else v[0]))
        self._log.info("##################################")
<|endoftext|>


<commit_before>import calendar
from datetime import datetime
from dateutil.relativedelta import relativedelta
from dateutil.rrule import *
from django.utils.translation import ugettext_noop
from corehq.apps.products.models import SQLProduct
from corehq.apps.programs.models import Program
from corehq.apps.reports.filters.base import BaseDrilldownOptionFilter, BaseSingleOptionFilter, \
    BaseMultipleOptionFilter, BaseReportFilter
from corehq.apps.reports.filters.fixtures import AsyncLocationFilter
from custom.common import ALL_OPTION
from corehq import Domain
from custom.ewsghana.utils import ews_date_format
import settings

class ProductByProgramFilter(BaseDrilldownOptionFilter):
    slug = "filter_by"
    single_option_select = 0
    template = "common/drilldown_options.html"
    label = ugettext_noop("Filter By")

    @property
    def drilldown_map(self):
        options = [{"val": ALL_OPTION, "text": "All", "next": []}]
        for program in Program.by_domain(self.domain):
            options.append({"val": program.get_id, "text": program.name})
        return options

    @classmethod
    def get_labels(cls):
        return [('Program', 'program')]

    @property
    def filter_context(self):
        controls = []
        for level, label in enumerate(self.rendered_labels):
            controls.append({
                'label': label[0],
                'slug': label[1],
                'level': level,
            })

        return {
            'option_map': self.drilldown_map,
            'controls': controls,
            'selected': self.selected,
            'use_last': self.use_only_last,
            'notifications': self.final_notifications,
            'empty_text': self.drilldown_empty_text,
            'is_empty': not self.drilldown_map,
            'single_option_select': self.single_option_select
        }

    @classmethod
    def _get_label_value(cls, request, label):
        slug = str(label[1])
        val = request.GET.getlist('%s_%s' % (cls.slug, str(label[1])))
        return {
            'slug': slug,
            'value': val,
        }

class ViewReportFilter(BaseSingleOptionFilter):
    default_text = ugettext_noop("Product Availability")
    slug = 'report_type'
    label = 'View'

    @property
    def options(self):
        return [
            ('stockouts', 'Stockouts'),
            ('asi', 'All Stock Information')
        ]

class ProductFilter(BaseSingleOptionFilter):
    slug = 'product_id'
    label = 'Product'
    default_text = ''

    @property
    def options(self):
        return SQLProduct.objects.filter(domain=self.domain, is_archived=False).values_list('product_id', 'name')\
            .order_by('name')

class MultiProductFilter(BaseMultipleOptionFilter):
    slug = 'product_id'
    label = 'Product'
    default_text = ''

    @property
    def options(self):
        return SQLProduct.objects.filter(domain=self.domain, is_archived=False).values_list('product_id', 'name')\
            .order_by('name')

class EWSLocationFilter(AsyncLocationFilter):
    template = "reports/filters/location_async.html"

    def reporting_types(self):
        return [
            location_type.name
            for location_type in Domain.get_by_name(self.domain).location_types
            if not location_type.administrative
        ]

    @property
    def filter_context(self):
        context = super(EWSLocationFilter, self).filter_context
        hierarchy = []
        for h in context['hierarchy']:
            if h[0] not in self.reporting_types():
                hierarchy.append(h)
        context['hierarchy'] = hierarchy

        return context

class EWSRestrictionLocationFilter(AsyncLocationFilter):
    template = "reports/filters/location_async.html"

class EWSDateFilter(BaseReportFilter):

    template = "ewsghana/datespan.html"
    slug = "datespan"
    label = "Filter By:"

    def selected(self, type):
        slug = '{0}_{1}'.format(self.slug, type)
        return self.request.GET.get(slug)

    @property
    def select_options(self):
        start_year = getattr(settings, 'START_YEAR', 2008)
        years = [dict(val=unicode(y), text=y) for y in range(start_year, datetime.utcnow().year + 1)]
        years.reverse()
        months = [dict(val="%02d" % m, text=calendar.month_name[m]) for m in range(1, 13)]
        now = datetime.now()
        three_month_earlier = now - relativedelta(months=3)

        first_friday = rrule(DAILY,
                             dtstart=datetime(three_month_earlier.year, three_month_earlier.month, 1),
                             until=now,
                             byweekday=FR)[0]
        if first_friday.day != 1:
            first_friday = first_friday - relativedelta(days=7)
        fridays = rrule(WEEKLY, dtstart=first_friday, until=now, byweekday=FR)
        weeks = []
        value = None
        text = None
        for idx, val in enumerate(fridays):
            try:
                value = '{0}|{1}'.format(val.strftime("%Y-%m-%d"), fridays[idx + 1].strftime("%Y-%m-%d"))
                text = '{0} - {1}'.format(ews_date_format(val), ews_date_format(fridays[idx + 1]))
            except IndexError:
                value = '{0}|{1}'.format(val.strftime("%Y-%m-%d"), now.strftime("%Y-%m-%d"))
                text = '{0} - {1}'.format(ews_date_format(val), ews_date_format(now))
            finally:
                weeks.append(dict(val=value, text=text))
        return [
            {
                'text': 'Week (Friday - Thursday)',
                'val': 2,
                'firstOptions': weeks[:-1],
                'secondOptions': []
            },
            {
                'text': 'Month',
                'val': 1,
                'firstOptions': months,
                'secondOptions': years
            }

        ]

    @property
    def default_week(self):
        now = datetime.utcnow()
        date = now - relativedelta(days=(7 - (4 - now.weekday())) % 7)
        return '{0}|{1}'.format((date - relativedelta(days=7)).strftime("%Y-%m-%d"), date.strftime("%Y-%m-%d"))

    @property
    def filter_context(self):
        return dict(
            select_options=self.select_options,
            selected_type=self.selected('type') if self.selected('type') else 2,
            selected_first=self.selected('first') if self.selected('first') else self.default_week,
            selected_second=self.selected('second') if self.selected('second') else ''
        )
<commit_msg>Display proper range in date filter<commit_after>import calendar
from datetime import datetime
from dateutil.relativedelta import relativedelta
from dateutil.rrule import *
from django.utils.translation import ugettext_noop
from corehq.apps.products.models import SQLProduct
from corehq.apps.programs.models import Program
from corehq.apps.reports.filters.base import BaseDrilldownOptionFilter, BaseSingleOptionFilter, \
    BaseMultipleOptionFilter, BaseReportFilter
from corehq.apps.reports.filters.fixtures import AsyncLocationFilter
from custom.common import ALL_OPTION
from corehq import Domain
from custom.ewsghana.utils import ews_date_format
import settings

class ProductByProgramFilter(BaseDrilldownOptionFilter):
    slug = "filter_by"
    single_option_select = 0
    template = "common/drilldown_options.html"
    label = ugettext_noop("Filter By")

    @property
    def drilldown_map(self):
        options = [{"val": ALL_OPTION, "text": "All", "next": []}]
        for program in Program.by_domain(self.domain):
            options.append({"val": program.get_id, "text": program.name})
        return options

    @classmethod
    def get_labels(cls):
        return [('Program', 'program')]

    @property
    def filter_context(self):
        controls = []
        for level, label in enumerate(self.rendered_labels):
            controls.append({
                'label': label[0],
                'slug': label[1],
                'level': level,
            })

        return {
            'option_map': self.drilldown_map,
            'controls': controls,
            'selected': self.selected,
            'use_last': self.use_only_last,
            'notifications': self.final_notifications,
            'empty_text': self.drilldown_empty_text,
            'is_empty': not self.drilldown_map,
            'single_option_select': self.single_option_select
        }

    @classmethod
    def _get_label_value(cls, request, label):
        slug = str(label[1])
        val = request.GET.getlist('%s_%s' % (cls.slug, str(label[1])))
        return {
            'slug': slug,
            'value': val,
        }

class ViewReportFilter(BaseSingleOptionFilter):
    default_text = ugettext_noop("Product Availability")
    slug = 'report_type'
    label = 'View'

    @property
    def options(self):
        return [
            ('stockouts', 'Stockouts'),
            ('asi', 'All Stock Information')
        ]

class ProductFilter(BaseSingleOptionFilter):
    slug = 'product_id'
    label = 'Product'
    default_text = ''

    @property
    def options(self):
        return SQLProduct.objects.filter(domain=self.domain, is_archived=False).values_list('product_id', 'name')\
            .order_by('name')

class MultiProductFilter(BaseMultipleOptionFilter):
    slug = 'product_id'
    label = 'Product'
    default_text = ''

    @property
    def options(self):
        return SQLProduct.objects.filter(domain=self.domain, is_archived=False).values_list('product_id', 'name')\
            .order_by('name')

class EWSLocationFilter(AsyncLocationFilter):
    template = "reports/filters/location_async.html"

    def reporting_types(self):
        return [
            location_type.name
            for location_type in Domain.get_by_name(self.domain).location_types
            if not location_type.administrative
        ]

    @property
    def filter_context(self):
        context = super(EWSLocationFilter, self).filter_context
        hierarchy = []
        for h in context['hierarchy']:
            if h[0] not in self.reporting_types():
                hierarchy.append(h)
        context['hierarchy'] = hierarchy

        return context

class EWSRestrictionLocationFilter(AsyncLocationFilter):
    template = "reports/filters/location_async.html"

class EWSDateFilter(BaseReportFilter):

    template = "ewsghana/datespan.html"
    slug = "datespan"
    label = "Filter By:"

    def selected(self, type):
        slug = '{0}_{1}'.format(self.slug, type)
        return self.request.GET.get(slug)

    @property
    def select_options(self):
        start_year = getattr(settings, 'START_YEAR', 2008)
        years = [dict(val=unicode(y), text=y) for y in range(start_year, datetime.utcnow().year + 1)]
        years.reverse()
        months = [dict(val="%02d" % m, text=calendar.month_name[m]) for m in range(1, 13)]
        now = datetime.now()
        three_month_earlier = now - relativedelta(months=3)

        first_friday = rrule(DAILY,
                             dtstart=datetime(three_month_earlier.year, three_month_earlier.month, 1),
                             until=now,
                             byweekday=FR)[0]
        if first_friday.day != 1:
            first_friday = first_friday - relativedelta(days=7)
        fridays = rrule(WEEKLY, dtstart=first_friday, until=now, byweekday=FR)
        weeks = []
        value = None
        text = None
        for idx, val in enumerate(fridays):
            try:
                value = '{0}|{1}'.format(val.strftime("%Y-%m-%d"), fridays[idx + 1].strftime("%Y-%m-%d"))
                text = '{0} - {1}'.format(
                    ews_date_format(val),
                    ews_date_format(fridays[idx + 1] - relativedelta(days=1))
                )
            except IndexError:
                value = '{0}|{1}'.format(val.strftime("%Y-%m-%d"), now.strftime("%Y-%m-%d"))
                text = '{0} - {1}'.format(ews_date_format(val), ews_date_format(now))
            finally:
                weeks.append(dict(val=value, text=text))
        return [
            {
                'text': 'Week (Friday - Thursday)',
                'val': 2,
                'firstOptions': weeks[:-1],
                'secondOptions': []
            },
            {
                'text': 'Month',
                'val': 1,
                'firstOptions': months,
                'secondOptions': years
            }

        ]

    @property
    def default_week(self):
        now = datetime.utcnow()
        date = now - relativedelta(days=(7 - (4 - now.weekday())) % 7)
        return '{0}|{1}'.format((date - relativedelta(days=7)).strftime("%Y-%m-%d"), date.strftime("%Y-%m-%d"))

    @property
    def filter_context(self):
        return dict(
            select_options=self.select_options,
            selected_type=self.selected('type') if self.selected('type') else 2,
            selected_first=self.selected('first') if self.selected('first') else self.default_week,
            selected_second=self.selected('second') if self.selected('second') else ''
        )
<|endoftext|>


<commit_before>#-------------------------------------------------------------------------------
# Name  	    	: AddFields.py
# ArcGIS Version	: ArcGIS 10.1 sp1
# Script Version	: 20130225
# Name of Company 	: Environmental System Research Institute
# Author        	: ESRI raster solution team
# Date          	: 16-09-2012
# Purpose 	    	: A component to create custom fields in mosaic datasets.
# Created	    	: 14-08-2012
# LastUpdated  		: 17-09-2012
# Required Argument 	: Not applicable
# Optional Argument 	: Not applicable
# Usage         	: Object of this class should be instantiated.
# Copyright	    	: (c) ESRI 2012
# License	    	: <your license>
#-------------------------------------------------------------------------------

import arcpy,os,sys
from xml.dom import minidom

import Base

class AddFields(Base.Base):
    geodatabase_ext = '.gdb'

    gdbName = ''
    gdbNameExt = ''
    doc = None
    workspace = ''
    gdbPath =  ''
    m_MD = ''
    fieldNameList = []
    fieldTypeList = []
    fieldLengthList = []
    m_base = None

    def __init__(self, base=None):
        if (base != None):
            self.setLog(base.m_log)
            self.workspace = base.m_workspace
            self.gdbNameExt = base.m_geodatabase
            self.m_MD = base.m_md

        self.m_base = base

    def CreateFields(self):

        self.log("Adding custom fields:", self.const_general_text)

        self.log("Using mosaic dataset:" + self.m_MD, self.const_general_text)
        try:
            mdPath = os.path.join(self.gdbPath, self.m_MD)
            if not arcpy.Exists(mdPath):
                self.log("Mosaic dataset is not found.", self.const_warning_text)
                return False

            self.log("\tCreating fields:", self.const_general_text)
            for j in range(len(self.fieldNameList)):
                self.log("\t\t" + self.fieldNameList[j], self.const_general_text)
                fieldExist = arcpy.ListFields(mdPath,self.fieldNameList[j])
                if len(fieldExist) == 0:
                    arcpy.AddField_management(mdPath,self.fieldNameList[j],self.fieldTypeList[j],"","",self.fieldLengthList[j])
##                    workspace = self.m_base.m_workspace
##                    geodatabase = self.m_base.m_geodatabase
##                    if (workspace == ''):
##                        workspace = self.workspace
##                    if (geodatabase == ''):
##                        geodatabase = self.gdbName
##                    lock_path_  = os.path.join(workspace, geodatabase + '.gdb')
##                    result_code_  = self.waitForLockRelease(lock_path_)
##                    if (result_code_ == -1):
##                        return False
##                    elif(result_code_ == -2):
##                        return False

        except:
            self.log("Error: " + arcpy.GetMessages(), self.const_critical_text)
            return False

        return True

    def init(self, config):

        try:
            self.doc = minidom.parse(config)
        except:
            self.log("Error: reading input config file:" + sys.argv[1] + "\nQuitting...", self.const_critical_text)
            return False

        # Step (1)
        #workspace/location on filesystem where the .gdb is created.
        if (self.workspace == ''):
            self.workspace = self.prefixFolderPath(self.getXMLNodeValue(self.doc, "WorkspacePath"), self.const_workspace_path_)

        if (self.gdbNameExt == ''):
            self.gdbNameExt =  self.getXMLNodeValue(self.doc, "Geodatabase")

        const_len_ext = 4
        if (self.gdbNameExt[-const_len_ext:].lower() != self.geodatabase_ext):
            self.gdbNameExt += '.gdb'

        self.gdbName = self.gdbNameExt[:len(self.gdbNameExt) - const_len_ext]       #.gdb

        Nodelist = self.doc.getElementsByTagName("MosaicDataset")
        if (Nodelist.length == 0):
            self.log("\nError: MosaicDataset node not found! Invalid schema.", self.const_critical_text)
            return False

        try:
            for node in Nodelist[0].childNodes:
                  node =  node.nextSibling
                  if (node != None and node.nodeType == minidom.Node.ELEMENT_NODE):
                    if (node.nodeName == 'Name'):
                        try:
                            if (self.m_MD == ''):
                                self.m_MD = node.firstChild.nodeValue
                            break
                        except:
                            Error = True
        except:
            self.log("\nError: reading MosaicDataset nodes.", self.const_critical_text)
            return False

        Nodelist = self.doc.getElementsByTagName("Fields")
        if (Nodelist.length == 0):
            self.log("Error: Fields node not found! Invalid schema.", self.const_critical_text)
            return False

        try:
            for node in Nodelist[0].childNodes:
              if (node.nodeType == minidom.Node.ELEMENT_NODE):
                for n in node.childNodes:
                    if(n.nodeType == minidom.Node.ELEMENT_NODE):
                        nodeName = n.nodeName.upper()
                        if (nodeName == 'NAME'):
                            self.fieldNameList.append(n.firstChild.nodeValue)
                        elif(nodeName == 'TYPE'):
                            self.fieldTypeList.append(n.firstChild.nodeValue)
                        elif(nodeName == 'LENGTH'):
                            try:
                                self.fieldLengthList.append(n.firstChild.nodeValue)
                            except:
                                self.fieldLengthList.append('')
        except:
            self.log("\nError: Reading fields information!", self.const_critical_text)
            return False

        fields_len = len(self.fieldNameList)
        if (len(self.fieldTypeList) != fields_len or len(self.fieldLengthList) != fields_len):
            self.log("\nError: Number of Field(Name, Type, Len) do not match!", self.const_critical_text)
            return False

        self.gdbPath = os.path.join(self.workspace, self.gdbNameExt)

        return True

<commit_msg>Update AddFields.py<commit_after>#-------------------------------------------------------------------------------
# Name  	    	: AddFields.py
# ArcGIS Version	: ArcGIS 10.1 sp1
# Script Version	: 20130225
# Name of Company 	: Environmental System Research Institute
# Author        	: ESRI raster solution team
# Date          	: 16-09-2012
# Purpose 	    	: A component to create custom fields in mosaic datasets.
# Created	    	: 14-08-2012
# LastUpdated  		: 17-09-2012
# Required Argument 	: Not applicable.
# Optional Argument 	: Not applicable.
# Usage         	: Object of this class should be instantiated.
# Copyright	    	: (c) ESRI 2012
# License	    	: <your license>
#-------------------------------------------------------------------------------

import arcpy,os,sys
from xml.dom import minidom

import Base

class AddFields(Base.Base):
    geodatabase_ext = '.gdb'

    gdbName = ''
    gdbNameExt = ''
    doc = None
    workspace = ''
    gdbPath =  ''
    m_MD = ''
    fieldNameList = []
    fieldTypeList = []
    fieldLengthList = []
    m_base = None

    def __init__(self, base=None):
        if (base != None):
            self.setLog(base.m_log)
            self.workspace = base.m_workspace
            self.gdbNameExt = base.m_geodatabase
            self.m_MD = base.m_md

        self.m_base = base

    def CreateFields(self):

        self.log("Adding custom fields:", self.const_general_text)

        self.log("Using mosaic dataset:" + self.m_MD, self.const_general_text)
        try:
            mdPath = os.path.join(self.gdbPath, self.m_MD)
            if not arcpy.Exists(mdPath):
                self.log("Mosaic dataset is not found.", self.const_warning_text)
                return False

            self.log("\tCreating fields:", self.const_general_text)
            for j in range(len(self.fieldNameList)):
                self.log("\t\t" + self.fieldNameList[j], self.const_general_text)
                fieldExist = arcpy.ListFields(mdPath,self.fieldNameList[j])
                if len(fieldExist) == 0:
                    arcpy.AddField_management(mdPath,self.fieldNameList[j],self.fieldTypeList[j],"","",self.fieldLengthList[j])
##                    workspace = self.m_base.m_workspace
##                    geodatabase = self.m_base.m_geodatabase
##                    if (workspace == ''):
##                        workspace = self.workspace
##                    if (geodatabase == ''):
##                        geodatabase = self.gdbName
##                    lock_path_  = os.path.join(workspace, geodatabase + '.gdb')
##                    result_code_  = self.waitForLockRelease(lock_path_)
##                    if (result_code_ == -1):
##                        return False
##                    elif(result_code_ == -2):
##                        return False

        except:
            self.log("Error: " + arcpy.GetMessages(), self.const_critical_text)
            return False

        return True

    def init(self, config):

        try:
            self.doc = minidom.parse(config)
        except:
            self.log("Error: reading input config file:" + sys.argv[1] + "\nQuitting...", self.const_critical_text)
            return False

        # Step (1)
        #workspace/location on filesystem where the .gdb is created.
        if (self.workspace == ''):
            self.workspace = self.prefixFolderPath(self.getXMLNodeValue(self.doc, "WorkspacePath"), self.const_workspace_path_)

        if (self.gdbNameExt == ''):
            self.gdbNameExt =  self.getXMLNodeValue(self.doc, "Geodatabase")

        const_len_ext = 4
        if (self.gdbNameExt[-const_len_ext:].lower() != self.geodatabase_ext):
            self.gdbNameExt += '.gdb'

        self.gdbName = self.gdbNameExt[:len(self.gdbNameExt) - const_len_ext]       #.gdb

        Nodelist = self.doc.getElementsByTagName("MosaicDataset")
        if (Nodelist.length == 0):
            self.log("\nError: MosaicDataset node not found! Invalid schema.", self.const_critical_text)
            return False

        try:
            for node in Nodelist[0].childNodes:
                  node =  node.nextSibling
                  if (node != None and node.nodeType == minidom.Node.ELEMENT_NODE):
                    if (node.nodeName == 'Name'):
                        try:
                            if (self.m_MD == ''):
                                self.m_MD = node.firstChild.nodeValue
                            break
                        except:
                            Error = True
        except:
            self.log("\nError: reading MosaicDataset nodes.", self.const_critical_text)
            return False

        Nodelist = self.doc.getElementsByTagName("Fields")
        if (Nodelist.length == 0):
            self.log("Error: Fields node not found! Invalid schema.", self.const_critical_text)
            return False

        try:
            for node in Nodelist[0].childNodes:
              if (node.nodeType == minidom.Node.ELEMENT_NODE):
                for n in node.childNodes:
                    if(n.nodeType == minidom.Node.ELEMENT_NODE):
                        nodeName = n.nodeName.upper()
                        if (nodeName == 'NAME'):
                            self.fieldNameList.append(n.firstChild.nodeValue)
                        elif(nodeName == 'TYPE'):
                            self.fieldTypeList.append(n.firstChild.nodeValue)
                        elif(nodeName == 'LENGTH'):
                            try:
                                self.fieldLengthList.append(n.firstChild.nodeValue)
                            except:
                                self.fieldLengthList.append('')
        except:
            self.log("\nError: Reading fields information!", self.const_critical_text)
            return False

        fields_len = len(self.fieldNameList)
        if (len(self.fieldTypeList) != fields_len or len(self.fieldLengthList) != fields_len):
            self.log("\nError: Number of Field(Name, Type, Len) do not match!", self.const_critical_text)
            return False

        self.gdbPath = os.path.join(self.workspace, self.gdbNameExt)

        return True

<|endoftext|>