<commit_before>import numpy as np

import cupy
from cupyx.scipy.ndimage import _util

def _get_output_fourier(output, input, complex_only=False):
    types = [cupy.complex64, cupy.complex128]
    if not complex_only:
        types += [cupy.float32, cupy.float64]

    if output is None:
        if input.dtype in types:
            output = cupy.zeros(input.shape, dtype=input.dtype)
        else:
            output = cupy.zeros(input.shape, dtype=types[-1])
    elif type(output) is type:
        if output not in types:
            raise RuntimeError("output type not supported")
        output = cupy.zeros(input.shape, dtype=output)
    elif output.shape != input.shape:
        raise RuntimeError("output shape not correct")
    return output

def _reshape_nd(arr, ndim, axis):
    """Promote a 1d array to ndim with non-singleton size along axis."""
    nd_shape = (1,) * axis + (arr.size,) + (1,) * (ndim - axis - 1)
    return arr.reshape(nd_shape)

def fourier_gaussian(input, sigma, n=-1, axis=-1, output=None):
    """Multidimensional Gaussian shift filter.

    The array is multiplied with the Fourier transform of a (separable)
    Gaussian kernel.

    Args:
        input (cupy.ndarray): The input array.
        sigma (float or sequence of float):  The sigma of the Gaussian kernel.
            If a float, `sigma` is the same for all axes. If a sequence,
            `sigma` has to contain one value for each axis.
        n (int, optional):  If `n` is negative (default), then the input is
            assumed to be the result of a complex fft. If `n` is larger than or
            equal to zero, the input is assumed to be the result of a real fft,
            and `n` gives the length of the array before transformation along
            the real transform direction.
        axis (int, optional): The axis of the real transform (only used when
            ``n > -1``).
        output (cupy.ndarray, optional):
            If given, the result of shifting the input is placed in this array.

    Returns:
        output (cupy.ndarray): The filtered output.
    """
    ndim = input.ndim
    output = _get_output_fourier(output, input)
    axis = cupy.util._normalize_axis_index(axis, ndim)
    sigmas = _util._fix_sequence_arg(sigma, ndim, 'sigma')

    output[...] = input
    for ax, (sigmak, ax_size) in enumerate(zip(sigmas, output.shape)):

        # compute the frequency grid in Hz
        if ax == axis and n > 0:
            arr = cupy.arange(ax_size, dtype=output.real.dtype)
            arr /= n
        else:
            arr = cupy.fft.fftfreq(ax_size)
        arr = arr.astype(output.real.dtype, copy=False)

        # compute the Gaussian weights
        arr *= arr
        scale = sigmak * sigmak / -2
        arr *= (4 * np.pi * np.pi) * scale
        cupy.exp(arr, out=arr)

        # reshape for broadcasting
        arr = _reshape_nd(arr, ndim=ndim, axis=ax)
        output *= arr

    return output

def fourier_uniform(input, size, n=-1, axis=-1, output=None):
    """Multidimensional uniform shift filter.

    The array is multiplied with the Fourier transform of a box of given size.

    Args:
        input (cupy.ndarray): The input array.
        size (float or sequence of float):  The sigma of the box used for
            filtering. If a float, `size` is the same for all axes. If a
            sequence, `size` has to contain one value for each axis.
        n (int, optional):  If `n` is negative (default), then the input is
            assumed to be the result of a complex fft. If `n` is larger than or
            equal to zero, the input is assumed to be the result of a real fft,
            and `n` gives the length of the array before transformation along
            the real transform direction.
        axis (int, optional): The axis of the real transform (only used when
            ``n > -1``).
        output (cupy.ndarray, optional):
            If given, the result of shifting the input is placed in this array.

    Returns:
        output (cupy.ndarray): The filtered output.
    """
    ndim = input.ndim
    output = _get_output_fourier(output, input)
    axis = cupy.util._normalize_axis_index(axis, ndim)
    sizes = _util._fix_sequence_arg(size, ndim, 'size')

    output[...] = input
    for ax, (size, ax_size) in enumerate(zip(sizes, output.shape)):

        # compute the frequency grid in Hz
        if ax == axis and n > 0:
            arr = cupy.arange(ax_size, dtype=output.real.dtype)
            arr /= n
        else:
            arr = cupy.fft.fftfreq(ax_size)
        arr = arr.astype(output.real.dtype, copy=False)

        # compute the uniform filter weights
        arr *= size
        cupy.sinc(arr, out=arr)

        # reshape for broadcasting
        arr = _reshape_nd(arr, ndim=ndim, axis=ax)
        output *= arr

    return output

def fourier_shift(input, shift, n=-1, axis=-1, output=None):
    """Multidimensional Fourier shift filter.

    The array is multiplied with the Fourier transform of a shift operation.

    Args:
        input (cupy.ndarray): The input array. This should be in the Fourier
            domain.
        shift (float or sequence of float):  The size of shift. If a float,
            `shift` is the same for all axes. If a sequence, `shift` has to
            contain one value for each axis.
        n (int, optional):  If `n` is negative (default), then the input is
            assumed to be the result of a complex fft. If `n` is larger than or
            equal to zero, the input is assumed to be the result of a real fft,
            and `n` gives the length of the array before transformation along
            the real transform direction.
        axis (int, optional): The axis of the real transform (only used when
            ``n > -1``).
        output (cupy.ndarray, optional):
            If given, the result of shifting the input is placed in this array.

    Returns:
        output (cupy.ndarray): The shifted output (in the Fourier domain).
    """
    ndim = input.ndim
    output = _get_output_fourier(output, input, complex_only=True)
    axis = cupy.util._normalize_axis_index(axis, ndim)
    shifts = _util._fix_sequence_arg(shift, ndim, 'shift')

    output[...] = input
    for ax, (shiftk, ax_size) in enumerate(zip(shifts, output.shape)):
        if shiftk == 0:
            continue
        if ax == axis and n > 0:
            # cp.fft.rfftfreq(ax_size) * (-2j * np.pi * shiftk *  ax_size / n)
            arr = cupy.arange(ax_size, dtype=output.dtype)
            arr *= -2j * np.pi * shiftk / n
        else:
            arr = cupy.fft.fftfreq(ax_size)
            arr = arr * (-2j * np.pi * shiftk)
        cupy.exp(arr, out=arr)

        # reshape for broadcasting
        arr = _reshape_nd(arr, ndim=ndim, axis=ax)
        output *= arr

    return output
<commit_msg>STYLE: np->numpy<commit_after>import numpy

import cupy
from cupyx.scipy.ndimage import _util

def _get_output_fourier(output, input, complex_only=False):
    types = [cupy.complex64, cupy.complex128]
    if not complex_only:
        types += [cupy.float32, cupy.float64]

    if output is None:
        if input.dtype in types:
            output = cupy.zeros(input.shape, dtype=input.dtype)
        else:
            output = cupy.zeros(input.shape, dtype=types[-1])
    elif type(output) is type:
        if output not in types:
            raise RuntimeError("output type not supported")
        output = cupy.zeros(input.shape, dtype=output)
    elif output.shape != input.shape:
        raise RuntimeError("output shape not correct")
    return output

def _reshape_nd(arr, ndim, axis):
    """Promote a 1d array to ndim with non-singleton size along axis."""
    nd_shape = (1,) * axis + (arr.size,) + (1,) * (ndim - axis - 1)
    return arr.reshape(nd_shape)

def fourier_gaussian(input, sigma, n=-1, axis=-1, output=None):
    """Multidimensional Gaussian shift filter.

    The array is multiplied with the Fourier transform of a (separable)
    Gaussian kernel.

    Args:
        input (cupy.ndarray): The input array.
        sigma (float or sequence of float):  The sigma of the Gaussian kernel.
            If a float, `sigma` is the same for all axes. If a sequence,
            `sigma` has to contain one value for each axis.
        n (int, optional):  If `n` is negative (default), then the input is
            assumed to be the result of a complex fft. If `n` is larger than or
            equal to zero, the input is assumed to be the result of a real fft,
            and `n` gives the length of the array before transformation along
            the real transform direction.
        axis (int, optional): The axis of the real transform (only used when
            ``n > -1``).
        output (cupy.ndarray, optional):
            If given, the result of shifting the input is placed in this array.

    Returns:
        output (cupy.ndarray): The filtered output.
    """
    ndim = input.ndim
    output = _get_output_fourier(output, input)
    axis = cupy.util._normalize_axis_index(axis, ndim)
    sigmas = _util._fix_sequence_arg(sigma, ndim, 'sigma')

    output[...] = input
    for ax, (sigmak, ax_size) in enumerate(zip(sigmas, output.shape)):

        # compute the frequency grid in Hz
        if ax == axis and n > 0:
            arr = cupy.arange(ax_size, dtype=output.real.dtype)
            arr /= n
        else:
            arr = cupy.fft.fftfreq(ax_size)
        arr = arr.astype(output.real.dtype, copy=False)

        # compute the Gaussian weights
        arr *= arr
        scale = sigmak * sigmak / -2
        arr *= (4 * numpy.pi * numpy.pi) * scale
        cupy.exp(arr, out=arr)

        # reshape for broadcasting
        arr = _reshape_nd(arr, ndim=ndim, axis=ax)
        output *= arr

    return output

def fourier_uniform(input, size, n=-1, axis=-1, output=None):
    """Multidimensional uniform shift filter.

    The array is multiplied with the Fourier transform of a box of given size.

    Args:
        input (cupy.ndarray): The input array.
        size (float or sequence of float):  The sigma of the box used for
            filtering. If a float, `size` is the same for all axes. If a
            sequence, `size` has to contain one value for each axis.
        n (int, optional):  If `n` is negative (default), then the input is
            assumed to be the result of a complex fft. If `n` is larger than or
            equal to zero, the input is assumed to be the result of a real fft,
            and `n` gives the length of the array before transformation along
            the real transform direction.
        axis (int, optional): The axis of the real transform (only used when
            ``n > -1``).
        output (cupy.ndarray, optional):
            If given, the result of shifting the input is placed in this array.

    Returns:
        output (cupy.ndarray): The filtered output.
    """
    ndim = input.ndim
    output = _get_output_fourier(output, input)
    axis = cupy.util._normalize_axis_index(axis, ndim)
    sizes = _util._fix_sequence_arg(size, ndim, 'size')

    output[...] = input
    for ax, (size, ax_size) in enumerate(zip(sizes, output.shape)):

        # compute the frequency grid in Hz
        if ax == axis and n > 0:
            arr = cupy.arange(ax_size, dtype=output.real.dtype)
            arr /= n
        else:
            arr = cupy.fft.fftfreq(ax_size)
        arr = arr.astype(output.real.dtype, copy=False)

        # compute the uniform filter weights
        arr *= size
        cupy.sinc(arr, out=arr)

        # reshape for broadcasting
        arr = _reshape_nd(arr, ndim=ndim, axis=ax)
        output *= arr

    return output

def fourier_shift(input, shift, n=-1, axis=-1, output=None):
    """Multidimensional Fourier shift filter.

    The array is multiplied with the Fourier transform of a shift operation.

    Args:
        input (cupy.ndarray): The input array. This should be in the Fourier
            domain.
        shift (float or sequence of float):  The size of shift. If a float,
            `shift` is the same for all axes. If a sequence, `shift` has to
            contain one value for each axis.
        n (int, optional):  If `n` is negative (default), then the input is
            assumed to be the result of a complex fft. If `n` is larger than or
            equal to zero, the input is assumed to be the result of a real fft,
            and `n` gives the length of the array before transformation along
            the real transform direction.
        axis (int, optional): The axis of the real transform (only used when
            ``n > -1``).
        output (cupy.ndarray, optional):
            If given, the result of shifting the input is placed in this array.

    Returns:
        output (cupy.ndarray): The shifted output (in the Fourier domain).
    """
    ndim = input.ndim
    output = _get_output_fourier(output, input, complex_only=True)
    axis = cupy.util._normalize_axis_index(axis, ndim)
    shifts = _util._fix_sequence_arg(shift, ndim, 'shift')

    output[...] = input
    for ax, (shiftk, ax_size) in enumerate(zip(shifts, output.shape)):
        if shiftk == 0:
            continue
        if ax == axis and n > 0:
            # cp.fft.rfftfreq(ax_size) * (-2j * numpy.pi * shiftk *  ax_size/n)
            arr = cupy.arange(ax_size, dtype=output.dtype)
            arr *= -2j * numpy.pi * shiftk / n
        else:
            arr = cupy.fft.fftfreq(ax_size)
            arr = arr * (-2j * numpy.pi * shiftk)
        cupy.exp(arr, out=arr)

        # reshape for broadcasting
        arr = _reshape_nd(arr, ndim=ndim, axis=ax)
        output *= arr

    return output
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Conservation views."""
from import_export import resources

from conservation import models as cons_models

class ConservationThreatResource(resources.ModelResource):
    """ConservationThreat resource class for CSV download."""

    class Meta:
        """Class opts."""

        model = cons_models.ConservationThreat
        fields = [
            "taxon_list",
            "com_list",
            "document__title",
            "occurrence_area_code",
            "category__label",
            "encountered_by__name",
            "encountered_on",
            "area_affected_percent",
            "current_impact,"
            "potential_impact",
            "potential_onset",
        ]

class ConservationActionResource(resources.ModelResource):
    """ConservationAction resource class for CSV download."""

    class Meta:
        """Class opts."""

        model = cons_models.ConservationAction
        fields = [
            "taxon_list",
            "com_list",
            "document__title",
            "occurrence_area_code",
            "category__label",
            "instructions",
            "implementation_notes",
            "completion_date",
            "expenditure",
            "status"
        ]
<commit_msg>Cons threat: resource typo<commit_after># -*- coding: utf-8 -*-
"""Conservation views."""
from import_export import resources

from conservation import models as cons_models

class ConservationThreatResource(resources.ModelResource):
    """ConservationThreat resource class for CSV download."""

    class Meta:
        """Class opts."""

        model = cons_models.ConservationThreat
        fields = [
            "taxon_list",
            "com_list",
            "document__title",
            "occurrence_area_code",
            "category__label",
            "encountered_by__name",
            "encountered_on",
            "area_affected_percent",
            "current_impact",
            "potential_impact",
            "potential_onset",
        ]

class ConservationActionResource(resources.ModelResource):
    """ConservationAction resource class for CSV download."""

    class Meta:
        """Class opts."""

        model = cons_models.ConservationAction
        fields = [
            "taxon_list",
            "com_list",
            "document__title",
            "occurrence_area_code",
            "category__label",
            "instructions",
            "implementation_notes",
            "completion_date",
            "expenditure",
            "status"
        ]
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
CUDA Library Installer

Installs the latest CUDA library supported by CuPy.
"""

# This script will also be used as a standalone script when building wheels.
# Keep the script runnable without CuPy dependency.

import argparse
import json
import os
import platform
import shutil
import sys
import tempfile
import urllib.request

_cudnn_records = []
_cutensor_records = []
_nccl_records = []
library_records = {}

def _make_cudnn_url(public_version, filename):
    # https://developer.download.nvidia.com/compute/redist/cudnn/v8.0.2/cudnn-11.0-linux-x64-v8.0.2.39.tgz
    return (
        'https://developer.download.nvidia.com/compute/redist/cudnn' +
        '/v{}/{}'.format(public_version, filename))

def _make_cudnn_record(
        cuda_version, public_version, filename_linux, filename_windows):
    major_version = public_version.split('.')[0]

    if major_version == '7':
        suffix_list = ['']
    elif major_version == '8':
        # Dependency order is documented at:
        # https://docs.nvidia.com/deeplearning/cudnn/api/index.html
        suffix_list = ['', '_ops_infer', '_ops_train',
                       '_cnn_infer', '_cnn_train',
                       '_adv_infer', '_adv_train']
    else:
        raise AssertionError

    return {
        'cuda': cuda_version,
        'cudnn': public_version,
        'assets': {
            'Linux': {
                'url': _make_cudnn_url(public_version, filename_linux),
                'filenames': [f'libcudnn{suffix}.so.{public_version}'
                              for suffix in suffix_list]
            },
            'Windows': {
                'url': _make_cudnn_url(public_version, filename_windows),
                'filenames': [f'cudnn{suffix}64_{major_version}.dll'
                              for suffix in suffix_list]
            },
        }
    }

# Latest cuDNN versions: https://developer.nvidia.com/rdp/cudnn-download
_cudnn_records.append(_make_cudnn_record(
    '11.4', '8.2.4',
    'cudnn-11.4-linux-x64-v8.2.4.15.tgz',
    'cudnn-11.4-windows-x64-v8.2.4.15.zip'))
_cudnn_records.append(_make_cudnn_record(
    '11.3', '8.2.4',
    'cudnn-11.4-linux-x64-v8.2.4.15.tgz',
    'cudnn-11.4-windows-x64-v8.2.4.15.zip'))
_cudnn_records.append(_make_cudnn_record(
    '11.2', '8.2.4',
    'cudnn-11.4-linux-x64-v8.2.4.15.tgz',
    'cudnn-11.4-windows-x64-v8.2.4.15.zip'))
_cudnn_records.append(_make_cudnn_record(
    '11.1', '8.2.4',
    'cudnn-11.4-linux-x64-v8.2.4.15.tgz',
    'cudnn-11.4-windows-x64-v8.2.4.15.zip'))
_cudnn_records.append(_make_cudnn_record(
    '11.0', '8.2.4',
    'cudnn-11.4-linux-x64-v8.2.4.15.tgz',
    'cudnn-11.4-windows-x64-v8.2.4.15.zip'))
_cudnn_records.append(_make_cudnn_record(
    '10.2', '8.2.4',
    'cudnn-10.2-linux-x64-v8.2.4.15.tgz',
    'cudnn-10.2-windows10-x64-v8.2.4.15.zip'))
library_records['cudnn'] = _cudnn_records

def _make_cutensor_url(public_version, platform, filename):
    if public_version == '1.2.2':  # v10.1
        # https://developer.download.nvidia.com/compute/cutensor/1.2.2/local_installers/libcutensor-linux-x86_64-1.2.2.5.tar.gz
        return (
            'https://developer.download.nvidia.com/compute/cutensor/' +
            f'{public_version}/local_installers/{filename}')
    else:
        # https://developer.download.nvidia.com/compute/cutensor/redist/libcutensor/linux-x86_64/libcutensor-linux-x86_64-1.3.3.2-archive.tar.xz
        return (
            'https://developer.download.nvidia.com/compute/cutensor/' +
            f'redist/libcutensor/{platform}-x86_64/{filename}')

def _make_cutensor_record(
        cuda_version, public_version, filename_linux, filename_windows):
    return {
        'cuda': cuda_version,
        'cutensor': public_version,
        'assets': {
            'Linux': {
                'url': _make_cutensor_url(
                    public_version, 'linux', filename_linux),
                'filenames': ['libcutensor.so.{}'.format(public_version)],
            },
            'Windows': {
                'url': _make_cutensor_url(
                    public_version, 'windows', filename_windows),
                'filenames': ['cutensor.dll'],
            },
        }
    }

_cutensor_records.append(_make_cutensor_record(
    '11.4', '1.3.3',
    'libcutensor-linux-x86_64-1.3.3.2-archive.tar.xz',
    'libcutensor-windows-x86_64-1.3.3.2-archive.zip'))
_cutensor_records.append(_make_cutensor_record(
    '11.3', '1.3.3',
    'libcutensor-linux-x86_64-1.3.3.2-archive.tar.xz',
    'libcutensor-windows-x86_64-1.3.3.2-archive.zip'))
_cutensor_records.append(_make_cutensor_record(
    '11.2', '1.3.3',
    'libcutensor-linux-x86_64-1.3.3.2-archive.tar.xz',
    'libcutensor-windows-x86_64-1.3.3.2-archive.zip'))
_cutensor_records.append(_make_cutensor_record(
    '11.1', '1.3.3',
    'libcutensor-linux-x86_64-1.3.3.2-archive.tar.xz',
    'libcutensor-windows-x86_64-1.3.3.2-archive.zip'))
_cutensor_records.append(_make_cutensor_record(
    '11.0', '1.3.3',
    'libcutensor-linux-x86_64-1.3.3.2-archive.tar.xz',
    'libcutensor-windows-x86_64-1.3.3.2-archive.zip'))
_cutensor_records.append(_make_cutensor_record(
    '10.2', '1.3.3',
    'libcutensor-linux-x86_64-1.3.3.2-archive.tar.xz',
    'libcutensor-windows-x86_64-1.3.3.2-archive.zip'))
library_records['cutensor'] = _cutensor_records

def _make_nccl_url(public_version, filename):
    # https://developer.download.nvidia.com/compute/redist/nccl/v2.8/nccl_2.8.4-1+cuda11.2_x86_64.txz
    return (
        'https://developer.download.nvidia.com/compute/redist/nccl/' +
        'v{}/{}'.format(public_version, filename))

def _make_nccl_record(
        cuda_version, full_version, public_version, filename_linux):
    return {
        'cuda': cuda_version,
        'nccl': full_version,
        'assets': {
            'Linux': {
                'url': _make_nccl_url(public_version, filename_linux),
                'filenames': ['libnccl.so.{}'.format(full_version)],
            },
        },
    }

_nccl_records.append(_make_nccl_record(
    '11.4', '2.11.4', '2.11',
    'nccl_2.11.4-1+cuda11.4_x86_64.txz'))
_nccl_records.append(_make_nccl_record(
    '11.3', '2.9.9', '2.9',
    'nccl_2.9.9-1+cuda11.3_x86_64.txz'))
_nccl_records.append(_make_nccl_record(
    '11.2', '2.8.4', '2.8',
    'nccl_2.8.4-1+cuda11.2_x86_64.txz'))
_nccl_records.append(_make_nccl_record(
    '11.1', '2.8.4', '2.8',
    'nccl_2.8.4-1+cuda11.1_x86_64.txz'))
_nccl_records.append(_make_nccl_record(
    '11.0', '2.11.4', '2.11',
    'nccl_2.11.4-1+cuda11.0_x86_64.txz'))
_nccl_records.append(_make_nccl_record(
    '10.2', '2.11.4', '2.11',
    'nccl_2.11.4-1+cuda10.2_x86_64.txz'))
library_records['nccl'] = _nccl_records

def install_lib(cuda, prefix, library):
    record = None
    lib_records = library_records
    for record in lib_records[library]:
        if record['cuda'] == cuda:
            break
    else:
        raise RuntimeError('''
The CUDA version specified is not supported.
Should be one of {}.'''.format(str([x['cuda'] for x in lib_records[library]])))
    if prefix is None:
        prefix = os.path.expanduser('~/.cupy/cuda_lib')
    destination = calculate_destination(prefix, cuda, library, record[library])

    if os.path.exists(destination):
        raise RuntimeError('''
The destination directory {} already exists.
Remove the directory first if you want to reinstall.'''.format(destination))

    target_platform = platform.system()
    asset = record['assets'].get(target_platform, None)
    if asset is None:
        raise RuntimeError('''
The current platform ({}) is not supported.'''.format(target_platform))

    print('Installing {} {} for CUDA {} to: {}'.format(
        library, record[library], record['cuda'], destination))

    url = asset['url']
    print('Downloading {}...'.format(url))
    with tempfile.TemporaryDirectory() as tmpdir:
        with open(os.path.join(tmpdir, os.path.basename(url)), 'wb') as f:
            with urllib.request.urlopen(url) as response:
                f.write(response.read())
        print('Extracting...')
        outdir = os.path.join(tmpdir, 'extract')
        shutil.unpack_archive(f.name, outdir)
        print('Installing...')
        if library == 'cudnn':
            shutil.move(os.path.join(outdir, 'cuda'), destination)
        elif library == 'cutensor':
            if cuda.startswith('11.') and cuda != '11.0':
                cuda = '11'
            if cuda == '10.1':  # v1.2.2
                dir_name = 'libcutensor'
                license = 'license.pdf'
            else:  # v1.3.3
                dir_name = os.path.basename(url)[:-7]  # remove '.tar.xz'
                license = 'LICENSE'
            shutil.move(
                os.path.join(outdir, dir_name, 'include'),
                os.path.join(destination, 'include'))
            shutil.move(
                os.path.join(outdir, dir_name, 'lib', cuda),
                os.path.join(destination, 'lib'))
            shutil.move(
                os.path.join(outdir, dir_name, license), destination)
        elif library == 'nccl':
            subdir = os.listdir(outdir)  # ['nccl_2.8.4-1+cuda11.2_x86_64']
            assert len(subdir) == 1
            shutil.move(os.path.join(outdir, subdir[0]), destination)
        else:
            assert False
        print('Cleaning up...')
    print('Done!')

def calculate_destination(prefix, cuda, lib, lib_ver):
    """Calculates the installation directory.

    ~/.cupy/cuda_lib/{cuda_version}/{library_name}/{library_version}
    """
    return os.path.join(prefix, cuda, lib, lib_ver)

def main(args):
    parser = argparse.ArgumentParser()

    # TODO(kmaehashi): support NCCL
    parser.add_argument('--library',
                        choices=['cudnn', 'cutensor', 'nccl'],
                        required=True,
                        help='Library to install')
    parser.add_argument('--cuda', type=str, required=True,
                        help='CUDA version')
    parser.add_argument('--prefix', type=str, default=None,
                        help='Install destination')
    parser.add_argument('--action', choices=['install', 'dump'],
                        default='install',
                        help='Action to perform')
    params = parser.parse_args(args)

    if params.prefix is not None:
        params.prefix = os.path.abspath(params.prefix)

    if params.action == 'install':
        install_lib(params.cuda, params.prefix, params.library)
    elif params.action == 'dump':
        print(json.dumps(library_records[params.library], indent=4))
    else:
        assert False

if __name__ == '__main__':
    main(sys.argv[1:])
<commit_msg>Remove code for CUDA 10.1<commit_after>#!/usr/bin/env python

"""
CUDA Library Installer

Installs the latest CUDA library supported by CuPy.
"""

# This script will also be used as a standalone script when building wheels.
# Keep the script runnable without CuPy dependency.

import argparse
import json
import os
import platform
import shutil
import sys
import tempfile
import urllib.request

_cudnn_records = []
_cutensor_records = []
_nccl_records = []
library_records = {}

def _make_cudnn_url(public_version, filename):
    # https://developer.download.nvidia.com/compute/redist/cudnn/v8.0.2/cudnn-11.0-linux-x64-v8.0.2.39.tgz
    return (
        'https://developer.download.nvidia.com/compute/redist/cudnn' +
        '/v{}/{}'.format(public_version, filename))

def _make_cudnn_record(
        cuda_version, public_version, filename_linux, filename_windows):
    major_version = public_version.split('.')[0]

    if major_version == '7':
        suffix_list = ['']
    elif major_version == '8':
        # Dependency order is documented at:
        # https://docs.nvidia.com/deeplearning/cudnn/api/index.html
        suffix_list = ['', '_ops_infer', '_ops_train',
                       '_cnn_infer', '_cnn_train',
                       '_adv_infer', '_adv_train']
    else:
        raise AssertionError

    return {
        'cuda': cuda_version,
        'cudnn': public_version,
        'assets': {
            'Linux': {
                'url': _make_cudnn_url(public_version, filename_linux),
                'filenames': [f'libcudnn{suffix}.so.{public_version}'
                              for suffix in suffix_list]
            },
            'Windows': {
                'url': _make_cudnn_url(public_version, filename_windows),
                'filenames': [f'cudnn{suffix}64_{major_version}.dll'
                              for suffix in suffix_list]
            },
        }
    }

# Latest cuDNN versions: https://developer.nvidia.com/rdp/cudnn-download
_cudnn_records.append(_make_cudnn_record(
    '11.4', '8.2.4',
    'cudnn-11.4-linux-x64-v8.2.4.15.tgz',
    'cudnn-11.4-windows-x64-v8.2.4.15.zip'))
_cudnn_records.append(_make_cudnn_record(
    '11.3', '8.2.4',
    'cudnn-11.4-linux-x64-v8.2.4.15.tgz',
    'cudnn-11.4-windows-x64-v8.2.4.15.zip'))
_cudnn_records.append(_make_cudnn_record(
    '11.2', '8.2.4',
    'cudnn-11.4-linux-x64-v8.2.4.15.tgz',
    'cudnn-11.4-windows-x64-v8.2.4.15.zip'))
_cudnn_records.append(_make_cudnn_record(
    '11.1', '8.2.4',
    'cudnn-11.4-linux-x64-v8.2.4.15.tgz',
    'cudnn-11.4-windows-x64-v8.2.4.15.zip'))
_cudnn_records.append(_make_cudnn_record(
    '11.0', '8.2.4',
    'cudnn-11.4-linux-x64-v8.2.4.15.tgz',
    'cudnn-11.4-windows-x64-v8.2.4.15.zip'))
_cudnn_records.append(_make_cudnn_record(
    '10.2', '8.2.4',
    'cudnn-10.2-linux-x64-v8.2.4.15.tgz',
    'cudnn-10.2-windows10-x64-v8.2.4.15.zip'))
library_records['cudnn'] = _cudnn_records

def _make_cutensor_url(platform, filename):
    # https://developer.download.nvidia.com/compute/cutensor/redist/libcutensor/linux-x86_64/libcutensor-linux-x86_64-1.3.3.2-archive.tar.xz
    return (
        'https://developer.download.nvidia.com/compute/cutensor/' +
        f'redist/libcutensor/{platform}-x86_64/{filename}')

def _make_cutensor_record(
        cuda_version, public_version, filename_linux, filename_windows):
    return {
        'cuda': cuda_version,
        'cutensor': public_version,
        'assets': {
            'Linux': {
                'url': _make_cutensor_url('linux', filename_linux),
                'filenames': ['libcutensor.so.{}'.format(public_version)],
            },
            'Windows': {
                'url': _make_cutensor_url('windows', filename_windows),
                'filenames': ['cutensor.dll'],
            },
        }
    }

_cutensor_records.append(_make_cutensor_record(
    '11.4', '1.3.3',
    'libcutensor-linux-x86_64-1.3.3.2-archive.tar.xz',
    'libcutensor-windows-x86_64-1.3.3.2-archive.zip'))
_cutensor_records.append(_make_cutensor_record(
    '11.3', '1.3.3',
    'libcutensor-linux-x86_64-1.3.3.2-archive.tar.xz',
    'libcutensor-windows-x86_64-1.3.3.2-archive.zip'))
_cutensor_records.append(_make_cutensor_record(
    '11.2', '1.3.3',
    'libcutensor-linux-x86_64-1.3.3.2-archive.tar.xz',
    'libcutensor-windows-x86_64-1.3.3.2-archive.zip'))
_cutensor_records.append(_make_cutensor_record(
    '11.1', '1.3.3',
    'libcutensor-linux-x86_64-1.3.3.2-archive.tar.xz',
    'libcutensor-windows-x86_64-1.3.3.2-archive.zip'))
_cutensor_records.append(_make_cutensor_record(
    '11.0', '1.3.3',
    'libcutensor-linux-x86_64-1.3.3.2-archive.tar.xz',
    'libcutensor-windows-x86_64-1.3.3.2-archive.zip'))
_cutensor_records.append(_make_cutensor_record(
    '10.2', '1.3.3',
    'libcutensor-linux-x86_64-1.3.3.2-archive.tar.xz',
    'libcutensor-windows-x86_64-1.3.3.2-archive.zip'))
library_records['cutensor'] = _cutensor_records

def _make_nccl_url(public_version, filename):
    # https://developer.download.nvidia.com/compute/redist/nccl/v2.8/nccl_2.8.4-1+cuda11.2_x86_64.txz
    return (
        'https://developer.download.nvidia.com/compute/redist/nccl/' +
        'v{}/{}'.format(public_version, filename))

def _make_nccl_record(
        cuda_version, full_version, public_version, filename_linux):
    return {
        'cuda': cuda_version,
        'nccl': full_version,
        'assets': {
            'Linux': {
                'url': _make_nccl_url(public_version, filename_linux),
                'filenames': ['libnccl.so.{}'.format(full_version)],
            },
        },
    }

_nccl_records.append(_make_nccl_record(
    '11.4', '2.11.4', '2.11',
    'nccl_2.11.4-1+cuda11.4_x86_64.txz'))
_nccl_records.append(_make_nccl_record(
    '11.3', '2.9.9', '2.9',
    'nccl_2.9.9-1+cuda11.3_x86_64.txz'))
_nccl_records.append(_make_nccl_record(
    '11.2', '2.8.4', '2.8',
    'nccl_2.8.4-1+cuda11.2_x86_64.txz'))
_nccl_records.append(_make_nccl_record(
    '11.1', '2.8.4', '2.8',
    'nccl_2.8.4-1+cuda11.1_x86_64.txz'))
_nccl_records.append(_make_nccl_record(
    '11.0', '2.11.4', '2.11',
    'nccl_2.11.4-1+cuda11.0_x86_64.txz'))
_nccl_records.append(_make_nccl_record(
    '10.2', '2.11.4', '2.11',
    'nccl_2.11.4-1+cuda10.2_x86_64.txz'))
library_records['nccl'] = _nccl_records

def install_lib(cuda, prefix, library):
    record = None
    lib_records = library_records
    for record in lib_records[library]:
        if record['cuda'] == cuda:
            break
    else:
        raise RuntimeError('''
The CUDA version specified is not supported.
Should be one of {}.'''.format(str([x['cuda'] for x in lib_records[library]])))
    if prefix is None:
        prefix = os.path.expanduser('~/.cupy/cuda_lib')
    destination = calculate_destination(prefix, cuda, library, record[library])

    if os.path.exists(destination):
        raise RuntimeError('''
The destination directory {} already exists.
Remove the directory first if you want to reinstall.'''.format(destination))

    target_platform = platform.system()
    asset = record['assets'].get(target_platform, None)
    if asset is None:
        raise RuntimeError('''
The current platform ({}) is not supported.'''.format(target_platform))

    print('Installing {} {} for CUDA {} to: {}'.format(
        library, record[library], record['cuda'], destination))

    url = asset['url']
    print('Downloading {}...'.format(url))
    with tempfile.TemporaryDirectory() as tmpdir:
        with open(os.path.join(tmpdir, os.path.basename(url)), 'wb') as f:
            with urllib.request.urlopen(url) as response:
                f.write(response.read())
        print('Extracting...')
        outdir = os.path.join(tmpdir, 'extract')
        shutil.unpack_archive(f.name, outdir)
        print('Installing...')
        if library == 'cudnn':
            shutil.move(os.path.join(outdir, 'cuda'), destination)
        elif library == 'cutensor':
            if cuda.startswith('11.') and cuda != '11.0':
                cuda = '11'
            dir_name = os.path.basename(url)[:-7]  # remove '.tar.xz'
            license = 'LICENSE'
            shutil.move(
                os.path.join(outdir, dir_name, 'include'),
                os.path.join(destination, 'include'))
            shutil.move(
                os.path.join(outdir, dir_name, 'lib', cuda),
                os.path.join(destination, 'lib'))
            shutil.move(
                os.path.join(outdir, dir_name, license), destination)
        elif library == 'nccl':
            subdir = os.listdir(outdir)  # ['nccl_2.8.4-1+cuda11.2_x86_64']
            assert len(subdir) == 1
            shutil.move(os.path.join(outdir, subdir[0]), destination)
        else:
            assert False
        print('Cleaning up...')
    print('Done!')

def calculate_destination(prefix, cuda, lib, lib_ver):
    """Calculates the installation directory.

    ~/.cupy/cuda_lib/{cuda_version}/{library_name}/{library_version}
    """
    return os.path.join(prefix, cuda, lib, lib_ver)

def main(args):
    parser = argparse.ArgumentParser()

    # TODO(kmaehashi): support NCCL
    parser.add_argument('--library',
                        choices=['cudnn', 'cutensor', 'nccl'],
                        required=True,
                        help='Library to install')
    parser.add_argument('--cuda', type=str, required=True,
                        help='CUDA version')
    parser.add_argument('--prefix', type=str, default=None,
                        help='Install destination')
    parser.add_argument('--action', choices=['install', 'dump'],
                        default='install',
                        help='Action to perform')
    params = parser.parse_args(args)

    if params.prefix is not None:
        params.prefix = os.path.abspath(params.prefix)

    if params.action == 'install':
        install_lib(params.cuda, params.prefix, params.library)
    elif params.action == 'dump':
        print(json.dumps(library_records[params.library], indent=4))
    else:
        assert False

if __name__ == '__main__':
    main(sys.argv[1:])
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
CUDA Library Installer

Installs the latest CUDA library supported by CuPy.
"""

# This script will also be used as a standalone script when building wheels.
# Keep the script runnable without CuPy dependency.

import argparse
import json
import os
import platform
import shutil
import sys
import tempfile
import urllib.request

_cudnn_records = []
_cutensor_records = []
_nccl_records = []
library_records = {}

def _make_cudnn_url(public_version, filename):
    # https://developer.download.nvidia.com/compute/redist/cudnn/v8.0.2/cudnn-11.0-linux-x64-v8.0.2.39.tgz
    return (
        'https://developer.download.nvidia.com/compute/redist/cudnn' +
        '/v{}/{}'.format(public_version, filename))

def _make_cudnn_record(
        cuda_version, public_version, filename_linux, filename_windows):
    major_version = public_version.split('.')[0]
    return {
        'cuda': cuda_version,
        'cudnn': public_version,
        'assets': {
            'Linux': {
                'url': _make_cudnn_url(public_version, filename_linux),
                'filename': 'libcudnn.so.{}'.format(public_version),
            },
            'Windows': {
                'url': _make_cudnn_url(public_version, filename_windows),
                'filename': 'cudnn64_{}.dll'.format(major_version),
            },
        }
    }

# Latest cuDNN versions: https://developer.nvidia.com/rdp/cudnn-download
_cudnn_records.append(_make_cudnn_record(
    '11.2', '8.1.1',
    'cudnn-11.2-linux-x64-v8.1.1.33.tgz',
    'cudnn-11.2-windows-x64-v8.1.1.33.zip'))
_cudnn_records.append(_make_cudnn_record(
    '11.1', '8.1.1',
    'cudnn-11.2-linux-x64-v8.1.1.33.tgz',
    'cudnn-11.2-windows-x64-v8.1.1.33.zip'))
_cudnn_records.append(_make_cudnn_record(
    '11.0', '8.1.1',
    'cudnn-11.2-linux-x64-v8.1.1.33.tgz',
    'cudnn-11.2-windows-x64-v8.1.1.33.zip'))
_cudnn_records.append(_make_cudnn_record(
    '10.2', '8.1.1',
    'cudnn-10.2-linux-x64-v8.1.1.33.tgz',
    'cudnn-10.2-windows10-x64-v8.1.1.33.zip'))
_cudnn_records.append(_make_cudnn_record(
    '10.1', '8.0.5',
    'cudnn-10.1-linux-x64-v8.0.5.39.tgz',
    'cudnn-10.1-windows10-x64-v8.0.5.39.zip'))
_cudnn_records.append(_make_cudnn_record(
    '10.0', '7.6.5',
    'cudnn-10.0-linux-x64-v7.6.5.32.tgz',
    'cudnn-10.0-windows10-x64-v7.6.5.32.zip'))
_cudnn_records.append(_make_cudnn_record(
    '9.2', '7.6.5',
    'cudnn-9.2-linux-x64-v7.6.5.32.tgz',
    'cudnn-9.2-windows10-x64-v7.6.5.32.zip'))
library_records['cudnn'] = _cudnn_records

def _make_cutensor_url(public_version, filename):
    # https://developer.download.nvidia.com/compute/cutensor/1.2.2/local_installers/libcutensor-linux-x86_64-1.2.2.5.tar.gz
    return (
        'https://developer.download.nvidia.com/compute/cutensor/' +
        '{}/local_installers/{}'.format(public_version, filename))

def _make_cutensor_record(
        cuda_version, public_version, filename_linux, filename_windows):
    return {
        'cuda': cuda_version,
        'cutensor': public_version,
        'assets': {
            'Linux': {
                'url': _make_cutensor_url(public_version, filename_linux),
                'filename': 'libcutensor.so.{}'.format(public_version),
            },
            'Windows': {
                'url': _make_cutensor_url(public_version, filename_windows),
                'filename': 'cutensor.dll',
            },
        }
    }

_cutensor_records.append(_make_cutensor_record(
    '11.2', '1.2.2',
    'libcutensor-linux-x86_64-1.2.2.5.tar.gz',
    'libcutensor-windows-x86_64-1.2.2.5.zip'))
_cutensor_records.append(_make_cutensor_record(
    '11.1', '1.2.2',
    'libcutensor-linux-x86_64-1.2.2.5.tar.gz',
    'libcutensor-windows-x86_64-1.2.2.5.zip'))
_cutensor_records.append(_make_cutensor_record(
    '11.0', '1.2.2',
    'libcutensor-linux-x86_64-1.2.2.5.tar.gz',
    'libcutensor-windows-x86_64-1.2.2.5.zip'))
_cutensor_records.append(_make_cutensor_record(
    '10.2', '1.2.2',
    'libcutensor-linux-x86_64-1.2.2.5.tar.gz',
    'libcutensor-windows-x86_64-1.2.2.5.zip'))
_cutensor_records.append(_make_cutensor_record(
    '10.1', '1.2.2',
    'libcutensor-linux-x86_64-1.2.2.5.tar.gz',
    'libcutensor-windows-x86_64-1.2.2.5.zip'))
library_records['cutensor'] = _cutensor_records

def _make_nccl_url(public_version, filename):
    # https://developer.download.nvidia.com/compute/redist/nccl/v2.8/nccl_2.8.4-1+cuda11.2_x86_64.txz
    return (
        'https://developer.download.nvidia.com/compute/redist/nccl/' +
        'v{}/{}'.format(public_version, filename))

def _make_nccl_record(
        cuda_version, full_version, public_version, filename_linux):
    return {
        'cuda': cuda_version,
        'nccl': full_version,
        'assets': {
            'Linux': {
                'url': _make_nccl_url(public_version, filename_linux),
                'filename': 'libnccl.so.{}'.format(full_version),
            },
        },
    }

_nccl_records.append(_make_nccl_record(
    '11.2', '2.8.4', '2.8',
    'nccl_2.8.4-1+cuda11.2_x86_64.txz'))
_nccl_records.append(_make_nccl_record(
    '11.1', '2.8.4', '2.8',
    'nccl_2.8.4-1+cuda11.1_x86_64.txz'))
_nccl_records.append(_make_nccl_record(
    '11.0', '2.8.4', '2.8',
    'nccl_2.8.4-1+cuda11.0_x86_64.txz'))
_nccl_records.append(_make_nccl_record(
    '10.2', '2.8.4', '2.8',
    'nccl_2.8.4-1+cuda10.2_x86_64.txz'))
_nccl_records.append(_make_nccl_record(
    '10.1', '2.8.3', '2.8',
    'nccl_2.8.3-1+cuda10.1_x86_64.txz'))
_nccl_records.append(_make_nccl_record(
    '10.0', '2.6.4', '2.6',
    'nccl_2.6.4-1+cuda10.0_x86_64.txz'))
_nccl_records.append(_make_nccl_record(
    '9.2', '2.4.8', '2.4',
    'nccl_2.4.8-1+cuda9.2_x86_64.txz'))
library_records['nccl'] = _nccl_records

def install_lib(cuda, prefix, library):
    record = None
    lib_records = library_records
    for record in lib_records[library]:
        if record['cuda'] == cuda:
            break
    else:
        raise RuntimeError('''
The CUDA version specified is not supported.
Should be one of {}.'''.format(str([x['cuda'] for x in lib_records[library]])))
    if prefix is None:
        prefix = os.path.expanduser('~/.cupy/cuda_lib')
    destination = calculate_destination(prefix, cuda, library, record[library])

    if os.path.exists(destination):
        raise RuntimeError('''
The destination directory {} already exists.
Remove the directory first if you want to reinstall.'''.format(destination))

    target_platform = platform.system()
    asset = record['assets'].get(target_platform, None)
    if asset is None:
        raise RuntimeError('''
The current platform ({}) is not supported.'''.format(target_platform))

    print('Installing {} {} for CUDA {} to: {}'.format(
        library, record[library], record['cuda'], destination))

    url = asset['url']
    print('Downloading {}...'.format(url))
    with tempfile.TemporaryDirectory() as tmpdir:
        with open(os.path.join(tmpdir, os.path.basename(url)), 'wb') as f:
            with urllib.request.urlopen(url) as response:
                f.write(response.read())
        print('Extracting...')
        outdir = os.path.join(tmpdir, 'extract')
        shutil.unpack_archive(f.name, outdir)
        print('Installing...')
        if library == 'cudnn':
            shutil.move(os.path.join(outdir, 'cuda'), destination)
        elif library == 'cutensor':
            if cuda.startswith('11.'):
                cuda = '11'
            shutil.move(
                os.path.join(outdir, 'libcutensor', 'include'),
                os.path.join(destination, 'include'))
            shutil.move(
                os.path.join(outdir, 'libcutensor', 'lib', cuda),
                os.path.join(destination, 'lib'))
            shutil.move(
                os.path.join(outdir, 'libcutensor', 'license.pdf'),
                destination)
        elif library == 'nccl':
            subdir = os.listdir(outdir)  # ['nccl_2.8.4-1+cuda11.2_x86_64']
            assert len(subdir) == 1
            shutil.move(os.path.join(outdir, subdir[0]), destination)
        else:
            assert False
        print('Cleaning up...')
    print('Done!')

def calculate_destination(prefix, cuda, lib, lib_ver):
    """Calculates the installation directory.

    ~/.cupy/cuda_lib/{cuda_version}/{library_name}/{library_version}
    """
    return os.path.join(prefix, cuda, lib, lib_ver)

def main(args):
    parser = argparse.ArgumentParser()

    # TODO(kmaehashi): support NCCL
    parser.add_argument('--library',
                        choices=['cudnn', 'cutensor', 'nccl'],
                        required=True,
                        help='Library to install')
    parser.add_argument('--cuda', type=str, required=True,
                        help='CUDA version')
    parser.add_argument('--prefix', type=str, default=None,
                        help='Install destination')
    parser.add_argument('--action', choices=['install', 'dump'],
                        default='install',
                        help='Action to perform')
    params = parser.parse_args(args)

    if params.prefix is not None:
        params.prefix = os.path.abspath(params.prefix)

    if params.action == 'install':
        install_lib(params.cuda, params.prefix, params.library)
    elif params.action == 'dump':
        print(json.dumps(library_records[params.library], indent=4))
    else:
        assert False

if __name__ == '__main__':
    main(sys.argv[1:])
<commit_msg>cuDNN v8.2.0<commit_after>#!/usr/bin/env python

"""
CUDA Library Installer

Installs the latest CUDA library supported by CuPy.
"""

# This script will also be used as a standalone script when building wheels.
# Keep the script runnable without CuPy dependency.

import argparse
import json
import os
import platform
import shutil
import sys
import tempfile
import urllib.request

_cudnn_records = []
_cutensor_records = []
_nccl_records = []
library_records = {}

def _make_cudnn_url(public_version, filename):
    # https://developer.download.nvidia.com/compute/redist/cudnn/v8.0.2/cudnn-11.0-linux-x64-v8.0.2.39.tgz
    return (
        'https://developer.download.nvidia.com/compute/redist/cudnn' +
        '/v{}/{}'.format(public_version, filename))

def _make_cudnn_record(
        cuda_version, public_version, filename_linux, filename_windows):
    major_version = public_version.split('.')[0]
    return {
        'cuda': cuda_version,
        'cudnn': public_version,
        'assets': {
            'Linux': {
                'url': _make_cudnn_url(public_version, filename_linux),
                'filename': 'libcudnn.so.{}'.format(public_version),
            },
            'Windows': {
                'url': _make_cudnn_url(public_version, filename_windows),
                'filename': 'cudnn64_{}.dll'.format(major_version),
            },
        }
    }

# Latest cuDNN versions: https://developer.nvidia.com/rdp/cudnn-download
_cudnn_records.append(_make_cudnn_record(
    '11.2', '8.2.0',
    'cudnn-11.3-linux-x64-v8.2.0.53.tgz',
    'cudnn-11.3-windows-x64-v8.2.0.53.zip'))
_cudnn_records.append(_make_cudnn_record(
    '11.1', '8.2.0',
    'cudnn-11.3-linux-x64-v8.2.0.53.tgz',
    'cudnn-11.3-windows-x64-v8.2.0.53.zip'))
_cudnn_records.append(_make_cudnn_record(
    '11.0', '8.2.0',
    'cudnn-11.3-linux-x64-v8.2.0.53.tgz',
    'cudnn-11.3-windows-x64-v8.2.0.53.zip'))
_cudnn_records.append(_make_cudnn_record(
    '10.2', '8.2.0',
    'cudnn-10.2-linux-x64-v8.2.0.53.tgz',
    'cudnn-10.2-windows10-x64-v8.2.0.53.zip'))
_cudnn_records.append(_make_cudnn_record(
    '10.1', '8.0.5',
    'cudnn-10.1-linux-x64-v8.0.5.39.tgz',
    'cudnn-10.1-windows10-x64-v8.0.5.39.zip'))
_cudnn_records.append(_make_cudnn_record(
    '10.0', '7.6.5',
    'cudnn-10.0-linux-x64-v7.6.5.32.tgz',
    'cudnn-10.0-windows10-x64-v7.6.5.32.zip'))
_cudnn_records.append(_make_cudnn_record(
    '9.2', '7.6.5',
    'cudnn-9.2-linux-x64-v7.6.5.32.tgz',
    'cudnn-9.2-windows10-x64-v7.6.5.32.zip'))
library_records['cudnn'] = _cudnn_records

def _make_cutensor_url(public_version, filename):
    # https://developer.download.nvidia.com/compute/cutensor/1.2.2/local_installers/libcutensor-linux-x86_64-1.2.2.5.tar.gz
    return (
        'https://developer.download.nvidia.com/compute/cutensor/' +
        '{}/local_installers/{}'.format(public_version, filename))

def _make_cutensor_record(
        cuda_version, public_version, filename_linux, filename_windows):
    return {
        'cuda': cuda_version,
        'cutensor': public_version,
        'assets': {
            'Linux': {
                'url': _make_cutensor_url(public_version, filename_linux),
                'filename': 'libcutensor.so.{}'.format(public_version),
            },
            'Windows': {
                'url': _make_cutensor_url(public_version, filename_windows),
                'filename': 'cutensor.dll',
            },
        }
    }

_cutensor_records.append(_make_cutensor_record(
    '11.2', '1.2.2',
    'libcutensor-linux-x86_64-1.2.2.5.tar.gz',
    'libcutensor-windows-x86_64-1.2.2.5.zip'))
_cutensor_records.append(_make_cutensor_record(
    '11.1', '1.2.2',
    'libcutensor-linux-x86_64-1.2.2.5.tar.gz',
    'libcutensor-windows-x86_64-1.2.2.5.zip'))
_cutensor_records.append(_make_cutensor_record(
    '11.0', '1.2.2',
    'libcutensor-linux-x86_64-1.2.2.5.tar.gz',
    'libcutensor-windows-x86_64-1.2.2.5.zip'))
_cutensor_records.append(_make_cutensor_record(
    '10.2', '1.2.2',
    'libcutensor-linux-x86_64-1.2.2.5.tar.gz',
    'libcutensor-windows-x86_64-1.2.2.5.zip'))
_cutensor_records.append(_make_cutensor_record(
    '10.1', '1.2.2',
    'libcutensor-linux-x86_64-1.2.2.5.tar.gz',
    'libcutensor-windows-x86_64-1.2.2.5.zip'))
library_records['cutensor'] = _cutensor_records

def _make_nccl_url(public_version, filename):
    # https://developer.download.nvidia.com/compute/redist/nccl/v2.8/nccl_2.8.4-1+cuda11.2_x86_64.txz
    return (
        'https://developer.download.nvidia.com/compute/redist/nccl/' +
        'v{}/{}'.format(public_version, filename))

def _make_nccl_record(
        cuda_version, full_version, public_version, filename_linux):
    return {
        'cuda': cuda_version,
        'nccl': full_version,
        'assets': {
            'Linux': {
                'url': _make_nccl_url(public_version, filename_linux),
                'filename': 'libnccl.so.{}'.format(full_version),
            },
        },
    }

_nccl_records.append(_make_nccl_record(
    '11.2', '2.8.4', '2.8',
    'nccl_2.8.4-1+cuda11.2_x86_64.txz'))
_nccl_records.append(_make_nccl_record(
    '11.1', '2.8.4', '2.8',
    'nccl_2.8.4-1+cuda11.1_x86_64.txz'))
_nccl_records.append(_make_nccl_record(
    '11.0', '2.8.4', '2.8',
    'nccl_2.8.4-1+cuda11.0_x86_64.txz'))
_nccl_records.append(_make_nccl_record(
    '10.2', '2.8.4', '2.8',
    'nccl_2.8.4-1+cuda10.2_x86_64.txz'))
_nccl_records.append(_make_nccl_record(
    '10.1', '2.8.3', '2.8',
    'nccl_2.8.3-1+cuda10.1_x86_64.txz'))
_nccl_records.append(_make_nccl_record(
    '10.0', '2.6.4', '2.6',
    'nccl_2.6.4-1+cuda10.0_x86_64.txz'))
_nccl_records.append(_make_nccl_record(
    '9.2', '2.4.8', '2.4',
    'nccl_2.4.8-1+cuda9.2_x86_64.txz'))
library_records['nccl'] = _nccl_records

def install_lib(cuda, prefix, library):
    record = None
    lib_records = library_records
    for record in lib_records[library]:
        if record['cuda'] == cuda:
            break
    else:
        raise RuntimeError('''
The CUDA version specified is not supported.
Should be one of {}.'''.format(str([x['cuda'] for x in lib_records[library]])))
    if prefix is None:
        prefix = os.path.expanduser('~/.cupy/cuda_lib')
    destination = calculate_destination(prefix, cuda, library, record[library])

    if os.path.exists(destination):
        raise RuntimeError('''
The destination directory {} already exists.
Remove the directory first if you want to reinstall.'''.format(destination))

    target_platform = platform.system()
    asset = record['assets'].get(target_platform, None)
    if asset is None:
        raise RuntimeError('''
The current platform ({}) is not supported.'''.format(target_platform))

    print('Installing {} {} for CUDA {} to: {}'.format(
        library, record[library], record['cuda'], destination))

    url = asset['url']
    print('Downloading {}...'.format(url))
    with tempfile.TemporaryDirectory() as tmpdir:
        with open(os.path.join(tmpdir, os.path.basename(url)), 'wb') as f:
            with urllib.request.urlopen(url) as response:
                f.write(response.read())
        print('Extracting...')
        outdir = os.path.join(tmpdir, 'extract')
        shutil.unpack_archive(f.name, outdir)
        print('Installing...')
        if library == 'cudnn':
            shutil.move(os.path.join(outdir, 'cuda'), destination)
        elif library == 'cutensor':
            if cuda.startswith('11.'):
                cuda = '11'
            shutil.move(
                os.path.join(outdir, 'libcutensor', 'include'),
                os.path.join(destination, 'include'))
            shutil.move(
                os.path.join(outdir, 'libcutensor', 'lib', cuda),
                os.path.join(destination, 'lib'))
            shutil.move(
                os.path.join(outdir, 'libcutensor', 'license.pdf'),
                destination)
        elif library == 'nccl':
            subdir = os.listdir(outdir)  # ['nccl_2.8.4-1+cuda11.2_x86_64']
            assert len(subdir) == 1
            shutil.move(os.path.join(outdir, subdir[0]), destination)
        else:
            assert False
        print('Cleaning up...')
    print('Done!')

def calculate_destination(prefix, cuda, lib, lib_ver):
    """Calculates the installation directory.

    ~/.cupy/cuda_lib/{cuda_version}/{library_name}/{library_version}
    """
    return os.path.join(prefix, cuda, lib, lib_ver)

def main(args):
    parser = argparse.ArgumentParser()

    # TODO(kmaehashi): support NCCL
    parser.add_argument('--library',
                        choices=['cudnn', 'cutensor', 'nccl'],
                        required=True,
                        help='Library to install')
    parser.add_argument('--cuda', type=str, required=True,
                        help='CUDA version')
    parser.add_argument('--prefix', type=str, default=None,
                        help='Install destination')
    parser.add_argument('--action', choices=['install', 'dump'],
                        default='install',
                        help='Action to perform')
    params = parser.parse_args(args)

    if params.prefix is not None:
        params.prefix = os.path.abspath(params.prefix)

    if params.action == 'install':
        install_lib(params.cuda, params.prefix, params.library)
    elif params.action == 'dump':
        print(json.dumps(library_records[params.library], indent=4))
    else:
        assert False

if __name__ == '__main__':
    main(sys.argv[1:])
<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import unicode_literals
from collections import namedtuple

from sqlagg.filters import ORFilter, EQFilter

from memoized import memoized

from corehq.apps.es import filters
from corehq.apps.locations.models import SQLLocation
from corehq.apps.reports_core.filters import DynamicChoiceListFilter
from corehq.apps.userreports.reports.filters.choice_providers import LocationChoiceProvider
from corehq.apps.userreports.reports.filters.specs import FilterSpec
from corehq.apps.userreports.reports.filters.values import FilterValue, dynamic_choice_list_url, SHOW_ALL_CHOICE

_LocationFilter = namedtuple("_LocationFilter", "column parameter_slug filter_value")

class ENikshayLocationHierarchyFilterValue(FilterValue):

    @memoized
    def get_hierarchy(self, location_id):
        """
        Return a list of _LocationFilter(column, parameter_slug, filter_value) objects to be used in constructing
        the filter.
        """
        relevant_location_types = ["sto", "cto", "dto", "tu", "phi"]
        location = SQLLocation.objects.get(location_id=location_id)

        if location.location_type.code not in relevant_location_types:
            # This report could be filtered by a user who is not assigned to one of the pertinent location types.
            # In that case, the report should show nothing, so return a filter that is guaranteed not to match
            # any records.
            return [_LocationFilter("phi", "{}_phi".format(self.filter.slug), location_id)]

        filters = []
        for ancestor in location.get_ancestors(include_self=True):
            if ancestor.location_type.code in relevant_location_types:
                column = ancestor.location_type.code
                parameter = "{slug}_{column}".format(slug=self.filter.slug, column=column)
                value = ancestor.location_id
                filters.append(
                    _LocationFilter(column, parameter, value)
                )

        below_column = "below_{}".format(location.location_type.code)
        below_parameter = "{slug}_{column}".format(slug=self.filter.slug, column=below_column)
        filters.append(
            _LocationFilter(below_column, below_parameter, location.location_id)
        )
        return filters

    @property
    def show_all(self):
        return SHOW_ALL_CHOICE in [choice.value for choice in self.value]

    def to_sql_filter(self):
        if self.show_all:
            return None
        location_id = self.value[0].value
        hierarchy = self.get_hierarchy(location_id)
        if len(hierarchy) == 1:
            f = self.get_hierarchy(location_id)[0]
            return EQFilter(f.column, f.parameter_slug)
        else:
            return ORFilter([
                EQFilter(x.column, x.parameter_slug) for x in self.get_hierarchy(location_id)
            ])

    def to_sql_values(self):
        if self.show_all:
            return {}
        location_id = self.value[0].value
        return {
            x.parameter_slug: x.filter_value for x in self.get_hierarchy(location_id)
        }

    def to_es_filter(self):
        if self.show_all:
            return None
        location_id = self.value[0].value
        fs = [
            filters.term(x.column, x.filter_value) for x in self.get_hierarchy(location_id)
        ]
        return filters.OR(fs)

class EnikshayLocationHiearachyFilter(DynamicChoiceListFilter):
    """
    A custom ucr report filter for the enikshay Referral Report.
    This filter will use the selected filter to filter against multiple fields.
    https://docs.google.com/document/d/1ejESqYoqSff0u4X72Fq0TYi7ll3IvH5KX3VWXzLyMQs/edit

    Each referral in the data source has fields like:
        cto
        dto
        sto
        ...
        below_cto
        below_dto
        below_sto
        ...
    corresponding to the location hieararchy levels.
    If a person was referred to a location below a given level, then the below_<location_level> column will
    contain the id of the location at <location_level>. The <location_level> column corresponding to the location
    that the person was referred to will contain the location id, and the other location level columns will be
    blank.
    """

    def __init__(self, name, label, choice_provider, url_generator, css_id=None):

        super(EnikshayLocationHiearachyFilter, self).__init__(
            name=name,
            field=None,
            datatype="string",
            label=label,
            show_all=None,
            choice_provider=choice_provider,
            url_generator=url_generator,
            css_id=css_id,
        )

def _build_enikshay_location_hierarchy(spec, report):
    wrapped = FilterSpec.wrap(spec)
    choice_provider = LocationChoiceProvider(report, wrapped.slug)
    choice_provider.configure({})
    return EnikshayLocationHiearachyFilter(
        name=wrapped.slug,
        label=wrapped.get_display(),
        choice_provider=choice_provider,
        url_generator=dynamic_choice_list_url,
    )
<commit_msg>refer correct slug<commit_after>from __future__ import absolute_import
from __future__ import unicode_literals
from collections import namedtuple

from sqlagg.filters import ORFilter, EQFilter

from memoized import memoized

from corehq.apps.es import filters
from corehq.apps.locations.models import SQLLocation
from corehq.apps.reports_core.filters import DynamicChoiceListFilter
from corehq.apps.userreports.reports.filters.choice_providers import LocationChoiceProvider
from corehq.apps.userreports.reports.filters.specs import FilterSpec
from corehq.apps.userreports.reports.filters.values import FilterValue, dynamic_choice_list_url, SHOW_ALL_CHOICE

_LocationFilter = namedtuple("_LocationFilter", "column parameter_slug filter_value")

class ENikshayLocationHierarchyFilterValue(FilterValue):

    @memoized
    def get_hierarchy(self, location_id):
        """
        Return a list of _LocationFilter(column, parameter_slug, filter_value) objects to be used in constructing
        the filter.
        """
        relevant_location_types = ["sto", "cto", "dto", "tu", "phi"]
        location = SQLLocation.objects.get(location_id=location_id)

        if location.location_type.code not in relevant_location_types:
            # This report could be filtered by a user who is not assigned to one of the pertinent location types.
            # In that case, the report should show nothing, so return a filter that is guaranteed not to match
            # any records.
            return [_LocationFilter("phi", "{}_phi".format(self.filter['slug']), location_id)]

        filters = []
        for ancestor in location.get_ancestors(include_self=True):
            if ancestor.location_type.code in relevant_location_types:
                column = ancestor.location_type.code
                parameter = "{slug}_{column}".format(slug=self.filter['slug'], column=column)
                value = ancestor.location_id
                filters.append(
                    _LocationFilter(column, parameter, value)
                )

        below_column = "below_{}".format(location.location_type.code)
        below_parameter = "{slug}_{column}".format(slug=self.filter['slug'], column=below_column)
        filters.append(
            _LocationFilter(below_column, below_parameter, location.location_id)
        )
        return filters

    @property
    def show_all(self):
        return SHOW_ALL_CHOICE in [choice.value for choice in self.value]

    def to_sql_filter(self):
        if self.show_all:
            return None
        location_id = self.value[0].value
        hierarchy = self.get_hierarchy(location_id)
        if len(hierarchy) == 1:
            f = self.get_hierarchy(location_id)[0]
            return EQFilter(f.column, f.parameter_slug)
        else:
            return ORFilter([
                EQFilter(x.column, x.parameter_slug) for x in self.get_hierarchy(location_id)
            ])

    def to_sql_values(self):
        if self.show_all:
            return {}
        location_id = self.value[0].value
        return {
            x.parameter_slug: x.filter_value for x in self.get_hierarchy(location_id)
        }

    def to_es_filter(self):
        if self.show_all:
            return None
        location_id = self.value[0].value
        fs = [
            filters.term(x.column, x.filter_value) for x in self.get_hierarchy(location_id)
        ]
        return filters.OR(fs)

class EnikshayLocationHiearachyFilter(DynamicChoiceListFilter):
    """
    A custom ucr report filter for the enikshay Referral Report.
    This filter will use the selected filter to filter against multiple fields.
    https://docs.google.com/document/d/1ejESqYoqSff0u4X72Fq0TYi7ll3IvH5KX3VWXzLyMQs/edit

    Each referral in the data source has fields like:
        cto
        dto
        sto
        ...
        below_cto
        below_dto
        below_sto
        ...
    corresponding to the location hieararchy levels.
    If a person was referred to a location below a given level, then the below_<location_level> column will
    contain the id of the location at <location_level>. The <location_level> column corresponding to the location
    that the person was referred to will contain the location id, and the other location level columns will be
    blank.
    """

    def __init__(self, name, label, choice_provider, url_generator, css_id=None):

        super(EnikshayLocationHiearachyFilter, self).__init__(
            name=name,
            field=None,
            datatype="string",
            label=label,
            show_all=None,
            choice_provider=choice_provider,
            url_generator=url_generator,
            css_id=css_id,
        )

def _build_enikshay_location_hierarchy(spec, report):
    wrapped = FilterSpec.wrap(spec)
    choice_provider = LocationChoiceProvider(report, wrapped.slug)
    choice_provider.configure({})
    return EnikshayLocationHiearachyFilter(
        name=wrapped.slug,
        label=wrapped.get_display(),
        choice_provider=choice_provider,
        url_generator=dynamic_choice_list_url,
    )
<|endoftext|>


<commit_before>from django.http import HttpResponse
from .models import Company
from .serializers_company import CompanySerializer
from .helper import auth_decorator
from rest_framework import status
from rest_framework.response import Response
from rest_framework.decorators import api_view
from django.db.models import Q

import logging
logger = logging.getLogger('ct-logger')

def index(request):
    return HttpResponse("Control is an illusion")

@api_view(['GET', 'POST'])
@auth_decorator
def all_companies(request,format=None, *args, **kwargs):
    """
    List all snippets, or create a new snippet.
    """
    if request.method == 'GET':
        companies = Company.objects.all()
        serializer = CompanySerializer(companies, many=True,  remove_fields=['contacts'])
        return Response(serializer.data)

    elif request.method == 'POST':
        serializer = CompanySerializer(data=request.data)
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data, status=status.HTTP_201_CREATED)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

@api_view(['GET', 'PUT', 'DELETE'])
@auth_decorator
def single_company(request, id, format=None):
    """
    Retrieve, update or delete a snippet instance.
    """
    try:
        company = Company.objects.get(id=id)
    except Company.DoesNotExist:
        return Response(status=status.HTTP_404_NOT_FOUND)

    if request.method == 'GET':
        serializer = CompanySerializer(company)
        return Response(serializer.data)

    elif request.method == 'PUT':
        serializer = CompanySerializer(company, data=request.data)
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

    elif request.method == 'DELETE':
        return Response(status=status.HTTP_400_BAD_REQUEST)

@api_view(['GET'])
@auth_decorator
def get_company_by_code(request, code, format=None):
    """
    Retrieve a company by code (first match is returned)
    """

    company = Company.objects.filter(Q(company_vat_number__iexact=code) | Q( company_tax_code__iexact=code)).first()

    if company:
        if request.method == 'GET':
            serializer = CompanySerializer(company)
            return Response(serializer.data)
        else:
            return Response(status=status.HTTP_400_BAD_REQUEST)
    else:
        print("company not found")
        company = Company.objects.filter(
            Q(company_vat_number__iendswith=code) | Q(company_tax_code__iendswith=code)).first()
        if company:
            if request.method == 'GET':
                serializer = CompanySerializer(company)
                return Response(serializer.data)
            else:
                print("company not found 2")
                return Response(status=status.HTTP_404_NOT_FOUND)

@api_view(['GET'])
@auth_decorator
def get_company_by_csid(request, csid, format=None):
    """
    Retrieve a company by Centralservices ID
    """
    companies = Company.objects.filter( company_centralservices_id=csid ).all()

    if request.method == 'GET':
        serializer = CompanySerializer(companies, many=True)
        return Response(serializer.data)
    else:
        return Response(status=status.HTTP_400_BAD_REQUEST)

@api_view(['GET'])
@auth_decorator
def get_company_freesearch(request, searchstring, format=None):
    """
    Retrieve a company by free search
    """
    companies = Company.objects.filter( Q(company_name__icontains=searchstring) | Q(company_short_name__icontains=searchstring) | Q(company_business_name__icontains=searchstring) | Q(company_website__icontains=searchstring) | Q(company_notes__icontains=searchstring) ).all()

    if request.method == 'GET':
        serializer = CompanySerializer(companies, many=True,  remove_fields=['contacts'])
        return Response(serializer.data)
    else:
        return Response(status=status.HTTP_400_BAD_REQUEST)

<commit_msg>little fix<commit_after>from django.http import HttpResponse
from .models import Company
from .serializers_company import CompanySerializer
from .helper import auth_decorator
from rest_framework import status
from rest_framework.response import Response
from rest_framework.decorators import api_view
from django.db.models import Q

import logging
logger = logging.getLogger('ct-logger')

def index(request):
    return HttpResponse("Control is an illusion")

@api_view(['GET', 'POST'])
@auth_decorator
def all_companies(request,format=None, *args, **kwargs):
    """
    List all snippets, or create a new snippet.
    """
    if request.method == 'GET':
        companies = Company.objects.all()
        serializer = CompanySerializer(companies, many=True,  remove_fields=['contacts'])
        return Response(serializer.data)

    elif request.method == 'POST':
        serializer = CompanySerializer(data=request.data)
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data, status=status.HTTP_201_CREATED)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

@api_view(['GET', 'PUT', 'DELETE'])
@auth_decorator
def single_company(request, id, format=None):
    """
    Retrieve, update or delete a snippet instance.
    """
    try:
        company = Company.objects.get(id=id)
    except Company.DoesNotExist:
        return Response(status=status.HTTP_404_NOT_FOUND)

    if request.method == 'GET':
        serializer = CompanySerializer(company)
        return Response(serializer.data)

    elif request.method == 'PUT':
        serializer = CompanySerializer(company, data=request.data)
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

    elif request.method == 'DELETE':
        return Response(status=status.HTTP_400_BAD_REQUEST)

@api_view(['GET'])
@auth_decorator
def get_company_by_code(request, code, format=None):
    """
    Retrieve a company by code (first match is returned)
    """

    company = Company.objects.filter(Q(company_vat_number__iexact=code) | Q( company_tax_code__iexact=code)).first()

    if company:
        if request.method == 'GET':
            serializer = CompanySerializer(company)
            return Response(serializer.data)
        else:
            return Response(status=status.HTTP_400_BAD_REQUEST)
    else:
        print("company not found")
        company = Company.objects.filter(
            Q(company_vat_number__iendswith=code) | Q(company_tax_code__iendswith=code)).first()
        if company:
            if request.method == 'GET':
                serializer = CompanySerializer(company)
                return Response(serializer.data)
        else:
            print("company not found 2")
            return Response(status=status.HTTP_404_NOT_FOUND)

@api_view(['GET'])
@auth_decorator
def get_company_by_csid(request, csid, format=None):
    """
    Retrieve a company by Centralservices ID
    """
    companies = Company.objects.filter( company_centralservices_id=csid ).all()

    if request.method == 'GET':
        serializer = CompanySerializer(companies, many=True)
        return Response(serializer.data)
    else:
        return Response(status=status.HTTP_400_BAD_REQUEST)

@api_view(['GET'])
@auth_decorator
def get_company_freesearch(request, searchstring, format=None):
    """
    Retrieve a company by free search
    """
    companies = Company.objects.filter( Q(company_name__icontains=searchstring) | Q(company_short_name__icontains=searchstring) | Q(company_business_name__icontains=searchstring) | Q(company_website__icontains=searchstring) | Q(company_notes__icontains=searchstring) ).all()

    if request.method == 'GET':
        serializer = CompanySerializer(companies, many=True,  remove_fields=['contacts'])
        return Response(serializer.data)
    else:
        return Response(status=status.HTTP_400_BAD_REQUEST)

<|endoftext|>


<commit_before>"""
Authors: Tim Bedin, Tim Erwin

Copyright 2014 CSIRO

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

Contains the PatternGenerator class.

"""

import os

from cwsl.configuration import configuration

class PatternGenerator(object):
    """ Class to pick the correct pattern (if one exists) and pass it into

    a VisTrails module.

    """

    def __init__(self, destination, data_type):
        self.destination = destination
        self.data_type = data_type

        # Get the user and project information.
        try:
            self.project = os.environ["PROJECT"]
        except KeyError:
            self.project = "NoProjectSet"
        self.user = os.environ["USER"]

        self.pbas = self.load_pbas()
        self.paths = self.load_paths()

        # Check the inputs to make sure that they make sense.
        self.check_inputs()

    def check_inputs(self):

        # Throw some errors if the parameters don't make sense.
        drs_only = ["downloaded"]
        if (self.destination != "drstree" and self.data_type in drs_only):
            raise BadCombinationError
        if self.data_type not in self.paths:
            raise PatternNotFoundError

    @property
    def pattern(self):
        """ Return the correct full pattern. """
        pba = self.pbas[self.destination]
        fullpath = self.paths[self.data_type]

        return os.path.join(pba, fullpath)

    def load_pbas(self):
        """ The paths before activity are pulled from the configuration object. """

        pba_dict = {}
        pba_dict["drstree"] = configuration.drs_basepath
        pba_dict["authoritative"] = configuration.authoritative_basepath
        pba_dict["user"] = configuration.user_basepath

        return pba_dict

    def load_paths(self):
        """ The patterns live here! """

        fullpath_dict = {}
        fullpath_dict["seasonal"] = os.path.join("%mip%/%product%/%grid%/%institute%/%model%/%experiment%/%frequency%/%realm%/%variable%/%ensemble%/",
                                                 "%variable%_%mip_table%_%model%_%experiment%_%ensemble%_%year_start%-%year_end%-%seas_agg%_%grid%.%suffix%")
        fullpath_dict["downloaded"] = os.path.join("%mip%/%product%/%institute%/%model%/%experiment%/%frequency%/%realm%/%variable%/%ensemble%/",
                                                   "%variable%_%mip_table%_%model%_%experiment%_%ensemble%_%origstart%-%origend%.nc")
        fullpath_dict["timeseries"] = os.path.join("%mip%/%product%/%institute%/%model%/%experiment%/%frequency%/%realm%/%variable%/%ensemble%/",
                                                   "%variable%_%mip_table%_%model%_%experiment%_%ensemble%_cdat-5-1-0.xml")
        fullpath_dict["cdat_lite_catalogue"] = os.path.join("%mip%/%product%/%institute%/%model%/%experiment%/%frequency%/%realm%/%variable%/%ensemble%/",
                                                            "%variable%_%mip_table%_%model%_%experiment%_%ensemble%_cdat-lite-6-0rc2-py2.7.xml")
        fullpath_dict["seasonal_aggregate"] = os.path.join("%mip%/%product%/%grid%/%institute%/%model%/%experiment%/%frequency%/%realm%/%variable%/%ensemble%/",
                                                           "%variable%_%mip_table%_%model%_%experiment%_%ensemble%_%year_start%-%year_end%-%seas_agg%_%grid%.nc")
        return fullpath_dict

# Here are some exceptions that get raised if the patterns are not correct.
class BadCombinationError(Exception):
    """ To be raised if a combination that doesn't make sense is requested:

    for example, user downloaded data .

    """
    # TODO (TB): actually, user downloaded data could be useful.

    pass

class PatternNotFoundError(Exception):
    """ Raised if a requested pattern is not found. """

    pass
<commit_msg>Added pattern for indicies<commit_after>"""
Authors: Tim Bedin, Tim Erwin

Copyright 2014 CSIRO

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

Contains the PatternGenerator class.

"""

import os

from cwsl.configuration import configuration

class PatternGenerator(object):
    """ Class to pick the correct pattern (if one exists) and pass it into

    a VisTrails module.

    """

    def __init__(self, destination, data_type):
        self.destination = destination
        self.data_type = data_type

        # Get the user and project information.
        try:
            self.project = os.environ["PROJECT"]
        except KeyError:
            self.project = "NoProjectSet"
        self.user = os.environ["USER"]

        self.pbas = self.load_pbas()
        self.paths = self.load_paths()

        # Check the inputs to make sure that they make sense.
        self.check_inputs()

    def check_inputs(self):

        # Throw some errors if the parameters don't make sense.
        drs_only = ["downloaded"]
        if (self.destination != "drstree" and self.data_type in drs_only):
            raise BadCombinationError
        if self.data_type not in self.paths:
            raise PatternNotFoundError

    @property
    def pattern(self):
        """ Return the correct full pattern. """
        pba = self.pbas[self.destination]
        fullpath = self.paths[self.data_type]

        return os.path.join(pba, fullpath)

    def load_pbas(self):
        """ The paths before activity are pulled from the configuration object. """

        pba_dict = {}
        pba_dict["drstree"] = configuration.drs_basepath
        pba_dict["authoritative"] = configuration.authoritative_basepath
        pba_dict["user"] = configuration.user_basepath

        return pba_dict

    def load_paths(self):
        """ The patterns live here! """

        fullpath_dict = {}
        fullpath_dict["seasonal"] = os.path.join("%mip%/%product%/%grid%/%institute%/%model%/%experiment%/%frequency%/%realm%/%variable%/%ensemble%/",
                                                 "%variable%_%mip_table%_%model%_%experiment%_%ensemble%_%year_start%-%year_end%-%seas_agg%_%grid%.%suffix%")
        fullpath_dict["indicies"] = os.path.join("%mip%/%product%/%grid%/%institute%/%model%/%experiment%/%frequency%/%realm%/%variable%/%ensemble%/",
                                    "%variable%_%mip_table%_%model%_%experiment%_%ensemble%_%year_start%-%year_end%-%seas_agg%_%index%_%grid%.%suffix%")
        fullpath_dict["downloaded"] = os.path.join("%mip%/%product%/%institute%/%model%/%experiment%/%frequency%/%realm%/%variable%/%ensemble%/",
                                                   "%variable%_%mip_table%_%model%_%experiment%_%ensemble%_%origstart%-%origend%.nc")
        fullpath_dict["timeseries"] = os.path.join("%mip%/%product%/%institute%/%model%/%experiment%/%frequency%/%realm%/%variable%/%ensemble%/",
                                                   "%variable%_%mip_table%_%model%_%experiment%_%ensemble%_cdat-5-1-0.xml")
        fullpath_dict["cdat_lite_catalogue"] = os.path.join("%mip%/%product%/%institute%/%model%/%experiment%/%frequency%/%realm%/%variable%/%ensemble%/",
                                                            "%variable%_%mip_table%_%model%_%experiment%_%ensemble%_cdat-lite-6-0rc2-py2.7.xml")
        fullpath_dict["seasonal_aggregate"] = os.path.join("%mip%/%product%/%grid%/%institute%/%model%/%experiment%/%frequency%/%realm%/%variable%/%ensemble%/",
                                                           "%variable%_%mip_table%_%model%_%experiment%_%ensemble%_%year_start%-%year_end%-%seas_agg%_%grid%.nc")
        return fullpath_dict

# Here are some exceptions that get raised if the patterns are not correct.
class BadCombinationError(Exception):
    """ To be raised if a combination that doesn't make sense is requested:

    for example, user downloaded data .

    """
    # TODO (TB): actually, user downloaded data could be useful.

    pass

class PatternNotFoundError(Exception):
    """ Raised if a requested pattern is not found. """

    pass
<|endoftext|>


<commit_before>from django.core.exceptions import ValidationError
from django.core.exceptions import ObjectDoesNotExist

from cyder.base.constants import IP_TYPE_4

import ipaddr
import functools
import re

is_attr = re.compile("^attr_\d+$")

def list_validator(things, validator):
    return all([validator(thing.strip(' ')) for thing in things.split(',')])

def is_valid_ip(ip, ip_type=IP_TYPE_4):
    try:
        ip_klass = ipaddr.IPv4Address if IP_TYPE_4 else ipaddr.IPv6Address
        ip_klass(ip)
        return True
    except:
        return False

def is_valid_domain(name):
    if name[-1] == '.':
        name = name[:-1]
    return all([re.match(r'^[_a-zA-Z\d]([_a-zA-Z0-9-]{0,61}[_a-zA-Z0-9])?$', \
            label) for label in name.split('.')]) and len(name) <= 253

def check_int(val, bits):
    return val.isdigit() and int(val) < (2 ** bits - 1)

def is__int8(val):
    return check_int(val, 8)

def is_int16(val):
    return check_int(val, 16)

def is_int32(val):
    return check_int(val, 32)

def is_bool(val):
    return val.lower() in ['on', 'off', 'true', 'false']

def is_ip_list(option_list, ip_type=IP_TYPE_4):
    return list_validator(
        option_list, functools.partial(is_valid_ip, ip_type=ip_type))

def is_bool_and_ip_list(option_list, ip_type=IP_TYPE_4):
    # FIXME: if option_list was stripped before being stored, strip() on the
    # next line is not necessary
    option_list_parts = option_list.strip(' ').split(' ')
    if is_bool(option_list_parts[0]) and len(option_list_parts) == 2:
        return is_ip_list(option_list_parts[1], ip_type)
    else:
        return is_ip_list(option_list, ip_type)

def is_domain_list(option_list, ip_type=IP_TYPE_4):
    return list_validator(option_list, is_valid_domain)

def is_int32_list(val):
    return list_validator(val, is_int32)

def get_dhcp_aa(obj):
    members = dir(obj)
    aa = []
    for member in members:
        if member.startswith("_aa"):
            member_name = member[4:].replace('_', '-')
            aa.append(member_name)
    return aa

def get_aa(obj):
    members = dir(obj)
    aa = []
    for member in members:
        if member.startswith("_aa"):
            aa.append(member[4:])
    return aa

def get_attrs(query_dict):
    kv = {}
    for param, values in query_dict.iteritems():
        if (is_attr.match(param) and
                "{0}_value".format(param) in query_dict):
                # u'attr_0': [u'<attr>']
                # u'key_attr_0': [u'<attr_value>']
                key = query_dict[param]
                value = query_dict["{0}_value".format(param)]

                if key in kv:
                    raise ValidationError("{0} is already an "
                                          "attribute.".format(key))
                kv[key] = value

    return kv

def update_attrs(kv, attrs, KVClass, obj, obj_name):
    """
    :param kv: The KV pairs being added to the object.
    :type kv: dict
    :param attrs: The existing KV pairs.
    :type attrs: :class:`QuerySet`
    :param KVClass: The KeyValue Class being used to validate new KV pairs.
    :type KVClass: class
    :param obj: The object the KV paris are being associated to.
    :type obj: object
    :param obj_name: The name of the foreign key back to obj
    :type obj_name: str
    """
    to_save = []
    for attr, value in kv.iteritems():
        if not attr and not value:
            continue
        if attrs.filter(key=attr):
            kv = attrs.get(key=attr)
            kv.value = value
            to_save.append(kv)
        else:
            # This kv is new. Let's create it and add it to the
            # to_save list for saving!
            kv = KVClass(key=attr, value=value)
            to_save.append(kv)

    for kv in to_save:
        setattr(kv, obj_name, obj)
        kv.clean()
        kv.save()

def dict_to_kv(kv, KVClass):
    # All attrs wraps the new and old kv pairs in a KVClass to ensure
    # that the template knows how to render them.
    all_attrs = []
    for k, v in kv.iteritems():
        all_attrs.append(KVClass(key=k, value=v))
    return all_attrs

def get_dhcp_docstrings(obj):
    members = dir(obj)
    docs = []
    for member in members:
        if member.startswith("_aa"):
            if obj.is_option or obj.is_statement:
                member_name = member[4:].replace('_', '-')
            else:
                member_name = member[4:]
            docs.append((member_name, getattr(obj, member).__doc__))
    return docs

def get_docstrings(obj):
    members = dir(obj)
    docs = []
    for member in members:
        if member.startswith("_aa"):
            member_name = member[4:]
            docs.append((member_name, getattr(obj, member).__doc__))
    return docs

class AuxAttr(object):
    """
    This class is a quick hack and is quite magical. This class abstracts the
    retreval and creation of Key Value pairs. Example usage:

        >>> aa = AuxAttr(StaticIntrKeyValue, intr, 'intr')

    This initializes an :class:`AuxAttr` instance with StaticIntrKeyValue,
    (a class that inheirits from :class:`KeyValue`), an object that has
    a KeyValue store (in this case the 'intr' object of type
    :class:`StaticInterface`), and the :class:`str` 'intr' (the name of the
    object in the 'StaticIntrKeyValue' class).

    See the StaticInterface class for example usage for this class. Also see
    the unit tests.
    """

    def __init__(self, KeyValueClass, obj, obj_name):
        # Bypass our overrides.
        super(AuxAttr, self).__setattr__('obj', obj)
        super(AuxAttr, self).__setattr__('obj_name', obj_name)
        super(AuxAttr, self).__setattr__('KVClass', KeyValueClass)
        super(AuxAttr, self).__setattr__('cache', {})

    def _get_aa(self, attr):
        if attr in self.cache:
            return self.cache[attr]
        else:
            try:
                kv = self.KVClass.objects.get(**{'key': attr, self.obj_name:
                                                 self.obj})
            except ObjectDoesNotExist, e:
                raise AttributeError("{0} AuxAttr has no attribute "
                                     "{1}".format(self.KVClass, attr))
            self.cache[attr] = kv.value
            return kv.value
        raise AttributeError()

    def __getattr__(self, attr):
        raise AttributeError()

    def __getattribute__(self, attr):
        try:
            return super(AuxAttr, self).__getattribute__(attr)
        except AttributeError:
            pass
        return self._get_aa(attr)

    def __setattr__(self, attr, value):

        try:
            if super(AuxAttr, self).__getattribute__(attr):
                return super(AuxAttr, self).__setattr__(attr, value)
        except AttributeError:
            pass
        try:
            kv = self.KVClass.objects.get(**{'key': attr, self.obj_name:
                                             self.obj})
        except ObjectDoesNotExist, e:
            kv = self.KVClass(**{'key': attr, self.obj_name: self.obj})
        kv.value = value
        kv.clean()
        kv.save()
        self.cache[attr] = value
        return

    def __delattr__(self, attr):
        try:
            if super(AuxAttr, self).__getattribute__(attr):
                return super(AuxAttr, self).__delattr__(attr)
        except AttributeError:
            pass
        if hasattr(self, attr):
            self.cache.pop(attr)
            kv = self.KVClass.objects.get(**{'key': attr, self.obj_name:
                                             self.obj})
            kv.delete()
            return
        else:
            raise AttributeError("{0} AuxAttr has no attribute "
                                 "{1}".format(self.KVClass, attr))
<commit_msg>Fix inconsistent function name<commit_after>from django.core.exceptions import ValidationError
from django.core.exceptions import ObjectDoesNotExist

from cyder.base.constants import IP_TYPE_4

import ipaddr
import functools
import re

is_attr = re.compile("^attr_\d+$")

def list_validator(things, validator):
    return all([validator(thing.strip(' ')) for thing in things.split(',')])

def is_valid_ip(ip, ip_type=IP_TYPE_4):
    try:
        ip_klass = ipaddr.IPv4Address if IP_TYPE_4 else ipaddr.IPv6Address
        ip_klass(ip)
        return True
    except:
        return False

def is_valid_domain(name):
    if name[-1] == '.':
        name = name[:-1]
    return all([re.match(r'^[_a-zA-Z\d]([_a-zA-Z0-9-]{0,61}[_a-zA-Z0-9])?$', \
            label) for label in name.split('.')]) and len(name) <= 253

def check_int(val, bits):
    return val.isdigit() and int(val) < (2 ** bits - 1)

def is_int8(val):
    return check_int(val, 8)

def is_int16(val):
    return check_int(val, 16)

def is_int32(val):
    return check_int(val, 32)

def is_bool(val):
    return val.lower() in ['on', 'off', 'true', 'false']

def is_ip_list(option_list, ip_type=IP_TYPE_4):
    return list_validator(
        option_list, functools.partial(is_valid_ip, ip_type=ip_type))

def is_bool_and_ip_list(option_list, ip_type=IP_TYPE_4):
    # FIXME: if option_list was stripped before being stored, strip() on the
    # next line is not necessary
    option_list_parts = option_list.strip(' ').split(' ')
    if is_bool(option_list_parts[0]) and len(option_list_parts) == 2:
        return is_ip_list(option_list_parts[1], ip_type)
    else:
        return is_ip_list(option_list, ip_type)

def is_domain_list(option_list, ip_type=IP_TYPE_4):
    return list_validator(option_list, is_valid_domain)

def is_int32_list(val):
    return list_validator(val, is_int32)

def get_dhcp_aa(obj):
    members = dir(obj)
    aa = []
    for member in members:
        if member.startswith("_aa"):
            member_name = member[4:].replace('_', '-')
            aa.append(member_name)
    return aa

def get_aa(obj):
    members = dir(obj)
    aa = []
    for member in members:
        if member.startswith("_aa"):
            aa.append(member[4:])
    return aa

def get_attrs(query_dict):
    kv = {}
    for param, values in query_dict.iteritems():
        if (is_attr.match(param) and
                "{0}_value".format(param) in query_dict):
                # u'attr_0': [u'<attr>']
                # u'key_attr_0': [u'<attr_value>']
                key = query_dict[param]
                value = query_dict["{0}_value".format(param)]

                if key in kv:
                    raise ValidationError("{0} is already an "
                                          "attribute.".format(key))
                kv[key] = value

    return kv

def update_attrs(kv, attrs, KVClass, obj, obj_name):
    """
    :param kv: The KV pairs being added to the object.
    :type kv: dict
    :param attrs: The existing KV pairs.
    :type attrs: :class:`QuerySet`
    :param KVClass: The KeyValue Class being used to validate new KV pairs.
    :type KVClass: class
    :param obj: The object the KV paris are being associated to.
    :type obj: object
    :param obj_name: The name of the foreign key back to obj
    :type obj_name: str
    """
    to_save = []
    for attr, value in kv.iteritems():
        if not attr and not value:
            continue
        if attrs.filter(key=attr):
            kv = attrs.get(key=attr)
            kv.value = value
            to_save.append(kv)
        else:
            # This kv is new. Let's create it and add it to the
            # to_save list for saving!
            kv = KVClass(key=attr, value=value)
            to_save.append(kv)

    for kv in to_save:
        setattr(kv, obj_name, obj)
        kv.clean()
        kv.save()

def dict_to_kv(kv, KVClass):
    # All attrs wraps the new and old kv pairs in a KVClass to ensure
    # that the template knows how to render them.
    all_attrs = []
    for k, v in kv.iteritems():
        all_attrs.append(KVClass(key=k, value=v))
    return all_attrs

def get_dhcp_docstrings(obj):
    members = dir(obj)
    docs = []
    for member in members:
        if member.startswith("_aa"):
            if obj.is_option or obj.is_statement:
                member_name = member[4:].replace('_', '-')
            else:
                member_name = member[4:]
            docs.append((member_name, getattr(obj, member).__doc__))
    return docs

def get_docstrings(obj):
    members = dir(obj)
    docs = []
    for member in members:
        if member.startswith("_aa"):
            member_name = member[4:]
            docs.append((member_name, getattr(obj, member).__doc__))
    return docs

class AuxAttr(object):
    """
    This class is a quick hack and is quite magical. This class abstracts the
    retreval and creation of Key Value pairs. Example usage:

        >>> aa = AuxAttr(StaticIntrKeyValue, intr, 'intr')

    This initializes an :class:`AuxAttr` instance with StaticIntrKeyValue,
    (a class that inheirits from :class:`KeyValue`), an object that has
    a KeyValue store (in this case the 'intr' object of type
    :class:`StaticInterface`), and the :class:`str` 'intr' (the name of the
    object in the 'StaticIntrKeyValue' class).

    See the StaticInterface class for example usage for this class. Also see
    the unit tests.
    """

    def __init__(self, KeyValueClass, obj, obj_name):
        # Bypass our overrides.
        super(AuxAttr, self).__setattr__('obj', obj)
        super(AuxAttr, self).__setattr__('obj_name', obj_name)
        super(AuxAttr, self).__setattr__('KVClass', KeyValueClass)
        super(AuxAttr, self).__setattr__('cache', {})

    def _get_aa(self, attr):
        if attr in self.cache:
            return self.cache[attr]
        else:
            try:
                kv = self.KVClass.objects.get(**{'key': attr, self.obj_name:
                                                 self.obj})
            except ObjectDoesNotExist, e:
                raise AttributeError("{0} AuxAttr has no attribute "
                                     "{1}".format(self.KVClass, attr))
            self.cache[attr] = kv.value
            return kv.value
        raise AttributeError()

    def __getattr__(self, attr):
        raise AttributeError()

    def __getattribute__(self, attr):
        try:
            return super(AuxAttr, self).__getattribute__(attr)
        except AttributeError:
            pass
        return self._get_aa(attr)

    def __setattr__(self, attr, value):

        try:
            if super(AuxAttr, self).__getattribute__(attr):
                return super(AuxAttr, self).__setattr__(attr, value)
        except AttributeError:
            pass
        try:
            kv = self.KVClass.objects.get(**{'key': attr, self.obj_name:
                                             self.obj})
        except ObjectDoesNotExist, e:
            kv = self.KVClass(**{'key': attr, self.obj_name: self.obj})
        kv.value = value
        kv.clean()
        kv.save()
        self.cache[attr] = value
        return

    def __delattr__(self, attr):
        try:
            if super(AuxAttr, self).__getattribute__(attr):
                return super(AuxAttr, self).__delattr__(attr)
        except AttributeError:
            pass
        if hasattr(self, attr):
            self.cache.pop(attr)
            kv = self.KVClass.objects.get(**{'key': attr, self.obj_name:
                                             self.obj})
            kv.delete()
            return
        else:
            raise AttributeError("{0} AuxAttr has no attribute "
                                 "{1}".format(self.KVClass, attr))
<|endoftext|>