<commit_before>import contextlib

from cupy import _util
from cupy.cuda.cufft import (CUFFT_C2C, CUFFT_C2R, CUFFT_R2C,
                             CUFFT_Z2Z, CUFFT_Z2D, CUFFT_D2Z,
                             CUFFT_CB_LD_COMPLEX, CUFFT_CB_LD_COMPLEX_DOUBLE,
                             CUFFT_CB_LD_REAL, CUFFT_CB_LD_REAL_DOUBLE,
                             CUFFT_CB_ST_COMPLEX, CUFFT_CB_ST_COMPLEX_DOUBLE,
                             CUFFT_CB_ST_REAL, CUFFT_CB_ST_REAL_DOUBLE,)
from cupy.cuda.device import get_compute_capability
from cupy_backends.cuda.api.driver import get_build_version

# expose cache handles to this module
from cupy.fft._cache import get_plan_cache  # NOQA
from cupy.fft._cache import clear_plan_cache  # NOQA
from cupy.fft._cache import get_plan_cache_size  # NOQA
from cupy.fft._cache import set_plan_cache_size  # NOQA
from cupy.fft._cache import get_plan_cache_max_memsize  # NOQA
from cupy.fft._cache import set_plan_cache_max_memsize  # NOQA
from cupy.fft._cache import show_plan_cache_info  # NOQA

enable_nd_planning = True
use_multi_gpus = False
_devices = None

def set_cufft_gpus(gpus):
    '''Set the GPUs to be used in multi-GPU FFT.

    Args:
        gpus (int or list of int): The number of GPUs or a list of GPUs
            to be used. For the former case, the first ``gpus`` GPUs
            will be used.

    .. warning::
        This API is currently experimental and may be changed in the future
        version.

    .. seealso:: `Multiple GPU cuFFT Transforms`_

    .. _Multiple GPU cuFFT Transforms:
        https://docs.nvidia.com/cuda/cufft/index.html#multiple-GPU-cufft-transforms
    '''
    _util.experimental('cupy.fft.config.set_cufft_gpus')
    global _devices

    if isinstance(gpus, int):
        devs = [i for i in range(gpus)]
    elif isinstance(gpus, list):
        devs = gpus
    else:
        raise ValueError("gpus must be an int or a list of int.")
    if len(devs) <= 1:
        raise ValueError("Must use at least 2 GPUs.")

    # make it hashable
    _devices = tuple(devs)

_callback_load = ''
_callback_store = ''
_callback_mgr = []
_callback_dev_code = None

class _CallbackManager:
    def __init__(self, plan_args, cb_load='', cb_store=''):
        import importlib
        import os
        import shutil
        import string
        import subprocess
        import sys
        import sysconfig
        import tempfile
        from cupy._environment import get_nvcc_path

        # TODO(leofang): cache the generated .so file based on:
        #   - Python version
        #   - CuPy version
        #   - cuFFT version
        #   - plan_args
        #   - cb_load & cb_store

        # TODO(leofang): would it be more robust if we use distutils +
        # setuptools here? I'm worried that the number of lines of code
        # might inflate too much...

        # TODO(leofang): make sure Cython is installed at runtime

        # TODO(leofang): make sure all needed source files are included
        # in sdist/wheel

        if not cb_load and not cb_store:
            raise ValueError('need to specify either cb_load or cb_store, '
                             'or both')
        if cb_load and 'd_loadCallbackPtr' not in cb_load:
            raise ValueError('need to specify d_loadCallbackPtr in cb_load')
        if cb_store and 'd_storeCallbackPtr' not in cb_store:
            raise ValueError('need to specify d_storeCallbackPtr in cb_store')
        self.plan_args = plan_args[1]
        self.cb_load = cb_load
        self.cb_store = cb_store

        python_include = sysconfig.get_paths()['include']
        nvcc = get_nvcc_path()
        CC = get_compute_capability()
        build_ver = get_build_version()
        source_dir = os.path.dirname(__file__) + '/../cuda/'

        # Set up temp directory; its lifetime is tied with the present instance
        self.dir_obj = tempfile.TemporaryDirectory()
        self.dir = self.dir_obj.name

        # Cythonize the Cython code; a c++ source cupy_callback.cpp is produced
        shutil.copyfile(source_dir+'/cupy_cufft.h', self.dir+'/cupy_cufft.h')
        shutil.copyfile(source_dir+'/cufft.pxd', self.dir+'/cupy_callback.pxd')
        shutil.copyfile(source_dir+'/cufft.pyx', self.dir+'/cupy_callback.pyx')
        p = subprocess.run(['cython', '-3', '--cplus',
                            '-E', 'use_hip=0',
                            '-E', 'CUDA_VERSION='+str(build_ver),
                            '-E', 'CUPY_CUFFT_STATIC=True',
                            self.dir+'/cupy_callback.pyx'],
                           env=os.environ)
        p.check_returncode()

        # Compile the Python module
        self.obj_host = self.dir+'/cupy_callback.o'
        shutil.copyfile(source_dir+'/cupy_cufftx.h', self.dir+'/cupy_cufftx.h')
        p = subprocess.run(['g++', '-I'+python_include,
                            '-fPIC', '-pthread', '-O2', '-std=c++11',
                            '-c', self.dir+'/cupy_callback.cpp',
                            '-o', self.obj_host],
                           env=os.environ)
        p.check_returncode()

        # Dump and compile device code using nvcc
        global _callback_dev_code
        if _callback_dev_code is None:
            with open(source_dir+'/cupy_cufftx.cu') as f:
                support = _callback_dev_code = f.read()
        else:
            support = _callback_dev_code
        with open(self.dir+'/cupy_cufftx.cu', 'w') as f:
            support = string.Template(support).substitute(
                dev_load_callback_ker=cb_load,
                dev_store_callback_ker=cb_store,
            )
            f.write(support)
        self.obj_dev = self.dir + '/cupy_callback_dev.o'
        cmd = [nvcc, '-ccbin', 'g++', '-arch=sm_'+CC, '-dc',
               '-c', self.dir+'/cupy_cufftx.cu',
               '-Xcompiler', '-fPIC', '-O2', '-std=c++11']
        if self.cb_load:
            cmd.append('-DHAS_LOAD_CALLBACK')
        if self.cb_store:
            cmd.append('-DHAS_STORE_CALLBACK')
        p = subprocess.run(cmd + ['-o', self.obj_dev], env=os.environ)
        p.check_returncode()

        # Use nvcc to link and generate a shared library
        # WARNING: CANNOT use host compiler to link!
        self.lib = self.dir + '/cupy_callback.so'
        p = subprocess.run([nvcc, '-ccbin', 'g++', '-shared', '-arch=sm_'+CC,
                            self.obj_dev, self.obj_host,
                            '-lcufft_static', '-lculibos', '-o', self.lib],
                           env=os.environ)
        p.check_returncode()

        # Load the Python module
        spec = importlib.util.spec_from_file_location(
            'cupy_callback', self.lib)
        self.mod = module = importlib.util.module_from_spec(spec)
        sys.modules['cupy_callback'] = module
        spec.loader.exec_module(module)

        # Create a cuFFT plan
        plan = plan_args[0]
        self.plan = getattr(self.mod, plan)(*plan_args[1])
        self.handle = self.plan.handle
        self.fft_type = self.plan.fft_type
        self.is_callback_set = False

    def set_callbacks(self, cb_load_type=-1, cb_store_type=-1):
        if self.is_callback_set:
            raise RuntimeError('callback cannot be reset')
        if self.cb_load:
            if cb_load_type == -1:
                raise ValueError('cb_load_type needs to be speficied')
            self.mod.setCallback(self.handle, cb_load_type, True)
        if self.cb_store:
            if cb_store_type == -1:
                raise ValueError('cb_store_type needs to be speficied')
            self.mod.setCallback(self.handle, cb_store_type, False)
        self.is_callback_set = True

    def __del__(self):
        self.dir_obj.cleanup()

@contextlib.contextmanager
def set_cufft_callbacks(cb_load='', cb_store=''):
    """A context manager for setting up load and/or store callbacks.

    Args:
        cb_load (str): A string contains the device kernel for the load
            callback. It must define ``d_loadCallbackPtr``.
        cb_store (str): A string contains the device kernel for the store
            callback. It must define ``d_storeCallbackPtr``.

    .. note::
        Any FFT calls living in this context will have callbacks set up. An
        example for a load callback is shown below:

        .. code-block:: python

            code = r'''
            __device__ cufftComplex CB_ConvertInputC(
                void *dataIn,
                size_t offset,
                void *callerInfo,
                void *sharedPtr) {
              // implementation
            }

            __device__ cufftCallbackLoadC d_loadCallbackPtr = CB_ConvertInputC;
            '''

            with cp.fft.config.set_cufft_callbacks(cb_load=code):
                out_arr = cp.fft.fft(in_arr, ...)

    .. warning::
        Using cuFFT callbacks requires compiling and loading a Python module at
        runtime as well as static linking for each distinct transform and
        callback, so the first invocation for each combination will be very
        slow. This is a limitation of cuFFT, so use this feature only when the
        callback-enabled transform is known more performant and can be reused
        to amortize the cost.

    """
    global _callback_load, _callback_store
    try:
        if cb_load:
            _callback_load = cb_load
        if cb_store:
            _callback_store = cb_store
        yield
    finally:
        _callback_load = ''
        _callback_store = ''

def _get_static_plan(plan_type, fft_type, keys):
    global _callback_load, _callback_store, _callback_mgr

    mgr = _CallbackManager(
        (plan_type, keys), cb_load=_callback_load, cb_store=_callback_store)
    if fft_type == CUFFT_C2C:
        cb_load_type = CUFFT_CB_LD_COMPLEX if _callback_load else -1
        cb_store_type = CUFFT_CB_ST_COMPLEX if _callback_store else -1
    elif fft_type == CUFFT_R2C:
        cb_load_type = CUFFT_CB_LD_REAL if _callback_load else -1
        cb_store_type = CUFFT_CB_ST_COMPLEX if _callback_store else -1
    elif fft_type == CUFFT_C2R:
        cb_load_type = CUFFT_CB_LD_COMPLEX if _callback_load else -1
        cb_store_type = CUFFT_CB_ST_REAL if _callback_store else -1
    elif fft_type == CUFFT_Z2Z:
        cb_load_type = CUFFT_CB_LD_COMPLEX_DOUBLE if _callback_load else -1
        cb_store_type = CUFFT_CB_ST_COMPLEX_DOUBLE if _callback_store else -1
    elif fft_type == CUFFT_D2Z:
        cb_load_type = CUFFT_CB_LD_REAL_DOUBLE if _callback_load else -1
        cb_store_type = CUFFT_CB_ST_COMPLEX_DOUBLE if _callback_store else -1
    elif fft_type == CUFFT_Z2D:
        cb_load_type = CUFFT_CB_LD_COMPLEX_DOUBLE if _callback_load else -1
        cb_store_type = CUFFT_CB_ST_REAL_DOUBLE if _callback_store else -1
    else:
        raise ValueError
    mgr.set_callbacks(cb_load_type, cb_store_type)

    _callback_load = ''
    _callback_store = ''
    _callback_mgr.append(mgr)  # keep the manager alive
    return mgr.plan
<commit_msg>cache python modules<commit_after>import contextlib
import hashlib
import importlib
import os
import shutil
import string
import subprocess
import sys
import sysconfig
import tempfile

from cupy import __version__
from cupy import _util
from cupy._environment import get_nvcc_path
from cupy.cuda.cufft import (CUFFT_C2C, CUFFT_C2R, CUFFT_R2C,
                             CUFFT_Z2Z, CUFFT_Z2D, CUFFT_D2Z,
                             CUFFT_CB_LD_COMPLEX, CUFFT_CB_LD_COMPLEX_DOUBLE,
                             CUFFT_CB_LD_REAL, CUFFT_CB_LD_REAL_DOUBLE,
                             CUFFT_CB_ST_COMPLEX, CUFFT_CB_ST_COMPLEX_DOUBLE,
                             CUFFT_CB_ST_REAL, CUFFT_CB_ST_REAL_DOUBLE,)
from cupy.cuda.cufft import getVersion as get_cufft_version
from cupy.cuda.device import get_compute_capability
from cupy_backends.cuda.api.driver import get_build_version

# expose cache handles to this module
from cupy.fft._cache import get_plan_cache  # NOQA
from cupy.fft._cache import clear_plan_cache  # NOQA
from cupy.fft._cache import get_plan_cache_size  # NOQA
from cupy.fft._cache import set_plan_cache_size  # NOQA
from cupy.fft._cache import get_plan_cache_max_memsize  # NOQA
from cupy.fft._cache import set_plan_cache_max_memsize  # NOQA
from cupy.fft._cache import show_plan_cache_info  # NOQA

enable_nd_planning = True
use_multi_gpus = False
_devices = None

def set_cufft_gpus(gpus):
    '''Set the GPUs to be used in multi-GPU FFT.

    Args:
        gpus (int or list of int): The number of GPUs or a list of GPUs
            to be used. For the former case, the first ``gpus`` GPUs
            will be used.

    .. warning::
        This API is currently experimental and may be changed in the future
        version.

    .. seealso:: `Multiple GPU cuFFT Transforms`_

    .. _Multiple GPU cuFFT Transforms:
        https://docs.nvidia.com/cuda/cufft/index.html#multiple-GPU-cufft-transforms
    '''
    _util.experimental('cupy.fft.config.set_cufft_gpus')
    global _devices

    if isinstance(gpus, int):
        devs = [i for i in range(gpus)]
    elif isinstance(gpus, list):
        devs = gpus
    else:
        raise ValueError("gpus must be an int or a list of int.")
    if len(devs) <= 1:
        raise ValueError("Must use at least 2 GPUs.")

    # make it hashable
    _devices = tuple(devs)

_callback_load = ''
_callback_store = ''
_callback_mgr = []
_callback_dev_code = None
_default_cache_dir = os.path.expanduser('~/.cupy/callback_cache')

def _get_cache_dir():
    return os.environ.get('CUPY_CACHE_DIR', _default_cache_dir)

# TODO(leofang): would it be more robust if we use distutils +
# setuptools here? I'm worried that the number of lines of code
# might inflate too much...
# TODO(leofang): make sure Cython is installed at runtime
# TODO(leofang): make sure all needed source files are included
# in sdist/wheel
# TODO(leofang): find a way to implement a lock-free method for
# cached shared libraries like what's done in cupy/cuda/compiler.py
# TODO(leofang): investigate if callerInfo can be supported. Looks
# like in that case we can't cache the Python modules?
class _CallbackManager:
    def __init__(self, plan_args, cb_load='', cb_store=''):
        # Sanity checks
        if not cb_load and not cb_store:
            raise ValueError('need to specify either cb_load or cb_store, '
                             'or both')
        if cb_load and 'd_loadCallbackPtr' not in cb_load:
            raise ValueError('need to specify d_loadCallbackPtr in cb_load')
        if cb_store and 'd_storeCallbackPtr' not in cb_store:
            raise ValueError('need to specify d_storeCallbackPtr in cb_store')
        self.plan_args = plan_args[1]
        self.cb_load = cb_load
        self.cb_store = cb_store

        # Set up some variables...
        python_include = sysconfig.get_paths()['include']
        nvcc = get_nvcc_path()
        CC = get_compute_capability()
        build_ver = get_build_version()
        source_dir = os.path.dirname(__file__) + '/../cuda/'
        sysinfo = sys.version_info

        # Generate module filename: all modules with the identical callbacks
        # are considered identical regardless of which plan was actually
        # executed at the time of generation
        mod_name = 'cupy_callback_' + __version__ + '_PY'
        for i in range(3):
            mod_name += str(sysinfo[i])
        mod_name += '_cuFFT' + str(get_cufft_version())
        if cb_load:
            mod_name += '_' + hashlib.md5(cb_load.encode()).hexdigest()
        if cb_store:
            mod_name += '_' + hashlib.md5(cb_store.encode()).hexdigest()
        mod_name = mod_name.replace('.', '')

        # Check if the module is already cached on disk
        cache_dir = _get_cache_dir() + '/'
        if not os.path.isdir(cache_dir):
            os.makedirs(cache_dir, exist_ok=True)
        path = os.path.join(cache_dir, mod_name + '.so')
        if not os.path.isfile(path):
            # Set up temp directory
            self.dir_obj = tempfile.TemporaryDirectory()
            self.dir = self.dir_obj.name + '/'

            # Cythonize the Cython code to produce a c++ source file
            shutil.copyfile(source_dir + '/cupy_cufft.h',
                            self.dir+'/cupy_cufft.h')
            shutil.copyfile(source_dir + '/cufft.pxd',
                            self.dir + mod_name + '.pxd')
            shutil.copyfile(source_dir + '/cufft.pyx',
                            self.dir + mod_name + '.pyx')
            p = subprocess.run(['cython', '-3', '--cplus',
                                '-E', 'use_hip=0',
                                '-E', 'CUDA_VERSION='+str(build_ver),
                                '-E', 'CUPY_CUFFT_STATIC=True',
                                self.dir + mod_name + '.pyx',
                                '-o', self.dir + mod_name + '.cpp'],
                               env=os.environ)
            p.check_returncode()

            # Compile the Python module
            self.obj_host = self.dir + mod_name + '.o'
            shutil.copyfile(source_dir + '/cupy_cufftx.h',
                            self.dir + '/cupy_cufftx.h')
            p = subprocess.run(['g++', '-I'+python_include,
                                '-fPIC', '-pthread', '-O2', '-std=c++11',
                                '-c', self.dir + mod_name + '.cpp',
                                '-o', self.obj_host],
                               env=os.environ)
            p.check_returncode()

            # Dump and compile device code using nvcc
            # TODO(leofang): use the nvcc utilities from cupy/cuda/compiler.py
            global _callback_dev_code
            if _callback_dev_code is None:
                with open(source_dir+'/cupy_cufftx.cu') as f:
                    support = _callback_dev_code = f.read()
            else:
                support = _callback_dev_code
            with open(self.dir+'/cupy_cufftx.cu', 'w') as f:
                support = string.Template(support).substitute(
                    dev_load_callback_ker=cb_load,
                    dev_store_callback_ker=cb_store,
                )
                f.write(support)
            self.obj_dev = self.dir + mod_name + '_dev.o'
            cmd = [nvcc, '-ccbin', 'g++', '-arch=sm_'+CC, '-dc',
                   '-c', self.dir+'/cupy_cufftx.cu',
                   '-Xcompiler', '-fPIC', '-O2', '-std=c++11']
            if self.cb_load:
                cmd.append('-DHAS_LOAD_CALLBACK')
            if self.cb_store:
                cmd.append('-DHAS_STORE_CALLBACK')
            p = subprocess.run(cmd + ['-o', self.obj_dev], env=os.environ)
            p.check_returncode()

            # Use nvcc to link and generate a shared library
            # WARNING: CANNOT use host compiler to link!
            p = subprocess.run([nvcc, '-ccbin', 'g++',
                                '-shared', '-arch=sm_'+CC,
                                self.obj_dev, self.obj_host,
                                '-lcufft_static', '-lculibos',
                                '-o', path],
                               env=os.environ)
            p.check_returncode()

            self.dir_obj.cleanup()
            del self.dir_obj
            del self.dir
            del self.obj_host
            del self.obj_dev

        # Load the Python module
        spec = importlib.util.spec_from_file_location(mod_name, path)
        self.mod = module = importlib.util.module_from_spec(spec)
        sys.modules[mod_name] = module
        spec.loader.exec_module(module)

        # Create a cuFFT plan
        plan = plan_args[0]
        self.plan = getattr(self.mod, plan)(*plan_args[1])
        self.handle = self.plan.handle
        self.fft_type = self.plan.fft_type
        self.is_callback_set = False

    def set_callbacks(self, cb_load_type=-1, cb_store_type=-1):
        if self.is_callback_set:
            raise RuntimeError('callback cannot be reset')
        if self.cb_load:
            if cb_load_type == -1:
                raise ValueError('cb_load_type needs to be speficied')
            self.mod.setCallback(self.handle, cb_load_type, True)
        if self.cb_store:
            if cb_store_type == -1:
                raise ValueError('cb_store_type needs to be speficied')
            self.mod.setCallback(self.handle, cb_store_type, False)
        self.is_callback_set = True

@contextlib.contextmanager
def set_cufft_callbacks(cb_load='', cb_store=''):
    """A context manager for setting up load and/or store callbacks.

    Args:
        cb_load (str): A string contains the device kernel for the load
            callback. It must define ``d_loadCallbackPtr``.
        cb_store (str): A string contains the device kernel for the store
            callback. It must define ``d_storeCallbackPtr``.

    .. note::
        Any FFT calls living in this context will have callbacks set up. An
        example for a load callback is shown below:

        .. code-block:: python

            code = r'''
            __device__ cufftComplex CB_ConvertInputC(
                void *dataIn,
                size_t offset,
                void *callerInfo,
                void *sharedPtr) {
              // implementation
            }

            __device__ cufftCallbackLoadC d_loadCallbackPtr = CB_ConvertInputC;
            '''

            with cp.fft.config.set_cufft_callbacks(cb_load=code):
                out_arr = cp.fft.fft(in_arr, ...)

    .. warning::
        Using cuFFT callbacks requires compiling and loading a Python module at
        runtime as well as static linking for each distinct transform and
        callback, so the first invocation for each combination will be very
        slow. This is a limitation of cuFFT, so use this feature only when the
        callback-enabled transform is known more performant and can be reused
        to amortize the cost.

    .. warning::
        The generated Python modules are by default cached in
        ``~/.cupy/callback_cache`` for possible reuse (with the same set of
        load/store callbacks). Due to static linking, however, the file sizes
        can be excessive! The cache position can be changed via setting
        ``CUPY_CACHE_DIR``.

    """
    global _callback_load, _callback_store
    try:
        if cb_load:
            _callback_load = cb_load
        if cb_store:
            _callback_store = cb_store
        yield
    finally:
        _callback_load = ''
        _callback_store = ''

def _get_static_plan(plan_type, fft_type, keys):
    global _callback_load, _callback_store, _callback_mgr

    mgr = _CallbackManager(
        (plan_type, keys), cb_load=_callback_load, cb_store=_callback_store)
    if fft_type == CUFFT_C2C:
        cb_load_type = CUFFT_CB_LD_COMPLEX if _callback_load else -1
        cb_store_type = CUFFT_CB_ST_COMPLEX if _callback_store else -1
    elif fft_type == CUFFT_R2C:
        cb_load_type = CUFFT_CB_LD_REAL if _callback_load else -1
        cb_store_type = CUFFT_CB_ST_COMPLEX if _callback_store else -1
    elif fft_type == CUFFT_C2R:
        cb_load_type = CUFFT_CB_LD_COMPLEX if _callback_load else -1
        cb_store_type = CUFFT_CB_ST_REAL if _callback_store else -1
    elif fft_type == CUFFT_Z2Z:
        cb_load_type = CUFFT_CB_LD_COMPLEX_DOUBLE if _callback_load else -1
        cb_store_type = CUFFT_CB_ST_COMPLEX_DOUBLE if _callback_store else -1
    elif fft_type == CUFFT_D2Z:
        cb_load_type = CUFFT_CB_LD_REAL_DOUBLE if _callback_load else -1
        cb_store_type = CUFFT_CB_ST_COMPLEX_DOUBLE if _callback_store else -1
    elif fft_type == CUFFT_Z2D:
        cb_load_type = CUFFT_CB_LD_COMPLEX_DOUBLE if _callback_load else -1
        cb_store_type = CUFFT_CB_ST_REAL_DOUBLE if _callback_store else -1
    else:
        raise ValueError
    mgr.set_callbacks(cb_load_type, cb_store_type)

    _callback_load = ''
    _callback_store = ''
    _callback_mgr.append(mgr)  # keep the manager alive
    return mgr.plan
<|endoftext|>


<commit_before>#!/usr/bin/env python

# MontyHall.py

# Library used - Numpy for its numpy.random() modules
# The Monty Hall problem:
# You are on a gameshow and are
# presented with three doors. Behind one door (chosen at rand) sits a prize.
# You must select (guess) a door. If you guess correctly, you get the prize.
# But, when you select your door, the show's host throws in a bit of a twist,
# opening one of the other two doors ... specifically a door that doesn't
# contain the prize. He asks you if you'd like to change your door selection,
# to the only other remaining door (not the one you picked originally and not
# the now open door). Do you change your selection? Why?

# Answer: Yes. ALWAYS change your selection. By sticking with your original
# door selection, you have a 1/3 chance of getting the prize. By switching, you
# end up with a 2/3 chance of getting the prize.

import numpy as np

# We need the Numpy module to use its random function value
#  which provides a Return random integers from low (inclusive) to high (exclusive).

def pick_door():
    """
    Function to pick a door. Returns an integer 1, 2, or 3 at random.
    """
    return np.random.randint(1,4)

class MontyHall:
    """
    Class to model the Monty Hall problem.
    """
    def __init__(self):
        """
        Class Constructor -initializes values to the program
        Creates an instance of the Monty-Hall problem. This will be called
        automatically when the class is formed.
        """

        # Pick a prize door randomly and store as a variable.

        self.prize_door = pick_door()

        # We'll create variables for the selected door and removed door as
        # well, but we won't set them just yet.

        self.selected_door = None
        self.removed_door = None

    def select_door(self):
        """
        randomly selects a door for the contestant.
        """
        self.selected_door = pick_door()

    def remove_door(self):
        """
        This is how the host removes a (non-prize/non-selected) door
        """
        # Pick a door at random.
        d = pick_door()
        # If that door is the prize door or the contestant's door, re-pick.
        while d == self.selected_door or d == self.prize_door:
            d = pick_door()
        # set the removed door to d
        self.removed_door = d

    def switch_choice(self):
        """
        Switches the selected door once a non-prize door is removed.
        """
        # 1+2+3=6. There's only one choice of door if we switch our selection.
        self.selected_door = 6 - self.selected_door - self.removed_door

    def user_wins(self):
        """
        Determine if the user wins. Return true on win, false on lose.
        """
        if self.selected_door == self.prize_door:
            return True
        else:
            return False

    def run_game(self, switch=True):
        """
        Once a problem is initialized, run the game.

        'switch' determines if the user changes selection during the game.
        """
        # The user selects a door.
        self.select_door()
        # The host removes a door.
        self.remove_door()
        # The user can switch selection of doors.
        if switch:
            self.switch_choice()
        # Determine if the user wins.
        return self.user_wins()

 def main() :
    # Now, we'll run the game. When asked if we want to switch door selection when
    # the door is removed by the host, we'll say 'yes' and switch our choice of
    # door. We'll run that experiment one million times, always switching choice
    # when given the chance. Here's what that looks like:

    wins, losses = 0, 0
    for i in range(1000000):
        # make an instance of the game, call it 'm'
        m = MontyHall()
        # run the game and switch choice of door.
        if m.run_game(switch=True):
            # a return value of True means we've won
            wins += 1
        else:
            # a return value of False means we've lost
            losses += 1

    # Now that the game has been run one million times, compute/display stats.
    perc_win = 100.0*wins / (wins+losses)

    print "\nOne million Monty Hall games (with switching):"
    print "  won:", wins, "games"
    print "  lost:", losses, "games"
    print "  odds: %.2f%% winning percentage" % perc_win

    # Now, we'll run the game one million times and always stick to our original
    # door selection every single time.

    wins, losses = 0, 0
    for i in range(1000000):
        # make an instance of the game, call it 'm'
        m = MontyHall()
        # run the game and switch choice of door.
        if m.run_game(switch=False):
            # a return value of True means we've won
            wins += 1
        else:
            # a return value of False means we've lost
            losses += 1

    # Now that the game has been run one million times, compute/display stats.

    perc_win = 100.0*wins / (wins+losses)

    print "\nOne million Monty Hall games (staying with original choice):"
    print "  won:", wins, "games"
    print "  lost:", losses, "games"
    print "  odds: %.2f%% winning percentage\n" % perc_win

if __name__ == '__main__':
    main()<commit_msg>Added the MontyHall problem with corrected errors<commit_after>#!/usr/bin/env python

# MontyHall.py

# Library used - Numpy for its numpy.random() modules
# The Monty Hall problem:
# You are on a gameshow and are
# presented with three doors. Behind one door (chosen at rand) sits a prize.
# You must select (guess) a door. If you guess correctly, you get the prize.
# But, when you select your door, the show's host throws in a bit of a twist,
# opening one of the other two doors ... specifically a door that doesn't
# contain the prize. He asks you if you'd like to change your door selection,
# to the only other remaining door (not the one you picked originally and not
# the now open door). Do you change your selection? Why?

# Answer: Yes. ALWAYS change your selection. By sticking with your original
# door selection, you have a 1/3 chance of getting the prize. By switching, you
# end up with a 2/3 chance of getting the prize.

import numpy as np

# We need the Numpy module to use its random function value
#  which provides a Return random integers from low (inclusive) to high (exclusive).

def pick_door():
    """
    Function to pick a door. Returns an integer 1, 2, or 3 at random.
    """
    return np.random.randint(1,4)

class MontyHall:
    """
    Class to model the Monty Hall problem.
    """
    def __init__(self):
        """
        Class Constructor -initializes values to the program
        Creates an instance of the Monty-Hall problem. This will be called
        automatically when the class is formed.
        """

        # Pick a prize door randomly and store as a variable.

        self.prize_door = pick_door()

        # We'll create variables for the selected door and removed door as
        # well, but we won't set them just yet.

        self.selected_door = None
        self.removed_door = None

    def select_door(self):
        """
        randomly selects a door for the contestant.
        """
        self.selected_door = pick_door()

    def remove_door(self):
        """
        This is how the host removes a (non-prize/non-selected) door
        """
        # Pick a door at random.
        d = pick_door()
        # If that door is the prize door or the contestant's door, re-pick.
        while d == self.selected_door or d == self.prize_door:
            d = pick_door()
        # set the removed door to d
        self.removed_door = d

    def switch_choice(self):
        """
        Switches the selected door once a non-prize door is removed.
        """
        # 1+2+3=6. There's only one choice of door if we switch our selection.
        self.selected_door = 6 - self.selected_door - self.removed_door

    def user_wins(self):
        """
        Determine if the user wins. Return true on win, false on lose.
        """
        if self.selected_door == self.prize_door:
            return True
        else:
            return False

    def run_game(self, switch=True):
        """
        Once a problem is initialized, run the game.

        'switch' determines if the user changes selection during the game.
        """
        # The user selects a door.
        self.select_door()
        # The host removes a door.
        self.remove_door()
        # The user can switch selection of doors.
        if switch:
            self.switch_choice()
        # Determine if the user wins.
        return self.user_wins()

def main():
    # Now, we'll run the game. When asked if we want to switch door selection when
    # the door is removed by the host, we'll say 'yes' and switch our choice of
    # door. We'll run that experiment one million times, always switching choice
    # when given the chance. Here's what that looks like:

    wins, losses = 0, 0

    for i in range(1000000):
        # make an instance of the game, call it 'm'
        m = MontyHall()
        # run the game and switch choice of door.
        if m.run_game(switch=True):
            # a return value of True means we've won
            wins += 1
        else:
            # a return value of False means we've lost
            losses += 1

    # Now that the game has been run one million times, compute/display stats.
    perc_win = 100.0*wins / (wins+losses)

    print "\nOne million Monty Hall games (with switching):"
    print "  won:", wins, "games"
    print "  lost:", losses, "games"
    print "  odds: %.2f%% winning percentage" % perc_win

    # Now, we'll run the game one million times and always stick to our original
    # door selection every single time.

    wins, losses = 0, 0
    for i in range(1000000):
        # make an instance of the game, call it 'm'
        m = MontyHall()
        # run the game and switch choice of door.
        if m.run_game(switch=False):
            # a return value of True means we've won
            wins += 1
        else:
            # a return value of False means we've lost
            losses += 1

    # Now that the game has been run one million times, compute/display stats.

    perc_win = 100.0*wins / (wins+losses)

    print "\nOne million Monty Hall games (staying with original choice):"
    print "  won:", wins, "games"
    print "  lost:", losses, "games"
    print "  odds: %.2f%% winning percentage\n" % perc_win

if __name__ == '__main__':
    main()<|endoftext|>


<commit_before>from django.conf.urls import url, patterns

from fabric_bolt.accounts import views

urlpatterns = patterns('',
    url(r'^login/$', views.Login.as_view(), name='accounts_user_login'),
    url(r'^logout/$', views.Logout.as_view(), name='accounts_user_logout'),

    url(r'^user/permissions/$', views.UserPermissions.as_view(), name='accounts_user_permissions'),
    url(r'^users/$', views.UserList.as_view(), name='accounts_user_list'),
    url(r'^user/add/$', views.UserAdd.as_view(), name='accounts_user_add'),
    url(r'^user/change/(?P<pk>.+)/$', views.UserChange.as_view(), name='accounts_user_change'),
    url(r'^user/delete/(?P<pk>.+)/$', views.UserDelete.as_view(), name='accounts_user_delete'),
    url(r'^user/(?P<pk>.+)/$', views.UserDetail.as_view(), name='accounts_user_view'),

    url(r'^password_change/$', views.PasswordChange.as_view(), name='accounts_password_change'),  # django.contrib.auth.views.password_change'),
    url(r'^password_change/done/$', 'django.contrib.auth.views.password_change_done'),
    url(r'^password_reset/$', 'django.contrib.auth.views.password_reset'),
    url(r'^password_reset/done/$', 'django.contrib.auth.views.password_reset_done'),
    url(r'^register/(?P<uidb64>[0-9A-Za-z]+)-(?P<token>.+)/$', views.PasswordCreate.as_view(), name='auth_password_reset_confirm'),
    url(r'^register/done/$', 'django.contrib.auth.views.password_reset_complete', name='password_reset_complete'),
)<commit_msg>Moved the user permission url in the urls file.<commit_after>from django.conf.urls import url, patterns

from fabric_bolt.accounts import views

urlpatterns = patterns('',
    url(r'^login/$', views.Login.as_view(), name='accounts_user_login'),
    url(r'^logout/$', views.Logout.as_view(), name='accounts_user_logout'),

    url(r'^user/permissions/$', views.UserPermissions.as_view(), name='accounts_user_permissions'),

    url(r'^users/$', views.UserList.as_view(), name='accounts_user_list'),
    url(r'^user/add/$', views.UserAdd.as_view(), name='accounts_user_add'),
    url(r'^user/change/(?P<pk>.+)/$', views.UserChange.as_view(), name='accounts_user_change'),
    url(r'^user/delete/(?P<pk>.+)/$', views.UserDelete.as_view(), name='accounts_user_delete'),
    url(r'^user/(?P<pk>.+)/$', views.UserDetail.as_view(), name='accounts_user_view'),

    url(r'^password_change/$', views.PasswordChange.as_view(), name='accounts_password_change'),  # django.contrib.auth.views.password_change'),
    url(r'^password_change/done/$', 'django.contrib.auth.views.password_change_done'),
    url(r'^password_reset/$', 'django.contrib.auth.views.password_reset'),
    url(r'^password_reset/done/$', 'django.contrib.auth.views.password_reset_done'),
    url(r'^register/(?P<uidb64>[0-9A-Za-z]+)-(?P<token>.+)/$', views.PasswordCreate.as_view(), name='auth_password_reset_confirm'),
    url(r'^register/done/$', 'django.contrib.auth.views.password_reset_complete', name='password_reset_complete'),
)<|endoftext|>


<commit_before>import math
import tensorflow as tf

from neuralmonkey.decoding_function import attention_decoder

class Decoder(object):
    """A class that manages parts of the computation graph that are
    used for the decoding.
    """

    # pylint: disable=too-many-instance-attributes,too-many-locals
    # Big decoder cannot be simpler. Not sure if refactoring
    # it into smaller units would be helpful
    # Some locals may be turned to attributes

    def __init__(self, encoders, vocabulary, data_id, **kwargs):
        """Creates a new instance of the decoder

        Required arguments:
            encoders: List of encoders whose outputs will be decoded
            vocabulary: Output vocabulary
            data_id: Identifier of the data series fed to this decoder

        Keyword arguments:
            embedding_size: Size of embedding vectors. Default 200
            max_output: Maximum length of the output. Default 20
            rnn_size: When projection is used or when no encoder is supplied,
                      this is the size of the projected vector.
            dropout_keep_prob: Dropout keep probability. Default 1 (no dropout)

        Flags:
            use_attention: Indicates whether to use attention from encoders
            reuse_word_embeddings: Boolean flag specifying whether to reuse
                                   word embeddings. If True, word embeddings
                                   from the first encoder will be used
            project_encoder_outputs: Boolean flag whether to project output
                                     states of encoders

        """
        self.encoders = encoders
        self.vocabulary = vocabulary
        self.data_id = data_id

        self.max_output = kwargs.get("max_output", 20)
        self.embedding_size = kwargs.get("embedding_size", 200)
        self.name = kwargs.get("name", "decoder")
        dropout_keep_prob = kwargs.get("dropout_keep_prob", 1.0)

        self.use_attention = kwargs.get("use_attention", False)
        self.reuse_word_embeddings = kwargs.get("reuse_word_embeddings", False)
        self.project_encoder_outputs = kwargs.get("project_encoder_outputs",
                                                  False)

        if self.project_encoder_outputs or len(self.encoders) == 0:
            self.rnn_size = kwargs.get("rnn_size", 200)
        else:
            self.rnn_size = sum([e.encoded.get_shape()[1].value
                                 for e in self.encoders])

        ### Initialize model

        self.dropout_placeholder = tf.placeholder_with_default(
            dropout_keep_prob, [], "decoder_dropout_placeholder")

        state = self.initial_state()

        self.weights, self.biases = self.state_to_output()
        self.embedding_matrix = self.input_embeddings()

        cell = self.get_rnn_cell()
        attention_objects = self.collect_attention_objects(self.encoders)

        self.train_inputs, self.train_weights = self.training_placeholders()
        self.batch_size = tf.shape(self.train_inputs[0])
        train_targets = self.train_inputs[1:]

        ### Perform computation

        embedded_train_inputs = self.embed_inputs(self.train_inputs[:-1])

        self.train_rnn_outputs, _ = attention_decoder(
            embedded_train_inputs, state, attention_objects,
            self.embedding_size, cell, scope="att_train")

        runtime_inputs = self.runtime_inputs()
        loop_function = self.get_loop_function()

        self.runtime_rnn_outputs, _ = attention_decoder(
            runtime_inputs, state, attention_objects, self.embedding_size,
            cell, loop_function=loop_function, scope="att_runtime")

        ### KONEC decoder scope

        _, train_logits = self.decode(self.train_rnn_outputs)
        self.decoded, runtime_logits = self.decode(self.runtime_rnn_outputs)

        self.train_loss = tf.nn.seq2seq.sequence_loss(
            train_logits, train_targets, self.train_weights,
            self.vocabulary_size)

        self.runtime_loss = tf.nn.seq2seq.sequence_loss(
            runtime_logits, train_targets, self.train_weights,
            self.vocabulary_size)

        ### Learning step
        ### TODO was here only because of scheduled sampling.
        ### needs to be refactored out
        self.learning_step = tf.Variable(0, name="learning_step",
                                         trainable=False)

        ### Summaries
        self.init_summaries()

    @property
    def vocabulary_size(self):
        return len(self.vocabulary)

    @property
    def cost(self):
        return self.train_loss

    def init_summaries(self):
        """Initialize the summaries of the decoder

        TensorBoard summaries are collected into the following
        collections:

        - summary_train: collects statistics from the train-time
        - sumarry_val: collects OAstatistics while being tested on the
                 development data
        """
        tf.scalar_summary("train_loss_with_decoded_inputs", self.runtime_loss,
                          collections=["summary_train"])

        tf.scalar_summary("train_optimization_cost", self.train_loss,
                          collections=["summary_train"])

    def decode(self, rnn_states):
        """Decodes a sequence from a list of hidden states

        Arguments:
            rnn_states: hidden states
        """
        logits = [self.logit_function(s) for s in rnn_states]
        decoded = [tf.argmax(l[:, 1:], 1) + 1 for l in logits]

        return decoded, logits

    def training_placeholders(self):
        """Defines data placeholders for training the decoder"""

        inputs = [tf.placeholder(tf.int64, [None], name="decoder_{}".format(i))
                  for i in range(self.max_output + 2)]

        # one less than inputs
        weights = [tf.placeholder(tf.float32, [None],
                                  name="decoder_padding_weights_{}".format(i))
                   for i in range(self.max_output + 1)]

        return inputs, weights

    def runtime_inputs(self):
        """Defines data inputs for running trained decoder"""
        go_symbols = tf.ones(self.batch_size, dtype=tf.int32)
        go_embeds = tf.nn.embedding_lookup(self.embedding_matrix, go_symbols)

        inputs = [self.dropout(go_embeds)]
        inputs += [None for _ in range(self.max_output)]

        return inputs

    def state_to_output(self):
        """Create variables for projection of states to output vectors"""

        weights = tf.get_variable("state_to_word_W", [self.rnn_size,
                                                      self.vocabulary_size])

        biases = tf.Variable(tf.fill([self.vocabulary_size],
                                     - math.log(self.vocabulary_size)),
                             name="state_to_word_b")

        return weights, biases

    def input_embeddings(self):
        """Create variables and operations for embedding of input words

        If we are reusing word embeddings, this function takes
        them from the first encoder
        """
        if self.reuse_word_embeddings:
            return self.encoders[0].word_embeddings

        return tf.get_variable("word_embeddings", [self.vocabulary_size,
                                                   self.embedding_size])

    def embed_inputs(self, inputs):
        """Embed inputs using the decoder"s word embedding matrix

        Arguments:
            inputs: List of (batched) input words to be embedded
        """
        embedded = [tf.nn.embedding_lookup(self.embedding_matrix, o)
                    for o in inputs]
        return [self.dropout(e) for e in embedded]

    def initial_state(self):
        """Create the initial state of the decoder."""
        if len(self.encoders) == 0:
            return tf.zeros([self.rnn_size])

        encoders_out = tf.concat(1, [e.encoded for e in self.encoders])

        if self.project_encoder_outputs:
            encoders_out = self.encoder_projection(encoders_out)

        return self.dropout(encoders_out)

    def encoder_projection(self, encoded_states):
        """Creates a projection of concatenated encoder states

        Arguments:
            encoded_states: Tensor of concatenated states of input encoders
                            (batch x sum(states))
            rnn_size: Size of the projected vector
        """

        input_size = encoded_states.get_shape()[1].value
        output_size = self.rnn_size

        weights = tf.get_variable("encoder_projection_W", [input_size,
                                                           output_size])
        biases = tf.Variable(tf.zeros([output_size]),
                             name="encoder_projection_b")

        dropped_input = self.dropout(encoded_states)
        return tf.matmul(dropped_input, weights) + biases

    def dropout(self, var):
        """Perform dropout on a variable

        Arguments:
            var: The variable to perform the dropout on
        """
        return tf.nn.dropout(var, self.dropout_placeholder)

    def logit_function(self, state):
        """Compute logits on the vocabulary given the state

        Arguments:
            state: the state of the decoder
        """
        return tf.matmul(self.dropout(state), self.weights) + self.biases

    def get_loop_function(self):
        """Constructs a loop function for the decoder"""

        def basic_loop(previous_state, _):
            """Basic loop function. Projects state to logits, take the
            argmax of the logits, embed the word and perform dropout on the
            embedding vector.

            Arguments:
                previous_state: The state of the decoder
                i: Unused argument, number of the time step
            """
            output_activation = self.logit_function(previous_state)
            previous_word = tf.argmax(output_activation, 1)
            input_embedding = tf.nn.embedding_lookup(self.embedding_matrix,
                                                     previous_word)

            return self.dropout(input_embedding)

        return basic_loop

    def get_rnn_cell(self):
        """Returns a RNNCell object for this decoder"""

        return tf.nn.rnn_cell.GRUCell(self.rnn_size)

    def collect_attention_objects(self, encoders):
        """Collect attention objects of the given encoders

        Arguments:
            encoders: Encoders from which to take attention objects
        """
        if self.use_attention:
            return [e.attention_object for e in encoders if e.attention_object]
        else:
            return []

    def feed_dict(self, dataset, train=False):
        """Populate the feed dictionary for the decoder object

        Decoder placeholders:
            decoder_{x} for x in range(max_output+2):
                Training data placeholders. Starts with <s> and ends with </s>

            decoder_padding_weights{x} for x in range(max_output+1):
                Weights used for padding. (Float) tensor of ones and zeros.
                This tensor is one-item shorter than the other one since the
                decoder does not produce the first <s>.

            dropout_placeholder: Scalar placeholder for dropout probability.
                Has value 'dropout_keep_prob' from the constructor or 1
                in case we are decoding at run-time
        """
        # pylint: disable=invalid-name
        # fd is the common name for feed dictionary
        fd = {}
        sentences = dataset.get_series(self.data_id, allow_none=True)

        if sentences is not None:
            inputs, weights = self.vocabulary.sentences_to_tensor(
                sentences, self.max_output)

            for placeholder, weight in zip(self.train_weights, weights):
                fd[placeholder] = weight

            for placeholder, tensor in zip(self.train_inputs, inputs):
                fd[placeholder] = tensor

        if not train:
            fd[self.dropout_placeholder] = 1.0

        return fd
<commit_msg>added lint checks to decoder<commit_after>#tests: lint

import math
import tensorflow as tf

from neuralmonkey.decoding_function import attention_decoder

class Decoder(object):
    """A class that manages parts of the computation graph that are
    used for the decoding.
    """

    # pylint: disable=too-many-instance-attributes,too-many-locals
    # Big decoder cannot be simpler. Not sure if refactoring
    # it into smaller units would be helpful
    # Some locals may be turned to attributes

    def __init__(self, encoders, vocabulary, data_id, **kwargs):
        """Creates a new instance of the decoder

        Required arguments:
            encoders: List of encoders whose outputs will be decoded
            vocabulary: Output vocabulary
            data_id: Identifier of the data series fed to this decoder

        Keyword arguments:
            embedding_size: Size of embedding vectors. Default 200
            max_output: Maximum length of the output. Default 20
            rnn_size: When projection is used or when no encoder is supplied,
                      this is the size of the projected vector.
            dropout_keep_prob: Dropout keep probability. Default 1 (no dropout)

        Flags:
            use_attention: Indicates whether to use attention from encoders
            reuse_word_embeddings: Boolean flag specifying whether to reuse
                                   word embeddings. If True, word embeddings
                                   from the first encoder will be used
            project_encoder_outputs: Boolean flag whether to project output
                                     states of encoders

        """
        self.encoders = encoders
        self.vocabulary = vocabulary
        self.data_id = data_id

        self.max_output = kwargs.get("max_output", 20)
        self.embedding_size = kwargs.get("embedding_size", 200)
        self.name = kwargs.get("name", "decoder")
        dropout_keep_prob = kwargs.get("dropout_keep_prob", 1.0)

        self.use_attention = kwargs.get("use_attention", False)
        self.reuse_word_embeddings = kwargs.get("reuse_word_embeddings", False)
        self.project_encoder_outputs = kwargs.get("project_encoder_outputs",
                                                  False)

        if self.project_encoder_outputs or len(self.encoders) == 0:
            self.rnn_size = kwargs.get("rnn_size", 200)
        else:
            self.rnn_size = sum([e.encoded.get_shape()[1].value
                                 for e in self.encoders])

        ### Initialize model

        self.dropout_placeholder = tf.placeholder_with_default(
            dropout_keep_prob, [], "decoder_dropout_placeholder")

        state = self.initial_state()

        self.weights, self.biases = self.state_to_output()
        self.embedding_matrix = self.input_embeddings()

        cell = self.get_rnn_cell()
        attention_objects = self.collect_attention_objects(self.encoders)

        self.train_inputs, self.train_weights = self.training_placeholders()
        self.batch_size = tf.shape(self.train_inputs[0])
        train_targets = self.train_inputs[1:]

        ### Perform computation

        embedded_train_inputs = self.embed_inputs(self.train_inputs[:-1])

        self.train_rnn_outputs, _ = attention_decoder(
            embedded_train_inputs, state, attention_objects,
            self.embedding_size, cell, scope="att_train")

        runtime_inputs = self.runtime_inputs()
        loop_function = self.get_loop_function()

        self.runtime_rnn_outputs, _ = attention_decoder(
            runtime_inputs, state, attention_objects, self.embedding_size,
            cell, loop_function=loop_function, scope="att_runtime")

        ### KONEC decoder scope

        _, train_logits = self.decode(self.train_rnn_outputs)
        self.decoded, runtime_logits = self.decode(self.runtime_rnn_outputs)

        self.train_loss = tf.nn.seq2seq.sequence_loss(
            train_logits, train_targets, self.train_weights,
            self.vocabulary_size)

        self.runtime_loss = tf.nn.seq2seq.sequence_loss(
            runtime_logits, train_targets, self.train_weights,
            self.vocabulary_size)

        ### Learning step
        ### TODO was here only because of scheduled sampling.
        ### needs to be refactored out
        self.learning_step = tf.Variable(0, name="learning_step",
                                         trainable=False)

        ### Summaries
        self.init_summaries()

    @property
    def vocabulary_size(self):
        return len(self.vocabulary)

    @property
    def cost(self):
        return self.train_loss

    def init_summaries(self):
        """Initialize the summaries of the decoder

        TensorBoard summaries are collected into the following
        collections:

        - summary_train: collects statistics from the train-time
        - sumarry_val: collects OAstatistics while being tested on the
                 development data
        """
        tf.scalar_summary("train_loss_with_decoded_inputs", self.runtime_loss,
                          collections=["summary_train"])

        tf.scalar_summary("train_optimization_cost", self.train_loss,
                          collections=["summary_train"])

    def decode(self, rnn_states):
        """Decodes a sequence from a list of hidden states

        Arguments:
            rnn_states: hidden states
        """
        logits = [self.logit_function(s) for s in rnn_states]
        decoded = [tf.argmax(l[:, 1:], 1) + 1 for l in logits]

        return decoded, logits

    def training_placeholders(self):
        """Defines data placeholders for training the decoder"""

        inputs = [tf.placeholder(tf.int64, [None], name="decoder_{}".format(i))
                  for i in range(self.max_output + 2)]

        # one less than inputs
        weights = [tf.placeholder(tf.float32, [None],
                                  name="decoder_padding_weights_{}".format(i))
                   for i in range(self.max_output + 1)]

        return inputs, weights

    def runtime_inputs(self):
        """Defines data inputs for running trained decoder"""
        go_symbols = tf.ones(self.batch_size, dtype=tf.int32)
        go_embeds = tf.nn.embedding_lookup(self.embedding_matrix, go_symbols)

        inputs = [self.dropout(go_embeds)]
        inputs += [None for _ in range(self.max_output)]

        return inputs

    def state_to_output(self):
        """Create variables for projection of states to output vectors"""

        weights = tf.get_variable("state_to_word_W", [self.rnn_size,
                                                      self.vocabulary_size])

        biases = tf.Variable(tf.fill([self.vocabulary_size],
                                     - math.log(self.vocabulary_size)),
                             name="state_to_word_b")

        return weights, biases

    def input_embeddings(self):
        """Create variables and operations for embedding of input words

        If we are reusing word embeddings, this function takes
        them from the first encoder
        """
        if self.reuse_word_embeddings:
            return self.encoders[0].word_embeddings

        return tf.get_variable("word_embeddings", [self.vocabulary_size,
                                                   self.embedding_size])

    def embed_inputs(self, inputs):
        """Embed inputs using the decoder"s word embedding matrix

        Arguments:
            inputs: List of (batched) input words to be embedded
        """
        embedded = [tf.nn.embedding_lookup(self.embedding_matrix, o)
                    for o in inputs]
        return [self.dropout(e) for e in embedded]

    def initial_state(self):
        """Create the initial state of the decoder."""
        if len(self.encoders) == 0:
            return tf.zeros([self.rnn_size])

        encoders_out = tf.concat(1, [e.encoded for e in self.encoders])

        if self.project_encoder_outputs:
            encoders_out = self.encoder_projection(encoders_out)

        return self.dropout(encoders_out)

    def encoder_projection(self, encoded_states):
        """Creates a projection of concatenated encoder states

        Arguments:
            encoded_states: Tensor of concatenated states of input encoders
                            (batch x sum(states))
            rnn_size: Size of the projected vector
        """

        input_size = encoded_states.get_shape()[1].value
        output_size = self.rnn_size

        weights = tf.get_variable("encoder_projection_W", [input_size,
                                                           output_size])
        biases = tf.Variable(tf.zeros([output_size]),
                             name="encoder_projection_b")

        dropped_input = self.dropout(encoded_states)
        return tf.matmul(dropped_input, weights) + biases

    def dropout(self, var):
        """Perform dropout on a variable

        Arguments:
            var: The variable to perform the dropout on
        """
        return tf.nn.dropout(var, self.dropout_placeholder)

    def logit_function(self, state):
        """Compute logits on the vocabulary given the state

        Arguments:
            state: the state of the decoder
        """
        return tf.matmul(self.dropout(state), self.weights) + self.biases

    def get_loop_function(self):
        """Constructs a loop function for the decoder"""

        def basic_loop(previous_state, _):
            """Basic loop function. Projects state to logits, take the
            argmax of the logits, embed the word and perform dropout on the
            embedding vector.

            Arguments:
                previous_state: The state of the decoder
                i: Unused argument, number of the time step
            """
            output_activation = self.logit_function(previous_state)
            previous_word = tf.argmax(output_activation, 1)
            input_embedding = tf.nn.embedding_lookup(self.embedding_matrix,
                                                     previous_word)

            return self.dropout(input_embedding)

        return basic_loop

    def get_rnn_cell(self):
        """Returns a RNNCell object for this decoder"""

        return tf.nn.rnn_cell.GRUCell(self.rnn_size)

    def collect_attention_objects(self, encoders):
        """Collect attention objects of the given encoders

        Arguments:
            encoders: Encoders from which to take attention objects
        """
        if self.use_attention:
            return [e.attention_object for e in encoders if e.attention_object]
        else:
            return []

    def feed_dict(self, dataset, train=False):
        """Populate the feed dictionary for the decoder object

        Decoder placeholders:
            decoder_{x} for x in range(max_output+2):
                Training data placeholders. Starts with <s> and ends with </s>

            decoder_padding_weights{x} for x in range(max_output+1):
                Weights used for padding. (Float) tensor of ones and zeros.
                This tensor is one-item shorter than the other one since the
                decoder does not produce the first <s>.

            dropout_placeholder: Scalar placeholder for dropout probability.
                Has value 'dropout_keep_prob' from the constructor or 1
                in case we are decoding at run-time
        """
        # pylint: disable=invalid-name
        # fd is the common name for feed dictionary
        fd = {}
        sentences = dataset.get_series(self.data_id, allow_none=True)

        if sentences is not None:
            inputs, weights = self.vocabulary.sentences_to_tensor(
                sentences, self.max_output)

            for placeholder, weight in zip(self.train_weights, weights):
                fd[placeholder] = weight

            for placeholder, tensor in zip(self.train_inputs, inputs):
                fd[placeholder] = tensor

        if not train:
            fd[self.dropout_placeholder] = 1.0

        return fd
<|endoftext|>


<commit_before>import numpy
try:
    import scipy.sparse
    _scipy_available = True
except ImportError:
    _scipy_available = False

import cupy
from cupy import cusparse
from cupy.sparse import base
from cupy.sparse import csr
from cupy.sparse import data as sparse_data
from cupy.sparse import util

class coo_matrix(sparse_data._data_matrix):

    """COOrdinate format sparse matrix.

    Now it has only one initializer format below:

    ``coo_matrix((M, N), [dtype])``
        It constructs an empty matrix whose shape is ``(M, N)``. Default dtype
        is float64.

    ``coo_matrix((data, (row, col))``
        All ``data``, ``row`` and ``col`` are one-dimenaional
        :class:`cupy.ndarray`.

    Args:
        arg1: Arguments for the initializer.
        shape (tuple): Shape of a matrix. Its length must be two.
        dtype: Data type. It must be an argument of :class:`numpy.dtype`.
        copy (bool): If ``True``, copies of given data are always used.

    .. see::
       :class:`scipy.sparse.coo_matrix`

    """

    format = 'coo'

    def __init__(self, arg1, shape=None, dtype=None, copy=False):
        if shape is not None and len(shape) != 2:
            raise ValueError(
                'Only two-dimensional sparse arrays are supported.')

        if util.isshape(arg1):
            m, n = arg1
            m, n = int(m), int(n)
            data = cupy.zeros(0, dtype if dtype else 'd')
            row = cupy.zeros(0, dtype='i')
            col = cupy.zeros(0, dtype='i')
            # shape and copy argument is ignored
            shape = (m, n)
            copy = False

        elif isinstance(arg1, tuple) and len(arg1) == 2:
            try:
                data, (row, col) = arg1
            except (TypeError, ValueError):
                raise TypeError('invalid input format')

            if not (base.isdense(data) and data.ndim == 1 and
                    base.isdense(row) and row.ndim == 1 and
                    base.isdense(col) and col.ndim == 1):
                raise ValueError('row, column, and data arrays must be 1-D')
            if not (len(data) == len(row) == len(col)):
                raise ValueError(
                    'row, column, and data array must all be the same length')

            has_canonical_format = False

        elif isspmatrix_coo(arg1):
            data = arg1.data
            row = arg1.row
            col = arg1.col

            if shape is None:
                shape = arg1.shape

            has_canonical_format = arg1.has_canonical_format

        else:
            raise ValueError(
                'Only (data, (row, col)) format is supported')

        if dtype is None:
            dtype = data.dtype
        else:
            dtype = numpy.dtype(dtype)

        if dtype != 'f' and dtype != 'd':
            raise ValueError('Only float32 and float64 are supported')

        data = data.astype(dtype, copy=copy)
        row = row.astype('i', copy=copy)
        col = col.astype('i', copy=copy)

        if shape is None:
            if len(row) == 0 or len(col) == 0:
                raise ValueError(
                    'cannot infer dimensions from zero sized index arrays')
            shape = (int(row.max()) + 1, int(col.max()) + 1)

        if len(data) > 0:
            if row.max() >= shape[0]:
                raise ValueError('row index exceeds matrix dimensions')
            if col.max() >= shape[1]:
                raise ValueError('column index exceeds matrix dimensions')
            if row.min() < 0:
                raise ValueError('negative row index found')
            if col.min() < 0:
                raise ValueError('negative column index found')

        sparse_data._data_matrix.__init__(self, data)
        self.row = row
        self.col = col
        self._shape = shape
        self._has_canonical_format = has_canonical_format

    def _with_data(self, data):
        return coo_matrix(
            (data, (self.row.copy(), self.col.copy())), shape=self.shape)

    @property
    def has_canonical_format(self):
        return self._has_canonical_format

    def get_shape(self):
        """Returns the shape of the matrix.

        Returns:
            tuple: Shape of the matrix.
        """
        return self._shape

    def getnnz(self, axis=None):
        """Returns the number of stored values, including explicit zeros."""
        if axis is None:
            return self.data.size
        else:
            raise ValueError

    def get(self, stream=None):
        """Returns a copy of the array on host memory.

        Args:
            stream (cupy.cuda.Stream): CUDA stream object. If it is given, the
                copy runs asynchronously. Otherwise, the copy is synchronous.

        Returns:
            scipy.sparse.coo_matrix: Copy of the array on host memory.

        """
        if not _scipy_available:
            raise RuntimeError('scipy is not available')

        data = self.data.get(stream)
        row = self.row.get(stream)
        col = self.col.get(stream)
        return scipy.sparse.coo_matrix(
            (data, (row, col)), shape=self.shape)

    def sum_duplicates(self):
        if self._has_canonical_format:
            return
        keys = cupy.stack([self.row, self.col])
        order = cupy.lexsort(keys)
        src_data = self.data[order]
        src_row = self.row[order]
        src_col = self.col[order]
        diff = cupy.ElementwiseKernel(
            'raw int32 row, raw int32 col',
            'int32 diff',
            '''
            int index;
            if (i == 0 || row[i - 1] == row[i] && col[i - 1] == col[i]) {
              diff = 0;
            } else {
              diff = 1;
            }
            ''',
            'sum_duplicates_diff'
        )(src_row, src_col, size=self.row.size)
        index = cupy.cumsum(diff, dtype='i')
        size = int(index[-1]) + 1
        data = cupy.zeros(size, dtype=self.data.dtype)
        row = cupy.empty(size, dtype='i')
        col = cupy.empty(size, dtype='i')
        cupy.ElementwiseKernel(
            'T src_data, int32 src_row, int32 src_col, int32 index',
            'raw T data, raw int32 row, raw int32 col',
            '''
            atomicAdd(&data[index], src_data);
            row[index] = src_row;
            col[index] = src_col;
            ''',
            'sum_duplicates_assign'
        )(src_data, src_row, src_col, index, data, row, col)

        self.data = data
        self.row = row
        self.col = col
        self._has_canonical_format = True

    def toarray(self, order=None, out=None):
        """Returns a dense matrix representing the same value.

        Args:
            order (str): Not supported.
            out: Not supported.

        Returns:
            cupy.ndarray: Dense array representing the same value.

        .. seealso:: :func:`cupy.sparse.coo_array.toarray`

        """
        return self.tocsr().toarray(order=order, out=out)

    def tocoo(self, copy=False):
        """Converts the matrix to COOdinate format.

        Args:
            copy (bool): If ``False``, it shares data arrays as much as
                possible.

        Returns:
            cupy.sparse.coo_matrix: Converted matrix.

        """
        if copy:
            return self.copy()
        else:
            return self

    def tocsc(self, copy=False):
        """Converts the matrix to Compressed Sparse Column format.

        Args:
            copy (bool): If ``False``, it shares data arrays as much as
                possible. Actually this option is ignored because all
                arrays in a matrix cannot be shared in coo to csc conversion.

        Returns:
            cupy.sparse.csc_matrix: Converted matrix.

        """
        return self.T.tocsr().T

    def tocsr(self, copy=False):
        """Converts the matrix to Compressed Sparse Row format.

        Args:
            copy (bool): If ``False``, it shares data arrays as much as
                possible. Actually this option is ignored because all
                arrays in a matrix cannot be shared in coo to csr conversion.

        Returns:
            cupy.sparse.csr_matrix: Converted matrix.

        """
        if self.nnz == 0:
            return csr.csr_matrix(self.shape, dtype=self.dtype)
        # copy is ignored because coosort method breaks an original.
        x = self.copy()
        cusparse.coosort(x)
        return cusparse.coo2csr(x)

    def transpose(self, axes=None, copy=False):
        """Returns a transpose matrix.

        Args:
            axes: This option is not supported.
            copy (bool): If ``True``, a returned matrix shares no data.
                Otherwise, it shared data arrays as much as possible.

        Returns:
            cupy.sparse.spmatrix: Transpose matrix.

        """
        if axes is not None:
            raise ValueError(
                'Sparse matrices do not support an \'axes\' parameter because '
                'swapping dimensions is the only logical permutation.')
        shape = self.shape[1], self.shape[0]
        return coo_matrix(
            (self.data, (self.col, self.row)), shape=shape, copy=copy)

def isspmatrix_coo(x):
    """Checks if a given matrix is of COO format.

    Returns:
        bool: Returns if ``x`` is :class:`cupy.sparse.coo_matrix`.

    """
    return isinstance(x, coo_matrix)
<commit_msg>Write docstring`<commit_after>import numpy
try:
    import scipy.sparse
    _scipy_available = True
except ImportError:
    _scipy_available = False

import cupy
from cupy import cusparse
from cupy.sparse import base
from cupy.sparse import csr
from cupy.sparse import data as sparse_data
from cupy.sparse import util

class coo_matrix(sparse_data._data_matrix):

    """COOrdinate format sparse matrix.

    Now it has only one initializer format below:

    ``coo_matrix((M, N), [dtype])``
        It constructs an empty matrix whose shape is ``(M, N)``. Default dtype
        is float64.

    ``coo_matrix((data, (row, col))``
        All ``data``, ``row`` and ``col`` are one-dimenaional
        :class:`cupy.ndarray`.

    Args:
        arg1: Arguments for the initializer.
        shape (tuple): Shape of a matrix. Its length must be two.
        dtype: Data type. It must be an argument of :class:`numpy.dtype`.
        copy (bool): If ``True``, copies of given data are always used.

    .. see::
       :class:`scipy.sparse.coo_matrix`

    """

    format = 'coo'

    def __init__(self, arg1, shape=None, dtype=None, copy=False):
        if shape is not None and len(shape) != 2:
            raise ValueError(
                'Only two-dimensional sparse arrays are supported.')

        if util.isshape(arg1):
            m, n = arg1
            m, n = int(m), int(n)
            data = cupy.zeros(0, dtype if dtype else 'd')
            row = cupy.zeros(0, dtype='i')
            col = cupy.zeros(0, dtype='i')
            # shape and copy argument is ignored
            shape = (m, n)
            copy = False

        elif isinstance(arg1, tuple) and len(arg1) == 2:
            try:
                data, (row, col) = arg1
            except (TypeError, ValueError):
                raise TypeError('invalid input format')

            if not (base.isdense(data) and data.ndim == 1 and
                    base.isdense(row) and row.ndim == 1 and
                    base.isdense(col) and col.ndim == 1):
                raise ValueError('row, column, and data arrays must be 1-D')
            if not (len(data) == len(row) == len(col)):
                raise ValueError(
                    'row, column, and data array must all be the same length')

            has_canonical_format = False

        elif isspmatrix_coo(arg1):
            data = arg1.data
            row = arg1.row
            col = arg1.col

            if shape is None:
                shape = arg1.shape

            has_canonical_format = arg1.has_canonical_format

        else:
            raise ValueError(
                'Only (data, (row, col)) format is supported')

        if dtype is None:
            dtype = data.dtype
        else:
            dtype = numpy.dtype(dtype)

        if dtype != 'f' and dtype != 'd':
            raise ValueError('Only float32 and float64 are supported')

        data = data.astype(dtype, copy=copy)
        row = row.astype('i', copy=copy)
        col = col.astype('i', copy=copy)

        if shape is None:
            if len(row) == 0 or len(col) == 0:
                raise ValueError(
                    'cannot infer dimensions from zero sized index arrays')
            shape = (int(row.max()) + 1, int(col.max()) + 1)

        if len(data) > 0:
            if row.max() >= shape[0]:
                raise ValueError('row index exceeds matrix dimensions')
            if col.max() >= shape[1]:
                raise ValueError('column index exceeds matrix dimensions')
            if row.min() < 0:
                raise ValueError('negative row index found')
            if col.min() < 0:
                raise ValueError('negative column index found')

        sparse_data._data_matrix.__init__(self, data)
        self.row = row
        self.col = col
        self._shape = shape
        self._has_canonical_format = has_canonical_format

    def _with_data(self, data):
        return coo_matrix(
            (data, (self.row.copy(), self.col.copy())), shape=self.shape)

    @property
    def has_canonical_format(self):
        return self._has_canonical_format

    def get_shape(self):
        """Returns the shape of the matrix.

        Returns:
            tuple: Shape of the matrix.
        """
        return self._shape

    def getnnz(self, axis=None):
        """Returns the number of stored values, including explicit zeros."""
        if axis is None:
            return self.data.size
        else:
            raise ValueError

    def get(self, stream=None):
        """Returns a copy of the array on host memory.

        Args:
            stream (cupy.cuda.Stream): CUDA stream object. If it is given, the
                copy runs asynchronously. Otherwise, the copy is synchronous.

        Returns:
            scipy.sparse.coo_matrix: Copy of the array on host memory.

        """
        if not _scipy_available:
            raise RuntimeError('scipy is not available')

        data = self.data.get(stream)
        row = self.row.get(stream)
        col = self.col.get(stream)
        return scipy.sparse.coo_matrix(
            (data, (row, col)), shape=self.shape)

    def sum_duplicates(self):
        """Eliminate duplicate matrix entries by adding them together.

        .. see::
           :func:`scipy.sparse.coo_matrix.sum_duplicates`

        """
        if self._has_canonical_format:
            return
        keys = cupy.stack([self.row, self.col])
        order = cupy.lexsort(keys)
        src_data = self.data[order]
        src_row = self.row[order]
        src_col = self.col[order]
        diff = cupy.ElementwiseKernel(
            'raw int32 row, raw int32 col',
            'int32 diff',
            '''
            int index;
            if (i == 0 || row[i - 1] == row[i] && col[i - 1] == col[i]) {
              diff = 0;
            } else {
              diff = 1;
            }
            ''',
            'sum_duplicates_diff'
        )(src_row, src_col, size=self.row.size)
        index = cupy.cumsum(diff, dtype='i')
        size = int(index[-1]) + 1
        data = cupy.zeros(size, dtype=self.data.dtype)
        row = cupy.empty(size, dtype='i')
        col = cupy.empty(size, dtype='i')
        cupy.ElementwiseKernel(
            'T src_data, int32 src_row, int32 src_col, int32 index',
            'raw T data, raw int32 row, raw int32 col',
            '''
            atomicAdd(&data[index], src_data);
            row[index] = src_row;
            col[index] = src_col;
            ''',
            'sum_duplicates_assign'
        )(src_data, src_row, src_col, index, data, row, col)

        self.data = data
        self.row = row
        self.col = col
        self._has_canonical_format = True

    def toarray(self, order=None, out=None):
        """Returns a dense matrix representing the same value.

        Args:
            order (str): Not supported.
            out: Not supported.

        Returns:
            cupy.ndarray: Dense array representing the same value.

        .. seealso:: :func:`cupy.sparse.coo_array.toarray`

        """
        return self.tocsr().toarray(order=order, out=out)

    def tocoo(self, copy=False):
        """Converts the matrix to COOdinate format.

        Args:
            copy (bool): If ``False``, it shares data arrays as much as
                possible.

        Returns:
            cupy.sparse.coo_matrix: Converted matrix.

        """
        if copy:
            return self.copy()
        else:
            return self

    def tocsc(self, copy=False):
        """Converts the matrix to Compressed Sparse Column format.

        Args:
            copy (bool): If ``False``, it shares data arrays as much as
                possible. Actually this option is ignored because all
                arrays in a matrix cannot be shared in coo to csc conversion.

        Returns:
            cupy.sparse.csc_matrix: Converted matrix.

        """
        return self.T.tocsr().T

    def tocsr(self, copy=False):
        """Converts the matrix to Compressed Sparse Row format.

        Args:
            copy (bool): If ``False``, it shares data arrays as much as
                possible. Actually this option is ignored because all
                arrays in a matrix cannot be shared in coo to csr conversion.

        Returns:
            cupy.sparse.csr_matrix: Converted matrix.

        """
        if self.nnz == 0:
            return csr.csr_matrix(self.shape, dtype=self.dtype)
        # copy is ignored because coosort method breaks an original.
        x = self.copy()
        cusparse.coosort(x)
        return cusparse.coo2csr(x)

    def transpose(self, axes=None, copy=False):
        """Returns a transpose matrix.

        Args:
            axes: This option is not supported.
            copy (bool): If ``True``, a returned matrix shares no data.
                Otherwise, it shared data arrays as much as possible.

        Returns:
            cupy.sparse.spmatrix: Transpose matrix.

        """
        if axes is not None:
            raise ValueError(
                'Sparse matrices do not support an \'axes\' parameter because '
                'swapping dimensions is the only logical permutation.')
        shape = self.shape[1], self.shape[0]
        return coo_matrix(
            (self.data, (self.col, self.row)), shape=shape, copy=copy)

def isspmatrix_coo(x):
    """Checks if a given matrix is of COO format.

    Returns:
        bool: Returns if ``x`` is :class:`cupy.sparse.coo_matrix`.

    """
    return isinstance(x, coo_matrix)
<|endoftext|>


<commit_before># No shebang line, this module is meant to be imported
#
# Copyright 2014 Oliver Palmer
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import sys
from platform import platform

try:
    from httplib import OK, CREATED
except ImportError:  # pragma: no cover
    from http.client import OK, CREATED

from pyfarm.agent.sysinfo.system import system_identifier, operating_system
from pyfarm.agent.testutil import TestCase
from pyfarm.agent.config import config
from pyfarm.agent.service import Agent
from pyfarm.agent.sysinfo import network

# TODO: need better tests, these are a little rudimentary at the moment
class TestAgentBasicMethods(TestCase):
    def test_agent_api_url(self):
        config["agent-id"] = 1
        agent = Agent()
        self.assertEqual(
            agent.agent_api(),
            "%s/agents/1" % config["master_api"])

    def test_agent_api_url_keyerror(self):
        agent = Agent()
        config.pop("agent-id")
        self.assertIsNone(agent.agent_api())

    def test_system_data(self):
        config["remote_ip"] = os.urandom(16).encode("hex")
        expected = {
            "current_assignments": {},
            "systemid": system_identifier(),
            "hostname": config["agent_hostname"],
            "version": config.version,
            "ram": config["agent_ram"],
            "cpus": config["agent_cpus"],
            "os_class": operating_system(),
            "os_fullname": platform(),
            "remote_ip": config["remote_ip"],
            "port": config["agent_api_port"],
            "time_offset": config["agent_time_offset"],
            "state": config["state"],
            "mac_addresses": list(network.mac_addresses())}

        agent = Agent()
        system_data = agent.system_data()
        self.assertApproximates(
            system_data.pop("free_ram"), config["free_ram"], 64)
        self.assertEqual(system_data, expected)

class TestRunAgent(TestCase):
    def test_created(self):
        self.skipTest("NOT IMPLEMENTED")

    def test_updated(self):
        self.skipTest("NOT IMPLEMENTED")
<commit_msg>Fix test<commit_after># No shebang line, this module is meant to be imported
#
# Copyright 2014 Oliver Palmer
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import sys
from platform import platform

try:
    from httplib import OK, CREATED
except ImportError:  # pragma: no cover
    from http.client import OK, CREATED

from pyfarm.agent.sysinfo.system import system_identifier, operating_system
from pyfarm.agent.sysinfo import cpu
from pyfarm.agent.testutil import TestCase
from pyfarm.agent.config import config
from pyfarm.agent.service import Agent
from pyfarm.agent.sysinfo import network

# TODO: need better tests, these are a little rudimentary at the moment
class TestAgentBasicMethods(TestCase):
    def test_agent_api_url(self):
        config["agent-id"] = 1
        agent = Agent()
        self.assertEqual(
            agent.agent_api(),
            "%s/agents/1" % config["master_api"])

    def test_agent_api_url_keyerror(self):
        agent = Agent()
        config.pop("agent-id")
        self.assertIsNone(agent.agent_api())

    def test_system_data(self):
        config["remote_ip"] = os.urandom(16).encode("hex")
        expected = {
            "current_assignments": {},
            "systemid": system_identifier(),
            "hostname": config["agent_hostname"],
            "version": config.version,
            "ram": config["agent_ram"],
            "cpus": config["agent_cpus"],
            "os_class": operating_system(),
            "os_fullname": platform(),
            "cpu_name": cpu.cpu_name(),
            "remote_ip": config["remote_ip"],
            "port": config["agent_api_port"],
            "time_offset": config["agent_time_offset"],
            "state": config["state"],
            "mac_addresses": list(network.mac_addresses())}

        agent = Agent()
        system_data = agent.system_data()
        self.assertApproximates(
            system_data.pop("free_ram"), config["free_ram"], 64)
        self.assertEqual(system_data, expected)

class TestRunAgent(TestCase):
    def test_created(self):
        self.skipTest("NOT IMPLEMENTED")

    def test_updated(self):
        self.skipTest("NOT IMPLEMENTED")
<|endoftext|>


<commit_before># imports
import django
import hashlib
from inspect import isfunction
import json
from lxml import etree, isoschematron
import os
import shutil
import sys
from types import ModuleType

# import Row from pyspark
from pyspark.sql import Row
from pyspark.sql.types import StringType, IntegerType
from pyspark.sql.functions import udf

# init django settings file to retrieve settings
os.environ['DJANGO_SETTINGS_MODULE'] = 'combine.settings'
sys.path.append('/opt/combine')
django.setup()
from django.conf import settings

# import select models from Core
from core.models import Job, ValidationScenario

class ValidationScenarioSpark(object):

	'''
	Class to organize methods and attributes used for running validation scenarios
	'''

	def __init__(self, spark=None, job=None, records_df=None, validation_scenarios=None):

		'''
		Args:
			spark (pyspark.sql.session.SparkSession): spark instance from static job methods
			job (core.models.Job): Job instance
			records_df (pyspark.sql.DataFrame): records as pyspark DataFrame
			validation_scenarios (list): list of ValidationScenario job ids as integers
		'''

		self.spark = spark
		self.job = job
		self.records_df = records_df
		self.validation_scenarios = validation_scenarios

	def run_record_validation_scenarios(self):

		'''
		Function to run validation scenarios
		Results are written to RecordValidation table, one result, per record, per failed validation test.

		Validation tests may be of type:
			- 'sch': Schematron based validation, performed with lxml etree
			- 'python': custom python code snippets

		Args:
			None

		Returns:
			None
				- writes validation fails to RecordValidation table
		'''

		# loop through validation scenarios and fire validation type specific method
		for vs_id in self.validation_scenarios:

			# get validation scenario
			vs = ValidationScenario.objects.get(pk=vs_id)
			vs_id = vs.id
			vs_filepath = vs.filepath

			# schematron based validation scenario
			if vs.validation_type == 'sch':

				# run udf map
				udf_func = self.validate_schematron_udf # must pass static method not attached to self
				validation_fails_rdd = self.records_df.rdd.\
					map(lambda row: udf_func(vs_id, vs_filepath, row))\
					.filter(lambda row: row is not None)

			# python based validation scenario
			elif vs.validation_type == 'python':

				# parse user defined functions from validation scenario payload
				temp_pyvs = ModuleType('temp_pyvs')
				exec(vs.payload, temp_pyvs.__dict__)

				# get defined functions
				pyvs_funcs = []
				for attr in dir(temp_pyvs):
					attr = getattr(temp_pyvs, attr)
					if isfunction(attr):
						pyvs_funcs.append(attr)

				udf_func = self.validate_python_udf # must pass static method not attached to self
				validation_fails_rdd = self.records_df.rdd.\
					map(lambda row: udf_func(vs_id, pyvs_funcs, row))\
					.filter(lambda row: row is not None)

			# finally, write to DB if validation failures
			if not validation_fails_rdd.isEmpty():
				validation_fails_rdd.toDF().write.jdbc(
					settings.COMBINE_DATABASE['jdbc_url'],
					'core_recordvalidation',
					properties=settings.COMBINE_DATABASE,
					mode='append')

	@staticmethod
	def validate_schematron_udf(vs_id, vs_filepath, row):

			# parse schematron
			sct_doc = etree.parse(vs_filepath)
			validator = isoschematron.Schematron(sct_doc, store_report=True)

			# get document xml
			record_xml = etree.fromstring(row.document.encode('utf-8'))

			# validate
			is_valid = validator.validate(record_xml)

			# if not valid, prepare Row
			if not is_valid:

				# prepare fail_dict
				fail_dict = {
					'count':0,
					'failures':[]
				}

				# get failures
				report_root = validator.validation_report.getroot()
				fails = report_root.findall('svrl:failed-assert', namespaces=report_root.nsmap)

				# log count
				fail_dict['count'] = len(fails)

				# loop through fails and add to dictionary
				for fail in fails:
					fail_text_elem = fail.find('svrl:text', namespaces=fail.nsmap)
					fail_dict['failures'].append(fail_text_elem.text)

				return Row(
					record_id=int(row.id),
					validation_scenario_id=int(vs_id),
					valid=0,
					results_payload=json.dumps(fail_dict),
					fail_count=fail_dict['count']
				)

	@staticmethod
	def validate_python_udf(vs_id, pyvs_funcs, row):

		'''
		Loop through test functions and aggregate in fail_dict to return with Row

		Args:
			vs_id (int): integer of validation scenario
			pyvs_funcs (list): list of functions imported from user created python validation scenario payload
			row ():
		'''

		# locally define class to be used
		class PythonRecordValidationBase(object):

			'''
			Simple class to provide an object with parsed metadata for user defined functions
			'''

			def __init__(self, row):

				# row
				self._row = row

				# get combine id
				self.id = row.id

				# get record id
				self.record_id = row.record_id

				# document string
				self.document = row.document.encode('utf-8')

				# parse XML string, save
				self.xml = etree.fromstring(self.document)

				# get namespace map, popping None values
				_nsmap = self.xml.nsmap.copy()
				try:
					_nsmap.pop(None)
				except:
					pass
				self.nsmap = _nsmap

		# prvb
		prvb = PythonRecordValidationBase(row)

		# prepare fail_dict
		fail_dict = {
			'count':0,
			'failures':[]
		}

		# loop through functions
		for func in pyvs_funcs:

			# run test
			test_result = func(prvb)

			# if fail, append
			if test_result != True:
				fail_dict['count'] += 1
				fail_dict['failures'].append(test_result)

		# if failures, return Row
		if fail_dict['count'] > 0:
			# return row
			return Row(
				record_id=int(row.id),
				validation_scenario_id=int(vs_id),
				valid=0,
				results_payload=json.dumps(fail_dict),
				fail_count=fail_dict['count']
			)

<commit_msg>pyvs funcs must begin with 'test'<commit_after># imports
import django
import hashlib
from inspect import isfunction
import json
from lxml import etree, isoschematron
import os
import shutil
import sys
from types import ModuleType

# import Row from pyspark
from pyspark.sql import Row
from pyspark.sql.types import StringType, IntegerType
from pyspark.sql.functions import udf

# init django settings file to retrieve settings
os.environ['DJANGO_SETTINGS_MODULE'] = 'combine.settings'
sys.path.append('/opt/combine')
django.setup()
from django.conf import settings

# import select models from Core
from core.models import Job, ValidationScenario

class ValidationScenarioSpark(object):

	'''
	Class to organize methods and attributes used for running validation scenarios
	'''

	def __init__(self, spark=None, job=None, records_df=None, validation_scenarios=None):

		'''
		Args:
			spark (pyspark.sql.session.SparkSession): spark instance from static job methods
			job (core.models.Job): Job instance
			records_df (pyspark.sql.DataFrame): records as pyspark DataFrame
			validation_scenarios (list): list of ValidationScenario job ids as integers
		'''

		self.spark = spark
		self.job = job
		self.records_df = records_df
		self.validation_scenarios = validation_scenarios

	def run_record_validation_scenarios(self):

		'''
		Function to run validation scenarios
		Results are written to RecordValidation table, one result, per record, per failed validation test.

		Validation tests may be of type:
			- 'sch': Schematron based validation, performed with lxml etree
			- 'python': custom python code snippets

		Args:
			None

		Returns:
			None
				- writes validation fails to RecordValidation table
		'''

		# loop through validation scenarios and fire validation type specific method
		for vs_id in self.validation_scenarios:

			# get validation scenario
			vs = ValidationScenario.objects.get(pk=vs_id)
			vs_id = vs.id
			vs_filepath = vs.filepath

			# schematron based validation scenario
			if vs.validation_type == 'sch':

				# run udf map
				udf_func = self.validate_schematron_udf # must pass static method not attached to self
				validation_fails_rdd = self.records_df.rdd.\
					map(lambda row: udf_func(vs_id, vs_filepath, row))\
					.filter(lambda row: row is not None)

			# python based validation scenario
			elif vs.validation_type == 'python':

				# parse user defined functions from validation scenario payload
				temp_pyvs = ModuleType('temp_pyvs')
				exec(vs.payload, temp_pyvs.__dict__)

				# get defined functions
				pyvs_funcs = []
				test_labeled_attrs = [ attr for attr in dir(temp_pyvs) if attr.lower().startswith('test') ]
				for attr in test_labeled_attrs:
					attr = getattr(temp_pyvs, attr)
					if isfunction(attr):
						pyvs_funcs.append(attr)

				udf_func = self.validate_python_udf # must pass static method not attached to self
				validation_fails_rdd = self.records_df.rdd.\
					map(lambda row: udf_func(vs_id, pyvs_funcs, row))\
					.filter(lambda row: row is not None)

			# finally, write to DB if validation failures
			if not validation_fails_rdd.isEmpty():
				validation_fails_rdd.toDF().write.jdbc(
					settings.COMBINE_DATABASE['jdbc_url'],
					'core_recordvalidation',
					properties=settings.COMBINE_DATABASE,
					mode='append')

	@staticmethod
	def validate_schematron_udf(vs_id, vs_filepath, row):

			# parse schematron
			sct_doc = etree.parse(vs_filepath)
			validator = isoschematron.Schematron(sct_doc, store_report=True)

			# get document xml
			record_xml = etree.fromstring(row.document.encode('utf-8'))

			# validate
			is_valid = validator.validate(record_xml)

			# if not valid, prepare Row
			if not is_valid:

				# prepare fail_dict
				fail_dict = {
					'count':0,
					'failures':[]
				}

				# get failures
				report_root = validator.validation_report.getroot()
				fails = report_root.findall('svrl:failed-assert', namespaces=report_root.nsmap)

				# log count
				fail_dict['count'] = len(fails)

				# loop through fails and add to dictionary
				for fail in fails:
					fail_text_elem = fail.find('svrl:text', namespaces=fail.nsmap)
					fail_dict['failures'].append(fail_text_elem.text)

				return Row(
					record_id=int(row.id),
					validation_scenario_id=int(vs_id),
					valid=0,
					results_payload=json.dumps(fail_dict),
					fail_count=fail_dict['count']
				)

	@staticmethod
	def validate_python_udf(vs_id, pyvs_funcs, row):

		'''
		Loop through test functions and aggregate in fail_dict to return with Row

		Args:
			vs_id (int): integer of validation scenario
			pyvs_funcs (list): list of functions imported from user created python validation scenario payload
			row ():
		'''

		# locally define class to be used
		class PythonRecordValidationBase(object):

			'''
			Simple class to provide an object with parsed metadata for user defined functions
			'''

			def __init__(self, row):

				# row
				self._row = row

				# get combine id
				self.id = row.id

				# get record id
				self.record_id = row.record_id

				# document string
				self.document = row.document.encode('utf-8')

				# parse XML string, save
				self.xml = etree.fromstring(self.document)

				# get namespace map, popping None values
				_nsmap = self.xml.nsmap.copy()
				try:
					_nsmap.pop(None)
				except:
					pass
				self.nsmap = _nsmap

		# prvb
		prvb = PythonRecordValidationBase(row)

		# prepare fail_dict
		fail_dict = {
			'count':0,
			'failures':[]
		}

		# loop through functions
		for func in pyvs_funcs:

			# run test
			test_result = func(prvb)

			# if fail, append
			if test_result != True:
				fail_dict['count'] += 1
				fail_dict['failures'].append(test_result)

		# if failures, return Row
		if fail_dict['count'] > 0:
			# return row
			return Row(
				record_id=int(row.id),
				validation_scenario_id=int(vs_id),
				valid=0,
				results_payload=json.dumps(fail_dict),
				fail_count=fail_dict['count']
			)

<|endoftext|>