<commit_before>from __future__ import absolute_import

import abc
from functools import reduce
from operator import mul

import numpy as np
from sympy import Eq
import ctypes
from ctypes.util import find_library
import mmap

from devito.parameters import configuration
from devito.tools import as_tuple, numpy_to_ctypes
from devito.logger import logger
import devito

__all__ = ['ALLOC_FLAT', 'ALLOC_NUMA_LOCAL', 'ALLOC_NUMA_ANY',
           'ALLOC_KNL_MCDRAM', 'ALLOC_KNL_DRAM', 'ALLOC_GUARD']

class MemoryAllocator(object):

    """Abstract class defining the interface to memory allocators."""

    __metaclass__ = abc.ABCMeta

    is_Posix = False
    is_Numa = False

    _attempted_init = False
    lib = None

    @classmethod
    def available(cls):
        if cls._attempted_init is False:
            cls.initialize()
            cls._attempted_init = True
        return cls.lib is not None

    @classmethod
    def initialize(cls):
        """
        Initialize the MemoryAllocator.

        .. note::

            This must be implemented by all subclasses of MemoryAllocator.
        """
        return

    def alloc(self, shape, dtype):
        """
        Allocate memory.

        :param shape: Shape of the array to allocate.
        :param dtype: Numpy datatype to allocate.

        :returns (pointer, free_handle): The first element of the tuple is the reference
                                         that can be used to access the data as a ctypes
                                         object. The second element is an opaque object
                                         that is needed only for the call to free.
        """
        size = int(reduce(mul, shape))
        ctype = numpy_to_ctypes(dtype)

        c_pointer, free_args = self._alloc_C_libcall(size, ctype)
        if c_pointer is None:
            raise RuntimeError("Unable to allocate %d elements in memory", str(size))

        c_pointer = ctypes.cast(c_pointer, np.ctypeslib.ndpointer(dtype=dtype,
                                                                  shape=shape))
        pointer = np.ctypeslib.as_array(c_pointer, shape=shape)

        return (pointer, free_args)

    @abc.abstractmethod
    def _alloc_C_libcall(self, size, ctype):
        """
        Perform the actual memory allocation by calling a C function.
        Should return a 2-tuple (c_pointer, free_args), where the free args
        are what is handed back to free() later to deallocate.

        .. note::

            This method should be implemented by a subclass.
        """
        return

    @abc.abstractmethod
    def free(self, *args):
        """
        Free memory previously allocated with ``self.alloc``.

        Arguments are provided exactly as returned in the second
        element of the tuple returned by _alloc_C_libcall

        .. note::

            This method should be implemented by a subclass.
        """
        return

class PosixAllocator(MemoryAllocator):

    """
    Memory allocator based on ``posix`` functions. The allocated memory is
    aligned to page boundaries.
    """

    is_Posix = True

    @classmethod
    def initialize(cls):
        handle = find_library('c')
        if handle is not None:
            cls.lib = ctypes.CDLL(handle)

    def _alloc_C_libcall(self, size, ctype):
        if not self.available():
            raise RuntimeError("Couldn't find `libc`'s `posix_memalign` to "
                               "allocate memory")
        c_bytesize = ctypes.c_ulong(size * ctypes.sizeof(ctype))
        c_pointer = ctypes.cast(ctypes.c_void_p(), ctypes.c_void_p)
        alignment = self.lib.getpagesize()
        ret = self.lib.posix_memalign(ctypes.byref(c_pointer), alignment, c_bytesize)
        if ret == 0:
            return c_pointer, (c_pointer, )
        else:
            return None, None

    def free(self, c_pointer):
        self.lib.free(c_pointer)

class GuardAllocator(MemoryAllocator):
    """
    Memory allocator based on ``posix`` functions. The allocated memory is
    aligned to page boundaries.  Additionally, it allocates extra memory
    before and after the data, and configures it so that an SEGV is thrown
    immediately if an out-of-bounds access occurs.

    Further, the remainder region of the last page (which cannot be
    protected), is poisoned with NaNs.
    """

    is_Posix = True

    @classmethod
    def initialize(cls):
        handle = find_library('c')
        if handle is not None:
            cls.lib = ctypes.CDLL(handle)

    def __init__(self, padding=1048576):
        self.padding = padding

    def _alloc_C_libcall(self, size, ctype):
        if not self.available():
            raise RuntimeError("Couldn't find `libc`'s `posix_memalign` to "
                               "allocate memory")

        pagesize = self.lib.getpagesize()
        assert self.padding % pagesize == 0

        npages_pad = self.padding // pagesize
        nbytes_user = size * ctypes.sizeof(ctype)
        npages_user = (nbytes_user + pagesize - 1) // pagesize

        npages_alloc = 2*npages_pad + npages_user

        c_bytesize = ctypes.c_ulong(npages_alloc * pagesize)
        c_pointer = ctypes.cast(ctypes.c_void_p(), ctypes.c_void_p)
        alignment = self.lib.getpagesize()
        ret = self.lib.posix_memalign(ctypes.byref(c_pointer), alignment, c_bytesize)
        if ret != 0:
            return None, None

        # generate pointers to the left padding, the user data, and the right pad
        padleft_pointer = c_pointer
        c_pointer = ctypes.c_void_p(c_pointer.value + self.padding)
        padright_pointer = ctypes.c_void_p(c_pointer.value + npages_user * pagesize)

        # and set the permissions on the pad memory to 0 (no access)
        # if these fail, don't worry about failing the entire allocation
        c_padsize = ctypes.c_ulong(self.padding)
        print("protect", padleft_pointer, padright_pointer)
        if self.lib.mprotect(padleft_pointer, c_padsize, ctypes.c_int(0)):
            logger.warning("couldn't protect memory")
        if self.lib.mprotect(padright_pointer, c_padsize, ctypes.c_int(0)):
            logger.warning("couldn't protect memory")

        # if there is a multiple of 4 bytes left, use the code below to poison
        # the memory
        if nbytes_user % 4 == 0:
            poison_size = npages_user*pagesize - nbytes_user
            intp_type = ctypes.POINTER(ctypes.c_int)
            poison_ptr = ctypes.cast(ctypes.c_void_p(c_pointer.value + nbytes_user),
                                     intp_type)

            # for both float32 and float64, a sequence of -100 int32s represents NaNs,
            # at least on little-endian architectures.  It shouldn't matter what we
            # put in there, anyway
            for i in range(poison_size // 4):
                poison_ptr[i] = -100

        return c_pointer, (padleft_pointer, c_bytesize)

    def free(self, c_pointer, total_size):
        # unprotect it, since free() accesses it, I think...
        self.lib.mprotect(c_pointer, total_size,
                          ctypes.c_int(mmap.PROT_READ | mmap.PROT_WRITE))
        self.lib.free(c_pointer)

class NumaAllocator(MemoryAllocator):

    """
    Memory allocator based on ``libnuma`` functions. The allocated memory is
    aligned to page boundaries. Through the argument ``node`` it is possible
    to specify a NUMA node in which memory allocation should be attempted first
    (will fall back to an arbitrary NUMA domain if not enough memory is available)

    :param node: Either an integer, indicating a specific NUMA node, or the special
                 keywords ``'local'`` or ``'any'``.
    """

    is_Numa = True

    @classmethod
    def initialize(cls):
        handle = find_library('numa')
        if handle is None:
            return
        lib = ctypes.CDLL(handle)
        if lib.numa_available() == -1:
            return
        # We are indeed on a NUMA system
        # Allow the kernel to allocate memory on other NUMA nodes when there isn't
        # enough free on the target node
        lib.numa_set_bind_policy(0)
        # Required because numa_alloc* functions return pointers
        lib.numa_alloc_onnode.restype = ctypes.c_void_p
        lib.numa_alloc_local.restype = ctypes.c_void_p
        lib.numa_alloc.restype = ctypes.c_void_p
        cls.lib = lib

    def __init__(self, node):
        super(NumaAllocator, self).__init__()
        self._node = node

    def _alloc_C_libcall(self, size, ctype):
        if not self.available():
            raise RuntimeError("Couldn't find `libnuma`'s `numa_alloc_*` to "
                               "allocate memory")
        c_bytesize = ctypes.c_ulong(size * ctypes.sizeof(ctype))
        if self.put_onnode:
            c_pointer = self.lib.numa_alloc_onnode(c_bytesize, self._node)
        elif self.put_local:
            c_pointer = self.lib.numa_alloc_local(c_bytesize)
        else:
            c_pointer = self.lib.numa_alloc(c_bytesize)
        if c_pointer == 0:
            return None, None
        else:
            return c_pointer, (c_pointer, size)

    def free(self, c_pointer, size):
        self.lib.numa_free(c_pointer, size)

    @property
    def node(self):
        return self._node

    @property
    def put_onnode(self):
        return isinstance(self._node, int)

    @property
    def put_local(self):
        return self._node == 'local'

ALLOC_GUARD = GuardAllocator(1048576)
ALLOC_FLAT = PosixAllocator()
ALLOC_KNL_DRAM = NumaAllocator(0)
ALLOC_KNL_MCDRAM = NumaAllocator(1)
ALLOC_NUMA_ANY = NumaAllocator('any')
ALLOC_NUMA_LOCAL = NumaAllocator('local')

def default_allocator():
    """
    Return a :class:`MemoryAllocator` for the architecture on which the
    process is running. Possible allocators are: ::

        * ALLOC_FLAT: Align memory to page boundaries using the posix function
                      ``posix_memalign``
        * ALLOC_NUMA_LOCAL: Allocate memory in the "closest" NUMA node. This only
                            makes sense on a NUMA architecture. Falls back to
                            allocation in an arbitrary NUMA node if there isn't
                            enough space.
        * ALLOC_NUMA_ANY: Allocate memory in an arbitrary NUMA node.
        * ALLOC_KNL_MCDRAM: On a Knights Landing platform, allocate memory in MCDRAM.
                            Falls back to DRAM if there isn't enough space.
        * ALLOC_KNL_DRAM: On a Knights Landing platform, allocate memory in DRAM.

    The default allocator is chosen based on the following algorithm: ::

        * If running in DEVELOP mode (env var DEVITO_DEVELOP), return ALLOC_FLAT;
        * If ``libnuma`` is not available on the system, return ALLOC_FLAT (though
          it typically is available, at least on relatively recent Linux distributions);
        * If on a Knights Landing platform (codename ``knl``, see ``print_defaults()``)
          return ALLOC_KNL_MCDRAM;
        * If on a multi-socket Intel Xeon platform, return ALLOC_NUMA_LOCAL;
        * In all other cases, return ALLOC_FLAT.
    """
    if configuration['develop-mode']:
        return ALLOC_GUARD
    elif NumaAllocator.available():
        if configuration['platform'] == 'knl':
            return ALLOC_KNL_MCDRAM
        else:
            return ALLOC_NUMA_LOCAL
    else:
        return ALLOC_FLAT

class Data(np.ndarray):

    """
    A special :class:`numpy.ndarray` allowing logical indexing.

    The type :class:`numpy.ndarray` is subclassed as indicated at: ::

        https://docs.scipy.org/doc/numpy-1.13.0/user/basics.subclassing.html

    :param shape: Shape of the domain region in grid points.
    :param dimensions: A tuple of :class:`Dimension`s, representing the dimensions
                       of the ``Data``.
    :param dtype: A ``numpy.dtype`` for the raw data.
    :param allocator: (Optional) a :class:`MemoryAllocator` to specialize memory
                      allocation. Defaults to ``ALLOC_FLAT``.

    .. note::

        This type supports logical indexing over modulo buffered dimensions.

    .. note::

        Any view or copy ``A`` created starting from ``self``, for instance via
        a slice operation or a universal function ("ufunc" in NumPy jargon), will
        still be of type :class:`Data`. However, if ``A``'s rank is different than
        ``self``'s rank, namely if ``A.ndim != self.ndim``, then the capability of
        performing logical indexing is lost.
    """

    def __new__(cls, shape, dimensions, dtype, allocator=ALLOC_FLAT):
        assert len(shape) == len(dimensions)
        ndarray, free_args = allocator.alloc(shape, dtype)
        obj = np.asarray(ndarray).view(cls)
        obj._allocator = allocator
        obj._free_args = free_args
        obj._modulo = tuple(True if i.is_Stepping else False for i in dimensions)
        return obj

    def __del__(self):
        if self._free_args is None:
            return
        self._allocator.free(*self._free_args)
        self._free_args = None

    def __array_finalize__(self, obj):
        # `self` is the newly created object
        # `obj` is the object from which `self` was created
        if obj is None:
            # `self` was created through __new__()
            return
        if type(obj) != Data or self.ndim != obj.ndim:
            self._modulo = tuple(False for i in range(self.ndim))
        else:
            self._modulo = obj._modulo
        # Views or references created via operations on `obj` do not get an
        # explicit reference to the underlying data (`_free_args`). This makes sure
        # that only one object (the "root" Data) will free the C-allocated memory
        self._free_args = None

    def __getitem__(self, index):
        index = self._convert_index(index)
        return super(Data, self).__getitem__(index)

    def __setitem__(self, index, val):
        index = self._convert_index(index)
        super(Data, self).__setitem__(index, val)

    def _convert_index(self, index):
        if isinstance(index, np.ndarray):
            # Advanced indexing, nothing special to do
            return index

        index = as_tuple(index)
        if len(index) > self.ndim:
            # Maybe user code is trying to add a new axis (see np.newaxis),
            # so the resulting array will have shape larger than `self`'s,
            # hence I can just let numpy deal with it, as by specification
            # we're gonna drop modulo indexing anyway
            return index

        # Index conversion
        wrapped = []
        for i, mod, use_modulo in zip(index, self.shape, self._modulo):
            if use_modulo is False:
                # Nothing special to do (no logical indexing)
                wrapped.append(i)
            elif isinstance(i, slice):
                if i.start is None:
                    start = i.start
                elif i.start >= 0:
                    start = i.start % mod
                else:
                    start = -(i.start % mod)
                if i.stop is None:
                    stop = i.stop
                elif i.stop >= 0:
                    stop = i.stop % (mod + 1)
                else:
                    stop = -(i.stop % (mod + 1))
                wrapped.append(slice(start, stop, i.step))
            elif isinstance(i, (tuple, list)):
                wrapped.append([k % mod for k in i])
            else:
                wrapped.append(i % mod)
        return wrapped[0] if len(index) == 1 else tuple(wrapped)

    def reset(self):
        """
        Set all grid entries to 0.
        """
        self[:] = 0.0

def first_touch(array):
    """
    Uses an Operator to initialize the given array in the same pattern that
    would later be used to access it.
    """
    devito.Operator(Eq(array, 0.))()
<commit_msg>fix feedback<commit_after>from __future__ import absolute_import

import abc
from functools import reduce
from operator import mul

import numpy as np
from sympy import Eq
import ctypes
from ctypes.util import find_library
import mmap

from devito.parameters import configuration
from devito.tools import as_tuple, numpy_to_ctypes
from devito.logger import logger
import devito

__all__ = ['ALLOC_FLAT', 'ALLOC_NUMA_LOCAL', 'ALLOC_NUMA_ANY',
           'ALLOC_KNL_MCDRAM', 'ALLOC_KNL_DRAM', 'ALLOC_GUARD']

class MemoryAllocator(object):

    """Abstract class defining the interface to memory allocators."""

    __metaclass__ = abc.ABCMeta

    is_Posix = False
    is_Numa = False

    _attempted_init = False
    lib = None

    @classmethod
    def available(cls):
        if cls._attempted_init is False:
            cls.initialize()
            cls._attempted_init = True
        return cls.lib is not None

    @classmethod
    def initialize(cls):
        """
        Initialize the MemoryAllocator.

        .. note::

            This must be implemented by all subclasses of MemoryAllocator.
        """
        return

    def alloc(self, shape, dtype):
        """
        Allocate memory.

        :param shape: Shape of the array to allocate.
        :param dtype: Numpy datatype to allocate.

        :returns (pointer, free_handle): The first element of the tuple is the reference
                                         that can be used to access the data as a ctypes
                                         object. The second element is an opaque object
                                         that is needed only for the call to free.
        """
        size = int(reduce(mul, shape))
        ctype = numpy_to_ctypes(dtype)

        c_pointer, memfree_args = self._alloc_C_libcall(size, ctype)
        if c_pointer is None:
            raise RuntimeError("Unable to allocate %d elements in memory", str(size))

        c_pointer = ctypes.cast(c_pointer, np.ctypeslib.ndpointer(dtype=dtype,
                                                                  shape=shape))
        pointer = np.ctypeslib.as_array(c_pointer, shape=shape)

        return (pointer, memfree_args)

    @abc.abstractmethod
    def _alloc_C_libcall(self, size, ctype):
        """
        Perform the actual memory allocation by calling a C function.
        Should return a 2-tuple (c_pointer, memfree_args), where the free args
        are what is handed back to free() later to deallocate.

        .. note::

            This method should be implemented by a subclass.
        """
        return

    @abc.abstractmethod
    def free(self, *args):
        """
        Free memory previously allocated with ``self.alloc``.

        Arguments are provided exactly as returned in the second
        element of the tuple returned by _alloc_C_libcall

        .. note::

            This method should be implemented by a subclass.
        """
        return

class PosixAllocator(MemoryAllocator):

    """
    Memory allocator based on ``posix`` functions. The allocated memory is
    aligned to page boundaries.
    """

    is_Posix = True

    @classmethod
    def initialize(cls):
        handle = find_library('c')
        if handle is not None:
            cls.lib = ctypes.CDLL(handle)

    def _alloc_C_libcall(self, size, ctype):
        if not self.available():
            raise RuntimeError("Couldn't find `libc`'s `posix_memalign` to "
                               "allocate memory")
        c_bytesize = ctypes.c_ulong(size * ctypes.sizeof(ctype))
        c_pointer = ctypes.cast(ctypes.c_void_p(), ctypes.c_void_p)
        alignment = self.lib.getpagesize()
        ret = self.lib.posix_memalign(ctypes.byref(c_pointer), alignment, c_bytesize)
        if ret == 0:
            return c_pointer, (c_pointer, )
        else:
            return None, None

    def free(self, c_pointer):
        self.lib.free(c_pointer)

class GuardAllocator(PosixAllocator):
    """
    Memory allocator based on ``posix`` functions. The allocated memory is
    aligned to page boundaries.  Additionally, it allocates extra memory
    before and after the data, and configures it so that an SEGV is thrown
    immediately if an out-of-bounds access occurs.

    Further, the remainder region of the last page (which cannot be
    protected), is poisoned with NaNs.
    """

    def __init__(self, padding_bytes=1024*1024):
        self.padding_bytes = padding_bytes

    def _alloc_C_libcall(self, size, ctype):
        if not self.available():
            raise RuntimeError("Couldn't find `libc`'s `posix_memalign` to "
                               "allocate memory")

        pagesize = self.lib.getpagesize()
        assert self.padding_bytes % pagesize == 0

        npages_pad = self.padding_bytes // pagesize
        nbytes_user = size * ctypes.sizeof(ctype)
        npages_user = (nbytes_user + pagesize - 1) // pagesize

        npages_alloc = 2*npages_pad + npages_user

        c_bytesize = ctypes.c_ulong(npages_alloc * pagesize)
        c_pointer = ctypes.cast(ctypes.c_void_p(), ctypes.c_void_p)
        alignment = self.lib.getpagesize()
        ret = self.lib.posix_memalign(ctypes.byref(c_pointer), alignment, c_bytesize)
        if ret != 0:
            return None, None

        # generate pointers to the left padding, the user data, and the right pad
        padleft_pointer = c_pointer
        c_pointer = ctypes.c_void_p(c_pointer.value + self.padding_bytes)
        padright_pointer = ctypes.c_void_p(c_pointer.value + npages_user * pagesize)

        # and set the permissions on the pad memory to 0 (no access)
        # if these fail, don't worry about failing the entire allocation
        c_padsize = ctypes.c_ulong(self.padding_bytes)
        if self.lib.mprotect(padleft_pointer, c_padsize, ctypes.c_int(0)):
            logger.warning("couldn't protect memory")
        if self.lib.mprotect(padright_pointer, c_padsize, ctypes.c_int(0)):
            logger.warning("couldn't protect memory")

        # if there is a multiple of 4 bytes left, use the code below to poison
        # the memory
        if nbytes_user % 4 == 0:
            poison_size = npages_user*pagesize - nbytes_user
            intp_type = ctypes.POINTER(ctypes.c_int)
            poison_ptr = ctypes.cast(ctypes.c_void_p(c_pointer.value + nbytes_user),
                                     intp_type)

            # for both float32 and float64, a sequence of -100 int32s represents NaNs,
            # at least on little-endian architectures.  It shouldn't matter what we
            # put in there, anyway
            for i in range(poison_size // 4):
                poison_ptr[i] = -100

        return c_pointer, (padleft_pointer, c_bytesize)

    def free(self, c_pointer, total_size):
        # unprotect it, since free() accesses it, I think...
        self.lib.mprotect(c_pointer, total_size,
                          ctypes.c_int(mmap.PROT_READ | mmap.PROT_WRITE))
        self.lib.free(c_pointer)

class NumaAllocator(MemoryAllocator):

    """
    Memory allocator based on ``libnuma`` functions. The allocated memory is
    aligned to page boundaries. Through the argument ``node`` it is possible
    to specify a NUMA node in which memory allocation should be attempted first
    (will fall back to an arbitrary NUMA domain if not enough memory is available)

    :param node: Either an integer, indicating a specific NUMA node, or the special
                 keywords ``'local'`` or ``'any'``.
    """

    is_Numa = True

    @classmethod
    def initialize(cls):
        handle = find_library('numa')
        if handle is None:
            return
        lib = ctypes.CDLL(handle)
        if lib.numa_available() == -1:
            return
        # We are indeed on a NUMA system
        # Allow the kernel to allocate memory on other NUMA nodes when there isn't
        # enough free on the target node
        lib.numa_set_bind_policy(0)
        # Required because numa_alloc* functions return pointers
        lib.numa_alloc_onnode.restype = ctypes.c_void_p
        lib.numa_alloc_local.restype = ctypes.c_void_p
        lib.numa_alloc.restype = ctypes.c_void_p
        cls.lib = lib

    def __init__(self, node):
        super(NumaAllocator, self).__init__()
        self._node = node

    def _alloc_C_libcall(self, size, ctype):
        if not self.available():
            raise RuntimeError("Couldn't find `libnuma`'s `numa_alloc_*` to "
                               "allocate memory")
        c_bytesize = ctypes.c_ulong(size * ctypes.sizeof(ctype))
        if self.put_onnode:
            c_pointer = self.lib.numa_alloc_onnode(c_bytesize, self._node)
        elif self.put_local:
            c_pointer = self.lib.numa_alloc_local(c_bytesize)
        else:
            c_pointer = self.lib.numa_alloc(c_bytesize)
        if c_pointer == 0:
            return None, None
        else:
            return c_pointer, (c_pointer, size)

    def free(self, c_pointer, size):
        self.lib.numa_free(c_pointer, size)

    @property
    def node(self):
        return self._node

    @property
    def put_onnode(self):
        return isinstance(self._node, int)

    @property
    def put_local(self):
        return self._node == 'local'

ALLOC_GUARD = GuardAllocator(1048576)
ALLOC_FLAT = PosixAllocator()
ALLOC_KNL_DRAM = NumaAllocator(0)
ALLOC_KNL_MCDRAM = NumaAllocator(1)
ALLOC_NUMA_ANY = NumaAllocator('any')
ALLOC_NUMA_LOCAL = NumaAllocator('local')

def default_allocator():
    """
    Return a :class:`MemoryAllocator` for the architecture on which the
    process is running. Possible allocators are: ::

        * ALLOC_FLAT: Align memory to page boundaries using the posix function
                      ``posix_memalign``
        * ALLOC_NUMA_LOCAL: Allocate memory in the "closest" NUMA node. This only
                            makes sense on a NUMA architecture. Falls back to
                            allocation in an arbitrary NUMA node if there isn't
                            enough space.
        * ALLOC_NUMA_ANY: Allocate memory in an arbitrary NUMA node.
        * ALLOC_KNL_MCDRAM: On a Knights Landing platform, allocate memory in MCDRAM.
                            Falls back to DRAM if there isn't enough space.
        * ALLOC_KNL_DRAM: On a Knights Landing platform, allocate memory in DRAM.

    The default allocator is chosen based on the following algorithm: ::

        * If running in DEVELOP mode (env var DEVITO_DEVELOP), return ALLOC_FLAT;
        * If ``libnuma`` is not available on the system, return ALLOC_FLAT (though
          it typically is available, at least on relatively recent Linux distributions);
        * If on a Knights Landing platform (codename ``knl``, see ``print_defaults()``)
          return ALLOC_KNL_MCDRAM;
        * If on a multi-socket Intel Xeon platform, return ALLOC_NUMA_LOCAL;
        * In all other cases, return ALLOC_FLAT.
    """
    if configuration['develop-mode']:
        return ALLOC_GUARD
    elif NumaAllocator.available():
        if configuration['platform'] == 'knl':
            return ALLOC_KNL_MCDRAM
        else:
            return ALLOC_NUMA_LOCAL
    else:
        return ALLOC_FLAT

class Data(np.ndarray):

    """
    A special :class:`numpy.ndarray` allowing logical indexing.

    The type :class:`numpy.ndarray` is subclassed as indicated at: ::

        https://docs.scipy.org/doc/numpy-1.13.0/user/basics.subclassing.html

    :param shape: Shape of the domain region in grid points.
    :param dimensions: A tuple of :class:`Dimension`s, representing the dimensions
                       of the ``Data``.
    :param dtype: A ``numpy.dtype`` for the raw data.
    :param allocator: (Optional) a :class:`MemoryAllocator` to specialize memory
                      allocation. Defaults to ``ALLOC_FLAT``.

    .. note::

        This type supports logical indexing over modulo buffered dimensions.

    .. note::

        Any view or copy ``A`` created starting from ``self``, for instance via
        a slice operation or a universal function ("ufunc" in NumPy jargon), will
        still be of type :class:`Data`. However, if ``A``'s rank is different than
        ``self``'s rank, namely if ``A.ndim != self.ndim``, then the capability of
        performing logical indexing is lost.
    """

    def __new__(cls, shape, dimensions, dtype, allocator=ALLOC_FLAT):
        assert len(shape) == len(dimensions)
        ndarray, memfree_args = allocator.alloc(shape, dtype)
        obj = np.asarray(ndarray).view(cls)
        obj._allocator = allocator
        obj._memfree_args = memfree_args
        obj._modulo = tuple(True if i.is_Stepping else False for i in dimensions)
        return obj

    def __del__(self):
        if self._memfree_args is None:
            return
        self._allocator.free(*self._memfree_args)
        self._memfree_args = None

    def __array_finalize__(self, obj):
        # `self` is the newly created object
        # `obj` is the object from which `self` was created
        if obj is None:
            # `self` was created through __new__()
            return
        if type(obj) != Data or self.ndim != obj.ndim:
            self._modulo = tuple(False for i in range(self.ndim))
        else:
            self._modulo = obj._modulo
        # Views or references created via operations on `obj` do not get an
        # explicit reference to the underlying data (`_memfree_args`). This makes sure
        # that only one object (the "root" Data) will free the C-allocated memory
        self._memfree_args = None

    def __getitem__(self, index):
        index = self._convert_index(index)
        return super(Data, self).__getitem__(index)

    def __setitem__(self, index, val):
        index = self._convert_index(index)
        super(Data, self).__setitem__(index, val)

    def _convert_index(self, index):
        if isinstance(index, np.ndarray):
            # Advanced indexing, nothing special to do
            return index

        index = as_tuple(index)
        if len(index) > self.ndim:
            # Maybe user code is trying to add a new axis (see np.newaxis),
            # so the resulting array will have shape larger than `self`'s,
            # hence I can just let numpy deal with it, as by specification
            # we're gonna drop modulo indexing anyway
            return index

        # Index conversion
        wrapped = []
        for i, mod, use_modulo in zip(index, self.shape, self._modulo):
            if use_modulo is False:
                # Nothing special to do (no logical indexing)
                wrapped.append(i)
            elif isinstance(i, slice):
                if i.start is None:
                    start = i.start
                elif i.start >= 0:
                    start = i.start % mod
                else:
                    start = -(i.start % mod)
                if i.stop is None:
                    stop = i.stop
                elif i.stop >= 0:
                    stop = i.stop % (mod + 1)
                else:
                    stop = -(i.stop % (mod + 1))
                wrapped.append(slice(start, stop, i.step))
            elif isinstance(i, (tuple, list)):
                wrapped.append([k % mod for k in i])
            else:
                wrapped.append(i % mod)
        return wrapped[0] if len(index) == 1 else tuple(wrapped)

    def reset(self):
        """
        Set all grid entries to 0.
        """
        self[:] = 0.0

def first_touch(array):
    """
    Uses an Operator to initialize the given array in the same pattern that
    would later be used to access it.
    """
    devito.Operator(Eq(array, 0.))()
<|endoftext|>


<commit_before>import Adafruit_GPIO as GPIO
import adafruit_spi_modified as SPI
import commands
import os
import Queue
import settings
import time
import threading
import traceback
import sys

BASE_PATH = os.path.dirname(os.path.realpath(__file__))
UPPER_PATH = os.path.split(os.path.dirname(os.path.realpath(__file__)))[0]
DEVICES_PATH = "%s/Hosts/" % (BASE_PATH )
THIRTYBIRDS_PATH = "%s/thirtybirds_2_0" % (UPPER_PATH )

sys.path.append(BASE_PATH)
sys.path.append(UPPER_PATH)

from thirtybirds_2_0.Network.manager import init as network_init
from thirtybirds_2_0.Updates.manager import init as updates_init

########################
## UTILS
########################

class Utils(object):
    def __init__(self, hostname):
        self.hostname = hostname
    def reboot(self):
        os.system("sudo reboot now")

    def get_shelf_id(self):
        return self.hostname[11:][:1]

    def get_camera_id(self):
        return self.hostname[12:]

    def create_image_file_name(self, timestamp, light_level, process_type):
        return "{}_{}_{}_{}_{}.png".format(timestamp, self.get_shelf_id() ,  self.get_camera_id(), light_level, process_type)

    def remote_update_git(self, oratio, thirtybirds, update, upgrade):
        if oratio:
            subprocess.call(['sudo', 'git', 'pull'], cwd='/home/pi/oratio')
        if thirtybirds:
            subprocess.call(['sudo', 'git', 'pull'], cwd='/home/pi/thirtybirds_2_0')
        return

    def remote_update_scripts(self):
        updates_init("/home/pi/oratio", False, True)
        return

    def get_update_script_version(self):
        (updates, ghStatus, bsStatus) = updates_init("/home/pi/oratio", False, False)
        return updates.read_version_pickle()

    def get_git_timestamp(self):
        return commands.getstatusoutput("cd /home/pi/oratio/; git log -1 --format=%cd")[1]

    def get_temp(self):
        return commands.getstatusoutput("/opt/vc/bin/vcgencmd measure_temp")[1]

    def get_cpu(self):
        bash_output = commands.getstatusoutput("uptime")[1]
        split_output = bash_output.split(" ")
        return split_output[12]

    def get_uptime(self):
        bash_output = commands.getstatusoutput("uptime")[1]
        split_output = bash_output.split(" ")
        return split_output[4]

    def get_disk(self):
        # stub for now
        return "0"

    def get_client_status(self):
        return (self.hostname, self.get_update_script_version(), self.get_git_timestamp(), self.get_temp(), self.get_cpu(), self.get_uptime(), self.get_disk())

class MCP3008s(object):
    def __init__(self, spi_clock_pin, miso_pin, mosi_pin, chip_select_pins):
        self.gpio = GPIO.get_platform_gpio()
        self.chip_select_pins = chip_select_pins
        self._spi = SPI.BitBang(self.gpio, spi_clock_pin, mosi_pin, miso_pin)
        self._spi.set_clock_hz(1000000)
        self._spi.set_mode(0)
        self._spi.set_bit_order(SPI.MSBFIRST)
        for chip_select_pin in chip_select_pins:
            self.gpio.setup(chip_select_pin, GPIO.OUT)
            self.gpio.set_high(chip_select_pin)

    def read(self, chip_select_pin, adc_number):
        assert 0 <= adc_number <= 7, 'ADC number must be a value of 0-7!'
        # Build a single channel read command.
        # For example channel zero = 0b11000000
        command = 0b11 << 6                  # Start bit, single channel read
        command |= (adc_number & 0x07) << 3  # Channel number (in 3 bits)
        # Note the bottom 3 bits of command are 0, this is to account for the
        # extra clock to do the conversion, and the low null bit returned at
        # the start of the response.
        resp = self._spi.transfer(chip_select_pin,[command, 0x0, 0x0])
        # Parse out the 10 bits of response data and return it.
        result = (resp[0] & 0x01) << 9
        result |= (resp[1] & 0xFF) << 1
        result |= (resp[2] & 0x80) >> 7
        return result & 0x3FF

    def scan_all(self):
        adcs = []
        for chip_select_pin in self.chip_select_pins:
            channels = []
            for adc_number in range(8):
                channels.append(self.read(chip_select_pin, adc_number))
            adcs.append(channels)
        return adcs

class Potentiometers(threading.Thread):
    def __init__(self, network_send_ref):
        threading.Thread.__init__(self)
        self.network_send_ref = network_send_ref
        self.noise_threshold = 10
        self.spi_clock_pin = 11
        self.miso_pin = 9
        self.mosi_pin = 10
        self.chip_select_pins = [8,7,12,16,20,21]

        self.potentiometers_layout  = [
            [
                "",
                "voice_1_overtone_1_harmonic",
                "voice_1_overtone_1_fine",
                "voice_1_overtone_1_volume",
                "",
                "voice_1_root_fine",
                "voice_1_root_half_steps",
                "voice_1_root_octave"
            ],
            [
                "",
                "",
                "",
                "",
                "voice_1_overtone_2_harmonic",
                "voice_1_overtone_2_fine",
                "voice_1_overtone_2_volume",
                ""
            ],

            [
                "voice_2_root_octave",
                "voice_2_root_half_steps",
                "voice_2_root_fine",
                "",
                "voice_2_overtone_1_volume",
                "voice_2_overtone_1_fine",
                "voice_2_overtone_1_harmonic",
                ""
            ],
            [
                "",
                "",
                "",
                "",
                "voice_2_overtone_2_harmonic",
                "voice_2_overtone_2_fine",
                "voice_2_overtone_2_volume"
                "",
            ],
            [
                "voice_3_root_octave",
                "voice_3_root_half_steps",
                "voice_3_root_fine",
                "",
                "voice_3_overtone_1_volume",
                "voice_3_overtone_1_fine",
                "voice_3_overtone_1_harmonic"
                ""
            ],
            [
                "",
                "",
                "",
                "",
                "voice_3_overtone_2_harmonic",
                "voice_3_overtone_2_fine",
                "voice_3_overtone_2_volume",
                "layer_speed"
            ],
        ]

        """
        self.potentiometers_layout  = [
            [
                "",
                "voice_1_overtone_2_harmonic",
                "voice_1_overtone_2_volume",
                "voice_1_overtone_1_fine",
                "voice_1_overtone_1_harmonic",
                "voice_1_overtone_1_volume",
                "voice_1_root_fine",
                "voice_1_root_harmonic",
            ],
            [
                "voice_1_root_half_steps",
                "voice_1_root_octave",
                "voice_1_formant_front_back",
                "voice_1_formant_open_close",
                "voice_1_formant_pitch",
                "voice_1_formant_volume",
                "voice_1_overtone_2_fine",
                ""
            ],
            [
                "voice_2_root_harmonic",
                "voice_2_root_fine",
                "voice_2_overtone_1_volume",
                "voice_2_overtone_1_harmonic",
                "voice_2_overtone_1_fine",
                "voice_2_overtone_2_volume",
                "voice_2_overtone_2_harmonic",
                ""
            ],
            [
                "voice_2_root_half_steps",
                "voice_2_root_octave",
                "voice_2_formant_front_back",
                "voice_2_formant_open_close",
                "voice_2_formant_pitch",
                "voice_2_formant_volume",
                "voice_2_overtone_2_fine",
                ""
            ],
            [
                "voice_3_root_harmonic",
                "voice_3_root_fine",
                "voice_3_overtone_1_volume",
                "voice_3_overtone_1_harmonic",
                "voice_3_overtone_1_fine",
                "voice_3_overtone_2_volume",
                "voice_3_overtone_2_harmonic",
                ""
            ],
            [
                "voice_3_root_half_steps",
                "voice_3_root_octave",
                "voice_3_formant_front_back",
                "voice_3_formant_open_close",
                "voice_3_formant_pitch",
                "voice_3_formant_volume",
                "voice_3_overtone_2_fine",
                "layer_speed"
            ]
        ]
        """

        self.potentiometer_last_value  = [
            [0,0,0,0,0,0,0,0],
            [0,0,0,0,0,0,0,0],
            [0,0,0,0,0,0,0,0],
            [0,0,0,0,0,0,0,0],
            [0,0,0,0,0,0,0,0],
            [0,0,0,0,0,0,0,0]
        ]
        self.mcp3008s = MCP3008s(self.spi_clock_pin, self.miso_pin, self.mosi_pin, self.chip_select_pins)

    def run(self):
        while True:
            all_adc_values =  self.mcp3008s.scan_all()
            #print all_adc_values
            for adc in range(len(all_adc_values)):
                for channel in range(8):
                    adc_value = 1023 - all_adc_values[adc][channel]
                    potentiometer_name = self.potentiometers_layout[adc][channel]
                    if potentiometer_name != "":
                        if abs(adc_value - self.potentiometer_last_value[adc][channel] ) > self.noise_threshold:
                            self.network_send_ref(potentiometer_name, adc_value/1023.0)
                            print adc, channel, self.potentiometers_layout[adc][channel], adc_value
                    self.potentiometer_last_value[adc][channel] = adc_value
            time.sleep(0.1)

class Network(object):
    def __init__(self, hostname, network_message_handler, network_status_handler):
        self.hostname = hostname
        self.thirtybirds = network_init(
            hostname=hostname,
            role="client",
            discovery_multicastGroup=settings.discovery_multicastGroup,
            discovery_multicastPort=settings.discovery_multicastPort,
            discovery_responsePort=settings.discovery_responsePort,
            pubsub_pubPort=settings.pubsub_pubPort,
            message_callback=network_message_handler,
            status_callback=network_status_handler
        )

# Main handles network send/recv and can see all other classes directly
class Main(threading.Thread):
    def __init__(self, hostname):
        threading.Thread.__init__(self)
        self.network = Network(hostname, self.network_message_handler, self.network_status_handler)
        self.queue = Queue.Queue()
        self.network.thirtybirds.subscribe_to_topic("door_closed")
        self.potentiometers = Potentiometers(self.network.thirtybirds.send) # self.network.thirtybirds.send
        self.potentiometers.daemon = True
        self.potentiometers.start()
        self.utils = Utils(hostname)
        self.network.thirtybirds.subscribe_to_topic("client_monitor_request")

    def network_message_handler(self, topic_msg):
        # this method runs in the thread of the caller, not the tread of Main
        topic, msg =  topic_msg # separating just to eval msg.  best to do it early.  it should be done in TB.
        if len(msg) > 0:
            msg = eval(msg)
        self.add_to_queue(topic, msg)

    def network_status_handler(self, topic_msg):
        # this method runs in the thread of the caller, not the tread of Main
        print "Main.network_status_handler", topic_msg

    def add_to_queue(self, topic, msg):
        self.queue.put((topic, msg))

    def run(self):
        while True:
            try:
                topic, msg = self.queue.get(True)
                if topic == "client_monitor_request":
                    self.network.thirtybirds.send("client_monitor_response", self.utils.get_client_status())

            except Exception as e:
                exc_type, exc_value, exc_traceback = sys.exc_info()
                print e, repr(traceback.format_exception(exc_type, exc_value,exc_traceback))

def init(hostname):
    main = Main(hostname)
    main.daemon = True
    main.start()
    return main

<commit_msg>re-arranging settings sliders. fixing index exception<commit_after>import Adafruit_GPIO as GPIO
import adafruit_spi_modified as SPI
import commands
import os
import Queue
import settings
import time
import threading
import traceback
import sys

BASE_PATH = os.path.dirname(os.path.realpath(__file__))
UPPER_PATH = os.path.split(os.path.dirname(os.path.realpath(__file__)))[0]
DEVICES_PATH = "%s/Hosts/" % (BASE_PATH )
THIRTYBIRDS_PATH = "%s/thirtybirds_2_0" % (UPPER_PATH )

sys.path.append(BASE_PATH)
sys.path.append(UPPER_PATH)

from thirtybirds_2_0.Network.manager import init as network_init
from thirtybirds_2_0.Updates.manager import init as updates_init

########################
## UTILS
########################

class Utils(object):
    def __init__(self, hostname):
        self.hostname = hostname
    def reboot(self):
        os.system("sudo reboot now")

    def get_shelf_id(self):
        return self.hostname[11:][:1]

    def get_camera_id(self):
        return self.hostname[12:]

    def create_image_file_name(self, timestamp, light_level, process_type):
        return "{}_{}_{}_{}_{}.png".format(timestamp, self.get_shelf_id() ,  self.get_camera_id(), light_level, process_type)

    def remote_update_git(self, oratio, thirtybirds, update, upgrade):
        if oratio:
            subprocess.call(['sudo', 'git', 'pull'], cwd='/home/pi/oratio')
        if thirtybirds:
            subprocess.call(['sudo', 'git', 'pull'], cwd='/home/pi/thirtybirds_2_0')
        return

    def remote_update_scripts(self):
        updates_init("/home/pi/oratio", False, True)
        return

    def get_update_script_version(self):
        (updates, ghStatus, bsStatus) = updates_init("/home/pi/oratio", False, False)
        return updates.read_version_pickle()

    def get_git_timestamp(self):
        return commands.getstatusoutput("cd /home/pi/oratio/; git log -1 --format=%cd")[1]

    def get_temp(self):
        return commands.getstatusoutput("/opt/vc/bin/vcgencmd measure_temp")[1]

    def get_cpu(self):
        bash_output = commands.getstatusoutput("uptime")[1]
        split_output = bash_output.split(" ")
        return split_output[12]

    def get_uptime(self):
        bash_output = commands.getstatusoutput("uptime")[1]
        split_output = bash_output.split(" ")
        return split_output[4]

    def get_disk(self):
        # stub for now
        return "0"

    def get_client_status(self):
        return (self.hostname, self.get_update_script_version(), self.get_git_timestamp(), self.get_temp(), self.get_cpu(), self.get_uptime(), self.get_disk())

class MCP3008s(object):
    def __init__(self, spi_clock_pin, miso_pin, mosi_pin, chip_select_pins):
        self.gpio = GPIO.get_platform_gpio()
        self.chip_select_pins = chip_select_pins
        self._spi = SPI.BitBang(self.gpio, spi_clock_pin, mosi_pin, miso_pin)
        self._spi.set_clock_hz(1000000)
        self._spi.set_mode(0)
        self._spi.set_bit_order(SPI.MSBFIRST)
        for chip_select_pin in chip_select_pins:
            self.gpio.setup(chip_select_pin, GPIO.OUT)
            self.gpio.set_high(chip_select_pin)

    def read(self, chip_select_pin, adc_number):
        assert 0 <= adc_number <= 7, 'ADC number must be a value of 0-7!'
        # Build a single channel read command.
        # For example channel zero = 0b11000000
        command = 0b11 << 6                  # Start bit, single channel read
        command |= (adc_number & 0x07) << 3  # Channel number (in 3 bits)
        # Note the bottom 3 bits of command are 0, this is to account for the
        # extra clock to do the conversion, and the low null bit returned at
        # the start of the response.
        resp = self._spi.transfer(chip_select_pin,[command, 0x0, 0x0])
        # Parse out the 10 bits of response data and return it.
        result = (resp[0] & 0x01) << 9
        result |= (resp[1] & 0xFF) << 1
        result |= (resp[2] & 0x80) >> 7
        return result & 0x3FF

    def scan_all(self):
        adcs = []
        for chip_select_pin in self.chip_select_pins:
            channels = []
            for adc_number in range(8):
                channels.append(self.read(chip_select_pin, adc_number))
            adcs.append(channels)
        return adcs

class Potentiometers(threading.Thread):
    def __init__(self, network_send_ref):
        threading.Thread.__init__(self)
        self.network_send_ref = network_send_ref
        self.noise_threshold = 10
        self.spi_clock_pin = 11
        self.miso_pin = 9
        self.mosi_pin = 10
        self.chip_select_pins = [8,7,12,16,20,21]

        self.potentiometers_layout  = [
            [
                "",
                "voice_1_overtone_1_harmonic",
                "voice_1_overtone_1_fine",
                "voice_1_overtone_1_volume",
                "",
                "voice_1_root_fine",
                "voice_1_root_half_steps",
                "voice_1_root_octave"
            ],
            [
                "",
                "",
                "",
                "",
                "voice_1_overtone_2_harmonic",
                "voice_1_overtone_2_fine",
                "voice_1_overtone_2_volume",
                ""
            ],

            [
                "voice_2_root_octave",
                "voice_2_root_half_steps",
                "voice_2_root_fine",
                "",
                "voice_2_overtone_1_volume",
                "voice_2_overtone_1_fine",
                "voice_2_overtone_1_harmonic",
                ""
            ],
            [
                "",
                "",
                "",
                "",
                "voice_2_overtone_2_harmonic",
                "voice_2_overtone_2_fine",
                "voice_2_overtone_2_volume"
                ""
            ],
            [
                "voice_3_root_octave",
                "voice_3_root_half_steps",
                "voice_3_root_fine",
                "",
                "voice_3_overtone_1_volume",
                "voice_3_overtone_1_fine",
                "voice_3_overtone_1_harmonic"
                ""
            ],
            [
                "",
                "",
                "",
                "",
                "voice_3_overtone_2_harmonic",
                "voice_3_overtone_2_fine",
                "voice_3_overtone_2_volume",
                "layer_speed"
            ],
        ]

        """
        self.potentiometers_layout  = [
            [
                "",
                "voice_1_overtone_2_harmonic",
                "voice_1_overtone_2_volume",
                "voice_1_overtone_1_fine",
                "voice_1_overtone_1_harmonic",
                "voice_1_overtone_1_volume",
                "voice_1_root_fine",
                "voice_1_root_harmonic",
            ],
            [
                "voice_1_root_half_steps",
                "voice_1_root_octave",
                "voice_1_formant_front_back",
                "voice_1_formant_open_close",
                "voice_1_formant_pitch",
                "voice_1_formant_volume",
                "voice_1_overtone_2_fine",
                ""
            ],
            [
                "voice_2_root_harmonic",
                "voice_2_root_fine",
                "voice_2_overtone_1_volume",
                "voice_2_overtone_1_harmonic",
                "voice_2_overtone_1_fine",
                "voice_2_overtone_2_volume",
                "voice_2_overtone_2_harmonic",
                ""
            ],
            [
                "voice_2_root_half_steps",
                "voice_2_root_octave",
                "voice_2_formant_front_back",
                "voice_2_formant_open_close",
                "voice_2_formant_pitch",
                "voice_2_formant_volume",
                "voice_2_overtone_2_fine",
                ""
            ],
            [
                "voice_3_root_harmonic",
                "voice_3_root_fine",
                "voice_3_overtone_1_volume",
                "voice_3_overtone_1_harmonic",
                "voice_3_overtone_1_fine",
                "voice_3_overtone_2_volume",
                "voice_3_overtone_2_harmonic",
                ""
            ],
            [
                "voice_3_root_half_steps",
                "voice_3_root_octave",
                "voice_3_formant_front_back",
                "voice_3_formant_open_close",
                "voice_3_formant_pitch",
                "voice_3_formant_volume",
                "voice_3_overtone_2_fine",
                "layer_speed"
            ]
        ]
        """

        self.potentiometer_last_value  = [
            [0,0,0,0,0,0,0,0],
            [0,0,0,0,0,0,0,0],
            [0,0,0,0,0,0,0,0],
            [0,0,0,0,0,0,0,0],
            [0,0,0,0,0,0,0,0],
            [0,0,0,0,0,0,0,0]
        ]
        self.mcp3008s = MCP3008s(self.spi_clock_pin, self.miso_pin, self.mosi_pin, self.chip_select_pins)

    def run(self):
        while True:
            all_adc_values =  self.mcp3008s.scan_all()
            #print all_adc_values
            for adc in range(len(all_adc_values)):
                for channel in range(8):
                    adc_value = 1023 - all_adc_values[adc][channel]
                    potentiometer_name = self.potentiometers_layout[adc][channel]
                    if potentiometer_name != "":
                        if abs(adc_value - self.potentiometer_last_value[adc][channel] ) > self.noise_threshold:
                            self.network_send_ref(potentiometer_name, adc_value/1023.0)
                            print adc, channel, self.potentiometers_layout[adc][channel], adc_value
                    self.potentiometer_last_value[adc][channel] = adc_value
            time.sleep(0.1)

class Network(object):
    def __init__(self, hostname, network_message_handler, network_status_handler):
        self.hostname = hostname
        self.thirtybirds = network_init(
            hostname=hostname,
            role="client",
            discovery_multicastGroup=settings.discovery_multicastGroup,
            discovery_multicastPort=settings.discovery_multicastPort,
            discovery_responsePort=settings.discovery_responsePort,
            pubsub_pubPort=settings.pubsub_pubPort,
            message_callback=network_message_handler,
            status_callback=network_status_handler
        )

# Main handles network send/recv and can see all other classes directly
class Main(threading.Thread):
    def __init__(self, hostname):
        threading.Thread.__init__(self)
        self.network = Network(hostname, self.network_message_handler, self.network_status_handler)
        self.queue = Queue.Queue()
        self.network.thirtybirds.subscribe_to_topic("door_closed")
        self.potentiometers = Potentiometers(self.network.thirtybirds.send) # self.network.thirtybirds.send
        self.potentiometers.daemon = True
        self.potentiometers.start()
        self.utils = Utils(hostname)
        self.network.thirtybirds.subscribe_to_topic("client_monitor_request")

    def network_message_handler(self, topic_msg):
        # this method runs in the thread of the caller, not the tread of Main
        topic, msg =  topic_msg # separating just to eval msg.  best to do it early.  it should be done in TB.
        if len(msg) > 0:
            msg = eval(msg)
        self.add_to_queue(topic, msg)

    def network_status_handler(self, topic_msg):
        # this method runs in the thread of the caller, not the tread of Main
        print "Main.network_status_handler", topic_msg

    def add_to_queue(self, topic, msg):
        self.queue.put((topic, msg))

    def run(self):
        while True:
            try:
                topic, msg = self.queue.get(True)
                if topic == "client_monitor_request":
                    self.network.thirtybirds.send("client_monitor_response", self.utils.get_client_status())

            except Exception as e:
                exc_type, exc_value, exc_traceback = sys.exc_info()
                print e, repr(traceback.format_exception(exc_type, exc_value,exc_traceback))

def init(hostname):
    main = Main(hostname)
    main.daemon = True
    main.start()
    return main

<|endoftext|>


<commit_before>#!/usr/local/bin/python2.7
"""list ISY vars demo app

Usage: %(program)s [options] [localhost:localport [remotehost:remoteport]]

Options:

    --list

"""

import ISY
import sys

import getopt

def list_vars_bash(myisy):
    fmt = "{0}={1}"
    for var in myisy.var_iter() :
	print fmt.format( var.name, var.value )

def list_vars(myisy) :
    fmt = "%-4s : %-19s%-5s\t%-5s\t%s"
    print fmt % ( "ID", "NAME", "VAL", "INIT", "DATE" )
    for var in myisy.var_iter() :
        print fmt % ( var.id, var.name, var.value, var.init, var.ts )

def usage(code, msg=''):
    print >> sys.stderr, __doc__ % globals()
    if msg:
        print >> sys.stderr, msg
    sys.exit(code)

class Options:
    debug = 0
    list = 0
    bash = 0

def parseargs():
    options = Options()

    # shortcut
    if len(sys.argv) == 1:
	options.list = 1
	return options

    try:
        opts, args = getopt.getopt(
            sys.argv[1:], "hbld",
            ['bash', 'list', 'help', 'debug='])
    except getopt.error, e:
        usage(1, e)

    for opt, arg in opts:
        if opt in ('-h', '--help'):
            usage(0)
        elif opt in ('-b', '--bash'):
            options.bash = 1
        elif opt in ('-l', '--list'):
            options.list = 1
        elif opt in ('-d', '--debug'):
            options.debug = arg

    return options, arg

def set_vars(isy, *arg):
    print "set_vars arg: ", arg
    for ar in arg:
	name, val  = str(ar).split('=')
	print "set ", name, " to ", val
	if str(val).isdigit :
	    try :
		isy.var_set_value(name, val)
	    except LookupError :
		print "bad Var name: ", ar
	else :
	    print "bad Value: ", ar
    return

if __name__ == '__main__':

    options, arg = parseargs()
    myisy = ISY.Isy( debug=options.debug )

    if options.list :
	list_vars(myisy)
    elif options.bash:
	list_vars_bash(myisy)

    if len(arg) :
	set_vars(myisy, *arg)

<commit_msg>more of the same<commit_after>#!/usr/local/bin/python2.7
"""list ISY vars demo app

Usage: %(program)s [options] [localhost:localport [remotehost:remoteport]]

Options:

    --list

"""

import ISY
import sys

import getopt

def list_vars_bash(myisy, csh=0):

    if csh :
	fmt = "set {0}={1}"
    else :
	fmt = "{0}={1}"

    for var in myisy.var_iter() :
	print fmt.format( var.name, var.value )

def list_vars(myisy) :
    fmt = "%-4s : %-19s%-5s\t%-5s\t%s"
    print fmt % ( "ID", "NAME", "VAL", "INIT", "DATE" )
    for var in myisy.var_iter() :
        print fmt % ( var.id, var.name, var.value, var.init, var.ts )

def usage(code, msg=''):
    print >> sys.stderr, __doc__ % globals()
    if msg:
        print >> sys.stderr, msg
    sys.exit(code)

class Options:
    debug = 0
    list = 0
    bash = 0
    csh = 0

def parseargs():
    options = Options()

    # shortcut
    if len(sys.argv) == 1:
	options.list = 1
	return options

    try:
        opts, args = getopt.getopt(
            sys.argv[1:], "hbldc",
            ['bash', 'csh', 'list', 'help', 'debug='])
    except getopt.error, e:
        usage(1, e)

    for opt, arg in opts:
        if opt in ('-h', '--help'):
            usage(0)
        elif opt in ('-b', '--bash'):
            options.bash = 1
        elif opt in ('-c', '--csh'):
            options.csh = 1
        elif opt in ('-l', '--list'):
            options.list = 1
        elif opt in ('-d', '--debug'):
            options.debug = arg

    return options, arg

def set_vars(isy, *arg):
    print "set_vars arg: ", arg
    for ar in arg:
	name, val  = str(ar).split('=')
	print "set ", name, " to ", val
	if str(val).isdigit :
	    try :
		isy.var_set_value(name, val)
	    except LookupError :
		print "bad Var name: ", ar
	else :
	    print "bad Value: ", ar
    return

if __name__ == '__main__':

    options, arg = parseargs()
    myisy = ISY.Isy( debug=options.debug )

    if options.list :
	list_vars(myisy)
    elif options.bash or options.csh:
	list_vars_bash(myisy, options.csh)

    if len(arg) :
	set_vars(myisy, *arg)

<|endoftext|>


<commit_before>import sqlite3
import pickle
import json
from time import time

"""
NoSQL interface to SQLite
"""

# TODO insert old values into history table

# PERF no serialization vs pickle -> 1.5 x writes per second, 1.1 x reads per second
# PERF json vs pickle -> 1.3 writes per second, 0.6 reads per second
# PERF optional pickle vs pickle -> 0.9 x reads and writes per second

class SERDES:
	"serialization-deserialization interface"
	def __init__(self):
		self.protocol = 3
	def serialize(self,v):
		return pickle.dumps(v,self.protocol)
		#return v
	def deserialize(self,v):
		return pickle.loads(v)
		#return v

class LINK:
	"link interface"
	def __init__(self):
		self.cursor.execute('create table if not exists tkv_link (link,k1,k2,ts)')
		self.cursor.execute('create unique index if not exists i_tkv_link1 on tkv_link (link,k1,k2)')
		self.cursor.execute('create unique index if not exists i_tkv_link2 on tkv_link (link,k2,k1)')
	def add_link(self,link,k1,k2):
		ts = time()
		self.cursor.execute('insert or replace into tkv_link values (?,?,?,?)', (link,k1,k2,ts))
	def get_linked(self,link,k1):
		for x in self.cursor.execute('select k2 from tkv_link where link=? and k1=?',(link,k1)):
			yield x[0]
	def get_linked_to(self,link,k2):
		for x in self.cursor.execute('select k1 from tkv_link where link=? and k2=?',(link,k2)):
			yield x[0]
	def get_links(self,link):
		for x in self.cursor.execute('select k1,k2 from tkv_link where link=?',(link,)):
			yield x
	def delete_links(self,k,any=[],left=[],right=[]):
		for link in any:
			self.cursor.execute('delete from tkv_link where link=? and (k1=? or k2=?)',(link,k,k))
		for link in left:
			self.cursor.execute('delete from tkv_link where link=? and k1=?',(link,k))
		for link in right:
			self.cursor.execute('delete from tkv_link where link=? and k2=?',(link,k))

# TODO separate file
class HIST:
	"history interface"
	def __init__(self):
		self.cursor.execute('create table if not exists tkv_hist (t,k,v,ts)')
		self.cursor.execute('create index if not exists i_tkv_hist on tkv (t,k,ts)')
	def set(self,t,k,v,ts):
		val = self.serialize(v)
		self.cursor.execute('insert into tkv_hist values (?,?,?,?)',(t,k,val,ts))
class NOHIST:
	def __init__(self): pass
	def set(self,t,k,v,ts): pass

class STAR:
	"star schema interface"
	def __init__(self):
		self.cursor.execute('create table if not exists tkv_star (s,k1,k2,k3,k4,k5,k6,k7,k8,k9)')
	def add_fact(self,star,*keys):
		pass

class TKV(SERDES,LINK,HIST):
	"Table-Key-Value database"
	def __init__(self,path=':memory:'):
		self.connection = sqlite3.connect(path)
		self.cursor = self.connection.cursor()
		self.cursor.execute('create table if not exists tkv (t,k,v,ts)')
		self.cursor.execute('create unique index if not exists i_tkv on tkv (t,k)')
		SERDES.__init__(self)
		LINK.__init__(self)
		HIST.__init__(self)
	### GET ###
	def get(self,t,k,default=None):
		results = self.cursor.execute('select v from tkv where t=? and k=?',(t,k))
		x = results.fetchone()
		return self.deserialize(x[0]) if x else default
	### SET ###
	def set(self,t,k,v):
		ts = time()
		val = self.serialize(v)
		self.cursor.execute('insert or replace into tkv values (?,?,?,?)',(t,k,val,ts))
		super().set(t,k,v,ts)
	### ITER ###
	def keys(self,t):
		for x in self.cursor.execute('select distinct k from tkv where t=?',(t,)):
			yield x[0]
	def values(self,t):
		for x in self.cursor.execute('select distinct v from tkv where t=?',(t,)):
			yield x[0]
	def items(self,t):
		for x in self.cursor.execute('select distinct k,v from tkv where t=?',(t,)):
			yield x[0],self.deserialize(x[1])
	### OTHER ###
	def delete(self,t,k):
		self.cursor.execute('delete from tkv where t=? and k=?',(t,k))
	def truncate(self,t):
		self.cursor.execute('delete from tkv where t=?',(t,))
	def commit(self):
		self.connection.commit()
	### ### ###
connect = TKV

if __name__=="__main__":
	db = connect()
	db.set('usr',1,'bob')
	db.set('usr',2,'alice')
	db.set('usr',3,'charlie')
	db.add_link('usr:knows:usr',1,2)
	db.add_link('usr:knows:usr',1,3)
	db.add_link('usr:knows:usr',2,3)
	print(dict(db.items('usr')))
	print(list(db.get_linked('usr:knows:usr',1)))
	print(list(db.get_links('usr:knows:usr')))
	db.delete_links(2,left=['usr:knows:usr'])
	print(list(db.get_links('usr:knows:usr')))
	if 0:
		CNT = 1000000
		t0=time()
		for i in range(CNT):
			db.set('test',i,str(i))
		dt = time()-t0
		print("db.set {0} elements -> {1:0.1f}s {2:0.0f}/s".format(CNT,dt,CNT/dt))
		t0=time()
		for i in range(CNT):
			db.get('test',i)
		dt = time()-t0
		print("db.get {0} elements -> {1:0.1f}s {2:0.0f}/s".format(CNT,dt,CNT/dt))

<commit_msg>tkv fix<commit_after>import sqlite3
import pickle
import json
from time import time

"""
NoSQL interface to SQLite
"""

# TODO insert old values into history table

# PERF no serialization vs pickle -> 1.5 x writes per second, 1.1 x reads per second
# PERF json vs pickle -> 1.3 writes per second, 0.6 reads per second
# PERF optional pickle vs pickle -> 0.9 x reads and writes per second

class SERDE:
	"serialization-deserialization interface"
	def __init__(self):
		self.protocol = 3
	def serialize(self,v):
		return pickle.dumps(v,self.protocol)
		#return v
	def deserialize(self,v):
		return pickle.loads(v)
		#return v

class LINK:
	"link interface"
	def __init__(self):
		self.cursor.execute('create table if not exists tkv_link (link,k1,k2,ts)')
		self.cursor.execute('create unique index if not exists i_tkv_link1 on tkv_link (link,k1,k2)')
		self.cursor.execute('create index if not exists i_tkv_link2 on tkv_link (link,k2)')
	def add_link(self,link,k1,k2):
		ts = time()
		self.cursor.execute('insert or replace into tkv_link values (?,?,?,?)', (link,k1,k2,ts))
	def get_linked(self,link,k1):
		for x in self.cursor.execute('select k2 from tkv_link where link=? and k1=?',(link,k1)):
			yield x[0]
	def get_linked_to(self,link,k2):
		for x in self.cursor.execute('select k1 from tkv_link where link=? and k2=?',(link,k2)):
			yield x[0]
	def get_links(self,link):
		for x in self.cursor.execute('select k1,k2 from tkv_link where link=?',(link,)):
			yield x
	def delete_links(self,k,any=[],left=[],right=[]):
		for link in any:
			self.cursor.execute('delete from tkv_link where link=? and (k1=? or k2=?)',(link,k,k))
		for link in left:
			self.cursor.execute('delete from tkv_link where link=? and k1=?',(link,k))
		for link in right:
			self.cursor.execute('delete from tkv_link where link=? and k2=?',(link,k))

# TODO separate file
class HIST:
	"history interface"
	def __init__(self):
		self.cursor.execute('create table if not exists tkv_hist (t,k,v,ts)')
		self.cursor.execute('create index if not exists i_tkv_hist on tkv (t,k,ts)')
	def set(self,t,k,v,ts):
		val = self.serialize(v)
		self.cursor.execute('insert into tkv_hist values (?,?,?,?)',(t,k,val,ts))
class NOHIST:
	def __init__(self): pass
	def set(self,t,k,v,ts): pass

# TODO
class STAR:
	"star schema interface"
	def __init__(self):
		self.cursor.execute('create table if not exists tkv_star (s,k1,k2,k3,k4,k5,k6,k7,k8,k9)')
	def add_fact(self,star,*keys):
		pass

# TODO HIST,SERDE mixin
class TKV(SERDE,LINK,NOHIST):
	"Table-Key-Value database"
	def __init__(self,path=':memory:'):
		self.connection = sqlite3.connect(path)
		self.cursor = self.connection.cursor()
		self.cursor.execute('create table if not exists tkv (t,k,v,ts)')
		self.cursor.execute('create unique index if not exists i_tkv on tkv (t,k)')
		SERDE.__init__(self)
		LINK.__init__(self)
		NOHIST.__init__(self)
	### GET ###
	def get(self,t,k,default=None):
		results = self.cursor.execute('select v from tkv where t=? and k=?',(t,k))
		x = results.fetchone()
		return self.deserialize(x[0]) if x else default
	### SET ###
	def set(self,t,k,v):
		ts = time()
		val = self.serialize(v)
		self.cursor.execute('insert or replace into tkv values (?,?,?,?)',(t,k,val,ts))
		super().set(t,k,v,ts)
	### ITER ###
	def keys(self,t):
		for k in self.cursor.execute('select distinct k from tkv where t=?',(t,)):
			yield k[0]
	def values(self,t):
		for v in self.cursor.execute('select distinct v from tkv where t=?',(t,)):
			yield self.deserialize(v[0])
	def items(self,t):
		for k,v in self.cursor.execute('select distinct k,v from tkv where t=?',(t,)):
			yield k,self.deserialize(v)
	### OTHER ###
	def delete(self,t,k):
		self.cursor.execute('delete from tkv where t=? and k=?',(t,k))
	def truncate(self,t):
		self.cursor.execute('delete from tkv where t=?',(t,))
	def commit(self):
		self.connection.commit()
	### ### ###
connect = TKV

if __name__=="__main__":
	db = connect()
	db.set('usr',1,'bob')
	db.set('usr',2,'alice')
	db.set('usr',3,'charlie')
	db.add_link('usr:knows:usr',1,2)
	db.add_link('usr:knows:usr',1,3)
	db.add_link('usr:knows:usr',2,3)
	print(dict(db.items('usr')))
	print(list(db.get_linked('usr:knows:usr',1)))
	print(list(db.get_links('usr:knows:usr')))
	db.delete_links(2,left=['usr:knows:usr'])
	print(list(db.get_links('usr:knows:usr')))
	print(list(db.values('usr')))
	if 0:
		CNT = 1000000
		t0=time()
		for i in range(CNT):
			db.set('test',i,str(i))
		dt = time()-t0
		print("db.set {0} elements -> {1:0.1f}s {2:0.0f}/s".format(CNT,dt,CNT/dt))
		t0=time()
		for i in range(CNT):
			db.get('test',i)
		dt = time()-t0
		print("db.get {0} elements -> {1:0.1f}s {2:0.0f}/s".format(CNT,dt,CNT/dt))

<|endoftext|>


<commit_before>__author__ = 'matt'

from databaseAccess import *
import spotipy

class UserLikesBroker():

    def __init__(self):
        self.spotify = spotipy.Spotify()

    def saveUserLikes(self, artists, user):
        try:
            UserLikes.get(UserLikes.userID == user)
        except:
            artistURIs = self.getArtistURIs(artists)

            for uri in artistURIs:
                userlike = UserLikes.create(userID=user,artistURI=uri)
                userlike.save()

    def getArtistURIs(self, artists):
        artistURIs = []

        for artist in artists:
            results = self.spotify.search(q='artist:' + artist, type='artist')
            if len(results) > 0:
                artistsData = results['artists']
                if len(artistsData) > 0:
                    if len(artistsData['items']) > 0:
                        artistInfo = artistsData['items'][0]
                        artistURIs.append(artistInfo["uri"])

        return artistURIs<commit_msg>userLikesBroker - changes to track down source of FB userID bug<commit_after>__author__ = 'matt'

from databaseAccess import *
import spotipy

class UserLikesBroker():

    def __init__(self):
        self.spotify = spotipy.Spotify()

    def saveUserLikes(self, artists, user):
        try:
            UserLikes.get(UserLikes.userID == user)
        except:
            artistURIs = self.getArtistURIs(artists)

            for uri in artistURIs:
                print user
                userlike = UserLikes.create(userID=user,artistURI=uri)
                userlike.save()

    def getArtistURIs(self, artists):
        artistURIs = []

        for artist in artists:
            results = self.spotify.search(q='artist:' + artist, type='artist')
            if len(results) > 0:
                artistsData = results['artists']
                if len(artistsData) > 0:
                    if len(artistsData['items']) > 0:
                        artistInfo = artistsData['items'][0]
                        artistURIs.append(artistInfo["uri"])

        return artistURIs<|endoftext|>


<commit_before># -*- coding:utf-8 -*-
#from django.conf.urls import patterns, url, include
from rest_framework.routers import DefaultRouter
from django.http import HttpResponseRedirect
from django.conf.urls import patterns, url

# class MyRouter(DefaultRouter):
#
#     def get_api_root_view(self):
# return lambda r:
# HttpResponseRedirect('https://github.com/globocom/database-as-a-service/wiki/Introduction-to-the-API')

router = DefaultRouter()

# physical

from .environment import EnvironmentAPI
router.register(r'environment', EnvironmentAPI)

from .plan import PlanAPI
router.register(r'plan', PlanAPI)

from .engine import EngineAPI
router.register(r'engine', EngineAPI)

from .engine_type import EngineTypeAPI
router.register(r'engine_type', EngineTypeAPI)

# logical
from .project import ProjectAPI
router.register(r'project', ProjectAPI)

from .database import DatabaseAPI
router.register(r'database', DatabaseAPI)

from .credential import CredentialAPI
router.register(r'credential', CredentialAPI)

from .extra_dns import ExtraDnsAPI
router.register(r'extra_dns', ExtraDnsAPI)

from .task import TaskAPI
router.register(r'task', TaskAPI, base_name="task")

from django.conf import settings
if settings.CLOUD_STACK_ENABLED:
    from .integration_type import CredentialTypeAPI
    router.register(r'integration_type', CredentialTypeAPI)

    from .integration_credential import IntegrationCredentialAPI
    router.register(r'integration_credential',
                    IntegrationCredentialAPI, base_name="integration_credential")

# account
from .team import TeamAPI
router.register(r'team', TeamAPI)

from .user import UserAPI
router.register(r'user', UserAPI)

from .snapshot import SnapshotAPI
router.register(r'snapshot', SnapshotAPI)

urlpatterns = router.urls

#from .task import TaskDetail
#urlpatterns += patterns( url('^task/(?P<task_id>.+)/$', TaskDetail.as_view()), )

# urlpatterns = patterns(
#     '',
#     # url(r'^$', 'api_help', name='api.index'),
#     # url(r'^', include(router.urls)),
#     # url(r'^api-auth/', include('rest_framework.urls', namespace='rest_framework'))
#     url('^logical', )
# )
<commit_msg>Add bundles cloudstack api on urls.py<commit_after># -*- coding:utf-8 -*-
#from django.conf.urls import patterns, url, include
from rest_framework.routers import DefaultRouter
from django.http import HttpResponseRedirect
from django.conf.urls import patterns, url, include

# class MyRouter(DefaultRouter):
#
#     def get_api_root_view(self):
# return lambda r:
# HttpResponseRedirect('https://github.com/globocom/database-as-a-service/wiki/Introduction-to-the-API')

router = DefaultRouter()
# physical
urlpatterns = []

from .environment import EnvironmentAPI
router.register(r'environment', EnvironmentAPI)

from .plan import PlanAPI
router.register(r'plan', PlanAPI)

from .engine import EngineAPI
router.register(r'engine', EngineAPI)

from .engine_type import EngineTypeAPI
router.register(r'engine_type', EngineTypeAPI)

# logical
from .project import ProjectAPI
router.register(r'project', ProjectAPI)

from .database import DatabaseAPI
router.register(r'database', DatabaseAPI)

from .credential import CredentialAPI
router.register(r'credential', CredentialAPI)

from .extra_dns import ExtraDnsAPI
router.register(r'extra_dns', ExtraDnsAPI)

from .task import TaskAPI
router.register(r'task', TaskAPI, base_name="task")

from django.conf import settings
if settings.CLOUD_STACK_ENABLED:
    from .integration_type import CredentialTypeAPI
    router.register(r'integration_type', CredentialTypeAPI)

    from .integration_credential import IntegrationCredentialAPI
    router.register(r'integration_credential',
                    IntegrationCredentialAPI, base_name="integration_credential")

    from dbaas_cloudstack.api import urls as cloudstack_api_urls
    urlpatterns += patterns('', url(r'^', include(cloudstack_api_urls)))

# account
from .team import TeamAPI
router.register(r'team', TeamAPI)

from .user import UserAPI
router.register(r'user', UserAPI)

from .snapshot import SnapshotAPI
router.register(r'snapshot', SnapshotAPI)

urlpatterns += router.urls

#from .task import TaskDetail
#urlpatterns += patterns( url('^task/(?P<task_id>.+)/$', TaskDetail.as_view()), )

# urlpatterns = patterns(
#     '',
#     # url(r'^$', 'api_help', name='api.index'),
#     # url(r'^', include(router.urls)),
#     # url(r'^api-auth/', include('rest_framework.urls', namespace='rest_framework'))
#     url('^logical', )
# )
<|endoftext|>


<commit_before>import uuid
import zlib
import commands
from pprint import pformat

from django.conf import settings
from django.core.urlresolvers import reverse
from django.http import HttpResponse

def generate_order_id():
    return uuid.uuid4().get_hex()

def get_redirect_url(request):
    url = "%s%s" % (settings.DOMAIN, reverse("dcavenue-callback"))
    if request.is_secure():
        return "https://%s" % (url, )
    else:
        return "http://%s" % (url, )

def checksum(redirect_url, amount, order_id):
    # "$MerchantId|$OrderId|$Amount|$redirectUrl|$WorkingKey"

    data = "%s|%s|%s|%s|%s" % (
        settings.DCAVENUE["MERCHANT_ID"], order_id, amount,
        redirect_url, settings.DCAVENUE["WORKING_KEY"]
    )

    return zlib.adler32(data, 1)

def enc_request(request, order_id):
    amount = request.REQUEST["Amount"]

    redirect_url = get_redirect_url(request)

    cca_request = "Merchant_Id=%s&Order_Id=%s&Redirect_Url=%s" % (
        settings.DCAVENUE["MERCHANT_ID"], order_id, redirect_url,
    )

    cca_request = "%s&%s" % (
        cca_request, "&".join(
            "%s=%s" % (k, v) for k, v in request.REQUEST.items()
        )
    )

    cca_request = "%s&TxnType=A&actionID=TXN&Checksum=%s" % (
        cca_request, checksum(redirect_url, amount, order_id)
    )

    return commands.getoutput(
        '%s -jar %s %s "%s" enc' % (
            settings.DCAVENUE.get("JAVA", "java"), settings.DCAVENUE["JAR"],
            settings.DCAVENUE["WORKING_KEY"], cca_request
        )
    )

#noinspection PyTypeChecker
def verify_checksum(data):
    # "$MerchantId|$OrderId|$Amount|$AuthDesc|$WorkingKey";
    data = "%s|%s|%s|%s|%s" % (
        settings.DCAVENUE["MERCHANT_ID"], data["OrderId"], data["Amount"],
        data["AuthDesc"], settings.DCAVENUE["WORKING_KEY"]
    )
    return zlib.adler32(data, 1) == data['Checksum']

def dec_response(request, response):
    response = commands.getoutput(
        '%s -jar %s %s "%s" dec' % (
            settings.DCAVENUE.get("JAVA", "java"), settings.DCAVENUE["JAR"],
            settings.DCAVENUE["WORKING_KEY"], response
        )
    )

    data = dict(
        part.split("=", 1) for part in response.split("&")
    )

    if not verify_checksum(data):
        return None

    return data

def default_callback(request, data):
    return HttpResponse(
        """
            <html>
                <body>
                    %s
                </body>
            </html>
        """ % pformat(data)
    )<commit_msg>minor<commit_after>import uuid
import zlib
import commands
from pprint import pformat

from django.conf import settings
from django.core.urlresolvers import reverse
from django.http import HttpResponse

def generate_order_id():
    return uuid.uuid4().get_hex()

def get_redirect_url(request):
    url = "%s%s" % (settings.DOMAIN, reverse("dcavenue-callback"))
    if request.is_secure():
        return "https://%s" % (url, )
    else:
        return "http://%s" % (url, )

def checksum(redirect_url, amount, order_id):
    # "$MerchantId|$OrderId|$Amount|$redirectUrl|$WorkingKey"

    data = "%s|%s|%s|%s|%s" % (
        settings.DCAVENUE["MERCHANT_ID"], order_id, amount,
        redirect_url, settings.DCAVENUE["WORKING_KEY"]
    )

    csum = zlib.adler32(data, 1)

    if csum < 0:
        csum += 2 ** 32

    return csum

def enc_request(request, order_id):
    amount = request.REQUEST["Amount"]

    redirect_url = get_redirect_url(request)

    cca_request = "Merchant_Id=%s&Order_Id=%s&Redirect_Url=%s" % (
        settings.DCAVENUE["MERCHANT_ID"], order_id, redirect_url,
    )

    cca_request = "%s&%s" % (
        cca_request, "&".join(
            "%s=%s" % (k, v) for k, v in request.REQUEST.items()
        )
    )

    cca_request = "%s&Checksum=%s" % (
        cca_request, checksum(redirect_url, amount, order_id)
    )

    return commands.getoutput(
        '%s -jar %s %s "%s" enc' % (
            settings.DCAVENUE.get("JAVA", "java"), settings.DCAVENUE["JAR"],
            settings.DCAVENUE["WORKING_KEY"], cca_request
        )
    )

#noinspection PyTypeChecker
def verify_checksum(data):
    # "$MerchantId|$OrderId|$Amount|$AuthDesc|$WorkingKey";
    data = "%s|%s|%s|%s|%s" % (
        settings.DCAVENUE["MERCHANT_ID"], data["OrderId"], data["Amount"],
        data["AuthDesc"], settings.DCAVENUE["WORKING_KEY"]
    )
    return zlib.adler32(data, 1) == data['Checksum']

def dec_response(request, response):
    response = commands.getoutput(
        '%s -jar %s %s "%s" dec' % (
            settings.DCAVENUE.get("JAVA", "java"), settings.DCAVENUE["JAR"],
            settings.DCAVENUE["WORKING_KEY"], response
        )
    )

    data = dict(
        part.split("=", 1) for part in response.split("&")
    )

    if not verify_checksum(data):
        return None

    return data

def default_callback(request, data):
    return HttpResponse(
        """
            <html>
                <body>
                    %s
                </body>
            </html>
        """ % pformat(data)
    )<|endoftext|>


<commit_before>"""This is a docstring for this container file"""

"""
This file is part of RAPD

Copyright (C) 2017, Cornell University
All rights reserved.

RAPD is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, version 3.

RAPD is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""

__created__ = "2017-05-10"
__maintainer__ = "Your name"
__email__ = "Your email"
__status__ = "Development"

# Standard imports
from pprint import pprint
import sys, os, signal
import shlex
#import time
from subprocess import Popen, PIPE, STDOUT
from multiprocessing import Pool

def local_subprocess(command,
                     logfile=False,
                     pid_queue=False,
                     result_queue=False,
                     tag=False):
    """
    Run job as subprocess on local machine. based on xutils.processLocal

    Arguments
    ---------
    commands - a dict with various data for running. Recognized fields are:
        command - the command to be run in the subprocess
        logfile - name of a logfile to be generated. The logfile will be STDOUT+STDERR
        pid_queue - a multiprocessing.Queue for placing PID of subprocess in
        tag - an identifying tag to be useful to the caller
    """

    proc = Popen(shlex.split(command),
                 stdout=PIPE,
                 stderr=PIPE,
                 #stderr=STDOUT,
                 )

    # Send back PID if have pid_queue
    if pid_queue:
        pid_queue.put(proc.pid)

    try:
        # Get the stdout and stderr from process
        stdout, stderr = proc.communicate()
        print stdout
    except KeyboardInterrupt:
        #sys.exit()
        os._exit()

    # Put results on a Queue, if given
    if result_queue:
        result = {
            "pid": proc.pid,
            "returncode": proc.returncode,
            "stdout": stdout,
            "stderr": stderr,
            "tag": tag
        }
        result_queue.put(result)

    # Write out a log file, if name passed in
    if logfile:
        print "In logfile %s" % logfile
        with open(logfile, "w") as out_file:
            out_file.write(stdout)
            out_file.write(stderr)

def mp_pool_FUTURE(nproc=8):
    """Setup and return a multiprocessing.Pool to launch jobs"""
    pool = Pool(processes=nproc)
    return pool
<commit_msg>Try using module directly<commit_after>"""This is a docstring for this container file"""

"""
This file is part of RAPD

Copyright (C) 2017, Cornell University
All rights reserved.

RAPD is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, version 3.

RAPD is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""

__created__ = "2017-05-10"
__maintainer__ = "Your name"
__email__ = "Your email"
__status__ = "Development"

# Standard imports
from pprint import pprint
import sys, os, signal
import shlex
#import time
import subprocess
from subprocess import Popen, PIPE, STDOUT
from multiprocessing import Pool

def local_subprocess(command,
                     logfile=False,
                     pid_queue=False,
                     result_queue=False,
                     tag=False):
    """
    Run job as subprocess on local machine. based on xutils.processLocal

    Arguments
    ---------
    commands - a dict with various data for running. Recognized fields are:
        command - the command to be run in the subprocess
        logfile - name of a logfile to be generated. The logfile will be STDOUT+STDERR
        pid_queue - a multiprocessing.Queue for placing PID of subprocess in
        tag - an identifying tag to be useful to the caller
    """

    print command
    proc = subprocess.Popen(shlex.split(command),
                            stdout=subprocess.PIPE,
                            stderr=subprocess.PIPE,
                            #stderr=STDOUT,
                            )

    # Send back PID if have pid_queue
    if pid_queue:
        pid_queue.put(proc.pid)

    try:
        # Get the stdout and stderr from process
        stdout, stderr = proc.communicate()
    except KeyboardInterrupt:
        #sys.exit()
        os._exit()

    # Put results on a Queue, if given
    if result_queue:
        result = {
            "pid": proc.pid,
            "returncode": proc.returncode,
            "stdout": stdout,
            "stderr": stderr,
            "tag": tag
        }
        result_queue.put(result)

    # Write out a log file, if name passed in
    if logfile:
        print "In logfile %s" % logfile
        with open(logfile, "w") as out_file:
            out_file.write(stdout)
            out_file.write(stderr)

def mp_pool_FUTURE(nproc=8):
    """Setup and return a multiprocessing.Pool to launch jobs"""
    pool = Pool(processes=nproc)
    return pool
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Created on Sat Apr 30 14:52:14 2016

@author: matsumi
"""

import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import load_digits
from sklearn.neighbors import KNeighborsClassifier

class K_NN(object):
    def __init__(self, learning_rate=0.5, max_iteration=100):
        pass

    def fit(self, X, y):
        pass

    def Kneighbors(self, X, n_neighbors=None, return_distance=True):
        pass

    def Kneighbors_graph(self, X=None, n_neighbors=None):
        pass

    def predict(self, X):
        pass

    def predict_proba(self, X):
        pass

    def score(self, X, y):
        pass

if __name__ == '__main__':

    digits = load_digits(2)
    X = digits.data
    num_sumples = len(X)
    T = digits.target

    # ライブラリを使用しない時
    classifier = K_NN()

    # ライブラリを使用した時
    lib_classifier = KNeighborsClassifier()
    lib_classifier.fit(X, T)
    lib_y = lib_classifier.predict(X)
    print "lib_y:", lib_y
    print "lib_classifier_accuracy:", lib_classifier.score(X, T)
<commit_msg>yのラベルを変更する関数を追加する score関数を追加する fit関数を追加する(WIP)<commit_after># -*- coding: utf-8 -*-
"""
Created on Sat Apr 30 14:52:14 2016

@author: matsumi
"""

import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import load_digits
from sklearn.neighbors import KNeighborsClassifier

class K_NN(object):
    def __init__(self, learning_rate=0.5, max_iteration=100):
        self.max_iteration = max_iteration

    def fit(self, X, y):
        y = self._convert_label(y)
        for (x_i, y_i) in zip(X, y):
           distance = x_i - y_i
           print distance
           return self

    def Kneighbors(self, X, n_neighbors=None, return_distance=True):
        pass

    def Kneighbors_graph(self, X=None, n_neighbors=None):
        pass

    def predict(self, X):
        pass

    def predict_proba(self, X):
        pass

    def score(self, X, y):
        predict_y = self.predict(X)
        correct_rate = np.mean(predict_y == y)
        return correct_rate

    def _convert_label(self, y):
        y = y.copy()
        y[y == 0] = -1
        return y

    def _revert_label(self, y):
        y = y.copy()
        y[y == -1] = 0
        return y

if __name__ == '__main__':

    digits = load_digits(2)
    X = digits.data
    num_sumples = len(X)
    T = digits.target

    # ライブラリを使用しない時
    classifier = K_NN()
    classifier.fit(X, T)

    # ライブラリを使用した時
    lib_classifier = KNeighborsClassifier()
    lib_classifier.fit(X, T)
    lib_y = lib_classifier.predict(X)
    print "lib_y:", lib_y
    print "lib_classifier_accuracy:", lib_classifier.score(X, T)
<|endoftext|>