<commit_before>"""
This module implements the class `truncated_gaussian` which
performs (conditional) UMPU tests for Gaussians
restricted to a set of intervals.

"""
import numpy as np
from scipy.stats import chi
from mpmath import fsum

import matplotlib
import matplotlib.pyplot as plt

import warnings

from abc import ABCMeta, abstractmethod

class truncated(object):
    """
    A distribution, truncated to a union of intervals

    HOW TO MAKE A SUBCLASS :
    You have to implement :

    __init__(self, args*) : It has to call the method from the base class
        Since the method is abstract, you can't have an instance of the
        subclass if the method __init__ is not implemented

    _cdf_notTruncated(self, a, b, dps) :

    With these two methods, you can use :
        -> cdf
        -> sf

    You should implement :

    _pdf_notTruncated(self, z, dps) : it allows you to use :
        -> pdf
        -> plt_pdf (if you also have _quantile_notTruncated)

    _quantile_notTruncated(self, q, tol) : it allows  you to use :
        -> quantile
        -> rvs
        -> plt_cdf
        -> plt_pdf (if you also have  _pdf_notTruncated)

    """

    __metaclass__ = ABCMeta

    @abstractmethod
    def __init__(self, intervals):
        """
        Create a new truncated distribution object
        This method is abstract : it has to be overriden

        Parameters
        ----------

        intervals : [(float, float)]
            The intervals the distribution is truncated to

        """
        self.intervals = intervals

        dps = 15
        not_precise = True
        while not_precise:
            dps *= 2.
            Q = [self._cdf_notTruncated(a, b, dps) for a, b in intervals]
            not_precise = (fsum(Q) == 0.)

        self._sumQ = fsum(Q)
        self._dps = dps
        self._Q = Q

    @abstractmethod
    def _cdf_notTruncated(self, a, b, dps=15):
        """
        Compute the probability of being in the interval (a, b)
        for a variable with this distribution (not truncated)

        Parameters
        ----------
        a, b : float
            Bounds of the interval. Can be infinite.

        dps : int
            Decimal precision (decimal places). Used in mpmath

        Returns
        -------
        p : float
            The probability of being in the intervals (a, b)
            P( a < X < b)
            for a non truncated variable

        WARNING : This is the fundamental method of the truncated class
        It has to be overriden for each distribution
        """
        pass

    def _quantile_notTruncated(self, q, tol=1.e-6):
        """
        Compute the quantile for the non truncated distribution

        Parameters
        ----------
        q : float
            quantile you want to compute. Between 0 and 1

        tol : float
            precision for the output

        Returns
        -------
        x : float
            x such that P(X < x) = q

        WARNING : This method is deprecated if not overriden. It can be
        very slow
        """

        warnings.warn("""Deprecated to use the general quantile_notTruncated
        method : it should be overrriden""", DeprecationWarning)

        dps = self._dps
        f = lambda x: cdf_notTruncated(-np.inf, x, dps)

        lb, ub = -1.e8, 1.e8
        x = find_root(f, y, lb, ub, tol=1e-6)

        return x

    def rvs(self, size=1):
        """
        Sample a random variable from the truncated disribution

        Parameters
        ----------
        size : int
           Number of samples. Default : 1

        Returns
        -------
        X : np.array
            array of sample

        """
        if not hasattr(self, '_quantile_notTruncated'):
            raise NotImplementedError( \
                """The 'quantile_notTruncated' method
                should be implemented in order to use the truncated
                rvs method"""
            )

        U = np.random.uniform(size=size)
        X = np.array([self.quantile(u) for u in U])
        return X

    def sf(self, z):
        """
        Compute the survival function of the truncated distribution

        Parameters
        ----------
        z : float
            Minimum bound of the interval

        Returns
        -------
        sf : float
            The survival function of the truncated distribution
            sf(z) = P( X > z | X is in intervals )

        """
        intervals = self.intervals
        Q, sumQ = self._Q, self._sumQ
        N = len(Q)
        dps = self._dps

        k, (a, b) = min( (k, (a, b))  for k, (a, b) in enumerate(intervals) if b > z)

        sf = fsum(Q[k+1:]) + self._cdf_notTruncated(max(a, z), b, dps)
        sf /= sumQ

        return sf

    def cdf(self, z):
        """
        Compute the survival function of the truncated distribution

        Parameters
        ----------
        z : float
            Minimum bound of the interval

        Returns
        -------
        cdf : float
            function  The cumulative distribution function of the
            truncated distribution
            cdf(z) = P( X < z | X is in intervals )

        WARNING : This method only use the sf method : it is never going to be
        more precise
        """
        cdf = 1. - self.sf(z)
        return cdf

    def pdf(self, z):
        """
        Compute the probability distribution funtion of the
        truncated distribution

        Parameters
        ----------
        z : float

        Returns
        -------
        p : float
            p(z) such that E[f(X)] = \int f(z)p(z)dz

        """

        if not hasattr(self, '_pdf_notTruncated'):
            raise NotImplementedError( \
                """The 'pdf_notTruncated'
                should be implemented in order to use the truncated
                pdf method"""
            )

        intervals = self.intervals
        dps = self._dps

        if intervals(z):
            p = self._pdf_notTruncated(z, dps)
            p /= self._sumQ
        else:
            p = 0

        return p

    def quantile(self, q, tol=1.e-6):
        if not hasattr(self, '_quantile_notTruncated'):
            raise NotImplementedError( \
                """The 'quantile_notTruncated' method
                should be implemented in order to use the truncated
                quantile method"""
            )

        Q = self._Q
        sumQ = self._sumQ
        intervals = self.intervals
        dps = self._dps

        cum_sum = np.cumsum( Q )

        k = min( i for i, c in enumerate(cum_sum) if c > q*sumQ )
        a, b = intervals[k]

        q_notTruncated = q*sumQ + self._cdf_notTruncated(-np.inf, a, dps)
        if k>0:
            q_notTruncated -= cum_sum[k-1]

        q_notTruncated = float(q_notTruncated)
        z = self._quantile_notTruncated(q_notTruncated, tol)

        return z

    def plt_cdf(self):
        import matplotlib
        import matplotlib.pyplot as plt
        print 1
        fig, ax = plt.subplots(1, 1)
        print 2
        l = self.quantile(0.01)
        u = self.quantile(0.99)
        print 2.5
        x = np.linspace(l, u, 100)
        print 3
        cdf_x = [self.cdf(t) for t in x]
        print 4
        ax.plot(x, cdf_x, 'r-', lw=5, alpha=0.6, label='cdf')
        print 5
        plt.show()

    def plt_pdf(self):
        import matplotlib
        import matplotlib.pyplot as plt
        fig, ax = plt.subplots(1, 1)
        x = np.linspace(self.quantile(0.01), self.quantile(0.99), 100)
        pdf_x = [self.pdf(t) for t in x]
        ax.plot(x, pdf_x, 'r-', lw=5, alpha=0.6, label='pdf')
        plt.show()

def find_root(f, y, lb, ub, tol=1e-6):
    """
    searches for solution to f(x) = y in (lb, ub), where
    f is a monotone decreasing function
    """

    # make sure solution is in range
    a, b   = lb, ub
    fa, fb = f(a), f(b)

    # assume a < b
    if fa > y and fb > y:
        while fb > y :
            b, fb = b + (b-a), f(b + (b-a))
    elif fa < y and fb < y:
        while fa < y :
            a, fa = a - (b-a), f(a - (b-a))

    # determine the necessary number of iterations
    try:
        max_iter = int( np.ceil( ( np.log(tol) - np.log(b-a) ) / np.log(0.5) ) )
    except OverflowError:
        warnings.warn('root finding failed, returning np.nan')
        return np.nan

    # bisect (slow but sure) until solution is obtained
    for _ in xrange(max_iter):
        try:
            c, fc  = (a+b)/2, f((a+b)/2)
            if fc > y: a = c
            elif fc < y: b = c
        except OverflowError:
            warnings.warn('root finding failed, returning np.nan')
            return np.nan

    return c

<commit_msg>removing matplotlib import<commit_after>"""
This module implements the class `truncated_gaussian` which
performs (conditional) UMPU tests for Gaussians
restricted to a set of intervals.

"""
import numpy as np
from scipy.stats import chi
from mpmath import fsum

import warnings

from abc import ABCMeta, abstractmethod

class truncated(object):
    """
    A distribution, truncated to a union of intervals

    HOW TO MAKE A SUBCLASS :
    You have to implement :

    __init__(self, args*) : It has to call the method from the base class
        Since the method is abstract, you can't have an instance of the
        subclass if the method __init__ is not implemented

    _cdf_notTruncated(self, a, b, dps) :

    With these two methods, you can use :
        -> cdf
        -> sf

    You should implement :

    _pdf_notTruncated(self, z, dps) : it allows you to use :
        -> pdf
        -> plt_pdf (if you also have _quantile_notTruncated)

    _quantile_notTruncated(self, q, tol) : it allows  you to use :
        -> quantile
        -> rvs
        -> plt_cdf
        -> plt_pdf (if you also have  _pdf_notTruncated)

    """

    __metaclass__ = ABCMeta

    @abstractmethod
    def __init__(self, intervals):
        """
        Create a new truncated distribution object
        This method is abstract : it has to be overriden

        Parameters
        ----------

        intervals : [(float, float)]
            The intervals the distribution is truncated to

        """
        self.intervals = intervals

        dps = 15
        not_precise = True
        while not_precise:
            dps *= 2.
            Q = [self._cdf_notTruncated(a, b, dps) for a, b in intervals]
            not_precise = (fsum(Q) == 0.)

        self._sumQ = fsum(Q)
        self._dps = dps
        self._Q = Q

    @abstractmethod
    def _cdf_notTruncated(self, a, b, dps=15):
        """
        Compute the probability of being in the interval (a, b)
        for a variable with this distribution (not truncated)

        Parameters
        ----------
        a, b : float
            Bounds of the interval. Can be infinite.

        dps : int
            Decimal precision (decimal places). Used in mpmath

        Returns
        -------
        p : float
            The probability of being in the intervals (a, b)
            P( a < X < b)
            for a non truncated variable

        WARNING : This is the fundamental method of the truncated class
        It has to be overriden for each distribution
        """
        pass

    def _quantile_notTruncated(self, q, tol=1.e-6):
        """
        Compute the quantile for the non truncated distribution

        Parameters
        ----------
        q : float
            quantile you want to compute. Between 0 and 1

        tol : float
            precision for the output

        Returns
        -------
        x : float
            x such that P(X < x) = q

        WARNING : This method is deprecated if not overriden. It can be
        very slow
        """

        warnings.warn("""Deprecated to use the general quantile_notTruncated
        method : it should be overrriden""", DeprecationWarning)

        dps = self._dps
        f = lambda x: cdf_notTruncated(-np.inf, x, dps)

        lb, ub = -1.e8, 1.e8
        x = find_root(f, y, lb, ub, tol=1e-6)

        return x

    def rvs(self, size=1):
        """
        Sample a random variable from the truncated disribution

        Parameters
        ----------
        size : int
           Number of samples. Default : 1

        Returns
        -------
        X : np.array
            array of sample

        """
        if not hasattr(self, '_quantile_notTruncated'):
            raise NotImplementedError( \
                """The 'quantile_notTruncated' method
                should be implemented in order to use the truncated
                rvs method"""
            )

        U = np.random.uniform(size=size)
        X = np.array([self.quantile(u) for u in U])
        return X

    def sf(self, z):
        """
        Compute the survival function of the truncated distribution

        Parameters
        ----------
        z : float
            Minimum bound of the interval

        Returns
        -------
        sf : float
            The survival function of the truncated distribution
            sf(z) = P( X > z | X is in intervals )

        """
        intervals = self.intervals
        Q, sumQ = self._Q, self._sumQ
        N = len(Q)
        dps = self._dps

        k, (a, b) = min( (k, (a, b))  for k, (a, b) in enumerate(intervals) if b > z)

        sf = fsum(Q[k+1:]) + self._cdf_notTruncated(max(a, z), b, dps)
        sf /= sumQ

        return sf

    def cdf(self, z):
        """
        Compute the survival function of the truncated distribution

        Parameters
        ----------
        z : float
            Minimum bound of the interval

        Returns
        -------
        cdf : float
            function  The cumulative distribution function of the
            truncated distribution
            cdf(z) = P( X < z | X is in intervals )

        WARNING : This method only use the sf method : it is never going to be
        more precise
        """
        cdf = 1. - self.sf(z)
        return cdf

    def pdf(self, z):
        """
        Compute the probability distribution funtion of the
        truncated distribution

        Parameters
        ----------
        z : float

        Returns
        -------
        p : float
            p(z) such that E[f(X)] = \int f(z)p(z)dz

        """

        if not hasattr(self, '_pdf_notTruncated'):
            raise NotImplementedError( \
                """The 'pdf_notTruncated'
                should be implemented in order to use the truncated
                pdf method"""
            )

        intervals = self.intervals
        dps = self._dps

        if intervals(z):
            p = self._pdf_notTruncated(z, dps)
            p /= self._sumQ
        else:
            p = 0

        return p

    def quantile(self, q, tol=1.e-6):
        if not hasattr(self, '_quantile_notTruncated'):
            raise NotImplementedError( \
                """The 'quantile_notTruncated' method
                should be implemented in order to use the truncated
                quantile method"""
            )

        Q = self._Q
        sumQ = self._sumQ
        intervals = self.intervals
        dps = self._dps

        cum_sum = np.cumsum( Q )

        k = min( i for i, c in enumerate(cum_sum) if c > q*sumQ )
        a, b = intervals[k]

        q_notTruncated = q*sumQ + self._cdf_notTruncated(-np.inf, a, dps)
        if k>0:
            q_notTruncated -= cum_sum[k-1]

        q_notTruncated = float(q_notTruncated)
        z = self._quantile_notTruncated(q_notTruncated, tol)

        return z

def find_root(f, y, lb, ub, tol=1e-6):
    """
    searches for solution to f(x) = y in (lb, ub), where
    f is a monotone decreasing function
    """

    # make sure solution is in range
    a, b   = lb, ub
    fa, fb = f(a), f(b)

    # assume a < b
    if fa > y and fb > y:
        while fb > y :
            b, fb = b + (b-a), f(b + (b-a))
    elif fa < y and fb < y:
        while fa < y :
            a, fa = a - (b-a), f(a - (b-a))

    # determine the necessary number of iterations
    try:
        max_iter = int( np.ceil( ( np.log(tol) - np.log(b-a) ) / np.log(0.5) ) )
    except OverflowError:
        warnings.warn('root finding failed, returning np.nan')
        return np.nan

    # bisect (slow but sure) until solution is obtained
    for _ in xrange(max_iter):
        try:
            c, fc  = (a+b)/2, f((a+b)/2)
            if fc > y: a = c
            elif fc < y: b = c
        except OverflowError:
            warnings.warn('root finding failed, returning np.nan')
            return np.nan

    return c

<|endoftext|>


<commit_before>import numpy as np
from random_tree import RandomClassifierTree
from util import timer, get_best_dtype, dtype_to_ctype, mk_kernel, mk_tex_kernel, compile_module
from pycuda import gpuarray
from pycuda import driver
from util import start_timer, end_timer, show_timings
from parakeet import jit
import math
import sys

@jit
def convert_result(tran_table, res):
    return np.array([tran_table[i] for i in res])

class RandomForestClassifier(object):
  """A random forest classifier.

    A random forest is a meta estimator that fits a number of classifical
    decision trees on various sub-samples of the dataset and use averaging
    to improve the predictive accuracy and control over-fitting.

    Usage:
    See RandomForestClassifier.fit
  """
  COMPUTE_THREADS_PER_BLOCK = 128
  RESHUFFLE_THREADS_PER_BLOCK = 256
  BFS_THREADS = 64
  MAX_BLOCK_PER_FEATURE = 50
  MAX_BLOCK_BFS = 10000

  def __init__(self, n_estimators = 10, max_features = None, min_samples_split = 1, bootstrap = True, verbose = False, debug = False):
    """Construce multiple trees in the forest.

    Parameters
    ----------
    n_estimator : integer, optional (default=10)
        The number of trees in the forest.

    max_features : int or None, optional (default="log2(n_features)")
        The number of features to consider when looking for the best split:
          - If None, then `max_features=log2(n_features)`.

    min_samples_split : integer, optional (default=1)
        The minimum number of samples required to split an internal node.
        Note: this parameter is tree-specific.

    bootstrap : boolean, optional (default=True)
        Whether use bootstrap samples

    verbose : boolean, optional (default=False)
        Display the time of each tree construction takes if verbose = True.

    Returns
    -------
    None
    """
    self.max_features = max_features
    self.min_samples_split = min_samples_split
    self.bootstrap = bootstrap
    self.verbose = verbose
    self.n_estimators = n_estimators
    self.debug = debug
    self.forest = list()

  def __compact_labels(self, target):
    def check_is_compacted(x):
      return x.size == int(np.max(x)) + 1 and int(np.min(x)) == 0
    def convert_to_dict(x):
      d = {}
      for i, val in enumerate(x):
        d[val] = i
      return d

    self.compt_table = np.unique(target)
    self.compt_table.sort()
    if not check_is_compacted(self.compt_table):
      trans_table = convert_to_dict(self.compt_table)
      for i, val in enumerate(target):
        target[i] = trans_table[val]

  def __init_bootstrap_kernel(self):
    """ Compile the kernels and GPUArrays needed to generate the bootstrap samples"""
    ctype_indices = dtype_to_ctype(self.dtype_indices)
    self.bootstrap_fill= mk_kernel((ctype_indices,), "bootstrap_fill",
        "bootstrap_fill.cu")
    self.bootstrap_reshuffle, tex_ref = mk_tex_kernel((ctype_indices, 128), "bootstrap_reshuffle",
        "tex_mark", "bootstrap_reshuffle.cu")

    self.bootstrap_fill.prepare("PPii")
    self.bootstrap_reshuffle.prepare("PPPi")
    self.mark_table.bind_to_texref_ext(tex_ref)

  def _get_sorted_indices(self, sorted_indices):
    """ Generate sorted indices, if bootstrap == False, then the sorted indices is as same as original sorted indices """

    sorted_indices_gpu_original = self.sorted_indices_gpu.copy()

    if not self.bootstrap:
      return sorted_indices_gpu_original, sorted_indices.shape[1]
    else:
      sorted_indices_gpu = gpuarray.empty((self.n_features, self.stride), dtype = self.dtype_indices)
      random_sample_idx = np.unique(np.random.randint(0, self.stride, size = self.stride)).astype(self.dtype_indices)
      random_sample_idx_gpu = gpuarray.to_gpu(random_sample_idx)
      n_samples = random_sample_idx.size

      self.bootstrap_fill.prepared_call(
                (1, 1),
                (512, 1, 1),
                random_sample_idx_gpu.ptr,
                self.mark_table.ptr,
                n_samples,
                self.stride)

      self.bootstrap_reshuffle.prepared_call(
                (self.n_features, 1),
                (128, 1, 1),
                self.mark_table.ptr,
                sorted_indices_gpu_original.ptr,
                sorted_indices_gpu.ptr,
                self.stride)

      return sorted_indices_gpu, n_samples

  def fit_init(self, samples, target):
    assert isinstance(samples, np.ndarray)
    assert isinstance(target, np.ndarray)
    assert samples.size / samples[0].size == target.size
    target = target.copy()
    self.__compact_labels(target)

    self.n_labels = self.compt_table.size
    self.dtype_indices = get_best_dtype(target.size)

    if self.dtype_indices == np.dtype(np.uint8):
      self.dtype_indices = np.dtype(np.uint16)

    self.dtype_counts = self.dtype_indices
    self.dtype_labels = get_best_dtype(self.n_labels)
    self.dtype_samples = samples.dtype

    samples = np.require(np.transpose(samples), requirements = 'C')
    target = np.require(np.transpose(target), dtype = self.dtype_labels, requirements = 'C')

    self.n_features = samples.shape[0]
    self.stride = target.size

    if self.COMPUTE_THREADS_PER_BLOCK > self.stride:
      self.COMPUTE_THREADS_PER_BLOCK = 32
    if self.RESHUFFLE_THREADS_PER_BLOCK > self.stride:
      self.RESHUFFLE_THREADS_PER_BLOCK = 32

    samples_gpu = gpuarray.to_gpu(samples)
    labels_gpu = gpuarray.to_gpu(target)

    sorted_indices = np.argsort(samples).astype(self.dtype_indices)
    self.sorted_indices_gpu = gpuarray.to_gpu(sorted_indices)
    self.mark_table = gpuarray.empty(self.stride, np.uint8)

    if self.max_features is None:
      self.max_features = int(math.ceil(np.sqrt(self.n_features)))

    self.__compile_kernels()

    if self.bootstrap:
      self.__init_bootstrap_kernel()

    self.sorted_indices = sorted_indices
    self.target = target
    self.samples = samples
    self.samples_gpu = samples_gpu
    self.labels_gpu = labels_gpu
    assert self.max_features > 0 and self.max_features <= self.n_features

  def fit_release(self):
    self.target = None
    self.samples = None
    self.samples_gpu = None
    self.labels_gpu = None
    self.sorted_indices_gpu = None
    self.mark_table = None
    self.sorted_indices = None

  def fit(self, samples, target, bfs_threshold = None):
    """Construce multiple trees in the forest.

    Parameters
    ----------
    samples:numpy.array of shape = [n_samples, n_features]
            The training input samples.

    target: numpy.array of shape = [n_samples]
            The training input labels.

    bfs_threshold: integer, optional (default= n_samples / 40)
            The n_samples threshold of changing to bfs

    Returns
    -------
    self : object
      Returns self
    """
    self.fit_init(samples, target)

    if bfs_threshold is None:
      n_classes = len(np.unique(target))
      n_samples = len(samples)
      bfs_threshold = int(3855 - 2 * n_classes + 0.076 * n_samples - 10.7 * self.max_features)
      # don't let it grow too big
      bfs_threshold = min(bfs_threshold, min(n_samples, 25000))
      # ...or too small
      bfs_threshold = max(bfs_threshold, 2000)

    if self.verbose:
      print "bsf_threadshold : %d; bootstrap : %r; min_samples_split : %d" % (bfs_threshold, self.bootstrap,
          self.min_samples_split)
      print "n_samples : %d; n_features : %d; n_labels : %d; max_features : %d" % (self.stride, self.n_features,
              self.n_labels, self.max_features)

    self.forest = [RandomClassifierTree(self.samples_gpu, self.labels_gpu, self.compt_table,
      self.dtype_labels,self.dtype_samples, self.dtype_indices, self.dtype_counts,
      self.n_features, self.stride, self.n_labels, self.COMPUTE_THREADS_PER_BLOCK,
      self.RESHUFFLE_THREADS_PER_BLOCK, self.max_features, self.min_samples_split, bfs_threshold, self.debug,
      self) for i in xrange(self.n_estimators)]

    for i, tree in enumerate(self.forest):
      si, n_samples = self._get_sorted_indices(self.sorted_indices)

      if self.verbose:
        with timer("Tree %s" % (i,)):
          tree.fit(self.samples, self.target, si, n_samples)
        print ""
      else:
        tree.fit(self.samples, self.target, si, n_samples)

    self.fit_release()
    return self

  def predict(self, x):
    """Predict labels for giving samples.

    Parameters
    ----------
    x:numpy.array of shape = [n_samples, n_features]
            The predicting input samples.

    Returns
    -------
    y: Array of shape [n_samples].
        The predicted labels.
    """
    x = np.require(x.copy(), requirements = "C")
    res = np.ndarray((len(self.forest), x.shape[0]), dtype = self.dtype_labels)

    for i, tree in enumerate(self.forest):
      res[i] =  tree.gpu_predict(x)

    res =  np.array([np.argmax(np.bincount(res[:,i])) for i in xrange(res.shape[1])])
    if hasattr(self, "compt_table"):
      res = convert_result(self.compt_table, res)

    return res

  def predict_proba(self, x):
    x = np.require(x.copy(), requirements = "C")
    res = np.ndarray((len(self.forest), x.shape[0]), dtype = self.dtype_labels)
    res_proba = np.ndarray((x.shape[0], self.n_labels), np.float64)

    for i, tree in enumerate(self.forest):
      res[i] =  tree.gpu_predict(x)

    for i in xrange(x.shape[0]):
      tmp_res = np.bincount(res[:, i])
      tmp_res.resize(self.n_labels)
      res_proba[i] = tmp_res.astype(np.float64) / len(self.forest)

    return res_proba

  def score(self, X, Y):
    return np.mean(self.predict(X) == Y)

  def __compile_kernels(self):
    ctype_indices = dtype_to_ctype(self.dtype_indices)
    ctype_labels = dtype_to_ctype(self.dtype_labels)
    ctype_counts = dtype_to_ctype(self.dtype_counts)
    ctype_samples = dtype_to_ctype(self.dtype_samples)
    n_labels = self.n_labels
    n_threads = self.COMPUTE_THREADS_PER_BLOCK
    n_shf_threads = self.RESHUFFLE_THREADS_PER_BLOCK

    """ DFS module """
    dfs_module = compile_module("dfs_module.cu", (n_threads, n_shf_threads, n_labels, ctype_samples,
      ctype_labels, ctype_counts, ctype_indices, self.MAX_BLOCK_PER_FEATURE, self.debug))

    const_stride = dfs_module.get_global("stride")[0]
    driver.memcpy_htod(const_stride, np.uint32(self.stride))

    self.find_min_kernel = dfs_module.get_function("find_min_imp")
    self.find_min_kernel.prepare("PPPi")

    self.fill_kernel = dfs_module.get_function("fill_table")
    self.fill_kernel.prepare("PiiP")

    self.scan_reshuffle_tex = dfs_module.get_function("scan_reshuffle")
    self.scan_reshuffle_tex.prepare("PPii")
    tex_ref = dfs_module.get_texref("tex_mark")
    self.mark_table.bind_to_texref_ext(tex_ref)

    self.comput_total_2d = dfs_module.get_function("compute_2d")
    self.comput_total_2d.prepare("PPPPPPPii")

    self.reduce_2d = dfs_module.get_function("reduce_2d")
    self.reduce_2d.prepare("PPPPPi")

    self.scan_total_2d = dfs_module.get_function("scan_gini_large")
    self.scan_total_2d.prepare("PPPPii")

    self.scan_reduce = dfs_module.get_function("scan_reduce")
    self.scan_reduce.prepare("Pi")

    """ BFS module """
    bfs_module = compile_module("bfs_module.cu", (self.BFS_THREADS, n_labels, ctype_samples,
      ctype_labels, ctype_counts, ctype_indices,  self.debug))

    const_stride = bfs_module.get_global("stride")[0]
    const_n_features = bfs_module.get_global("n_features")[0]
    const_max_features = bfs_module.get_global("max_features")[0]
    driver.memcpy_htod(const_stride, np.uint32(self.stride))
    driver.memcpy_htod(const_n_features, np.uint16(self.n_features))
    driver.memcpy_htod(const_max_features, np.uint16(self.max_features))

    self.scan_total_bfs = bfs_module.get_function("scan_bfs")
    self.scan_total_bfs.prepare("PPPP")

    self.comput_bfs_2d = bfs_module.get_function("compute_2d")
    self.comput_bfs_2d.prepare("PPPPPPPPP")

    self.fill_bfs = bfs_module.get_function("fill_table")
    self.fill_bfs.prepare("PPPPP")

    self.reshuffle_bfs = bfs_module.get_function("scan_reshuffle")
    tex_ref = bfs_module.get_texref("tex_mark")
    self.mark_table.bind_to_texref_ext(tex_ref)
    self.reshuffle_bfs.prepare("PPP")

    self.reduce_bfs_2d = bfs_module.get_function("reduce")
    self.reduce_bfs_2d.prepare("PPPPPPi")

    self.get_thresholds = bfs_module.get_function("get_thresholds")
    self.get_thresholds.prepare("PPPPP")

    self.predict_kernel = mk_kernel(
        params = (ctype_indices, ctype_samples, ctype_labels),
        func_name = "predict",
        kernel_file = "predict.cu",
        prepare_args = "PPPPPPPii")

    self.bfs_module = bfs_module
    self.dfs_module = dfs_module
<commit_msg>comment<commit_after>import numpy as np
from random_tree import RandomClassifierTree
from util import timer, get_best_dtype, dtype_to_ctype, mk_kernel, mk_tex_kernel, compile_module
from pycuda import gpuarray
from pycuda import driver
from util import start_timer, end_timer, show_timings
from parakeet import jit
import math
import sys

@jit
def convert_result(tran_table, res):
    return np.array([tran_table[i] for i in res])

class RandomForestClassifier(object):
  """A random forest classifier.

    A random forest is a meta estimator that fits a number of classifical
    decision trees on various sub-samples of the dataset and use averaging
    to improve the predictive accuracy and control over-fitting.

    Usage:
    See RandomForestClassifier.fit
  """
  COMPUTE_THREADS_PER_BLOCK = 128
  RESHUFFLE_THREADS_PER_BLOCK = 256
  BFS_THREADS = 64
  MAX_BLOCK_PER_FEATURE = 50
  MAX_BLOCK_BFS = 10000

  def __init__(self, n_estimators = 10, max_features = None, min_samples_split = 1, bootstrap = True, verbose = False, debug = False):
    """Construce multiple trees in the forest.

    Parameters
    ----------
    n_estimator : integer, optional (default=10)
        The number of trees in the forest.

    max_features : int or None, optional (default="log2(n_features)")
        The number of features to consider when looking for the best split:
          - If None, then `max_features=log2(n_features)`.

    min_samples_split : integer, optional (default=1)
        The minimum number of samples required to split an internal node.
        Note: this parameter is tree-specific.

    bootstrap : boolean, optional (default=True)
        Whether use bootstrap samples

    verbose : boolean, optional (default=False)
        Display the time of each tree construction takes if verbose = True.

    Returns
    -------
    None
    """
    self.max_features = max_features
    self.min_samples_split = min_samples_split
    self.bootstrap = bootstrap
    self.verbose = verbose
    self.n_estimators = n_estimators
    self.debug = debug
    self.forest = list()

  def __compact_labels(self, target):
    def check_is_compacted(x):
      return x.size == int(np.max(x)) + 1 and int(np.min(x)) == 0
    def convert_to_dict(x):
      d = {}
      for i, val in enumerate(x):
        d[val] = i
      return d

    self.compt_table = np.unique(target)
    self.compt_table.sort()
    if not check_is_compacted(self.compt_table):
      trans_table = convert_to_dict(self.compt_table)
      for i, val in enumerate(target):
        target[i] = trans_table[val]

  def __init_bootstrap_kernel(self):
    """ Compile the kernels and GPUArrays needed to generate the bootstrap samples"""
    ctype_indices = dtype_to_ctype(self.dtype_indices)
    self.bootstrap_fill= mk_kernel((ctype_indices,), "bootstrap_fill",
        "bootstrap_fill.cu")
    self.bootstrap_reshuffle, tex_ref = mk_tex_kernel((ctype_indices, 128), "bootstrap_reshuffle",
        "tex_mark", "bootstrap_reshuffle.cu")

    self.bootstrap_fill.prepare("PPii")
    self.bootstrap_reshuffle.prepare("PPPi")
    self.mark_table.bind_to_texref_ext(tex_ref)

  def _get_sorted_indices(self, sorted_indices):
    """ Generate sorted indices, if bootstrap == False, then the sorted indices is as same as original sorted indices """

    sorted_indices_gpu_original = self.sorted_indices_gpu.copy()

    if not self.bootstrap:
      return sorted_indices_gpu_original, sorted_indices.shape[1]
    else:
      sorted_indices_gpu = gpuarray.empty((self.n_features, self.stride), dtype = self.dtype_indices)
      random_sample_idx = np.unique(np.random.randint(0, self.stride, size = self.stride)).astype(self.dtype_indices)
      random_sample_idx_gpu = gpuarray.to_gpu(random_sample_idx)
      n_samples = random_sample_idx.size

      self.bootstrap_fill.prepared_call(
                (1, 1),
                (512, 1, 1),
                random_sample_idx_gpu.ptr,
                self.mark_table.ptr,
                n_samples,
                self.stride)

      self.bootstrap_reshuffle.prepared_call(
                (self.n_features, 1),
                (128, 1, 1),
                self.mark_table.ptr,
                sorted_indices_gpu_original.ptr,
                sorted_indices_gpu.ptr,
                self.stride)

      return sorted_indices_gpu, n_samples

  def fit_init(self, samples, target):
    assert isinstance(samples, np.ndarray)
    assert isinstance(target, np.ndarray)
    assert samples.size / samples[0].size == target.size
    target = target.copy()
    self.__compact_labels(target)

    self.n_labels = self.compt_table.size
    self.dtype_indices = get_best_dtype(target.size)

    if self.dtype_indices == np.dtype(np.uint8):
      self.dtype_indices = np.dtype(np.uint16)

    self.dtype_counts = self.dtype_indices
    self.dtype_labels = get_best_dtype(self.n_labels)
    self.dtype_samples = samples.dtype

    samples = np.require(np.transpose(samples), requirements = 'C')
    target = np.require(np.transpose(target), dtype = self.dtype_labels, requirements = 'C')

    self.n_features = samples.shape[0]
    self.stride = target.size

    if self.COMPUTE_THREADS_PER_BLOCK > self.stride:
      self.COMPUTE_THREADS_PER_BLOCK = 32
    if self.RESHUFFLE_THREADS_PER_BLOCK > self.stride:
      self.RESHUFFLE_THREADS_PER_BLOCK = 32

    samples_gpu = gpuarray.to_gpu(samples)
    labels_gpu = gpuarray.to_gpu(target)

    sorted_indices = np.argsort(samples).astype(self.dtype_indices)
    self.sorted_indices_gpu = gpuarray.to_gpu(sorted_indices)
    self.mark_table = gpuarray.empty(self.stride, np.uint8)

    if self.max_features is None:
      self.max_features = int(math.ceil(np.sqrt(self.n_features)))

    self.__compile_kernels()

    if self.bootstrap:
      self.__init_bootstrap_kernel()

    self.sorted_indices = sorted_indices
    self.target = target
    self.samples = samples
    self.samples_gpu = samples_gpu
    self.labels_gpu = labels_gpu
    assert self.max_features > 0 and self.max_features <= self.n_features

  def fit_release(self):
    self.target = None
    self.samples = None
    self.samples_gpu = None
    self.labels_gpu = None
    self.sorted_indices_gpu = None
    self.mark_table = None
    self.sorted_indices = None

  def fit(self, samples, target, bfs_threshold = None):
    """Construce multiple trees in the forest.

    Parameters
    ----------
    samples:numpy.array of shape = [n_samples, n_features]
            The training input samples.

    target: numpy.array of shape = [n_samples]
            The training input labels.

    bfs_threshold: integer, optional (default= n_samples / 40)
            The n_samples threshold of changing to bfs

    Returns
    -------
    self : object
      Returns self
    """
    self.fit_init(samples, target)

    if bfs_threshold is None:
      n_classes = len(np.unique(target))
      n_samples = len(samples)
      # coefficients estimated by regression over best thresholds for randomly generated data sets
      bfs_threshold = int(3855 - 2 * n_classes + 0.076 * n_samples - 10.7 * self.max_features)
      # don't let it grow too big
      bfs_threshold = min(bfs_threshold, min(n_samples, 25000))
      # ...or too small
      bfs_threshold = max(bfs_threshold, 2000)

    if self.verbose:
      print "bsf_threadshold : %d; bootstrap : %r; min_samples_split : %d" % (bfs_threshold, self.bootstrap,
          self.min_samples_split)
      print "n_samples : %d; n_features : %d; n_labels : %d; max_features : %d" % (self.stride, self.n_features,
              self.n_labels, self.max_features)

    self.forest = [RandomClassifierTree(self.samples_gpu, self.labels_gpu, self.compt_table,
      self.dtype_labels,self.dtype_samples, self.dtype_indices, self.dtype_counts,
      self.n_features, self.stride, self.n_labels, self.COMPUTE_THREADS_PER_BLOCK,
      self.RESHUFFLE_THREADS_PER_BLOCK, self.max_features, self.min_samples_split, bfs_threshold, self.debug,
      self) for i in xrange(self.n_estimators)]

    for i, tree in enumerate(self.forest):
      si, n_samples = self._get_sorted_indices(self.sorted_indices)

      if self.verbose:
        with timer("Tree %s" % (i,)):
          tree.fit(self.samples, self.target, si, n_samples)
        print ""
      else:
        tree.fit(self.samples, self.target, si, n_samples)

    self.fit_release()
    return self

  def predict(self, x):
    """Predict labels for giving samples.

    Parameters
    ----------
    x:numpy.array of shape = [n_samples, n_features]
            The predicting input samples.

    Returns
    -------
    y: Array of shape [n_samples].
        The predicted labels.
    """
    x = np.require(x.copy(), requirements = "C")
    res = np.ndarray((len(self.forest), x.shape[0]), dtype = self.dtype_labels)

    for i, tree in enumerate(self.forest):
      res[i] =  tree.gpu_predict(x)

    res =  np.array([np.argmax(np.bincount(res[:,i])) for i in xrange(res.shape[1])])
    if hasattr(self, "compt_table"):
      res = convert_result(self.compt_table, res)

    return res

  def predict_proba(self, x):
    x = np.require(x.copy(), requirements = "C")
    res = np.ndarray((len(self.forest), x.shape[0]), dtype = self.dtype_labels)
    res_proba = np.ndarray((x.shape[0], self.n_labels), np.float64)

    for i, tree in enumerate(self.forest):
      res[i] =  tree.gpu_predict(x)

    for i in xrange(x.shape[0]):
      tmp_res = np.bincount(res[:, i])
      tmp_res.resize(self.n_labels)
      res_proba[i] = tmp_res.astype(np.float64) / len(self.forest)

    return res_proba

  def score(self, X, Y):
    return np.mean(self.predict(X) == Y)

  def __compile_kernels(self):
    ctype_indices = dtype_to_ctype(self.dtype_indices)
    ctype_labels = dtype_to_ctype(self.dtype_labels)
    ctype_counts = dtype_to_ctype(self.dtype_counts)
    ctype_samples = dtype_to_ctype(self.dtype_samples)
    n_labels = self.n_labels
    n_threads = self.COMPUTE_THREADS_PER_BLOCK
    n_shf_threads = self.RESHUFFLE_THREADS_PER_BLOCK

    """ DFS module """
    dfs_module = compile_module("dfs_module.cu", (n_threads, n_shf_threads, n_labels, ctype_samples,
      ctype_labels, ctype_counts, ctype_indices, self.MAX_BLOCK_PER_FEATURE, self.debug))

    const_stride = dfs_module.get_global("stride")[0]
    driver.memcpy_htod(const_stride, np.uint32(self.stride))

    self.find_min_kernel = dfs_module.get_function("find_min_imp")
    self.find_min_kernel.prepare("PPPi")

    self.fill_kernel = dfs_module.get_function("fill_table")
    self.fill_kernel.prepare("PiiP")

    self.scan_reshuffle_tex = dfs_module.get_function("scan_reshuffle")
    self.scan_reshuffle_tex.prepare("PPii")
    tex_ref = dfs_module.get_texref("tex_mark")
    self.mark_table.bind_to_texref_ext(tex_ref)

    self.comput_total_2d = dfs_module.get_function("compute_2d")
    self.comput_total_2d.prepare("PPPPPPPii")

    self.reduce_2d = dfs_module.get_function("reduce_2d")
    self.reduce_2d.prepare("PPPPPi")

    self.scan_total_2d = dfs_module.get_function("scan_gini_large")
    self.scan_total_2d.prepare("PPPPii")

    self.scan_reduce = dfs_module.get_function("scan_reduce")
    self.scan_reduce.prepare("Pi")

    """ BFS module """
    bfs_module = compile_module("bfs_module.cu", (self.BFS_THREADS, n_labels, ctype_samples,
      ctype_labels, ctype_counts, ctype_indices,  self.debug))

    const_stride = bfs_module.get_global("stride")[0]
    const_n_features = bfs_module.get_global("n_features")[0]
    const_max_features = bfs_module.get_global("max_features")[0]
    driver.memcpy_htod(const_stride, np.uint32(self.stride))
    driver.memcpy_htod(const_n_features, np.uint16(self.n_features))
    driver.memcpy_htod(const_max_features, np.uint16(self.max_features))

    self.scan_total_bfs = bfs_module.get_function("scan_bfs")
    self.scan_total_bfs.prepare("PPPP")

    self.comput_bfs_2d = bfs_module.get_function("compute_2d")
    self.comput_bfs_2d.prepare("PPPPPPPPP")

    self.fill_bfs = bfs_module.get_function("fill_table")
    self.fill_bfs.prepare("PPPPP")

    self.reshuffle_bfs = bfs_module.get_function("scan_reshuffle")
    tex_ref = bfs_module.get_texref("tex_mark")
    self.mark_table.bind_to_texref_ext(tex_ref)
    self.reshuffle_bfs.prepare("PPP")

    self.reduce_bfs_2d = bfs_module.get_function("reduce")
    self.reduce_bfs_2d.prepare("PPPPPPi")

    self.get_thresholds = bfs_module.get_function("get_thresholds")
    self.get_thresholds.prepare("PPPPP")

    self.predict_kernel = mk_kernel(
        params = (ctype_indices, ctype_samples, ctype_labels),
        func_name = "predict",
        kernel_file = "predict.cu",
        prepare_args = "PPPPPPPii")

    self.bfs_module = bfs_module
    self.dfs_module = dfs_module
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Copyright 2010 Steven Dee. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from os import chdir, sep
from subprocess import call, PIPE

# Closure library directory relative to project root.
closuredir = sep.join(['..', 'closure-library', 'closure'])
# Project directory relative to the location of base.js (see below.)
projdir = sep.join(['..', '..', '..', 'jsprettify'])

# Per erik.arvidsson@gmail.com:
# > It is a bit tricky to get the paths correct but I found that running
# > the script from the same directory as base.js works for me.
# > Something like:
# >
# > ../bin/calcdeps.py -o deps -p ../../../test/ > ../../../test/deps.js
chdir(sep.join([closuredir, 'goog']))
calcdeps = sep.join(['..', 'bin', 'calcdeps.py'])
with open(sep.join([projdir, 'deps.js']), 'w') as depsjs:
    call([calcdeps, '-o', 'deps', '-p', projdir], stdout=depsjs, stderr=PIPE)
<commit_msg>Guess projdir based on script execution directory<commit_after>#!/usr/bin/env python

# Copyright 2010 Steven Dee. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from os import chdir, sep
from os.path import basename
from sys import path
from subprocess import call, PIPE

# Closure library directory relative to project root.
closuredir = sep.join(['..', 'closure-library', 'closure'])
# Project directory relative to the location of base.js (see below.)
projdir = sep.join(['..', '..', '..', basename(path[0])])

# Per erik.arvidsson@gmail.com:
# > It is a bit tricky to get the paths correct but I found that running
# > the script from the same directory as base.js works for me.
# > Something like:
# >
# > ../bin/calcdeps.py -o deps -p ../../../test/ > ../../../test/deps.js
chdir(sep.join([closuredir, 'goog']))
calcdeps = sep.join(['..', 'bin', 'calcdeps.py'])
with open(sep.join([projdir, 'deps.js']), 'w') as depsjs:
    call([calcdeps, '-o', 'deps', '-p', projdir], stdout=depsjs, stderr=PIPE)
<|endoftext|>


<commit_before>import sys

def traceBlame(offset, count):
	lines = []
	for i in range(count):
		try:
			caller   = sys._getframe(offset+(count-i-1))
			name	 = caller.f_code.co_name
			lineno   = caller.f_lineno
			filename = caller.f_code.co_filename

			lines.append("%s:%d in %s" % (filename, lineno, name))
		finally:
			del caller # Destroy a circular reference

	return tuple(lines)
<commit_msg>Fixed bug in util.debug.blame<commit_after>import sys

def traceBlame(offset, count):
	lines = []
	for i in range(count):
		try:
			caller   = sys._getframe(offset+(count-i-1))
			name	 = caller.f_code.co_name
			lineno   = caller.f_lineno
			filename = caller.f_code.co_filename
			del caller # Destroy a circular reference

			lines.append("%s:%d in %s" % (filename, lineno, name))
		except:
                        pass

	return tuple(lines)
<|endoftext|>


<commit_before>import cupy
from cupy import core

# TODO(okuta): Implement asfarray

def asfortranarray(a, dtype=None):
    """Return an array laid out in Fortran order in memory.

    Args:
        a (~cupy.ndarray): The input array.
        dtype (str or dtype object, optional): By default, the data-type is
            inferred from the input data.

    Returns:
        ~cupy.ndarray: The input `a` in Fortran, or column-major, order.

    .. seealso:: :func:`numpy.asfortranarray`

    """
    return core.asfortranarray(a, dtype)

# TODO(okuta): Implement asarray_chkfinite

def require(a, dtype=None, requirements=None):
    """Return an array which satisfies the requirements.

    Args:
        a (~cupy.ndarray): The input array.
        dtype (str or dtype object, optional): The required data-type.
            If None preserve the current dtype.
        requirements (str or list of str): The requirements can be any
            of the following

            * 'F_CONTIGUOUS' ('F', 'FORTRAN') - ensure a Fortran-contiguous array.

            * 'C_CONTIGUOUS' ('C', 'CONTIGUOUS') - ensure a C-contiguous array.

            * 'OWNDATA' ('O')      - ensure an array that owns its own data.

    Returns:
        ~cupy.ndarray: The input array ``a`` with specified requirements and type if provided.

    .. seealso:: :func:`numpy.require`

    """

    possible_flags = {'C': 'C', 'C_CONTIGUOUS': 'C', 'CONTIGUOUS': 'C',
                      'F': 'F', 'F_CONTIGUOUS': 'F', 'FORTRAN': 'F',
                      'O': 'OWNDATA', 'OWNDATA': 'OWNDATA'}

    if not requirements:
        return cupy.asanyarray(a, dtype=dtype)
    else:
        requirements = {possible_flags[x.upper()] for x in requirements}

    order = 'A'
    if requirements >= {'C', 'F'}:
        raise ValueError('Cannot specify both "C" and "F" order')
    elif 'F' in requirements:
        order = 'F_CONTIGUOUS'
        requirements.remove('F')
    elif 'C' in requirements:
        order = 'C_CONTIGUOUS'
        requirements.remove('C')

    arr = cupy.array(a, dtype=dtype, order=order, copy=False, subok=False)

    for prop in requirements:
        if not arr.flags[prop]:
            arr = arr.copy(order)
            break

    return arr<commit_msg>style fixes<commit_after>import cupy
from cupy import core

# TODO(okuta): Implement asfarray

def asfortranarray(a, dtype=None):
    """Return an array laid out in Fortran order in memory.

    Args:
        a (~cupy.ndarray): The input array.
        dtype (str or dtype object, optional): By default, the data-type is
            inferred from the input data.

    Returns:
        ~cupy.ndarray: The input `a` in Fortran, or column-major, order.

    .. seealso:: :func:`numpy.asfortranarray`

    """
    return core.asfortranarray(a, dtype)

# TODO(okuta): Implement asarray_chkfinite

def require(a, dtype=None, requirements=None):
    """Return an array which satisfies the requirements.

    Args:
        a (~cupy.ndarray): The input array.
        dtype (str or dtype object, optional): The required data-type.
            If None preserve the current dtype.
        requirements (str or list of str): The requirements can be any
            of the following

            * 'F_CONTIGUOUS' ('F', 'FORTRAN') - ensure a Fortran-contiguous \
                array. \

            * 'C_CONTIGUOUS' ('C', 'CONTIGUOUS') - ensure a C-contiguous array.

            * 'OWNDATA' ('O')      - ensure an array that owns its own data.

    Returns:
        ~cupy.ndarray: The input array ``a`` with specified requirements and
            type if provided.

    .. seealso:: :func:`numpy.require`

    """

    possible_flags = {'C': 'C', 'C_CONTIGUOUS': 'C', 'CONTIGUOUS': 'C',
                      'F': 'F', 'F_CONTIGUOUS': 'F', 'FORTRAN': 'F',
                      'O': 'OWNDATA', 'OWNDATA': 'OWNDATA'}

    if not requirements:
        return cupy.asanyarray(a, dtype=dtype)
    else:
        requirements = {possible_flags[x.upper()] for x in requirements}

    order = 'A'
    if requirements >= {'C', 'F'}:
        raise ValueError('Cannot specify both "C" and "F" order')
    elif 'F' in requirements:
        order = 'F_CONTIGUOUS'
        requirements.remove('F')
    elif 'C' in requirements:
        order = 'C_CONTIGUOUS'
        requirements.remove('C')

    arr = cupy.array(a, dtype=dtype, order=order, copy=False, subok=False)

    for prop in requirements:
        if not arr.flags[prop]:
            arr = arr.copy(order)
            break

    return arr
<|endoftext|>


<commit_before># seleniumbase package
__version__ = "2.4.26"
<commit_msg>Version 2.4.27<commit_after># seleniumbase package
__version__ = "2.4.27"
<|endoftext|>


<commit_before>#!/usr/bin/env python
from __future__ import unicode_literals

import os
import time
from datetime import datetime
from threading import Thread
from decimal import Decimal

import paho.mqtt.subscribe as subscribe
from django.core.management.base import BaseCommand

import orchid_app.sensors.anemometer as anemometer
import orchid_app.sensors.max44009 as light
import orchid_app.sensors.yf_201s as water
import orchid_app.sensors.mlx90614 as mlx
import orchid_app.sensors.bme280 as bme

import orchid_app.controller as controller
from orchid_app.models import Sensors

import warnings
warnings.filterwarnings('ignore')

POLL_PERIOD = 600  # seconds = 10 minutes
POLL_PERIOD_MIN = POLL_PERIOD / 60  # minutes
MAX_FLOW_RATE = 2.0  # L/minute.  This is threshold for emergency water leakage detection. If more than the threshold then close the valves.
MAX_LEAK_RATE = 0.008
MAX_SEND_COUNT = POLL_PERIOD / 10  # Send leakage message once in hour
send_counter = 0

def avg(l):
    '''Convert values of list from str to float if needed. Return average of the collected values.'''

    if not l:
        return 0.0

    pre = [float(i) for i in l]
    return round(sum(pre)/len(pre), 2)

class Command(BaseCommand):
    help = 'Polls sensors and writes data into the DB.'

    # def add_arguments(self, parser):
    #     parser.add_argument('poll_id', nargs='+', type=int)

    def handle(self, *args, **options):

        # Run measure and publishing of GPIO data in background.
        threads = [Thread(target=water.run),
                   Thread(target=anemometer.run),
                   ]
        for t in threads:
            t.setDaemon(True)
            t.start()

        os.system('logger orchid_runner has started')
        # Shut down on system start/restart everything could be open.
        controller.activate(reason='System startup', force=True, mist=False, drip=False, fan=False, light=False, heat=False)

        # Keep preliminary data for averaging
        data = {'wind': [], 'water': 0.0, 't_amb': [], 't_obj': [], 'hpa': [], 'rh': [], 'lux': []}
        ts = time.time()

        while True:
            if time.time() - ts < POLL_PERIOD:
                try:  # Catch sensor reading data, stay running
                    # Wait for MQTT data
                    topic = "shm/orchid/wind/last_min"
                    data['wind'].append(float(subscribe.simple(topic, keepalive=65, will={'topic': topic, 'payload': 0.0}).payload))
                    topic = "shm/orchid/water/last_min"
                    last_water = float(subscribe.simple(topic, keepalive=65, will={'topic': topic, 'payload': 0.0}).payload)
                    check_water_flow(last_water)
                    data['water'] += last_water
                    # Read i2c sensors
                    a, b, c = bme.readBME280All()
                    data['t_amb'].append(a)
                    data['hpa'].append(b)
                    data['rh'].append(c)
                    data['t_obj'].append(mlx.Melexis().readObject1())
                    data['lux'].append(light.readLight())
                except Exception as e:
                    self.stderr.write('On read: %s (%s)' % (e.message, type(e)))
                    time.sleep(60)  # Wait 1 minute before retry.
            else:
                n = datetime.now()
                s = Sensors()
                s.date = n.replace(minute=n.minute / POLL_PERIOD_MIN * POLL_PERIOD_MIN, second=0, microsecond=0)  # reduce to poll period resolution
                # Data conditioning by model/DB requirements
                s.t_amb = Decimal('{:.1f}'.format(avg(data['t_amb'])))
                s.t_obj = Decimal('{:.1f}'.format(avg(data['t_obj'])))
                s.water = Decimal('{:.1f}'.format(data['water']))
                s.wind = Decimal('{:.1f}'.format(avg(data['wind'])))
                s.hpa = Decimal('{:.1f}'.format(avg(data['hpa'])))
                s.rh = int(avg(data['rh']))
                s.lux = int(avg(data['lux']))
                self.stdout.write(str(s))
                try:  # Catch sensor reading data, stay running
                    # Write data to the DB
                    s.save()
                    self.stdout.write('Sensor Records: ' + repr(Sensors.objects.count()))
                except Exception as e:
                    self.stderr.write('On write: %s (%s)' % (e.message, type(e)))
                    time.sleep(60)  # Wait 1 minute before retry.
                # Reset the data structure
                data = {'wind': [], 'water': 0.0, 't_amb': [], 't_obj': [], 'hpa': [], 'rh': [], 'lux': []}
                ts = time.time()

                # Calculate current state
                controller.read_current_state()

            # Example of catch bad data
            # try:
            #     poll = Poll.objects.get(pk=poll_id)
            # except Poll.DoesNotExist:
            #     raise CommandError('Poll "%s" does not exist' % poll_id)
            # self.stdout.write(self.style.SUCCESS('Successfully closed poll "%s"' % poll_id))

def check_water_flow(liters):
    # Take emergency actions
    # Find out which valve is open
    la = controller.get_last_action()
    if la.mist or la.water:
        if liters > MAX_FLOW_RATE:
            # Try to shut open valve off
            controller.activate(reason='Emergency shut off', force=True, mist=False, drip=False,
                                fan=la.fan, light=la.light, heat=la.heat)

            # Build emergency message
            msg = 'Water leakage is detected in circuit(s): '
            msg += 'drip ' if la.water else ''
            msg += 'mist' if la.mist else ''
            msg += '\nOpened valve closed. This may impact watering and/or temperature conditions.\nTake actions immediately.'
            subj = 'Orchid farm emergency: water leakage detected'
            controller.send_message(subj, msg)

    # Check leakage when all valves closed
    elif liters > MAX_LEAK_RATE:
        global send_counter
        if send_counter == 0:
            # Try to shut open valve off
            controller.activate(reason='Emergency shut off', force=True, mist=False, drip=False,
                                fan=la.fan, light=la.light, heat=la.heat)

            # Build emergency message
            msg = 'Water leakage is detected while all valves should be closed.'
            msg += '\nTried to close all valves. This may impact watering and/or temperature conditions.\nTake actions immediately.'
            subj = 'Orchid farm emergency: water leakage detected'
            controller.send_message(subj, msg)
            send_counter += 1
        else:
            if send_counter < MAX_SEND_COUNT:
                send_counter += 1
            else:
                send_counter = 0

<commit_msg>Reduce sensitivity of water max flow rate detection.<commit_after>#!/usr/bin/env python
from __future__ import unicode_literals

import os
import time
from datetime import datetime
from threading import Thread
from decimal import Decimal

import paho.mqtt.subscribe as subscribe
from django.core.management.base import BaseCommand

import orchid_app.sensors.anemometer as anemometer
import orchid_app.sensors.max44009 as light
import orchid_app.sensors.yf_201s as water
import orchid_app.sensors.mlx90614 as mlx
import orchid_app.sensors.bme280 as bme

import orchid_app.controller as controller
from orchid_app.models import Sensors

import warnings
warnings.filterwarnings('ignore')

POLL_PERIOD = 600  # seconds = 10 minutes
POLL_PERIOD_MIN = POLL_PERIOD / 60  # minutes
MAX_FLOW_RATE = 2.5  # L/minute.  This is threshold for emergency water leakage detection. If more than the threshold then close the valves.
MAX_LEAK_RATE = 0.008
MAX_SEND_COUNT = POLL_PERIOD / 10  # Send leakage message once in hour
send_counter = 0

def avg(l):
    '''Convert values of list from str to float if needed. Return average of the collected values.'''

    if not l:
        return 0.0

    pre = [float(i) for i in l]
    return round(sum(pre)/len(pre), 2)

class Command(BaseCommand):
    help = 'Polls sensors and writes data into the DB.'

    # def add_arguments(self, parser):
    #     parser.add_argument('poll_id', nargs='+', type=int)

    def handle(self, *args, **options):

        # Run measure and publishing of GPIO data in background.
        threads = [Thread(target=water.run),
                   Thread(target=anemometer.run),
                   ]
        for t in threads:
            t.setDaemon(True)
            t.start()

        # TODO: Add here init for activators threads.

        os.system('logger orchid_runner has started')
        # Shut down on system start/restart everything could be open.
        controller.activate(reason='System startup', force=True, mist=False, drip=False, fan=False, light=False, heat=False)

        # Keep preliminary data for averaging
        data = {'wind': [], 'water': 0.0, 't_amb': [], 't_obj': [], 'hpa': [], 'rh': [], 'lux': []}
        ts = time.time()

        while True:
            if time.time() - ts < POLL_PERIOD:
                try:  # Catch sensor reading data, stay running
                    # Wait for MQTT data
                    topic = "shm/orchid/wind/last_min"
                    data['wind'].append(float(subscribe.simple(topic, keepalive=65, will={'topic': topic, 'payload': 0.0}).payload))
                    topic = "shm/orchid/water/last_min"
                    last_water = float(subscribe.simple(topic, keepalive=65, will={'topic': topic, 'payload': 0.0}).payload)
                    check_water_flow(last_water)
                    data['water'] += last_water
                    # Read i2c sensors
                    a, b, c = bme.readBME280All()
                    data['t_amb'].append(a)
                    data['hpa'].append(b)
                    data['rh'].append(c)
                    data['t_obj'].append(mlx.Melexis().readObject1())
                    data['lux'].append(light.readLight())
                except Exception as e:
                    self.stderr.write('On read: %s (%s)' % (e.message, type(e)))
                    time.sleep(60)  # Wait 1 minute before retry.
            else:
                n = datetime.now()
                s = Sensors()
                s.date = n.replace(minute=n.minute / POLL_PERIOD_MIN * POLL_PERIOD_MIN, second=0, microsecond=0)  # reduce to poll period resolution
                # Data conditioning by model/DB requirements
                s.t_amb = Decimal('{:.1f}'.format(avg(data['t_amb'])))
                s.t_obj = Decimal('{:.1f}'.format(avg(data['t_obj'])))
                s.water = Decimal('{:.1f}'.format(data['water']))
                s.wind = Decimal('{:.1f}'.format(avg(data['wind'])))
                s.hpa = Decimal('{:.1f}'.format(avg(data['hpa'])))
                s.rh = int(avg(data['rh']))
                s.lux = int(avg(data['lux']))
                self.stdout.write(str(s))
                try:  # Catch sensor reading data, stay running
                    # Write data to the DB
                    s.save()
                    self.stdout.write('Sensor Records: ' + repr(Sensors.objects.count()))
                except Exception as e:
                    self.stderr.write('On write: %s (%s)' % (e.message, type(e)))
                    time.sleep(60)  # Wait 1 minute before retry.
                # Reset the data structure
                data = {'wind': [], 'water': 0.0, 't_amb': [], 't_obj': [], 'hpa': [], 'rh': [], 'lux': []}
                ts = time.time()

                # Calculate current state
                controller.read_current_state()

            # Example of catch bad data
            # try:
            #     poll = Poll.objects.get(pk=poll_id)
            # except Poll.DoesNotExist:
            #     raise CommandError('Poll "%s" does not exist' % poll_id)
            # self.stdout.write(self.style.SUCCESS('Successfully closed poll "%s"' % poll_id))

def check_water_flow(liters):
    # Take emergency actions
    # Find out which valve is open
    la = controller.get_last_action()
    if la.mist or la.water:
        if liters > MAX_FLOW_RATE:
            # Try to shut open valve off
            controller.activate(reason='Emergency shut off', force=True, mist=False, drip=False,
                                fan=la.fan, light=la.light, heat=la.heat)

            # Build emergency message
            msg = 'Water leakage is detected in circuit(s): '
            msg += 'drip ' if la.water else ''
            msg += 'mist' if la.mist else ''
            msg += '\nOpened valve closed. This may impact watering and/or temperature conditions.\nTake actions immediately.'
            subj = 'Orchid farm emergency: water leakage detected'
            controller.send_message(subj, msg)

    # Check leakage when all valves closed
    elif liters > MAX_LEAK_RATE:
        global send_counter
        if send_counter == 0:
            # Try to shut open valve off
            controller.activate(reason='Emergency shut off', force=True, mist=False, drip=False,
                                fan=la.fan, light=la.light, heat=la.heat)

            # Build emergency message
            msg = 'Water leakage is detected while all valves should be closed.'
            msg += '\nTried to close all valves. This may impact watering and/or temperature conditions.\nTake actions immediately.'
            subj = 'Orchid farm emergency: water leakage detected'
            controller.send_message(subj, msg)
            send_counter += 1
        else:
            if send_counter < MAX_SEND_COUNT:
                send_counter += 1
            else:
                send_counter = 0

<|endoftext|>


<commit_before>'''
Kivy framework
==============

Kivy is an open source library for developing multi-touch applications. It is
completely cross-platform (Linux/OSX/Win) and released under the terms of the
GNU LGPL.

It comes with native support for many multi-touch input devices, a growing
library of multi-touch aware widgets, hardware accelerated OpenGL drawing,
and an architecture that is designed to let you focus on building custom and
highly interactive applications as quickly and easily as possible.

Thanks to Kivy's pure Python interface, you can take advantage of its highly
dynamic nature and use any of the thousands of high quality Python libraries
out there.
At the same time, performance-critical sections are internally implemented
on the C-level to maximize performance.

See http://kivy.org for more information.
'''

__all__ = (
    'require',
    'kivy_configure', 'kivy_register_post_configuration',
    'kivy_options', 'kivy_base_dir', 'kivy_libs_dir',
    'kivy_modules_dir', 'kivy_data_dir', 'kivy_shader_dir',
    'kivy_providers_dir', 'kivy_icons_dir', 'kivy_home_dir',
    'kivy_config_fn', 'kivy_usermodules_dir',
)

__version__ = '1.0.6'

import sys
import shutil
from getopt import getopt, GetoptError
from os import environ, mkdir
from os.path import dirname, join, basename, exists, expanduser
from kivy.logger import Logger, LOG_LEVELS

# internals for post-configuration
__kivy_post_configuration = []

if sys.platform == 'darwin' and sys.maxint < 9223372036854775807:
    r ='''Unsupported Python version detected!:
    On Mac OS X Kivy requires a 64 bit version of Python. We strongly advise you
    to use the version of Python that is provided by Apple (and neither ports,
    fink nor homebrew unless you know what you're doing).
    See http://kivy.org/docs/installation/installation-macosx.html for details.
    '''
    Logger.critical(r)

def require(version):
    '''Require can be used to check the minimum version required to run a Kivy
    application. For example, you can start your application like this::

        import kivy
        kivy.require('1.0.1')

    If a user attempts to run your application with a version of Kivy that is
    older than the version you specified, an Exception will be raised.

    The Kivy version is built like this::

        X.Y.Z[-tag[-tagrevision]]

        X is the major version
        Y is the minor version
        Z is the bugfixes revision

    The tag is optional, but may be one of 'dev', 'alpha', or 'beta'.
    The tagrevision is the revision of the tag.

    .. warning::

        You must not ask for a version with a tag, except -dev. Asking for a
        'dev' version will just warn the user if the current Kivy version is not
        a -dev, but it will never raise an exception.
        You must not ask for a version with a tagrevision.

    '''

    def parse_version(version):
        # check for tag
        tag = None
        tagrev = None
        if '-' in version:
            l = version.split('-')
            if len(l) == 2:
                version, tag = l
            elif len(l) == 3:
                version, tag, tagrev = l
            else:
                raise Exception('Revision format must be X.Y.Z[-tag]')

        # check x y z
        l = version.split('.')
        if len(l) != 3:
            raise Exception('Revision format must be X.Y.Z[-tag]')
        return [int(x) for x in l], tag, tagrev

    # user version
    revision, tag, tagrev = parse_version(version)
    # current version
    sysrevision, systag, systagrev = parse_version(__version__)

    # ensure that the required version don't contain tag, except dev
    if tag not in (None, 'dev'):
        raise Exception('Revision format must not have any tag except "dev"')
    if tag == 'dev' and systag != 'dev':
        Logger.warning('Application requested a -dev version of Kivy. '
                       '(You have %s, but the application requires %s)' % (
                            __version__, version))
    # not tag rev (-alpha-1, -beta-x) allowed.
    if tagrev is not None:
        raise Exception('Revision format must not contain any tagrevision')

    # finally, checking revision
    if sysrevision < revision:
        raise Exception('The version of Kivy installed on this system '
                        'is too old. '
                        '(You have %s, but the application requires %s)' % (
                            __version__, version))

def kivy_configure():
    '''Call post-configuration of Kivy.
    This function must be called if you create the window yourself.
    '''
    for callback in __kivy_post_configuration:
        callback()

def kivy_register_post_configuration(callback):
    '''Register a function to be called when kivy_configure() will be called.

    .. warning::
        Internal use only.
    '''
    __kivy_post_configuration.append(callback)

def kivy_usage():
    '''Kivy Usage: %s [OPTION...] ::

        -h, --help
            Prints this help message.
        -d, --debug
            Shows debug log
        -a, --auto-fullscreen
            Force run in 'auto' fullscreen (no resolution change) mode.
            Uses your display's resolution. This is most likely what you want.
        -c, --config section:key[:value]
            Set a custom [section] key=value in the configuration object
        -f, --fullscreen
            Force running in fullscreen mode.
        -k, --fake-fullscreen
            Force running in 'fake' fullscreen (no border) mode.
            Uses the resolution specified by width and height in your config.
        -w, --windowed
            Force running in a window.
        -p, --provider id:provider[,options]
            Add an input provider (eg: ccvtable1:tuio,192.168.0.1:3333).
        -m mod, --module=mod
            Activate a module (use "list" to get a list of available modules).
        -r, --rotation
            Rotate the window's contents (0, 90, 180, 270).
        -s, --save
            Save current Kivy configuration.
        --size=640x480
            Size of window geometry.
    '''
    print kivy_usage.__doc__ % (basename(sys.argv[0]))

# Start !
Logger.info('Kivy v%s' % (__version__))

#: Global settings options for kivy
kivy_options = {
    'use_accelerate': True,
    'shadow_window': True,
    'window': ('pygame', 'glut'),
    'text': ('pil', 'cairo', 'pygame'),
    'video': ('gstreamer', 'pyglet'),
    'audio': ('pygame', 'gstreamer', ),
    'image': ('pil', 'pygame'),
    'camera': ('opencv', 'gstreamer', 'videocapture'),
    'svg': ('squirtle', ),
    'spelling': ('enchant', 'osxappkit', ),
    'clipboard': ('pygame', 'dummy'), }

# Read environment
for option in kivy_options:
    key = 'KIVY_%s' % option.upper()
    if key in environ:
        try:
            if type(kivy_options[option]) in (list, tuple):
                kivy_options[option] = (str(environ[key]), )
            else:
                kivy_options[option] = environ[key].lower() in \
                    ('true', '1', 'yes', 'yup')
        except Exception:
            Logger.warning('Core: Wrong value for %s environment key' % key)
            Logger.exception('')

# Extract all needed path in kivy
#: Kivy directory
kivy_base_dir = dirname(sys.modules[__name__].__file__)
#: Kivy external libraries directory
kivy_libs_dir = join(kivy_base_dir, 'lib')
#: Kivy modules directory
kivy_modules_dir = join(kivy_base_dir, 'modules')
#: Kivy extension directory
kivy_exts_dir = join(kivy_base_dir, 'extensions')
#: Kivy data directory
kivy_data_dir = join(kivy_base_dir, 'data')
#: Kivy glsl shader directory
kivy_shader_dir = join(kivy_data_dir, 'glsl')
#: Kivy input provider directory
kivy_providers_dir = join(kivy_base_dir, 'input', 'providers')
#: Kivy icons config path (don't remove the last '')
kivy_icons_dir = join(kivy_data_dir, 'icons', '')
#: Kivy user-home storage directory
kivy_home_dir = None
#: Kivy configuration filename
kivy_config_fn = None
#: Kivy user modules directory
kivy_usermodules_dir = None

# Add libs to pythonpath
sys.path = [kivy_libs_dir] + sys.path

# Don't go further if we generate documentation
if basename(sys.argv[0]) in ('sphinx-build', 'autobuild.py'):
    environ['KIVY_DOC'] = '1'
if basename(sys.argv[0]) in ('sphinx-build', ):
    environ['KIVY_DOC_INCLUDE'] = '1'
if basename(sys.argv[0]) in ('nosetests', ) or 'nosetests' in sys.argv:
    environ['KIVY_UNITTEST'] = '1'
if not 'KIVY_DOC_INCLUDE' in environ:
    # Configuration management
    user_home_dir = expanduser('~')
    kivy_home_dir = join(user_home_dir, '.kivy')
    kivy_config_fn = join(kivy_home_dir, 'config.ini')

    if not exists(kivy_home_dir):
        mkdir(kivy_home_dir)

    kivy_usermodules_dir = join(kivy_home_dir, 'mods')
    if not exists(kivy_usermodules_dir):
        mkdir(kivy_usermodules_dir)

    kivy_userexts_dir = join(kivy_home_dir, 'extensions')
    if not exists(kivy_userexts_dir):
        mkdir(kivy_userexts_dir)

    icon_dir = join(kivy_home_dir, 'icon')
    if not exists(icon_dir):
        try:
            shutil.copytree(join(kivy_data_dir, 'logo'), icon_dir)
        except shutil.Error, e:
            Logger.exception('Error when copying logo directory')

    # configuration
    from kivy.config import Config

    # Set level of logger
    level = LOG_LEVELS.get(Config.get('kivy', 'log_level'))
    Logger.setLevel(level=level)

    # Can be overrided in command line
    if 'KIVY_UNITTEST' not in environ:

        # save sys argv, otherwize, gstreamer use it and display help..
        sys_argv = sys.argv
        sys.argv = sys.argv[:1]

        try:
            opts, args = getopt(sys_argv[1:], 'hp:fkawFem:snr:dc:',
                ['help', 'fullscreen', 'windowed', 'fps', 'event',
                 'module=', 'save', 'fake-fullscreen', 'auto-fullscreen',
                 'display=', 'size=', 'rotate=', 'config=', 'debug'])

        except GetoptError, err:
            Logger.error('Core: %s' % str(err))
            kivy_usage()
            sys.exit(2)

        # set argv to the non-read args
        sys.argv = sys_argv[0:1] + args
    else:
        opts = []
        args = []

    need_save = False
    for opt, arg in opts:
        if opt in ('-h', '--help'):
            kivy_usage()
            sys.exit(0)
        elif opt in ('-p', '--provider'):
            pid, args = arg.split(':', 1)
            Config.set('input', pid, args)
        elif opt in ('-a', '--auto-fullscreen'):
            Config.set('graphics', 'fullscreen', 'auto')
        elif opt in ('-c', '--config'):
            l = arg.split(':', 2)
            if len(l) == 2:
                Config.set(l[0], l[1], '')
            elif len(l) == 3:
                Config.set(l[0], l[1], l[2])
            else:
                raise Exception('Invalid --config value')
            if l[0] == 'kivy' and l[1] == 'log_level':
                level = LOG_LEVELS.get(Config.get('kivy', 'log_level'))
                Logger.setLevel(level=level)
        elif opt in ('-k', '--fake-fullscreen'):
            Config.set('graphics', 'fullscreen', 'fake')
        elif opt in ('-f', '--fullscreen'):
            Config.set('graphics', 'fullscreen', '1')
        elif opt in ('-w', '--windowed'):
            Config.set('graphics', 'fullscreen', '0')
        elif opt in ('--size', ):
            w, h = str(arg).split('x')
            Config.set('graphics', 'width', w)
            Config.set('graphics', 'height', h)
        elif opt in ('--display', ):
            Config.set('graphics', 'display', str(arg))
        elif opt in ('-m', '--module'):
            if str(arg) == 'list':
                from kivy.modules import Modules
                Modules.usage_list()
                sys.exit(0)
            args = arg.split(':', 1)
            if len(args) == 1:
                args += ['']
            Config.set('modules', args[0], args[1])
        elif opt in ('-s', '--save'):
            need_save = True
        elif opt in ('-r', '--rotation'):
            Config.set('graphics', 'rotation', arg)
        elif opt in ('-n', ):
            kivy_options['shadow_window'] = False
        elif opt in ('-d', '--debug'):
            level = LOG_LEVELS.get('debug')
            Logger.setLevel(level=level)

    if need_save:
        try:
            with open(kivy_config_fn, 'w') as fd:
                Config.write(fd)
        except Exception, e:
            Logger.exception('Core: error while saving default'
                             'configuration file:', str(e))
        Logger.info('Core: Kivy configuration saved.')
        sys.exit(0)

<commit_msg>bump to 1.0.7-dev<commit_after>'''
Kivy framework
==============

Kivy is an open source library for developing multi-touch applications. It is
completely cross-platform (Linux/OSX/Win) and released under the terms of the
GNU LGPL.

It comes with native support for many multi-touch input devices, a growing
library of multi-touch aware widgets, hardware accelerated OpenGL drawing,
and an architecture that is designed to let you focus on building custom and
highly interactive applications as quickly and easily as possible.

Thanks to Kivy's pure Python interface, you can take advantage of its highly
dynamic nature and use any of the thousands of high quality Python libraries
out there.
At the same time, performance-critical sections are internally implemented
on the C-level to maximize performance.

See http://kivy.org for more information.
'''

__all__ = (
    'require',
    'kivy_configure', 'kivy_register_post_configuration',
    'kivy_options', 'kivy_base_dir', 'kivy_libs_dir',
    'kivy_modules_dir', 'kivy_data_dir', 'kivy_shader_dir',
    'kivy_providers_dir', 'kivy_icons_dir', 'kivy_home_dir',
    'kivy_config_fn', 'kivy_usermodules_dir',
)

__version__ = '1.0.7-dev'

import sys
import shutil
from getopt import getopt, GetoptError
from os import environ, mkdir
from os.path import dirname, join, basename, exists, expanduser
from kivy.logger import Logger, LOG_LEVELS

# internals for post-configuration
__kivy_post_configuration = []

if sys.platform == 'darwin' and sys.maxint < 9223372036854775807:
    r ='''Unsupported Python version detected!:
    On Mac OS X Kivy requires a 64 bit version of Python. We strongly advise you
    to use the version of Python that is provided by Apple (and neither ports,
    fink nor homebrew unless you know what you're doing).
    See http://kivy.org/docs/installation/installation-macosx.html for details.
    '''
    Logger.critical(r)

def require(version):
    '''Require can be used to check the minimum version required to run a Kivy
    application. For example, you can start your application like this::

        import kivy
        kivy.require('1.0.1')

    If a user attempts to run your application with a version of Kivy that is
    older than the version you specified, an Exception will be raised.

    The Kivy version is built like this::

        X.Y.Z[-tag[-tagrevision]]

        X is the major version
        Y is the minor version
        Z is the bugfixes revision

    The tag is optional, but may be one of 'dev', 'alpha', or 'beta'.
    The tagrevision is the revision of the tag.

    .. warning::

        You must not ask for a version with a tag, except -dev. Asking for a
        'dev' version will just warn the user if the current Kivy version is not
        a -dev, but it will never raise an exception.
        You must not ask for a version with a tagrevision.

    '''

    def parse_version(version):
        # check for tag
        tag = None
        tagrev = None
        if '-' in version:
            l = version.split('-')
            if len(l) == 2:
                version, tag = l
            elif len(l) == 3:
                version, tag, tagrev = l
            else:
                raise Exception('Revision format must be X.Y.Z[-tag]')

        # check x y z
        l = version.split('.')
        if len(l) != 3:
            raise Exception('Revision format must be X.Y.Z[-tag]')
        return [int(x) for x in l], tag, tagrev

    # user version
    revision, tag, tagrev = parse_version(version)
    # current version
    sysrevision, systag, systagrev = parse_version(__version__)

    # ensure that the required version don't contain tag, except dev
    if tag not in (None, 'dev'):
        raise Exception('Revision format must not have any tag except "dev"')
    if tag == 'dev' and systag != 'dev':
        Logger.warning('Application requested a -dev version of Kivy. '
                       '(You have %s, but the application requires %s)' % (
                            __version__, version))
    # not tag rev (-alpha-1, -beta-x) allowed.
    if tagrev is not None:
        raise Exception('Revision format must not contain any tagrevision')

    # finally, checking revision
    if sysrevision < revision:
        raise Exception('The version of Kivy installed on this system '
                        'is too old. '
                        '(You have %s, but the application requires %s)' % (
                            __version__, version))

def kivy_configure():
    '''Call post-configuration of Kivy.
    This function must be called if you create the window yourself.
    '''
    for callback in __kivy_post_configuration:
        callback()

def kivy_register_post_configuration(callback):
    '''Register a function to be called when kivy_configure() will be called.

    .. warning::
        Internal use only.
    '''
    __kivy_post_configuration.append(callback)

def kivy_usage():
    '''Kivy Usage: %s [OPTION...] ::

        -h, --help
            Prints this help message.
        -d, --debug
            Shows debug log
        -a, --auto-fullscreen
            Force run in 'auto' fullscreen (no resolution change) mode.
            Uses your display's resolution. This is most likely what you want.
        -c, --config section:key[:value]
            Set a custom [section] key=value in the configuration object
        -f, --fullscreen
            Force running in fullscreen mode.
        -k, --fake-fullscreen
            Force running in 'fake' fullscreen (no border) mode.
            Uses the resolution specified by width and height in your config.
        -w, --windowed
            Force running in a window.
        -p, --provider id:provider[,options]
            Add an input provider (eg: ccvtable1:tuio,192.168.0.1:3333).
        -m mod, --module=mod
            Activate a module (use "list" to get a list of available modules).
        -r, --rotation
            Rotate the window's contents (0, 90, 180, 270).
        -s, --save
            Save current Kivy configuration.
        --size=640x480
            Size of window geometry.
    '''
    print kivy_usage.__doc__ % (basename(sys.argv[0]))

# Start !
Logger.info('Kivy v%s' % (__version__))

#: Global settings options for kivy
kivy_options = {
    'use_accelerate': True,
    'shadow_window': True,
    'window': ('pygame', 'glut'),
    'text': ('pil', 'cairo', 'pygame'),
    'video': ('gstreamer', 'pyglet'),
    'audio': ('pygame', 'gstreamer', ),
    'image': ('pil', 'pygame'),
    'camera': ('opencv', 'gstreamer', 'videocapture'),
    'svg': ('squirtle', ),
    'spelling': ('enchant', 'osxappkit', ),
    'clipboard': ('pygame', 'dummy'), }

# Read environment
for option in kivy_options:
    key = 'KIVY_%s' % option.upper()
    if key in environ:
        try:
            if type(kivy_options[option]) in (list, tuple):
                kivy_options[option] = (str(environ[key]), )
            else:
                kivy_options[option] = environ[key].lower() in \
                    ('true', '1', 'yes', 'yup')
        except Exception:
            Logger.warning('Core: Wrong value for %s environment key' % key)
            Logger.exception('')

# Extract all needed path in kivy
#: Kivy directory
kivy_base_dir = dirname(sys.modules[__name__].__file__)
#: Kivy external libraries directory
kivy_libs_dir = join(kivy_base_dir, 'lib')
#: Kivy modules directory
kivy_modules_dir = join(kivy_base_dir, 'modules')
#: Kivy extension directory
kivy_exts_dir = join(kivy_base_dir, 'extensions')
#: Kivy data directory
kivy_data_dir = join(kivy_base_dir, 'data')
#: Kivy glsl shader directory
kivy_shader_dir = join(kivy_data_dir, 'glsl')
#: Kivy input provider directory
kivy_providers_dir = join(kivy_base_dir, 'input', 'providers')
#: Kivy icons config path (don't remove the last '')
kivy_icons_dir = join(kivy_data_dir, 'icons', '')
#: Kivy user-home storage directory
kivy_home_dir = None
#: Kivy configuration filename
kivy_config_fn = None
#: Kivy user modules directory
kivy_usermodules_dir = None

# Add libs to pythonpath
sys.path = [kivy_libs_dir] + sys.path

# Don't go further if we generate documentation
if basename(sys.argv[0]) in ('sphinx-build', 'autobuild.py'):
    environ['KIVY_DOC'] = '1'
if basename(sys.argv[0]) in ('sphinx-build', ):
    environ['KIVY_DOC_INCLUDE'] = '1'
if basename(sys.argv[0]) in ('nosetests', ) or 'nosetests' in sys.argv:
    environ['KIVY_UNITTEST'] = '1'
if not 'KIVY_DOC_INCLUDE' in environ:
    # Configuration management
    user_home_dir = expanduser('~')
    kivy_home_dir = join(user_home_dir, '.kivy')
    kivy_config_fn = join(kivy_home_dir, 'config.ini')

    if not exists(kivy_home_dir):
        mkdir(kivy_home_dir)

    kivy_usermodules_dir = join(kivy_home_dir, 'mods')
    if not exists(kivy_usermodules_dir):
        mkdir(kivy_usermodules_dir)

    kivy_userexts_dir = join(kivy_home_dir, 'extensions')
    if not exists(kivy_userexts_dir):
        mkdir(kivy_userexts_dir)

    icon_dir = join(kivy_home_dir, 'icon')
    if not exists(icon_dir):
        try:
            shutil.copytree(join(kivy_data_dir, 'logo'), icon_dir)
        except shutil.Error, e:
            Logger.exception('Error when copying logo directory')

    # configuration
    from kivy.config import Config

    # Set level of logger
    level = LOG_LEVELS.get(Config.get('kivy', 'log_level'))
    Logger.setLevel(level=level)

    # Can be overrided in command line
    if 'KIVY_UNITTEST' not in environ:

        # save sys argv, otherwize, gstreamer use it and display help..
        sys_argv = sys.argv
        sys.argv = sys.argv[:1]

        try:
            opts, args = getopt(sys_argv[1:], 'hp:fkawFem:snr:dc:',
                ['help', 'fullscreen', 'windowed', 'fps', 'event',
                 'module=', 'save', 'fake-fullscreen', 'auto-fullscreen',
                 'display=', 'size=', 'rotate=', 'config=', 'debug'])

        except GetoptError, err:
            Logger.error('Core: %s' % str(err))
            kivy_usage()
            sys.exit(2)

        # set argv to the non-read args
        sys.argv = sys_argv[0:1] + args
    else:
        opts = []
        args = []

    need_save = False
    for opt, arg in opts:
        if opt in ('-h', '--help'):
            kivy_usage()
            sys.exit(0)
        elif opt in ('-p', '--provider'):
            pid, args = arg.split(':', 1)
            Config.set('input', pid, args)
        elif opt in ('-a', '--auto-fullscreen'):
            Config.set('graphics', 'fullscreen', 'auto')
        elif opt in ('-c', '--config'):
            l = arg.split(':', 2)
            if len(l) == 2:
                Config.set(l[0], l[1], '')
            elif len(l) == 3:
                Config.set(l[0], l[1], l[2])
            else:
                raise Exception('Invalid --config value')
            if l[0] == 'kivy' and l[1] == 'log_level':
                level = LOG_LEVELS.get(Config.get('kivy', 'log_level'))
                Logger.setLevel(level=level)
        elif opt in ('-k', '--fake-fullscreen'):
            Config.set('graphics', 'fullscreen', 'fake')
        elif opt in ('-f', '--fullscreen'):
            Config.set('graphics', 'fullscreen', '1')
        elif opt in ('-w', '--windowed'):
            Config.set('graphics', 'fullscreen', '0')
        elif opt in ('--size', ):
            w, h = str(arg).split('x')
            Config.set('graphics', 'width', w)
            Config.set('graphics', 'height', h)
        elif opt in ('--display', ):
            Config.set('graphics', 'display', str(arg))
        elif opt in ('-m', '--module'):
            if str(arg) == 'list':
                from kivy.modules import Modules
                Modules.usage_list()
                sys.exit(0)
            args = arg.split(':', 1)
            if len(args) == 1:
                args += ['']
            Config.set('modules', args[0], args[1])
        elif opt in ('-s', '--save'):
            need_save = True
        elif opt in ('-r', '--rotation'):
            Config.set('graphics', 'rotation', arg)
        elif opt in ('-n', ):
            kivy_options['shadow_window'] = False
        elif opt in ('-d', '--debug'):
            level = LOG_LEVELS.get('debug')
            Logger.setLevel(level=level)

    if need_save:
        try:
            with open(kivy_config_fn, 'w') as fd:
                Config.write(fd)
        except Exception, e:
            Logger.exception('Core: error while saving default'
                             'configuration file:', str(e))
        Logger.info('Core: Kivy configuration saved.')
        sys.exit(0)

<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# This script is used to compile ANTs binaries
# Inputs: ANTs folder, must be a sct fork of original ANTs folder.
# Outputs: binaries that are directly put into sct
# ants_scripts = ['ANTSLandmarksBSplineTransform',
# 				'antsApplyTransforms',
# 				'antsRegistration',
# 				'antsSliceRegularizedRegistration',
# 				'ANTSUseLandmarkImagesToGetAffineTransform',
# 				'ComposeMultiTransform']

import sct_utils as sct
import os
import getopt
import sys

#=======================================================================================================================
# usage
#=======================================================================================================================
def usage():
    print 'USAGE: \n' \
        ''+os.path.basename(__file__)+'\n' \
        '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n' \
        'Part of the Spinal Cord Toolbox <https://sourceforge.net/projects/spinalcordtoolbox>\n' \
        '\n'\
        'DESCRIPTION\n' \
        '  Compile ANTs binaries for the Spinal Cord Toolbox.\n' \
        '\n' \
        'USAGE\n' \
        '  '+os.path.basename(__file__)+'\n' \
        '\n' \
        'MANDATORY ARGUMENTS\n' \
        '  -a <path2ants>	 directory in which ANTs is. Must contains ANTs/ folder.\n' \
        '  -s <OS name>      name of the OS {osx,linux}.\n' \
        '  -f <script>		 name of scripts to compile. Separated by commas.\n' \
        '\n'\
        'EXAMPLE:\n' \
        '  compile_ants.py -s linux\n'
    sys.exit(2)

listOS = ['osx', 'linux']
os_target = ''

path_ants = ''
scripts_target = ''
ants_scripts = ['ANTSLandmarksBSplineTransform',
 				'antsApplyTransforms',
 				'antsRegistration',
 				'antsSliceRegularizedRegistration',
 				'ANTSUseLandmarkImagesToGetAffineTransform',
 				'ComposeMultiTransform']

# Check input param
try:
    opts, args = getopt.getopt(sys.argv[1:],'ha:s:f:')
except getopt.GetoptError as err:
    print str(err)
    usage()
for opt, arg in opts:
    if opt == '-h':
        usage()
    elif opt in ('-a'):
        path_ants = str(arg)
    elif opt in ('-s'):
        os_target = str(arg)
    elif opt in ('-f'):
        scripts_target = str(arg)

if os_target not in listOS:
    print 'ERROR: OS name should be one of the following: '+'[%s]' % ', '.join(map(str,listOS))
    usage()

if not path_ants:
	print 'ERROR: ANTs folder must be specified.'
	usage()
else:
	path_ants = sct.slash_at_the_end(path_ants) + '/'
	if not os.path.isdir(path_ants + 'ANTs/'):
		print 'ERROR: Path to ANTs must be a directory containing the folder ANTs/.'
		print 'Path specified: ' + path_ants
		usage()

# process list of scripts
scripts = ants_scripts
if scripts_target:
	scripts = []
	scripts_temp = scripts_target.split(',')
	for sc in scripts_temp:
		if sc in ants_scripts:
			scripts.append(sc)
		else:
			print 'WARNING: the ANTs binary ' + sc + ' is not part of SCT. It wasn\'t included into SCT.'

if not os.path.isdir(path_ants + 'antsbin/'):
	os.makedirs(path_ants + 'antsbin/')
	os.chdir(path_ants + 'antsbin/')
	status, output = sct.run('cmake ../ANTs', verbose=2)
else:
	os.chdir(path_ants + 'antsbin/')

status, output = sct.run('make -j 8', verbose=2)
status, path_sct = sct.run('echo $SCT_DIR')

for script in scripts:
	sct.run('cp ' + path_ants + 'antsbin/bin/' + script +' ' + path_sct + '/bin/' + os_target + '/isct_' + script, verbose=2)
<commit_msg>INST: compile ants<commit_after>#!/usr/bin/env python
#
# This script is used to compile ANTs binaries
# Inputs: ANTs folder, must be a sct fork of original ANTs folder.
# Outputs: binaries that are directly put into sct
# ants_scripts = ['ANTSLandmarksBSplineTransform',
# 				'antsApplyTransforms',
# 				'antsRegistration',
# 				'antsSliceRegularizedRegistration',
# 				'ANTSUseLandmarkImagesToGetAffineTransform',
# 				'ComposeMultiTransform']

import sct_utils as sct
import os
import getopt
import sys

def usage():
    print 'USAGE: \n' \
        ''+os.path.basename(__file__)+'\n' \
        '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n' \
        'Part of the Spinal Cord Toolbox <https://sourceforge.net/projects/spinalcordtoolbox>\n' \
        '\n'\
        'DESCRIPTION\n' \
        '  Compile ANTs binaries for the Spinal Cord Toolbox.\n' \
        '\n' \
        'USAGE\n' \
        '  '+os.path.basename(__file__)+'\n' \
        '\n' \
        'MANDATORY ARGUMENTS\n' \
        '  -a <path2ants>	 directory in which ANTs is. Must contains ANTs/ folder.\n' \
        '  -s <OS name>      name of the OS {osx,linux}.\n' \
        '  -f <script>		 name of scripts to compile. Separated by commas.\n' \
        '\n'\
        'EXAMPLE:\n' \
        '  compile_ants.py -s linux\n'
    sys.exit(2)

listOS = ['osx', 'linux']
os_target = ''

path_ants = ''
scripts_target = ''
ants_scripts = ['ANTSLandmarksBSplineTransform',
                'antsApplyTransforms',
                'antsRegistration',
                'antsSliceRegularizedRegistration',
                'ANTSUseLandmarkImagesToGetAffineTransform',
                'ComposeMultiTransform']

# Check input param
try:
    opts, args = getopt.getopt(sys.argv[1:], 'ha:s:f:')
except getopt.GetoptError as err:
    print str(err)
    usage()
for opt, arg in opts:
    if opt == '-h':
        usage()
    elif opt in '-a':
        path_ants = str(arg)
    elif opt in '-s':
        os_target = str(arg)
    elif opt in '-f':
        scripts_target = str(arg)

if os_target not in listOS:
    print 'ERROR: OS name should be one of the following: ' + '[%s]' % ', '.join(map(str, listOS))
    usage()

if not path_ants:
    print 'ERROR: ANTs folder must be specified.'
    usage()
else:
    path_ants = sct.slash_at_the_end(path_ants) + '/'
    if not os.path.isdir(path_ants + 'ANTs/'):
        print 'ERROR: Path to ANTs must be a directory containing the folder ANTs/.'
        print 'Path specified: ' + path_ants
        usage()

# process list of scripts
scripts = ants_scripts
if scripts_target:
    scripts = []
    scripts_temp = scripts_target.split(',')
    for sc in scripts_temp:
        if sc in ants_scripts:
            scripts.append(sc)
        else:
            print 'WARNING: the ANTs binary ' + sc + ' is not part of SCT. It wasn\'t included into SCT.'

if not scripts:
    print 'ERROR: No scripts to compile. Please check input.'
    usage()

if not os.path.isdir(path_ants + 'antsbin/'):
    os.makedirs(path_ants + 'antsbin/')
    os.chdir(path_ants + 'antsbin/')
    sct.run('cmake ../ANTs', verbose=2)
else:
    os.chdir(path_ants + 'antsbin/')

sct.run('make -j 8', verbose=2)
status, path_sct = sct.run('echo $SCT_DIR')

for script in scripts:
    sct.run('cp ' + path_ants + 'antsbin/bin/' + script + ' ' + path_sct + '/bin/' + os_target + '/isct_' + script,
            verbose=2)
<|endoftext|>