<commit_before>"""Meta-estimators for parallelizing scikit-learn."""
import logging
from timeit import default_timer as tic

import dask.array as da
import dask.dataframe as dd
import dask.delayed
import numpy as np
import sklearn.base
import sklearn.metrics
from sklearn.utils.validation import check_is_fitted

from ._partial import fit
from ._utils import copy_learned_attributes
from .metrics import get_scorer, check_scoring

logger = logging.getLogger(__name__)

class ParallelPostFit(sklearn.base.BaseEstimator):
    """Meta-estimator for parallel predict and transform.

    Parameters
    ----------
    estimator : Estimator
        The underlying estimator that is fit.

    scoring : string or callable, optional
        A single string (see :ref:`scoring_parameter`) or a callable
        (see :ref:`scoring`) to evaluate the predictions on the test set.

        For evaluating multiple metrics, either give a list of (unique)
        strings or a dict with names as keys and callables as values.

        NOTE that when using custom scorers, each scorer should return a
        single value. Metric functions returning a list/array of values
        can be wrapped into multiple scorers that return one value each.

        See :ref:`multimetric_grid_search` for an example.

        .. warning::

           If None, the estimator's default scorer (if available) is used.
           Most scikit-learn estimators will convert large Dask arrays to
           a single NumPy array, which may exhaust the memory of your worker.
           You probably want to always specify `scoring`.

    Notes
    -----

    .. warning::

       This class is not appropriate for parallel or distributed *training*
       on large datasets. For that, see :class:`Incremental`, which provides
       distributed (but sequential) training.

    This estimator does not parallelize the training step. This simply calls
    the underlying estimators's ``fit`` method called and copies over the
    learned attributes to ``self`` afterwards.

    It is helpful for situations where your training dataset is relatively
    small (fits on a single machine) but you need to predict or transform
    a much larger dataset. ``predict``, ``predict_proba`` and ``transform``
    will be done in parallel (potentially distributed if you've connected
    to a ``dask.distributed.Client``).

    Note that many scikit-learn estimators already predict and transform in
    parallel. This meta-estimator may still be useful in those cases when your
    dataset is larger than memory, as the distributed scheduler will ensure the
    data isn't all read into memory at once.

    See Also
    --------
    Incremental

    Examples
    --------
    >>> from sklearn.ensemble import GradientBoostingClassifier
    >>> import sklearn.datasets
    >>> import dask_ml.datasets

    Make a small 1,000 sample 2 training dataset and fit normally.

    >>> X, y = sklearn.datasets.make_classification(n_samples=1000,
    ...                                             random_state=0)
    >>> clf = ParallelPostFit(estimator=GradientBoostingClassifier(),
    ...                       scoring='accuracy')
    >>> clf.fit(X, y)
    ParallelPostFit(estimator=GradientBoostingClassifier(...))

    >>> clf.classes_
    array([0, 1])

    Transform and predict return dask outputs for dask inputs.

    >>> X_big, y_big = dask_ml.datasets.make_classification(n_samples=100000,
                                                            random_state=0)

    >>> clf.predict(X)
    dask.array<predict, shape=(10000,), dtype=int64, chunksize=(1000,)>

    Which can be computed in parallel.

    >>> clf.predict_proba(X).compute()
    array([[0.99141094, 0.00858906],
           [0.93178389, 0.06821611],
           [0.99129105, 0.00870895],
           ...,
           [0.97996652, 0.02003348],
           [0.98087444, 0.01912556],
           [0.99407016, 0.00592984]])
    """

    def __init__(self, estimator=None, scoring=None):
        self.estimator = estimator
        self.scoring = scoring

    @property
    def _postfit_estimator(self):
        # The estimator instance to use for postfit tasks like score
        return self.estimator

    def fit(self, X, y=None, **kwargs):
        """Fit the underlying estimator.

        Parameters
        ----------
        X, y : array-like
        **kwargs
            Additional fit-kwargs for the underlying estimator.

        Returns
        -------
        self : object
        """
        start = tic()
        logger.info("Starting fit")
        result = self.estimator.fit(X, y, **kwargs)
        stop = tic()
        logger.info("Finished fit, %0.2f", stop - start)

        # Copy over learned attributes
        copy_learned_attributes(result, self)
        copy_learned_attributes(result, self.estimator)
        return self

    def transform(self, X):
        """Transform block or partition-wise for dask inputs.

        For dask inputs, a dask array or dataframe is returned. For other
        inputs (NumPy array, pandas dataframe, scipy sparse matrix), the
        regular return value is returned.

        If the underlying estimator does not have a ``transform`` method, then
        an ``AttributeError`` is raised.

        Parameters
        ----------
        X : array-like

        Returns
        -------
        transformed : array-like
        """
        transform = self._check_method('transform')

        if isinstance(X, da.Array):
            return X.map_blocks(transform)
        elif isinstance(X, dd._Frame):
            return _apply_partitionwise(X, transform)
        else:
            return transform(X)

    def score(self, X, y):
        """Returns the score on the given data.

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]
            Input data, where n_samples is the number of samples and
            n_features is the number of features.

        y : array-like, shape = [n_samples] or [n_samples, n_output], optional
            Target relative to X for classification or regression;
            None for unsupervised learning.

        Returns
        -------
        score : float
                return self.estimator.score(X, y)
        """
        if self.scoring:
            if (not dask.is_dask_collection(X) and
                    not dask.is_dask_collection(y)):
                scorer = sklearn.metrics.get_scorer(self.scoring)
            else:
                scorer = get_scorer(self.scoring)
            return scorer(self, X, y)
        else:
            return self._postfit_estimator.score(X, y)

    def predict(self, X):
        """Predict for X.

        For dask inputs, a dask array or dataframe is returned. For other
        inputs (NumPy array, pandas dataframe, scipy sparse matrix), the
        regular return value is returned.

        Parameters
        ----------
        X : array-like

        Returns
        -------
        y : array-like
        """
        predict = self._check_method('predict')

        if isinstance(X, da.Array):
            return X.map_blocks(predict, dtype='int', drop_axis=1)

        elif isinstance(X, dd._Frame):
            return _apply_partitionwise(X, predict)

        else:
            return predict(X)

    def predict_proba(self, X):
        """Predict for X.

        For dask inputs, a dask array or dataframe is returned. For other
        inputs (NumPy array, pandas dataframe, scipy sparse matrix), the
        regular return value is returned.

        If the underlying estimator does not have a ``predict_proba``
        method, then an ``AttributeError`` is raised.

        Parameters
        ----------
        X : array or dataframe

        Returns
        -------
        y : array-like
        """

        predict_proba = self._check_method('predict_proba')

        if isinstance(X, da.Array):
            # XXX: multiclass
            return X.map_blocks(predict_proba,
                                dtype='float',
                                chunks=(X.chunks[0], len(self.classes_)))
        elif isinstance(X, dd._Frame):
            return _apply_partitionwise(X, predict_proba)
        else:
            return predict_proba(X)

    def _check_method(self, method):
        """Check if self.estimator has 'method'.

        Raises
        ------
        AttributeError
        """
        estimator = self._postfit_estimator
        if not hasattr(estimator, method):
            msg = ("The wrapped estimator '{}' does not have a "
                   "'{}' method.".format(estimator, method))
            raise AttributeError(msg)
        return getattr(estimator, method)

class Incremental(ParallelPostFit):
    """Metaestimator for feeding Dask Arrays to an estimator blockwise.

    This wrapper provides a bridge between Dask objects and estimators
    implementing the ``partial_fit`` API. These *incremental learners* can
    train on batches of data. This fits well with Dask's blocked data
    structures.

    See the `list of incremental learners`_ in the scikit-learn documentation
    for a list of estimators that implement the ``partial_fit`` API. Note that
    `Incremental` is not limited to just these classes, it will work on any
    estimator implementing ``partial_fit``, including those defined outside of
    scikit-learn itself.

    Calling :meth:`Incremental.fit` with a Dask Array will pass each block of
    the Dask array or arrays to ``estimator.partial_fit`` *sequentially*.

    Like :class:`ParallelPostFit`, the methods available after fitting (e.g.
    :meth:`Incremental.predict`, etc.) are all parallel and delayed.

    The ``estimator_`` attribute is a clone of `estimator` that was actually
    used during the call to ``fit``. All attributes learned during training
    are available on ``Incremental`` directly.

    .. _list of incremental learners: http://scikit-learn.org/stable/modules/scaling_strategies.html#incremental-learning  # noqa

    Parameters
    ----------
    estimator : Estimator
        Any object supporting the scikit-learn ``parital_fit`` API.

    scoring : string or callable, optional
        A single string (see :ref:`scoring_parameter`) or a callable
        (see :ref:`scoring`) to evaluate the predictions on the test set.

        For evaluating multiple metrics, either give a list of (unique)
        strings or a dict with names as keys and callables as values.

        NOTE that when using custom scorers, each scorer should return a
        single value. Metric functions returning a list/array of values
        can be wrapped into multiple scorers that return one value each.

        See :ref:`multimetric_grid_search` for an example.

        .. warning::

           If None, the estimator's default scorer (if available) is used.
           Most scikit-learn estimators will convert large Dask arrays to
           a single NumPy array, which may exhaust the memory of your worker.
           You probably want to always specify `scoring`.

    max_iter : int, default 1
        The maximum number of passes over the training data (aka epochs).
        It only impacts the behavior in the ``fit`` method, and not
        `partial_fit`.

    Attributes
    ----------
    estimator_ : Estimator
        A clone of `estimator` that was actually fit during the ``.fit`` call.

    See Also
    --------
    ParallelPostFit

    Examples
    --------
    >>> from dask_ml.wrappers import Incremental
    >>> from dask_ml.datasets import make_classification
    >>> import sklearn.linear_model
    >>> X, y = make_classification(chunks=25)
    >>> est = sklearn.linear_model.SGDClassifier()
    >>> clf = Incremental(est, scoring='accuracy')
    >>> clf.fit(X, y, classes=[0, 1])

    When used inside a grid search, prefix the underlying estimator's
    parameter names with ``estimator__``.

    >>> from sklearn.model_selection import GridSearchCV
    >>> param_grid = {"estimator__alpha": [0.1, 1.0, 10.0]}
    >>> gs = GridSearchCV(clf, param_grid)
    >>> gs.fit(X, y, classes=[0, 1])
    """

    def __init__(self, estimator=None, scoring=None, max_iter=1):
        self.max_iter = max_iter
        super().__init__(estimator=estimator, scoring=scoring)

    @property
    def _postfit_estimator(self):
        check_is_fitted(self, 'estimator_')
        return self.estimator_

    def _fit_for_estimator(self, estimator, X, y, **fit_kwargs):
        check_scoring(estimator, self.scoring)
        if not dask.is_dask_collection(X) and not dask.is_dask_collection(y):
            result = estimator.partial_fit(X=X, y=y, **fit_kwargs)
        else:
            result = fit(estimator, X, y, **fit_kwargs)

        copy_learned_attributes(result, self)
        self.estimator_ = estimator
        return self

    def fit(self, X, y=None, **fit_kwargs):
        estimator = sklearn.base.clone(self.estimator)

        for i in range(self.max_iter):
            logger.info("Starting iteration %d", i)
            start = tic()
            self._fit_for_estimator(estimator, X, y, **fit_kwargs)
            stop = tic()
            logger.info("Finished iteration %d, %0.2f", i, stop - start)

        return self

    def partial_fit(self, X, y=None, **fit_kwargs):
        """Fit the underlying estimator.

        If this estimator has not been previously fit, this is identical to
        :meth:`Incremental.fit`. If it has been previously fit,
        ``self.estimator_`` is used as the starting point.

        Parameters
        ----------
        X, y : array-like
        **kwargs
            Additional fit-kwargs for the underlying estimator.

        Returns
        -------
        self : object
        """
        estimator = getattr(self, 'estimator_', None)
        if estimator is None:
            estimator = sklearn.base.clone(self.estimator)
        return self._fit_for_estimator(estimator, X, y, **fit_kwargs)

def _first_block(dask_object):
    """Extract the first block / partition from a dask object
    """
    if isinstance(dask_object, da.Array):
        if dask_object.ndim > 1 and dask_object.numblocks[-1] != 1:
            raise NotImplementedError("IID estimators require that the array "
                                      "blocked only along the first axis. "
                                      "Rechunk your array before fitting.")
        shape = (dask_object.chunks[0][0],)
        if dask_object.ndim > 1:
            shape = shape + (dask_object.chunks[1][0],)

        return da.from_delayed(dask_object.to_delayed().flatten()[0],
                               shape,
                               dask_object.dtype)

    if isinstance(dask_object, dd._Frame):
        return dask_object.get_partition(0)

    else:
        return dask_object

def _apply_partitionwise(X, func):
    """Apply a prediction partition-wise to a dask.dataframe"""
    sample = func(X._meta_nonempty)
    if sample.ndim <= 1:
        p = ()
    else:
        p = (sample.shape[1],)

    if isinstance(sample, np.ndarray):
        blocks = X.to_delayed()
        arrays = [
            da.from_delayed(dask.delayed(func)(block),
                            shape=(np.nan,) + p,
                            dtype=sample.dtype)
            for block in blocks
        ]
        return da.concatenate(arrays)
    else:
        return X.map_partitions(func, meta=sample)
<commit_msg>Linting<commit_after>"""Meta-estimators for parallelizing scikit-learn."""
import logging
from timeit import default_timer as tic

import dask.array as da
import dask.dataframe as dd
import dask.delayed
import numpy as np
import sklearn.base
import sklearn.metrics
from sklearn.utils.validation import check_is_fitted

from ._partial import fit
from ._utils import copy_learned_attributes
from .metrics import get_scorer, check_scoring

logger = logging.getLogger(__name__)

class ParallelPostFit(sklearn.base.BaseEstimator):
    """Meta-estimator for parallel predict and transform.

    Parameters
    ----------
    estimator : Estimator
        The underlying estimator that is fit.

    scoring : string or callable, optional
        A single string (see :ref:`scoring_parameter`) or a callable
        (see :ref:`scoring`) to evaluate the predictions on the test set.

        For evaluating multiple metrics, either give a list of (unique)
        strings or a dict with names as keys and callables as values.

        NOTE that when using custom scorers, each scorer should return a
        single value. Metric functions returning a list/array of values
        can be wrapped into multiple scorers that return one value each.

        See :ref:`multimetric_grid_search` for an example.

        .. warning::

           If None, the estimator's default scorer (if available) is used.
           Most scikit-learn estimators will convert large Dask arrays to
           a single NumPy array, which may exhaust the memory of your worker.
           You probably want to always specify `scoring`.

    Notes
    -----

    .. warning::

       This class is not appropriate for parallel or distributed *training*
       on large datasets. For that, see :class:`Incremental`, which provides
       distributed (but sequential) training.

    This estimator does not parallelize the training step. This simply calls
    the underlying estimators's ``fit`` method called and copies over the
    learned attributes to ``self`` afterwards.

    It is helpful for situations where your training dataset is relatively
    small (fits on a single machine) but you need to predict or transform
    a much larger dataset. ``predict``, ``predict_proba`` and ``transform``
    will be done in parallel (potentially distributed if you've connected
    to a ``dask.distributed.Client``).

    Note that many scikit-learn estimators already predict and transform in
    parallel. This meta-estimator may still be useful in those cases when your
    dataset is larger than memory, as the distributed scheduler will ensure the
    data isn't all read into memory at once.

    See Also
    --------
    Incremental

    Examples
    --------
    >>> from sklearn.ensemble import GradientBoostingClassifier
    >>> import sklearn.datasets
    >>> import dask_ml.datasets

    Make a small 1,000 sample 2 training dataset and fit normally.

    >>> X, y = sklearn.datasets.make_classification(n_samples=1000,
    ...                                             random_state=0)
    >>> clf = ParallelPostFit(estimator=GradientBoostingClassifier(),
    ...                       scoring='accuracy')
    >>> clf.fit(X, y)
    ParallelPostFit(estimator=GradientBoostingClassifier(...))

    >>> clf.classes_
    array([0, 1])

    Transform and predict return dask outputs for dask inputs.

    >>> X_big, y_big = dask_ml.datasets.make_classification(n_samples=100000,
                                                            random_state=0)

    >>> clf.predict(X)
    dask.array<predict, shape=(10000,), dtype=int64, chunksize=(1000,)>

    Which can be computed in parallel.

    >>> clf.predict_proba(X).compute()
    array([[0.99141094, 0.00858906],
           [0.93178389, 0.06821611],
           [0.99129105, 0.00870895],
           ...,
           [0.97996652, 0.02003348],
           [0.98087444, 0.01912556],
           [0.99407016, 0.00592984]])
    """

    def __init__(self, estimator=None, scoring=None):
        self.estimator = estimator
        self.scoring = scoring

    @property
    def _postfit_estimator(self):
        # The estimator instance to use for postfit tasks like score
        return self.estimator

    def fit(self, X, y=None, **kwargs):
        """Fit the underlying estimator.

        Parameters
        ----------
        X, y : array-like
        **kwargs
            Additional fit-kwargs for the underlying estimator.

        Returns
        -------
        self : object
        """
        start = tic()
        logger.info("Starting fit")
        result = self.estimator.fit(X, y, **kwargs)
        stop = tic()
        logger.info("Finished fit, %0.2f", stop - start)

        # Copy over learned attributes
        copy_learned_attributes(result, self)
        copy_learned_attributes(result, self.estimator)
        return self

    def transform(self, X):
        """Transform block or partition-wise for dask inputs.

        For dask inputs, a dask array or dataframe is returned. For other
        inputs (NumPy array, pandas dataframe, scipy sparse matrix), the
        regular return value is returned.

        If the underlying estimator does not have a ``transform`` method, then
        an ``AttributeError`` is raised.

        Parameters
        ----------
        X : array-like

        Returns
        -------
        transformed : array-like
        """
        transform = self._check_method('transform')

        if isinstance(X, da.Array):
            return X.map_blocks(transform)
        elif isinstance(X, dd._Frame):
            return _apply_partitionwise(X, transform)
        else:
            return transform(X)

    def score(self, X, y):
        """Returns the score on the given data.

        Parameters
        ----------
        X : array-like, shape = [n_samples, n_features]
            Input data, where n_samples is the number of samples and
            n_features is the number of features.

        y : array-like, shape = [n_samples] or [n_samples, n_output], optional
            Target relative to X for classification or regression;
            None for unsupervised learning.

        Returns
        -------
        score : float
                return self.estimator.score(X, y)
        """
        if self.scoring:
            if (not dask.is_dask_collection(X) and
                    not dask.is_dask_collection(y)):
                scorer = sklearn.metrics.get_scorer(self.scoring)
            else:
                scorer = get_scorer(self.scoring)
            return scorer(self, X, y)
        else:
            return self._postfit_estimator.score(X, y)

    def predict(self, X):
        """Predict for X.

        For dask inputs, a dask array or dataframe is returned. For other
        inputs (NumPy array, pandas dataframe, scipy sparse matrix), the
        regular return value is returned.

        Parameters
        ----------
        X : array-like

        Returns
        -------
        y : array-like
        """
        predict = self._check_method('predict')

        if isinstance(X, da.Array):
            return X.map_blocks(predict, dtype='int', drop_axis=1)

        elif isinstance(X, dd._Frame):
            return _apply_partitionwise(X, predict)

        else:
            return predict(X)

    def predict_proba(self, X):
        """Predict for X.

        For dask inputs, a dask array or dataframe is returned. For other
        inputs (NumPy array, pandas dataframe, scipy sparse matrix), the
        regular return value is returned.

        If the underlying estimator does not have a ``predict_proba``
        method, then an ``AttributeError`` is raised.

        Parameters
        ----------
        X : array or dataframe

        Returns
        -------
        y : array-like
        """

        predict_proba = self._check_method('predict_proba')

        if isinstance(X, da.Array):
            # XXX: multiclass
            return X.map_blocks(predict_proba,
                                dtype='float',
                                chunks=(X.chunks[0], len(self.classes_)))
        elif isinstance(X, dd._Frame):
            return _apply_partitionwise(X, predict_proba)
        else:
            return predict_proba(X)

    def _check_method(self, method):
        """Check if self.estimator has 'method'.

        Raises
        ------
        AttributeError
        """
        estimator = self._postfit_estimator
        if not hasattr(estimator, method):
            msg = ("The wrapped estimator '{}' does not have a "
                   "'{}' method.".format(estimator, method))
            raise AttributeError(msg)
        return getattr(estimator, method)

class Incremental(ParallelPostFit):
    """Metaestimator for feeding Dask Arrays to an estimator blockwise.

    This wrapper provides a bridge between Dask objects and estimators
    implementing the ``partial_fit`` API. These *incremental learners* can
    train on batches of data. This fits well with Dask's blocked data
    structures.

    See the `list of incremental learners`_ in the scikit-learn documentation
    for a list of estimators that implement the ``partial_fit`` API. Note that
    `Incremental` is not limited to just these classes, it will work on any
    estimator implementing ``partial_fit``, including those defined outside of
    scikit-learn itself.

    Calling :meth:`Incremental.fit` with a Dask Array will pass each block of
    the Dask array or arrays to ``estimator.partial_fit`` *sequentially*.

    Like :class:`ParallelPostFit`, the methods available after fitting (e.g.
    :meth:`Incremental.predict`, etc.) are all parallel and delayed.

    The ``estimator_`` attribute is a clone of `estimator` that was actually
    used during the call to ``fit``. All attributes learned during training
    are available on ``Incremental`` directly.

    .. _list of incremental learners: http://scikit-learn.org/stable/modules/scaling_strategies.html#incremental-learning  # noqa

    Parameters
    ----------
    estimator : Estimator
        Any object supporting the scikit-learn ``parital_fit`` API.

    scoring : string or callable, optional
        A single string (see :ref:`scoring_parameter`) or a callable
        (see :ref:`scoring`) to evaluate the predictions on the test set.

        For evaluating multiple metrics, either give a list of (unique)
        strings or a dict with names as keys and callables as values.

        NOTE that when using custom scorers, each scorer should return a
        single value. Metric functions returning a list/array of values
        can be wrapped into multiple scorers that return one value each.

        See :ref:`multimetric_grid_search` for an example.

        .. warning::

           If None, the estimator's default scorer (if available) is used.
           Most scikit-learn estimators will convert large Dask arrays to
           a single NumPy array, which may exhaust the memory of your worker.
           You probably want to always specify `scoring`.

    max_iter : int, default 1
        The maximum number of passes over the training data (aka epochs).
        It only impacts the behavior in the ``fit`` method, and not
        `partial_fit`.

    Attributes
    ----------
    estimator_ : Estimator
        A clone of `estimator` that was actually fit during the ``.fit`` call.

    See Also
    --------
    ParallelPostFit

    Examples
    --------
    >>> from dask_ml.wrappers import Incremental
    >>> from dask_ml.datasets import make_classification
    >>> import sklearn.linear_model
    >>> X, y = make_classification(chunks=25)
    >>> est = sklearn.linear_model.SGDClassifier()
    >>> clf = Incremental(est, scoring='accuracy')
    >>> clf.fit(X, y, classes=[0, 1])

    When used inside a grid search, prefix the underlying estimator's
    parameter names with ``estimator__``.

    >>> from sklearn.model_selection import GridSearchCV
    >>> param_grid = {"estimator__alpha": [0.1, 1.0, 10.0]}
    >>> gs = GridSearchCV(clf, param_grid)
    >>> gs.fit(X, y, classes=[0, 1])
    """

    def __init__(self, estimator=None, scoring=None, max_iter=1):
        self.max_iter = max_iter
        super().__init__(estimator=estimator, scoring=scoring)

    @property
    def _postfit_estimator(self):
        check_is_fitted(self, 'estimator_')
        return self.estimator_

    def _fit_for_estimator(self, estimator, X, y, **fit_kwargs):
        check_scoring(estimator, self.scoring)
        if not dask.is_dask_collection(X) and not dask.is_dask_collection(y):
            result = estimator.partial_fit(X=X, y=y, **fit_kwargs)
        else:
            result = fit(estimator, X, y, **fit_kwargs)

        copy_learned_attributes(result, self)
        self.estimator_ = estimator
        return self

    def fit(self, X, y=None, **fit_kwargs):
        estimator = sklearn.base.clone(self.estimator)

        for i in range(self.max_iter):
            logger.info("Starting iteration %d", i)
            start = tic()
            self._fit_for_estimator(estimator, X, y, **fit_kwargs)
            stop = tic()
            logger.info("Finished iteration %d, %0.2f", i, stop - start)

        return self

    def partial_fit(self, X, y=None, **fit_kwargs):
        """Fit the underlying estimator.

        If this estimator has not been previously fit, this is identical to
        :meth:`Incremental.fit`. If it has been previously fit,
        ``self.estimator_`` is used as the starting point.

        Parameters
        ----------
        X, y : array-like
        **kwargs
            Additional fit-kwargs for the underlying estimator.

        Returns
        -------
        self : object
        """
        estimator = getattr(self, 'estimator_', None)
        if estimator is None:
            estimator = sklearn.base.clone(self.estimator)
        return self._fit_for_estimator(estimator, X, y, **fit_kwargs)

def _first_block(dask_object):
    """Extract the first block / partition from a dask object
    """
    if isinstance(dask_object, da.Array):
        if dask_object.ndim > 1 and dask_object.numblocks[-1] != 1:
            raise NotImplementedError("IID estimators require that the array "
                                      "blocked only along the first axis. "
                                      "Rechunk your array before fitting.")
        shape = (dask_object.chunks[0][0],)
        if dask_object.ndim > 1:
            shape = shape + (dask_object.chunks[1][0],)

        return da.from_delayed(dask_object.to_delayed().flatten()[0],
                               shape,
                               dask_object.dtype)

    if isinstance(dask_object, dd._Frame):
        return dask_object.get_partition(0)

    else:
        return dask_object

def _apply_partitionwise(X, func):
    """Apply a prediction partition-wise to a dask.dataframe"""
    sample = func(X._meta_nonempty)
    if sample.ndim <= 1:
        p = ()
    else:
        p = (sample.shape[1],)

    if isinstance(sample, np.ndarray):
        blocks = X.to_delayed()
        arrays = [
            da.from_delayed(dask.delayed(func)(block),
                            shape=(np.nan,) + p,
                            dtype=sample.dtype)
            for block in blocks
        ]
        return da.concatenate(arrays)
    else:
        return X.map_partitions(func, meta=sample)
<|endoftext|>


<commit_before># stdlib
import operator
from typing import Any
from typing import Dict as TypeDict
from typing import Final
from typing import Union

# third party
import numpy as np
import pytest
import torch

# syft absolute
import syft as sy
from syft.core.tensor.smpc.mpc_tensor import MPCTensor
from syft.core.tensor.smpc.share_tensor import ShareTensor

# TODO: The functionality for VMs was removed to focus on the Hagrid
vms = [sy.VirtualMachine(name=name) for name in ["alice", "bob", "theo", "andrew"]]
clients = [vm.get_client() for vm in vms]

PUBLIC_VALUES: Final[TypeDict[str, Union[np.ndarray, torch.Tensor, int]]] = {
    "numpy_array": np.array([32], dtype=np.int64),
    "torch_tensor": torch.tensor([42], dtype=torch.int64),
    "int": 42,
}

@pytest.mark.parametrize("public_value_type", ["int", "torch_tensor", "numpy_array"])
def test_remote_sharing(public_value_type: str) -> None:
    value = np.array([[1, 2, 3, 4, -5]], dtype=np.int64)
    remote_value = clients[0].syft.core.tensor.tensor.Tensor(value)

    mpc_tensor = MPCTensor(
        parties=clients, secret=remote_value, shape=(1, 5), seed_shares=52
    )

    assert len(mpc_tensor.child) == len(clients)

    shares = [share.get_copy() for share in mpc_tensor.child]
    assert all(isinstance(share, ShareTensor) for share in shares)
    assert (mpc_tensor.reconstruct() == value).all()

@pytest.mark.parametrize("op_str", ["add", "sub"])
def test_mpc_private_private_op(op_str: str) -> None:
    value_1 = np.array([[1, 2, 3, 4, -5]], dtype=np.int64)
    value_2 = np.array([42], dtype=np.int64)

    remote_value_1 = clients[0].syft.core.tensor.tensor.Tensor(value_1)
    remote_value_2 = clients[2].syft.core.tensor.tensor.Tensor(value_2)

    mpc_tensor_1 = MPCTensor(
        parties=clients, secret=remote_value_1, shape=(1, 5), seed_shares=52
    )

    mpc_tensor_2 = MPCTensor(
        parties=clients, secret=remote_value_2, shape=(1,), seed_shares=42
    )

    op = getattr(operator, op_str)

    res = op(mpc_tensor_1, mpc_tensor_2).reconstruct()
    expected = op(value_1, value_2)

    assert (res == expected).all()

@pytest.mark.parametrize("public_value_type", ["int", "torch_tensor", "numpy_array"])
@pytest.mark.parametrize("op_str", ["add", "sub", "mul"])
def test_mpc_private_public_op(op_str: str, public_value_type: str) -> None:
    value_1 = np.array([[1, 2, 3, 4, -5]], dtype=np.int64)
    value_2 = PUBLIC_VALUES[public_value_type]

    remote_value_1 = clients[0].syft.core.tensor.tensor.Tensor(value_1)

    mpc_tensor_1 = MPCTensor(
        parties=clients, secret=remote_value_1, shape=(1, 5), seed_shares=52
    )

    op = getattr(operator, op_str)

    res = op(mpc_tensor_1, value_2).reconstruct()

    # TODO: Conversion to numpy is required because numpy op torch_tensor
    # gives back
    # TypeError: Concatenation operation is not implemented for NumPy arrays, use np.concatenate() instead.
    expected = op(value_1, np.array(value_2))

    assert (res == expected).all()

@pytest.mark.parametrize("public_value_type", ["int", "torch_tensor", "numpy_array"])
@pytest.mark.parametrize("op_str", ["add", "sub", "mul"])
def test_mpc_public_private_op(op_str: str, public_value_type: str) -> None:
    value_1 = PUBLIC_VALUES[public_value_type]
    value_2 = np.array([[1, 2, 3, 4, -5]], dtype=np.int64)

    remote_value_2 = clients[0].syft.core.tensor.tensor.Tensor(value_2)

    mpc_tensor_2 = MPCTensor(
        parties=clients, secret=remote_value_2, shape=(1, 5), seed_shares=52
    )

    op = getattr(operator, op_str)

    # TODO: Conversion to numpy is required because numpy op torch_tensor
    # gives back
    # TypeError: Concatenation operation is not implemented for NumPy arrays, use np.concatenate() instead.
    res = op(value_1, mpc_tensor_2).reconstruct()
    expected = op(np.array(value_1), value_2)

    assert (res == np.array(expected)).all()

@pytest.mark.parametrize(
    "method_str, kwargs", [("sum", {"axis": 0}), ("sum", {"axis": 1})]
)
def test_mpc_forward_methods(method_str: str, kwargs: TypeDict[str, Any]) -> None:
    value = np.array([[1, 2, 3, 4, -5], [5, 6, 7, 8, 9]], dtype=np.int64)

    remote_value = clients[0].syft.core.tensor.tensor.Tensor(value)

    mpc_tensor = MPCTensor(
        parties=clients, secret=remote_value, shape=(2, 5), seed_shares=52
    )

    op_mpc = getattr(mpc_tensor, method_str)
    op = getattr(value, method_str)

    res = op_mpc(**kwargs).reconstruct()
    expected = op(**kwargs)

    assert (res == expected).all()
<commit_msg>Add conversion to numpy array<commit_after># stdlib
import operator
from typing import Any
from typing import Dict as TypeDict
from typing import Final
from typing import Union

# third party
import numpy as np
import pytest
import torch

# syft absolute
import syft as sy
from syft.core.tensor.smpc.mpc_tensor import MPCTensor
from syft.core.tensor.smpc.share_tensor import ShareTensor

# TODO: The functionality for VMs was removed to focus on the Hagrid
vms = [sy.VirtualMachine(name=name) for name in ["alice", "bob", "theo", "andrew"]]
clients = [vm.get_client() for vm in vms]

PUBLIC_VALUES: Final[TypeDict[str, Union[np.ndarray, torch.Tensor, int]]] = {
    "numpy_array": np.array([32], dtype=np.int64),
    "torch_tensor": torch.tensor([42], dtype=torch.int64),
    "int": 42,
}

@pytest.mark.parametrize("public_value_type", ["int", "torch_tensor", "numpy_array"])
def test_remote_sharing(public_value_type: str) -> None:
    value = np.array([[1, 2, 3, 4, -5]], dtype=np.int64)
    remote_value = clients[0].syft.core.tensor.tensor.Tensor(value)

    mpc_tensor = MPCTensor(
        parties=clients, secret=remote_value, shape=(1, 5), seed_shares=52
    )

    assert len(mpc_tensor.child) == len(clients)

    shares = [share.get_copy() for share in mpc_tensor.child]
    assert all(isinstance(share, ShareTensor) for share in shares)
    assert (mpc_tensor.reconstruct() == value).all()

@pytest.mark.parametrize("op_str", ["add", "sub"])
def test_mpc_private_private_op(op_str: str) -> None:
    value_1 = np.array([[1, 2, 3, 4, -5]], dtype=np.int64)
    value_2 = np.array([42], dtype=np.int64)

    remote_value_1 = clients[0].syft.core.tensor.tensor.Tensor(value_1)
    remote_value_2 = clients[2].syft.core.tensor.tensor.Tensor(value_2)

    mpc_tensor_1 = MPCTensor(
        parties=clients, secret=remote_value_1, shape=(1, 5), seed_shares=52
    )

    mpc_tensor_2 = MPCTensor(
        parties=clients, secret=remote_value_2, shape=(1,), seed_shares=42
    )

    op = getattr(operator, op_str)

    res = op(mpc_tensor_1, mpc_tensor_2).reconstruct()
    expected = op(value_1, value_2)

    assert (res == expected).all()

@pytest.mark.parametrize("public_value_type", ["int", "torch_tensor", "numpy_array"])
@pytest.mark.parametrize("op_str", ["add", "sub", "mul"])
def test_mpc_private_public_op(op_str: str, public_value_type: str) -> None:
    value_1 = np.array([[1, 2, 3, 4, -5]], dtype=np.int64)
    value_2 = PUBLIC_VALUES[public_value_type]

    remote_value_1 = clients[0].syft.core.tensor.tensor.Tensor(value_1)

    mpc_tensor_1 = MPCTensor(
        parties=clients, secret=remote_value_1, shape=(1, 5), seed_shares=52
    )

    op = getattr(operator, op_str)

    res = op(mpc_tensor_1, value_2).reconstruct()

    # TODO: Conversion to numpy is required because numpy op torch_tensor
    # gives back
    # TypeError: Concatenation operation is not implemented for NumPy arrays, use np.concatenate() instead.
    expected = op(value_1, np.array(value_2))

    assert (res == expected).all()

@pytest.mark.parametrize("public_value_type", ["int", "torch_tensor", "numpy_array"])
@pytest.mark.parametrize("op_str", ["add", "sub", "mul"])
def test_mpc_public_private_op(op_str: str, public_value_type: str) -> None:
    value_1 = PUBLIC_VALUES[public_value_type]
    value_2 = np.array([[1, 2, 3, 4, -5]], dtype=np.int64)

    remote_value_2 = clients[0].syft.core.tensor.tensor.Tensor(value_2)

    mpc_tensor_2 = MPCTensor(
        parties=clients, secret=remote_value_2, shape=(1, 5), seed_shares=52
    )

    op = getattr(operator, op_str)

    # TODO: Conversion to numpy is required because numpy op torch_tensor
    # gives back
    # TypeError: Concatenation operation is not implemented for NumPy arrays, use np.concatenate() instead.
    res = op(np.array(value_1), mpc_tensor_2).reconstruct()
    expected = op(np.array(value_1), value_2)

    assert (res == np.array(expected)).all()

@pytest.mark.parametrize(
    "method_str, kwargs", [("sum", {"axis": 0}), ("sum", {"axis": 1})]
)
def test_mpc_forward_methods(method_str: str, kwargs: TypeDict[str, Any]) -> None:
    value = np.array([[1, 2, 3, 4, -5], [5, 6, 7, 8, 9]], dtype=np.int64)

    remote_value = clients[0].syft.core.tensor.tensor.Tensor(value)

    mpc_tensor = MPCTensor(
        parties=clients, secret=remote_value, shape=(2, 5), seed_shares=52
    )

    op_mpc = getattr(mpc_tensor, method_str)
    op = getattr(value, method_str)

    res = op_mpc(**kwargs).reconstruct()
    expected = op(**kwargs)

    assert (res == expected).all()
<|endoftext|>


<commit_before>#
# ovirt-engine-setup -- ovirt engine setup
# Copyright (C) 2013 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""Misc plugin."""

import os
import logging
import gettext
_ = lambda m: gettext.dgettext(message=m, domain='ovirt-engine-setup')

from otopi import constants as otopicons
from otopi import util
from otopi import plugin

from ovirt_engine_setup import constants as osetupcons
from ovirt_engine_setup import dialog

@util.export
class Plugin(plugin.PluginBase):
    """Misc plugin."""

    def __init__(self, context):
        super(Plugin, self).__init__(context=context)

    @plugin.event(
        stage=plugin.Stages.STAGE_BOOT,
        before=(
            otopicons.Stages.CORE_CONFIG_INIT,
        ),
    )
    def _preinit(self):
        self.environment.setdefault(
            otopicons.CoreEnv.CONFIG_FILE_NAME,
            self.resolveFile(
                os.environ.get(
                    otopicons.SystemEnvironment.CONFIG,
                    self.resolveFile(
                        osetupcons.FileLocations.OVIRT_OVIRT_SETUP_CONFIG_FILE
                    )
                )
            )
        )
        logging.getLogger('ovirt').setLevel(logging.DEBUG)

    @plugin.event(
        stage=plugin.Stages.STAGE_INIT,
        priority=plugin.Stages.PRIORITY_HIGH,
    )
    def _init(self):

        self.environment.setdefault(
            osetupcons.CoreEnv.GENERATED_BY_VERSION,
            None
        )

        self.environment[
            osetupcons.CoreEnv.ORIGINAL_GENERATED_BY_VERSION
        ] = self.environment[
            osetupcons.CoreEnv.GENERATED_BY_VERSION
        ]

        self.environment[
            osetupcons.CoreEnv.GENERATED_BY_VERSION
        ] = osetupcons.Const.PACKAGE_VERSION

        self.environment.setdefault(
            osetupcons.CoreEnv.DEVELOPER_MODE,
            None
        )
        self.environment.setdefault(
            osetupcons.CoreEnv.UPGRADE_SUPPORTED_VERSIONS,
            '3.0,3.1,3.2,3.3,3.4'
        )

        self.logger.debug(
            'Package: %s-%s (%s)',
            osetupcons.Const.PACKAGE_NAME,
            osetupcons.Const.PACKAGE_VERSION,
            osetupcons.Const.DISPLAY_VERSION,
        )

        self.environment[
            osetupcons.CoreEnv.SETUP_ATTRS_MODULES
        ] = [osetupcons]

        if self.environment[osetupcons.CoreEnv.DEVELOPER_MODE] is None:
            self.environment[osetupcons.CoreEnv.DEVELOPER_MODE] = False
            if os.geteuid() != 0:
                if not dialog.queryBoolean(
                    dialog=self.dialog,
                    name='OVESETUP_SYSTEM_UNPRIVILEGED',
                    note=_(
                        'Setup was run under unprivileged user '
                        'this will produce development installation '
                        'do you wish to proceed? (@VALUES@) [@DEFAULT@]: '
                    ),
                    prompt=True,
                    default=False,
                ):
                    raise RuntimeError(_('Aborted by user'))
                self.environment[osetupcons.CoreEnv.DEVELOPER_MODE] = True

        if (
            not self.environment[osetupcons.CoreEnv.DEVELOPER_MODE] and
            os.geteuid() != 0
        ):
            raise RuntimeError(
                _('Running as non root and not in development mode')
            )

    @plugin.event(
        stage=plugin.Stages.STAGE_CLEANUP,
    )
    def _cleanup(self):
        self.dialog.note(
            text=_('Log file is located at {path}').format(
                path=self.environment[
                    otopicons.CoreEnv.LOG_FILE_NAME
                ],
            ),
        )

    @plugin.event(
        stage=plugin.Stages.STAGE_TERMINATE,
    )
    def _terminate(self):
        if self.environment[otopicons.BaseEnv.ERROR]:
            self.logger.error(
                _('Execution of {action} failed').format(
                    action=self.environment[
                        osetupcons.CoreEnv.ACTION
                    ],
                ),
            )
        else:
            self.logger.info(
                _('Execution of {action} completed successfully').format(
                    action=self.environment[
                        osetupcons.CoreEnv.ACTION
                    ],
                ),
            )

# vim: expandtab tabstop=4 shiftwidth=4
<commit_msg>packaging: setup: Allow upgrading from 3.5<commit_after>#
# ovirt-engine-setup -- ovirt engine setup
# Copyright (C) 2013 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""Misc plugin."""

import os
import logging
import gettext
_ = lambda m: gettext.dgettext(message=m, domain='ovirt-engine-setup')

from otopi import constants as otopicons
from otopi import util
from otopi import plugin

from ovirt_engine_setup import constants as osetupcons
from ovirt_engine_setup import dialog

@util.export
class Plugin(plugin.PluginBase):
    """Misc plugin."""

    def __init__(self, context):
        super(Plugin, self).__init__(context=context)

    @plugin.event(
        stage=plugin.Stages.STAGE_BOOT,
        before=(
            otopicons.Stages.CORE_CONFIG_INIT,
        ),
    )
    def _preinit(self):
        self.environment.setdefault(
            otopicons.CoreEnv.CONFIG_FILE_NAME,
            self.resolveFile(
                os.environ.get(
                    otopicons.SystemEnvironment.CONFIG,
                    self.resolveFile(
                        osetupcons.FileLocations.OVIRT_OVIRT_SETUP_CONFIG_FILE
                    )
                )
            )
        )
        logging.getLogger('ovirt').setLevel(logging.DEBUG)

    @plugin.event(
        stage=plugin.Stages.STAGE_INIT,
        priority=plugin.Stages.PRIORITY_HIGH,
    )
    def _init(self):

        self.environment.setdefault(
            osetupcons.CoreEnv.GENERATED_BY_VERSION,
            None
        )

        self.environment[
            osetupcons.CoreEnv.ORIGINAL_GENERATED_BY_VERSION
        ] = self.environment[
            osetupcons.CoreEnv.GENERATED_BY_VERSION
        ]

        self.environment[
            osetupcons.CoreEnv.GENERATED_BY_VERSION
        ] = osetupcons.Const.PACKAGE_VERSION

        self.environment.setdefault(
            osetupcons.CoreEnv.DEVELOPER_MODE,
            None
        )
        self.environment.setdefault(
            osetupcons.CoreEnv.UPGRADE_SUPPORTED_VERSIONS,
            '3.0,3.1,3.2,3.3,3.4,3.5'
        )

        self.logger.debug(
            'Package: %s-%s (%s)',
            osetupcons.Const.PACKAGE_NAME,
            osetupcons.Const.PACKAGE_VERSION,
            osetupcons.Const.DISPLAY_VERSION,
        )

        self.environment[
            osetupcons.CoreEnv.SETUP_ATTRS_MODULES
        ] = [osetupcons]

        if self.environment[osetupcons.CoreEnv.DEVELOPER_MODE] is None:
            self.environment[osetupcons.CoreEnv.DEVELOPER_MODE] = False
            if os.geteuid() != 0:
                if not dialog.queryBoolean(
                    dialog=self.dialog,
                    name='OVESETUP_SYSTEM_UNPRIVILEGED',
                    note=_(
                        'Setup was run under unprivileged user '
                        'this will produce development installation '
                        'do you wish to proceed? (@VALUES@) [@DEFAULT@]: '
                    ),
                    prompt=True,
                    default=False,
                ):
                    raise RuntimeError(_('Aborted by user'))
                self.environment[osetupcons.CoreEnv.DEVELOPER_MODE] = True

        if (
            not self.environment[osetupcons.CoreEnv.DEVELOPER_MODE] and
            os.geteuid() != 0
        ):
            raise RuntimeError(
                _('Running as non root and not in development mode')
            )

    @plugin.event(
        stage=plugin.Stages.STAGE_CLEANUP,
    )
    def _cleanup(self):
        self.dialog.note(
            text=_('Log file is located at {path}').format(
                path=self.environment[
                    otopicons.CoreEnv.LOG_FILE_NAME
                ],
            ),
        )

    @plugin.event(
        stage=plugin.Stages.STAGE_TERMINATE,
    )
    def _terminate(self):
        if self.environment[otopicons.BaseEnv.ERROR]:
            self.logger.error(
                _('Execution of {action} failed').format(
                    action=self.environment[
                        osetupcons.CoreEnv.ACTION
                    ],
                ),
            )
        else:
            self.logger.info(
                _('Execution of {action} completed successfully').format(
                    action=self.environment[
                        osetupcons.CoreEnv.ACTION
                    ],
                ),
            )

# vim: expandtab tabstop=4 shiftwidth=4
<|endoftext|>


<commit_before>from pprint import pprint

def rows_to_dict_list(cursor):
    columns = [i[0] for i in cursor.description]
    return [dict(zip(columns, row)) for row in cursor]

def rows_to_dict_list_lower(cursor):
    columns = [i[0].lower() for i in cursor.description]
    return [dict(zip(columns, row)) for row in cursor]

def queryset_to_dict_list_lower(qs):
    result = []
    for obj in qs:
        result.append({
            name.lower(): obj.__dict__[name] for name in obj.__dict__
            if name[0] != '_'
        })
    return result

def dict_list_to_lower(data):
    data_lower = []
    for row in data:
        row_lower = {}
        for key in row.keys():
            row_lower[key.lower()] = row[key]
        data_lower.append(row_lower)
    return data_lower

def dict_def_options(dictionary, default, *args):
    """
    Return dictionary[arg] for first arg in args that exists in dictionary.
    Otherwise, return default value.
    """
    for arg in args:
        if arg in dictionary:
            return dictionary[arg]
    return default

def dict_options(dictionary, *args):
    """
    Call dict_def_options with default value None
    """
    return dict_def_options(dictionary, None, *args)

class DataSql(object):
    """docstring for DataDim."""
    def __init__(self, cursor, args=[], sql=''):
        self._cursor = cursor
        self.args = args
        if sql:
            self.sql = sql

    def execute(self, sql):
        self._cursor.execute(sql, self.args)
        return rows_to_dict_list(self._cursor)

    @property
    def sql(self): pass

    @sql.setter
    def sql(self, sql):
        self.data = self.execute(sql)

class ExecSql(object):
    """
    Initialize with cursor, args=[] (or *arguments) and parameters:
    - receive data when run DataSql.execute n times, passing only the sql
    Or pass sql also in inicialization:
    - receive imediatly data
    Parameters can be:
    - result_case:
      - '' or upper: process cursor with rows_to_dict_list
      - lower: process cursor with rows_to_dict_list_lower
      - cursor; not process the cursor
    """
    def __init__(self, cursor, *arguments, args=None, sql='', **kwargs):
        self._cursor = cursor
        if len(arguments) == 0:
            self.args = []
        else:
            self.args = arguments
        if args is not None:
            self.args.append(args)
        self.result_case = 'upper'
        if 'result_case' in {k.lower() for k in kwargs}:
            self.result_case = {
                k.lower(): v for k, v in kwargs.items()}['result_case']
        if sql:
            self.sql = sql

    def execute(self, sql=''):
        if sql:
            self.sql = sql
        self._cursor.execute(self.sql, self.args)
        if self.result_case == 'lower':
            return rows_to_dict_list_lower(self._cursor)
        elif self.result_case == 'cursor':
            return self._cursor
        else:
            return rows_to_dict_list(self._cursor)

    @property
    def sql(self): pass

    @sql.setter
    def sql(self, sql):
        self.data = self.execute(sql)

class GradeQtd(object):
    """docstring for GradeQtd."""

    class DataDim(DataSql):
        """docstring for DataDim."""
        def __init__(self, cursor, args, sql, **kwargs):
            self.id_field = \
                dict_def_options(kwargs, '', 'id', 'id_field')
            self.facade_field = \
                dict_def_options(kwargs, '', 'facade', 'facade_field',
                                             'id', 'id_field')
            self.name = kwargs.get('name', '')
            self.name_plural = kwargs.get('name_plural', self.name+'s')
            self.total = kwargs.get('total', '')
            self.forca_total = kwargs.get('forca_total', False)
            super(GradeQtd.DataDim, self).__init__(cursor, args, sql)

        def execute(self, sql):
            data_dim = super(GradeQtd.DataDim, self).execute(sql)
            self.niveis = len(data_dim)
            if self.total == '' or \
                    (self.niveis == 1 and not self.forca_total):
                self.idx_total = -1
            else:
                self.idx_total = len(data_dim)
                column = {self.id_field: self.total}
                if self.facade_field != self.id_field:
                    column[self.facade_field] = self.total
                data_dim.append(column)
            return data_dim

    def __init__(self, cursor, args=[]):
        self._cursor = cursor
        self.args = args
        self.total = 0
        self._table_data = None

    def row(self, **kwargs):
        if 'sql' in kwargs:
            self._row = self.DataDim(
                self._cursor, self.args, **kwargs)
            if len(self._row.data) > 1 and self._row.name_plural:
                self._row.name = self._row.name_plural
            return self._row.data
        return None

    def col(self, **kwargs):
        if 'sql' in kwargs:
            self._col = self.DataDim(
                self._cursor, self.args, **kwargs)
            if len(self._col.data) > 1 and self._col.name_plural:
                self._col.name = self._col.name_plural
            return self._col.data
        return None

    def value(self, **kwargs):
        if 'sql' in kwargs:
            self._value = self.DataDim(
                self._cursor, self.args, **kwargs)
            return self._value.data
        return None

    def do_table_data(self):
        self._table_data = {
            'header': [],
            'fields': [],
            'data': [],
            'row_tot': False,
            'col_tot': False,
            'style': {},
        }
        right_style = 'text-align: right;'
        bold_style = 'font-weight: bold;'

        if len(self._col.data) != 0 and len(self._row.data) != 0:
            self._table_data['header'] = \
                ['{} / {}'.format(self._row.name, self._col.name)]
            self._table_data['fields'] = [self._row.id_field]
            for col in self._col.data:
                self._table_data['header'].append(col[self._col.facade_field])
                self._table_data['fields'].append(col[self._col.id_field])

            row_tots = {}
            if self._col.idx_total != -1:
                self._table_data['row_tot'] = True
                for i in range(0, self._row.niveis):
                    row_tots[i] = 0

            col_tots = {}
            if self._row.idx_total != -1:
                self._table_data['col_tot'] = True
                for i in range(0, self._col.niveis):
                    col_tots[i+1] = 0

            if self._col.idx_total != -1:
                self._col.idx_total += 1

            for i_row, row in enumerate(self._row.data):
                for i_field, field in enumerate(self._table_data['fields']):
                    if i_field == 0:
                        self._table_data['data'].append({})
                        self._table_data['data'][i_row][field] = \
                            row[self._row.facade_field]
                    else:
                        tot_row = i_row == self._row.idx_total
                        tot_col = i_field == self._col.idx_total

                        if not tot_row and not tot_col:
                            value_list = [
                                v[self._value.id_field]
                                for v in self._value.data
                                if v[self._row.id_field] ==
                                row[self._row.id_field]
                                and v[self._col.id_field] == field]
                            if len(value_list) == 1:
                                value = value_list[0]
                            else:
                                value = 0

                            self._table_data['data'][i_row][field] = value
                            self.total += value
                            if i_row in row_tots:
                                row_tots[i_row] += value
                            if i_field in col_tots:
                                col_tots[i_field] += value

                        if tot_row and not tot_col:
                            self._table_data['data'][i_row][field] = \
                                col_tots[i_field]

                        if not tot_row and tot_col:
                            self._table_data['data'][i_row][field] = \
                                row_tots[i_row]

                        if tot_row and tot_col:
                            self._table_data['data'][i_row][field] = self.total

            len_fields = len(self._table_data['fields'])
            for i in range(2, len_fields+1):
                self._table_data['style'][i] = right_style

            if row_tots != {}:
                self._table_data['style'][len_fields] = (
                    self._table_data['style'][len_fields] + bold_style)

            if col_tots != {}:
                self._table_data['data'][-1]['|STYLE'] = bold_style

        return self._table_data

    @property
    def table_data(self):
        if self._table_data is None:
            return self.do_table_data()
        return self._table_data

    @table_data.setter
    def table_data(self, value):
        self._table_data = value
<commit_msg>criado rows_to_key_dict<commit_after>from pprint import pprint

def rows_to_key_dict(cursor, keys):

    def fkeys(row, keys):
        return tuple((key, row[key]) for key in keys)

    def fvalue(row, keys):
        return row[keys[0]]

    def fdict(row, keys):
        return {key: row[key] for key in keys}

    if isinstance(keys, tuple):
        fkey = fkeys
    else:
        keys = (keys, )
        fkey = fvalue

    columns = [i[0] for i in cursor.description]
    no_keys = tuple(column for column in columns if column not in keys)

    if len(no_keys) == 1:
        fvalue = fvalue
    else:
        fvalue = fdict

    return {fkey(values, keys): fvalue(values, no_keys)
            for values in [dict(zip(columns, row))
                           for row in cursor]}

def rows_to_dict_list(cursor):
    columns = [i[0] for i in cursor.description]
    return [dict(zip(columns, row)) for row in cursor]

def rows_to_dict_list_lower(cursor):
    columns = [i[0].lower() for i in cursor.description]
    return [dict(zip(columns, row)) for row in cursor]

def queryset_to_dict_list_lower(qs):
    result = []
    for obj in qs:
        result.append({
            name.lower(): obj.__dict__[name] for name in obj.__dict__
            if name[0] != '_'
        })
    return result

def dict_list_to_lower(data):
    data_lower = []
    for row in data:
        row_lower = {}
        for key in row.keys():
            row_lower[key.lower()] = row[key]
        data_lower.append(row_lower)
    return data_lower

def dict_def_options(dictionary, default, *args):
    """
    Return dictionary[arg] for first arg in args that exists in dictionary.
    Otherwise, return default value.
    """
    for arg in args:
        if arg in dictionary:
            return dictionary[arg]
    return default

def dict_options(dictionary, *args):
    """
    Call dict_def_options with default value None
    """
    return dict_def_options(dictionary, None, *args)

class DataSql(object):
    """docstring for DataDim."""
    def __init__(self, cursor, args=[], sql=''):
        self._cursor = cursor
        self.args = args
        if sql:
            self.sql = sql

    def execute(self, sql):
        self._cursor.execute(sql, self.args)
        return rows_to_dict_list(self._cursor)

    @property
    def sql(self): pass

    @sql.setter
    def sql(self, sql):
        self.data = self.execute(sql)

class ExecSql(object):
    """
    Initialize with cursor, args=[] (or *arguments) and parameters:
    - receive data when run DataSql.execute n times, passing only the sql
    Or pass sql also in inicialization:
    - receive imediatly data
    Parameters can be:
    - result_case:
      - '' or upper: process cursor with rows_to_dict_list
      - lower: process cursor with rows_to_dict_list_lower
      - cursor; not process the cursor
    """
    def __init__(self, cursor, *arguments, args=None, sql='', **kwargs):
        self._cursor = cursor
        if len(arguments) == 0:
            self.args = []
        else:
            self.args = arguments
        if args is not None:
            self.args.append(args)
        self.result_case = 'upper'
        if 'result_case' in {k.lower() for k in kwargs}:
            self.result_case = {
                k.lower(): v for k, v in kwargs.items()}['result_case']
        if sql:
            self.sql = sql

    def execute(self, sql=''):
        if sql:
            self.sql = sql
        self._cursor.execute(self.sql, self.args)
        if self.result_case == 'lower':
            return rows_to_dict_list_lower(self._cursor)
        elif self.result_case == 'cursor':
            return self._cursor
        else:
            return rows_to_dict_list(self._cursor)

    @property
    def sql(self): pass

    @sql.setter
    def sql(self, sql):
        self.data = self.execute(sql)

class GradeQtd(object):
    """docstring for GradeQtd."""

    class DataDim(DataSql):
        """docstring for DataDim."""
        def __init__(self, cursor, args, sql, **kwargs):
            self.id_field = \
                dict_def_options(kwargs, '', 'id', 'id_field')
            self.facade_field = \
                dict_def_options(kwargs, '', 'facade', 'facade_field',
                                             'id', 'id_field')
            self.name = kwargs.get('name', '')
            self.name_plural = kwargs.get('name_plural', self.name+'s')
            self.total = kwargs.get('total', '')
            self.forca_total = kwargs.get('forca_total', False)
            super(GradeQtd.DataDim, self).__init__(cursor, args, sql)

        def execute(self, sql):
            data_dim = super(GradeQtd.DataDim, self).execute(sql)
            self.niveis = len(data_dim)
            if self.total == '' or \
                    (self.niveis == 1 and not self.forca_total):
                self.idx_total = -1
            else:
                self.idx_total = len(data_dim)
                column = {self.id_field: self.total}
                if self.facade_field != self.id_field:
                    column[self.facade_field] = self.total
                data_dim.append(column)
            return data_dim

    def __init__(self, cursor, args=[]):
        self._cursor = cursor
        self.args = args
        self.total = 0
        self._table_data = None

    def row(self, **kwargs):
        if 'sql' in kwargs:
            self._row = self.DataDim(
                self._cursor, self.args, **kwargs)
            if len(self._row.data) > 1 and self._row.name_plural:
                self._row.name = self._row.name_plural
            return self._row.data
        return None

    def col(self, **kwargs):
        if 'sql' in kwargs:
            self._col = self.DataDim(
                self._cursor, self.args, **kwargs)
            if len(self._col.data) > 1 and self._col.name_plural:
                self._col.name = self._col.name_plural
            return self._col.data
        return None

    def value(self, **kwargs):
        if 'sql' in kwargs:
            self._value = self.DataDim(
                self._cursor, self.args, **kwargs)
            return self._value.data
        return None

    def do_table_data(self):
        self._table_data = {
            'header': [],
            'fields': [],
            'data': [],
            'row_tot': False,
            'col_tot': False,
            'style': {},
        }
        right_style = 'text-align: right;'
        bold_style = 'font-weight: bold;'

        if len(self._col.data) != 0 and len(self._row.data) != 0:
            self._table_data['header'] = \
                ['{} / {}'.format(self._row.name, self._col.name)]
            self._table_data['fields'] = [self._row.id_field]
            for col in self._col.data:
                self._table_data['header'].append(col[self._col.facade_field])
                self._table_data['fields'].append(col[self._col.id_field])

            row_tots = {}
            if self._col.idx_total != -1:
                self._table_data['row_tot'] = True
                for i in range(0, self._row.niveis):
                    row_tots[i] = 0

            col_tots = {}
            if self._row.idx_total != -1:
                self._table_data['col_tot'] = True
                for i in range(0, self._col.niveis):
                    col_tots[i+1] = 0

            if self._col.idx_total != -1:
                self._col.idx_total += 1

            for i_row, row in enumerate(self._row.data):
                for i_field, field in enumerate(self._table_data['fields']):
                    if i_field == 0:
                        self._table_data['data'].append({})
                        self._table_data['data'][i_row][field] = \
                            row[self._row.facade_field]
                    else:
                        tot_row = i_row == self._row.idx_total
                        tot_col = i_field == self._col.idx_total

                        if not tot_row and not tot_col:
                            value_list = [
                                v[self._value.id_field]
                                for v in self._value.data
                                if v[self._row.id_field] ==
                                row[self._row.id_field]
                                and v[self._col.id_field] == field]
                            if len(value_list) == 1:
                                value = value_list[0]
                            else:
                                value = 0

                            self._table_data['data'][i_row][field] = value
                            self.total += value
                            if i_row in row_tots:
                                row_tots[i_row] += value
                            if i_field in col_tots:
                                col_tots[i_field] += value

                        if tot_row and not tot_col:
                            self._table_data['data'][i_row][field] = \
                                col_tots[i_field]

                        if not tot_row and tot_col:
                            self._table_data['data'][i_row][field] = \
                                row_tots[i_row]

                        if tot_row and tot_col:
                            self._table_data['data'][i_row][field] = self.total

            len_fields = len(self._table_data['fields'])
            for i in range(2, len_fields+1):
                self._table_data['style'][i] = right_style

            if row_tots != {}:
                self._table_data['style'][len_fields] = (
                    self._table_data['style'][len_fields] + bold_style)

            if col_tots != {}:
                self._table_data['data'][-1]['|STYLE'] = bold_style

        return self._table_data

    @property
    def table_data(self):
        if self._table_data is None:
            return self.do_table_data()
        return self._table_data

    @table_data.setter
    def table_data(self, value):
        self._table_data = value
<|endoftext|>


<commit_before>"""
Test the printing of anonymous and named namespace variables.
"""

from __future__ import print_function

import os, time
import lldb
from lldbsuite.test.lldbtest import *
import lldbsuite.test.lldbutil as lldbutil

class NamespaceLookupTestCase(TestBase):

    mydir = TestBase.compute_mydir(__file__)

    def setUp(self):
        # Call super's setUp().
        TestBase.setUp(self)
        # Break inside different scopes and evaluate value
        self.line_break_global_scope = line_number('ns.cpp', '// BP_global_scope')
        self.line_break_file_scope = line_number('ns2.cpp', '// BP_file_scope')
        self.line_break_ns_scope = line_number('ns2.cpp', '// BP_ns_scope')
        self.line_break_nested_ns_scope = line_number('ns2.cpp', '// BP_nested_ns_scope')
        self.line_break_nested_ns_scope_after_using = line_number('ns2.cpp', '// BP_nested_ns_scope_after_using')
        self.line_break_before_using_directive = line_number('ns3.cpp', '// BP_before_using_directive')
        self.line_break_after_using_directive = line_number('ns3.cpp', '// BP_after_using_directive')

    def runToBkpt(self, command):
        self.runCmd(command, RUN_SUCCEEDED)
        # The stop reason of the thread should be breakpoint.
        self.expect("thread list", STOPPED_DUE_TO_BREAKPOINT,
            substrs = ['stopped',
                       'stop reason = breakpoint'])

    def test_scope_lookup_with_run_command(self):
        """Test scope lookup of functions in lldb."""
        self.build()
        self.runCmd("file a.out", CURRENT_EXECUTABLE_SET)

        lldbutil.run_break_set_by_file_and_line (self, "ns.cpp", self.line_break_global_scope, num_expected_locations=1, loc_exact=False)
        lldbutil.run_break_set_by_file_and_line (self, "ns2.cpp", self.line_break_ns_scope, num_expected_locations=1, loc_exact=False)
        lldbutil.run_break_set_by_file_and_line (self, "ns2.cpp", self.line_break_nested_ns_scope, num_expected_locations=1, loc_exact=False)
        lldbutil.run_break_set_by_file_and_line (self, "ns2.cpp", self.line_break_nested_ns_scope_after_using, num_expected_locations=1, loc_exact=False)
        lldbutil.run_break_set_by_file_and_line (self, "ns3.cpp", self.line_break_before_using_directive, num_expected_locations=1, loc_exact=False)
        lldbutil.run_break_set_by_file_and_line (self, "ns3.cpp", self.line_break_after_using_directive, num_expected_locations=1, loc_exact=False)

        # Run to BP_global_scope at global scope
        self.runToBkpt("run")
        # Evaluate func() - should call ::func()
        self.expect("expr -- func()", startstr = "(int) $0 = 1")
        # Evaluate A::B::func() - should call A::B::func()
        self.expect("expr -- A::B::func()", startstr = "(int) $1 = 4")
        # Evaluate func(10) - should call ::func(int)
        self.expect("expr -- func(10)", startstr = "(int) $2 = 11")
        # Evaluate ::func() - should call A::func()
        self.expect("expr -- ::func()", startstr = "(int) $3 = 1")
        # Evaluate A::foo() - should call A::foo()
        self.expect("expr -- A::foo()", startstr = "(int) $4 = 42")

        # Continue to BP_ns_scope at ns scope
        self.runToBkpt("continue")
        # Evaluate func(10) - should call A::func(int)
        self.expect("expr -- func(10)", startstr = "(int) $5 = 13")
        # Evaluate B::func() - should call B::func()
        self.expect("expr -- B::func()", startstr = "(int) $6 = 4")
        # Evaluate func() - should call A::func()
        self.expect("expr -- func()", startstr = "(int) $7 = 3")

        # Continue to BP_nested_ns_scope at nested ns scope
        self.runToBkpt("continue")
        # Evaluate func() - should call A::B::func()
        self.expect("expr -- func()", startstr = "(int) $8 = 4")
        # Evaluate A::func() - should call A::func()
        self.expect("expr -- A::func()", startstr = "(int) $9 = 3")

        # Evaluate func(10) - should call A::func(10)
        # NOTE: Under the rules of C++, this test would normally get an error
        # because A::B::func() hides A::func(), but lldb intentionally
        # disobeys these rules so that the intended overload can be found
        # by only removing duplicates if they have the same type.
        self.expect("expr -- func(10)", startstr = "(int) $10 = 13")

        # Continue to BP_nested_ns_scope_after_using at nested ns scope after using declaration
        self.runToBkpt("continue")
        # Evaluate A::func(10) - should call A::func(int)
        self.expect("expr -- A::func(10)", startstr = "(int) $11 = 13")

        # Continue to BP_before_using_directive at global scope before using declaration
        self.runToBkpt("continue")
        # Evaluate ::func() - should call ::func()
        self.expect("expr -- ::func()", startstr = "(int) $12 = 1")
        # Evaluate B::func() - should call B::func()
        self.expect("expr -- B::func()", startstr = "(int) $13 = 4")

        # Continue to BP_after_using_directive at global scope after using declaration
        self.runToBkpt("continue")
        # Evaluate ::func() - should call ::func()
        self.expect("expr -- ::func()", startstr = "(int) $14 = 1")
        # Evaluate B::func() - should call B::func()
        self.expect("expr -- B::func()", startstr = "(int) $15 = 4")

    @unittest2.expectedFailure("lldb scope lookup of functions bugs")
    def test_function_scope_lookup_with_run_command(self):
        """Test scope lookup of functions in lldb."""
        self.build()
        self.runCmd("file a.out", CURRENT_EXECUTABLE_SET)

        lldbutil.run_break_set_by_file_and_line (self, "ns.cpp", self.line_break_global_scope, num_expected_locations=1, loc_exact=False)
        lldbutil.run_break_set_by_file_and_line (self, "ns2.cpp", self.line_break_ns_scope, num_expected_locations=1, loc_exact=False)

        # Run to BP_global_scope at global scope
        self.runToBkpt("run")
        # Evaluate foo() - should call ::foo()
        # FIXME: lldb finds Y::foo because lookup for variables is done
        # before functions.
        self.expect("expr -- foo()", startstr = "(int) $0 = 42")
        # Evaluate ::foo() - should call ::foo()
        # FIXME: lldb finds Y::foo because lookup for variables is done
        # before functions and :: is ignored.
        self.expect("expr -- ::foo()", startstr = "(int) $1 = 42")

        # Continue to BP_ns_scope at ns scope
        self.runToBkpt("continue")
        # Evaluate foo() - should call A::foo()
        # FIXME: lldb finds Y::foo because lookup for variables is done
        # before functions.
        self.expect("expr -- foo()", startstr = "(int) $2 = 42")

    @unittest2.expectedFailure("lldb file scope lookup bugs")
    def test_file_scope_lookup_with_run_command(self):
        """Test file scope lookup in lldb."""
        self.build()
        self.runCmd("file a.out", CURRENT_EXECUTABLE_SET)

        lldbutil.run_break_set_by_file_and_line (self, "ns2.cpp", self.line_break_file_scope, num_expected_locations=1, loc_exact=False)

        # Run to BP_file_scope at file scope
        self.runToBkpt("run")
        # Evaluate func() - should call static ns2.cpp:func()
        # FIXME: This test fails because lldb doesn't know about file scopes so
        # finds the global ::func().
        self.expect("expr -- func()", startstr = "(int) $0 = 2")

    def test_scope_lookup_before_using_with_run_command(self):
        """Test scope lookup before using in lldb."""
        self.build()
        self.runCmd("file a.out", CURRENT_EXECUTABLE_SET)

        lldbutil.run_break_set_by_file_and_line (self, "ns3.cpp", self.line_break_before_using_directive, num_expected_locations=1, loc_exact=False)

        # Run to BP_before_using_directive at global scope before using declaration
        self.runToBkpt("run")
        # Evaluate func() - should call ::func()
        self.expect("expr -- func()", startstr = "(int) $0 = 1")

    # NOTE: this test may fail on older systems that don't emit import
    # emtries in DWARF - may need to add checks for compiler versions here.
    def test_scope_after_using_directive_lookup_with_run_command(self):
        """Test scope lookup after using directive in lldb."""
        self.build()
        self.runCmd("file a.out", CURRENT_EXECUTABLE_SET)

        lldbutil.run_break_set_by_file_and_line (self, "ns3.cpp", self.line_break_after_using_directive, num_expected_locations=1, loc_exact=False)

        # Run to BP_after_using_directive at global scope after using declaration
        self.runToBkpt("run")
        # Evaluate func2() - should call A::func2()
        self.expect("expr -- func2()", startstr = "(int) $0 = 3")

    @unittest2.expectedFailure("lldb scope lookup after using declaration bugs")
    # NOTE: this test may fail on older systems that don't emit import
    # emtries in DWARF - may need to add checks for compiler versions here.
    def test_scope_after_using_declaration_lookup_with_run_command(self):
        """Test scope lookup after using declaration in lldb."""
        self.build()
        self.runCmd("file a.out", CURRENT_EXECUTABLE_SET)

        lldbutil.run_break_set_by_file_and_line (self, "ns2.cpp", self.line_break_nested_ns_scope_after_using, num_expected_locations=1, loc_exact=False)

        # Run to BP_nested_ns_scope_after_using at nested ns scope after using declaration
        self.runToBkpt("run")
        # Evaluate func() - should call A::func()
        self.expect("expr -- func()", startstr = "(int) $0 = 3")

    @unittest2.expectedFailure("lldb scope lookup ambiguity after using bugs")
    def test_scope_ambiguity_after_using_lookup_with_run_command(self):
        """Test scope lookup ambiguity after using in lldb."""
        self.build()
        self.runCmd("file a.out", CURRENT_EXECUTABLE_SET)

        lldbutil.run_break_set_by_file_and_line (self, "ns3.cpp", self.line_break_after_using_directive, num_expected_locations=1, loc_exact=False)

        # Run to BP_after_using_directive at global scope after using declaration
        self.runToBkpt("run")
        # Evaluate func() - should get error: ambiguous
        # FIXME: This test fails because lldb removes duplicates if they have
        # the same type.
        self.expect("expr -- func()", startstr = "error")

    def test_scope_lookup_shadowed_by_using_with_run_command(self):
        """Test scope lookup shadowed by using in lldb."""
        self.build()
        self.runCmd("file a.out", CURRENT_EXECUTABLE_SET)

        lldbutil.run_break_set_by_file_and_line (self, "ns2.cpp", self.line_break_nested_ns_scope, num_expected_locations=1, loc_exact=False)

        # Run to BP_nested_ns_scope at nested ns scope
        self.runToBkpt("run")
        # Evaluate func(10) - should call A::func(10)
        # NOTE: Under the rules of C++, this test would normally get an error
        # because A::B::func() shadows A::func(), but lldb intentionally
        # disobeys these rules so that the intended overload can be found
        # by only removing duplicates if they have the same type.
        self.expect("expr -- func(10)", startstr = "(int) $0 = 13")

<commit_msg>XFAIL TestNamespaceLookup for linux<commit_after>"""
Test the printing of anonymous and named namespace variables.
"""

from __future__ import print_function

import os, time
import lldb
from lldbsuite.test.lldbtest import *
import lldbsuite.test.lldbutil as lldbutil

class NamespaceLookupTestCase(TestBase):

    mydir = TestBase.compute_mydir(__file__)

    def setUp(self):
        # Call super's setUp().
        TestBase.setUp(self)
        # Break inside different scopes and evaluate value
        self.line_break_global_scope = line_number('ns.cpp', '// BP_global_scope')
        self.line_break_file_scope = line_number('ns2.cpp', '// BP_file_scope')
        self.line_break_ns_scope = line_number('ns2.cpp', '// BP_ns_scope')
        self.line_break_nested_ns_scope = line_number('ns2.cpp', '// BP_nested_ns_scope')
        self.line_break_nested_ns_scope_after_using = line_number('ns2.cpp', '// BP_nested_ns_scope_after_using')
        self.line_break_before_using_directive = line_number('ns3.cpp', '// BP_before_using_directive')
        self.line_break_after_using_directive = line_number('ns3.cpp', '// BP_after_using_directive')

    def runToBkpt(self, command):
        self.runCmd(command, RUN_SUCCEEDED)
        # The stop reason of the thread should be breakpoint.
        self.expect("thread list", STOPPED_DUE_TO_BREAKPOINT,
            substrs = ['stopped',
                       'stop reason = breakpoint'])

    @expectedFailureLinux("llvm.org/pr25819")
    def test_scope_lookup_with_run_command(self):
        """Test scope lookup of functions in lldb."""
        self.build()
        self.runCmd("file a.out", CURRENT_EXECUTABLE_SET)

        lldbutil.run_break_set_by_file_and_line (self, "ns.cpp", self.line_break_global_scope, num_expected_locations=1, loc_exact=False)
        lldbutil.run_break_set_by_file_and_line (self, "ns2.cpp", self.line_break_ns_scope, num_expected_locations=1, loc_exact=False)
        lldbutil.run_break_set_by_file_and_line (self, "ns2.cpp", self.line_break_nested_ns_scope, num_expected_locations=1, loc_exact=False)
        lldbutil.run_break_set_by_file_and_line (self, "ns2.cpp", self.line_break_nested_ns_scope_after_using, num_expected_locations=1, loc_exact=False)
        lldbutil.run_break_set_by_file_and_line (self, "ns3.cpp", self.line_break_before_using_directive, num_expected_locations=1, loc_exact=False)
        lldbutil.run_break_set_by_file_and_line (self, "ns3.cpp", self.line_break_after_using_directive, num_expected_locations=1, loc_exact=False)

        # Run to BP_global_scope at global scope
        self.runToBkpt("run")
        # Evaluate func() - should call ::func()
        self.expect("expr -- func()", startstr = "(int) $0 = 1")
        # Evaluate A::B::func() - should call A::B::func()
        self.expect("expr -- A::B::func()", startstr = "(int) $1 = 4")
        # Evaluate func(10) - should call ::func(int)
        self.expect("expr -- func(10)", startstr = "(int) $2 = 11")
        # Evaluate ::func() - should call A::func()
        self.expect("expr -- ::func()", startstr = "(int) $3 = 1")
        # Evaluate A::foo() - should call A::foo()
        self.expect("expr -- A::foo()", startstr = "(int) $4 = 42")

        # Continue to BP_ns_scope at ns scope
        self.runToBkpt("continue")
        # Evaluate func(10) - should call A::func(int)
        self.expect("expr -- func(10)", startstr = "(int) $5 = 13")
        # Evaluate B::func() - should call B::func()
        self.expect("expr -- B::func()", startstr = "(int) $6 = 4")
        # Evaluate func() - should call A::func()
        self.expect("expr -- func()", startstr = "(int) $7 = 3")

        # Continue to BP_nested_ns_scope at nested ns scope
        self.runToBkpt("continue")
        # Evaluate func() - should call A::B::func()
        self.expect("expr -- func()", startstr = "(int) $8 = 4")
        # Evaluate A::func() - should call A::func()
        self.expect("expr -- A::func()", startstr = "(int) $9 = 3")

        # Evaluate func(10) - should call A::func(10)
        # NOTE: Under the rules of C++, this test would normally get an error
        # because A::B::func() hides A::func(), but lldb intentionally
        # disobeys these rules so that the intended overload can be found
        # by only removing duplicates if they have the same type.
        self.expect("expr -- func(10)", startstr = "(int) $10 = 13")

        # Continue to BP_nested_ns_scope_after_using at nested ns scope after using declaration
        self.runToBkpt("continue")
        # Evaluate A::func(10) - should call A::func(int)
        self.expect("expr -- A::func(10)", startstr = "(int) $11 = 13")

        # Continue to BP_before_using_directive at global scope before using declaration
        self.runToBkpt("continue")
        # Evaluate ::func() - should call ::func()
        self.expect("expr -- ::func()", startstr = "(int) $12 = 1")
        # Evaluate B::func() - should call B::func()
        self.expect("expr -- B::func()", startstr = "(int) $13 = 4")

        # Continue to BP_after_using_directive at global scope after using declaration
        self.runToBkpt("continue")
        # Evaluate ::func() - should call ::func()
        self.expect("expr -- ::func()", startstr = "(int) $14 = 1")
        # Evaluate B::func() - should call B::func()
        self.expect("expr -- B::func()", startstr = "(int) $15 = 4")

    @unittest2.expectedFailure("lldb scope lookup of functions bugs")
    def test_function_scope_lookup_with_run_command(self):
        """Test scope lookup of functions in lldb."""
        self.build()
        self.runCmd("file a.out", CURRENT_EXECUTABLE_SET)

        lldbutil.run_break_set_by_file_and_line (self, "ns.cpp", self.line_break_global_scope, num_expected_locations=1, loc_exact=False)
        lldbutil.run_break_set_by_file_and_line (self, "ns2.cpp", self.line_break_ns_scope, num_expected_locations=1, loc_exact=False)

        # Run to BP_global_scope at global scope
        self.runToBkpt("run")
        # Evaluate foo() - should call ::foo()
        # FIXME: lldb finds Y::foo because lookup for variables is done
        # before functions.
        self.expect("expr -- foo()", startstr = "(int) $0 = 42")
        # Evaluate ::foo() - should call ::foo()
        # FIXME: lldb finds Y::foo because lookup for variables is done
        # before functions and :: is ignored.
        self.expect("expr -- ::foo()", startstr = "(int) $1 = 42")

        # Continue to BP_ns_scope at ns scope
        self.runToBkpt("continue")
        # Evaluate foo() - should call A::foo()
        # FIXME: lldb finds Y::foo because lookup for variables is done
        # before functions.
        self.expect("expr -- foo()", startstr = "(int) $2 = 42")

    @unittest2.expectedFailure("lldb file scope lookup bugs")
    def test_file_scope_lookup_with_run_command(self):
        """Test file scope lookup in lldb."""
        self.build()
        self.runCmd("file a.out", CURRENT_EXECUTABLE_SET)

        lldbutil.run_break_set_by_file_and_line (self, "ns2.cpp", self.line_break_file_scope, num_expected_locations=1, loc_exact=False)

        # Run to BP_file_scope at file scope
        self.runToBkpt("run")
        # Evaluate func() - should call static ns2.cpp:func()
        # FIXME: This test fails because lldb doesn't know about file scopes so
        # finds the global ::func().
        self.expect("expr -- func()", startstr = "(int) $0 = 2")

    @expectedFailureAll(bugnumber="llvm.org/pr25819", oslist=["linux"], compiler="gcc")
    def test_scope_lookup_before_using_with_run_command(self):
        """Test scope lookup before using in lldb."""
        self.build()
        self.runCmd("file a.out", CURRENT_EXECUTABLE_SET)

        lldbutil.run_break_set_by_file_and_line (self, "ns3.cpp", self.line_break_before_using_directive, num_expected_locations=1, loc_exact=False)

        # Run to BP_before_using_directive at global scope before using declaration
        self.runToBkpt("run")
        # Evaluate func() - should call ::func()
        self.expect("expr -- func()", startstr = "(int) $0 = 1")

    # NOTE: this test may fail on older systems that don't emit import
    # emtries in DWARF - may need to add checks for compiler versions here.
    @expectedFailureLinux("llvm.org/pr25819")
    def test_scope_after_using_directive_lookup_with_run_command(self):
        """Test scope lookup after using directive in lldb."""
        self.build()
        self.runCmd("file a.out", CURRENT_EXECUTABLE_SET)

        lldbutil.run_break_set_by_file_and_line (self, "ns3.cpp", self.line_break_after_using_directive, num_expected_locations=1, loc_exact=False)

        # Run to BP_after_using_directive at global scope after using declaration
        self.runToBkpt("run")
        # Evaluate func2() - should call A::func2()
        self.expect("expr -- func2()", startstr = "(int) $0 = 3")

    @unittest2.expectedFailure("lldb scope lookup after using declaration bugs")
    # NOTE: this test may fail on older systems that don't emit import
    # emtries in DWARF - may need to add checks for compiler versions here.
    def test_scope_after_using_declaration_lookup_with_run_command(self):
        """Test scope lookup after using declaration in lldb."""
        self.build()
        self.runCmd("file a.out", CURRENT_EXECUTABLE_SET)

        lldbutil.run_break_set_by_file_and_line (self, "ns2.cpp", self.line_break_nested_ns_scope_after_using, num_expected_locations=1, loc_exact=False)

        # Run to BP_nested_ns_scope_after_using at nested ns scope after using declaration
        self.runToBkpt("run")
        # Evaluate func() - should call A::func()
        self.expect("expr -- func()", startstr = "(int) $0 = 3")

    @unittest2.expectedFailure("lldb scope lookup ambiguity after using bugs")
    def test_scope_ambiguity_after_using_lookup_with_run_command(self):
        """Test scope lookup ambiguity after using in lldb."""
        self.build()
        self.runCmd("file a.out", CURRENT_EXECUTABLE_SET)

        lldbutil.run_break_set_by_file_and_line (self, "ns3.cpp", self.line_break_after_using_directive, num_expected_locations=1, loc_exact=False)

        # Run to BP_after_using_directive at global scope after using declaration
        self.runToBkpt("run")
        # Evaluate func() - should get error: ambiguous
        # FIXME: This test fails because lldb removes duplicates if they have
        # the same type.
        self.expect("expr -- func()", startstr = "error")

    @expectedFailureLinux("llvm.org/pr25819")
    def test_scope_lookup_shadowed_by_using_with_run_command(self):
        """Test scope lookup shadowed by using in lldb."""
        self.build()
        self.runCmd("file a.out", CURRENT_EXECUTABLE_SET)

        lldbutil.run_break_set_by_file_and_line (self, "ns2.cpp", self.line_break_nested_ns_scope, num_expected_locations=1, loc_exact=False)

        # Run to BP_nested_ns_scope at nested ns scope
        self.runToBkpt("run")
        # Evaluate func(10) - should call A::func(10)
        # NOTE: Under the rules of C++, this test would normally get an error
        # because A::B::func() shadows A::func(), but lldb intentionally
        # disobeys these rules so that the intended overload can be found
        # by only removing duplicates if they have the same type.
        self.expect("expr -- func(10)", startstr = "(int) $0 = 13")

<|endoftext|>