<commit_before>from itertools import product

import numpy as np

from .. import confusion_matrix
from ...utils import check_matplotlib_support
from ...utils.multiclass import unique_labels
from ...utils.validation import _deprecate_positional_args
from ...base import is_classifier

class ConfusionMatrixDisplay:
    """Confusion Matrix visualization.

    It is recommend to use :func:`~sklearn.metrics.plot_confusion_matrix` to
    create a :class:`ConfusionMatrixDisplay`. All parameters are stored as
    attributes.

    Read more in the :ref:`User Guide <visualizations>`.

    Parameters
    ----------
    confusion_matrix : ndarray of shape (n_classes, n_classes)
        Confusion matrix.

    display_labels : ndarray of shape (n_classes,), default=None
        Display labels for plot. If None, display labels are set from 0 to
        `n_classes - 1`.

    Attributes
    ----------
    im_ : matplotlib AxesImage
        Image representing the confusion matrix.

    text_ : ndarray of shape (n_classes, n_classes), dtype=matplotlib Text, \
            or None
        Array of matplotlib axes. `None` if `include_values` is false.

    ax_ : matplotlib Axes
        Axes with confusion matrix.

    figure_ : matplotlib Figure
        Figure containing the confusion matrix.

    See Also
    --------
    confusion_matrix : Compute Confusion Matrix to evaluate the accuracy of a
        classification.
    plot_confusion_matrix : Plot Confusion Matrix.

    Examples
    --------
    >>> from sklearn.datasets import make_classification
    >>> from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay
    >>> from sklearn.model_selection import train_test_split
    >>> from sklearn.svm import SVC
    >>> X, y = make_classification(random_state=0)
    >>> X_train, X_test, y_train, y_test = train_test_split(X, y,
    ...                                                     random_state=0)
    >>> clf = SVC(random_state=0)
    >>> clf.fit(X_train, y_train)
    SVC(random_state=0)
    >>> predictions = clf.predict(X_test)
    >>> cm = confusion_matrix(y_test, predictions, labels=clf.classes_)
    >>> disp = ConfusionMatrixDisplay(confusion_matrix=cm,
    ...                               display_labels=clf.classes_)
    >>> disp.plot() # doctest: +SKIP
    """
    @_deprecate_positional_args
    def __init__(self, confusion_matrix, *, display_labels=None):
        self.confusion_matrix = confusion_matrix
        self.display_labels = display_labels

    @_deprecate_positional_args
    def plot(self, *, include_values=True, cmap='viridis',
             xticks_rotation='horizontal', values_format=None,
             ax=None, colorbar=True):
        """Plot visualization.

        Parameters
        ----------
        include_values : bool, default=True
            Includes values in confusion matrix.

        cmap : str or matplotlib Colormap, default='viridis'
            Colormap recognized by matplotlib.

        xticks_rotation : {'vertical', 'horizontal'} or float, \
                         default='horizontal'
            Rotation of xtick labels.

        values_format : str, default=None
            Format specification for values in confusion matrix. If `None`,
            the format specification is 'd' or '.2g' whichever is shorter.

        ax : matplotlib axes, default=None
            Axes object to plot on. If `None`, a new figure and axes is
            created.

        colorbar : bool, default=True
            Whether or not to add a colorbar to the plot.

        Returns
        -------
        display : :class:`~sklearn.metrics.ConfusionMatrixDisplay`
        """
        check_matplotlib_support("ConfusionMatrixDisplay.plot")
        import matplotlib.pyplot as plt

        if ax is None:
            fig, ax = plt.subplots()
        else:
            fig = ax.figure

        cm = self.confusion_matrix
        n_classes = cm.shape[0]
        self.im_ = ax.imshow(cm, interpolation='nearest', cmap=cmap)
        self.text_ = None
        cmap_min, cmap_max = self.im_.cmap(0), self.im_.cmap(256)

        if include_values:
            self.text_ = np.empty_like(cm, dtype=object)

            # print text with appropriate color depending on background
            thresh = (cm.max() + cm.min()) / 2.0

            for i, j in product(range(n_classes), range(n_classes)):
                color = cmap_max if cm[i, j] < thresh else cmap_min

                if values_format is None:
                    text_cm = format(cm[i, j], '.2g')
                    if cm.dtype.kind != 'f':
                        text_d = format(cm[i, j], 'd')
                        if len(text_d) < len(text_cm):
                            text_cm = text_d
                else:
                    text_cm = format(cm[i, j], values_format)

                self.text_[i, j] = ax.text(
                    j, i, text_cm,
                    ha="center", va="center",
                    color=color)

        if self.display_labels is None:
            display_labels = np.arange(n_classes)
        else:
            display_labels = self.display_labels
        if colorbar:
            fig.colorbar(self.im_, ax=ax)
        ax.set(xticks=np.arange(n_classes),
               yticks=np.arange(n_classes),
               xticklabels=display_labels,
               yticklabels=display_labels,
               ylabel="True label",
               xlabel="Predicted label")

        ax.set_ylim((n_classes - 0.5, -0.5))
        plt.setp(ax.get_xticklabels(), rotation=xticks_rotation)

        self.figure_ = fig
        self.ax_ = ax
        return self

@_deprecate_positional_args
def plot_confusion_matrix(estimator, X, y_true, *, labels=None,
                          sample_weight=None, normalize=None,
                          display_labels=None, include_values=True,
                          xticks_rotation='horizontal',
                          values_format=None,
                          cmap='viridis', ax=None, colorbar=True):
    """Plot Confusion Matrix.

    Read more in the :ref:`User Guide <confusion_matrix>`.

    Parameters
    ----------
    estimator : estimator instance
        Fitted classifier or a fitted :class:`~sklearn.pipeline.Pipeline`
        in which the last estimator is a classifier.

    X : {array-like, sparse matrix} of shape (n_samples, n_features)
        Input values.

    y : array-like of shape (n_samples,)
        Target values.

    labels : array-like of shape (n_classes,), default=None
        List of labels to index the matrix. This may be used to reorder or
        select a subset of labels. If `None` is given, those that appear at
        least once in `y_true` or `y_pred` are used in sorted order.

    sample_weight : array-like of shape (n_samples,), default=None
        Sample weights.

    normalize : {'true', 'pred', 'all'}, default=None
        Normalizes confusion matrix over the true (rows), predicted (columns)
        conditions or all the population. If None, confusion matrix will not be
        normalized.

    display_labels : array-like of shape (n_classes,), default=None
        Target names used for plotting. By default, `labels` will be used if
        it is defined, otherwise the unique labels of `y_true` and `y_pred`
        will be used.

    include_values : bool, default=True
        Includes values in confusion matrix.

    xticks_rotation : {'vertical', 'horizontal'} or float, \
                        default='horizontal'
        Rotation of xtick labels.

    values_format : str, default=None
        Format specification for values in confusion matrix. If `None`,
        the format specification is 'd' or '.2g' whichever is shorter.

    cmap : str or matplotlib Colormap, default='viridis'
        Colormap recognized by matplotlib.

    ax : matplotlib Axes, default=None
        Axes object to plot on. If `None`, a new figure and axes is
        created.

    colorbar : bool, default=True
        Whether or not to add a colorbar to the plot.

        .. versionadded:: 0.24

    Returns
    -------
    display : :class:`~sklearn.metrics.ConfusionMatrixDisplay`

    See Also
    --------
    confusion_matrix : Compute Confusion Matrix to evaluate the accuracy of a
        classification.
    ConfusionMatrixDisplay : Confusion Matrix visualization.

    Examples
    --------
    >>> import matplotlib.pyplot as plt  # doctest: +SKIP
    >>> from sklearn.datasets import make_classification
    >>> from sklearn.metrics import plot_confusion_matrix
    >>> from sklearn.model_selection import train_test_split
    >>> from sklearn.svm import SVC
    >>> X, y = make_classification(random_state=0)
    >>> X_train, X_test, y_train, y_test = train_test_split(
    ...         X, y, random_state=0)
    >>> clf = SVC(random_state=0)
    >>> clf.fit(X_train, y_train)
    SVC(random_state=0)
    >>> plot_confusion_matrix(clf, X_test, y_test)  # doctest: +SKIP
    >>> plt.show()  # doctest: +SKIP
    """
    check_matplotlib_support("plot_confusion_matrix")

    if not is_classifier(estimator):
        raise ValueError("plot_confusion_matrix only supports classifiers")

    y_pred = estimator.predict(X)
    cm = confusion_matrix(y_true, y_pred, sample_weight=sample_weight,
                          labels=labels, normalize=normalize)

    if display_labels is None:
        if labels is None:
            display_labels = unique_labels(y_true, y_pred)
        else:
            display_labels = labels

    disp = ConfusionMatrixDisplay(confusion_matrix=cm,
                                  display_labels=display_labels)
    return disp.plot(include_values=include_values,
                     cmap=cmap, ax=ax, xticks_rotation=xticks_rotation,
                     values_format=values_format, colorbar=colorbar)
<commit_msg>DOC Updates confusion_matrix.py to match parameter in code(#18938)<commit_after>from itertools import product

import numpy as np

from .. import confusion_matrix
from ...utils import check_matplotlib_support
from ...utils.multiclass import unique_labels
from ...utils.validation import _deprecate_positional_args
from ...base import is_classifier

class ConfusionMatrixDisplay:
    """Confusion Matrix visualization.

    It is recommend to use :func:`~sklearn.metrics.plot_confusion_matrix` to
    create a :class:`ConfusionMatrixDisplay`. All parameters are stored as
    attributes.

    Read more in the :ref:`User Guide <visualizations>`.

    Parameters
    ----------
    confusion_matrix : ndarray of shape (n_classes, n_classes)
        Confusion matrix.

    display_labels : ndarray of shape (n_classes,), default=None
        Display labels for plot. If None, display labels are set from 0 to
        `n_classes - 1`.

    Attributes
    ----------
    im_ : matplotlib AxesImage
        Image representing the confusion matrix.

    text_ : ndarray of shape (n_classes, n_classes), dtype=matplotlib Text, \
            or None
        Array of matplotlib axes. `None` if `include_values` is false.

    ax_ : matplotlib Axes
        Axes with confusion matrix.

    figure_ : matplotlib Figure
        Figure containing the confusion matrix.

    See Also
    --------
    confusion_matrix : Compute Confusion Matrix to evaluate the accuracy of a
        classification.
    plot_confusion_matrix : Plot Confusion Matrix.

    Examples
    --------
    >>> from sklearn.datasets import make_classification
    >>> from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay
    >>> from sklearn.model_selection import train_test_split
    >>> from sklearn.svm import SVC
    >>> X, y = make_classification(random_state=0)
    >>> X_train, X_test, y_train, y_test = train_test_split(X, y,
    ...                                                     random_state=0)
    >>> clf = SVC(random_state=0)
    >>> clf.fit(X_train, y_train)
    SVC(random_state=0)
    >>> predictions = clf.predict(X_test)
    >>> cm = confusion_matrix(y_test, predictions, labels=clf.classes_)
    >>> disp = ConfusionMatrixDisplay(confusion_matrix=cm,
    ...                               display_labels=clf.classes_)
    >>> disp.plot() # doctest: +SKIP
    """
    @_deprecate_positional_args
    def __init__(self, confusion_matrix, *, display_labels=None):
        self.confusion_matrix = confusion_matrix
        self.display_labels = display_labels

    @_deprecate_positional_args
    def plot(self, *, include_values=True, cmap='viridis',
             xticks_rotation='horizontal', values_format=None,
             ax=None, colorbar=True):
        """Plot visualization.

        Parameters
        ----------
        include_values : bool, default=True
            Includes values in confusion matrix.

        cmap : str or matplotlib Colormap, default='viridis'
            Colormap recognized by matplotlib.

        xticks_rotation : {'vertical', 'horizontal'} or float, \
                         default='horizontal'
            Rotation of xtick labels.

        values_format : str, default=None
            Format specification for values in confusion matrix. If `None`,
            the format specification is 'd' or '.2g' whichever is shorter.

        ax : matplotlib axes, default=None
            Axes object to plot on. If `None`, a new figure and axes is
            created.

        colorbar : bool, default=True
            Whether or not to add a colorbar to the plot.

        Returns
        -------
        display : :class:`~sklearn.metrics.ConfusionMatrixDisplay`
        """
        check_matplotlib_support("ConfusionMatrixDisplay.plot")
        import matplotlib.pyplot as plt

        if ax is None:
            fig, ax = plt.subplots()
        else:
            fig = ax.figure

        cm = self.confusion_matrix
        n_classes = cm.shape[0]
        self.im_ = ax.imshow(cm, interpolation='nearest', cmap=cmap)
        self.text_ = None
        cmap_min, cmap_max = self.im_.cmap(0), self.im_.cmap(256)

        if include_values:
            self.text_ = np.empty_like(cm, dtype=object)

            # print text with appropriate color depending on background
            thresh = (cm.max() + cm.min()) / 2.0

            for i, j in product(range(n_classes), range(n_classes)):
                color = cmap_max if cm[i, j] < thresh else cmap_min

                if values_format is None:
                    text_cm = format(cm[i, j], '.2g')
                    if cm.dtype.kind != 'f':
                        text_d = format(cm[i, j], 'd')
                        if len(text_d) < len(text_cm):
                            text_cm = text_d
                else:
                    text_cm = format(cm[i, j], values_format)

                self.text_[i, j] = ax.text(
                    j, i, text_cm,
                    ha="center", va="center",
                    color=color)

        if self.display_labels is None:
            display_labels = np.arange(n_classes)
        else:
            display_labels = self.display_labels
        if colorbar:
            fig.colorbar(self.im_, ax=ax)
        ax.set(xticks=np.arange(n_classes),
               yticks=np.arange(n_classes),
               xticklabels=display_labels,
               yticklabels=display_labels,
               ylabel="True label",
               xlabel="Predicted label")

        ax.set_ylim((n_classes - 0.5, -0.5))
        plt.setp(ax.get_xticklabels(), rotation=xticks_rotation)

        self.figure_ = fig
        self.ax_ = ax
        return self

@_deprecate_positional_args
def plot_confusion_matrix(estimator, X, y_true, *, labels=None,
                          sample_weight=None, normalize=None,
                          display_labels=None, include_values=True,
                          xticks_rotation='horizontal',
                          values_format=None,
                          cmap='viridis', ax=None, colorbar=True):
    """Plot Confusion Matrix.

    Read more in the :ref:`User Guide <confusion_matrix>`.

    Parameters
    ----------
    estimator : estimator instance
        Fitted classifier or a fitted :class:`~sklearn.pipeline.Pipeline`
        in which the last estimator is a classifier.

    X : {array-like, sparse matrix} of shape (n_samples, n_features)
        Input values.

    y_true : array-like of shape (n_samples,)
        Target values.

    labels : array-like of shape (n_classes,), default=None
        List of labels to index the matrix. This may be used to reorder or
        select a subset of labels. If `None` is given, those that appear at
        least once in `y_true` or `y_pred` are used in sorted order.

    sample_weight : array-like of shape (n_samples,), default=None
        Sample weights.

    normalize : {'true', 'pred', 'all'}, default=None
        Normalizes confusion matrix over the true (rows), predicted (columns)
        conditions or all the population. If None, confusion matrix will not be
        normalized.

    display_labels : array-like of shape (n_classes,), default=None
        Target names used for plotting. By default, `labels` will be used if
        it is defined, otherwise the unique labels of `y_true` and `y_pred`
        will be used.

    include_values : bool, default=True
        Includes values in confusion matrix.

    xticks_rotation : {'vertical', 'horizontal'} or float, \
                        default='horizontal'
        Rotation of xtick labels.

    values_format : str, default=None
        Format specification for values in confusion matrix. If `None`,
        the format specification is 'd' or '.2g' whichever is shorter.

    cmap : str or matplotlib Colormap, default='viridis'
        Colormap recognized by matplotlib.

    ax : matplotlib Axes, default=None
        Axes object to plot on. If `None`, a new figure and axes is
        created.

    colorbar : bool, default=True
        Whether or not to add a colorbar to the plot.

        .. versionadded:: 0.24

    Returns
    -------
    display : :class:`~sklearn.metrics.ConfusionMatrixDisplay`

    See Also
    --------
    confusion_matrix : Compute Confusion Matrix to evaluate the accuracy of a
        classification.
    ConfusionMatrixDisplay : Confusion Matrix visualization.

    Examples
    --------
    >>> import matplotlib.pyplot as plt  # doctest: +SKIP
    >>> from sklearn.datasets import make_classification
    >>> from sklearn.metrics import plot_confusion_matrix
    >>> from sklearn.model_selection import train_test_split
    >>> from sklearn.svm import SVC
    >>> X, y = make_classification(random_state=0)
    >>> X_train, X_test, y_train, y_test = train_test_split(
    ...         X, y, random_state=0)
    >>> clf = SVC(random_state=0)
    >>> clf.fit(X_train, y_train)
    SVC(random_state=0)
    >>> plot_confusion_matrix(clf, X_test, y_test)  # doctest: +SKIP
    >>> plt.show()  # doctest: +SKIP
    """
    check_matplotlib_support("plot_confusion_matrix")

    if not is_classifier(estimator):
        raise ValueError("plot_confusion_matrix only supports classifiers")

    y_pred = estimator.predict(X)
    cm = confusion_matrix(y_true, y_pred, sample_weight=sample_weight,
                          labels=labels, normalize=normalize)

    if display_labels is None:
        if labels is None:
            display_labels = unique_labels(y_true, y_pred)
        else:
            display_labels = labels

    disp = ConfusionMatrixDisplay(confusion_matrix=cm,
                                  display_labels=display_labels)
    return disp.plot(include_values=include_values,
                     cmap=cmap, ax=ax, xticks_rotation=xticks_rotation,
                     values_format=values_format, colorbar=colorbar)
<|endoftext|>


<commit_before>#! /usr/bin/env python
#-*- coding:utf-8 -*-
#
# movie.py ---
#
# Filename: movie.py
# Description:
# Author: Werther Zhang
# Maintainer:
# Created: Fri Sep 29 13:25:41 2017 (+0800)
#

# Change Log:
#
#

if __name__ == '__main__' and __package__ is None:
    from os import sys, path
    sys.path.append(path.dirname(path.dirname(path.abspath(__file__))))
    import utils.browser as b
else:
    from ...utils import browser as b

from BeautifulSoup import BeautifulSoup

import datetime
import time
import logging
import sys
import os
import re

_LOGGER = logging.getLogger(__name__)
_LOGGER.setLevel(logging.INFO)

class dysfz(object):
    """"""
    IGNORE_LIST='save/dysfz_ignore.txt'
    def __init__(self):
        self._url = 'http://www.dysfz.cc/{}?o=2'
        self.ignore_list = []
        self._load_ignore_list()

    def _load_ignore_list(self):
        if not os.path.exists(dysfz.IGNORE_LIST):
            return
        with open(dysfz.IGNORE_LIST, 'r') as f:
            for l in f.readlines():
                self.ignore_list.append(l[:-1])

    def _save_ignore_list(self):
        if len(self.ignore_list) > 50:
            self.ignore_list = self.ignore_list[15:]
        with open(dysfz.IGNORE_LIST, 'w') as f:
            f.write('\n'.join(self.ignore_list) + '\n')

    def _create_soup(self, url):
        buff = ""
        with b.browser_base_urlopen(url) as f:
            buff = f.read()

        return BeautifulSoup(buff)

    def _get_dbsccore(self, dburl):
        soup = self._create_soup(dburl)
        item = soup.find(attrs={'class':'ll rating_num'})
        dbscore = item.string
        _LOGGER.debug('{}'.format(dbscore))
        if dbscore:
            return float(dbscore)
        return 0.0

    def _get_movie_list(self, soup):
        """MOVEL (name, movie_url, db_url)"""
        movie_list = []
        lis = soup.findAll('li')
        _LOGGER.debug('{}'.format(lis))
        for li in lis:
            # db = li.find(attrs={'class':'dbscore'})
            # if not db:
            #     continue
            # dbscore = float(db.b.string)
            year = datetime.datetime.now().strftime('%Y')
            year = int(year)
            #if today != li.p.span.string:
            #    continue
            _LOGGER.debug('LI {}'.format(li))
            if li.h2 == None:
                continue
            if li.p.find(attrs={'rel':'nofollow', 'target':'_blank'}) == None:
                continue
            movie_url = li.h2.a['href']
            db_url = li.p.find(attrs={'rel':'nofollow', 'target':'_blank'})['href']
            dbscore = self._get_dbsccore(db_url)
            movie_name = '{}({})'.format(li.h2.a.string.encode('utf-8'), dbscore)
            all_year = re.findall(r'【\d+】', movie_name)
            movie_year = year
            if len(all_year) > 0:
                movie_year = int(all_year[0][3:-3])
                _LOGGER.debug('movie_year: {}'.format(movie_year))

            if db_url in self.ignore_list:
                continue

            if dbscore <= 7:
                continue

            if movie_year < year - 1:
                continue

            _LOGGER.debug('dbscore: {}'.format(dbscore))
            _LOGGER.debug('{}'.format(li.p.span.string))
            _LOGGER.debug('{}\n{}\n{}'.format(movie_name, movie_url, db_url))
            movie_list.append((movie_name, movie_url, db_url))
            self.ignore_list.append(db_url)

        return movie_list

    def refresh(self):
        soups = []
        contents = []
        for i in (1, 2, 3):
            url = self._url.format(i)
            _LOGGER.debug('url: {}'.format(url))
            soups.append(self._create_soup(url))
        for soup in soups:
            contents += self._get_movie_list(soup)
        if len(contents) != 0:
            self._save_ignore_list()
        return contents

if __name__ == '__main__':
    logging.basicConfig()
    _LOGGER.setLevel(logging.DEBUG)
    m = dysfz()
    items = m.refresh()
    for (name, url, db) in items:
        print('{}\n{}\n{}\n'.format(name, url, db))
    sys.exit()
<commit_msg>update dysfz.py to tell the title changed<commit_after>#! /usr/bin/env python
#-*- coding:utf-8 -*-
#
# movie.py ---
#
# Filename: movie.py
# Description:
# Author: Werther Zhang
# Maintainer:
# Created: Fri Sep 29 13:25:41 2017 (+0800)
#

# Change Log:
#
#

if __name__ == '__main__' and __package__ is None:
    from os import sys, path
    sys.path.append(path.dirname(path.dirname(path.abspath(__file__))))
    import utils.browser as b
else:
    from ...utils import browser as b

from BeautifulSoup import BeautifulSoup

import datetime
import time
import logging
import sys
import os
import re
import hashlib

_LOGGER = logging.getLogger(__name__)
_LOGGER.setLevel(logging.INFO)

def md5sum(contents):
    _LOGGER.debug(contents)
    hash = hashlib.md5()
    hash.update(contents)
    return hash.hexdigest()

class dysfz(object):
    """"""
    IGNORE_LIST='save/dysfz_ignore.txt'
    def __init__(self):
        self._url = 'http://www.dysfz.cc/{}?o=2'
        self.ignore_list = []
        self._load_ignore_list()

    def _load_ignore_list(self):
        if not os.path.exists(dysfz.IGNORE_LIST):
            return
        with open(dysfz.IGNORE_LIST, 'r') as f:
            for l in f.readlines():
                self.ignore_list.append(l[:-1])

    def _save_ignore_list(self):
        if len(self.ignore_list) > 50:
            self.ignore_list = self.ignore_list[15:]
        with open(dysfz.IGNORE_LIST, 'w') as f:
            f.write('\n'.join(self.ignore_list) + '\n')

    def _create_soup(self, url):
        buff = ""
        with b.browser_base_urlopen(url) as f:
            buff = f.read()

        return BeautifulSoup(buff)

    def _get_dbsccore(self, dburl):
        soup = self._create_soup(dburl)
        item = soup.find(attrs={'class':'ll rating_num'})
        dbscore = item.string
        _LOGGER.debug('{}'.format(dbscore))
        if dbscore:
            return float(dbscore)
        return 0.0

    def _get_movie_list(self, soup):
        """MOVEL (name, movie_url, db_url)"""
        movie_list = []
        lis = soup.findAll('li')
        _LOGGER.debug('{}'.format(lis))
        for li in lis:
            # db = li.find(attrs={'class':'dbscore'})
            # if not db:
            #     continue
            # dbscore = float(db.b.string)
            year = datetime.datetime.now().strftime('%Y')
            year = int(year)
            #if today != li.p.span.string:
            #    continue
            _LOGGER.debug('LI {}'.format(li))
            if li.h2 == None:
                continue
            if li.p.find(attrs={'rel':'nofollow', 'target':'_blank'}) == None:
                continue
            movie_url = li.h2.a['href']
            db_url = li.p.find(attrs={'rel':'nofollow', 'target':'_blank'})['href'].encode('utf-8')
            dbscore = self._get_dbsccore(db_url)
            movie_name = '{}({})'.format(li.h2.a.string.encode('utf-8'), dbscore)
            movie_title = li.h2.a.string.encode('utf-8')
            movie_hash = md5sum(movie_title+db_url)
            all_year = re.findall(r'【\d+】', movie_name)
            movie_year = year
            if len(all_year) > 0:
                movie_year = int(all_year[0][3:-3])
                _LOGGER.debug('movie_year: {}'.format(movie_year))

            if movie_hash in self.ignore_list:
                continue

            if dbscore <= 7:
                continue

            if movie_year < year - 1:
                continue

            _LOGGER.debug('dbscore: {}'.format(dbscore))
            _LOGGER.debug('{}'.format(li.p.span.string))
            _LOGGER.debug('{}\n{}\n{}'.format(movie_name, movie_url, db_url))
            movie_list.append((movie_name, movie_url, db_url))
            self.ignore_list.append(movie_hash)

        return movie_list

    def refresh(self):
        soups = []
        contents = []
        for i in (1, 2, 3):
            url = self._url.format(i)
            _LOGGER.debug('url: {}'.format(url))
            soups.append(self._create_soup(url))
        for soup in soups:
            contents += self._get_movie_list(soup)
        if len(contents) != 0:
            self._save_ignore_list()
        return contents

if __name__ == '__main__':
    logging.basicConfig()
    _LOGGER.setLevel(logging.DEBUG)
    m = dysfz()
    items = m.refresh()
    for (name, url, db) in items:
        print('{}\n{}\n{}\n'.format(name, url, db))
    sys.exit()
<|endoftext|>


<commit_before># Copyright 2016 Cloudbase Solutions Srl
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from oslo_policy import policy

from nova.policies import base

BASE_POLICY_NAME = 'os_compute_api:os-cloudpipe'
POLICY_ROOT = 'os_compute_api:os-cloudpipe:%s'

cloudpipe_policies = [
    policy.RuleDefault(
        name=BASE_POLICY_NAME,
        check_str=base.RULE_ADMIN_API),
    policy.RuleDefault(
        name=POLICY_ROOT % 'discoverable',
        check_str=base.RULE_ANY),
]

def list_rules():
    return cloudpipe_policies
<commit_msg>Add description to policies in cloudpipe.py<commit_after># Copyright 2016 Cloudbase Solutions Srl
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from oslo_policy import policy

from nova.policies import base

BASE_POLICY_NAME = 'os_compute_api:os-cloudpipe'
POLICY_ROOT = 'os_compute_api:os-cloudpipe:%s'

cloudpipe_policies = [
    base.create_rule_default(
        BASE_POLICY_NAME,
        base.RULE_ADMIN_API,
        """List, create and update cloud pipes.

os-cloudpipe API is deprecated as this works only with nova-network which \
itself is deprecated.
""",
        [
            {
                'method': 'GET',
                'path': '/os-cloudpipe'
            },
            {
                'method': 'POST',
                'path': '/os-cloudpipe'
            },
            {
                'method': 'PUT',
                'path': '/os-cloudpipe/configure-project'
            }
        ]),
    policy.RuleDefault(
        name=POLICY_ROOT % 'discoverable',
        check_str=base.RULE_ANY),
]

def list_rules():
    return cloudpipe_policies
<|endoftext|>


<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright (c) 2010 Citrix Systems, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""Stubouts, mocks and fixtures for the test suite"""

import contextlib
import random
import sys

from nova.openstack.common import jsonutils
from nova import test
import nova.tests.image.fake
from nova.virt.xenapi import driver as xenapi_conn
from nova.virt.xenapi import fake
from nova.virt.xenapi import vm_utils
from nova.virt.xenapi import vmops

def stubout_firewall_driver(stubs, conn):

    def fake_none(self, *args):
        return

    vmops = conn._vmops
    stubs.Set(vmops.firewall_driver, 'prepare_instance_filter', fake_none)
    stubs.Set(vmops.firewall_driver, 'instance_filter_exists', fake_none)

def stubout_instance_snapshot(stubs):
    def fake_fetch_image(context, session, instance, image, type):
        return {'root': dict(uuid=_make_fake_vdi(), file=None),
                'kernel': dict(uuid=_make_fake_vdi(), file=None),
                'ramdisk': dict(uuid=_make_fake_vdi(), file=None)}

    stubs.Set(vm_utils, '_fetch_image', fake_fetch_image)

    def fake_wait_for_vhd_coalesce(*args):
        #TODO(sirp): Should we actually fake out the data here
        return "fakeparent", "fakebase"

    stubs.Set(vm_utils, '_wait_for_vhd_coalesce', fake_wait_for_vhd_coalesce)

def stubout_session(stubs, cls, product_version=(5, 6, 2), **opt_args):
    """Stubs out methods from XenAPISession"""
    stubs.Set(xenapi_conn.XenAPISession, '_create_session',
              lambda s, url: cls(url, **opt_args))
    stubs.Set(xenapi_conn.XenAPISession, '_get_product_version',
              lambda s: product_version)

def stubout_get_this_vm_uuid(stubs):
    def f():
        vms = [rec['uuid'] for ref, rec
               in fake.get_all_records('VM').iteritems()
               if rec['is_control_domain']]
        return vms[0]
    stubs.Set(vm_utils, 'get_this_vm_uuid', f)

def stubout_image_service_download(stubs):
    def fake_download(*args, **kwargs):
        pass
    stubs.Set(nova.tests.image.fake._FakeImageService,
        'download', fake_download)

def stubout_stream_disk(stubs):
    def fake_stream_disk(*args, **kwargs):
        pass
    stubs.Set(vm_utils, '_stream_disk', fake_stream_disk)

def stubout_is_vdi_pv(stubs):
    def f(_1):
        return False
    stubs.Set(vm_utils, '_is_vdi_pv', f)

def stubout_determine_is_pv_objectstore(stubs):
    """Assumes VMs stu have PV kernels"""

    def f(*args):
        return False
    stubs.Set(vm_utils, '_determine_is_pv_objectstore', f)

def stubout_is_snapshot(stubs):
    """ Always returns true
        xenapi fake driver does not create vmrefs for snapshots """

    def f(*args):
        return True
    stubs.Set(vm_utils, 'is_snapshot', f)

def stubout_lookup_image(stubs):
    """Simulates a failure in lookup image."""
    def f(_1, _2, _3, _4):
        raise Exception("Test Exception raised by fake lookup_image")
    stubs.Set(vm_utils, 'lookup_image', f)

def stubout_fetch_disk_image(stubs, raise_failure=False):
    """Simulates a failure in fetch image_glance_disk."""

    def _fake_fetch_disk_image(context, session, instance, image,
                                      image_type):
        if raise_failure:
            raise fake.Failure("Test Exception raised by "
                               "fake fetch_image_glance_disk")
        elif image_type == vm_utils.ImageType.KERNEL:
            filename = "kernel"
        elif image_type == vm_utils.ImageType.RAMDISK:
            filename = "ramdisk"
        else:
            filename = "unknown"

        vdi_type = vm_utils.ImageType.to_string(image_type)
        return {vdi_type: dict(uuid=None, file=filename)}

    stubs.Set(vm_utils, '_fetch_disk_image', _fake_fetch_disk_image)

def stubout_create_vm(stubs):
    """Simulates a failure in create_vm."""

    def f(*args):
        raise fake.Failure("Test Exception raised by " +
                           "fake create_vm")
    stubs.Set(vm_utils, 'create_vm', f)

def _make_fake_vdi():
    sr_ref = fake.get_all('SR')[0]
    vdi_ref = fake.create_vdi('', sr_ref)
    vdi_rec = fake.get_record('VDI', vdi_ref)
    return vdi_rec['uuid']

class FakeSessionForVMTests(fake.SessionBase):
    """ Stubs out a XenAPISession for VM tests """

    _fake_iptables_save_output = ("# Generated by iptables-save v1.4.10 on "
                                  "Sun Nov  6 22:49:02 2011\n"
                                  "*filter\n"
                                  ":INPUT ACCEPT [0:0]\n"
                                  ":FORWARD ACCEPT [0:0]\n"
                                  ":OUTPUT ACCEPT [0:0]\n"
                                  "COMMIT\n"
                                  "# Completed on Sun Nov  6 22:49:02 2011\n")

    def __init__(self, uri):
        super(FakeSessionForVMTests, self).__init__(uri)

    def host_call_plugin(self, _1, _2, plugin, method, _5):
        if (plugin, method) == ('glance', 'download_vhd'):
            root_uuid = _make_fake_vdi()
            return jsonutils.dumps(dict(root=dict(uuid=root_uuid)))
        elif (plugin, method) == ("xenhost", "iptables_config"):
            return fake.as_json(out=self._fake_iptables_save_output,
                                err='')
        else:
            return (super(FakeSessionForVMTests, self).
                    host_call_plugin(_1, _2, plugin, method, _5))

    def host_call_plugin_swap(self, _1, _2, plugin, method, _5):
        if (plugin, method) == ('glance', 'download_vhd'):
            root_uuid = _make_fake_vdi()
            swap_uuid = _make_fake_vdi()
            return jsonutils.dumps(dict(root=dict(uuid=root_uuid),
                                        swap=dict(uuid=swap_uuid)))
        else:
            return (super(FakeSessionForVMTests, self).
                    host_call_plugin(_1, _2, plugin, method, _5))

    def VM_start(self, _1, ref, _2, _3):
        vm = fake.get_record('VM', ref)
        if vm['power_state'] != 'Halted':
            raise fake.Failure(['VM_BAD_POWER_STATE', ref, 'Halted',
                                vm['power_state']])
        vm['power_state'] = 'Running'
        vm['is_a_template'] = False
        vm['is_control_domain'] = False
        vm['domid'] = random.randrange(1, 1 << 16)
        return vm

    def VM_start_on(self, _1, vm_ref, host_ref, _2, _3):
        vm_rec = self.VM_start(_1, vm_ref, _2, _3)
        vm_rec['resident_on'] = host_ref

    def VM_snapshot(self, session_ref, vm_ref, label):
        status = "Running"
        template_vm_ref = fake.create_vm(label, status, is_a_template=True,
            is_control_domain=False)

        sr_ref = "fakesr"
        template_vdi_ref = fake.create_vdi(label, sr_ref, read_only=True)

        template_vbd_ref = fake.create_vbd(template_vm_ref, template_vdi_ref)
        return template_vm_ref

    def SR_scan(self, session_ref, sr_ref):
        pass

class FakeSessionForFirewallTests(FakeSessionForVMTests):
    """ Stubs out a XenApi Session for doing IPTable Firewall tests """

    def __init__(self, uri, test_case=None):
        super(FakeSessionForFirewallTests, self).__init__(uri)
        if hasattr(test_case, '_in_filter_rules'):
            self._in_filter_rules = test_case._in_filter_rules
        if hasattr(test_case, '_in6_filter_rules'):
            self._in6_filter_rules = test_case._in6_filter_rules
        if hasattr(test_case, '_in_nat_rules'):
            self._in_nat_rules = test_case._in_nat_rules
        self._test_case = test_case

    def host_call_plugin(self, _1, _2, plugin, method, args):
        """Mock method four host_call_plugin to be used in unit tests
           for the dom0 iptables Firewall drivers for XenAPI

        """
        if plugin == "xenhost" and method == "iptables_config":
            # The command to execute is a json-encoded list
            cmd_args = args.get('cmd_args', None)
            cmd = jsonutils.loads(cmd_args)
            if not cmd:
                ret_str = ''
            else:
                output = ''
                process_input = args.get('process_input', None)
                if cmd == ['ip6tables-save', '-t', 'filter']:
                    output = '\n'.join(self._in6_filter_rules)
                if cmd == ['iptables-save', '-t', 'filter']:
                    output = '\n'.join(self._in_filter_rules)
                if cmd == ['iptables-save', '-t', 'nat']:
                    output = '\n'.join(self._in_nat_rules)
                if cmd == ['iptables-restore', ]:
                    lines = process_input.split('\n')
                    if '*filter' in lines:
                        if self._test_case is not None:
                            self._test_case._out_rules = lines
                        output = '\n'.join(lines)
                if cmd == ['ip6tables-restore', ]:
                    lines = process_input.split('\n')
                    if '*filter' in lines:
                        output = '\n'.join(lines)
                ret_str = fake.as_json(out=output, err='')
        return ret_str

def stub_out_vm_methods(stubs):
    def fake_acquire_bootlock(self, vm):
        pass

    def fake_release_bootlock(self, vm):
        pass

    def fake_generate_ephemeral(*args):
        pass

    def fake_wait_for_device(dev):
        pass

    stubs.Set(vmops.VMOps, "_acquire_bootlock", fake_acquire_bootlock)
    stubs.Set(vmops.VMOps, "_release_bootlock", fake_release_bootlock)
    stubs.Set(vm_utils, 'generate_ephemeral', fake_generate_ephemeral)
    stubs.Set(vm_utils, '_wait_for_device', fake_wait_for_device)

class FakeSessionForVolumeTests(fake.SessionBase):
    """ Stubs out a XenAPISession for Volume tests """
    def __init__(self, uri):
        super(FakeSessionForVolumeTests, self).__init__(uri)

    def VDI_introduce(self, _1, uuid, _2, _3, _4, _5,
                      _6, _7, _8, _9, _10, _11):
        valid_vdi = False
        refs = fake.get_all('VDI')
        for ref in refs:
            rec = fake.get_record('VDI', ref)
            if rec['uuid'] == uuid:
                valid_vdi = True
        if not valid_vdi:
            raise fake.Failure([['INVALID_VDI', 'session', self._session]])

class FakeSessionForVolumeFailedTests(FakeSessionForVolumeTests):
    """ Stubs out a XenAPISession for Volume tests: it injects failures """
    def __init__(self, uri):
        super(FakeSessionForVolumeFailedTests, self).__init__(uri)

    def VDI_introduce(self, _1, uuid, _2, _3, _4, _5,
                      _6, _7, _8, _9, _10, _11):
        # This is for testing failure
        raise fake.Failure([['INVALID_VDI', 'session', self._session]])

    def PBD_unplug(self, _1, ref):
        rec = fake.get_record('PBD', ref)
        rec['currently-attached'] = False

    def SR_forget(self, _1, ref):
        pass

def stub_out_migration_methods(stubs):
    @contextlib.contextmanager
    def fake_snapshot_attached_here(session, instance, vm_ref, label):
        yield dict(image='foo', snap='bar')

    def fake_move_disks(self, instance, disk_info):
        vdi_ref = fake.create_vdi(instance['name'], 'fake')
        vdi_rec = fake.get_record('VDI', vdi_ref)
        vdi_rec['other_config']['nova_disk_type'] = 'root'
        return {'uuid': vdi_rec['uuid'], 'ref': vdi_ref}

    def fake_get_vdi(session, vm_ref):
        vdi_ref = fake.create_vdi('derp', 'herp')
        vdi_rec = session.call_xenapi("VDI.get_record", vdi_ref)
        return vdi_ref, {'uuid': vdi_rec['uuid'], }

    def fake_sr(session, *args):
        pass

    def fake_get_sr_path(*args):
        return "fake"

    def fake_destroy(*args, **kwargs):
        pass

    def fake_generate_ephemeral(*args):
        pass

    stubs.Set(vmops.VMOps, '_destroy', fake_destroy)
    stubs.Set(vmops.VMOps, '_move_disks', fake_move_disks)
    stubs.Set(vm_utils, 'scan_default_sr', fake_sr)
    stubs.Set(vm_utils, '_scan_sr', fake_sr)
    stubs.Set(vm_utils, 'snapshot_attached_here', fake_snapshot_attached_here)
    stubs.Set(vm_utils, 'get_vdi_for_vm_safely', fake_get_vdi)
    stubs.Set(vm_utils, 'get_sr_path', fake_get_sr_path)
    stubs.Set(vm_utils, 'generate_ephemeral', fake_generate_ephemeral)

class XenAPITestBase(test.TestCase):
    def setUp(self):
        super(XenAPITestBase, self).setUp()

        self.orig_XenAPI = sys.modules.get('XenAPI')
        sys.modules['XenAPI'] = fake

        fake.reset()

    def tearDown(self):
        if self.orig_XenAPI is not None:
            sys.modules['XenAPI'] = self.orig_XenAPI
            self.orig_XenAPI = None
        else:
            sys.modules.pop('XenAPI')

        super(XenAPITestBase, self).tearDown()
<commit_msg>Remove VDI chain limit for snapshots.<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright (c) 2010 Citrix Systems, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""Stubouts, mocks and fixtures for the test suite"""

import contextlib
import random
import sys

from nova.openstack.common import jsonutils
from nova import test
import nova.tests.image.fake
from nova.virt.xenapi import driver as xenapi_conn
from nova.virt.xenapi import fake
from nova.virt.xenapi import vm_utils
from nova.virt.xenapi import vmops

def stubout_firewall_driver(stubs, conn):

    def fake_none(self, *args):
        return

    vmops = conn._vmops
    stubs.Set(vmops.firewall_driver, 'prepare_instance_filter', fake_none)
    stubs.Set(vmops.firewall_driver, 'instance_filter_exists', fake_none)

def stubout_instance_snapshot(stubs):
    def fake_fetch_image(context, session, instance, image, type):
        return {'root': dict(uuid=_make_fake_vdi(), file=None),
                'kernel': dict(uuid=_make_fake_vdi(), file=None),
                'ramdisk': dict(uuid=_make_fake_vdi(), file=None)}

    stubs.Set(vm_utils, '_fetch_image', fake_fetch_image)

    def fake_wait_for_vhd_coalesce(*args):
        #TODO(sirp): Should we actually fake out the data here
        return "fakeparent", "fakebase"

    stubs.Set(vm_utils, '_wait_for_vhd_coalesce', fake_wait_for_vhd_coalesce)

def stubout_session(stubs, cls, product_version=(5, 6, 2), **opt_args):
    """Stubs out methods from XenAPISession"""
    stubs.Set(xenapi_conn.XenAPISession, '_create_session',
              lambda s, url: cls(url, **opt_args))
    stubs.Set(xenapi_conn.XenAPISession, '_get_product_version',
              lambda s: product_version)

def stubout_get_this_vm_uuid(stubs):
    def f():
        vms = [rec['uuid'] for ref, rec
               in fake.get_all_records('VM').iteritems()
               if rec['is_control_domain']]
        return vms[0]
    stubs.Set(vm_utils, 'get_this_vm_uuid', f)

def stubout_image_service_download(stubs):
    def fake_download(*args, **kwargs):
        pass
    stubs.Set(nova.tests.image.fake._FakeImageService,
        'download', fake_download)

def stubout_stream_disk(stubs):
    def fake_stream_disk(*args, **kwargs):
        pass
    stubs.Set(vm_utils, '_stream_disk', fake_stream_disk)

def stubout_is_vdi_pv(stubs):
    def f(_1):
        return False
    stubs.Set(vm_utils, '_is_vdi_pv', f)

def stubout_determine_is_pv_objectstore(stubs):
    """Assumes VMs stu have PV kernels"""

    def f(*args):
        return False
    stubs.Set(vm_utils, '_determine_is_pv_objectstore', f)

def stubout_is_snapshot(stubs):
    """ Always returns true
        xenapi fake driver does not create vmrefs for snapshots """

    def f(*args):
        return True
    stubs.Set(vm_utils, 'is_snapshot', f)

def stubout_lookup_image(stubs):
    """Simulates a failure in lookup image."""
    def f(_1, _2, _3, _4):
        raise Exception("Test Exception raised by fake lookup_image")
    stubs.Set(vm_utils, 'lookup_image', f)

def stubout_fetch_disk_image(stubs, raise_failure=False):
    """Simulates a failure in fetch image_glance_disk."""

    def _fake_fetch_disk_image(context, session, instance, image,
                                      image_type):
        if raise_failure:
            raise fake.Failure("Test Exception raised by "
                               "fake fetch_image_glance_disk")
        elif image_type == vm_utils.ImageType.KERNEL:
            filename = "kernel"
        elif image_type == vm_utils.ImageType.RAMDISK:
            filename = "ramdisk"
        else:
            filename = "unknown"

        vdi_type = vm_utils.ImageType.to_string(image_type)
        return {vdi_type: dict(uuid=None, file=filename)}

    stubs.Set(vm_utils, '_fetch_disk_image', _fake_fetch_disk_image)

def stubout_create_vm(stubs):
    """Simulates a failure in create_vm."""

    def f(*args):
        raise fake.Failure("Test Exception raised by " +
                           "fake create_vm")
    stubs.Set(vm_utils, 'create_vm', f)

def _make_fake_vdi():
    sr_ref = fake.get_all('SR')[0]
    vdi_ref = fake.create_vdi('', sr_ref)
    vdi_rec = fake.get_record('VDI', vdi_ref)
    return vdi_rec['uuid']

class FakeSessionForVMTests(fake.SessionBase):
    """ Stubs out a XenAPISession for VM tests """

    _fake_iptables_save_output = ("# Generated by iptables-save v1.4.10 on "
                                  "Sun Nov  6 22:49:02 2011\n"
                                  "*filter\n"
                                  ":INPUT ACCEPT [0:0]\n"
                                  ":FORWARD ACCEPT [0:0]\n"
                                  ":OUTPUT ACCEPT [0:0]\n"
                                  "COMMIT\n"
                                  "# Completed on Sun Nov  6 22:49:02 2011\n")

    def __init__(self, uri):
        super(FakeSessionForVMTests, self).__init__(uri)

    def host_call_plugin(self, _1, _2, plugin, method, _5):
        if (plugin, method) == ('glance', 'download_vhd'):
            root_uuid = _make_fake_vdi()
            return jsonutils.dumps(dict(root=dict(uuid=root_uuid)))
        elif (plugin, method) == ("xenhost", "iptables_config"):
            return fake.as_json(out=self._fake_iptables_save_output,
                                err='')
        else:
            return (super(FakeSessionForVMTests, self).
                    host_call_plugin(_1, _2, plugin, method, _5))

    def host_call_plugin_swap(self, _1, _2, plugin, method, _5):
        if (plugin, method) == ('glance', 'download_vhd'):
            root_uuid = _make_fake_vdi()
            swap_uuid = _make_fake_vdi()
            return jsonutils.dumps(dict(root=dict(uuid=root_uuid),
                                        swap=dict(uuid=swap_uuid)))
        else:
            return (super(FakeSessionForVMTests, self).
                    host_call_plugin(_1, _2, plugin, method, _5))

    def VM_start(self, _1, ref, _2, _3):
        vm = fake.get_record('VM', ref)
        if vm['power_state'] != 'Halted':
            raise fake.Failure(['VM_BAD_POWER_STATE', ref, 'Halted',
                                vm['power_state']])
        vm['power_state'] = 'Running'
        vm['is_a_template'] = False
        vm['is_control_domain'] = False
        vm['domid'] = random.randrange(1, 1 << 16)
        return vm

    def VM_start_on(self, _1, vm_ref, host_ref, _2, _3):
        vm_rec = self.VM_start(_1, vm_ref, _2, _3)
        vm_rec['resident_on'] = host_ref

    def VM_snapshot(self, session_ref, vm_ref, label):
        status = "Running"
        template_vm_ref = fake.create_vm(label, status, is_a_template=True,
            is_control_domain=False)

        sr_ref = "fakesr"
        template_vdi_ref = fake.create_vdi(label, sr_ref, read_only=True)

        template_vbd_ref = fake.create_vbd(template_vm_ref, template_vdi_ref)
        return template_vm_ref

    def SR_scan(self, session_ref, sr_ref):
        pass

class FakeSessionForFirewallTests(FakeSessionForVMTests):
    """ Stubs out a XenApi Session for doing IPTable Firewall tests """

    def __init__(self, uri, test_case=None):
        super(FakeSessionForFirewallTests, self).__init__(uri)
        if hasattr(test_case, '_in_filter_rules'):
            self._in_filter_rules = test_case._in_filter_rules
        if hasattr(test_case, '_in6_filter_rules'):
            self._in6_filter_rules = test_case._in6_filter_rules
        if hasattr(test_case, '_in_nat_rules'):
            self._in_nat_rules = test_case._in_nat_rules
        self._test_case = test_case

    def host_call_plugin(self, _1, _2, plugin, method, args):
        """Mock method four host_call_plugin to be used in unit tests
           for the dom0 iptables Firewall drivers for XenAPI

        """
        if plugin == "xenhost" and method == "iptables_config":
            # The command to execute is a json-encoded list
            cmd_args = args.get('cmd_args', None)
            cmd = jsonutils.loads(cmd_args)
            if not cmd:
                ret_str = ''
            else:
                output = ''
                process_input = args.get('process_input', None)
                if cmd == ['ip6tables-save', '-t', 'filter']:
                    output = '\n'.join(self._in6_filter_rules)
                if cmd == ['iptables-save', '-t', 'filter']:
                    output = '\n'.join(self._in_filter_rules)
                if cmd == ['iptables-save', '-t', 'nat']:
                    output = '\n'.join(self._in_nat_rules)
                if cmd == ['iptables-restore', ]:
                    lines = process_input.split('\n')
                    if '*filter' in lines:
                        if self._test_case is not None:
                            self._test_case._out_rules = lines
                        output = '\n'.join(lines)
                if cmd == ['ip6tables-restore', ]:
                    lines = process_input.split('\n')
                    if '*filter' in lines:
                        output = '\n'.join(lines)
                ret_str = fake.as_json(out=output, err='')
        return ret_str

def stub_out_vm_methods(stubs):
    def fake_acquire_bootlock(self, vm):
        pass

    def fake_release_bootlock(self, vm):
        pass

    def fake_generate_ephemeral(*args):
        pass

    def fake_wait_for_device(dev):
        pass

    stubs.Set(vmops.VMOps, "_acquire_bootlock", fake_acquire_bootlock)
    stubs.Set(vmops.VMOps, "_release_bootlock", fake_release_bootlock)
    stubs.Set(vm_utils, 'generate_ephemeral', fake_generate_ephemeral)
    stubs.Set(vm_utils, '_wait_for_device', fake_wait_for_device)

class FakeSessionForVolumeTests(fake.SessionBase):
    """ Stubs out a XenAPISession for Volume tests """
    def __init__(self, uri):
        super(FakeSessionForVolumeTests, self).__init__(uri)

    def VDI_introduce(self, _1, uuid, _2, _3, _4, _5,
                      _6, _7, _8, _9, _10, _11):
        valid_vdi = False
        refs = fake.get_all('VDI')
        for ref in refs:
            rec = fake.get_record('VDI', ref)
            if rec['uuid'] == uuid:
                valid_vdi = True
        if not valid_vdi:
            raise fake.Failure([['INVALID_VDI', 'session', self._session]])

class FakeSessionForVolumeFailedTests(FakeSessionForVolumeTests):
    """ Stubs out a XenAPISession for Volume tests: it injects failures """
    def __init__(self, uri):
        super(FakeSessionForVolumeFailedTests, self).__init__(uri)

    def VDI_introduce(self, _1, uuid, _2, _3, _4, _5,
                      _6, _7, _8, _9, _10, _11):
        # This is for testing failure
        raise fake.Failure([['INVALID_VDI', 'session', self._session]])

    def PBD_unplug(self, _1, ref):
        rec = fake.get_record('PBD', ref)
        rec['currently-attached'] = False

    def SR_forget(self, _1, ref):
        pass

def stub_out_migration_methods(stubs):
    @contextlib.contextmanager
    def fake_snapshot_attached_here(session, instance, vm_ref, label):
        yield ['bar', 'foo']

    def fake_move_disks(self, instance, disk_info):
        vdi_ref = fake.create_vdi(instance['name'], 'fake')
        vdi_rec = fake.get_record('VDI', vdi_ref)
        vdi_rec['other_config']['nova_disk_type'] = 'root'
        return {'uuid': vdi_rec['uuid'], 'ref': vdi_ref}

    def fake_get_vdi(session, vm_ref):
        vdi_ref = fake.create_vdi('derp', 'herp')
        vdi_rec = session.call_xenapi("VDI.get_record", vdi_ref)
        return vdi_ref, {'uuid': vdi_rec['uuid'], }

    def fake_sr(session, *args):
        pass

    def fake_get_sr_path(*args):
        return "fake"

    def fake_destroy(*args, **kwargs):
        pass

    def fake_generate_ephemeral(*args):
        pass

    stubs.Set(vmops.VMOps, '_destroy', fake_destroy)
    stubs.Set(vmops.VMOps, '_move_disks', fake_move_disks)
    stubs.Set(vm_utils, 'scan_default_sr', fake_sr)
    stubs.Set(vm_utils, '_scan_sr', fake_sr)
    stubs.Set(vm_utils, 'snapshot_attached_here', fake_snapshot_attached_here)
    stubs.Set(vm_utils, 'get_vdi_for_vm_safely', fake_get_vdi)
    stubs.Set(vm_utils, 'get_sr_path', fake_get_sr_path)
    stubs.Set(vm_utils, 'generate_ephemeral', fake_generate_ephemeral)

class XenAPITestBase(test.TestCase):
    def setUp(self):
        super(XenAPITestBase, self).setUp()

        self.orig_XenAPI = sys.modules.get('XenAPI')
        sys.modules['XenAPI'] = fake

        fake.reset()

    def tearDown(self):
        if self.orig_XenAPI is not None:
            sys.modules['XenAPI'] = self.orig_XenAPI
            self.orig_XenAPI = None
        else:
            sys.modules.pop('XenAPI')

        super(XenAPITestBase, self).tearDown()
<|endoftext|>


<commit_before>def incident_edge_label(graph_list=None, output_attribute='type', separator='', level=1):
    """
    level: int
    level=1 considers all incident edges
    level=2 considers all edges incident on the neighbors
    """
    for g in graph_list:
        # iterate over nodes
        for n, d in g.nodes_iter(data=True):
            # for all neighbors
            edge_labels = []
            if level == 1:
                edge_labels += [ed.get('label', 'N/A') for u, v, ed in g.edges_iter(n, data=True)]
            elif level == 2:
                neighbors = g.neighbors(n)
                for nn in neighbors:
                    # extract list of edge labels
                    edge_labels += [ed.get('label', 'N/A') for u, v, ed in g.edges_iter(nn, data=True)]
            else:
                raise Exception('Unknown level: %s' % level)
            # consider the sorted serialization of all labels as a type
            vertex_type = separator.join(sorted(edge_labels))
            g.node[n][output_attribute] = vertex_type
        yield g

def incident_node_label(graph_list=None, output_attribute='type', separator='', level=1):
    """
    level: int
    level=1 considers all incident nodes
    level=2 considers all nodes incident on the neighbors
    """
    for g in graph_list:
        # iterate over nodes
        for n, d in g.nodes_iter(data=True):
            # for all neighbors
            node_labels = []
            if level == 1:
                node_labels += [g.node[u].get('label', 'N/A') for u in g.neighbors(n)]
            elif level == 2:
                neighbors = g.neighbors(n)
                for nn in neighbors:
                    # extract list of labels
                    node_labels += [g.node[u].get('label', 'N/A') for u in g.neighbors(nn)]
            else:
                raise Exception('Unknown level: %s' % level)
            # consider the sorted serialization of all labels as a type
            vertex_type = separator.join(sorted(node_labels))
            g.node[n][output_attribute] = vertex_type
        yield g

def translate(graph_list=None, input_attribute='label', output_attribute='label', label_map=dict(), default=' '):
    original_attribute = input_attribute + '_original'
    for g in graph_list:
        # iterate over nodes
        for n, d in g.nodes_iter(data=True):
            key = d.get(input_attribute, default)
            g.node[n][original_attribute] = key
            mapped_attribute = label_map.get(key, default)
            g.node[n][output_attribute] = mapped_attribute
        yield g

def colorize(graph_list=None, output_attribute='level', labels=['A', 'U', 'C', 'G']):
    values = np.linspace(0.0, 1.0, num=len(labels))
    color_dict = dict(zip(labels, values))
    for g in graph_list:
        # iterate over nodes
        for n, d in g.nodes_iter(data=True):
            g.node[n][output_attribute] = color_dict.get(d['label'], 0)
        yield g

def colorize_binary(graph_list=None, output_attribute='color_value', input_attribute='weight', level=0):
    for g in graph_list:
        # iterate over nodes
        for n, d in g.nodes_iter(data=True):
            val = d.get(input_attribute, 0)
            if val <= level:
                color_value = 0
            else:
                color_value = 1
            g.node[n][output_attribute] = color_value
        yield g

def trapezoidal_reweighting(graph_list=None, high_weight=1.0, low_weight=0.1, high_weight_window_start=0, high_weight_window_end=1, low_weight_window_start=0, low_weight_window_end=1):
    """
    Piece wise linear weight function between two levels with specified start end positions.
    high   ___
    low __/   \__
    """
# assert high_ weight > low_weight
    if high_weight < low_weight:
        raise Exception('high_weight (%f) must be higher than low_weight (%f)' % (high_weight, low_weight))

    # assert low_weight boundaries includes high_weight boundaries
    if high_weight_window_start > low_weight_window_end:
        raise Exception('high_weight_window_start (%d) must be lower than low_weight_window_end (%d)' %
                        (high_weight_window_start, low_weight_window_end))
    if high_weight_window_start < low_weight_window_start:
        raise Exception('high_weight_window_start (%d) must be higher than low_weight_window_start (%d)' %
                        (high_weight_window_start, low_weight_window_start))
    if high_weight_window_end < low_weight_window_start:
        raise Exception('high_weight_window_end (%d) must be higher than low_weight_window_start (%d)' %
                        (high_weight_window_end, low_weight_window_start))
    if high_weight_window_end > low_weight_window_end:
        raise Exception('high_weight_window_end (%d) must be higher than low_weight_window_end (%d)' %
                        (high_weight_window_end, low_weight_window_end))

    for g in graph_list:
        # iterate over nodes
        for n, d in g.nodes_iter(data=True):
            if 'position' not in d:
                # assert nodes must have position attribute
                raise Exception('Nodes must have "position" attribute')
            # given the 'position' attribute of node assign weight according to piece wise linear weight function between two levels
            pos = d['position']
            if pos < low_weight_window_start:
                """
                   ___
                __/   \__
                |
                """
                g.node[n]["weight"] = low_weight
            elif pos >= low_weight_window_start and pos < high_weight_window_start:
                """
                   ___
                __/   \__
                  |
                """
                g.node[n]["weight"] = (high_weight - low_weight) / (high_weight_window_start - low_weight_window_start) * \
                    (pos - low_weight_window_start) + low_weight
            elif pos >= high_weight_window_start and pos < high_weight_window_end:
                """
                   ___
                __/   \__
                    |
                """
                g.node[n]["weight"] = high_weight
            elif pos >= high_weight_window_end and pos < low_weight_window_end:
                """
                   ___
                __/   \__
                      |
                """
                g.node[n]["weight"] = high_weight - \
                    (high_weight - low_weight) / (low_weight_window_end - high_weight_window_end) * (pos - high_weight_window_end)
            else:
                """
                   ___
                __/   \__
                        |
                """
                g.node[n]["weight"] = low_weight
        yield g
<commit_msg>cleanup<commit_after>import numpy as np

def incident_edge_label(graph_list=None, output_attribute='type', separator='', level=1):
    """
    level: int
    level=1 considers all incident edges
    level=2 considers all edges incident on the neighbors
    """
    for g in graph_list:
        # iterate over nodes
        for n, d in g.nodes_iter(data=True):
            # for all neighbors
            edge_labels = []
            if level == 1:
                edge_labels += [ed.get('label', 'N/A') for u, v, ed in g.edges_iter(n, data=True)]
            elif level == 2:
                neighbors = g.neighbors(n)
                for nn in neighbors:
                    # extract list of edge labels
                    edge_labels += [ed.get('label', 'N/A') for u, v, ed in g.edges_iter(nn, data=True)]
            else:
                raise Exception('Unknown level: %s' % level)
            # consider the sorted serialization of all labels as a type
            vertex_type = separator.join(sorted(edge_labels))
            g.node[n][output_attribute] = vertex_type
        yield g

def incident_node_label(graph_list=None, output_attribute='type', separator='', level=1):
    """
    level: int
    level=1 considers all incident nodes
    level=2 considers all nodes incident on the neighbors
    """
    for g in graph_list:
        # iterate over nodes
        for n, d in g.nodes_iter(data=True):
            # for all neighbors
            node_labels = []
            if level == 1:
                node_labels += [g.node[u].get('label', 'N/A') for u in g.neighbors(n)]
            elif level == 2:
                neighbors = g.neighbors(n)
                for nn in neighbors:
                    # extract list of labels
                    node_labels += [g.node[u].get('label', 'N/A') for u in g.neighbors(nn)]
            else:
                raise Exception('Unknown level: %s' % level)
            # consider the sorted serialization of all labels as a type
            vertex_type = separator.join(sorted(node_labels))
            g.node[n][output_attribute] = vertex_type
        yield g

def translate(graph_list=None, input_attribute='label', output_attribute='label', label_map=dict(), default=' '):
    original_attribute = input_attribute + '_original'
    for g in graph_list:
        # iterate over nodes
        for n, d in g.nodes_iter(data=True):
            key = d.get(input_attribute, default)
            g.node[n][original_attribute] = key
            mapped_attribute = label_map.get(key, default)
            g.node[n][output_attribute] = mapped_attribute
        yield g

def colorize(graph_list=None, output_attribute='level', labels=['A', 'U', 'C', 'G']):
    values = np.linspace(0.0, 1.0, num=len(labels))
    color_dict = dict(zip(labels, values))
    for g in graph_list:
        # iterate over nodes
        for n, d in g.nodes_iter(data=True):
            g.node[n][output_attribute] = color_dict.get(d['label'], 0)
        yield g

def colorize_binary(graph_list=None, output_attribute='color_value', input_attribute='weight', level=0):
    for g in graph_list:
        # iterate over nodes
        for n, d in g.nodes_iter(data=True):
            val = d.get(input_attribute, 0)
            if val <= level:
                color_value = 0
            else:
                color_value = 1
            g.node[n][output_attribute] = color_value
        yield g

def trapezoidal_reweighting(graph_list=None, high_weight=1.0, low_weight=0.1, high_weight_window_start=0, high_weight_window_end=1, low_weight_window_start=0, low_weight_window_end=1):
    """
    Piece wise linear weight function between two levels with specified start end positions.
    high   ___
    low __/   \__
    """
# assert high_ weight > low_weight
    if high_weight < low_weight:
        raise Exception('high_weight (%f) must be higher than low_weight (%f)' % (high_weight, low_weight))

    # assert low_weight boundaries includes high_weight boundaries
    if high_weight_window_start > low_weight_window_end:
        raise Exception('high_weight_window_start (%d) must be lower than low_weight_window_end (%d)' %
                        (high_weight_window_start, low_weight_window_end))
    if high_weight_window_start < low_weight_window_start:
        raise Exception('high_weight_window_start (%d) must be higher than low_weight_window_start (%d)' %
                        (high_weight_window_start, low_weight_window_start))
    if high_weight_window_end < low_weight_window_start:
        raise Exception('high_weight_window_end (%d) must be higher than low_weight_window_start (%d)' %
                        (high_weight_window_end, low_weight_window_start))
    if high_weight_window_end > low_weight_window_end:
        raise Exception('high_weight_window_end (%d) must be higher than low_weight_window_end (%d)' %
                        (high_weight_window_end, low_weight_window_end))

    for g in graph_list:
        # iterate over nodes
        for n, d in g.nodes_iter(data=True):
            if 'position' not in d:
                # assert nodes must have position attribute
                raise Exception('Nodes must have "position" attribute')
            # given the 'position' attribute of node assign weight according to piece wise linear weight function between two levels
            pos = d['position']
            if pos < low_weight_window_start:
                """
                   ___
                __/   \__
                |
                """
                g.node[n]["weight"] = low_weight
            elif pos >= low_weight_window_start and pos < high_weight_window_start:
                """
                   ___
                __/   \__
                  |
                """
                g.node[n]["weight"] = (high_weight - low_weight) / (high_weight_window_start - low_weight_window_start) * \
                    (pos - low_weight_window_start) + low_weight
            elif pos >= high_weight_window_start and pos < high_weight_window_end:
                """
                   ___
                __/   \__
                    |
                """
                g.node[n]["weight"] = high_weight
            elif pos >= high_weight_window_end and pos < low_weight_window_end:
                """
                   ___
                __/   \__
                      |
                """
                g.node[n]["weight"] = high_weight - \
                    (high_weight - low_weight) / (low_weight_window_end - high_weight_window_end) * (pos - high_weight_window_end)
            else:
                """
                   ___
                __/   \__
                        |
                """
                g.node[n]["weight"] = low_weight
        yield g
<|endoftext|>


<commit_before>import unittest
"""
Given two binary arrays arr1[] and arr2[] of same size n, find the length of longest common span (i, j)
where j >= i such that arr1[i] + arr1[i+1] + ... + arr1[j] = arr2[i] + arr2[i+1] + ... + arr2[j].
Input: arr1[]: 0 1 0 0 0 0
       arr2[]: 1 0 1 0 0 1
Output: 4
The longest span is from index 1 to 4.
"""

"""
Approach:
1. We find the prefix sums in the two arrays and calculate their difference.
2. Consider the start and end of any span with same sum.
3. The difference of prefix sums at start and end will be the same. This is because the span has same
   sum in both the arrays and hence, does not change the difference from start to end.
4. Since both binary arrays are of same length, difference in sums can vary from -n to n ie 2n+1 values.
5. If prefix difference is zero, then max_length is current_index+1.
6. If prefix difference is encountered for first time, we store the current_index.
7. If prefix difference is encountered before, we compute difference between stored value and current_index.
"""

def longest_span_with_same_sum(first_bin_array, second_bin_array):

    assert len(first_bin_array) == len(second_bin_array)
    end = len(first_bin_array)
    difference_list = [-1] * (2*end + 1)
    first_prefix_sum = 0
    second_prefix_sum = 0
    max_length = 0

    for j in range(end):
        first_prefix_sum += first_bin_array[j]
        second_prefix_sum += second_bin_array[j]
        difference = end + (first_prefix_sum - second_prefix_sum)
        if difference == 0:
            max_length = j+1
        if difference_list[difference] == -1:
            difference_list[difference] = j
        else:
            i = difference_list[difference]
            if j-i > max_length:
                max_length = j-i

    return max_length

class TestLongestSpan(unittest.TestCase):

    def test_longest_span(self):
        first_bin_array = [0, 1, 0, 1, 1, 1, 1]
        second_bin_array = [1, 1, 1, 1, 1, 0, 1]
        self.assertEqual(longest_span_with_same_sum(first_bin_array, second_bin_array), 6)
        first_bin_array = [0, 0, 1, 0]
        second_bin_array = [1, 1, 1, 1]
        self.assertEqual(longest_span_with_same_sum(first_bin_array, second_bin_array), 1)
<commit_msg>--Refactor<commit_after>import unittest
"""
Given two binary arrays arr1[] and arr2[] of same size n, find the length of longest common span (i, j)
where j >= i such that arr1[i] + arr1[i+1] + ... + arr1[j] = arr2[i] + arr2[i+1] + ... + arr2[j].
Input: arr1[]: 0 1 0 0 0 0
       arr2[]: 1 0 1 0 0 1
Output: 4
The longest span is from index 1 to 4.
"""

"""
Approach:
1. We find the prefix sums in the two arrays and calculate their difference.
2. Consider the start and end of any span with same sum.
3. The difference of prefix sums at start and end will be the same. This is because the span has same
   sum in both the arrays and hence, does not change the difference from start to end.
4. Since both binary arrays are of same length, difference in sums can vary from -n to n ie 2n+1 values.
5. If prefix difference is zero, then max_length is current_index+1 # span goes from 0 to current_index+1
6. If prefix difference is encountered for first time, we store the current_index.
7. If prefix difference is encountered before, we compute difference between stored value and current_index.
"""

def longest_span_with_same_sum(first_bin_array, second_bin_array):

    assert len(first_bin_array) == len(second_bin_array)
    end = len(first_bin_array)
    difference_list = [-1] * (2*end + 1)
    first_prefix_sum = 0
    second_prefix_sum = 0
    max_length = 0

    for j in range(end):
        first_prefix_sum += first_bin_array[j]
        second_prefix_sum += second_bin_array[j]
        difference = end + (first_prefix_sum - second_prefix_sum)
        if difference == 0:
            max_length = j+1
        if difference_list[difference] == -1:
            difference_list[difference] = j
        else:
            i = difference_list[difference]
            if j-i > max_length:
                max_length = j-i

    return max_length

class TestLongestSpan(unittest.TestCase):

    def test_longest_span(self):
        first_bin_array = [0, 1, 0, 1, 1, 1, 1]
        second_bin_array = [1, 1, 1, 1, 1, 0, 1]
        self.assertEqual(longest_span_with_same_sum(first_bin_array, second_bin_array), 6)
        first_bin_array = [0, 0, 1, 0]
        second_bin_array = [1, 1, 1, 1]
        self.assertEqual(longest_span_with_same_sum(first_bin_array, second_bin_array), 1)
<|endoftext|>


<commit_before>##
# Copyright (c) 2011-2012 Sprymix Inc.
# All rights reserved.
#
# See LICENSE for details.
##

import imp
import importlib
import os.path
import sys

from semantix.utils.algos import topological

from .context import ImportContext
from . import module as module_types

def cache_from_source(path, debug_override=None, cache_ext=None):
    cachepath = imp.cache_from_source(path, debug_override)

    if cache_ext is not None:
        pathext = cache_ext
    else:
        pathpre, pathext = os.path.splitext(path)

    if pathext != '.py' or cache_ext is not None:
        cachepre, cacheext = os.path.splitext(cachepath)
        cachepath = cachepre + pathext + 'c'

    return cachepath

def source_from_cache(path):
    cachepath = path
    path = imp.source_from_cache(cachepath)

    cachepre, cacheext = os.path.splitext(cachepath)
    if cacheext != '.pyc':
        pathpre, pathext = os.path.splitext(path)
        path = pathpre + cacheext[:-1]

    return path

def import_module(full_module_name, *, loader=None):
    if loader is not None:
        if isinstance(full_module_name, ImportContext):
            context = full_module_name.__class__.copy(full_module_name)
            context.loader = loader
        else:
            context = ImportContext(full_module_name, loader=loader)

        full_module_name = context

    return importlib.import_module(full_module_name)

def reload(module):
    if isinstance(module, module_types.BaseProxyModule):
        sys.modules[module.__name__] = module.__wrapped__

        # XXX: imp.reload has a hardcoded check that fails on instances of module subclasses
        try:
            new_mod = module.__wrapped__.__loader__.load_module(module.__name__)

            if isinstance(new_mod, module_types.BaseProxyModule):
                module.__wrapped__ = new_mod.__wrapped__
            else:
                module.__wrapped__ = new_mod

        finally:
            sys.modules[module.__name__] = module

        return module

    else:
        return imp.reload(module)

def modules_from_import_statements(package, imports):
    modules = []

    for name, fromlist in imports:
        level = 0

        while level < len(name) and name[level] == '.':
            level += 1

        if level > 0:
            steps = package.rsplit('.', level - 1)
            if len(steps) < level:
                raise ValueError('relative import reaches beyond top-level package')

            suffix = name[level:]

            if suffix:
                fq_name = '{}.{}'.format(steps[0], name[level:])
            else:
                fq_name = steps[0]
        else:
            fq_name = name

        path = None
        steps = fq_name.split('.')

        add_package = True

        for i in range(len(steps)):
            modname = '.'.join(steps[:i + 1])

            loader = importlib.find_loader(modname, path=path)

            if loader is None:
                raise ValueError('could not find loader for module {}'.format(modname))

            if not loader.is_package(modname):
                break

            modfile = loader.get_filename(modname)
            # os.path.dirname(__file__) is a common importlib assumption for __path__
            path = [os.path.dirname(modfile)]
        else:
            if fromlist:
                add_package = False

                for entry in fromlist:
                    modname = '{}.{}'.format(fq_name, entry)
                    entry_loader = importlib.find_loader(modname, path=path)

                    if entry_loader is not None and entry_loader.path != loader.path:
                        modules.append(modname)
                    else:
                        add_package = True

        if add_package:
            modules.append(fq_name)

    return modules

def modified_modules():
    for module in list(sys.modules.values()):
        try:
            loader = module.__loader__
        except AttributeError:
            # Weird custom module, skip
            continue

        try:
            loaded_modver = module.__sx_modversion__
        except AttributeError:
            # Unmanaged module
            continue

        try:
            imports = module.__sx_imports__
        except AttributeError:
            imports = ()

        current_modver = loader.get_module_version(module.__name__, imports)

        if loaded_modver != current_modver:
            yield module

def reload_modified(modified=None):
    if modified is None:
        modified = modified_modules()

    modified = tuple(modified)
    modified_names = {m.__name__ for m in modified}

    modg = {}

    for module in modified:
        imports = set(getattr(module, '__sx_imports__', ())) & modified_names
        modg[module.__name__] = {'item': module, 'deps': imports}

    for module in topological.sort(modg):
        try:
            reload(module)
        except ImportError as e:
            if isinstance(e.__cause__, FileNotFoundError):
                del sys.modules[module.__name__]
            else:
                raise
<commit_msg>utils.lang.import_: Fix modules_from_import_statements to work with namespace loaders<commit_after>##
# Copyright (c) 2011-2012 Sprymix Inc.
# All rights reserved.
#
# See LICENSE for details.
##

import imp
import importlib
import os.path
import sys

from semantix.utils.algos import topological

from .context import ImportContext
from . import module as module_types

def cache_from_source(path, debug_override=None, cache_ext=None):
    cachepath = imp.cache_from_source(path, debug_override)

    if cache_ext is not None:
        pathext = cache_ext
    else:
        pathpre, pathext = os.path.splitext(path)

    if pathext != '.py' or cache_ext is not None:
        cachepre, cacheext = os.path.splitext(cachepath)
        cachepath = cachepre + pathext + 'c'

    return cachepath

def source_from_cache(path):
    cachepath = path
    path = imp.source_from_cache(cachepath)

    cachepre, cacheext = os.path.splitext(cachepath)
    if cacheext != '.pyc':
        pathpre, pathext = os.path.splitext(path)
        path = pathpre + cacheext[:-1]

    return path

def import_module(full_module_name, *, loader=None):
    if loader is not None:
        if isinstance(full_module_name, ImportContext):
            context = full_module_name.__class__.copy(full_module_name)
            context.loader = loader
        else:
            context = ImportContext(full_module_name, loader=loader)

        full_module_name = context

    return importlib.import_module(full_module_name)

def reload(module):
    if isinstance(module, module_types.BaseProxyModule):
        sys.modules[module.__name__] = module.__wrapped__

        # XXX: imp.reload has a hardcoded check that fails on instances of module subclasses
        try:
            new_mod = module.__wrapped__.__loader__.load_module(module.__name__)

            if isinstance(new_mod, module_types.BaseProxyModule):
                module.__wrapped__ = new_mod.__wrapped__
            else:
                module.__wrapped__ = new_mod

        finally:
            sys.modules[module.__name__] = module

        return module

    else:
        return imp.reload(module)

def modules_from_import_statements(package, imports):
    modules = []

    for name, fromlist in imports:
        level = 0

        while level < len(name) and name[level] == '.':
            level += 1

        if level > 0:
            steps = package.rsplit('.', level - 1)
            if len(steps) < level:
                raise ValueError('relative import reaches beyond top-level package')

            suffix = name[level:]

            if suffix:
                fq_name = '{}.{}'.format(steps[0], name[level:])
            else:
                fq_name = steps[0]
        else:
            fq_name = name

        path = None
        steps = fq_name.split('.')

        add_package = True

        for i in range(len(steps)):
            modname = '.'.join(steps[:i + 1])

            loader = importlib.find_loader(modname, path=path)

            if loader is None:
                raise ValueError('could not find loader for module {}'.format(modname))

            if not isinstance(loader, importlib._bootstrap.NamespaceLoader) and not loader.is_package(modname):
                break

            if isinstance(loader, importlib._bootstrap.NamespaceLoader):
                path = loader._path._path
            else:
                modfile = loader.get_filename(modname)
                # os.path.dirname(__file__) is a common importlib assumption for __path__
                path = [os.path.dirname(modfile)]
        else:
            if fromlist:
                add_package = False

                for entry in fromlist:
                    modname = '{}.{}'.format(fq_name, entry)
                    entry_loader = importlib.find_loader(modname, path=path)

                    if entry_loader is not None and entry_loader.path != loader.path:
                        modules.append(modname)
                    else:
                        add_package = True

        if add_package:
            modules.append(fq_name)

    return modules

def modified_modules():
    for module in list(sys.modules.values()):
        try:
            loader = module.__loader__
        except AttributeError:
            # Weird custom module, skip
            continue

        try:
            loaded_modver = module.__sx_modversion__
        except AttributeError:
            # Unmanaged module
            continue

        try:
            imports = module.__sx_imports__
        except AttributeError:
            imports = ()

        current_modver = loader.get_module_version(module.__name__, imports)

        if loaded_modver != current_modver:
            yield module

def reload_modified(modified=None):
    if modified is None:
        modified = modified_modules()

    modified = tuple(modified)
    modified_names = {m.__name__ for m in modified}

    modg = {}

    for module in modified:
        imports = set(getattr(module, '__sx_imports__', ())) & modified_names
        modg[module.__name__] = {'item': module, 'deps': imports}

    for module in topological.sort(modg):
        try:
            reload(module)
        except ImportError as e:
            if isinstance(e.__cause__, FileNotFoundError):
                del sys.modules[module.__name__]
            else:
                raise
<|endoftext|>