<commit_before>"""
==========================
Model Complexity Influence
==========================

Demonstrate how model complexity influences both prediction accuracy and
computational performance.

We will be using two datasets:
    - :ref:`diabetes_dataset` for regression.
      This dataset consists of 10 measurements taken from diabetes patients.
      The task is to predict disease progression;
    - :ref:`20newsgroups_dataset` for classification. This dataset consists of
      newsgroup posts. The task is to predict on which topic (out of 20 topics)
      the post is written about.

We will model the complexity influence on three different estimators:
    - :class:`~sklearn.linear_model.SGDClassifier` (for classification data)
      which implements stochastic gradient descent learning;

    - :class:`~sklearn.svm.NuSVR` (for regression data) which implements
      Nu support vector regression;

    - :class:`~sklearn.ensemble.GradientBoostingRegressor` (for regression
      data) which builds an additive model in a forward stage-wise fashion.

We make the model complexity vary through the choice of relevant model
parameters in each of our selected models. Next, we will measure the influence
on both computational performance (latency) and predictive power (MSE or
Hamming Loss).

"""

# Authors: Eustache Diemert <eustache@diemert.fr>
#          Maria Telenczuk <https://github.com/maikia>
#          Guillaume Lemaitre <g.lemaitre58@gmail.com>
# License: BSD 3 clause

import time
import numpy as np
import matplotlib.pyplot as plt

from sklearn import datasets
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error
from sklearn.svm import NuSVR
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.linear_model import SGDClassifier
from sklearn.metrics import hamming_loss

# Initialize random generator
np.random.seed(0)

##############################################################################
# Load the data
# -------------
#
# First we load both datasets.
#
# .. note:: We are using
#    :func:`~sklearn.datasets.fetch_20newsgroups_vectorized` to download 20
#    newsgroups dataset. It returns ready-to-use features.
#
# .. note:: ``X`` of the 20 newsgroups dataset is a sparse matrix while ``X``
#    of diabetes dataset is a numpy array.
#

def generate_data(case):
    """Generate regression/classification data."""
    if case == "regression":
        X, y = datasets.load_diabetes(return_X_y=True)
        train_size = 0.8
    elif case == "classification":
        X, y = datasets.fetch_20newsgroups_vectorized(subset="all", return_X_y=True)
        train_size = 0.4  # to make the example run faster

    X_train, X_test, y_train, y_test = train_test_split(
        X, y, train_size=train_size, random_state=0
    )

    data = {"X_train": X_train, "X_test": X_test, "y_train": y_train, "y_test": y_test}
    return data

regression_data = generate_data("regression")
classification_data = generate_data("classification")

##############################################################################
# Benchmark influence
# -------------------
# Next, we can calculate the influence of the parameters on the given
# estimator. In each round, we will set the estimator with the new value of
# ``changing_param`` and we will be collecting the prediction times, prediction
# performance and complexities to see how those changes affect the estimator.
# We will calculate the complexity using ``complexity_computer`` passed as a
# parameter.
#

def benchmark_influence(conf):
    """
    Benchmark influence of `changing_param` on both MSE and latency.
    """
    prediction_times = []
    prediction_powers = []
    complexities = []
    for param_value in conf["changing_param_values"]:
        conf["tuned_params"][conf["changing_param"]] = param_value
        estimator = conf["estimator"](**conf["tuned_params"])

        print("Benchmarking %s" % estimator)
        estimator.fit(conf["data"]["X_train"], conf["data"]["y_train"])
        conf["postfit_hook"](estimator)
        complexity = conf["complexity_computer"](estimator)
        complexities.append(complexity)
        start_time = time.time()
        for _ in range(conf["n_samples"]):
            y_pred = estimator.predict(conf["data"]["X_test"])
        elapsed_time = (time.time() - start_time) / float(conf["n_samples"])
        prediction_times.append(elapsed_time)
        pred_score = conf["prediction_performance_computer"](
            conf["data"]["y_test"], y_pred
        )
        prediction_powers.append(pred_score)
        print(
            "Complexity: %d | %s: %.4f | Pred. Time: %fs\n"
            % (
                complexity,
                conf["prediction_performance_label"],
                pred_score,
                elapsed_time,
            )
        )
    return prediction_powers, prediction_times, complexities

##############################################################################
# Choose parameters
# -----------------
#
# We choose the parameters for each of our estimators by making
# a dictionary with all the necessary values.
# ``changing_param`` is the name of the parameter which will vary in each
# estimator.
# Complexity will be defined by the ``complexity_label`` and calculated using
# `complexity_computer`.
# Also note that depending on the estimator type we are passing
# different data.
#

def _count_nonzero_coefficients(estimator):
    a = estimator.coef_.toarray()
    return np.count_nonzero(a)

configurations = [
    {
        "estimator": SGDClassifier,
        "tuned_params": {
            "penalty": "elasticnet",
            "alpha": 0.001,
            "loss": "modified_huber",
            "fit_intercept": True,
            "tol": 1e-3,
        },
        "changing_param": "l1_ratio",
        "changing_param_values": [0.25, 0.5, 0.75, 0.9],
        "complexity_label": "non_zero coefficients",
        "complexity_computer": _count_nonzero_coefficients,
        "prediction_performance_computer": hamming_loss,
        "prediction_performance_label": "Hamming Loss (Misclassification Ratio)",
        "postfit_hook": lambda x: x.sparsify(),
        "data": classification_data,
        "n_samples": 5,
    },
    {
        "estimator": NuSVR,
        "tuned_params": {"C": 1e3, "gamma": 2 ** -15},
        "changing_param": "nu",
        "changing_param_values": [0.05, 0.1, 0.2, 0.35, 0.5],
        "complexity_label": "n_support_vectors",
        "complexity_computer": lambda x: len(x.support_vectors_),
        "data": regression_data,
        "postfit_hook": lambda x: x,
        "prediction_performance_computer": mean_squared_error,
        "prediction_performance_label": "MSE",
        "n_samples": 15,
    },
    {
        "estimator": GradientBoostingRegressor,
        "tuned_params": {
            "loss": "squared_error",
            "learning_rate": 0.05,
            "max_depth": 2,
        },
        "changing_param": "n_estimators",
        "changing_param_values": [10, 25, 50, 75, 100],
        "complexity_label": "n_trees",
        "complexity_computer": lambda x: x.n_estimators,
        "data": regression_data,
        "postfit_hook": lambda x: x,
        "prediction_performance_computer": mean_squared_error,
        "prediction_performance_label": "MSE",
        "n_samples": 15,
    },
]

##############################################################################
# Run the code and plot the results
# ---------------------------------
#
# We defined all the functions required to run our benchmark. Now, we will loop
# over the different configurations that we defined previously. Subsequently,
# we can analyze the plots obtained from the benchmark:
# Relaxing the `L1` penalty in the SGD classifier reduces the prediction error
# but leads to an increase in the training time.
# We can draw a similar analysis regarding the training time which increases
# with the number of support vectors with a Nu-SVR. However, we observed that
# there is an optimal number of support vectors which reduces the prediction
# error. Indeed, too few support vectors lead to an under-fitted model while
# too many support vectors lead to an over-fitted model.
# The exact same conclusion can be drawn for the gradient-boosting model. The
# only the difference with the Nu-SVR is that having too many trees in the
# ensemble is not as detrimental.
#

def plot_influence(conf, mse_values, prediction_times, complexities):
    """
    Plot influence of model complexity on both accuracy and latency.
    """

    fig = plt.figure()
    fig.subplots_adjust(right=0.75)

    # first axes (prediction error)
    ax1 = fig.add_subplot(111)
    line1 = ax1.plot(complexities, mse_values, c="tab:blue", ls="-")[0]
    ax1.set_xlabel("Model Complexity (%s)" % conf["complexity_label"])
    y1_label = conf["prediction_performance_label"]
    ax1.set_ylabel(y1_label)

    ax1.spines["left"].set_color(line1.get_color())
    ax1.yaxis.label.set_color(line1.get_color())
    ax1.tick_params(axis="y", colors=line1.get_color())

    # second axes (latency)
    ax2 = fig.add_subplot(111, sharex=ax1, frameon=False)
    line2 = ax2.plot(complexities, prediction_times, c="tab:orange", ls="-")[0]
    ax2.yaxis.tick_right()
    ax2.yaxis.set_label_position("right")
    y2_label = "Time (s)"
    ax2.set_ylabel(y2_label)
    ax1.spines["right"].set_color(line2.get_color())
    ax2.yaxis.label.set_color(line2.get_color())
    ax2.tick_params(axis="y", colors=line2.get_color())

    plt.legend(
        (line1, line2), ("prediction error", "prediction latency"), loc="upper right"
    )

    plt.title(
        "Influence of varying '%s' on %s"
        % (conf["changing_param"], conf["estimator"].__name__)
    )

for conf in configurations:
    prediction_performances, prediction_times, complexities = benchmark_influence(conf)
    plot_influence(conf, prediction_performances, prediction_times, complexities)
plt.show()

##############################################################################
# Conclusion
# ----------
#
# As a conclusion, we can deduce the following insights:
#
# * a model which is more complex (or expressive) will require a larger
#   training time;
# * a more complex model does not guarantee to reduce the prediction error.
#
# These aspects are related to model generalization and avoiding model
# under-fitting or over-fitting.
<commit_msg>MNT Accelerate plot_model_complexity_influence.py modifying the SGDClassifier parameters (#21963)<commit_after>"""
==========================
Model Complexity Influence
==========================

Demonstrate how model complexity influences both prediction accuracy and
computational performance.

We will be using two datasets:
    - :ref:`diabetes_dataset` for regression.
      This dataset consists of 10 measurements taken from diabetes patients.
      The task is to predict disease progression;
    - :ref:`20newsgroups_dataset` for classification. This dataset consists of
      newsgroup posts. The task is to predict on which topic (out of 20 topics)
      the post is written about.

We will model the complexity influence on three different estimators:
    - :class:`~sklearn.linear_model.SGDClassifier` (for classification data)
      which implements stochastic gradient descent learning;

    - :class:`~sklearn.svm.NuSVR` (for regression data) which implements
      Nu support vector regression;

    - :class:`~sklearn.ensemble.GradientBoostingRegressor` (for regression
      data) which builds an additive model in a forward stage-wise fashion.

We make the model complexity vary through the choice of relevant model
parameters in each of our selected models. Next, we will measure the influence
on both computational performance (latency) and predictive power (MSE or
Hamming Loss).

"""

# Authors: Eustache Diemert <eustache@diemert.fr>
#          Maria Telenczuk <https://github.com/maikia>
#          Guillaume Lemaitre <g.lemaitre58@gmail.com>
# License: BSD 3 clause

import time
import numpy as np
import matplotlib.pyplot as plt

from sklearn import datasets
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error
from sklearn.svm import NuSVR
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.linear_model import SGDClassifier
from sklearn.metrics import hamming_loss

# Initialize random generator
np.random.seed(0)

##############################################################################
# Load the data
# -------------
#
# First we load both datasets.
#
# .. note:: We are using
#    :func:`~sklearn.datasets.fetch_20newsgroups_vectorized` to download 20
#    newsgroups dataset. It returns ready-to-use features.
#
# .. note:: ``X`` of the 20 newsgroups dataset is a sparse matrix while ``X``
#    of diabetes dataset is a numpy array.
#

def generate_data(case):
    """Generate regression/classification data."""
    if case == "regression":
        X, y = datasets.load_diabetes(return_X_y=True)
        train_size = 0.8
    elif case == "classification":
        X, y = datasets.fetch_20newsgroups_vectorized(subset="all", return_X_y=True)
        train_size = 0.4  # to make the example run faster

    X_train, X_test, y_train, y_test = train_test_split(
        X, y, train_size=train_size, random_state=0
    )

    data = {"X_train": X_train, "X_test": X_test, "y_train": y_train, "y_test": y_test}
    return data

regression_data = generate_data("regression")
classification_data = generate_data("classification")

##############################################################################
# Benchmark influence
# -------------------
# Next, we can calculate the influence of the parameters on the given
# estimator. In each round, we will set the estimator with the new value of
# ``changing_param`` and we will be collecting the prediction times, prediction
# performance and complexities to see how those changes affect the estimator.
# We will calculate the complexity using ``complexity_computer`` passed as a
# parameter.
#

def benchmark_influence(conf):
    """
    Benchmark influence of `changing_param` on both MSE and latency.
    """
    prediction_times = []
    prediction_powers = []
    complexities = []
    for param_value in conf["changing_param_values"]:
        conf["tuned_params"][conf["changing_param"]] = param_value
        estimator = conf["estimator"](**conf["tuned_params"])

        print("Benchmarking %s" % estimator)
        estimator.fit(conf["data"]["X_train"], conf["data"]["y_train"])
        conf["postfit_hook"](estimator)
        complexity = conf["complexity_computer"](estimator)
        complexities.append(complexity)
        start_time = time.time()
        for _ in range(conf["n_samples"]):
            y_pred = estimator.predict(conf["data"]["X_test"])
        elapsed_time = (time.time() - start_time) / float(conf["n_samples"])
        prediction_times.append(elapsed_time)
        pred_score = conf["prediction_performance_computer"](
            conf["data"]["y_test"], y_pred
        )
        prediction_powers.append(pred_score)
        print(
            "Complexity: %d | %s: %.4f | Pred. Time: %fs\n"
            % (
                complexity,
                conf["prediction_performance_label"],
                pred_score,
                elapsed_time,
            )
        )
    return prediction_powers, prediction_times, complexities

##############################################################################
# Choose parameters
# -----------------
#
# We choose the parameters for each of our estimators by making
# a dictionary with all the necessary values.
# ``changing_param`` is the name of the parameter which will vary in each
# estimator.
# Complexity will be defined by the ``complexity_label`` and calculated using
# `complexity_computer`.
# Also note that depending on the estimator type we are passing
# different data.
#

def _count_nonzero_coefficients(estimator):
    a = estimator.coef_.toarray()
    return np.count_nonzero(a)

configurations = [
    {
        "estimator": SGDClassifier,
        "tuned_params": {
            "penalty": "elasticnet",
            "alpha": 0.001,
            "loss": "modified_huber",
            "fit_intercept": True,
            "tol": 1e-1,
            "n_iter_no_change": 2,
        },
        "changing_param": "l1_ratio",
        "changing_param_values": [0.25, 0.5, 0.75, 0.9],
        "complexity_label": "non_zero coefficients",
        "complexity_computer": _count_nonzero_coefficients,
        "prediction_performance_computer": hamming_loss,
        "prediction_performance_label": "Hamming Loss (Misclassification Ratio)",
        "postfit_hook": lambda x: x.sparsify(),
        "data": classification_data,
        "n_samples": 5,
    },
    {
        "estimator": NuSVR,
        "tuned_params": {"C": 1e3, "gamma": 2 ** -15},
        "changing_param": "nu",
        "changing_param_values": [0.05, 0.1, 0.2, 0.35, 0.5],
        "complexity_label": "n_support_vectors",
        "complexity_computer": lambda x: len(x.support_vectors_),
        "data": regression_data,
        "postfit_hook": lambda x: x,
        "prediction_performance_computer": mean_squared_error,
        "prediction_performance_label": "MSE",
        "n_samples": 15,
    },
    {
        "estimator": GradientBoostingRegressor,
        "tuned_params": {
            "loss": "squared_error",
            "learning_rate": 0.05,
            "max_depth": 2,
        },
        "changing_param": "n_estimators",
        "changing_param_values": [10, 25, 50, 75, 100],
        "complexity_label": "n_trees",
        "complexity_computer": lambda x: x.n_estimators,
        "data": regression_data,
        "postfit_hook": lambda x: x,
        "prediction_performance_computer": mean_squared_error,
        "prediction_performance_label": "MSE",
        "n_samples": 15,
    },
]

##############################################################################
# Run the code and plot the results
# ---------------------------------
#
# We defined all the functions required to run our benchmark. Now, we will loop
# over the different configurations that we defined previously. Subsequently,
# we can analyze the plots obtained from the benchmark:
# Relaxing the `L1` penalty in the SGD classifier reduces the prediction error
# but leads to an increase in the training time.
# We can draw a similar analysis regarding the training time which increases
# with the number of support vectors with a Nu-SVR. However, we observed that
# there is an optimal number of support vectors which reduces the prediction
# error. Indeed, too few support vectors lead to an under-fitted model while
# too many support vectors lead to an over-fitted model.
# The exact same conclusion can be drawn for the gradient-boosting model. The
# only the difference with the Nu-SVR is that having too many trees in the
# ensemble is not as detrimental.
#

def plot_influence(conf, mse_values, prediction_times, complexities):
    """
    Plot influence of model complexity on both accuracy and latency.
    """

    fig = plt.figure()
    fig.subplots_adjust(right=0.75)

    # first axes (prediction error)
    ax1 = fig.add_subplot(111)
    line1 = ax1.plot(complexities, mse_values, c="tab:blue", ls="-")[0]
    ax1.set_xlabel("Model Complexity (%s)" % conf["complexity_label"])
    y1_label = conf["prediction_performance_label"]
    ax1.set_ylabel(y1_label)

    ax1.spines["left"].set_color(line1.get_color())
    ax1.yaxis.label.set_color(line1.get_color())
    ax1.tick_params(axis="y", colors=line1.get_color())

    # second axes (latency)
    ax2 = fig.add_subplot(111, sharex=ax1, frameon=False)
    line2 = ax2.plot(complexities, prediction_times, c="tab:orange", ls="-")[0]
    ax2.yaxis.tick_right()
    ax2.yaxis.set_label_position("right")
    y2_label = "Time (s)"
    ax2.set_ylabel(y2_label)
    ax1.spines["right"].set_color(line2.get_color())
    ax2.yaxis.label.set_color(line2.get_color())
    ax2.tick_params(axis="y", colors=line2.get_color())

    plt.legend(
        (line1, line2), ("prediction error", "prediction latency"), loc="upper right"
    )

    plt.title(
        "Influence of varying '%s' on %s"
        % (conf["changing_param"], conf["estimator"].__name__)
    )

for conf in configurations:
    prediction_performances, prediction_times, complexities = benchmark_influence(conf)
    plot_influence(conf, prediction_performances, prediction_times, complexities)
plt.show()

##############################################################################
# Conclusion
# ----------
#
# As a conclusion, we can deduce the following insights:
#
# * a model which is more complex (or expressive) will require a larger
#   training time;
# * a more complex model does not guarantee to reduce the prediction error.
#
# These aspects are related to model generalization and avoiding model
# under-fitting or over-fitting.
<|endoftext|>


<commit_before>import os

import jinja2.ext
from flask import Flask, redirect, url_for, request, flash, render_template, current_app
from flask_login import current_user
from flask_babel import Babel
from jinja2 import StrictUndefined
from werkzeug.datastructures import ImmutableDict

from hades_logs import HadesLogs
from pycroft.helpers.i18n import gettext
from pycroft.model import session
from web import api
from . import template_filters
from . import template_tests
from .blueprints import (
    finance, infrastructure, properties, user, facilities, login)
from .blueprints.login import login_manager
from .templates import page_resources

class PycroftFlask(Flask):
    """
    Extend the Flask class to set Jinja options.
    """
    jinja_options = ImmutableDict(
        Flask.jinja_options,
        extensions=[
            jinja2.ext.autoescape,
            jinja2.ext.do,
            jinja2.ext.loopcontrols,
            jinja2.ext.with_,
        ],
        undefined=StrictUndefined
    )

    def __init__(self, *a, **kw):
        super().__init__(*a, **kw)
        # config keys to support:
        self.maybe_add_config_from_env([
            'PYCROFT_API_KEY',
            'HADES_CELERY_APP_NAME',
            'HADES_BROKER_URI',
            'HADES_RESULT_BACKEND_URI',
            'HADES_TIMEOUT',
            'HADES_ROUTING_KEY',
        ])

    def maybe_add_config_from_env(self, keys):
        """Write keys from the environment to the app's config

        If a key does not exist in the environment, it will just be
        skipped.

        :param keys: An iterable of strings
        """
        for key in keys:
            try:
                self.config[key] = os.environ[key]
            except KeyError:
                self.logger.debug("Config key %s not present in environment, skipping", key)
                continue
            else:
                self.logger.debug("Config key %s successfuly read from environment", key)

def make_app(debug=False):
    """  Create and configure the main? Flask app object

    :return: The fully configured app object
    """
    app = PycroftFlask(__name__)
    app.debug = debug

    #initialization code
    login_manager.init_app(app)
    app.register_blueprint(user.bp, url_prefix="/user")
    app.register_blueprint(facilities.bp, url_prefix="/facilities")
    app.register_blueprint(infrastructure.bp, url_prefix="/infrastructure")
    app.register_blueprint(properties.bp, url_prefix="/properties")
    app.register_blueprint(finance.bp, url_prefix="/finance")
    app.register_blueprint(login.bp)
    app.register_blueprint(api.bp, url_prefix="/api/v0")

    template_filters.register_filters(app)
    template_tests.register_checks(app)

    babel = Babel(app)
    try:
        HadesLogs(app)
    except KeyError as e:
        app.logger.info("HadesLogs configuration incomplete, skipping.")
        app.logger.info("Original error: %s", str(e))

    page_resources.init_app(app)

    user.nav.register_on(app)
    finance.nav.register_on(app)
    facilities.nav.register_on(app)
    infrastructure.nav.register_on(app)
    properties.nav.register_on(app)

    @app.errorhandler(403)
    @app.errorhandler(404)
    @app.errorhandler(500)
    def errorpage(e):
        """Handle errors according to their error code

        :param e: The error from the errorhandler
        """
        # We need this path hard-coding because the global app errorhandlers have higher
        # precedence than anything registered to a blueprint.
        # A clean solution would be flask supporting nested blueprints (see flask #539)
        if request.path.startswith('/api/'):
            return api.errorpage(e)

        if not hasattr(e, 'code'):
            code = 500
        else:
            code = e.code
        if code == 500:
            message = str(e)
        elif code == 403:
            message = gettext(u"You are not allowed to access this page.")
        elif code == 404:
            message = gettext(u"Page not found.")
        else:
            raise AssertionError()
        return render_template('error.html', error=message), code

    @app.route('/')
    def redirect_to_index():
        return redirect(url_for('user.overview'))

    @app.teardown_request
    def shutdown_session(exception=None):
        session.Session.remove()

    @app.before_request
    def require_login():
        """Request a login for every page
        except the login blueprint and the static folder.

        Blueprint "None" is needed for "/static/*" GET requests.
        """
        if current_user.is_anonymous and request.blueprint not in ("login", 'api', None):
            return current_app.login_manager.unauthorized()

    return app
<commit_msg>Optimize imports<commit_after>import os

import jinja2.ext
from flask import (
    Flask, current_app, redirect, render_template, request, url_for,
)
from flask_babel import Babel
from flask_login import current_user
from werkzeug.datastructures import ImmutableDict

from hades_logs import HadesLogs
from pycroft.helpers.i18n import gettext
from pycroft.model import session
from web import api
from . import template_filters, template_tests
from .blueprints import (
    facilities, finance, infrastructure, login, properties, user,
)
from .blueprints.login import login_manager
from .templates import page_resources

class PycroftFlask(Flask):
    """
    Extend the Flask class to set Jinja options.
    """
    jinja_options = ImmutableDict(
        Flask.jinja_options,
        extensions=[
            jinja2.ext.autoescape,
            jinja2.ext.do,
            jinja2.ext.loopcontrols,
            jinja2.ext.with_,
        ],
        undefined=jinja2.StrictUndefined,
    )

    def __init__(self, *a, **kw):
        super().__init__(*a, **kw)
        # config keys to support:
        self.maybe_add_config_from_env([
            'PYCROFT_API_KEY',
            'HADES_CELERY_APP_NAME',
            'HADES_BROKER_URI',
            'HADES_RESULT_BACKEND_URI',
            'HADES_TIMEOUT',
            'HADES_ROUTING_KEY',
        ])

    def maybe_add_config_from_env(self, keys):
        """Write keys from the environment to the app's config

        If a key does not exist in the environment, it will just be
        skipped.

        :param keys: An iterable of strings
        """
        for key in keys:
            try:
                self.config[key] = os.environ[key]
            except KeyError:
                self.logger.debug("Config key %s not present in environment, skipping", key)
                continue
            else:
                self.logger.debug("Config key %s successfuly read from environment", key)

def make_app(debug=False):
    """  Create and configure the main? Flask app object

    :return: The fully configured app object
    """
    app = PycroftFlask(__name__)
    app.debug = debug

    #initialization code
    login_manager.init_app(app)
    app.register_blueprint(user.bp, url_prefix="/user")
    app.register_blueprint(facilities.bp, url_prefix="/facilities")
    app.register_blueprint(infrastructure.bp, url_prefix="/infrastructure")
    app.register_blueprint(properties.bp, url_prefix="/properties")
    app.register_blueprint(finance.bp, url_prefix="/finance")
    app.register_blueprint(login.bp)
    app.register_blueprint(api.bp, url_prefix="/api/v0")

    template_filters.register_filters(app)
    template_tests.register_checks(app)

    babel = Babel(app)
    try:
        HadesLogs(app)
    except KeyError as e:
        app.logger.info("HadesLogs configuration incomplete, skipping.")
        app.logger.info("Original error: %s", str(e))

    page_resources.init_app(app)

    user.nav.register_on(app)
    finance.nav.register_on(app)
    facilities.nav.register_on(app)
    infrastructure.nav.register_on(app)
    properties.nav.register_on(app)

    @app.errorhandler(403)
    @app.errorhandler(404)
    @app.errorhandler(500)
    def errorpage(e):
        """Handle errors according to their error code

        :param e: The error from the errorhandler
        """
        # We need this path hard-coding because the global app errorhandlers have higher
        # precedence than anything registered to a blueprint.
        # A clean solution would be flask supporting nested blueprints (see flask #539)
        if request.path.startswith('/api/'):
            return api.errorpage(e)

        if not hasattr(e, 'code'):
            code = 500
        else:
            code = e.code
        if code == 500:
            message = str(e)
        elif code == 403:
            message = gettext(u"You are not allowed to access this page.")
        elif code == 404:
            message = gettext(u"Page not found.")
        else:
            raise AssertionError()
        return render_template('error.html', error=message), code

    @app.route('/')
    def redirect_to_index():
        return redirect(url_for('user.overview'))

    @app.teardown_request
    def shutdown_session(exception=None):
        session.Session.remove()

    @app.before_request
    def require_login():
        """Request a login for every page
        except the login blueprint and the static folder.

        Blueprint "None" is needed for "/static/*" GET requests.
        """
        if current_user.is_anonymous and request.blueprint not in ("login", 'api', None):
            return current_app.login_manager.unauthorized()

    return app
<|endoftext|>


<commit_before>from django.conf import settings
from django.http import HttpResponseRedirect
from django.shortcuts import render_to_response
from hiicart import validate_gateway
from hiicart.gateway.base import SubmitResult
from hiicart.models import *
from pprint import PrettyPrinter

from apps.basic import forms

def index(request):
    context = {}
    if request.method == 'POST':
        form = forms.BasicPurchaseForm(request.POST)
        if form.is_valid():
            cart = HiiCart.objects.create()
            if form.cleaned_data['fancypants']:
                LineItem.objects.create(cart=cart, quantity=1, **forms.PANTS)
            if form.cleaned_data['monkeybutler']:
                LineItem.objects.create(cart=cart, quantity=1, **forms.MONKEYBUTLER)
            if form.cleaned_data['subscription']:
                RecurringLineItem.objects.create(cart=cart, quantity=1, **forms.SUBSCRIPTION)
            result = cart.submit(form.cleaned_data['gateway'])
            if result.type == 'url':
                return HttpResponseRedirect(result.url)
            else:
                return HttpResponseRedirect('/paypal_redirect')
    else:
        form = forms.BasicPurchaseForm()
    p = PrettyPrinter(width=40)
    context['paypal'] = (p.pformat(settings.HIICART_SETTINGS['PAYPAL']),
                         validate_gateway('paypal'))
    context['form'] = form
    return render_to_response('index.html', context)

def paypal_receipt(request):
    if 'cart' not in request.GET:
        return HttpResponseRedirect('/')
    context = {
            'form' :  forms.PaypalReceiptForm(request.GET),
            'cart': HiiCart.objects.get(_cart_uuid=request.GET['cart']),
            }
    return render_to_response('paypal_receipt.html', context)
<commit_msg>Revert accidental changes to example app<commit_after>from django.conf import settings
from django.http import HttpResponseRedirect
from django.shortcuts import render_to_response
from hiicart import validate_gateway
from hiicart.gateway.base import SubmitResult
from hiicart.models import *
from pprint import PrettyPrinter

from apps.basic import forms

def index(request):
    context = {}
    if request.method == 'POST':
        form = forms.BasicPurchaseForm(request.POST)
        if form.is_valid():
            cart = HiiCart.objects.create()
            if form.cleaned_data['fancypants']:
                LineItem.objects.create(cart=cart, quantity=1, **forms.PANTS)
            if form.cleaned_data['monkeybutler']:
                LineItem.objects.create(cart=cart, quantity=1, **forms.MONKEYBUTLER)
            if form.cleaned_data['subscription']:
                RecurringLineItem.objects.create(cart=cart, quantity=1, **forms.SUBSCRIPTION)
            result = cart.submit(form.cleaned_data['gateway'])
            if result.type == 'url':
                return HttpResponseRedirect(result.url)
            else:
                return HttpResponseRedirect('/paypal_redirect')
    else:
        form = forms.BasicPurchaseForm()
    p = PrettyPrinter(width=40)
    context['paypal'] = (p.pformat(settings.HIICART_SETTINGS['PAYPAL']),
                         validate_gateway('paypal'))
    context['paypal2'] = (p.pformat(settings.HIICART_SETTINGS['PAYPAL2']),
                          validate_gateway('paypal2'))
    context['paypal_adaptive'] = (p.pformat(settings.HIICART_SETTINGS['PAYPAL_ADAPTIVE']),
                                  validate_gateway('paypal_adaptive'))
    context['google'] = (p.pformat(settings.HIICART_SETTINGS['GOOGLE']),
                         validate_gateway('google'))
    context['amazon'] = (p.pformat(settings.HIICART_SETTINGS['AMAZON']),
                         validate_gateway('amazon'))
    context['form'] = form
    return render_to_response('index.html', context)

def paypal_receipt(request):
    if 'cart' not in request.GET:
        return HttpResponseRedirect('/')
    context = {
            'form' :  forms.PaypalReceiptForm(request.GET),
            'cart': HiiCart.objects.get(_cart_uuid=request.GET['cart']),
            }
    return render_to_response('paypal_receipt.html', context)
<|endoftext|>


<commit_before>from Texture import Texture
from os.path import join as pjoin, dirname

from PIL import Image

if __name__ == '__main__':
    texture = Texture(2, 2, 2)
    image = Image.open(pjoin(dirname(__file__), 'stripes.png')).convert('L')
    width, height = image.size

    texture.pick_texture_sample({
        0: (0, 1),
        1: (0, 0)
    }, 0)
    texture.pick_texture_sample({
        0: (1, 0),
        1: (1, 1)
    }, 0)
    texture.pick_texture_sample({
        0: (0, 0),
        1: (0, 0)
    }, 0)

    texture.pick_texture_sample({
        1: (0, 1),
        0: (0, 0)
    }, 1)
    texture.pick_texture_sample({
        1: (1, 0),
        0: (1, 1)
    }, 1)

    data = list(image.getdata())
    results = []
    for x in xrange(width):
        row = []
        for y in xrange(height):
            cur = data[x*width+y]
            cur = 0 if cur == 0 else 1

            params = {}
            if y < width-1:
                r = data[x*width+(y+1)]
                r = 0 if r == 0 else 1
                params[0] = (cur, r)
            if x < height-1:
                b = data[(x+1)*width+y]
                b = 0 if b == 0 else 1
                params[1] = (cur, b)

            t = texture.recognize_texture(params)
            row.append(t)
        results += row

    results = [((200, 50, 50) if r == 0 else (50, 50, 200)) for r in results]
    result = Image.new('RGB', image.size)
    result.putdata(results)
    for x in xrange(width):
        print results[x*height:(x+1)*height]
    result.save(pjoin(dirname(__file__), 'result.png'))

<commit_msg>Improved LSS Texture Recognizer example<commit_after>from Texture import Texture
from os.path import join as pjoin, dirname

from PIL import Image

if __name__ == '__main__':
    texture = Texture(2, 2, 2)
    image = Image.open(pjoin(dirname(__file__), 'stripes.png')).convert('L')
    mask = Image.open(pjoin(dirname(__file__), 'mask.png'))
    width, height = image.size

    data = list(image.getdata())
    mask_data = list(mask.getdata())
    for x in xrange(height-1):
        for y in xrange(width-1):
            cur = data[x*width+y]
            cur = 0 if cur == 0 else 1

            params = {}

            r = data[x*width+(y+1)]
            r = 0 if r == 0 else 1
            params[0] = (cur, r)

            b = data[(x+1)*width+y]
            b = 0 if b == 0 else 1
            params[1] = (cur, b)

            current_class = 0 if mask_data[x*width+y][0] == 255 else 1
            texture.pick_texture_sample(params, current_class)

    test_image = Image.open(pjoin(dirname(__file__), 'test.png')).convert('L')
    test_width, test_height = test_image.size
    test_data = list(test_image.getdata())
    results = []
    for x in xrange(test_height):
        row = []
        for y in xrange(test_width):
            cur = test_data[x*test_width+y]
            cur = 0 if cur == 0 else 1

            params = {}
            if y < test_width-1:
                r = test_data[x*test_width+(y+1)]
                r = 0 if r == 0 else 1
                params[0] = (cur, r)
            if x < test_height-1:
                b = test_data[(x+1)*test_width+y]
                b = 0 if b == 0 else 1
                params[1] = (cur, b)

            t = texture.recognize_texture(params)
            row.append(t)
        results += row

    results = [((200, 50, 50) if r == 0 else (50, 50, 200)) for r in results]
    result = Image.new('RGB', test_image.size)
    result.putdata(results)
    result.save(pjoin(dirname(__file__), 'result.png'))

<|endoftext|>


<commit_before>from cellnopt.core import cnodata
from cno.feeder import Feeder

def test_feeder():

    f = Feeder()
    pknmodel = cnodata("PKN-ToyMMB.sif")
    midas = cnodata("MD-ToyMMB.csv")

    f.run(model=pknmodel, data=midas)
    f.newlinks
    print(f)

    f.plot()

<commit_msg>Fixing import<commit_after>from cno import cnodata
from cno.feeder import Feeder

def test_feeder():

    f = Feeder()
    pknmodel = cnodata("PKN-ToyMMB.sif")
    midas = cnodata("MD-ToyMMB.csv")

    f.run(model=pknmodel, data=midas)
    f.newlinks
    print(f)

    f.plot()

<|endoftext|>


<commit_before>#!/usr/bin/env python

'''A tiny library for parsing, modifying, and composing SRT files.'''

import functools
import re
from datetime import timedelta
from itertools import groupby

SUBTITLE_PATTERN = r'(\d+)\n(\d+:\d+:\d+,\d+) --> (\d+:\d+:\d+,\d+)\n(.+?)\n\n'
SUBTITLE_REGEX = re.compile(SUBTITLE_PATTERN, re.MULTILINE | re.DOTALL)

@functools.total_ordering
class Subtitle(object):
    '''
    The metadata relating to a single subtitle.

    Attributes:
        index: The SRT index for this subtitle.
        start: A timedelta object representing the time that the subtitle
            should start being shown
        end: A timedelta object representing the time that the subtitle should
            stop being shown
        content: The subtitle content
    '''

    def __init__(self, index, start, end, content):
        self.index = index
        self.start = start
        self.end = end
        self.content = content

    def __eq__(self, other):
        return self.__dict__ == other.__dict__

    def __lt__(self, other):
        return self.start < other.start

    def to_srt(self):
        '''
        Convert the current subtitle to an SRT block.

        Returns:
            A string with the metadata of the current Subtitle object as an SRT
            formatted subtitle block.
        '''
        return '%d\n%s --> %s\n%s\n\n' % (
            self.index, timedelta_to_srt_timestamp(self.start),
            timedelta_to_srt_timestamp(self.end), self.content,
        )

def timedelta_to_srt_timestamp(timedelta_timestamp):
    '''
    Convert a timedelta to an SRT timestamp.

    Args:
        timedelta_timestamp: A timestamp represented as a datetime timedelta.

    Returns:
        An SRT formatted (HH:MM:SS,mmm) string representing the same timestamp
        passed in.
    '''
    hrs, remainder = divmod(timedelta_timestamp.seconds, 3600)
    mins, secs = divmod(remainder, 60)
    msecs = timedelta_timestamp.microseconds // 1000
    return '%02d:%02d:%02d,%03d' % (hrs, mins, secs, msecs)

def srt_timestamp_to_timedelta(srt_timestamp):
    '''
    Convert an SRT timestamp to a timedelta.

    Args:
        srt_timestamp: A timestamp in SRT format (HH:MM:SS,mmm).

    Returns:
        A timedelta object representing the same timestamp passed in.
    '''
    hrs, mins, secs, msecs = map(int, re.split('[,:]', srt_timestamp))
    return timedelta(hours=hrs, minutes=mins, seconds=secs, milliseconds=msecs)

def parse(srt):
    '''
    Convert an SRT formatted string to a generator of Subtitle objects.

    Args:
        srt: A string containing SRT formatted data.

    Returns:
        A generator of the subtitles contained in the SRT file as Subtitle
        objects.
    '''
    for match in SUBTITLE_REGEX.finditer(srt):
        raw_index, raw_start, raw_end, content = match.groups()
        yield Subtitle(
            index=int(raw_index), start=srt_timestamp_to_timedelta(raw_start),
            end=srt_timestamp_to_timedelta(raw_end), content=content,
        )

def parse_file(srt_stream):
    '''
    Parse an SRT formatted stream into Subtitle objects.

    Args:
        srt_stream: A stream containing SRT formatted data.

    Returns:
        A generator of the subtitles contained in the SRT file as Subtitle
        objects.
    '''
    srt_stream_chomped = (line.rstrip('\n') for line in srt_stream)
    srt_blocks = [
        '\n'.join(srt_lines) + '\n\n'
        for line_has_content, srt_lines in groupby(srt_stream_chomped, bool)
        if line_has_content
    ]

    for srt_block in srt_blocks:
        subtitle, = parse(srt_block)
        yield subtitle

def compose(subtitles):
    '''
    Convert an iterator of Subtitle objects to a string of joined SRT blocks.

    Args:
        subtitles: An iterator of Subtitle objects, in the order they should be
            in the output.

    Returns:
        A single SRT formatted string, with each input Subtitle represented as
        an SRT block.
    '''
    return ''.join(subtitle.to_srt() for subtitle in subtitles)

def compose_file(subtitles, output_stream):
    '''
    Stream a sequence of Subtitle objects into an SRT formatted stream.

    Args:
        subtitles: An iterator of Subtitle objects, in the order they should be
            written to the file.
        output_stream: A stream to write the resulting SRT blocks to.

    Returns:
        The number of subtitles that were written to the stream.
    '''
    for num_written, subtitle in enumerate(subtitles, start=1):
        output_stream.write(subtitle.to_srt())
    return num_written
<commit_msg>Add note on `compose` about memory usage<commit_after>#!/usr/bin/env python

'''A tiny library for parsing, modifying, and composing SRT files.'''

import functools
import re
from datetime import timedelta
from itertools import groupby

SUBTITLE_PATTERN = r'(\d+)\n(\d+:\d+:\d+,\d+) --> (\d+:\d+:\d+,\d+)\n(.+?)\n\n'
SUBTITLE_REGEX = re.compile(SUBTITLE_PATTERN, re.MULTILINE | re.DOTALL)

@functools.total_ordering
class Subtitle(object):
    '''
    The metadata relating to a single subtitle.

    Attributes:
        index: The SRT index for this subtitle.
        start: A timedelta object representing the time that the subtitle
            should start being shown
        end: A timedelta object representing the time that the subtitle should
            stop being shown
        content: The subtitle content
    '''

    def __init__(self, index, start, end, content):
        self.index = index
        self.start = start
        self.end = end
        self.content = content

    def __eq__(self, other):
        return self.__dict__ == other.__dict__

    def __lt__(self, other):
        return self.start < other.start

    def to_srt(self):
        '''
        Convert the current subtitle to an SRT block.

        Returns:
            A string with the metadata of the current Subtitle object as an SRT
            formatted subtitle block.
        '''
        return '%d\n%s --> %s\n%s\n\n' % (
            self.index, timedelta_to_srt_timestamp(self.start),
            timedelta_to_srt_timestamp(self.end), self.content,
        )

def timedelta_to_srt_timestamp(timedelta_timestamp):
    '''
    Convert a timedelta to an SRT timestamp.

    Args:
        timedelta_timestamp: A timestamp represented as a datetime timedelta.

    Returns:
        An SRT formatted (HH:MM:SS,mmm) string representing the same timestamp
        passed in.
    '''
    hrs, remainder = divmod(timedelta_timestamp.seconds, 3600)
    mins, secs = divmod(remainder, 60)
    msecs = timedelta_timestamp.microseconds // 1000
    return '%02d:%02d:%02d,%03d' % (hrs, mins, secs, msecs)

def srt_timestamp_to_timedelta(srt_timestamp):
    '''
    Convert an SRT timestamp to a timedelta.

    Args:
        srt_timestamp: A timestamp in SRT format (HH:MM:SS,mmm).

    Returns:
        A timedelta object representing the same timestamp passed in.
    '''
    hrs, mins, secs, msecs = map(int, re.split('[,:]', srt_timestamp))
    return timedelta(hours=hrs, minutes=mins, seconds=secs, milliseconds=msecs)

def parse(srt):
    '''
    Convert an SRT formatted string to a generator of Subtitle objects.

    Args:
        srt: A string containing SRT formatted data.

    Returns:
        A generator of the subtitles contained in the SRT file as Subtitle
        objects.
    '''
    for match in SUBTITLE_REGEX.finditer(srt):
        raw_index, raw_start, raw_end, content = match.groups()
        yield Subtitle(
            index=int(raw_index), start=srt_timestamp_to_timedelta(raw_start),
            end=srt_timestamp_to_timedelta(raw_end), content=content,
        )

def parse_file(srt_stream):
    '''
    Parse an SRT formatted stream into Subtitle objects.

    Args:
        srt_stream: A stream containing SRT formatted data.

    Returns:
        A generator of the subtitles contained in the SRT file as Subtitle
        objects.
    '''
    srt_stream_chomped = (line.rstrip('\n') for line in srt_stream)
    srt_blocks = [
        '\n'.join(srt_lines) + '\n\n'
        for line_has_content, srt_lines in groupby(srt_stream_chomped, bool)
        if line_has_content
    ]

    for srt_block in srt_blocks:
        subtitle, = parse(srt_block)
        yield subtitle

def compose(subtitles):
    '''
    Convert an iterator of Subtitle objects to a string of joined SRT blocks.

    This may be a convienient interface when converting back and forth between
    Python objects and SRT formatted blocks, but you may wish to consider using
    the provided stream interface, `compose_file`, instead, which is
    considerably more memory efficient when dealing with particularly large SRT
    data.

    Args:
        subtitles: An iterator of Subtitle objects, in the order they should be
            in the output.

    Returns:
        A single SRT formatted string, with each input Subtitle represented as
        an SRT block.
    '''
    return ''.join(subtitle.to_srt() for subtitle in subtitles)

def compose_file(subtitles, output_stream):
    '''
    Stream a sequence of Subtitle objects into an SRT formatted stream.

    Args:
        subtitles: An iterator of Subtitle objects, in the order they should be
            written to the file.
        output_stream: A stream to write the resulting SRT blocks to.

    Returns:
        The number of subtitles that were written to the stream.
    '''
    for num_written, subtitle in enumerate(subtitles, start=1):
        output_stream.write(subtitle.to_srt())
    return num_written
<|endoftext|>


<commit_before># Copyright (C) 2011 Nippon Telegraph and Telephone Corporation.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging
import struct

from ryu.base import app_manager
from ryu.controller import mac_to_port
from ryu.controller import ofp_event
from ryu.controller.handler import MAIN_DISPATCHER
from ryu.controller.handler import set_ev_cls
from ryu.ofproto import ofproto_v1_2, ofproto_v1_2_parser
from ryu.lib.mac import haddr_to_str

# TODO: we should split the handler into two parts, protocol
# independent and dependant parts.

# TODO: can we use dpkt python library?

# TODO: we need to move the followings to something like db

class SimpleSwitch(app_manager.RyuApp):
    OFP_VERSIONS = [ofproto_v1_2.OFP_VERSION]

    def __init__(self, *args, **kwargs):
        super(SimpleSwitch, self).__init__(*args, **kwargs)
        self.mac_to_port = {}

    def add_flow(self, datapath, in_port, dst, actions):
        ofproto = datapath.ofproto

        #wildcards = ofproto_v1_2.OFPFW_ALL
        #wildcards &= ~ofproto_v1_2.OFPFW_IN_PORT
        #wildcards &= ~ofproto_v1_2.OFPFW_DL_DST
        # old standard matching - deprecated from 1.2 onwards

        #match = datapath.ofproto_parser.OFPMatch(
        #    wildcards, in_port, 0, dst,
        #    0, 0, 0, 0, 0, 0, 0, 0, 0)

        match = datapath.ofproto_parser.OFPMatch()
        match.set_in_port(in_port)
        match.set_dl_dst(dst)

        mod = datapath.ofproto_parser.OFPFlowMod(
            datapath=datapath, match=match, cookie=0,
            command=ofproto.OFPFC_ADD, idle_timeout=0, hard_timeout=0,
            priority=ofproto.OFP_DEFAULT_PRIORITY,
            flags=ofproto.OFPFF_SEND_FLOW_REM, actions=actions)
        datapath.send_msg(mod)

    @set_ev_cls(ofp_event.EventOFPPacketIn, MAIN_DISPATCHER)
    def _packet_in_handler(self, ev):
        msg = ev.msg
        datapath = msg.datapath
        ofproto = datapath.ofproto

        dst, src, _eth_type = struct.unpack_from('!6s6sH', buffer(msg.data), 0)

        dpid = datapath.id
        self.mac_to_port.setdefault(dpid, {})

        match = msg.match
        in_port = 0
        #iterate through fields because WHY IS THIS SO HARD
        #packet in dpid 20015998343868 from 08:00:27:15:d4:53 to ff:ff:ff:ff:ff:ff log_port 0 phy_port 0
        #Field MTInPort(header=2147483652,length=8,n_bytes=4,value=2)
        #Field MTEthType(header=2147486210,length=6,n_bytes=2,value=2054)
        #Field MTArpOp(header=2147494402,length=6,n_bytes=2,value=1)
        #Field MTMetadata(header=2147484680,length=12,n_bytes=8,value=18446744073709551615L)
        #Field MTArpSha(header=2147495942,length=10,n_bytes=6,value="\x08\x00'\x15\xd4S")
        #Field MTEthDst(header=2147485190,length=10,n_bytes=6,value='\xff\xff\xff\xff\xff\xff')
        #Field MTArpSpa(header=2147494916,length=8,n_bytes=4,value=167772161)
        #Field MTArpTha(header=2147496454,length=10,n_bytes=6,value='\x00\x00\x00\x00\x00\x00')

        for o in match.fields:
            if isinstance(o, ofproto_v1_2_parser.MTInPort):
                in_port = o.value
                break

        self.logger.info("packet in dpid %s from %s to %s log_port %s",
                         dpid, haddr_to_str(src), haddr_to_str(dst),
                         in_port)

        # learn the mac address to avoid FLOOD next time.
        self.mac_to_port[dpid][src] = in_port

        # update on our list of mac addresses
        print self.mac_to_port[dpid]

        if dst in self.mac_to_port[dpid]:
            out_port = self.mac_to_port[dpid][dst]
        else:
            out_port = ofproto_v1_2.OFPP_FLOOD

        actions = [datapath.ofproto_parser.OFPActionOutput(out_port, 1500)]

        # install a flow to avoid packet_in next time
        if out_port != ofproto_v1_2.OFPP_FLOOD:
            self.add_flow(datapath, in_port, dst, actions)

        out = datapath.ofproto_parser.OFPPacketOut(
            datapath=datapath, buffer_id=msg.buffer_id, in_port=in_port,
            actions=actions)
        datapath.send_msg(out)

    @set_ev_cls(ofp_event.EventOFPPortStatus, MAIN_DISPATCHER)
    def _port_status_handler(self, ev):
        msg = ev.msg
        reason = msg.reason
        port_no = msg.desc.port_no

        ofproto = msg.datapath.ofproto
        if reason == ofproto.OFPPR_ADD:
            self.logger.info("port added %s", port_no)
        elif reason == ofproto.OFPPR_DELETE:
            self.logger.info("port deleted %s", port_no)
        elif reason == ofproto.OFPPR_MODIFY:
            self.logger.info("port modified %s", port_no)
        else:
            self.logger.info("Illeagal port state %s %s", port_no, reason)
<commit_msg>hardcoding priority - not defined in OF1.2 module<commit_after># Copyright (C) 2011 Nippon Telegraph and Telephone Corporation.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging
import struct

from ryu.base import app_manager
from ryu.controller import mac_to_port
from ryu.controller import ofp_event
from ryu.controller.handler import MAIN_DISPATCHER
from ryu.controller.handler import set_ev_cls
from ryu.ofproto import ofproto_v1_2, ofproto_v1_2_parser
from ryu.lib.mac import haddr_to_str

# TODO: we should split the handler into two parts, protocol
# independent and dependant parts.

# TODO: can we use dpkt python library?

# TODO: we need to move the followings to something like db

class SimpleSwitch(app_manager.RyuApp):
    OFP_VERSIONS = [ofproto_v1_2.OFP_VERSION]

    def __init__(self, *args, **kwargs):
        super(SimpleSwitch, self).__init__(*args, **kwargs)
        self.mac_to_port = {}

    def add_flow(self, datapath, in_port, dst, actions):
        ofproto = datapath.ofproto

        #wildcards = ofproto_v1_2.OFPFW_ALL
        #wildcards &= ~ofproto_v1_2.OFPFW_IN_PORT
        #wildcards &= ~ofproto_v1_2.OFPFW_DL_DST
        # old standard matching - deprecated from 1.2 onwards

        #match = datapath.ofproto_parser.OFPMatch(
        #    wildcards, in_port, 0, dst,
        #    0, 0, 0, 0, 0, 0, 0, 0, 0)

        match = datapath.ofproto_parser.OFPMatch()
        match.set_in_port(in_port)
        match.set_dl_dst(dst)

        mod = datapath.ofproto_parser.OFPFlowMod(
            datapath=datapath, match=match, cookie=0,
            command=ofproto.OFPFC_ADD, idle_timeout=0, hard_timeout=0,
            priority=0x8000,
            flags=ofproto.OFPFF_SEND_FLOW_REM, actions=actions)
        datapath.send_msg(mod)

    @set_ev_cls(ofp_event.EventOFPPacketIn, MAIN_DISPATCHER)
    def _packet_in_handler(self, ev):
        msg = ev.msg
        datapath = msg.datapath
        ofproto = datapath.ofproto

        dst, src, _eth_type = struct.unpack_from('!6s6sH', buffer(msg.data), 0)

        dpid = datapath.id
        self.mac_to_port.setdefault(dpid, {})

        match = msg.match
        in_port = 0
        #iterate through fields because WHY IS THIS SO HARD
        #packet in dpid 20015998343868 from 08:00:27:15:d4:53 to ff:ff:ff:ff:ff:ff log_port 0 phy_port 0
        #Field MTInPort(header=2147483652,length=8,n_bytes=4,value=2)
        #Field MTEthType(header=2147486210,length=6,n_bytes=2,value=2054)
        #Field MTArpOp(header=2147494402,length=6,n_bytes=2,value=1)
        #Field MTMetadata(header=2147484680,length=12,n_bytes=8,value=18446744073709551615L)
        #Field MTArpSha(header=2147495942,length=10,n_bytes=6,value="\x08\x00'\x15\xd4S")
        #Field MTEthDst(header=2147485190,length=10,n_bytes=6,value='\xff\xff\xff\xff\xff\xff')
        #Field MTArpSpa(header=2147494916,length=8,n_bytes=4,value=167772161)
        #Field MTArpTha(header=2147496454,length=10,n_bytes=6,value='\x00\x00\x00\x00\x00\x00')

        for o in match.fields:
            if isinstance(o, ofproto_v1_2_parser.MTInPort):
                in_port = o.value
                break

        self.logger.info("packet in dpid %s from %s to %s log_port %s",
                         dpid, haddr_to_str(src), haddr_to_str(dst),
                         in_port)

        # learn the mac address to avoid FLOOD next time.
        self.mac_to_port[dpid][src] = in_port

        # update on our list of mac addresses
        print self.mac_to_port[dpid]

        if dst in self.mac_to_port[dpid]:
            out_port = self.mac_to_port[dpid][dst]
        else:
            out_port = ofproto_v1_2.OFPP_FLOOD

        actions = [datapath.ofproto_parser.OFPActionOutput(out_port, 1500)]

        # install a flow to avoid packet_in next time
        if out_port != ofproto_v1_2.OFPP_FLOOD:
            self.add_flow(datapath, in_port, dst, actions)

        out = datapath.ofproto_parser.OFPPacketOut(
            datapath=datapath, buffer_id=msg.buffer_id, in_port=in_port,
            actions=actions)
        datapath.send_msg(out)

    @set_ev_cls(ofp_event.EventOFPPortStatus, MAIN_DISPATCHER)
    def _port_status_handler(self, ev):
        msg = ev.msg
        reason = msg.reason
        port_no = msg.desc.port_no

        ofproto = msg.datapath.ofproto
        if reason == ofproto.OFPPR_ADD:
            self.logger.info("port added %s", port_no)
        elif reason == ofproto.OFPPR_DELETE:
            self.logger.info("port deleted %s", port_no)
        elif reason == ofproto.OFPPR_MODIFY:
            self.logger.info("port modified %s", port_no)
        else:
            self.logger.info("Illeagal port state %s %s", port_no, reason)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8

import json
import logging

import tornado.web
import tornado.options
from jinja2 import Environment, FileSystemLoader

from workin.conf import settings
from workin.exceptions import ImproperlyConfigured
from workin.middleware import MiddlewareManager
from workin.mixins.flash_message import FlashMessageMixin
from workin.mixins.jinja2 import Jinja2Mixin
from workin.session import Session
from workin.utils import importlib
from workin.routes import Route

class Application(tornado.web.Application):

    db = None
    session_engine = None
    jinja_env = None
    handlers = []
    context_processors = []

    def __init__(self, settings_module):
        settings.configure(settings_module, True)
        self.settings = settings.to_dict()
        self._setup_extensions()
        self._setup_session_engine()
        self._setup_database_engine()
        self._setup_template_loaders()
        self._setup_installed_apps()
        self._setup_context_processors()
        self._setup_uimodules()

        tornado.web.Application.__init__(self, handlers=self.handlers,
                ui_modules=self.ui_modules,
                **self.settings)

        # Due to middleware run_init_hooks will call Application, so execute
        # this at last.
        self._setup_middlewares()

    def __call__(self, request):
        try:
            self.middleware_manager.run_call_hooks(request)
            handler = tornado.web.Application.__call__(self, request)
            self.middleware_manager.run_endcall_hooks(handler)
        except Exception, e:
            logging.error(e)
            raise

    def _setup_uimodules(self):
        from workin import uimodules
        self.ui_modules = uimodules

    def _setup_middlewares(self):
        self.middleware_manager = MiddlewareManager(**self.settings)
        self.middleware_manager.run_init_hooks(self)

    def _setup_context_processors(self):
        # load context processors
        processors = self.settings.get('context_processors', [])
        self.context_processors = [importlib.load_class(x) for x in set(processors)]

    def _setup_database_engine(self):
        from sqlalchemy import create_engine
        from sqlalchemy.orm import scoped_session, sessionmaker
        from workin.database import Base

        engine_url = self.settings['sqlalchemy_engine_url']
        engine_kwargs = self.settings['sqlalchemy_engine_kwargs']

        if not engine_url:
            return

        self.engine = create_engine(engine_url, **engine_kwargs)
        self.db = scoped_session(sessionmaker(bind=self.engine))

        Base.metadata.create_all(bind=self.engine)

    def _setup_session_engine(self):
        if 'session_engine' in self.settings:
            engine = importlib.load_class(self.settings['session_engine'])
            self.session_engine = engine(self.settings["session_secret"],
                    self.settings["session_options"],
                    self.settings["session_timeout"])

    def _setup_template_loaders(self):
        if "template_loader" in self.settings:
            loader = self.settings['template_loader']
        else:
            loader = FileSystemLoader(self.settings['template_dirs'])
        autoescape = bool(self.settings['autoescape'])
        self.jinja_env = Environment(
            loader=loader,
            auto_reload=self.settings['debug'],
            extensions=['workin.utils.jinja2htmlcompress.SelectiveHTMLCompress'],
            autoescape=autoescape, )

    def _setup_installed_apps(self):
        """Auto discovery handlers"""
        if not self.settings.get('installed_apps'):
            return

        for app in self.settings['installed_apps']:
            try:
                importlib.import_module(app + '.handlers')
            except ImportError, e:
                logging.warn("No handlers found in app '%s':"
                        "%s" % (app, e))
            try:
                importlib.import_module(app + '.models')
            except ImportError, e:
                logging.warn("No models/handlers found in app '%s':"
                        "%s" % (app, e))

            try:
                importlib.import_module(app + '.uimodules')
            except ImportError, e:
                logging.warn("No uimodels found in app '%s':"
                        "%s" % (app, e))

        self.handlers.extend(Route.routes())

    def _setup_extensions(self):
        """Auto discovery workin extensions"""
        if not self.settings.get('workin_extensions'):
            return

        from workin.extensions import BaseDiscover, find_extensions
        for ext in self.settings['workin_extensions']:
            discovery = find_extensions(ext)
            if isinstance(discovery, BaseDiscover):
                logging.info("Install workin.exts.admin ...")
                discovery.execute(self)

class BaseHandler(Jinja2Mixin, tornado.web.RequestHandler):

    def _populate_context_from_ctxprocessors(self, context):
        ctx = {}
        ctx.update(context)

        for ctx_processor in self.application.context_processors:
            ctx.update(ctx_processor(self))

        return ctx

    def render_string(self, template_name, **context):
        context = self._populate_context_from_ctxprocessors(context)
        return super(BaseHandler, self).render_string(template_name, **context)

class RequestHandler(BaseHandler, FlashMessageMixin):

    @property
    def db(self):
        return self.application.db

    @property
    def session(self):
        if not hasattr(self, '_session'):
            self._session = Session(self.application.session_engine, self)
        return self._session

    def on_finish(self):
        if self.session:  # and self.session.is_modified:
            self.session.save()

        # remove the sqla-session at the end of each request.
        self.db.remove()

        tornado.web.RequestHandler.on_finish(self)

    def write_json(self, data=None, encoder=json.JSONEncoder):
        if not data:
            data = {}
        self.set_header("Content-Type", "application/json")
        self.write(json.dumps(data, encoder))

    def get_args(self, key, default=None, type=None):
        if type == list:
            if default is None:
                default = []
            return self.get_arguments(key, default)
        value = self.get_argument(key, default)
        if value and type:
            try:
                value = type(value)
            except ValueError:
                value = default
        return value

    @property
    def is_xhr(self):
        '''True if the request was triggered via a JavaScript XMLHttpRequest.
        This only works with libraries that support the `X-Requested-With`
        header and set it to "XMLHttpRequest".  Libraries that do that are
        prototype, jQuery and Mochikit and probably some more.'''
        return self.request.headers.get('X-Requested-With', '') \
                           .lower() == 'xmlhttprequest'

class UIModules(object):
    _uimodules = {}

    def register(self, module_class, name):
        if name in self._uimodules:
            raise ImproperlyConfigured('Duplicate uimodule registered with \
                    name %s' % name)
        self._uimodules.update({name: module_class})

uimodules = UIModules()
<commit_msg>去掉不用的ui modules<commit_after>#!/usr/bin/env python
# encoding: utf-8

import json
import logging

import tornado.web
import tornado.options
from jinja2 import Environment, FileSystemLoader

from workin.conf import settings
from workin.middleware import MiddlewareManager
from workin.mixins.flash_message import FlashMessageMixin
from workin.mixins.jinja2 import Jinja2Mixin
from workin.session import Session
from workin.utils import importlib
from workin.routes import Route

class Application(tornado.web.Application):

    db = None
    session_engine = None
    jinja_env = None
    handlers = []
    context_processors = []

    def __init__(self, settings_module):
        settings.configure(settings_module, True)
        self.settings = settings.to_dict()
        self._setup_extensions()
        self._setup_session_engine()
        self._setup_database_engine()
        self._setup_template_loaders()
        self._setup_installed_apps()
        self._setup_context_processors()

        tornado.web.Application.__init__(self, handlers=self.handlers,
                **self.settings)

        # Due to middleware run_init_hooks will call Application, so execute
        # this at last.
        self._setup_middlewares()

    def __call__(self, request):
        try:
            self.middleware_manager.run_call_hooks(request)
            handler = tornado.web.Application.__call__(self, request)
            self.middleware_manager.run_endcall_hooks(handler)
        except Exception, e:
            logging.error(e)
            raise

    def _setup_middlewares(self):
        self.middleware_manager = MiddlewareManager(**self.settings)
        self.middleware_manager.run_init_hooks(self)

    def _setup_context_processors(self):
        # load context processors
        processors = self.settings.get('context_processors', [])
        self.context_processors = [importlib.load_class(x) for x in set(processors)]

    def _setup_database_engine(self):
        from sqlalchemy import create_engine
        from sqlalchemy.orm import scoped_session, sessionmaker
        from workin.database import Base

        engine_url = self.settings['sqlalchemy_engine_url']
        engine_kwargs = self.settings['sqlalchemy_engine_kwargs']

        if not engine_url:
            return

        self.engine = create_engine(engine_url, **engine_kwargs)
        self.db = scoped_session(sessionmaker(bind=self.engine))

        Base.metadata.create_all(bind=self.engine)

    def _setup_session_engine(self):
        if 'session_engine' in self.settings:
            engine = importlib.load_class(self.settings['session_engine'])
            self.session_engine = engine(self.settings["session_secret"],
                    self.settings["session_options"],
                    self.settings["session_timeout"])

    def _setup_template_loaders(self):
        if "template_loader" in self.settings:
            loader = self.settings['template_loader']
        else:
            loader = FileSystemLoader(self.settings['template_dirs'])
        autoescape = bool(self.settings['autoescape'])
        self.jinja_env = Environment(
            loader=loader,
            auto_reload=self.settings['debug'],
            extensions=['workin.utils.jinja2htmlcompress.SelectiveHTMLCompress'],
            autoescape=autoescape, )

    def _setup_installed_apps(self):
        """Auto discovery handlers"""
        if not self.settings.get('installed_apps'):
            return

        for app in self.settings['installed_apps']:
            try:
                importlib.import_module(app + '.handlers')
            except ImportError, e:
                logging.warn("No handlers found in app '%s':"
                        "%s" % (app, e))
            try:
                importlib.import_module(app + '.models')
            except ImportError, e:
                logging.warn("No models/handlers found in app '%s':"
                        "%s" % (app, e))

        self.handlers.extend(Route.routes())

    def _setup_extensions(self):
        """Auto discovery workin extensions"""
        if not self.settings.get('workin_extensions'):
            return

        from workin.extensions import BaseDiscover, find_extensions
        for ext in self.settings['workin_extensions']:
            discovery = find_extensions(ext)
            if isinstance(discovery, BaseDiscover):
                logging.info("Install workin.exts.admin ...")
                discovery.execute(self)

class BaseHandler(Jinja2Mixin, tornado.web.RequestHandler):

    def _populate_context_from_ctxprocessors(self, context):
        ctx = {}
        ctx.update(context)

        for ctx_processor in self.application.context_processors:
            ctx.update(ctx_processor(self))

        return ctx

    def render_string(self, template_name, **context):
        context = self._populate_context_from_ctxprocessors(context)
        return super(BaseHandler, self).render_string(template_name, **context)

class RequestHandler(BaseHandler, FlashMessageMixin):

    @property
    def db(self):
        return self.application.db

    @property
    def session(self):
        if not hasattr(self, '_session'):
            self._session = Session(self.application.session_engine, self)
        return self._session

    def on_finish(self):
        if self.session:  # and self.session.is_modified:
            self.session.save()

        # remove the sqla-session at the end of each request.
        self.db.remove()

        tornado.web.RequestHandler.on_finish(self)

    def write_json(self, data=None, encoder=json.JSONEncoder):
        if not data:
            data = {}
        self.set_header("Content-Type", "application/json")
        self.write(json.dumps(data, encoder))

    def get_args(self, key, default=None, type=None):
        if type == list:
            if default is None:
                default = []
            return self.get_arguments(key, default)
        value = self.get_argument(key, default)
        if value and type:
            try:
                value = type(value)
            except ValueError:
                value = default
        return value

    @property
    def is_xhr(self):
        '''True if the request was triggered via a JavaScript XMLHttpRequest.
        This only works with libraries that support the `X-Requested-With`
        header and set it to "XMLHttpRequest".  Libraries that do that are
        prototype, jQuery and Mochikit and probably some more.'''
        return self.request.headers.get('X-Requested-With', '') \
                           .lower() == 'xmlhttprequest'
<|endoftext|>


<commit_before>"""cleverhans.model.Model implementation of mnist_challenge.model.Model

This re-implementation factors variable creation apart from forward
propagation so it is possible to run forward propagation more than once
in the same model.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from collections import OrderedDict
import tensorflow as tf
from cleverhans.model import Model
from cleverhans.utils import deterministic_dict

class MadryMNIST(Model):

  def __init__(self, nb_classes=10):
    # NOTE: for compatibility with Madry Lab downloadable checkpoints,
    # we cannot use scopes, give these variables names, etc.
    self.W_conv1 = self._weight_variable([5, 5, 1, 32])
    self.b_conv1 = self._bias_variable([32])
    self.W_conv2 = self._weight_variable([5, 5, 32, 64])
    self.b_conv2 = self._bias_variable([64])
    self.W_fc1 = self._weight_variable([7 * 7 * 64, 1024])
    self.b_fc1 = self._bias_variable([1024])
    self.W_fc2 = self._weight_variable([1024, nb_classes])
    self.b_fc2 = self._bias_variable([nb_classes])
    Model.__init__(self, '', nb_classes, {})

  def fprop(self, x):

    output = OrderedDict()
    # first convolutional layer
    h_conv1 = tf.nn.relu(self._conv2d(x, self.W_conv1) + self.b_conv1)
    h_pool1 = self._max_pool_2x2(h_conv1)

    # second convolutional layer
    h_conv2 = tf.nn.relu(
        self._conv2d(h_pool1, self.W_conv2) + self.b_conv2)
    h_pool2 = self._max_pool_2x2(h_conv2)

    # first fully connected layer

    h_pool2_flat = tf.reshape(h_pool2, [-1, 7 * 7 * 64])
    h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, self.W_fc1) + self.b_fc1)

    # output layer
    logits = tf.matmul(h_fc1, self.W_fc2) + self.b_fc2

    output = deterministic_dict(locals())
    del output["self"]
    output[self.O_PROBS] = tf.nn.softmax(logits=logits)

    return output

  @staticmethod
  def _weight_variable(shape):
    initial = tf.truncated_normal(shape, stddev=0.1)
    return tf.Variable(initial)

  @staticmethod
  def _bias_variable(shape):
    initial = tf.constant(0.1, shape=shape)
    return tf.Variable(initial)

  @staticmethod
  def _conv2d(x, W):
    return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')

  @staticmethod
  def _max_pool_2x2(x):
    return tf.nn.max_pool(x,
                          ksize=[1, 2, 2, 1],
                          strides=[1, 2, 2, 1],
                          padding='SAME')
<commit_msg>maintain compatibility of Madry model<commit_after>"""cleverhans.model.Model implementation of mnist_challenge.model.Model

This re-implementation factors variable creation apart from forward
propagation so it is possible to run forward propagation more than once
in the same model.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from collections import OrderedDict
import tensorflow as tf
from cleverhans.model import Model
from cleverhans.utils import deterministic_dict
from cleverhans.dataset import Factory, MNIST

class MadryMNIST(Model):

  def __init__(self, nb_classes=10):
    # NOTE: for compatibility with Madry Lab downloadable checkpoints,
    # we cannot use scopes, give these variables names, etc.
    self.W_conv1 = self._weight_variable([5, 5, 1, 32])
    self.b_conv1 = self._bias_variable([32])
    self.W_conv2 = self._weight_variable([5, 5, 32, 64])
    self.b_conv2 = self._bias_variable([64])
    self.W_fc1 = self._weight_variable([7 * 7 * 64, 1024])
    self.b_fc1 = self._bias_variable([1024])
    self.W_fc2 = self._weight_variable([1024, nb_classes])
    self.b_fc2 = self._bias_variable([nb_classes])
    Model.__init__(self, '', nb_classes, {})
    self.dataset_factory = Factory(MNIST, {"center": False})

  def get_params(self):
    return [
        self.W_conv1,
        self.b_conv1,
        self.W_conv2,
        self.b_conv2,
        self.W_fc1,
        self.b_fc1,
        self.W_fc2,
        self.b_fc2,
        ]

  def fprop(self, x):

    output = OrderedDict()
    # first convolutional layer
    h_conv1 = tf.nn.relu(self._conv2d(x, self.W_conv1) + self.b_conv1)
    h_pool1 = self._max_pool_2x2(h_conv1)

    # second convolutional layer
    h_conv2 = tf.nn.relu(
        self._conv2d(h_pool1, self.W_conv2) + self.b_conv2)
    h_pool2 = self._max_pool_2x2(h_conv2)

    # first fully connected layer

    h_pool2_flat = tf.reshape(h_pool2, [-1, 7 * 7 * 64])
    h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, self.W_fc1) + self.b_fc1)

    # output layer
    logits = tf.matmul(h_fc1, self.W_fc2) + self.b_fc2

    output = deterministic_dict(locals())
    del output["self"]
    output[self.O_PROBS] = tf.nn.softmax(logits=logits)

    return output

  @staticmethod
  def _weight_variable(shape):
    initial = tf.truncated_normal(shape, stddev=0.1)
    return tf.Variable(initial)

  @staticmethod
  def _bias_variable(shape):
    initial = tf.constant(0.1, shape=shape)
    return tf.Variable(initial)

  @staticmethod
  def _conv2d(x, W):
    return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')

  @staticmethod
  def _max_pool_2x2(x):
    return tf.nn.max_pool(x,
                          ksize=[1, 2, 2, 1],
                          strides=[1, 2, 2, 1],
                          padding='SAME')
<|endoftext|>


<commit_before>import os

import besapi

def main():
    """Execution starts here"""
    print("main()")
    bes_conn = besapi.besapi.get_bes_conn_using_config_file()
    bes_conn.login()

    os.mkdir("export")

    os.chdir("export")

    bes_conn.export_all_sites()

if __name__ == "__main__":
    main()
<commit_msg>add comments<commit_after>"""
This will export all bigfix sites to a folder called `export`

This is equivalent of running `python -m besapi export_all_sites`
"""

import os

import besapi

def main():
    """Execution starts here"""
    print("main()")
    bes_conn = besapi.besapi.get_bes_conn_using_config_file()
    bes_conn.login()

    os.mkdir("export")

    os.chdir("export")

    bes_conn.export_all_sites()

if __name__ == "__main__":
    main()
<|endoftext|>