<commit_before>"""
================================================
Categorical Feature Support in Gradient Boosting
================================================

.. currentmodule:: sklearn

In this example, we will compare the training times and prediction
performances of :class:`~ensemble.HistGradientBoostingRegressor` with
different encoding strategies for categorical features. In
particular, we will evaluate:

- dropping the categorical features
- using a :class:`~preprocessing.OneHotEncoder`
- using an :class:`~preprocessing.OrdinalEncoder` and treat categories as
  ordered, equidistant quantities
- using an :class:`~preprocessing.OrdinalEncoder` and rely on the :ref:`native
  category support <categorical_support_gbdt>` of the
  :class:`~ensemble.HistGradientBoostingRegressor` estimator.

We will work with the Ames Lowa Housing dataset which consists of numerical
and categorical features, where the houses' sales prices is the target.

"""

# %%
# Load Ames Housing dataset
# -------------------------
# First, we load the ames housing data as a pandas dataframe. The features
# are either categorical or numerical:
from sklearn.datasets import fetch_openml

X, y = fetch_openml(data_id=41211, as_frame=True, return_X_y=True)

n_categorical_features = (X.dtypes == "category").sum()
n_numerical_features = (X.dtypes == "float").sum()
print(f"Number of samples: {X.shape[0]}")
print(f"Number of features: {X.shape[1]}")
print(f"Number of categorical features: {n_categorical_features}")
print(f"Number of numerical features: {n_numerical_features}")

# %%
# Gradient boosting estimator with dropped categorical features
# -------------------------------------------------------------
# As a baseline, we create an estimator where the categorical features are
# dropped:

from sklearn.ensemble import HistGradientBoostingRegressor
from sklearn.pipeline import make_pipeline
from sklearn.compose import make_column_transformer
from sklearn.compose import make_column_selector

dropper = make_column_transformer(
    ("drop", make_column_selector(dtype_include="category")), remainder="passthrough"
)
hist_dropped = make_pipeline(dropper, HistGradientBoostingRegressor(random_state=42))

# %%
# Gradient boosting estimator with one-hot encoding
# -------------------------------------------------
# Next, we create a pipeline that will one-hot encode the categorical features
# and let the rest of the numerical data to passthrough:

from sklearn.preprocessing import OneHotEncoder

one_hot_encoder = make_column_transformer(
    (
        OneHotEncoder(sparse=False, handle_unknown="ignore"),
        make_column_selector(dtype_include="category"),
    ),
    remainder="passthrough",
)

hist_one_hot = make_pipeline(
    one_hot_encoder, HistGradientBoostingRegressor(random_state=42)
)

# %%
# Gradient boosting estimator with ordinal encoding
# -------------------------------------------------
# Next, we create a pipeline that will treat categorical features as if they
# were ordered quantities, i.e. the categories will be encoded as 0, 1, 2,
# etc., and treated as continuous features.

from sklearn.preprocessing import OrdinalEncoder
import numpy as np

ordinal_encoder = make_column_transformer(
    (
        OrdinalEncoder(handle_unknown="use_encoded_value", unknown_value=np.nan),
        make_column_selector(dtype_include="category"),
    ),
    remainder="passthrough",
)

hist_ordinal = make_pipeline(
    ordinal_encoder, HistGradientBoostingRegressor(random_state=42)
)

# %%
# Gradient boosting estimator with native categorical support
# -----------------------------------------------------------
# We now create a :class:`~ensemble.HistGradientBoostingRegressor` estimator
# that will natively handle categorical features. This estimator will not treat
# categorical features as ordered quantities.
#
# Since the :class:`~ensemble.HistGradientBoostingRegressor` requires category
# values to be encoded in `[0, n_unique_categories - 1]`, we still rely on an
# :class:`~preprocessing.OrdinalEncoder` to pre-process the data.
#
# The main difference between this pipeline and the previous one is that in
# this one, we let the :class:`~ensemble.HistGradientBoostingRegressor` know
# which features are categorical.

# The ordinal encoder will first output the categorical features, and then the
# continuous (passed-through) features
categorical_mask = [True] * n_categorical_features + [False] * n_numerical_features
hist_native = make_pipeline(
    ordinal_encoder,
    HistGradientBoostingRegressor(
        random_state=42, categorical_features=categorical_mask
    ),
)

# %%
# Model comparison
# ----------------
# Finally, we evaluate the models using cross validation. Here we compare the
# models performance in terms of
# :func:`~metrics.mean_absolute_percentage_error` and fit times.

from sklearn.model_selection import cross_validate
import matplotlib.pyplot as plt

scoring = "neg_mean_absolute_percentage_error"
dropped_result = cross_validate(hist_dropped, X, y, cv=3, scoring=scoring)
one_hot_result = cross_validate(hist_one_hot, X, y, cv=3, scoring=scoring)
ordinal_result = cross_validate(hist_ordinal, X, y, cv=3, scoring=scoring)
native_result = cross_validate(hist_native, X, y, cv=3, scoring=scoring)

def plot_results(figure_title):
    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 8))

    plot_info = [
        ("fit_time", "Fit times (s)", ax1, None),
        ("test_score", "Mean Absolute Percentage Error", ax2, (0, 0.20)),
    ]

    x, width = np.arange(4), 0.9
    for key, title, ax, y_limit in plot_info:
        items = [
            dropped_result[key],
            one_hot_result[key],
            ordinal_result[key],
            native_result[key],
        ]
        ax.bar(
            x,
            [np.mean(np.abs(item)) for item in items],
            width,
            yerr=[np.std(item) for item in items],
            color=["C0", "C1", "C2", "C3"],
        )
        ax.set(
            xlabel="Model",
            title=title,
            xticks=x,
            xticklabels=["Dropped", "One Hot", "Ordinal", "Native"],
            ylim=y_limit,
        )
    fig.suptitle(figure_title)

plot_results("Gradient Boosting on Adult Census")

# %%
# We see that the model with one-hot-encoded data is by far the slowest. This
# is to be expected, since one-hot-encoding creates one additional feature per
# category value (for each categorical feature), and thus more split points
# need to be considered during fitting. In theory, we expect the native
# handling of categorical features to be slightly slower than treating
# categories as ordered quantities ('Ordinal'), since native handling requires
# :ref:`sorting categories <categorical_support_gbdt>`. Fitting times should
# however be close when the number of categories is small, and this may not
# always be reflected in practice.
#
# In terms of prediction performance, dropping the categorical features leads
# to poorer performance. The three models that use categorical features have
# comparable error rates, with a slight edge for the native handling.

# %%
# Limitting the number of splits
# ------------------------------
#
# In general, one can expect poorer predictions from one-hot-encoded data,
# especially when the tree depths or the number of nodes are limited: with
# one-hot-encoded data, one needs more split points, i.e. more depth, in order
# to recover an equivalent split that could be obtained in one single split
# point with native handling.
#
# This is also true when categories are treated as ordinal quantities: if
# categories are `A..F` and the best split is `ACF - BDE` the one-hot-encoder
# model will need 3 split points (one per category in the left node), and the
# ordinal non-native model will need 4 splits: 1 split to isolate `A`, 1 split
# to isolate `F`, and 2 splits to isolate `C` from `BCDE`.
#
# How strongly the models' performances differ in practice will depend on the
# dataset and on the flexibility of the trees.
#
# To see this, let us re-run the same analysis with under-fitting models where
# we artificially limit the total number of splits by both limitting the number
# of trees and the depth of each tree.

for pipe in (hist_dropped, hist_one_hot, hist_ordinal, hist_native):
    pipe.set_params(
        histgradientboostingregressor__max_depth=3,
        histgradientboostingregressor__max_iter=15,
    )

dropped_result = cross_validate(hist_dropped, X, y, cv=3, scoring=scoring)
one_hot_result = cross_validate(hist_one_hot, X, y, cv=3, scoring=scoring)
ordinal_result = cross_validate(hist_ordinal, X, y, cv=3, scoring=scoring)
native_result = cross_validate(hist_native, X, y, cv=3, scoring=scoring)

plot_results("Gradient Boosting on Adult Census (few and small trees)")

plt.show()

# %%
# The results for these under-fitting models confirm our previous intuition:
# the native category handling strategy performs the best when the splitting
# budget is constrained. The two other strategies (one-hot encoding and
# treating categories as ordinal values) lead to error values comparable
# to the baseline model that just dropped the categorical features altogether.
<commit_msg>DOC Speed up example on plot_gradient_boosting_categorical.py  (#21634)<commit_after>"""
================================================
Categorical Feature Support in Gradient Boosting
================================================

.. currentmodule:: sklearn

In this example, we will compare the training times and prediction
performances of :class:`~ensemble.HistGradientBoostingRegressor` with
different encoding strategies for categorical features. In
particular, we will evaluate:

- dropping the categorical features
- using a :class:`~preprocessing.OneHotEncoder`
- using an :class:`~preprocessing.OrdinalEncoder` and treat categories as
  ordered, equidistant quantities
- using an :class:`~preprocessing.OrdinalEncoder` and rely on the :ref:`native
  category support <categorical_support_gbdt>` of the
  :class:`~ensemble.HistGradientBoostingRegressor` estimator.

We will work with the Ames Lowa Housing dataset which consists of numerical
and categorical features, where the houses' sales prices is the target.

"""

# %%
# Load Ames Housing dataset
# -------------------------
# First, we load the Ames Housing data as a pandas dataframe. The features
# are either categorical or numerical:
from sklearn.datasets import fetch_openml

X, y = fetch_openml(data_id=41211, as_frame=True, return_X_y=True)

# Select only a subset of features of X to make the example faster to run
categorical_columns_subset = [
    "Bldg_Type",
    "Garage_Finish",
    "Lot_Config",
    "Functional",
    "Mas_Vnr_Type",
    "House_Style",
    "Fireplace_Qu",
    "Exter_Cond",
    "Exter_Qual",
    "Pool_QC",
]

numerical_columns_subset = [
    "Three_season_porch",
    "Fireplaces",
    "Bsmt_Half_Bath",
    "Half_Bath",
    "Garage_Cars",
    "TotRms_AbvGrd",
    "BsmtFin_SF_1",
    "BsmtFin_SF_2",
    "Gr_Liv_Area",
    "Screen_Porch",
]

X = X[categorical_columns_subset + numerical_columns_subset]

n_categorical_features = X.select_dtypes(include="category").shape[1]
n_numerical_features = X.select_dtypes(include="number").shape[1]

print(f"Number of samples: {X.shape[0]}")
print(f"Number of features: {X.shape[1]}")
print(f"Number of categorical features: {n_categorical_features}")
print(f"Number of numerical features: {n_numerical_features}")

# %%
# Gradient boosting estimator with dropped categorical features
# -------------------------------------------------------------
# As a baseline, we create an estimator where the categorical features are
# dropped:

from sklearn.ensemble import HistGradientBoostingRegressor
from sklearn.pipeline import make_pipeline
from sklearn.compose import make_column_transformer
from sklearn.compose import make_column_selector

dropper = make_column_transformer(
    ("drop", make_column_selector(dtype_include="category")), remainder="passthrough"
)
hist_dropped = make_pipeline(dropper, HistGradientBoostingRegressor(random_state=42))

# %%
# Gradient boosting estimator with one-hot encoding
# -------------------------------------------------
# Next, we create a pipeline that will one-hot encode the categorical features
# and let the rest of the numerical data to passthrough:

from sklearn.preprocessing import OneHotEncoder

one_hot_encoder = make_column_transformer(
    (
        OneHotEncoder(sparse=False, handle_unknown="ignore"),
        make_column_selector(dtype_include="category"),
    ),
    remainder="passthrough",
)

hist_one_hot = make_pipeline(
    one_hot_encoder, HistGradientBoostingRegressor(random_state=42)
)

# %%
# Gradient boosting estimator with ordinal encoding
# -------------------------------------------------
# Next, we create a pipeline that will treat categorical features as if they
# were ordered quantities, i.e. the categories will be encoded as 0, 1, 2,
# etc., and treated as continuous features.

from sklearn.preprocessing import OrdinalEncoder
import numpy as np

ordinal_encoder = make_column_transformer(
    (
        OrdinalEncoder(handle_unknown="use_encoded_value", unknown_value=np.nan),
        make_column_selector(dtype_include="category"),
    ),
    remainder="passthrough",
)

hist_ordinal = make_pipeline(
    ordinal_encoder, HistGradientBoostingRegressor(random_state=42)
)

# %%
# Gradient boosting estimator with native categorical support
# -----------------------------------------------------------
# We now create a :class:`~ensemble.HistGradientBoostingRegressor` estimator
# that will natively handle categorical features. This estimator will not treat
# categorical features as ordered quantities.
#
# Since the :class:`~ensemble.HistGradientBoostingRegressor` requires category
# values to be encoded in `[0, n_unique_categories - 1]`, we still rely on an
# :class:`~preprocessing.OrdinalEncoder` to pre-process the data.
#
# The main difference between this pipeline and the previous one is that in
# this one, we let the :class:`~ensemble.HistGradientBoostingRegressor` know
# which features are categorical.

# The ordinal encoder will first output the categorical features, and then the
# continuous (passed-through) features

categorical_mask = [True] * n_categorical_features + [False] * n_numerical_features
hist_native = make_pipeline(
    ordinal_encoder,
    HistGradientBoostingRegressor(
        random_state=42, categorical_features=categorical_mask
    ),
)

# %%
# Model comparison
# ----------------
# Finally, we evaluate the models using cross validation. Here we compare the
# models performance in terms of
# :func:`~metrics.mean_absolute_percentage_error` and fit times.

from sklearn.model_selection import cross_validate
import matplotlib.pyplot as plt

scoring = "neg_mean_absolute_percentage_error"
n_cv_folds = 3

dropped_result = cross_validate(hist_dropped, X, y, cv=n_cv_folds, scoring=scoring)
one_hot_result = cross_validate(hist_one_hot, X, y, cv=n_cv_folds, scoring=scoring)
ordinal_result = cross_validate(hist_ordinal, X, y, cv=n_cv_folds, scoring=scoring)
native_result = cross_validate(hist_native, X, y, cv=n_cv_folds, scoring=scoring)

def plot_results(figure_title):
    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 8))

    plot_info = [
        ("fit_time", "Fit times (s)", ax1, None),
        ("test_score", "Mean Absolute Percentage Error", ax2, None),
    ]

    x, width = np.arange(4), 0.9
    for key, title, ax, y_limit in plot_info:
        items = [
            dropped_result[key],
            one_hot_result[key],
            ordinal_result[key],
            native_result[key],
        ]

        mape_cv_mean = [np.mean(np.abs(item)) for item in items]
        mape_cv_std = [np.std(item) for item in items]

        ax.bar(
            x=x,
            height=mape_cv_mean,
            width=width,
            yerr=mape_cv_std,
            color=["C0", "C1", "C2", "C3"],
        )
        ax.set(
            xlabel="Model",
            title=title,
            xticks=x,
            xticklabels=["Dropped", "One Hot", "Ordinal", "Native"],
            ylim=y_limit,
        )
    fig.suptitle(figure_title)

plot_results("Gradient Boosting on Ames Housing")

# %%
# We see that the model with one-hot-encoded data is by far the slowest. This
# is to be expected, since one-hot-encoding creates one additional feature per
# category value (for each categorical feature), and thus more split points
# need to be considered during fitting. In theory, we expect the native
# handling of categorical features to be slightly slower than treating
# categories as ordered quantities ('Ordinal'), since native handling requires
# :ref:`sorting categories <categorical_support_gbdt>`. Fitting times should
# however be close when the number of categories is small, and this may not
# always be reflected in practice.
#
# In terms of prediction performance, dropping the categorical features leads
# to poorer performance. The three models that use categorical features have
# comparable error rates, with a slight edge for the native handling.

# %%
# Limitting the number of splits
# ------------------------------
#
# In general, one can expect poorer predictions from one-hot-encoded data,
# especially when the tree depths or the number of nodes are limited: with
# one-hot-encoded data, one needs more split points, i.e. more depth, in order
# to recover an equivalent split that could be obtained in one single split
# point with native handling.
#
# This is also true when categories are treated as ordinal quantities: if
# categories are `A..F` and the best split is `ACF - BDE` the one-hot-encoder
# model will need 3 split points (one per category in the left node), and the
# ordinal non-native model will need 4 splits: 1 split to isolate `A`, 1 split
# to isolate `F`, and 2 splits to isolate `C` from `BCDE`.
#
# How strongly the models' performances differ in practice will depend on the
# dataset and on the flexibility of the trees.
#
# To see this, let us re-run the same analysis with under-fitting models where
# we artificially limit the total number of splits by both limitting the number
# of trees and the depth of each tree.

for pipe in (hist_dropped, hist_one_hot, hist_ordinal, hist_native):
    pipe.set_params(
        histgradientboostingregressor__max_depth=3,
        histgradientboostingregressor__max_iter=15,
    )

dropped_result = cross_validate(hist_dropped, X, y, cv=n_cv_folds, scoring=scoring)
one_hot_result = cross_validate(hist_one_hot, X, y, cv=n_cv_folds, scoring=scoring)
ordinal_result = cross_validate(hist_ordinal, X, y, cv=n_cv_folds, scoring=scoring)
native_result = cross_validate(hist_native, X, y, cv=n_cv_folds, scoring=scoring)

plot_results("Gradient Boosting on Ames Housing (few and small trees)")

plt.show()

# %%
# The results for these under-fitting models confirm our previous intuition:
# the native category handling strategy performs the best when the splitting
# budget is constrained. The two other strategies (one-hot encoding and
# treating categories as ordinal values) lead to error values comparable
# to the baseline model that just dropped the categorical features altogether.
<|endoftext|>


<commit_before>#! /usr/bin/env python

import sys
import os
import logging
file_path = os.path.dirname(__file__)
sys.path.insert(0, file_path)

from flask import Flask, render_template, jsonify, Response
from backend import Backend

app = Flask(__name__)

backend = Backend()

API_DESCRIPTION = """
API DESCRIPTION

		Routes
		Description

		/list
		List all sensors paired with this controller

		/rglist
		Modern list of all sensors paired with this controller, with more info available

		/networkinfo
		Return some debug values about the network and all the values contained in nodes

		/configuration

		Send config values to every awake nodes (e.g.: Interval in seconds to report Sensors Values)

		/restart

        	Restart Z-Wave Network

		/start

        	Start the Z-Wave Network

		/stop

        	Stop the Z-Wave Network

		/healNetworkAndRoutes
		Heal network by asking nodes to rediscover their neighbours + Init new routes

		/healNetworkOnly
		Heal network by asking nodes to rediscover their neighbours

		/healNodeAndRoutes/{$sensor_id}
		Heal sensor $sensor_id by rediscovering its neighbours + Init new routes

		/healNodeOnly/{$sensor_id}
		Heal sensor $sensor_id by rediscovering its neighbours

		/refreshNetworkInfo
		Refresh the informations held by nodes in the network (config & sensor values)

		/sensor/{$sensor_id}/get_temperature
		Return current temperature for sensor $sensor_id

		/sensor/{$sensor_id}/get_humidity
		Return current humidity for sensor $sensor_id

		/sensor/{$sensor_id}/get_luminance
		Return current luminance for sensor $sensor_id

		/sensor/{$sensor_id}/get_motion
		Return presence detection value for sensor $sensor_id

	JSON format :

	{

	  "controller": CONTROLLER_NAME,

          "location": SENSOR_LOCATION,

	  "sensor": SENSOR_ID,

	  "type": SENSOR_TYPE,

	  "updateTime": TIMESTAMP,

	  "value": SENSOR_VALUE

	}

	Fields description :

	controller : String desribing the controller (Raspberry name)

	location : Precise location of the sensor in his network

	sensor : Value describing the sensor (integer value)

	type : String describing the sensor, possible values : "temperature", "humidity", "presence", "luminosity"

	updateTime : Timestamp of last update

	value : Value of the sensor, possible types : float for temperature(deg. celsius), humidity(%age) and luminosity(lux) ; boolean for presence

"""

@app.route('/')
def index():
    return API_DESCRIPTION

@app.route('/list')
def devices_list():
    devices = backend.get_devices()
    if type(devices) is str:
	return devices
    devices_list = ""
    for key, val in devices.items():
	    devices_list += str(key) + "=" + str(val) + "\n"
    return Response(devices_list, mimetype="text/plain")
    #return render_template("index.html")

@app.route('/rglist')
def rglist():
    return backend.rgList()

@app.route('/networkinfo')
def networkinfo():
    return backend.network_info()

@app.route('/restart')
def restart():
    backend.stop()
    time.sleep(5)
    backend.start()
    return "Z-Wave Network Restarted"

@app.route('/start')
def start():
    backend.start()
    return "Z-Wave Network Started"

@app.route('/stop')
def stop():
    backend.stop()
    time.sleep(5)
    return "Z-Wave Network Stopped"

@app.route('/healNetworkAndRoutes')
def healNetworkAndRoutes():
    return backend.healNetwork(True)

@app.route('/healNetworkOnly')
def healNetworkOnly():
    return backend.healNetwork()

@app.route('/healNodeAndRoutes/<node>')
def healNodeAndRoutes(node):
    return backend.healNode(node, True)

@app.route('/healNodeOnly/<node>')
def healNodeOnly(node):
    return backend.healNode(node)

@app.route('/refreshNetworkInfo')
def refreshNetworkInfo():
    return backend.refreshNodes()

@app.route('/sensor/<node>/get_temperature')
def get_temperature(node):
    return backend.temperature(node)

@app.route('/sensor/<node>/get_humidity')
def get_humidity(node):
    return backend.humidity(node)

@app.route('/sensor/<node>/get_luminance')
def get_luminance(node):
    return backend.luminance(node)

@app.route('/sensor/<node>/get_motion')
def get_motion(node):
    return backend.motion(node)

@app.route('/rgtest')
def rgtest():
    return backend.rgtest_method()

#@app.route('/updateNeighbours/<node>')
#def upN(node):
#    return backend.updateNeighboursList(node)

# OLD METHODS

@app.route('/temperature/<node>')
def temperature(node):
    return backend.get_temperature(node)

@app.route('/humidity/<node>')
def humidity(node):
    return backend.get_humidity(node)

@app.route('/luminance/<node>')
def luminance(node):
    return backend.get_brightness(node)

@app.route('/motion/<node>')
def motion(node):
    return backend.get_motion(node)

# CONFIGURATION

@app.route('/configuration')
def configuration():
    return backend.set_sensor()

@app.route('/logtest')
def log():
    return 1/0

from logging import FileHandler, Formatter

if __name__ == '__main__':
    try:
        backend.start()
    	file_handler = FileHandler("flask.log")
    	file_handler.setLevel(logging.DEBUG)
    	file_handler.setFormatter(Formatter('%(asctime)s %(levelname)s: %(message)s [in %(pathname)s:%(lineno)d]'))
    	app.logger.addHandler(file_handler)

        app.run(host='::', debug=False, use_reloader=False)

        #app.run(host='2001:620:607:5e16::314', debug=True, use_reloader=False)
        #app.run(host='pi1.smarthepia.distantaccess.com', debug=True, use_reloader=False)
    except KeyboardInterrupt:
        backend.stop()
<commit_msg>Update flask-main.py<commit_after>#! /usr/bin/env python

import sys
import os
import logging
import time

file_path = os.path.dirname(__file__)
sys.path.insert(0, file_path)

from flask import Flask, render_template, jsonify, Response
from backend import Backend

app = Flask(__name__)

backend = Backend()

API_DESCRIPTION = """
API DESCRIPTION

Network Routes

		Routes
		Description

		/list
		List all sensors paired with this controller

		/rglist
		Modern list of all sensors paired with this controller, with more info available

		/network/info
		Return some debug values about the network and all the values contained in nodes

		/network/configure

		Send config values to every awake nodes (e.g.: Interval in seconds to report Sensors Values)

                /network/configuration

                List the config values in every awake nodes (e.g.: Interval in seconds to report Sensors Values)

                /network/restart

                Restart Z-Wave Network

                /network/start

                Start the Z-Wave Network

                /network/stop

                Stop the Z-Wave Network

                /network/healWithRoutes
                Heal network by asking nodes to rediscover their neighbours + Init new routes

                /network/healOnly
                Heal network by asking nodes to rediscover their neighbours

                /network/refresh
                Refresh the informations held by nodes in the network (config & sensor values)

Sensors Routes

                Routes
                Description

                /sensors/{$sensor_id}/healWithRoutes
                Heal sensor $sensor_id by rediscovering its neighbours + Init new routes

                /sensors/{$sensor_id}/healOnly
                Heal sensor $sensor_id by rediscovering its neighbours

                /sensors/{$sensor_id}/refresh
                Refresh node info for sensor $sensor_id

		/sensor/{$sensor_id}/get_all
		Return all measures (temp, hum, lum, motion) for sensor $sensor_id

		/sensor/{$sensor_id}/get_temperature
		Return current temperature for sensor $sensor_id

		/sensor/{$sensor_id}/get_humidity
		Return current humidity for sensor $sensor_id

		/sensor/{$sensor_id}/get_luminance
		Return current luminance for sensor $sensor_id

		/sensor/{$sensor_id}/get_motion
		Return presence detection value for sensor $sensor_id

                /sensor/add
                Add a device when its add button is pressed

                /sensor/remove
                Remove a device when its button is pressed

                /sensor/{$sensor_id}/set/{$param_index}/to/{$param_value}
                Set $param_value as value for the parameter $param_index

                /sensor/{$sensor_id}/get/{$param_index}
                Get the value for the parameter $param_index

        JSON format :

        {

          "controller": CONTROLLER_NAME,

          "location": SENSOR_LOCATION,

          "sensor": SENSOR_ID,

          "type": SENSOR_TYPE,

          "updateTime": TIMESTAMP,

          "value": SENSOR_VALUE

        }

        Fields description :

        controller : String desribing the controller (Raspberry name)

        location : Precise location of the sensor in his network

        sensor : Value describing the sensor (integer value)

        type : String describing the sensor, possible values : "temperature", "humidity", "presence", "luminosity"

        updateTime : Timestamp of last update

        value : Value of the sensor, possible types : float for temperature(deg. celsius), humidity(%age) and luminosity(lux) ; boolea$

"""

@app.route('/')
def index():
    return API_DESCRIPTION

@app.route('/list')
def devices_list():
    devices = backend.get_devices()
    if type(devices) is str:
	return devices
    devices_list = ""
    for key, val in devices.items():
	    devices_list += str(key) + "=" + str(val) + "\n"
    return Response(devices_list, mimetype="text/plain")
    #return render_template("index.html")

@app.route('/rglist')
def rglist():
    return backend.rgList()

@app.route('/network/info')
def networkinfo():
    return backend.network_info()

@app.route('/network/timestamps')
def networktimestamp():
    return backend.network_timestamp()

@app.route('/network/restart')
def restart():
    backend.stop()
    time.sleep(5)
    backend.start()
    return "Z-Wave Network Restarted"

@app.route('/network/start')
def start():
    backend.start()
    return "Z-Wave Network Started"

@app.route('/network/stop')
def stop():
    backend.stop()
    time.sleep(5)
    return "Z-Wave Network Stopped"

@app.route('/network/healOnly')
def healNetworkOnly():
    return backend.healNetwork()

@app.route('/network/healWithRoutes')
def healNetworkAndRoutes():
    return backend.healNetwork(True)

@app.route('/network/configuration')
def networkConfiguration():
    return backend.networkConfiguration()

@app.route('/network/refresh')
def refreshNetworkInfo():
    return backend.refreshNodes()

############# SENSORS #################

@app.route('/sensor/<node>/healOnly')
def healNodeOnly(node):
    return backend.healNode(node)

@app.route('/sensor/<node>/healWithRoutes')
def healNodeAndRoutes(node):
    return backend.healNode(node, True)

@app.route('/sensor/<node>/refresh')
def refreshNode(node):
    return backend.refreshNode(node)

@app.route('/sensor/<node>/set/<int:param>/to/<int:value>')
def set_config_param(node, param, value):
    return backend.set_node_config_param(node, param, value)

@app.route('/sensor/<node>/set/<int:param>')
def get_config_param(node, param):
    return backend.get_node_config_param(node, param)

@app.route('/sensor/add')
def add_device():
    return backend.addDevice()

@app.route('/sensor/remove')
def remove_device():
    return backend.removeDevice()

@app.route('/sensor/<node>/get_all')
def get_all_measures(node):
    return backend.allMeasures(node)

@app.route('/sensor/<node>/get_temperature')
def get_temperature(node):
    return backend.temperature(node)

@app.route('/sensor/<node>/get_humidity')
def get_humidity(node):
    return backend.humidity(node)

@app.route('/sensor/<node>/get_luminance')
def get_luminance(node):
    return backend.luminance(node)

@app.route('/sensor/<node>/get_motion')
def get_motion(node):
    return backend.motion(node)

@app.route('/rgtest')
def rgtest():
    return backend.rgtest_method()

#@app.route('/updateNeighbours/<node>')
#def upN(node):
#    return backend.updateNeighboursList(node)

# OLD METHODS

@app.route('/temperature/<node>')
def temperature(node):
    return backend.get_temperature(node)

@app.route('/humidity/<node>')
def humidity(node):
    return backend.get_humidity(node)

@app.route('/luminance/<node>')
def luminance(node):
    return backend.get_brightness(node)

@app.route('/motion/<node>')
def motion(node):
    return backend.get_motion(node)

# CONFIGURATION

@app.route('/network/configure')
def configuration():
    return backend.set_sensor()

@app.route('/logtest')
def log():
    return 1/0

from logging import FileHandler, Formatter

if __name__ == '__main__':
    try:
        backend.start()
    	file_handler = FileHandler("flask.log")
    	file_handler.setLevel(logging.DEBUG)
    	file_handler.setFormatter(Formatter('%(asctime)s %(levelname)s: %(message)s [in %(pathname)s:%(lineno)d]'))
    	app.logger.addHandler(file_handler)

        app.run(host='::', debug=False, use_reloader=False)

        #app.run(host='2001:620:607:5e16::314', debug=True, use_reloader=False)
        #app.run(host='pi1.smarthepia.distantaccess.com', debug=True, use_reloader=False)
    except KeyboardInterrupt:
        backend.stop()
<|endoftext|>


<commit_before>#!/usr/bin/env python
#pylint: disable=missing-docstring
#pylint: disable=bad-continuation
#pylint: disable=too-many-instance-attributes

import os
import asyncore
import logging
import hashlib
import random
from Crypto.PublicKey import RSA

# For unit tests
import unittest
import tempfile
import httplib
import shutil
import threading
import simplejson as json
import time

from flock import dbconn
from flock import dht
from flock import nat
from flock import async
from flock import http
from flock import api
from flock import syncgroup

logger = logging.getLogger('http') # pylint: disable=invalid-name

class Node(asyncore.dispatcher):
    def __load_nid(self):
        # Create/load nid
        nid_file = os.path.join(self.store_dir, 'nid')
        if not os.path.exists(nid_file):
            nid = ''.join(chr(random.randint(0, 255)) for _ in range(20))
            with open(nid_file, 'w') as nfile:
                nfile.write(nid)
        with open(nid_file, 'r') as nfile:
            self.nid = nfile.read()
        if len(self.nid) != 20:
            raise ValueError('Invalid nid')

    def __load_syncgroups(self):
        self.syncgroups = {}
        for bname in os.listdir(self.store_dir):
            if len(bname) != 40:
                continue
            try:
                tid = bname.decode('hex')
            except Exception: # pylint: disable=broad-except
                continue
            dbc = dbconn.DbConn(os.path.join(self.store_dir, bname))
            self.syncgroups[tid] = syncgroup.SyncGroup(self.asm, tid, self.nid, dbc)

    def __setup_dht(self):
        # Create DHT object
        self.dht = dht.Dht(self.asm, self.nid, self.cfg)

        # Bootstrap DHT
        bootstrap = [
            ("dht.transmissionbt.com", 6881),
            ("router.bittorrent.com", 6881),
            ("cz.magnets.im", 6881),
            ("de.magnets.im", 6881),
        ]
        for addr in bootstrap:
            self.dht.bootstrap_node(addr)

        # Make a DHT location for each store
        for tid, _ in self.syncgroups.iteritems():
            self.dht.add_location(tid, self.net_conn.ext_port,
                lambda addr, tid=tid: self.on_peer(tid, addr))

    def __init__(self, store_dir, cfg):
        # Create/find path
        self.store_dir = store_dir
        self.cfg = cfg
        if not os.path.exists(self.store_dir):
            os.makedirs(self.store_dir)

        # Make asm
        self.asm = async.AsyncMgr()

        # Load nid + syncgroups
        self.__load_nid()
        self.__load_syncgroups()

        # Nat punch out
        self.net_conn = nat.setup_network(self.cfg)
        if self.net_conn is None:
            logging.error("Unable to find internet connection, bailing")
            # TODO: Is this really the right exception
            raise RuntimeError("Unable to find internet")

        # Attach natted connection to myself
        asyncore.dispatcher.__init__(self, sock=self.net_conn.sock, map=self.asm.async_map)
        self.listen(5)

        # Setup DHT is needed
        if self.cfg.get('use_dht', True):
            self.__setup_dht()
        else:
            self.dht = None

        # Setup API goo
        self.api = api.Api(self)
        self.http = http.HttpServer(self.asm, self.api, self.cfg)

        # Kick off self timer
        self.on_timer()

    def on_timer(self):
        self.asm.add_timer(time.time() + 1, self.on_timer)
        for _, group in self.syncgroups.iteritems():
            group.dbc.commit()
            group.on_timer()

    def poke(self, tid):
        self.syncgroups[tid].poke()

    def handle_accept(self):
        pair = self.accept()
        if pair is None:
            return
        (sock, addr) = pair # pylint: disable=unpacking-non-sequence
        logger.debug("Incoming node connection, addr = %s", addr)
        syncgroup.IncomingSync(self.asm, sock, lambda tid: self.syncgroups[tid])

    def on_peer(self, tid, addr):
        if addr[0] == self.net_conn.ext_ip and addr[1] == self.net_conn.ext_port:
            # Ignore self connections
            return
        if tid not in self.syncgroups:
            # Ignore stuff I don't care about
            return
        self.syncgroups[tid].add_peer(addr)

    def get_store(self, tid):
        if tid not in self.syncgroups:
            return None
        return self.syncgroups[tid].store

    def create_app(self):
        priv_key = RSA.generate(2048)
        pub_key = priv_key.publickey()
        encoded = pub_key.exportKey('DER')
        hid = hashlib.sha256(encoded).digest()
        tid = hid[0:20]
        store_path = os.path.join(self.store_dir, tid.encode('hex'))
        dbc = dbconn.DbConn(store_path)
        self.syncgroups[tid] = syncgroup.SyncGroup(self.asm, tid, self.nid, dbc)
        self.syncgroups[tid].store.set_priv_key(priv_key)
        if self.dht is not None:
            self.dht.add_location(tid, self.net_conn.ext_port,
                lambda addr: self.on_peer(tid, addr))
        return tid

    def join_app(self, tid):
        store_path = os.path.join(self.store_dir, tid.encode('hex'))
        dbc = dbconn.DbConn(store_path)
        self.syncgroups[tid] = syncgroup.SyncGroup(self.asm, tid, self.nid, dbc)
        if self.dht is not None:
            self.dht.add_location(tid, self.net_conn.ext_port,
                lambda addr: self.on_peer(tid, addr))

    def run(self):
        self.asm.run()

class TestNodes(unittest.TestCase):
    def setUp(self):
        logger.debug("Doing test setup")
        self.cur_port = 30000
        self.next_node = 1
        self.nodes = []
        self.threads = []
        self.store_dir = tempfile.mkdtemp()

    def tearDown(self):
        logger.debug("Shutting down")
        for node in self.nodes:
            node.asm.stop()
        for thread in self.threads:
            thread.join()
        logger.debug("Threads stopped")
        shutil.rmtree(self.store_dir)

    def __next_port(self):
        result = self.cur_port
        self.cur_port += 1
        return result

    def setup_node(self):
        cfg = {}
        cfg['sync_local'] = True
        cfg['use_dht'] = False
        cfg['sync_port'] = self.__next_port()
        cfg['http_port'] = self.__next_port()
        store_dir = os.path.join(self.store_dir, str(self.next_node))
        self.next_node += 1
        node = Node(store_dir, cfg)
        thread = threading.Thread(target=node.run)
        thread.daemon = True
        thread.start()
        self.nodes.append(node)
        self.threads.append(thread)
        return node

    def send_post(self, node, loc, obj):
        _ = self
        conn = httplib.HTTPConnection("localhost:" + str(node.http.port))
        headers = {"Content-type": "application/json"}
        body = json.dumps(obj)
        conn.request("POST", loc, body, headers)
        response = conn.getresponse()
        if response.status != 200:
            conn.close()
            logger.warn("Got non 200 response to post: %d %s", response.status, response.reason)
            return None
        body = response.read()
        conn.close()
        return json.loads(body)

    def send_put(self, node, tid, key, value):
        _ = self
        conn = httplib.HTTPConnection("localhost:" + str(node.http.port))
        headers = {"Content-type": "application/json"}
        conn.request("PUT", '/' + tid + '/' + key, value, headers)
        response = conn.getresponse()
        _ = response.read()
        conn.close()
        return response.status

    def send_get(self, node, tid, key):
        _ = self
        conn = httplib.HTTPConnection("localhost:" + str(node.http.port))
        conn.request("GET", '/' + tid + '/' + key)
        response = conn.getresponse()
        value = response.read()
        conn.close()
        if response.status != 200:
            return None
        return value

    def connect(self, tid, node1, node2):
        _ = self
        port = node2.net_conn.int_port
        result = self.send_post(node1, '/' + tid + '/add_peer',
            {'addr' : '127.0.0.1', 'port' : port})
        if not result['success']:
            raise Exception("Failed to issue add_peer")

    def test_create_store(self):
        test_schema1 = json.dumps({
            "schema": {
                "test_table": {
                    "foo" : "int",
                }
            },
            "indexes": [
                ["test_table", "foo"]
            ]
        })
        test_schema2 = json.dumps({
            "schema": {
                "test_table": {
                    "foo" : "int",
                    "bar" : "text"
                }
            },
            "indexes": [
                ["test_table", "bar"]
            ]
        })
        json_record1 = {
            "_table" : "test_table",
            "foo" : 10,
            "bar" : "hello"
        }
        json_record2 = {
            "_table" : "test_table",
            "foo" : 20,
            "bar" : "world"
        }

        node1 = self.setup_node()
        node2 = self.setup_node()
        time.sleep(1)
        resp = self.send_post(node1, "/create_app", {'max_size' : 100000})
        self.assertTrue(resp['success'])
        logger.debug("Got resp: %s", resp)
        tid = resp['tid']
        status = self.send_put(node1, tid, 'foo', 'Hello')
        self.assertTrue(status == 204)
        status = self.send_put(node1, tid, '_schema', test_schema1)
        self.assertTrue(status == 204)
        resp = self.send_post(node1, "/" + tid + "/add_record", json_record1)
        self.assertTrue(resp['success'])
        resp = self.send_post(node2, "/" + tid + "/join_app", {'max_size' : 100000})
        self.assertTrue(resp['success'])
        self.connect(tid, node1, node2)
        self.connect(tid, node2, node1)
        time.sleep(5)
        val = self.send_get(node2, tid, 'foo')
        self.assertTrue(val == 'Hello')
        resp = self.send_post(node2, "/" + tid + "/query", {
            "query" : "SELECT id, foo FROM test_table"
        })
        self.assertTrue(resp['results'][0][1] == 10)
        status = self.send_put(node1, tid, '_schema', test_schema2)
        self.assertTrue(status == 204)
        status = self.send_put(node1, tid, 'bar', 'World')
        self.assertTrue(status == 204)
        resp = self.send_post(node1, "/" + tid + "/add_record", json_record2)
        self.assertTrue(resp['success'])
        time.sleep(5)
        val = self.send_get(node2, tid, 'bar')
        self.assertTrue(val == 'World')
        resp = self.send_post(node2, "/" + tid + "/query", {
            "query" : "SELECT bar FROM test_table WHERE foo = ?",
            "params" : [20]
        })
        logger.debug("Send resp: %s", resp)
        self.assertTrue(resp['results'][0][0] == 'world')

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)
    unittest.main()

<commit_msg>Better error message<commit_after>#!/usr/bin/env python
#pylint: disable=missing-docstring
#pylint: disable=bad-continuation
#pylint: disable=too-many-instance-attributes

import os
import asyncore
import logging
import hashlib
import random
from Crypto.PublicKey import RSA

# For unit tests
import unittest
import tempfile
import httplib
import shutil
import threading
import simplejson as json
import time

from flock import dbconn
from flock import dht
from flock import nat
from flock import async
from flock import http
from flock import api
from flock import syncgroup

logger = logging.getLogger('http') # pylint: disable=invalid-name

class Node(asyncore.dispatcher):
    def __load_nid(self):
        # Create/load nid
        nid_file = os.path.join(self.store_dir, 'nid')
        if not os.path.exists(nid_file):
            nid = ''.join(chr(random.randint(0, 255)) for _ in range(20))
            with open(nid_file, 'w') as nfile:
                nfile.write(nid)
        with open(nid_file, 'r') as nfile:
            self.nid = nfile.read()
        if len(self.nid) != 20:
            raise ValueError('Invalid nid')

    def __load_syncgroups(self):
        self.syncgroups = {}
        for bname in os.listdir(self.store_dir):
            if len(bname) != 40:
                continue
            try:
                tid = bname.decode('hex')
            except Exception: # pylint: disable=broad-except
                continue
            dbc = dbconn.DbConn(os.path.join(self.store_dir, bname))
            self.syncgroups[tid] = syncgroup.SyncGroup(self.asm, tid, self.nid, dbc)

    def __setup_dht(self):
        # Create DHT object
        self.dht = dht.Dht(self.asm, self.nid, self.cfg)

        # Bootstrap DHT
        bootstrap = [
            ("dht.transmissionbt.com", 6881),
            ("router.bittorrent.com", 6881),
            ("cz.magnets.im", 6881),
            ("de.magnets.im", 6881),
        ]
        for addr in bootstrap:
            self.dht.bootstrap_node(addr)

        # Make a DHT location for each store
        for tid, _ in self.syncgroups.iteritems():
            self.dht.add_location(tid, self.net_conn.ext_port,
                lambda addr, tid=tid: self.on_peer(tid, addr))

    def __init__(self, store_dir, cfg):
        # Create/find path
        self.store_dir = store_dir
        self.cfg = cfg
        if not os.path.exists(self.store_dir):
            os.makedirs(self.store_dir)

        # Make asm
        self.asm = async.AsyncMgr()

        # Load nid + syncgroups
        self.__load_nid()
        self.__load_syncgroups()

        # Nat punch out
        self.net_conn = nat.setup_network(self.cfg)
        if self.net_conn is None:
            logging.error("Unable to find internet connection, bailing")
            # TODO: Is this really the right exception
            raise RuntimeError("Unable to find internet")

        # Attach natted connection to myself
        asyncore.dispatcher.__init__(self, sock=self.net_conn.sock, map=self.asm.async_map)
        self.listen(5)

        # Setup DHT is needed
        if self.cfg.get('use_dht', True):
            self.__setup_dht()
        else:
            self.dht = None

        # Setup API goo
        self.api = api.Api(self)
        self.http = http.HttpServer(self.asm, self.api, self.cfg)

        # Kick off self timer
        self.on_timer()

    def on_timer(self):
        self.asm.add_timer(time.time() + 1, self.on_timer)
        for _, group in self.syncgroups.iteritems():
            group.dbc.commit()
            group.on_timer()

    def poke(self, tid):
        self.syncgroups[tid].poke()

    def handle_accept(self):
        pair = self.accept()
        if pair is None:
            return
        (sock, addr) = pair # pylint: disable=unpacking-non-sequence
        logger.debug("Incoming node connection, addr = %s", addr)
        syncgroup.IncomingSync(self.asm, sock, self.tid_to_group)

    def tid_to_group(self, tid):
        if tid not in self.syncgroups:
            raise ValueError("Incoming conneciton, unknown sync group")
        return self.syncgroups[tid]

    def on_peer(self, tid, addr):
        if addr[0] == self.net_conn.ext_ip and addr[1] == self.net_conn.ext_port:
            # Ignore self connections
            return
        if tid not in self.syncgroups:
            # Ignore stuff I don't care about
            return
        self.syncgroups[tid].add_peer(addr)

    def get_store(self, tid):
        if tid not in self.syncgroups:
            return None
        return self.syncgroups[tid].store

    def create_app(self):
        priv_key = RSA.generate(2048)
        pub_key = priv_key.publickey()
        encoded = pub_key.exportKey('DER')
        hid = hashlib.sha256(encoded).digest()
        tid = hid[0:20]
        store_path = os.path.join(self.store_dir, tid.encode('hex'))
        dbc = dbconn.DbConn(store_path)
        self.syncgroups[tid] = syncgroup.SyncGroup(self.asm, tid, self.nid, dbc)
        self.syncgroups[tid].store.set_priv_key(priv_key)
        if self.dht is not None:
            self.dht.add_location(tid, self.net_conn.ext_port,
                lambda addr: self.on_peer(tid, addr))
        return tid

    def join_app(self, tid):
        store_path = os.path.join(self.store_dir, tid.encode('hex'))
        dbc = dbconn.DbConn(store_path)
        self.syncgroups[tid] = syncgroup.SyncGroup(self.asm, tid, self.nid, dbc)
        if self.dht is not None:
            self.dht.add_location(tid, self.net_conn.ext_port,
                lambda addr: self.on_peer(tid, addr))

    def run(self):
        self.asm.run()

class TestNodes(unittest.TestCase):
    def setUp(self):
        logger.debug("Doing test setup")
        self.cur_port = 30000
        self.next_node = 1
        self.nodes = []
        self.threads = []
        self.store_dir = tempfile.mkdtemp()

    def tearDown(self):
        logger.debug("Shutting down")
        for node in self.nodes:
            node.asm.stop()
        for thread in self.threads:
            thread.join()
        logger.debug("Threads stopped")
        shutil.rmtree(self.store_dir)

    def __next_port(self):
        result = self.cur_port
        self.cur_port += 1
        return result

    def setup_node(self):
        cfg = {}
        cfg['sync_local'] = True
        cfg['use_dht'] = False
        cfg['sync_port'] = self.__next_port()
        cfg['http_port'] = self.__next_port()
        store_dir = os.path.join(self.store_dir, str(self.next_node))
        self.next_node += 1
        node = Node(store_dir, cfg)
        thread = threading.Thread(target=node.run)
        thread.daemon = True
        thread.start()
        self.nodes.append(node)
        self.threads.append(thread)
        return node

    def send_post(self, node, loc, obj):
        _ = self
        conn = httplib.HTTPConnection("localhost:" + str(node.http.port))
        headers = {"Content-type": "application/json"}
        body = json.dumps(obj)
        conn.request("POST", loc, body, headers)
        response = conn.getresponse()
        if response.status != 200:
            conn.close()
            logger.warn("Got non 200 response to post: %d %s", response.status, response.reason)
            return None
        body = response.read()
        conn.close()
        return json.loads(body)

    def send_put(self, node, tid, key, value):
        _ = self
        conn = httplib.HTTPConnection("localhost:" + str(node.http.port))
        headers = {"Content-type": "application/json"}
        conn.request("PUT", '/' + tid + '/' + key, value, headers)
        response = conn.getresponse()
        _ = response.read()
        conn.close()
        return response.status

    def send_get(self, node, tid, key):
        _ = self
        conn = httplib.HTTPConnection("localhost:" + str(node.http.port))
        conn.request("GET", '/' + tid + '/' + key)
        response = conn.getresponse()
        value = response.read()
        conn.close()
        if response.status != 200:
            return None
        return value

    def connect(self, tid, node1, node2):
        _ = self
        port = node2.net_conn.int_port
        result = self.send_post(node1, '/' + tid + '/add_peer',
            {'addr' : '127.0.0.1', 'port' : port})
        if not result['success']:
            raise Exception("Failed to issue add_peer")

    def test_create_store(self):
        test_schema1 = json.dumps({
            "schema": {
                "test_table": {
                    "foo" : "int",
                }
            },
            "indexes": [
                ["test_table", "foo"]
            ]
        })
        test_schema2 = json.dumps({
            "schema": {
                "test_table": {
                    "foo" : "int",
                    "bar" : "text"
                }
            },
            "indexes": [
                ["test_table", "bar"]
            ]
        })
        json_record1 = {
            "_table" : "test_table",
            "foo" : 10,
            "bar" : "hello"
        }
        json_record2 = {
            "_table" : "test_table",
            "foo" : 20,
            "bar" : "world"
        }

        node1 = self.setup_node()
        node2 = self.setup_node()
        time.sleep(1)
        resp = self.send_post(node1, "/create_app", {'max_size' : 100000})
        self.assertTrue(resp['success'])
        logger.debug("Got resp: %s", resp)
        tid = resp['tid']
        status = self.send_put(node1, tid, 'foo', 'Hello')
        self.assertTrue(status == 204)
        status = self.send_put(node1, tid, '_schema', test_schema1)
        self.assertTrue(status == 204)
        resp = self.send_post(node1, "/" + tid + "/add_record", json_record1)
        self.assertTrue(resp['success'])
        resp = self.send_post(node2, "/" + tid + "/join_app", {'max_size' : 100000})
        self.assertTrue(resp['success'])
        self.connect(tid, node1, node2)
        self.connect(tid, node2, node1)
        time.sleep(5)
        val = self.send_get(node2, tid, 'foo')
        self.assertTrue(val == 'Hello')
        resp = self.send_post(node2, "/" + tid + "/query", {
            "query" : "SELECT id, foo FROM test_table"
        })
        self.assertTrue(resp['results'][0][1] == 10)
        status = self.send_put(node1, tid, '_schema', test_schema2)
        self.assertTrue(status == 204)
        status = self.send_put(node1, tid, 'bar', 'World')
        self.assertTrue(status == 204)
        resp = self.send_post(node1, "/" + tid + "/add_record", json_record2)
        self.assertTrue(resp['success'])
        time.sleep(5)
        val = self.send_get(node2, tid, 'bar')
        self.assertTrue(val == 'World')
        resp = self.send_post(node2, "/" + tid + "/query", {
            "query" : "SELECT bar FROM test_table WHERE foo = ?",
            "params" : [20]
        })
        logger.debug("Send resp: %s", resp)
        self.assertTrue(resp['results'][0][0] == 'world')

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)
    unittest.main()

<|endoftext|>


<commit_before>"""Send mails to interested parties about pre-specified conditions."""
# =============================================================================
# CONTENTS
# -----------------------------------------------------------------------------
# abdmail_mailer
#
# Public Classes:
#   Mailer
#    .noUsersOnBranch
#
# -----------------------------------------------------------------------------
# (this contents block is generated, edits will be lost)
# =============================================================================
from __future__ import absolute_import

import types

class Mailer(object):

    """Send mails to interested parties about pre-specified conditions."""

    def __init__(self, mail_sender, admin_emails, repository_name, uri):
        """Initialise, simply store the supplied parameters.

        :mail_sender: supports send(to, cc, subject, message)
        :admin_emails: list of string, who to tell when no appropriate users
        :repository_name: the repository that is in context
        :uri: the address of the phabricator instance

        """
        self._mail_sender = mail_sender
        assert not isinstance(
            admin_emails, types.StringTypes), "should be list not string"
        self._admin_emails = admin_emails
        self._repository_name = repository_name
        self._uri = uri

    def noUsersOnBranch(self, branch_name, branch_base, emails):
        # TODO: determine which of 'emails' we're permitted to send to
        msg = ""
        msg += "No registered Phabricator users were found when\n"
        msg += "trying to create a review from a branch.\n"
        msg += "\n"
        msg += "repository:     " + self._repository_name + "\n"
        msg += "branch:         " + branch_name + "\n"
        msg += "base branch:    " + branch_base + "\n"
        msg += "unknown emails: " + str(emails) + "\n"
        msg += "\n"
        msg += "If you appear in the 'unknown emails' list then\n"
        msg += "please register by visiting this link, simply\n"
        msg += "logging in will resolve the issue:\n"
        msg += "  " + self._uri + "\n"
        msg += "\n"
        msg += "You are receiving this message because you are\n"
        msg += "either in the unknown email list or an admin.\n"
        to = []
        to.extend(self._admin_emails)
        to.extend(emails)
        self._mail_sender.send(
            to_addresses=to,
            subject="user exception",
            message=msg)

#------------------------------------------------------------------------------
# Copyright (C) 2012 Bloomberg L.P.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
#------------------------------- END-OF-FILE ----------------------------------
<commit_msg>abdmail_mailer: refactor, format whole string<commit_after>"""Send mails to interested parties about pre-specified conditions."""
# =============================================================================
# CONTENTS
# -----------------------------------------------------------------------------
# abdmail_mailer
#
# Public Classes:
#   Mailer
#    .noUsersOnBranch
#
# -----------------------------------------------------------------------------
# (this contents block is generated, edits will be lost)
# =============================================================================
from __future__ import absolute_import

import textwrap
import types

class Mailer(object):

    """Send mails to interested parties about pre-specified conditions."""

    def __init__(self, mail_sender, admin_emails, repository_name, uri):
        """Initialise, simply store the supplied parameters.

        :mail_sender: supports send(to, cc, subject, message)
        :admin_emails: list of string, who to tell when no appropriate users
        :repository_name: the repository that is in context
        :uri: the address of the phabricator instance

        """
        self._mail_sender = mail_sender
        assert not isinstance(
            admin_emails, types.StringTypes), "should be list not string"
        self._admin_emails = admin_emails
        self._repository_name = repository_name
        self._uri = uri

    def noUsersOnBranch(self, branch_name, branch_base, emails):
        # TODO: determine which of 'emails' we're permitted to send to
        msg = textwrap.dedent("""\
            No registered Phabricator users were found when
            trying to create a review from a branch.

            repository:     {repo}
            branch:         {branch}
            base branch:    {base_branch}
            unknown emails: {emails}

            If you appear in the 'unknown emails' list then
            please register by visiting this link, simply
            logging in will resolve the issue:

                {uri}

            You are receiving this message because you are
            either in the unknown email list or an admin.
        """).format(
            repo=self._repository_name,
            branch=branch_name,
            base_branch=branch_base,
            emails=str(emails),
            uri=self._uri
        )
        to = []
        to.extend(self._admin_emails)
        to.extend(emails)
        self._mail_sender.send(
            to_addresses=to,
            subject="user exception",
            message=msg)

#------------------------------------------------------------------------------
# Copyright (C) 2012 Bloomberg L.P.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
#------------------------------- END-OF-FILE ----------------------------------
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2010 Tiny SPRL (<http://tiny.be>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
import time

import netsvc
from osv import osv, fields
from tools.translate import _

class account_automatic_reconcile(osv.osv_memory):
    _name = 'account.automatic.reconcile'
    _description = 'Automatic Reconcile'

    _columns = {
        'account_ids': fields.many2many('account.account', 'reconcile_account_rel', 'reconcile_id', 'account_id', 'Account to reconcile', domain = [('reconcile','=',1)], \
                                        help = 'If no account is specified, the reconciliation will be made using every accounts that can be reconcilied'),
        'writeoff_acc_id': fields.many2one('account.account', 'Account', required=True),
        'journal_id': fields.many2one('account.journal', 'Journal', required=True),
        'period_id': fields.many2one('account.period', 'Period', required=True),
        'max_amount': fields.float('Maximum write-off amount'),
        'power': fields.selection([(p, str(p)) for p in range(2, 10)], required=True),
        'date1': fields.date('Start of period', required=True),
        'date2': fields.date('End of period', required=True),
        'reconciled': fields.integer('Reconciled transactions', readonly=True),
        'unreconciled': fields.integer('Not reconciled transactions', readonly=True),
    }

    def _get_reconciled(self, cr, uid, context={}):
        return context.get('reconciled', 0)

    def _get_unreconciled(self, cr, uid, context={}):
        return context.get('unreconciled', 0)

    _defaults = {
        'date1': time.strftime('%Y-01-01'),
        'date2': time.strftime('%Y-%m-%d'),
        'reconciled': _get_reconciled,
        'unreconciled': _get_unreconciled,
    }

    #TODO: cleanup and comment this code... For now, it is awfulllll
    # (way too complex, and really slow)...
    def do_reconcile(self, cr, uid, credits, debits, max_amount, power, writeoff_acc_id, period_id, journal_id, context=None):
    # for one value of a credit, check all debits, and combination of them
    # depending on the power. It starts with a power of one and goes up
    # to the max power allowed
        move_line_obj = self.pool.get('account.move.line')
        if context is None:
            context = {}
        def check2(value, move_list, power):
            def check(value, move_list, power):
                for i in range(len(move_list)):
                    move = move_list[i]
                    if power == 1:
                        if abs(value - move[1]) <= max_amount + 0.00001:
                            return [move[0]]
                    else:
                        del move_list[i]
                        res = check(value - move[1], move_list, power-1)
                        move_list[i:i] = [move]
                        if res:
                            res.append(move[0])
                            return res
                return False

            for p in range(1, power+1):
                res = check(value, move_list, p)
                if res:
                    return res
            return False

        # for a list of credit and debit and a given power, check if there
        # are matching tuples of credit and debits, check all debits, and combination of them
        # depending on the power. It starts with a power of one and goes up
        # to the max power allowed
        def check4(list1, list2, power):
            def check3(value, list1, list2, list1power, power):
                for i in range(len(list1)):
                    move = list1[i]
                    if list1power == 1:
                        res = check2(value + move[1], list2, power - 1)
                        if res:
                            return ([move[0]], res)
                    else:
                        del list1[i]
                        res = check3(value + move[1], list1, list2, list1power-1, power-1)
                        list1[i:i] = [move]
                        if res:
                            x, y = res
                            x.append(move[0])
                            return (x, y)
                return False

            for p in range(1, power):
                res = check3(0, list1, list2, p, power)
                if res:
                    return res
            return False

        def check5(list1, list2, max_power):
            for p in range(2, max_power+1):
                res = check4(list1, list2, p)
                if res:
                    return res

        ok = True
        reconciled = 0
        while credits and debits and ok:
            res = check5(credits, debits, power)
            if res:
                move_line_obj.reconcile(cr, uid, res[0] + res[1], 'auto', writeoff_acc_id, period_id, journal_id, context)
                reconciled += len(res[0]) + len(res[1])
                credits = [(id, credit) for (id, credit) in credits if id not in res[0]]
                debits = [(id, debit) for (id, debit) in debits if id not in res[1]]
            else:
                ok = False
        return (reconciled, len(credits)+len(debits))

    def reconcile(self, cr, uid, ids, context=None):
        service = netsvc.LocalService("object_proxy")
        move_line_obj = self.pool.get('account.move.line')
        obj_model = self.pool.get('ir.model.data')
        if context is None:
            context = {}
        form = self.read(cr, uid, ids, [])[0]
        max_amount = form.get('max_amount', 0.0)
        power = form['power']
        reconciled = unreconciled = 0
        if not form['account_ids']:
            raise osv.except_osv(_('UserError'), _('You must select accounts to reconcile'))
        for account_id in form['account_ids']:
            # reconcile automatically all transactions from partners whose balance is 0
            cr.execute(
                "SELECT partner_id " \
                "FROM account_move_line " \
                "WHERE account_id=%s " \
                "AND reconcile_id IS NULL " \
                "AND state <> 'draft' " \
                "GROUP BY partner_id " \
                "HAVING ABS(SUM(debit-credit)) < %s AND count(*)>0",
                (account_id, max_amount or 0.0))
            partner_ids = [id for (id,) in cr.fetchall()]
            for partner_id in partner_ids:
                cr.execute(
                    "SELECT id " \
                    "FROM account_move_line " \
                    "WHERE account_id=%s " \
                    "AND partner_id=%s " \
                    "AND state <> 'draft' " \
                    "AND reconcile_id IS NULL",
                    (account_id, partner_id))
                line_ids = [id for (id,) in cr.fetchall()]
                if len(line_ids):
                    move_line_obj.reconcile(cr, uid, line_ids, 'auto', form['writeoff_acc_id'], form['period_id'], form['journal_id'], context)
                    reconciled += len(line_ids)

            # get the list of partners who have more than one unreconciled transaction
            cr.execute(
                "SELECT partner_id " \
                "FROM account_move_line " \
                "WHERE account_id=%s " \
                "AND reconcile_id IS NULL " \
                "AND state <> 'draft' " \
                "AND partner_id IS NOT NULL " \
                "GROUP BY partner_id " \
                "HAVING count(*)>1",
                (account_id,))
            partner_ids = [id for (id,) in cr.fetchall()]
            #filter?
            for partner_id in partner_ids:
                # get the list of unreconciled 'debit transactions' for this partner
                cr.execute(
                    "SELECT id, debit " \
                    "FROM account_move_line " \
                    "WHERE account_id=%s " \
                    "AND partner_id=%s " \
                    "AND reconcile_id IS NULL " \
                    "AND state <> 'draft' " \
                    "AND debit > 0",
                    (account_id, partner_id))
                debits = cr.fetchall()

                # get the list of unreconciled 'credit transactions' for this partner
                cr.execute(
                    "SELECT id, credit " \
                    "FROM account_move_line " \
                    "WHERE account_id=%s " \
                    "AND partner_id=%s " \
                    "AND reconcile_id IS NULL " \
                    "AND state <> 'draft' " \
                    "AND credit > 0",
                    (account_id, partner_id))
                credits = cr.fetchall()

                (rec, unrec) = self.do_reconcile(cr, uid, credits, debits, max_amount, power, form['writeoff_acc_id'], form['period_id'], form['journal_id'], context)
                reconciled += rec
                unreconciled += unrec

            # add the number of transactions for partners who have only one
            # unreconciled transactions to the unreconciled count
            partner_filter = partner_ids and 'AND partner_id not in (%s)' % ','.join(map(str, filter(None, partner_ids))) or ''
            cr.execute(
                "SELECT count(*) " \
                "FROM account_move_line " \
                "WHERE account_id=%s " \
                "AND reconcile_id IS NULL " \
                "AND state <> 'draft' " + partner_filter,
                (account_id,))
            additional_unrec = cr.fetchone()[0]
        context.update({'reconciled': reconciled, 'unreconciled': unreconciled + additional_unrec})
        model_data_ids = obj_model.search(cr,uid,[('model','=','ir.ui.view'),('name','=','account_automatic_reconcile_view1')])
        resource_id = obj_model.read(cr, uid, model_data_ids, fields=['res_id'])[0]['res_id']
        return {
            'view_type': 'form',
            'view_mode': 'form',
            'res_model': 'account.automatic.reconcile',
            'views': [(resource_id,'form')],
            'type': 'ir.actions.act_window',
            'target': 'new',
            'context': context,
            'nodestroy':True,
        }

account_automatic_reconcile()

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:<commit_msg>[IMP] account: Add Label to field in account_automatic_reconcile wizard<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2010 Tiny SPRL (<http://tiny.be>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
import time

import netsvc
from osv import osv, fields
from tools.translate import _

class account_automatic_reconcile(osv.osv_memory):
    _name = 'account.automatic.reconcile'
    _description = 'Automatic Reconcile'

    _columns = {
        'account_ids': fields.many2many('account.account', 'reconcile_account_rel', 'reconcile_id', 'account_id', 'Account to reconcile', domain = [('reconcile','=',1)], \
                                        help = 'If no account is specified, the reconciliation will be made using every accounts that can be reconcilied'),
        'writeoff_acc_id': fields.many2one('account.account', 'Account', required=True),
        'journal_id': fields.many2one('account.journal', 'Journal', required=True),
        'period_id': fields.many2one('account.period', 'Period', required=True),
        'max_amount': fields.float('Maximum write-off amount'),
        'power': fields.selection([(p, str(p)) for p in range(2, 10)], 'Power', required=True),
        'date1': fields.date('Start of period', required=True),
        'date2': fields.date('End of period', required=True),
        'reconciled': fields.integer('Reconciled transactions', readonly=True),
        'unreconciled': fields.integer('Not reconciled transactions', readonly=True),
    }

    def _get_reconciled(self, cr, uid, context={}):
        return context.get('reconciled', 0)

    def _get_unreconciled(self, cr, uid, context={}):
        return context.get('unreconciled', 0)

    _defaults = {
        'date1': time.strftime('%Y-01-01'),
        'date2': time.strftime('%Y-%m-%d'),
        'reconciled': _get_reconciled,
        'unreconciled': _get_unreconciled,
    }

    #TODO: cleanup and comment this code... For now, it is awfulllll
    # (way too complex, and really slow)...
    def do_reconcile(self, cr, uid, credits, debits, max_amount, power, writeoff_acc_id, period_id, journal_id, context=None):
    # for one value of a credit, check all debits, and combination of them
    # depending on the power. It starts with a power of one and goes up
    # to the max power allowed
        move_line_obj = self.pool.get('account.move.line')
        if context is None:
            context = {}
        def check2(value, move_list, power):
            def check(value, move_list, power):
                for i in range(len(move_list)):
                    move = move_list[i]
                    if power == 1:
                        if abs(value - move[1]) <= max_amount + 0.00001:
                            return [move[0]]
                    else:
                        del move_list[i]
                        res = check(value - move[1], move_list, power-1)
                        move_list[i:i] = [move]
                        if res:
                            res.append(move[0])
                            return res
                return False

            for p in range(1, power+1):
                res = check(value, move_list, p)
                if res:
                    return res
            return False

        # for a list of credit and debit and a given power, check if there
        # are matching tuples of credit and debits, check all debits, and combination of them
        # depending on the power. It starts with a power of one and goes up
        # to the max power allowed
        def check4(list1, list2, power):
            def check3(value, list1, list2, list1power, power):
                for i in range(len(list1)):
                    move = list1[i]
                    if list1power == 1:
                        res = check2(value + move[1], list2, power - 1)
                        if res:
                            return ([move[0]], res)
                    else:
                        del list1[i]
                        res = check3(value + move[1], list1, list2, list1power-1, power-1)
                        list1[i:i] = [move]
                        if res:
                            x, y = res
                            x.append(move[0])
                            return (x, y)
                return False

            for p in range(1, power):
                res = check3(0, list1, list2, p, power)
                if res:
                    return res
            return False

        def check5(list1, list2, max_power):
            for p in range(2, max_power+1):
                res = check4(list1, list2, p)
                if res:
                    return res

        ok = True
        reconciled = 0
        while credits and debits and ok:
            res = check5(credits, debits, power)
            if res:
                move_line_obj.reconcile(cr, uid, res[0] + res[1], 'auto', writeoff_acc_id, period_id, journal_id, context)
                reconciled += len(res[0]) + len(res[1])
                credits = [(id, credit) for (id, credit) in credits if id not in res[0]]
                debits = [(id, debit) for (id, debit) in debits if id not in res[1]]
            else:
                ok = False
        return (reconciled, len(credits)+len(debits))

    def reconcile(self, cr, uid, ids, context=None):
        service = netsvc.LocalService("object_proxy")
        move_line_obj = self.pool.get('account.move.line')
        obj_model = self.pool.get('ir.model.data')
        if context is None:
            context = {}
        form = self.read(cr, uid, ids, [])[0]
        max_amount = form.get('max_amount', 0.0)
        power = form['power']
        reconciled = unreconciled = 0
        if not form['account_ids']:
            raise osv.except_osv(_('UserError'), _('You must select accounts to reconcile'))
        for account_id in form['account_ids']:
            # reconcile automatically all transactions from partners whose balance is 0
            cr.execute(
                "SELECT partner_id " \
                "FROM account_move_line " \
                "WHERE account_id=%s " \
                "AND reconcile_id IS NULL " \
                "AND state <> 'draft' " \
                "GROUP BY partner_id " \
                "HAVING ABS(SUM(debit-credit)) < %s AND count(*)>0",
                (account_id, max_amount or 0.0))
            partner_ids = [id for (id,) in cr.fetchall()]
            for partner_id in partner_ids:
                cr.execute(
                    "SELECT id " \
                    "FROM account_move_line " \
                    "WHERE account_id=%s " \
                    "AND partner_id=%s " \
                    "AND state <> 'draft' " \
                    "AND reconcile_id IS NULL",
                    (account_id, partner_id))
                line_ids = [id for (id,) in cr.fetchall()]
                if len(line_ids):
                    move_line_obj.reconcile(cr, uid, line_ids, 'auto', form['writeoff_acc_id'], form['period_id'], form['journal_id'], context)
                    reconciled += len(line_ids)

            # get the list of partners who have more than one unreconciled transaction
            cr.execute(
                "SELECT partner_id " \
                "FROM account_move_line " \
                "WHERE account_id=%s " \
                "AND reconcile_id IS NULL " \
                "AND state <> 'draft' " \
                "AND partner_id IS NOT NULL " \
                "GROUP BY partner_id " \
                "HAVING count(*)>1",
                (account_id,))
            partner_ids = [id for (id,) in cr.fetchall()]
            #filter?
            for partner_id in partner_ids:
                # get the list of unreconciled 'debit transactions' for this partner
                cr.execute(
                    "SELECT id, debit " \
                    "FROM account_move_line " \
                    "WHERE account_id=%s " \
                    "AND partner_id=%s " \
                    "AND reconcile_id IS NULL " \
                    "AND state <> 'draft' " \
                    "AND debit > 0",
                    (account_id, partner_id))
                debits = cr.fetchall()

                # get the list of unreconciled 'credit transactions' for this partner
                cr.execute(
                    "SELECT id, credit " \
                    "FROM account_move_line " \
                    "WHERE account_id=%s " \
                    "AND partner_id=%s " \
                    "AND reconcile_id IS NULL " \
                    "AND state <> 'draft' " \
                    "AND credit > 0",
                    (account_id, partner_id))
                credits = cr.fetchall()

                (rec, unrec) = self.do_reconcile(cr, uid, credits, debits, max_amount, power, form['writeoff_acc_id'], form['period_id'], form['journal_id'], context)
                reconciled += rec
                unreconciled += unrec

            # add the number of transactions for partners who have only one
            # unreconciled transactions to the unreconciled count
            partner_filter = partner_ids and 'AND partner_id not in (%s)' % ','.join(map(str, filter(None, partner_ids))) or ''
            cr.execute(
                "SELECT count(*) " \
                "FROM account_move_line " \
                "WHERE account_id=%s " \
                "AND reconcile_id IS NULL " \
                "AND state <> 'draft' " + partner_filter,
                (account_id,))
            additional_unrec = cr.fetchone()[0]
        context.update({'reconciled': reconciled, 'unreconciled': unreconciled + additional_unrec})
        model_data_ids = obj_model.search(cr,uid,[('model','=','ir.ui.view'),('name','=','account_automatic_reconcile_view1')])
        resource_id = obj_model.read(cr, uid, model_data_ids, fields=['res_id'])[0]['res_id']
        return {
            'view_type': 'form',
            'view_mode': 'form',
            'res_model': 'account.automatic.reconcile',
            'views': [(resource_id,'form')],
            'type': 'ir.actions.act_window',
            'target': 'new',
            'context': context,
            'nodestroy':True,
        }

account_automatic_reconcile()

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:<|endoftext|>