<commit_before># No shebang line, this module is meant to be imported
#
# Copyright 2014 Oliver Palmer
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from os import urandom
from random import randint
from uuid import UUID

try:
    from httplib import ACCEPTED, BAD_REQUEST, CONFLICT, SERVICE_UNAVAILABLE
except ImportError:  # pragma: no cover
    from http.client import ACCEPTED, BAD_REQUEST, CONFLICT, SERVICE_UNAVAILABLE

from twisted.web.server import NOT_DONE_YET

from pyfarm.agent.config import config
from pyfarm.agent.http.api.assign import Assign
from pyfarm.agent.sysinfo.memory import total_ram
from pyfarm.agent.sysinfo.cpu import total_cpus
from pyfarm.agent.testutil import BaseAPITestCase
from pyfarm.jobtypes.core.jobtype import JobType

FAKE_JOBTYPE = """
from twisted.internet.defer import Deferred
from pyfarm.jobtypes.core.jobtype import JobType

class FakeJobType(JobType):
    def __init__(self, assignment):
        super(FakeJobType, self).__init__(assignment)
        self.fake_started = Deferred()
        self.fake_stopped = Deferred()

    def stop(self):
        return self.fake_stopped

    def start(self):
        return self.fake_started
"""

FAKE_JOBTYPE_BAD_TYPE = """
from twisted.internet.defer import Deferred

class FakeJobType(object):
    def __init__(self, assignment):
        self.star_called = False
        self.started = Deferred()

    def start(self):
        self.star_called = True
        return self.started
"""

class TestAssign(BaseAPITestCase):
    URI = "/assign"
    CLASS = Assign

    def setUp(self):
        super(TestAssign, self).setUp()
        self.data = {
            "job": {
                "title": urandom(16).encode("hex"),
                "id": randint(0, 1024),
                "by": 1},
            "jobtype": {
                "name": "TestJobType" + urandom(16).encode("hex"),
                "version": randint(1, 256)},
            "tasks": [
                {"id": randint(0, 1024), "frame": randint(0, 1024)},
                {"id": randint(0, 1024), "frame": randint(0, 1024)},
                {"id": randint(0, 1024), "frame": randint(0, 1024)}]}

    def prepare_config(self):
        super(TestAssign, self).prepare_config()
        config.update({
            "cpus": randint(1, 16),
            "agent-id": randint(1, 2048)})

    def test_restarting(self):
        config["restart_requested"] = True
        request = self.post(
            data=self.data,
            headers={"User-Agent": config["master_user_agent"]})
        assign = Assign()
        result = assign.render(request)
        self.assertTrue(request.finished)
        self.assertEqual(request.code, SERVICE_UNAVAILABLE)
        self.assertEqual(result, NOT_DONE_YET)
        self.assertEqual(
            request.response()["error"],
            "Agent cannot accept assignments because of a pending restart")

    def test_agent_id_not_set(self):
        config.pop("agent-id", None)
        request = self.post(
            data=self.data,
            headers={"User-Agent": config["master_user_agent"]})
        assign = Assign()
        result = assign.render(request)
        self.assertTrue(request.finished)
        self.assertEqual(request.code, SERVICE_UNAVAILABLE)
        self.assertEqual(result, NOT_DONE_YET)
        self.assertEqual(
            request.response()["error"],
            "agent-id has not been set in the config")

    def test_not_enough_ram(self):
        self.data["job"]["ram"] = total_ram() * 10
        request = self.post(
            data=self.data,
            headers={"User-Agent": config["master_user_agent"]})
        assign = Assign()
        result = assign.render(request)
        self.assertTrue(request.finished)
        self.assertEqual(request.code, BAD_REQUEST)
        self.assertEqual(result, NOT_DONE_YET)
        self.assertEqual(request.response()["error"], "Not enough ram")

    def test_not_enough_cpus(self):
        self.data["job"]["cpus"] = int(total_cpus() * 10)
        request = self.post(
            data=self.data,
            headers={"User-Agent": config["master_user_agent"]})
        assign = Assign()
        result = assign.render(request)
        self.assertTrue(request.finished)
        self.assertEqual(request.code, BAD_REQUEST)
        self.assertEqual(result, NOT_DONE_YET)
        response = request.response()
        self.assertEqual(response["requires_cpus"], int(total_cpus() * 10))
        self.assertEqual(response["error"], "Not enough cpus")

    def test_duplicate_task(self):
        # Add the assignments in data to the config so we can make sure
        # the endpoint will reject the request
        tasks = []
        config["current_assignments"] = {}
        assignment = config["current_assignments"][
            self.data["job"]["id"]] = {"tasks": []}
        for task in self.data["tasks"]:
            tasks.append(task["id"])
            assignment["tasks"].append(task)

        request = self.post(
            data=self.data,
            headers={"User-Agent": config["master_user_agent"]})
        assign = self.instance_class()
        result = assign.render(request)
        self.assertTrue(request.finished)
        self.assertEqual(request.code, CONFLICT)
        self.assertEqual(result, NOT_DONE_YET)
        response = request.response()
        self.assertEqual(set(response["duplicate_tasks"]), set(tasks))
        self.assertEqual(response["error"], "Double assignment of tasks")

    def test_accepted(self):
        # Cache the fake job type and make sure the config
        # turns off caching
        jobtype = {
            "classname": "FakeJobType",
            "code": FAKE_JOBTYPE,
            "name": self.data["jobtype"]["name"],
            "version": self.data["jobtype"]["version"]}
        JobType.cache[(self.data["jobtype"]["name"],
                       self.data["jobtype"]["version"])] = (jobtype, None)
        config.update(
            jobtype_enable_cache=False,
            current_assignments={})
        request = self.post(
            data=self.data,
            headers={"User-Agent": config["master_user_agent"]})
        assign = self.instance_class()
        result = assign.render(request)
        self.assertEqual(result, NOT_DONE_YET)
        self.assertTrue(request.finished)
        response = request.response()
        self.assertEqual(request.code, ACCEPTED)
        response_id = UUID(response["id"])
        self.assertIn(response_id, config["current_assignments"])

        # An assignment uuid has been added
        test_data = self.data.copy()
        current_assignment = config["current_assignments"][response_id].copy()

        # Update the original test data with the new assignment data
        # and make sure it matches
        test_data.update(id=response_id)
        # TODO: this fails with the latest code, may have something to
        # do with the new deferreds
        test_data["jobtype"].update(id=current_assignment["jobtype"]["id"])
        self.assertEqual(current_assignment, test_data)
        self.assertIn(current_assignment["jobtype"]["id"], config["jobtypes"])

        # Now trigger the started callback so we can make sure the job
        # type gets removed
        jobtype = config["jobtypes"][current_assignment["jobtype"]["id"]]
        jobtype.fake_started.callback(None)
        jobtype.fake_stopped.callback(None)
        self.assertNotIn(response_id, config["current_assignments"])

    def test_accepted_type_error(self):
        # Cache the fake job type and make sure the config
        # turns off caching
        jobtype = {
            "classname": "FakeJobType",
            "code": FAKE_JOBTYPE_BAD_TYPE,
            "name": self.data["jobtype"]["name"],
            "version": self.data["jobtype"]["version"]}
        JobType.cache[(self.data["jobtype"]["name"],
                       self.data["jobtype"]["version"])] = (jobtype, None)

        # with self.assertRaises(TypeError):
        config.update(
            jobtype_enable_cache=False,
            current_assignments={})
        request = self.post(
            data=self.data,
            headers={"User-Agent": config["master_user_agent"]})
        assign = self.instance_class()
        result = assign.render(request)
        self.assertEqual(result, NOT_DONE_YET)
        self.assertTrue(request.finished)
        self.assertEqual(request.code, ACCEPTED)
        response = request.response()
        response_id = UUID(response["id"])
        self.assertNotIn(response_id, config["current_assignments"])
<commit_msg>Provide and use a factory for Assign resources<commit_after># No shebang line, this module is meant to be imported
#
# Copyright 2014 Oliver Palmer
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from os import urandom
from random import randint
from uuid import UUID

try:
    from httplib import ACCEPTED, BAD_REQUEST, CONFLICT, SERVICE_UNAVAILABLE
except ImportError:  # pragma: no cover
    from http.client import ACCEPTED, BAD_REQUEST, CONFLICT, SERVICE_UNAVAILABLE

from twisted.web.server import NOT_DONE_YET

from pyfarm.agent.config import config
from pyfarm.agent.http.api.assign import Assign
from pyfarm.agent.sysinfo.memory import total_ram
from pyfarm.agent.sysinfo.cpu import total_cpus
from pyfarm.agent.testutil import BaseAPITestCase
from pyfarm.jobtypes.core.jobtype import JobType

FAKE_JOBTYPE = """
from twisted.internet.defer import Deferred
from pyfarm.jobtypes.core.jobtype import JobType

class FakeJobType(JobType):
    def __init__(self, assignment):
        super(FakeJobType, self).__init__(assignment)
        self.fake_started = Deferred()
        self.fake_stopped = Deferred()

    def stop(self):
        return self.fake_stopped

    def start(self):
        return self.fake_started
"""

FAKE_JOBTYPE_BAD_TYPE = """
from twisted.internet.defer import Deferred

class FakeJobType(object):
    def __init__(self, assignment):
        self.star_called = False
        self.started = Deferred()

    def start(self):
        self.star_called = True
        return self.started
"""

class FakeAgent(object):
    def __init__(self):
       self.shutting_down = False

fake_agent = FakeAgent()

class AssignFactory(object):
    def __init__(self, fake_agent):
        self.fake_agent = fake_agent

    def __call__(self):
        return Assign(self.fake_agent)

class TestAssign(BaseAPITestCase):
    URI = "/assign"
    CLASS_FACTORY = AssignFactory(fake_agent)
    CLASS = Assign

    def setUp(self):
        super(TestAssign, self).setUp()
        self.data = {
            "job": {
                "title": urandom(16).encode("hex"),
                "id": randint(0, 1024),
                "by": 1},
            "jobtype": {
                "name": "TestJobType" + urandom(16).encode("hex"),
                "version": randint(1, 256)},
            "tasks": [
                {"id": randint(0, 1024), "frame": randint(0, 1024)},
                {"id": randint(0, 1024), "frame": randint(0, 1024)},
                {"id": randint(0, 1024), "frame": randint(0, 1024)}]}

    def prepare_config(self):
        super(TestAssign, self).prepare_config()
        config.update({
            "cpus": randint(1, 16),
            "agent-id": randint(1, 2048)})

    def test_restarting(self):
        config["restart_requested"] = True
        request = self.post(
            data=self.data,
            headers={"User-Agent": config["master_user_agent"]})
        assign = self.instance_class()
        result = assign.render(request)
        self.assertTrue(request.finished)
        self.assertEqual(request.code, SERVICE_UNAVAILABLE)
        self.assertEqual(result, NOT_DONE_YET)
        self.assertEqual(
            request.response()["error"],
            "Agent cannot accept assignments because of a pending restart")

    def test_agent_id_not_set(self):
        config.pop("agent-id", None)
        request = self.post(
            data=self.data,
            headers={"User-Agent": config["master_user_agent"]})
        assign = self.instance_class()
        result = assign.render(request)
        self.assertTrue(request.finished)
        self.assertEqual(request.code, SERVICE_UNAVAILABLE)
        self.assertEqual(result, NOT_DONE_YET)
        self.assertEqual(
            request.response()["error"],
            "agent-id has not been set in the config")

    def test_not_enough_ram(self):
        self.data["job"]["ram"] = total_ram() * 10
        request = self.post(
            data=self.data,
            headers={"User-Agent": config["master_user_agent"]})
        assign = self.instance_class()
        result = assign.render(request)
        self.assertTrue(request.finished)
        self.assertEqual(request.code, BAD_REQUEST)
        self.assertEqual(result, NOT_DONE_YET)
        self.assertEqual(request.response()["error"], "Not enough ram")

    def test_not_enough_cpus(self):
        self.data["job"]["cpus"] = int(total_cpus() * 10)
        request = self.post(
            data=self.data,
            headers={"User-Agent": config["master_user_agent"]})
        assign = self.instance_class()
        result = assign.render(request)
        self.assertTrue(request.finished)
        self.assertEqual(request.code, BAD_REQUEST)
        self.assertEqual(result, NOT_DONE_YET)
        response = request.response()
        self.assertEqual(response["requires_cpus"], int(total_cpus() * 10))
        self.assertEqual(response["error"], "Not enough cpus")

    def test_duplicate_task(self):
        # Add the assignments in data to the config so we can make sure
        # the endpoint will reject the request
        tasks = []
        config["current_assignments"] = {}
        assignment = config["current_assignments"][
            self.data["job"]["id"]] = {"tasks": []}
        for task in self.data["tasks"]:
            tasks.append(task["id"])
            assignment["tasks"].append(task)

        request = self.post(
            data=self.data,
            headers={"User-Agent": config["master_user_agent"]})
        assign = self.instance_class()
        result = assign.render(request)
        self.assertTrue(request.finished)
        self.assertEqual(request.code, CONFLICT)
        self.assertEqual(result, NOT_DONE_YET)
        response = request.response()
        self.assertEqual(set(response["duplicate_tasks"]), set(tasks))
        self.assertEqual(response["error"], "Double assignment of tasks")

    def test_accepted(self):
        # Cache the fake job type and make sure the config
        # turns off caching
        jobtype = {
            "classname": "FakeJobType",
            "code": FAKE_JOBTYPE,
            "name": self.data["jobtype"]["name"],
            "version": self.data["jobtype"]["version"]}
        JobType.cache[(self.data["jobtype"]["name"],
                       self.data["jobtype"]["version"])] = (jobtype, None)
        config.update(
            jobtype_enable_cache=False,
            current_assignments={})
        request = self.post(
            data=self.data,
            headers={"User-Agent": config["master_user_agent"]})
        assign = self.instance_class()
        result = assign.render(request)
        self.assertEqual(result, NOT_DONE_YET)
        self.assertTrue(request.finished)
        response = request.response()
        self.assertEqual(request.code, ACCEPTED)
        response_id = UUID(response["id"])
        self.assertIn(response_id, config["current_assignments"])

        # An assignment uuid has been added
        test_data = self.data.copy()
        current_assignment = config["current_assignments"][response_id].copy()

        # Update the original test data with the new assignment data
        # and make sure it matches
        test_data.update(id=response_id)
        # TODO: this fails with the latest code, may have something to
        # do with the new deferreds
        test_data["jobtype"].update(id=current_assignment["jobtype"]["id"])
        self.assertEqual(current_assignment, test_data)
        self.assertIn(current_assignment["jobtype"]["id"], config["jobtypes"])

        # Now trigger the started callback so we can make sure the job
        # type gets removed
        jobtype = config["jobtypes"][current_assignment["jobtype"]["id"]]
        jobtype.fake_started.callback(None)
        jobtype.fake_stopped.callback(None)
        self.assertNotIn(response_id, config["current_assignments"])

    def test_accepted_type_error(self):
        # Cache the fake job type and make sure the config
        # turns off caching
        jobtype = {
            "classname": "FakeJobType",
            "code": FAKE_JOBTYPE_BAD_TYPE,
            "name": self.data["jobtype"]["name"],
            "version": self.data["jobtype"]["version"]}
        JobType.cache[(self.data["jobtype"]["name"],
                       self.data["jobtype"]["version"])] = (jobtype, None)

        # with self.assertRaises(TypeError):
        config.update(
            jobtype_enable_cache=False,
            current_assignments={})
        request = self.post(
            data=self.data,
            headers={"User-Agent": config["master_user_agent"]})
        assign = self.instance_class()
        result = assign.render(request)
        self.assertEqual(result, NOT_DONE_YET)
        self.assertTrue(request.finished)
        self.assertEqual(request.code, ACCEPTED)
        response = request.response()
        response_id = UUID(response["id"])
        self.assertNotIn(response_id, config["current_assignments"])
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import
from functools import partial
from pprint import pprint

import pytest
from sklearn.datasets import make_regression
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.ensemble import (
    ExtraTreesClassifier,
    ExtraTreesRegressor,
    GradientBoostingClassifier,
    GradientBoostingRegressor,
    RandomForestClassifier,
    RandomForestRegressor,
)
from sklearn.linear_model import (
    ElasticNet,
    ElasticNetCV,
    HuberRegressor,
    Lars,
    LarsCV,
    Lasso,
    LassoCV,
    LassoLars,
    LassoLarsCV,
    LassoLarsIC,
    LinearRegression,
    LogisticRegression,
    LogisticRegressionCV,
    OrthogonalMatchingPursuit,
    OrthogonalMatchingPursuitCV,
    PassiveAggressiveClassifier,
    PassiveAggressiveRegressor,
    Perceptron,
    Ridge,
    RidgeClassifier,
    RidgeClassifierCV,
    RidgeCV,
    SGDClassifier,
    SGDRegressor,
    TheilSenRegressor,
)
from sklearn.svm import LinearSVC, LinearSVR
from sklearn.multiclass import OneVsRestClassifier
from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor

from eli5 import explain_prediction
from eli5.formatters import format_as_text, fields
from eli5.sklearn.utils import has_intercept
from .utils import (
    format_as_all, strip_blanks, get_all_features, check_targets_scores)

format_as_all = partial(format_as_all, show_feature_values=True)

def assert_multiclass_linear_classifier_explained(newsgroups_train, clf,
                                                  explain_prediction):
    docs, y, target_names = newsgroups_train
    vec = TfidfVectorizer()

    X = vec.fit_transform(docs)
    clf.fit(X, y)

    get_res = lambda: explain_prediction(
        clf, docs[0], vec=vec, target_names=target_names, top=20)
    res = get_res()
    pprint(res)
    expl_text, expl_html = format_as_all(res, clf)

    for e in res.targets:
        if e.target != 'comp.graphics':
            continue
        pos = get_all_features(e.feature_weights.pos)
        assert 'file' in pos

    for expl in [expl_text, expl_html]:
        for label in target_names:
            assert str(label) in expl
        assert 'file' in expl

    assert res == get_res()

def assert_linear_regression_explained(boston_train, reg, explain_prediction,
                                       atol=1e-8):
    X, y, feature_names = boston_train
    reg.fit(X, y)
    res = explain_prediction(reg, X[0], feature_names=feature_names)
    expl_text, expl_html = expls = format_as_all(res, reg)

    assert len(res.targets) == 1
    target = res.targets[0]
    assert target.target == 'y'
    pos, neg = (get_all_features(target.feature_weights.pos),
                get_all_features(target.feature_weights.neg))
    assert 'LSTAT' in pos or 'LSTAT' in neg

    if has_intercept(reg):
        assert '<BIAS>' in pos or '<BIAS>' in neg
        assert '<BIAS>' in expl_text
        assert '<BIAS>' in expl_html
    else:
        assert '<BIAS>' not in pos and '<BIAS>' not in neg
        assert '<BIAS>' not in expl_text
        assert 'BIAS' not in expl_html

    for expl in [expl_text, expl_html]:
        assert 'LSTAT' in expl
        assert '(score' in expl
    assert "'y'" in expl_text
    assert 'y' in strip_blanks(expl_html)

    for expl in expls:
        assert_feature_values_present(expl, feature_names, X[0])

    assert res == explain_prediction(reg, X[0], feature_names=feature_names)
    check_targets_scores(res, atol=atol)

def assert_multitarget_linear_regression_explained(reg, explain_prediction):
    X, y = make_regression(n_samples=100, n_targets=3, n_features=10,
                           random_state=42)
    reg.fit(X, y)
    res = explain_prediction(reg, X[0])
    expl_text, expl_html = format_as_all(res, reg)

    assert len(res.targets) == 3
    target = res.targets[1]
    assert target.target == 'y1'
    pos, neg = (get_all_features(target.feature_weights.pos),
                get_all_features(target.feature_weights.neg))
    assert 'x8' in pos or 'x8' in neg
    assert '<BIAS>' in pos or '<BIAS>' in neg

    assert 'x8' in expl_text
    assert '<BIAS>' in expl_text
    assert "'y2'" in expl_text

    assert res == explain_prediction(reg, X[0])
    check_targets_scores(res)

def assert_feature_values_present(expl, feature_names, x):
    assert 'Value' in expl
    any_features = False
    for feature, value in zip(feature_names, x):
        if feature in expl:
            assert '{:+.3f}'.format(value) in expl
            any_features = True
    assert any_features

@pytest.mark.parametrize(['clf'], [
    [LogisticRegression(random_state=42)],
    [LogisticRegression(random_state=42, multi_class='multinomial', solver='lbfgs')],
    [LogisticRegression(random_state=42, fit_intercept=False)],
    [LogisticRegressionCV(random_state=42)],
    [SGDClassifier(random_state=42)],
    [SGDClassifier(loss='log', random_state=42)],
    [PassiveAggressiveClassifier(random_state=42)],
    [Perceptron(random_state=42)],
    [RidgeClassifier(random_state=42)],
    [RidgeClassifierCV()],
    [LinearSVC(random_state=42)],
    [OneVsRestClassifier(LogisticRegression(random_state=42))],
])
def test_explain_linear(newsgroups_train, clf):
    assert_multiclass_linear_classifier_explained(newsgroups_train, clf,
                                                  explain_prediction)

@pytest.mark.parametrize(['reg'], [
    [ElasticNet(random_state=42)],
    [ElasticNetCV(random_state=42)],
    [HuberRegressor()],
    [Lars()],
    [LarsCV(max_n_alphas=10)],
    [Lasso(random_state=42)],
    [LassoCV(n_alphas=10)],
    [LassoLars(alpha=0.1)],
    [LassoLarsCV(max_n_alphas=10)],
    [LassoLarsIC()],
    [LinearRegression()],
    [LinearSVR(random_state=42)],
    [OrthogonalMatchingPursuit(n_nonzero_coefs=10)],
    [OrthogonalMatchingPursuitCV()],
    [PassiveAggressiveRegressor(C=0.1)],
    [Ridge(random_state=42)],
    [RidgeCV()],
    [SGDRegressor(random_state=42)],
    [TheilSenRegressor()],
])
def test_explain_linear_regression(boston_train, reg):
    assert_linear_regression_explained(boston_train, reg, explain_prediction)

@pytest.mark.parametrize(['reg'], [
    [ElasticNet(random_state=42)],
    [Lars()],
    [Lasso(random_state=42)],
    [LinearRegression()],
    [Ridge(random_state=42)],
])
def test_explain_linear_regression_multitarget(reg):
    assert_multitarget_linear_regression_explained(reg, explain_prediction)

@pytest.mark.parametrize(['clf'], [
    [DecisionTreeClassifier()],
    [ExtraTreesClassifier()],
    [GradientBoostingClassifier(learning_rate=0.075)],
    [RandomForestClassifier()],
])
def test_explain_tree_clf_multiclass(clf, iris_train):
    X, y, feature_names, target_names = iris_train
    clf.fit(X, y)
    res = explain_prediction(
        clf, X[0], target_names=target_names, feature_names=feature_names)
    for expl in format_as_all(res, clf):
        for target in target_names:
            assert target in expl
        assert 'BIAS' in expl
        assert any(f in expl for f in feature_names)
        assert_feature_values_present(expl, feature_names, X[0])
    check_targets_scores(res)

@pytest.mark.parametrize(['clf'], [
    [DecisionTreeClassifier()],
    [ExtraTreesClassifier()],
    [GradientBoostingClassifier(learning_rate=0.075)],
    [RandomForestClassifier()],
])
def test_explain_tree_clf_binary(clf, iris_train_binary):
    X, y, feature_names = iris_train_binary
    clf.fit(X, y)
    res = explain_prediction(clf, X[0], feature_names=feature_names)
    format_as_all(res, clf)
    all_expls = []
    for x in X[:5]:
        res = explain_prediction(clf, x, feature_names=feature_names)
        text_expl = format_as_text(res, show=fields.WEIGHTS)
        print(text_expl)
        assert '<BIAS>' in text_expl
        check_targets_scores(res)
        all_expls.append(text_expl)
    assert any(f in ''.join(all_expls) for f in feature_names)

@pytest.mark.parametrize(['reg'], [
    [DecisionTreeRegressor()],
    [ExtraTreesRegressor()],
    [RandomForestRegressor()],
])
def test_explain_tree_regressor_multitarget(reg):
    X, y = make_regression(n_samples=100, n_targets=3, n_features=10,
                           random_state=42)
    reg.fit(X, y)
    res = explain_prediction(reg, X[0])
    for expl in format_as_all(res, reg):
        for target in ['y0', 'y1', 'y2']:
            assert target in expl
        assert 'BIAS' in expl
        assert any('x%d' % i in expl for i in range(10))
    check_targets_scores(res)

@pytest.mark.parametrize(['reg'], [
    [DecisionTreeRegressor()],
    [ExtraTreesRegressor()],
    [GradientBoostingRegressor(learning_rate=0.075)],
    [RandomForestRegressor()],
])
def test_explain_tree_regressor(reg, boston_train):
    X, y, feature_names = boston_train
    reg.fit(X, y)
    res = explain_prediction(reg, X[0], feature_names=feature_names)
    for expl in format_as_all(res, reg):
        assert_feature_values_present(expl, feature_names, X[0])
    all_expls = []
    for i, x in enumerate(X[:5]):
        res = explain_prediction(reg, x, feature_names=feature_names)
        text_expl = format_as_text(res, show=fields.WEIGHTS)
        print(text_expl)
        assert '<BIAS>' in text_expl
        check_targets_scores(res)
        all_expls.append(text_expl)
    assert any(f in ''.join(all_expls) for f in feature_names)

@pytest.mark.parametrize(['clf'], [
    [DecisionTreeClassifier()],
    [ExtraTreesClassifier()],
    [RandomForestClassifier()],
])
def test_explain_tree_classifier_text(clf, newsgroups_train_big):
    docs, y, target_names = newsgroups_train_big
    vec = CountVectorizer(binary=True, stop_words='english')
    X = vec.fit_transform(docs)
    clf.fit(X, y)
    res = explain_prediction(clf, docs[0], vec=vec, target_names=target_names)
    check_targets_scores(res)
    format_as_all(res, clf)
<commit_msg>TST make tests more stable<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import
from functools import partial
from pprint import pprint

import pytest
from sklearn.datasets import make_regression
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.ensemble import (
    ExtraTreesClassifier,
    ExtraTreesRegressor,
    GradientBoostingClassifier,
    GradientBoostingRegressor,
    RandomForestClassifier,
    RandomForestRegressor,
)
from sklearn.linear_model import (
    ElasticNet,
    ElasticNetCV,
    HuberRegressor,
    Lars,
    LarsCV,
    Lasso,
    LassoCV,
    LassoLars,
    LassoLarsCV,
    LassoLarsIC,
    LinearRegression,
    LogisticRegression,
    LogisticRegressionCV,
    OrthogonalMatchingPursuit,
    OrthogonalMatchingPursuitCV,
    PassiveAggressiveClassifier,
    PassiveAggressiveRegressor,
    Perceptron,
    Ridge,
    RidgeClassifier,
    RidgeClassifierCV,
    RidgeCV,
    SGDClassifier,
    SGDRegressor,
    TheilSenRegressor,
)
from sklearn.svm import LinearSVC, LinearSVR
from sklearn.multiclass import OneVsRestClassifier
from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor

from eli5 import explain_prediction
from eli5.formatters import format_as_text, fields
from eli5.sklearn.utils import has_intercept
from .utils import (
    format_as_all, strip_blanks, get_all_features, check_targets_scores)

format_as_all = partial(format_as_all, show_feature_values=True)

def assert_multiclass_linear_classifier_explained(newsgroups_train, clf,
                                                  explain_prediction):
    docs, y, target_names = newsgroups_train
    vec = TfidfVectorizer()

    X = vec.fit_transform(docs)
    clf.fit(X, y)

    get_res = lambda: explain_prediction(
        clf, docs[0], vec=vec, target_names=target_names, top=20)
    res = get_res()
    pprint(res)
    expl_text, expl_html = format_as_all(res, clf)

    for e in res.targets:
        if e.target != 'comp.graphics':
            continue
        pos = get_all_features(e.feature_weights.pos)
        assert 'file' in pos

    for expl in [expl_text, expl_html]:
        for label in target_names:
            assert str(label) in expl
        assert 'file' in expl

    assert res == get_res()

def assert_linear_regression_explained(boston_train, reg, explain_prediction,
                                       atol=1e-8):
    X, y, feature_names = boston_train
    reg.fit(X, y)
    res = explain_prediction(reg, X[0], feature_names=feature_names)
    expl_text, expl_html = expls = format_as_all(res, reg)

    assert len(res.targets) == 1
    target = res.targets[0]
    assert target.target == 'y'
    pos, neg = (get_all_features(target.feature_weights.pos),
                get_all_features(target.feature_weights.neg))
    assert 'LSTAT' in pos or 'LSTAT' in neg

    if has_intercept(reg):
        assert '<BIAS>' in pos or '<BIAS>' in neg
        assert '<BIAS>' in expl_text
        assert '<BIAS>' in expl_html
    else:
        assert '<BIAS>' not in pos and '<BIAS>' not in neg
        assert '<BIAS>' not in expl_text
        assert 'BIAS' not in expl_html

    for expl in [expl_text, expl_html]:
        assert 'LSTAT' in expl
        assert '(score' in expl
    assert "'y'" in expl_text
    assert 'y' in strip_blanks(expl_html)

    for expl in expls:
        assert_feature_values_present(expl, feature_names, X[0])

    assert res == explain_prediction(reg, X[0], feature_names=feature_names)
    check_targets_scores(res, atol=atol)

def assert_multitarget_linear_regression_explained(reg, explain_prediction):
    X, y = make_regression(n_samples=100, n_targets=3, n_features=10,
                           random_state=42)
    reg.fit(X, y)
    res = explain_prediction(reg, X[0])
    expl_text, expl_html = format_as_all(res, reg)

    assert len(res.targets) == 3
    target = res.targets[1]
    assert target.target == 'y1'
    pos, neg = (get_all_features(target.feature_weights.pos),
                get_all_features(target.feature_weights.neg))
    assert 'x8' in pos or 'x8' in neg
    assert '<BIAS>' in pos or '<BIAS>' in neg

    assert 'x8' in expl_text
    assert '<BIAS>' in expl_text
    assert "'y2'" in expl_text

    assert res == explain_prediction(reg, X[0])
    check_targets_scores(res)

def assert_feature_values_present(expl, feature_names, x):
    assert 'Value' in expl
    any_features = False
    for feature, value in zip(feature_names, x):
        if feature in expl:
            assert '{:+.3f}'.format(value) in expl
            any_features = True
    assert any_features

@pytest.mark.parametrize(['clf'], [
    [LogisticRegression(random_state=42)],
    [LogisticRegression(random_state=42, multi_class='multinomial', solver='lbfgs')],
    [LogisticRegression(random_state=42, fit_intercept=False)],
    [LogisticRegressionCV(random_state=42)],
    [SGDClassifier(random_state=42)],
    [SGDClassifier(loss='log', random_state=42)],
    [PassiveAggressiveClassifier(random_state=42)],
    [Perceptron(random_state=42)],
    [RidgeClassifier(random_state=42)],
    [RidgeClassifierCV()],
    [LinearSVC(random_state=42)],
    [OneVsRestClassifier(LogisticRegression(random_state=42))],
])
def test_explain_linear(newsgroups_train, clf):
    assert_multiclass_linear_classifier_explained(newsgroups_train, clf,
                                                  explain_prediction)

@pytest.mark.parametrize(['reg'], [
    [ElasticNet(random_state=42)],
    [ElasticNetCV(random_state=42)],
    [HuberRegressor()],
    [Lars()],
    [LarsCV(max_n_alphas=10)],
    [Lasso(random_state=42)],
    [LassoCV(n_alphas=10)],
    [LassoLars(alpha=0.1)],
    [LassoLarsCV(max_n_alphas=10)],
    [LassoLarsIC()],
    [LinearRegression()],
    [LinearSVR(random_state=42)],
    [OrthogonalMatchingPursuit(n_nonzero_coefs=10)],
    [OrthogonalMatchingPursuitCV()],
    [PassiveAggressiveRegressor(C=0.1)],
    [Ridge(random_state=42)],
    [RidgeCV()],
    [SGDRegressor(random_state=42)],
    [TheilSenRegressor()],
])
def test_explain_linear_regression(boston_train, reg):
    assert_linear_regression_explained(boston_train, reg, explain_prediction)

@pytest.mark.parametrize(['reg'], [
    [ElasticNet(random_state=42)],
    [Lars()],
    [Lasso(random_state=42)],
    [LinearRegression()],
    [Ridge(random_state=42)],
])
def test_explain_linear_regression_multitarget(reg):
    assert_multitarget_linear_regression_explained(reg, explain_prediction)

@pytest.mark.parametrize(['clf'], [
    [DecisionTreeClassifier(random_state=42)],
    [ExtraTreesClassifier(random_state=42)],
    [GradientBoostingClassifier(learning_rate=0.075, random_state=42)],
    [RandomForestClassifier(random_state=42)],
])
def test_explain_tree_clf_multiclass(clf, iris_train):
    X, y, feature_names, target_names = iris_train
    clf.fit(X, y)
    res = explain_prediction(
        clf, X[0], target_names=target_names, feature_names=feature_names)
    for expl in format_as_all(res, clf):
        for target in target_names:
            assert target in expl
        assert 'BIAS' in expl
        assert any(f in expl for f in feature_names)
        assert_feature_values_present(expl, feature_names, X[0])
    check_targets_scores(res)

@pytest.mark.parametrize(['clf'], [
    [DecisionTreeClassifier(random_state=42)],
    [ExtraTreesClassifier(random_state=42)],
    [GradientBoostingClassifier(learning_rate=0.075, random_state=42)],
    [RandomForestClassifier(random_state=42)],
])
def test_explain_tree_clf_binary(clf, iris_train_binary):
    X, y, feature_names = iris_train_binary
    clf.fit(X, y)
    res = explain_prediction(clf, X[0], feature_names=feature_names)
    format_as_all(res, clf)
    all_expls = []
    for x in X[:5]:
        res = explain_prediction(clf, x, feature_names=feature_names)
        text_expl = format_as_text(res, show=fields.WEIGHTS)
        print(text_expl)
        assert '<BIAS>' in text_expl
        check_targets_scores(res)
        all_expls.append(text_expl)
    assert any(f in ''.join(all_expls) for f in feature_names)

@pytest.mark.parametrize(['reg'], [
    [DecisionTreeRegressor(random_state=42)],
    [ExtraTreesRegressor(random_state=42)],
    [RandomForestRegressor(random_state=42)],
])
def test_explain_tree_regressor_multitarget(reg):
    X, y = make_regression(n_samples=100, n_targets=3, n_features=10,
                           random_state=42)
    reg.fit(X, y)
    res = explain_prediction(reg, X[0])
    for expl in format_as_all(res, reg):
        for target in ['y0', 'y1', 'y2']:
            assert target in expl
        assert 'BIAS' in expl
        assert any('x%d' % i in expl for i in range(10))
    check_targets_scores(res)

@pytest.mark.parametrize(['reg'], [
    [DecisionTreeRegressor(random_state=42)],
    [ExtraTreesRegressor(random_state=42)],
    [GradientBoostingRegressor(learning_rate=0.075, random_state=42)],
    [RandomForestRegressor(random_state=42)],
])
def test_explain_tree_regressor(reg, boston_train):
    X, y, feature_names = boston_train
    reg.fit(X, y)
    res = explain_prediction(reg, X[0], feature_names=feature_names)
    for expl in format_as_all(res, reg):
        assert_feature_values_present(expl, feature_names, X[0])
    all_expls = []
    for i, x in enumerate(X[:5]):
        res = explain_prediction(reg, x, feature_names=feature_names)
        text_expl = format_as_text(res, show=fields.WEIGHTS)
        print(text_expl)
        assert '<BIAS>' in text_expl
        check_targets_scores(res)
        all_expls.append(text_expl)
    assert any(f in ''.join(all_expls) for f in feature_names)

@pytest.mark.parametrize(['clf'], [
    [DecisionTreeClassifier(random_state=42)],
    [ExtraTreesClassifier(random_state=42)],
    [RandomForestClassifier(random_state=42)],
])
def test_explain_tree_classifier_text(clf, newsgroups_train_big):
    docs, y, target_names = newsgroups_train_big
    vec = CountVectorizer(binary=True, stop_words='english')
    X = vec.fit_transform(docs)
    clf.fit(X, y)
    res = explain_prediction(clf, docs[0], vec=vec, target_names=target_names)
    check_targets_scores(res)
    format_as_all(res, clf)
<|endoftext|>


<commit_before># coding=utf-8
# Copyright 2020 The TF-Agents Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Lint as: python3
r"""Example training DQN using actor/learner in a gym environment.

To run DQN on CartPole:

```bash
tensorboard --logdir $HOME/tmp/dqn/gym/CartPole-v0/ --port 2223 &
python tf_agents/experimental/examples/dqn/dqn_train_eval.py \
  --root_dir=/tmp/dqn_cartpole
```

"""

import functools
import os

from absl import app
from absl import flags
from absl import logging

import gin
import reverb
import tensorflow.compat.v2 as tf

from tf_agents.agents.dqn import dqn_agent
from tf_agents.environments import suite_gym
from tf_agents.metrics import py_metrics
from tf_agents.networks import sequential
from tf_agents.policies import py_tf_eager_policy
from tf_agents.policies import random_py_policy
from tf_agents.replay_buffers import reverb_replay_buffer
from tf_agents.replay_buffers import reverb_utils
from tf_agents.specs import tensor_spec
from tf_agents.system import system_multiprocessing as multiprocessing
from tf_agents.train import actor
from tf_agents.train import learner
from tf_agents.train import triggers
from tf_agents.train.utils import train_utils
from tf_agents.utils import common

FLAGS = flags.FLAGS

flags.DEFINE_string('root_dir', os.getenv('TEST_UNDECLARED_OUTPUTS_DIR'),
                    'Root directory for writing logs/summaries/checkpoints.')
flags.DEFINE_integer(
    'reverb_port', None,
    'Port for reverb server, if None, use a randomly chosen unused port.')
flags.DEFINE_integer('num_iterations', 100000,
                     'Total number train/eval iterations to perform.')
flags.DEFINE_integer(
    'eval_interval', 1000,
    'Number of train steps between evaluations. Set to 0 to skip.')
flags.DEFINE_multi_string('gin_file', None, 'Paths to the gin-config files.')
flags.DEFINE_multi_string('gin_bindings', None, 'Gin binding parameters.')

@gin.configurable
def train_eval(
    root_dir,
    env_name='CartPole-v0',
    # Training params
    initial_collect_steps=1000,
    num_iterations=100000,
    fc_layer_params=(100,),
    # Agent params
    epsilon_greedy=0.1,
    batch_size=64,
    learning_rate=1e-3,
    n_step_update=1,
    gamma=0.99,
    target_update_tau=0.05,
    target_update_period=5,
    reward_scale_factor=1.0,
    # Replay params
    reverb_port=None,
    replay_capacity=100000,
    # Others
    policy_save_interval=1000,
    eval_interval=1000,
    eval_episodes=10):
  """Trains and evaluates DQN."""
  collect_env = suite_gym.load(env_name)
  eval_env = suite_gym.load(env_name)

  time_step_tensor_spec = tensor_spec.from_spec(collect_env.time_step_spec())
  action_tensor_spec = tensor_spec.from_spec(collect_env.action_spec())

  train_step = train_utils.create_train_step()
  num_actions = action_tensor_spec.maximum - action_tensor_spec.minimum + 1

  # Define a helper function to create Dense layers configured with the right
  # activation and kernel initializer.
  def dense_layer(num_units):
    return tf.keras.layers.Dense(
        num_units,
        activation=tf.keras.activations.relu,
        kernel_initializer=tf.keras.initializers.VarianceScaling(
            scale=2.0, mode='fan_in', distribution='truncated_normal'))

  # QNetwork consists of a sequence of Dense layers followed by a dense layer
  # with `num_actions` units to generate one q_value per available action as
  # it's output.
  dense_layers = [dense_layer(num_units) for num_units in fc_layer_params]
  q_values_layer = tf.keras.layers.Dense(
      num_actions,
      activation=None,
      kernel_initializer=tf.keras.initializers.RandomUniform(
          minval=-0.03, maxval=0.03),
      bias_initializer=tf.keras.initializers.Constant(-0.2))
  q_net = sequential.Sequential(dense_layers + [q_values_layer])

  agent = dqn_agent.DqnAgent(
      time_step_tensor_spec,
      action_tensor_spec,
      q_network=q_net,
      epsilon_greedy=epsilon_greedy,
      n_step_update=n_step_update,
      target_update_tau=target_update_tau,
      target_update_period=target_update_period,
      optimizer=tf.keras.optimizers.Adam(learning_rate=learning_rate),
      td_errors_loss_fn=common.element_wise_squared_loss,
      gamma=gamma,
      reward_scale_factor=reward_scale_factor,
      train_step_counter=train_step)

  table_name = 'uniform_table'
  table = reverb.Table(
      table_name,
      max_size=replay_capacity,
      sampler=reverb.selectors.Uniform(),
      remover=reverb.selectors.Fifo(),
      rate_limiter=reverb.rate_limiters.MinSize(1))
  reverb_server = reverb.Server([table], port=reverb_port)
  reverb_replay = reverb_replay_buffer.ReverbReplayBuffer(
      agent.collect_data_spec,
      sequence_length=2,
      table_name=table_name,
      local_server=reverb_server)
  rb_observer = reverb_utils.ReverbAddTrajectoryObserver(
      reverb_replay.py_client, table_name,
      sequence_length=2,
      stride_length=1)

  dataset = reverb_replay.as_dataset(
      num_parallel_calls=3, sample_batch_size=batch_size,
      num_steps=2).prefetch(3)
  experience_dataset_fn = lambda: dataset

  saved_model_dir = os.path.join(root_dir, learner.POLICY_SAVED_MODEL_DIR)
  env_step_metric = py_metrics.EnvironmentSteps()

  learning_triggers = [
      triggers.PolicySavedModelTrigger(
          saved_model_dir,
          agent,
          train_step,
          interval=policy_save_interval,
          metadata_metrics={triggers.ENV_STEP_METADATA_KEY: env_step_metric}),
      triggers.StepPerSecondLogTrigger(train_step, interval=100),
  ]

  dqn_learner = learner.Learner(
      root_dir,
      train_step,
      agent,
      experience_dataset_fn,
      triggers=learning_triggers)

  # If we haven't trained yet make sure we collect some random samples first to
  # fill up the Replay Buffer with some experience.
  random_policy = random_py_policy.RandomPyPolicy(collect_env.time_step_spec(),
                                                  collect_env.action_spec())
  initial_collect_actor = actor.Actor(
      collect_env,
      random_policy,
      train_step,
      steps_per_run=initial_collect_steps,
      observers=[rb_observer])
  logging.info('Doing initial collect.')
  initial_collect_actor.run()

  tf_collect_policy = agent.collect_policy
  collect_policy = py_tf_eager_policy.PyTFEagerPolicy(tf_collect_policy,
                                                      use_tf_function=True)

  collect_actor = actor.Actor(
      collect_env,
      collect_policy,
      train_step,
      steps_per_run=1,
      observers=[rb_observer, env_step_metric],
      metrics=actor.collect_metrics(10),
      summary_dir=os.path.join(root_dir, learner.TRAIN_DIR),
  )

  tf_greedy_policy = agent.policy
  greedy_policy = py_tf_eager_policy.PyTFEagerPolicy(tf_greedy_policy,
                                                     use_tf_function=True)

  eval_actor = actor.Actor(
      eval_env,
      greedy_policy,
      train_step,
      episodes_per_run=eval_episodes,
      metrics=actor.eval_metrics(eval_episodes),
      summary_dir=os.path.join(root_dir, 'eval'),
  )

  if eval_interval:
    logging.info('Evaluating.')
    eval_actor.run_and_log()

  logging.info('Training.')
  for _ in range(num_iterations):
    collect_actor.run()
    dqn_learner.run(iterations=1)

    if eval_interval and dqn_learner.train_step_numpy % eval_interval == 0:
      logging.info('Evaluating.')
      eval_actor.run_and_log()

  rb_observer.close()
  reverb_server.stop()

def main(_):
  logging.set_verbosity(logging.INFO)
  tf.enable_v2_behavior()

  gin.parse_config_files_and_bindings(FLAGS.gin_file, FLAGS.gin_bindings)

  train_eval(
      FLAGS.root_dir,
      num_iterations=FLAGS.num_iterations,
      reverb_port=FLAGS.reverb_port,
      eval_interval=FLAGS.eval_interval)

if __name__ == '__main__':
  flags.mark_flag_as_required('root_dir')
  multiprocessing.handle_main(functools.partial(app.run, main))
<commit_msg>Update path in dqn_train_eval.py example docstring<commit_after># coding=utf-8
# Copyright 2020 The TF-Agents Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Lint as: python3
r"""Example training DQN using actor/learner in a gym environment.

To run DQN on CartPole:

```bash
tensorboard --logdir $HOME/tmp/dqn_cartpole --port 2223 &
python tf_agents/examples/dqn/dqn_train_eval.py \
  --root_dir=$HOME/tmp/dqn_cartpole
```

"""

import functools
import os

from absl import app
from absl import flags
from absl import logging

import gin
import reverb
import tensorflow.compat.v2 as tf

from tf_agents.agents.dqn import dqn_agent
from tf_agents.environments import suite_gym
from tf_agents.metrics import py_metrics
from tf_agents.networks import sequential
from tf_agents.policies import py_tf_eager_policy
from tf_agents.policies import random_py_policy
from tf_agents.replay_buffers import reverb_replay_buffer
from tf_agents.replay_buffers import reverb_utils
from tf_agents.specs import tensor_spec
from tf_agents.system import system_multiprocessing as multiprocessing
from tf_agents.train import actor
from tf_agents.train import learner
from tf_agents.train import triggers
from tf_agents.train.utils import train_utils
from tf_agents.utils import common

FLAGS = flags.FLAGS

flags.DEFINE_string('root_dir', os.getenv('TEST_UNDECLARED_OUTPUTS_DIR'),
                    'Root directory for writing logs/summaries/checkpoints.')
flags.DEFINE_integer(
    'reverb_port', None,
    'Port for reverb server, if None, use a randomly chosen unused port.')
flags.DEFINE_integer('num_iterations', 100000,
                     'Total number train/eval iterations to perform.')
flags.DEFINE_integer(
    'eval_interval', 1000,
    'Number of train steps between evaluations. Set to 0 to skip.')
flags.DEFINE_multi_string('gin_file', None, 'Paths to the gin-config files.')
flags.DEFINE_multi_string('gin_bindings', None, 'Gin binding parameters.')

@gin.configurable
def train_eval(
    root_dir,
    env_name='CartPole-v0',
    # Training params
    initial_collect_steps=1000,
    num_iterations=100000,
    fc_layer_params=(100,),
    # Agent params
    epsilon_greedy=0.1,
    batch_size=64,
    learning_rate=1e-3,
    n_step_update=1,
    gamma=0.99,
    target_update_tau=0.05,
    target_update_period=5,
    reward_scale_factor=1.0,
    # Replay params
    reverb_port=None,
    replay_capacity=100000,
    # Others
    policy_save_interval=1000,
    eval_interval=1000,
    eval_episodes=10):
  """Trains and evaluates DQN."""
  collect_env = suite_gym.load(env_name)
  eval_env = suite_gym.load(env_name)

  time_step_tensor_spec = tensor_spec.from_spec(collect_env.time_step_spec())
  action_tensor_spec = tensor_spec.from_spec(collect_env.action_spec())

  train_step = train_utils.create_train_step()
  num_actions = action_tensor_spec.maximum - action_tensor_spec.minimum + 1

  # Define a helper function to create Dense layers configured with the right
  # activation and kernel initializer.
  def dense_layer(num_units):
    return tf.keras.layers.Dense(
        num_units,
        activation=tf.keras.activations.relu,
        kernel_initializer=tf.keras.initializers.VarianceScaling(
            scale=2.0, mode='fan_in', distribution='truncated_normal'))

  # QNetwork consists of a sequence of Dense layers followed by a dense layer
  # with `num_actions` units to generate one q_value per available action as
  # it's output.
  dense_layers = [dense_layer(num_units) for num_units in fc_layer_params]
  q_values_layer = tf.keras.layers.Dense(
      num_actions,
      activation=None,
      kernel_initializer=tf.keras.initializers.RandomUniform(
          minval=-0.03, maxval=0.03),
      bias_initializer=tf.keras.initializers.Constant(-0.2))
  q_net = sequential.Sequential(dense_layers + [q_values_layer])

  agent = dqn_agent.DqnAgent(
      time_step_tensor_spec,
      action_tensor_spec,
      q_network=q_net,
      epsilon_greedy=epsilon_greedy,
      n_step_update=n_step_update,
      target_update_tau=target_update_tau,
      target_update_period=target_update_period,
      optimizer=tf.keras.optimizers.Adam(learning_rate=learning_rate),
      td_errors_loss_fn=common.element_wise_squared_loss,
      gamma=gamma,
      reward_scale_factor=reward_scale_factor,
      train_step_counter=train_step)

  table_name = 'uniform_table'
  table = reverb.Table(
      table_name,
      max_size=replay_capacity,
      sampler=reverb.selectors.Uniform(),
      remover=reverb.selectors.Fifo(),
      rate_limiter=reverb.rate_limiters.MinSize(1))
  reverb_server = reverb.Server([table], port=reverb_port)
  reverb_replay = reverb_replay_buffer.ReverbReplayBuffer(
      agent.collect_data_spec,
      sequence_length=2,
      table_name=table_name,
      local_server=reverb_server)
  rb_observer = reverb_utils.ReverbAddTrajectoryObserver(
      reverb_replay.py_client, table_name,
      sequence_length=2,
      stride_length=1)

  dataset = reverb_replay.as_dataset(
      num_parallel_calls=3, sample_batch_size=batch_size,
      num_steps=2).prefetch(3)
  experience_dataset_fn = lambda: dataset

  saved_model_dir = os.path.join(root_dir, learner.POLICY_SAVED_MODEL_DIR)
  env_step_metric = py_metrics.EnvironmentSteps()

  learning_triggers = [
      triggers.PolicySavedModelTrigger(
          saved_model_dir,
          agent,
          train_step,
          interval=policy_save_interval,
          metadata_metrics={triggers.ENV_STEP_METADATA_KEY: env_step_metric}),
      triggers.StepPerSecondLogTrigger(train_step, interval=100),
  ]

  dqn_learner = learner.Learner(
      root_dir,
      train_step,
      agent,
      experience_dataset_fn,
      triggers=learning_triggers)

  # If we haven't trained yet make sure we collect some random samples first to
  # fill up the Replay Buffer with some experience.
  random_policy = random_py_policy.RandomPyPolicy(collect_env.time_step_spec(),
                                                  collect_env.action_spec())
  initial_collect_actor = actor.Actor(
      collect_env,
      random_policy,
      train_step,
      steps_per_run=initial_collect_steps,
      observers=[rb_observer])
  logging.info('Doing initial collect.')
  initial_collect_actor.run()

  tf_collect_policy = agent.collect_policy
  collect_policy = py_tf_eager_policy.PyTFEagerPolicy(tf_collect_policy,
                                                      use_tf_function=True)

  collect_actor = actor.Actor(
      collect_env,
      collect_policy,
      train_step,
      steps_per_run=1,
      observers=[rb_observer, env_step_metric],
      metrics=actor.collect_metrics(10),
      summary_dir=os.path.join(root_dir, learner.TRAIN_DIR),
  )

  tf_greedy_policy = agent.policy
  greedy_policy = py_tf_eager_policy.PyTFEagerPolicy(tf_greedy_policy,
                                                     use_tf_function=True)

  eval_actor = actor.Actor(
      eval_env,
      greedy_policy,
      train_step,
      episodes_per_run=eval_episodes,
      metrics=actor.eval_metrics(eval_episodes),
      summary_dir=os.path.join(root_dir, 'eval'),
  )

  if eval_interval:
    logging.info('Evaluating.')
    eval_actor.run_and_log()

  logging.info('Training.')
  for _ in range(num_iterations):
    collect_actor.run()
    dqn_learner.run(iterations=1)

    if eval_interval and dqn_learner.train_step_numpy % eval_interval == 0:
      logging.info('Evaluating.')
      eval_actor.run_and_log()

  rb_observer.close()
  reverb_server.stop()

def main(_):
  logging.set_verbosity(logging.INFO)
  tf.enable_v2_behavior()

  gin.parse_config_files_and_bindings(FLAGS.gin_file, FLAGS.gin_bindings)

  train_eval(
      FLAGS.root_dir,
      num_iterations=FLAGS.num_iterations,
      reverb_port=FLAGS.reverb_port,
      eval_interval=FLAGS.eval_interval)

if __name__ == '__main__':
  flags.mark_flag_as_required('root_dir')
  multiprocessing.handle_main(functools.partial(app.run, main))
<|endoftext|>


<commit_before># Copyright (c) 2010 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

{
  # Always provide a target, so we can put the logic about whether there's
  # anything to be done in this file (instead of a higher-level .gyp file).
  'targets': [
    {
      'target_name': 'flash_player',
      'type': 'none',
      'conditions': [
        [ 'branding == "Chrome"', {
          'copies': [{
            'destination': '<(PRODUCT_DIR)',
            'files': [],
            'conditions': [
              [ 'chromeos == 1', {
                'files': [
                  'binaries/chromeos/libgcflashplayer.so',
                ]
              }],
              [ 'OS == "linux" and target_arch == "ia32" and chromeos == 0', {
                'files': [
                  'binaries/linux/libgcflashplayer.so',
                  'binaries/linux/plugin.vch',
                ]
              }],
              [ 'OS == "mac"', {
                'files': [
                  'binaries/mac/Flash Player Plugin for Chrome.plugin',
                  'binaries/mac/plugin.vch',
                ]
              }],
              [ 'OS == "win"', {
                'files': [
                  'binaries/win/gcswf32.dll',
                  'binaries/win/plugin.vch',
                ]
              }],
            ],
          }],
        }],
      ],
    },
  ],
}

# Local Variables:
# tab-width:2
# indent-tabs-mode:nil
# End:
# vim: set expandtab tabstop=2 shiftwidth=2:
<commit_msg>Fix for ChromeOS build, included linux version of plugin.vch file for now to please FILES.cfg requirements (we don't dump this file in ChromeOS image right now anyway).^M<commit_after># Copyright (c) 2010 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

{
  # Always provide a target, so we can put the logic about whether there's
  # anything to be done in this file (instead of a higher-level .gyp file).
  'targets': [
    {
      'target_name': 'flash_player',
      'type': 'none',
      'conditions': [
        [ 'branding == "Chrome"', {
          'copies': [{
            'destination': '<(PRODUCT_DIR)',
            'files': [],
            'conditions': [
              [ 'chromeos == 1', {
                'files': [
                  'binaries/chromeos/libgcflashplayer.so',
                  'binaries/linux/plugin.vch',
                ]
              }],
              [ 'OS == "linux" and target_arch == "ia32" and chromeos == 0', {
                'files': [
                  'binaries/linux/libgcflashplayer.so',
                  'binaries/linux/plugin.vch',
                ]
              }],
              [ 'OS == "mac"', {
                'files': [
                  'binaries/mac/Flash Player Plugin for Chrome.plugin',
                  'binaries/mac/plugin.vch',
                ]
              }],
              [ 'OS == "win"', {
                'files': [
                  'binaries/win/gcswf32.dll',
                  'binaries/win/plugin.vch',
                ]
              }],
            ],
          }],
        }],
      ],
    },
  ],
}

# Local Variables:
# tab-width:2
# indent-tabs-mode:nil
# End:
# vim: set expandtab tabstop=2 shiftwidth=2:
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# © 2016 Therp BV <http://therp.nl>
# © 2017 ThinkOpen Solutions <https://tkobr.com>
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).

import logging
import os
import shutil
import subprocess
import tempfile
import threading
import time

import odoo
from odoo.tests import common

ADMIN_USER_ID = common.ADMIN_USER_ID

from odoo import api, fields, models, _
from odoo.exceptions import UserError

_logger = logging.getLogger(__name__)
_MARKER_PHRASE = '[[waiting for OCR]]'
_PDF_OCR_DOCUMENTS_THREADS = []
OCR_LANGUAGE = [('afr', 'Afrikaans'),
                ('amh', 'Amharic'),
                ('ara', 'Arabic'),
                ('asm', 'Assamese'),
                ('aze', 'Azerbaijani'),
                ('aze_cyrl', 'Azerbaijani - Cyrilic'),
                ('bel', 'Belarusian'),
                ('ben', 'Bengali'),
                ('bod', 'Tibetan'),
                ('bos', 'Bosnian'),
                ('bul', 'Bulgarian'),
                ('cat', 'Catalan; Valencian'),
                ('ceb', 'Cebuano'),
                ('ces', 'Czech'),
                ('chi_sim', 'Chinese - Simplified'),
                ('chi_tra', 'Chinese - Traditional'),
                ('chr', 'Cherokee'),
                ('cym', 'Welsh'),
                ('dan', 'Danish'),
                ('dan_frak', 'Danish - Fraktur'),
                ('deu', 'German'),
                ('deu_frak', 'German - Fraktur'),
                ('dzo', 'Dzongkha'),
                ('ell', 'Greek, Modern (1453-)'),
                ('eng', 'English'),
                ('enm', 'English, Middle (1100-1500)'),
                ('epo', 'Esperanto'),
                ('equ', 'Math / equation detection module'),
                ('est', 'Estonian'),
                ('eus', 'Basque'),
                ('fas', 'Persian'),
                ('fin', 'Finnish'),
                ('fra', 'French'),
                ('frk', 'Frankish'),
                ('frm', 'French, Middle (ca.1400-1600)'),
                ('gle', 'Irish'),
                ('glg', 'Galician'),
                ('grc', 'Greek, Ancient (to 1453)'),
                ('guj', 'Gujarati'),
                ('hat', 'Haitian; Haitian Creole'),
                ('heb', 'Hebrew'),
                ('hin', 'Hindi'),
                ('hrv', 'Croatian'),
                ('hun', 'Hungarian'),
                ('iku', 'Inuktitut'),
                ('ind', 'Indonesian'),
                ('isl', 'Icelandic'),
                ('ita', 'Italian'),
                ('ita_old', 'Italian - Old'),
                ('jav', 'Javanese'),
                ('jpn', 'Japanese'),
                ('kan', 'Kannada'),
                ('kat', 'Georgian'),
                ('kat_old', 'Georgian - Old'),
                ('kaz', 'Kazakh'),
                ('khm', 'Central Khmer'),
                ('kir', 'Kirghiz; Kyrgyz'),
                ('kor', 'Korean'),
                ('kur', 'Kurdish'),
                ('lao', 'Lao'),
                ('lat', 'Latin'),
                ('lav', 'Latvian'),
                ('lit', 'Lithuanian'),
                ('mal', 'Malayalam'),
                ('mar', 'Marathi'),
                ('mkd', 'Macedonian'),
                ('mlt', 'Maltese'),
                ('msa', 'Malay'),
                ('mya', 'Burmese'),
                ('nep', 'Nepali'),
                ('nld', 'Dutch; Flemish'),
                ('nor', 'Norwegian'),
                ('ori', 'Oriya'),
                ('osd', 'Orientation and script detection module'),
                ('pan', 'Panjabi; Punjabi'),
                ('pol', 'Polish'),
                ('por', 'Portuguese'),
                ('pus', 'Pushto; Pashto'),
                ('ron', 'Romanian; Moldavian; Moldovan'),
                ('rus', 'Russian'),
                ('san', 'Sanskrit'),
                ('sin', 'Sinhala; Sinhalese'),
                ('slk', 'Slovak'),
                ('slk_frak', 'Slovak - Fraktur'),
                ('slv', 'Slovenian'),
                ('spa', 'Spanish; Castilian'),
                ('spa_old', 'Spanish; Castilian - Old'),
                ('sqi', 'Albanian'),
                ('srp', 'Serbian'),
                ('srp_latn', 'Serbian - Latin'),
                ('swa', 'Swahili'),
                ('swe', 'Swedish'),
                ('syr', 'Syriac'),
                ('tam', 'Tamil'),
                ('tel', 'Telugu'),
                ('tgk', 'Tajik'),
                ('tgl', 'Tagalog'),
                ('tha', 'Thai'),
                ('tir', 'Tigrinya'),
                ('tur', 'Turkish'),
                ('uig', 'Uighur; Uyghur'),
                ('ukr', 'Ukrainian'),
                ('urd', 'Urdu'),
                ('uzb', 'Uzbek'),
                ('uzb_cyrl', 'Uzbek - Cyrilic'),
                ('vie', 'Vietnamese'),
                ('yid', 'Yiddish'), ]

def ncpus():
    # for Linux, Unix and MacOS
    if hasattr(os, "sysconf"):
        if os.sysconf_names.has_key("SC_NPROCESSORS_ONLN"):
            # Linux and Unix
            ncpus = os.sysconf("SC_NPROCESSORS_ONLN")
            if isinstance(ncpus, int) and ncpus > 0:
                return ncpus
        else:
            # MacOS X
            return int(os.popen2("sysctl -n hw.ncpu")[1].read())
    # for Windows
    if os.environ.has_key("NUMBER_OF_PROCESSORS"):
        ncpus = int(os.environ["NUMBER_OF_PROCESSORS"])
        if ncpus > 0:
            return ncpus
    # return the default value
    return 1

_SEMAPHORES_POOL = threading.BoundedSemaphore(ncpus())

class IrAttachment(models.Model):
    _inherit = 'ir.attachment'

    language = fields.Selection(OCR_LANGUAGE, 'Language',
                                default=lambda self:
                                self.env['ir.config_parameter'].get_param(
                                    'document_ocr.language', 'eng'))
    # We need to redefine index_content field to be able to update it
    # on the onchange_language() in form
    index_content = fields.Text('Indexed Content',
                                readonly=False,
                                prefetch=False)
    index_content_rel = fields.Text(related='index_content',
                                    string='Indexed Content Rel')
    processing_time = fields.Char('Processing Time',
                                  readonly=True,
                                  copy=False,
                                  help='Processing time.\n'
                                       '(00:00:00 means less than one second)')

    @api.onchange('language')
    def onchange_language(self):
        process = subprocess.Popen(['tesseract', '--list-langs'],
                                   stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE)
        stdout, stderr = process.communicate()
        if self.language not in stderr.split('\n')[1:-1]:
            raise UserError(_(
                "Language not installed."
                " Please ask your system administrator to"
                " install tesseract '%s' language." %
                self.language))
        if self.store_fname:
            bin_data = self._file_read(self.store_fname)
        else:
            bin_data = self.db_datas
        if bin_data:
            index_content = self._index(
                bin_data.decode('base64'), self.datas_fname, self.mimetype)
            return {'value': {
                'index_content': index_content}}
        return {'value': {}}

    @api.model
    def _index(self, bin_data, datas_fname, mimetype):
        content = super(IrAttachment, self)._index(
            bin_data, datas_fname, mimetype)
        if not content or content == 'image':
            has_synchr_param = self.env['ir.config_parameter'].get_param(
                'document_ocr.synchronous', 'False') == 'True'
            has_force_flag = self.env.context.get('document_ocr_force')
            synchr = has_synchr_param or has_force_flag
            if synchr:
                content = self._index_ocr(bin_data)
            else:
                content = _MARKER_PHRASE
        return content

    def _index_ocr(self, bin_data):
        if self.datas_fname:
            process = subprocess.Popen(
                ['tesseract', 'stdin', 'stdout', '-l', self.language],
                stdin=subprocess.PIPE, stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
            )
            stdout, stderr = process.communicate(bin_data)
            if stderr:
                _logger.warning('Error during OCR: %s', stderr)
            return stdout
        else:
            _logger.warning('OCR IMAGE "%s", no image to process...',
                            self.name)
        return False

    def _ocr_image_thread(self, i, t, image):
        global ocr_images_text
        with _SEMAPHORES_POOL:
            with threading.Lock():
                _logger.info('OCR PDF INFO "%s" image %d/%d to text...',
                             self.name, i, t)
                ocr_images_text[self.id][i] = self._index_ocr(image)

    def _index_doc_pdf_thread(self, bin_data):
        global ocr_images_text
        ocr_images_text[self.id] = {}
        buf = _MARKER_PHRASE
        tmpdir = tempfile.mkdtemp()
        # with api.Environment.manage():
        #     with odoo.registry(
        #             self.env.cr.dbname).cursor() as new_cr:
        #         new_env = api.Environment(new_cr, self.env.uid,
        #                                   self.env.context)
                # try:
                # with _SEMAPHORES_POOL:
                #     with threading.Lock():
        _logger.info('OCR PDF INFO "%s"...', self.name)
        time_start = time.time()
        stdout, stderr = subprocess.Popen(
            ['pdftotext', '-layout', '-nopgbrk', '-', '-'],
            stdin=subprocess.PIPE, stdout=subprocess.PIPE,
            stderr=subprocess.PIPE).communicate(bin_data)
        if stderr:
            _logger.warning('OCR PDF ERROR to text: %s',
                            stderr)
        buf = stdout
        # OCR PDF Images
        stdout, stderr = subprocess.Popen(
            ['pdfimages', '-p', '-', tmpdir + '/ocr'],
            stdin=subprocess.PIPE, stdout=subprocess.PIPE,
            stderr=subprocess.PIPE).communicate(bin_data)
        if stderr:
            _logger.warning('OCR PDF WARNING Images: %s',
                            stderr)
        # OCR every image greater than 50Kb
        filelist = sorted([(file) for file
                           in os.listdir(tmpdir)
                           if os.path.getsize(
                os.path.join(tmpdir, file)) > 50000])
        filelist_size = len(filelist)
        count = 1
        workers = []
        for pdf in filelist:
            img_file = os.path.join(tmpdir, pdf)
            image = open(img_file, 'rb').read()
            t = threading.Thread(target=self._ocr_image_thread,
                                 name=u'ocr_image_' + str(count),
                                 args=(count,
                                       filelist_size,
                                       image))
            t.start()
            count += 1
            workers.append(t)
        for t in workers:
            t.join()
        index_content = buf
        for text in sorted(ocr_images_text[self.id]):
            try:
                index_content = \
                    u'%s\n%s' % (
                        index_content,
                        ocr_images_text[self.id][text].decode(
                            'utf8'))
            except:
                index_content = \
                    u'%s\n%s' % (
                        index_content.decode('utf8'),
                        ocr_images_text[self.id][text])
        ocr_images_text.pop(self.id)  # release memory
        m, s = divmod((time.time() - time_start), 60)
        h, m = divmod(m, 60)
        self.index_content = index_content
        self.processing_time = "%02d:%02d:%02d" % (h, m, s)
        shutil.rmtree(tmpdir)
        # new_env.cr.commit()
                # except:
                #     new_env.cr.rollback()
        return self.index_content

    def _index_pdf(self, bin_data):
        global ocr_images_text
        buf = _MARKER_PHRASE
        has_synchr_param = self.env['ir.config_parameter'].get_param(
            'document_ocr.synchronous', 'False') == 'True'
        has_force_flag = self.env.context.get('document_ocr_force')
        synchr = has_synchr_param or has_force_flag
        try:
            if ocr_images_text:
                pass
        except:
            ocr_images_text = {}
        if synchr:
            buf = self._index_doc_pdf_thread(bin_data)
            # t = threading.Thread(target=self._index_doc_pdf_thread,
            #                      name=u'index_pdf_' + str(self.id),
            #                      args=[bin_data])
            # t.start()
            # t.join()
            # _PDF_OCR_DOCUMENTS_THREADS.append(t)
        else:
            buf = _MARKER_PHRASE
        return buf

    @api.model
    def _ocr_cron(self):
        for this in self.with_context(document_ocr_force=True).search(
                [('index_content', '=', _MARKER_PHRASE)]):
            if not this.datas:
                continue
            index_content = this._index(
                this.datas.decode('base64'), this.datas_fname, this.mimetype)
            this.write({
                'index_content': index_content,
            })
<commit_msg>Remove old comments<commit_after># -*- coding: utf-8 -*-
# © 2016 Therp BV <http://therp.nl>
# © 2017 ThinkOpen Solutions <https://tkobr.com>
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).

import logging
import os
import shutil
import subprocess
import tempfile
import threading
import time

import odoo
from odoo.tests import common

ADMIN_USER_ID = common.ADMIN_USER_ID

from odoo import api, fields, models, _
from odoo.exceptions import UserError

_logger = logging.getLogger(__name__)
_MARKER_PHRASE = '[[waiting for OCR]]'
_PDF_OCR_DOCUMENTS_THREADS = []
OCR_LANGUAGE = [('afr', 'Afrikaans'),
                ('amh', 'Amharic'),
                ('ara', 'Arabic'),
                ('asm', 'Assamese'),
                ('aze', 'Azerbaijani'),
                ('aze_cyrl', 'Azerbaijani - Cyrilic'),
                ('bel', 'Belarusian'),
                ('ben', 'Bengali'),
                ('bod', 'Tibetan'),
                ('bos', 'Bosnian'),
                ('bul', 'Bulgarian'),
                ('cat', 'Catalan; Valencian'),
                ('ceb', 'Cebuano'),
                ('ces', 'Czech'),
                ('chi_sim', 'Chinese - Simplified'),
                ('chi_tra', 'Chinese - Traditional'),
                ('chr', 'Cherokee'),
                ('cym', 'Welsh'),
                ('dan', 'Danish'),
                ('dan_frak', 'Danish - Fraktur'),
                ('deu', 'German'),
                ('deu_frak', 'German - Fraktur'),
                ('dzo', 'Dzongkha'),
                ('ell', 'Greek, Modern (1453-)'),
                ('eng', 'English'),
                ('enm', 'English, Middle (1100-1500)'),
                ('epo', 'Esperanto'),
                ('equ', 'Math / equation detection module'),
                ('est', 'Estonian'),
                ('eus', 'Basque'),
                ('fas', 'Persian'),
                ('fin', 'Finnish'),
                ('fra', 'French'),
                ('frk', 'Frankish'),
                ('frm', 'French, Middle (ca.1400-1600)'),
                ('gle', 'Irish'),
                ('glg', 'Galician'),
                ('grc', 'Greek, Ancient (to 1453)'),
                ('guj', 'Gujarati'),
                ('hat', 'Haitian; Haitian Creole'),
                ('heb', 'Hebrew'),
                ('hin', 'Hindi'),
                ('hrv', 'Croatian'),
                ('hun', 'Hungarian'),
                ('iku', 'Inuktitut'),
                ('ind', 'Indonesian'),
                ('isl', 'Icelandic'),
                ('ita', 'Italian'),
                ('ita_old', 'Italian - Old'),
                ('jav', 'Javanese'),
                ('jpn', 'Japanese'),
                ('kan', 'Kannada'),
                ('kat', 'Georgian'),
                ('kat_old', 'Georgian - Old'),
                ('kaz', 'Kazakh'),
                ('khm', 'Central Khmer'),
                ('kir', 'Kirghiz; Kyrgyz'),
                ('kor', 'Korean'),
                ('kur', 'Kurdish'),
                ('lao', 'Lao'),
                ('lat', 'Latin'),
                ('lav', 'Latvian'),
                ('lit', 'Lithuanian'),
                ('mal', 'Malayalam'),
                ('mar', 'Marathi'),
                ('mkd', 'Macedonian'),
                ('mlt', 'Maltese'),
                ('msa', 'Malay'),
                ('mya', 'Burmese'),
                ('nep', 'Nepali'),
                ('nld', 'Dutch; Flemish'),
                ('nor', 'Norwegian'),
                ('ori', 'Oriya'),
                ('osd', 'Orientation and script detection module'),
                ('pan', 'Panjabi; Punjabi'),
                ('pol', 'Polish'),
                ('por', 'Portuguese'),
                ('pus', 'Pushto; Pashto'),
                ('ron', 'Romanian; Moldavian; Moldovan'),
                ('rus', 'Russian'),
                ('san', 'Sanskrit'),
                ('sin', 'Sinhala; Sinhalese'),
                ('slk', 'Slovak'),
                ('slk_frak', 'Slovak - Fraktur'),
                ('slv', 'Slovenian'),
                ('spa', 'Spanish; Castilian'),
                ('spa_old', 'Spanish; Castilian - Old'),
                ('sqi', 'Albanian'),
                ('srp', 'Serbian'),
                ('srp_latn', 'Serbian - Latin'),
                ('swa', 'Swahili'),
                ('swe', 'Swedish'),
                ('syr', 'Syriac'),
                ('tam', 'Tamil'),
                ('tel', 'Telugu'),
                ('tgk', 'Tajik'),
                ('tgl', 'Tagalog'),
                ('tha', 'Thai'),
                ('tir', 'Tigrinya'),
                ('tur', 'Turkish'),
                ('uig', 'Uighur; Uyghur'),
                ('ukr', 'Ukrainian'),
                ('urd', 'Urdu'),
                ('uzb', 'Uzbek'),
                ('uzb_cyrl', 'Uzbek - Cyrilic'),
                ('vie', 'Vietnamese'),
                ('yid', 'Yiddish'), ]

def ncpus():
    # for Linux, Unix and MacOS
    if hasattr(os, "sysconf"):
        if os.sysconf_names.has_key("SC_NPROCESSORS_ONLN"):
            # Linux and Unix
            ncpus = os.sysconf("SC_NPROCESSORS_ONLN")
            if isinstance(ncpus, int) and ncpus > 0:
                return ncpus
        else:
            # MacOS X
            return int(os.popen2("sysctl -n hw.ncpu")[1].read())
    # for Windows
    if os.environ.has_key("NUMBER_OF_PROCESSORS"):
        ncpus = int(os.environ["NUMBER_OF_PROCESSORS"])
        if ncpus > 0:
            return ncpus
    # return the default value
    return 1

_SEMAPHORES_POOL = threading.BoundedSemaphore(ncpus())

class IrAttachment(models.Model):
    _inherit = 'ir.attachment'

    language = fields.Selection(OCR_LANGUAGE, 'Language',
                                default=lambda self:
                                self.env['ir.config_parameter'].get_param(
                                    'document_ocr.language', 'eng'))
    # We need to redefine index_content field to be able to update it
    # on the onchange_language() in form
    index_content = fields.Text('Indexed Content',
                                readonly=False,
                                prefetch=False)
    index_content_rel = fields.Text(related='index_content',
                                    string='Indexed Content Rel')
    processing_time = fields.Char('Processing Time',
                                  readonly=True,
                                  copy=False,
                                  help='Processing time.\n'
                                       '(00:00:00 means less than one second)')

    @api.onchange('language')
    def onchange_language(self):
        process = subprocess.Popen(['tesseract', '--list-langs'],
                                   stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE)
        stdout, stderr = process.communicate()
        if self.language not in stderr.split('\n')[1:-1]:
            raise UserError(_(
                "Language not installed."
                " Please ask your system administrator to"
                " install tesseract '%s' language." %
                self.language))
        if self.store_fname:
            bin_data = self._file_read(self.store_fname)
        else:
            bin_data = self.db_datas
        if bin_data:
            index_content = self._index(
                bin_data.decode('base64'), self.datas_fname, self.mimetype)
            return {'value': {
                'index_content': index_content}}
        return {'value': {}}

    @api.model
    def _index(self, bin_data, datas_fname, mimetype):
        content = super(IrAttachment, self)._index(
            bin_data, datas_fname, mimetype)
        if not content or content == 'image':
            has_synchr_param = self.env['ir.config_parameter'].get_param(
                'document_ocr.synchronous', 'False') == 'True'
            has_force_flag = self.env.context.get('document_ocr_force')
            synchr = has_synchr_param or has_force_flag
            if synchr:
                content = self._index_ocr(bin_data)
            else:
                content = _MARKER_PHRASE
        return content

    def _index_ocr(self, bin_data):
        if self.datas_fname:
            process = subprocess.Popen(
                ['tesseract', 'stdin', 'stdout', '-l', self.language],
                stdin=subprocess.PIPE, stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
            )
            stdout, stderr = process.communicate(bin_data)
            if stderr:
                _logger.warning('Error during OCR: %s', stderr)
            return stdout
        else:
            _logger.warning('OCR IMAGE "%s", no image to process...',
                            self.name)
        return False

    def _ocr_image_thread(self, i, t, image):
        global ocr_images_text
        with _SEMAPHORES_POOL:
            with threading.Lock():
                _logger.info('OCR PDF INFO "%s" image %d/%d to text...',
                             self.name, i, t)
                ocr_images_text[self.id][i] = self._index_ocr(image)

    def _index_doc_pdf_thread(self, bin_data):
        global ocr_images_text
        ocr_images_text[self.id] = {}
        buf = _MARKER_PHRASE
        tmpdir = tempfile.mkdtemp()
        _logger.info('OCR PDF INFO "%s"...', self.name)
        time_start = time.time()
        stdout, stderr = subprocess.Popen(
            ['pdftotext', '-layout', '-nopgbrk', '-', '-'],
            stdin=subprocess.PIPE, stdout=subprocess.PIPE,
            stderr=subprocess.PIPE).communicate(bin_data)
        if stderr:
            _logger.warning('OCR PDF ERROR to text: %s',
                            stderr)
        buf = stdout
        # OCR PDF Images
        stdout, stderr = subprocess.Popen(
            ['pdfimages', '-p', '-', tmpdir + '/ocr'],
            stdin=subprocess.PIPE, stdout=subprocess.PIPE,
            stderr=subprocess.PIPE).communicate(bin_data)
        if stderr:
            _logger.warning('OCR PDF WARNING Images: %s',
                            stderr)
        # OCR every image greater than 50Kb
        filelist = sorted([(file) for file
                           in os.listdir(tmpdir)
                           if os.path.getsize(
                os.path.join(tmpdir, file)) > 50000])
        filelist_size = len(filelist)
        count = 1
        workers = []
        for pdf in filelist:
            img_file = os.path.join(tmpdir, pdf)
            image = open(img_file, 'rb').read()
            t = threading.Thread(target=self._ocr_image_thread,
                                 name=u'ocr_image_' + str(count),
                                 args=(count,
                                       filelist_size,
                                       image))
            t.start()
            count += 1
            workers.append(t)
        for t in workers:
            t.join()
        index_content = buf
        for text in sorted(ocr_images_text[self.id]):
            try:
                index_content = \
                    u'%s\n%s' % (
                        index_content,
                        ocr_images_text[self.id][text].decode(
                            'utf8'))
            except:
                index_content = \
                    u'%s\n%s' % (
                        index_content.decode('utf8'),
                        ocr_images_text[self.id][text])
        ocr_images_text.pop(self.id)  # release memory
        m, s = divmod((time.time() - time_start), 60)
        h, m = divmod(m, 60)
        self.index_content = index_content
        self.processing_time = "%02d:%02d:%02d" % (h, m, s)
        shutil.rmtree(tmpdir)
        return self.index_content

    def _index_pdf(self, bin_data):
        global ocr_images_text
        buf = _MARKER_PHRASE
        has_synchr_param = self.env['ir.config_parameter'].get_param(
            'document_ocr.synchronous', 'False') == 'True'
        has_force_flag = self.env.context.get('document_ocr_force')
        synchr = has_synchr_param or has_force_flag
        try:
            if ocr_images_text:
                pass
        except:
            ocr_images_text = {}
        if synchr:
            buf = self._index_doc_pdf_thread(bin_data)
        else:
            buf = _MARKER_PHRASE
        return buf

    @api.model
    def _ocr_cron(self):
        for this in self.with_context(document_ocr_force=True).search(
                [('index_content', '=', _MARKER_PHRASE)]):
            if not this.datas:
                continue
            index_content = this._index(
                this.datas.decode('base64'), this.datas_fname, this.mimetype)
            this.write({
                'index_content': index_content,
            })
<|endoftext|>


<commit_before>#!/usr/bin/env python
import utils
from utils import memoize
import chpl_locale_model
import third_party_utils

@memoize
def get_uniq_cfg_path():
    return '{0}-{1}'.format(third_party_utils.default_uniq_cfg_path(),
                            chpl_locale_model.get())

@memoize
def get_link_args():
    return third_party_utils.default_get_link_args('qthread',
                                                   ucp=get_uniq_cfg_path(),
                                                   libs=['libqthread_chpl.la',
                                                         '-lchpl',
                                                         'libqthread.la',
                                                         '-lrt'])
<commit_msg>Only add -lrt in cases where we definitely need it and adding it is safe.<commit_after>#!/usr/bin/env python
import utils
from utils import memoize
import chpl_compiler, chpl_llvm, chpl_locale_model
import third_party_utils

@memoize
def get_uniq_cfg_path():
    return '{0}-{1}'.format(third_party_utils.default_uniq_cfg_path(),
                            chpl_locale_model.get())

@memoize
def get_link_args():
    link_args = \
        third_party_utils.default_get_link_args('qthread',
                                                ucp=get_uniq_cfg_path(),
                                                libs=['libqthread_chpl.la',
                                                      '-lchpl',
                                                      'libqthread.la'])
    if ( chpl_compiler.get('target').startswith('cray-prgenv') and
         chpl_llvm.get() == 'llvm' ):
        link_args.append('-lrt')
    return link_args
<|endoftext|>