<commit_before>from __future__ import unicode_literals

import uuid
from decimal import Decimal
import datetime as dt
from marshmallow.compat import basestring
import dj_database_url
import dj_email_url

try:
    import urllib.parse as urlparse
except ImportError:
    # Python 2
    import urlparse

import pytest
from marshmallow import fields, validate

import environs

@pytest.fixture
def set_env(monkeypatch):
    def _set_env(envvars):
        for key, val in envvars.items():
            monkeypatch.setenv(key, val)

    return _set_env

@pytest.fixture(scope="function")
def env():
    return environs.Env()

class TestCasting:
    def test_call(self, set_env, env):
        set_env({"STR": "foo", "INT": "42"})
        assert env("STR") == "foo"
        assert env("NOT_SET", "mydefault") == "mydefault"

    def test_call_with_default(self, env):
        assert env("NOT_SET", default="mydefault") == "mydefault"
        assert env("NOT_SET", "mydefault") == "mydefault"

    def test_basic(self, set_env, env):
        set_env({"STR": "foo"})
        assert env.str("STR") == "foo"

    def test_int_cast(self, set_env, env):
        set_env({"INT": "42"})
        assert env.int("INT") == 42

    def test_invalid_int(self, set_env, env):
        set_env({"INT": "invalid"})
        with pytest.raises(environs.EnvError) as excinfo:
            env.int("INT")
        assert 'Environment variable "INT" invalid' in excinfo.value.args[0]

    def test_float_cast(self, set_env, env):
        set_env({"FLOAT": "33.3"})
        assert env.float("FLOAT") == 33.3

    def test_list_cast(self, set_env, env):
        set_env({"LIST": "1,2,3"})
        assert env.list("LIST") == ["1", "2", "3"]

    def test_list_with_subcast(self, set_env, env):
        set_env({"LIST": "1,2,3"})
        assert env.list("LIST", subcast=int) == [1, 2, 3]
        assert env.list("LIST", subcast=float) == [1.0, 2.0, 3.0]

    def test_bool(self, set_env, env):
        set_env({"TRUTHY": "1", "FALSY": "0"})
        assert env.bool("TRUTHY") is True
        assert env.bool("FALSY") is False

        set_env({"TRUTHY2": "True", "FALSY2": "False"})
        assert env.bool("TRUTHY2") is True
        assert env.bool("FALSY2") is False

    def test_list_with_spaces(self, set_env, env):
        set_env({"LIST": " 1,  2,3"})
        assert env.list("LIST", subcast=int) == [1, 2, 3]

    def test_dict(self, set_env, env):
        set_env({"DICT": "key1=1,key2=2"})
        assert env.dict("DICT") == {"key1": "1", "key2": "2"}

    def test_dict_with_subcast(self, set_env, env):
        set_env({"DICT": "key1=1,key2=2"})
        assert env.dict("DICT", subcast=int) == {"key1": 1, "key2": 2}

    def test_decimat_cast(self, set_env, env):
        set_env({"DECIMAL": "12.34"})
        assert env.decimal("DECIMAL") == Decimal("12.34")

    def test_missing_raises_error(self, env):
        with pytest.raises(environs.EnvError) as exc:
            env.str("FOO")
        assert exc.value.args[0] == 'Environment variable "FOO" not set'

    def test_default_set(self, env):
        assert env.str("FOO", missing="foo") == "foo"
        # Passed positionally
        assert env.str("FOO", "foo") == "foo"

    def test_json_cast(self, set_env, env):
        set_env({"JSON": '{"foo": "bar", "baz": [1, 2, 3]}'})
        assert env.json("JSON") == {"foo": "bar", "baz": [1, 2, 3]}

    def test_datetime_cast(self, set_env, env):
        dtime = dt.datetime.utcnow()
        set_env({"DTIME": dtime.isoformat()})
        result = env.datetime("DTIME")
        assert type(result) is dt.datetime
        assert result.year == dtime.year
        assert result.month == dtime.month
        assert result.day == dtime.day

    def test_date_cast(self, set_env, env):
        date = dt.date.today()
        set_env({"DATE": date.isoformat()})
        assert env.date("DATE") == date

    def test_timedelta_cast(self, set_env, env):
        set_env({"TIMEDELTA": "42"})
        assert env.timedelta("TIMEDELTA") == dt.timedelta(seconds=42)

    def test_uuid_cast(self, set_env, env):
        uid = uuid.uuid1()
        set_env({"UUID": str(uid)})
        assert env.uuid("UUID") == uid

    def test_url_cast(self, set_env, env):
        set_env({"URL": "http://stevenloria.com/projects/?foo=42"})
        res = env.url("URL")
        assert isinstance(res, urlparse.ParseResult)

    @pytest.mark.parametrize("url", ["foo", "42", "foo@bar"])
    def test_invalid_url(self, url, set_env, env):
        set_env({"URL": url})
        with pytest.raises(environs.EnvError) as excinfo:
            env.url("URL")
        assert 'Environment variable "URL" invalid' in excinfo.value.args[0]

class TestProxiedVariables:
    def test_reading_proxied_variable(self, set_env, env):
        set_env(
            {
                "MAILGUN_SMTP_LOGIN": "sloria",
                "SMTP_LOGIN": "{{MAILGUN_SMTP_LOGIN}}",
                "SMTP_LOGIN_LPADDED": "{{ MAILGUN_SMTP_LOGIN}}",
                "SMTP_LOGIN_RPADDED": "{{MAILGUN_SMTP_LOGIN }}",
            }
        )
        for key in (
            "MAILGUN_SMTP_LOGIN",
            "SMTP_LOGIN",
            "SMTP_LOGIN_LPADDED",
            "SMTP_LOGIN_RPADDED",
        ):
            assert env(key) == "sloria"
            assert env.dump()[key] == "sloria"

    def test_reading_missing_proxied_variable(self, set_env, env):
        set_env({"SMTP_LOGIN": "{{MAILGUN_SMTP_LOGIN}}"})
        with pytest.raises(environs.EnvError) as excinfo:
            env("SMTP_LOGIN")
        assert (
            excinfo.value.args[0] == 'Environment variable "MAILGUN_SMTP_LOGIN" not set'
        )
        assert env("SMTP_LOGIN", "default") == "default"

class TestEnvFileReading:
    def test_read_env(self, env):
        assert env("STRING", "default") == "default"  # sanity check
        env.read_env()
        assert env("STRING") == "foo"
        assert env.list("LIST") == ["wat", "wer", "wen"]
        assert env("PROXIED") == "foo"

    def test_read_env_not_found_with_verbose_warns(self, env):
        with pytest.warns(UserWarning) as record:
            env.read_env("notfound", recurse=False, verbose=True)
        assert "File doesn't exist" in record[0].message.args[0]

def always_fail(value):
    raise environs.EnvError("something went wrong")

class TestValidation:
    def test_can_add_validator(self, set_env, env):
        set_env({"NUM": 3})

        with pytest.raises(environs.EnvError) as excinfo:
            env.int("NUM", validate=lambda n: n > 3)
        assert "Invalid value." in excinfo.value.args[0]

    def test_can_add_marshmallow_validator(self, set_env, env):
        set_env({"NODE_ENV": "invalid"})
        with pytest.raises(environs.EnvError) as excinfo:
            env("NODE_ENV", validate=validate.OneOf(["development", "production"]))
        assert "Not a valid choice." in excinfo.value.args[0]

    def test_validator_can_raise_enverror(self, set_env, env):
        with pytest.raises(environs.EnvError) as excinfo:
            env("NODE_ENV", "development", validate=always_fail)
        assert "something went wrong" in excinfo.value.args[0]

    def test_failed_vars_are_not_serialized(self, set_env, env):
        set_env({"FOO": "42"})
        try:
            env("FOO", validate=always_fail)
        except environs.EnvError:
            pass
        assert "FOO" not in env.dump()

class TestCustomTypes:
    def test_add_parser(self, set_env, env):
        set_env({"URL": "test.test/"})

        def url(value):
            return "https://" + value

        env.add_parser("url", url)
        assert env.url("URL") == "https://test.test/"
        with pytest.raises(environs.EnvError) as excinfo:
            env.url("NOT_SET")
        assert excinfo.value.args[0] == 'Environment variable "NOT_SET" not set'

        assert env.url("NOT_SET", "default.test/") == "https://default.test/"

    def test_parser_for(self, set_env, env):
        set_env({"URL": "test.test/"})

        @env.parser_for("url")
        def url(value):
            return "https://" + value

        assert env.url("URL") == "https://test.test/"

        with pytest.raises(environs.EnvError) as excinfo:
            env.url("NOT_SET")
        assert excinfo.value.args[0] == 'Environment variable "NOT_SET" not set'

        assert env.url("NOT_SET", "default.test/") == "https://default.test/"

    def test_parser_function_can_take_extra_arguments(self, set_env, env):
        set_env({"ENV": "dev"})

        @env.parser_for("enum")
        def enum_parser(value, choices):
            if value not in choices:
                raise environs.EnvError("Invalid!")
            return value

        assert env.enum("ENV", choices=["dev", "prod"]) == "dev"

        set_env({"ENV": "invalid"})
        with pytest.raises(environs.EnvError):
            env.enum("ENV", choices=["dev", "prod"])

    def test_add_parser_from_field(self, set_env, env):
        class MyURL(fields.Field):
            def _deserialize(self, value, *args, **kwargs):
                return "https://" + value

        env.add_parser_from_field("url", MyURL)

        set_env({"URL": "test.test/"})
        assert env.url("URL") == "https://test.test/"

        with pytest.raises(environs.EnvError) as excinfo:
            env.url("NOT_SET")
        assert excinfo.value.args[0] == 'Environment variable "NOT_SET" not set'

class TestDumping:
    def test_dump(self, set_env, env):
        dtime = dt.datetime.utcnow()
        set_env(
            {
                "STR": "foo",
                "INT": "42",
                "DTIME": dtime.isoformat(),
                "URLPARSE": "http://stevenloria.com/projects/?foo=42",
            }
        )

        env.str("STR")
        env.int("INT")
        env.datetime("DTIME")
        env.url("URLPARSE")

        result = env.dump()
        assert result["STR"] == "foo"
        assert result["INT"] == 42
        assert "DTIME" in result
        assert type(result["DTIME"]) is str
        assert isinstance(result["URLPARSE"], basestring)
        assert result["URLPARSE"] == "http://stevenloria.com/projects/?foo=42"

    def test_env_with_custom_parser(self, set_env, env):
        @env.parser_for("url")
        def url(value):
            return "https://" + value

        set_env({"URL": "test.test"})

        env.url("URL")

        assert env.dump() == {"URL": "https://test.test"}

def test_repr(set_env, env):
    set_env({"FOO": "foo", "BAR": 42})
    env.str("FOO")
    assert repr(env) == "<Env {}>".format({"FOO": "foo"})

def test_str(set_env, env):
    set_env({"FOO": "foo", "BAR": 42})
    env.str("FOO")
    assert repr(env) == "<Env {}>".format({"FOO": "foo"})

def test_env_isolation(set_env):
    set_env({"FOO": "foo"})
    env1 = environs.Env()

    @env1.parser_for("foo")
    def foo(value):
        return value

    env2 = environs.Env()

    # env1 has a parser for foo, but env2 does not
    assert env1.foo("FOO") == "foo"
    with pytest.raises(AttributeError):
        env2.foo("FOO")

class TestPrefix:
    @pytest.fixture(autouse=True)
    def default_environ(self, set_env):
        set_env({"APP_STR": "foo", "APP_INT": "42"})

    def test_prefixed(self, env):
        with env.prefixed("APP_"):
            assert env.str("STR") == "foo"
            assert env.int("INT") == 42
            assert env("NOT_FOUND", "mydefault") == "mydefault"

    def test_dump_with_prefixed(self, env):
        with env.prefixed("APP_"):
            env.str("STR") == "foo"
            env.int("INT") == 42
            env("NOT_FOUND", "mydefault") == "mydefault"
        assert env.dump() == {
            "APP_STR": "foo",
            "APP_INT": 42,
            "APP_NOT_FOUND": "mydefault",
        }

class TestNestedPrefix:
    @pytest.fixture(autouse=True)
    def default_environ(self, set_env):
        set_env({"APP_STR": "foo", "APP_NESTED_INT": "42"})

    def test_nested_prefixed(self, env):
        with env.prefixed("APP_"):
            with env.prefixed("NESTED_"):
                assert env.int("INT") == 42
                assert env("NOT_FOUND", "mydefault") == "mydefault"
            assert env.str("STR") == "foo"
            assert env("NOT_FOUND", "mydefault") == "mydefault"

    def test_dump_with_nested_prefixed(self, env):
        with env.prefixed("APP_"):
            with env.prefixed("NESTED_"):
                env.int("INT") == 42
                env("NOT_FOUND", "mydefault") == "mydefault"
            env.str("STR") == "foo"
            env("NOT_FOUND", "mydefault") == "mydefault"
        assert env.dump() == {
            "APP_STR": "foo",
            "APP_NOT_FOUND": "mydefault",
            "APP_NESTED_INT": 42,
            "APP_NESTED_NOT_FOUND": "mydefault",
        }

class TestDjango:
    def test_dj_db_url(self, env, set_env):
        db_url = "postgresql://localhost:5432/mydb"
        set_env({"DATABASE_URL": db_url})
        res = env.dj_db_url("DATABASE_URL")
        assert res == dj_database_url.parse(db_url)

    def test_dj_db_url_passes_kwargs(self, env, set_env):
        db_url = "postgresql://localhost:5432/mydb"
        set_env({"DATABASE_URL": db_url})
        res = env.dj_db_url("DATABASE_URL", conn_max_age=600)
        assert res == dj_database_url.parse(db_url, conn_max_age=600)

    def test_dj_email_url(self, env, set_env):
        email_url = "smtp://user@domain.com:pass@smtp.example.com:465/?ssl=True"
        set_env({"EMAIL_URL": email_url})
        res = env.dj_email_url("EMAIL_URL")
        assert res == dj_email_url.parse(email_url)
<commit_msg>Address pytest warnings re: string envvars<commit_after>from __future__ import unicode_literals

import uuid
from decimal import Decimal
import datetime as dt
from marshmallow.compat import basestring
import dj_database_url
import dj_email_url

try:
    import urllib.parse as urlparse
except ImportError:
    # Python 2
    import urlparse

import pytest
from marshmallow import fields, validate

import environs

@pytest.fixture
def set_env(monkeypatch):
    def _set_env(envvars):
        for key, val in envvars.items():
            monkeypatch.setenv(key, val)

    return _set_env

@pytest.fixture(scope="function")
def env():
    return environs.Env()

class TestCasting:
    def test_call(self, set_env, env):
        set_env({"STR": "foo", "INT": "42"})
        assert env("STR") == "foo"
        assert env("NOT_SET", "mydefault") == "mydefault"

    def test_call_with_default(self, env):
        assert env("NOT_SET", default="mydefault") == "mydefault"
        assert env("NOT_SET", "mydefault") == "mydefault"

    def test_basic(self, set_env, env):
        set_env({"STR": "foo"})
        assert env.str("STR") == "foo"

    def test_int_cast(self, set_env, env):
        set_env({"INT": "42"})
        assert env.int("INT") == 42

    def test_invalid_int(self, set_env, env):
        set_env({"INT": "invalid"})
        with pytest.raises(environs.EnvError) as excinfo:
            env.int("INT")
        assert 'Environment variable "INT" invalid' in excinfo.value.args[0]

    def test_float_cast(self, set_env, env):
        set_env({"FLOAT": "33.3"})
        assert env.float("FLOAT") == 33.3

    def test_list_cast(self, set_env, env):
        set_env({"LIST": "1,2,3"})
        assert env.list("LIST") == ["1", "2", "3"]

    def test_list_with_subcast(self, set_env, env):
        set_env({"LIST": "1,2,3"})
        assert env.list("LIST", subcast=int) == [1, 2, 3]
        assert env.list("LIST", subcast=float) == [1.0, 2.0, 3.0]

    def test_bool(self, set_env, env):
        set_env({"TRUTHY": "1", "FALSY": "0"})
        assert env.bool("TRUTHY") is True
        assert env.bool("FALSY") is False

        set_env({"TRUTHY2": "True", "FALSY2": "False"})
        assert env.bool("TRUTHY2") is True
        assert env.bool("FALSY2") is False

    def test_list_with_spaces(self, set_env, env):
        set_env({"LIST": " 1,  2,3"})
        assert env.list("LIST", subcast=int) == [1, 2, 3]

    def test_dict(self, set_env, env):
        set_env({"DICT": "key1=1,key2=2"})
        assert env.dict("DICT") == {"key1": "1", "key2": "2"}

    def test_dict_with_subcast(self, set_env, env):
        set_env({"DICT": "key1=1,key2=2"})
        assert env.dict("DICT", subcast=int) == {"key1": 1, "key2": 2}

    def test_decimat_cast(self, set_env, env):
        set_env({"DECIMAL": "12.34"})
        assert env.decimal("DECIMAL") == Decimal("12.34")

    def test_missing_raises_error(self, env):
        with pytest.raises(environs.EnvError) as exc:
            env.str("FOO")
        assert exc.value.args[0] == 'Environment variable "FOO" not set'

    def test_default_set(self, env):
        assert env.str("FOO", missing="foo") == "foo"
        # Passed positionally
        assert env.str("FOO", "foo") == "foo"

    def test_json_cast(self, set_env, env):
        set_env({"JSON": '{"foo": "bar", "baz": [1, 2, 3]}'})
        assert env.json("JSON") == {"foo": "bar", "baz": [1, 2, 3]}

    def test_datetime_cast(self, set_env, env):
        dtime = dt.datetime.utcnow()
        set_env({"DTIME": dtime.isoformat()})
        result = env.datetime("DTIME")
        assert type(result) is dt.datetime
        assert result.year == dtime.year
        assert result.month == dtime.month
        assert result.day == dtime.day

    def test_date_cast(self, set_env, env):
        date = dt.date.today()
        set_env({"DATE": date.isoformat()})
        assert env.date("DATE") == date

    def test_timedelta_cast(self, set_env, env):
        set_env({"TIMEDELTA": "42"})
        assert env.timedelta("TIMEDELTA") == dt.timedelta(seconds=42)

    def test_uuid_cast(self, set_env, env):
        uid = uuid.uuid1()
        set_env({"UUID": str(uid)})
        assert env.uuid("UUID") == uid

    def test_url_cast(self, set_env, env):
        set_env({"URL": "http://stevenloria.com/projects/?foo=42"})
        res = env.url("URL")
        assert isinstance(res, urlparse.ParseResult)

    @pytest.mark.parametrize("url", ["foo", "42", "foo@bar"])
    def test_invalid_url(self, url, set_env, env):
        set_env({"URL": url})
        with pytest.raises(environs.EnvError) as excinfo:
            env.url("URL")
        assert 'Environment variable "URL" invalid' in excinfo.value.args[0]

class TestProxiedVariables:
    def test_reading_proxied_variable(self, set_env, env):
        set_env(
            {
                "MAILGUN_SMTP_LOGIN": "sloria",
                "SMTP_LOGIN": "{{MAILGUN_SMTP_LOGIN}}",
                "SMTP_LOGIN_LPADDED": "{{ MAILGUN_SMTP_LOGIN}}",
                "SMTP_LOGIN_RPADDED": "{{MAILGUN_SMTP_LOGIN }}",
            }
        )
        for key in (
            "MAILGUN_SMTP_LOGIN",
            "SMTP_LOGIN",
            "SMTP_LOGIN_LPADDED",
            "SMTP_LOGIN_RPADDED",
        ):
            assert env(key) == "sloria"
            assert env.dump()[key] == "sloria"

    def test_reading_missing_proxied_variable(self, set_env, env):
        set_env({"SMTP_LOGIN": "{{MAILGUN_SMTP_LOGIN}}"})
        with pytest.raises(environs.EnvError) as excinfo:
            env("SMTP_LOGIN")
        assert (
            excinfo.value.args[0] == 'Environment variable "MAILGUN_SMTP_LOGIN" not set'
        )
        assert env("SMTP_LOGIN", "default") == "default"

class TestEnvFileReading:
    def test_read_env(self, env):
        assert env("STRING", "default") == "default"  # sanity check
        env.read_env()
        assert env("STRING") == "foo"
        assert env.list("LIST") == ["wat", "wer", "wen"]
        assert env("PROXIED") == "foo"

    def test_read_env_not_found_with_verbose_warns(self, env):
        with pytest.warns(UserWarning) as record:
            env.read_env("notfound", recurse=False, verbose=True)
        assert "File doesn't exist" in record[0].message.args[0]

def always_fail(value):
    raise environs.EnvError("something went wrong")

class TestValidation:
    def test_can_add_validator(self, set_env, env):
        set_env({"NUM": "3"})

        with pytest.raises(environs.EnvError) as excinfo:
            env.int("NUM", validate=lambda n: n > 3)
        assert "Invalid value." in excinfo.value.args[0]

    def test_can_add_marshmallow_validator(self, set_env, env):
        set_env({"NODE_ENV": "invalid"})
        with pytest.raises(environs.EnvError) as excinfo:
            env("NODE_ENV", validate=validate.OneOf(["development", "production"]))
        assert "Not a valid choice." in excinfo.value.args[0]

    def test_validator_can_raise_enverror(self, set_env, env):
        with pytest.raises(environs.EnvError) as excinfo:
            env("NODE_ENV", "development", validate=always_fail)
        assert "something went wrong" in excinfo.value.args[0]

    def test_failed_vars_are_not_serialized(self, set_env, env):
        set_env({"FOO": "42"})
        try:
            env("FOO", validate=always_fail)
        except environs.EnvError:
            pass
        assert "FOO" not in env.dump()

class TestCustomTypes:
    def test_add_parser(self, set_env, env):
        set_env({"URL": "test.test/"})

        def url(value):
            return "https://" + value

        env.add_parser("url", url)
        assert env.url("URL") == "https://test.test/"
        with pytest.raises(environs.EnvError) as excinfo:
            env.url("NOT_SET")
        assert excinfo.value.args[0] == 'Environment variable "NOT_SET" not set'

        assert env.url("NOT_SET", "default.test/") == "https://default.test/"

    def test_parser_for(self, set_env, env):
        set_env({"URL": "test.test/"})

        @env.parser_for("url")
        def url(value):
            return "https://" + value

        assert env.url("URL") == "https://test.test/"

        with pytest.raises(environs.EnvError) as excinfo:
            env.url("NOT_SET")
        assert excinfo.value.args[0] == 'Environment variable "NOT_SET" not set'

        assert env.url("NOT_SET", "default.test/") == "https://default.test/"

    def test_parser_function_can_take_extra_arguments(self, set_env, env):
        set_env({"ENV": "dev"})

        @env.parser_for("enum")
        def enum_parser(value, choices):
            if value not in choices:
                raise environs.EnvError("Invalid!")
            return value

        assert env.enum("ENV", choices=["dev", "prod"]) == "dev"

        set_env({"ENV": "invalid"})
        with pytest.raises(environs.EnvError):
            env.enum("ENV", choices=["dev", "prod"])

    def test_add_parser_from_field(self, set_env, env):
        class MyURL(fields.Field):
            def _deserialize(self, value, *args, **kwargs):
                return "https://" + value

        env.add_parser_from_field("url", MyURL)

        set_env({"URL": "test.test/"})
        assert env.url("URL") == "https://test.test/"

        with pytest.raises(environs.EnvError) as excinfo:
            env.url("NOT_SET")
        assert excinfo.value.args[0] == 'Environment variable "NOT_SET" not set'

class TestDumping:
    def test_dump(self, set_env, env):
        dtime = dt.datetime.utcnow()
        set_env(
            {
                "STR": "foo",
                "INT": "42",
                "DTIME": dtime.isoformat(),
                "URLPARSE": "http://stevenloria.com/projects/?foo=42",
            }
        )

        env.str("STR")
        env.int("INT")
        env.datetime("DTIME")
        env.url("URLPARSE")

        result = env.dump()
        assert result["STR"] == "foo"
        assert result["INT"] == 42
        assert "DTIME" in result
        assert type(result["DTIME"]) is str
        assert isinstance(result["URLPARSE"], basestring)
        assert result["URLPARSE"] == "http://stevenloria.com/projects/?foo=42"

    def test_env_with_custom_parser(self, set_env, env):
        @env.parser_for("url")
        def url(value):
            return "https://" + value

        set_env({"URL": "test.test"})

        env.url("URL")

        assert env.dump() == {"URL": "https://test.test"}

def test_repr(set_env, env):
    set_env({"FOO": "foo", "BAR": "42"})
    env.str("FOO")
    assert repr(env) == "<Env {}>".format({"FOO": "foo"})

def test_str(set_env, env):
    set_env({"FOO": "foo", "BAR": "42"})
    env.str("FOO")
    assert repr(env) == "<Env {}>".format({"FOO": "foo"})

def test_env_isolation(set_env):
    set_env({"FOO": "foo"})
    env1 = environs.Env()

    @env1.parser_for("foo")
    def foo(value):
        return value

    env2 = environs.Env()

    # env1 has a parser for foo, but env2 does not
    assert env1.foo("FOO") == "foo"
    with pytest.raises(AttributeError):
        env2.foo("FOO")

class TestPrefix:
    @pytest.fixture(autouse=True)
    def default_environ(self, set_env):
        set_env({"APP_STR": "foo", "APP_INT": "42"})

    def test_prefixed(self, env):
        with env.prefixed("APP_"):
            assert env.str("STR") == "foo"
            assert env.int("INT") == 42
            assert env("NOT_FOUND", "mydefault") == "mydefault"

    def test_dump_with_prefixed(self, env):
        with env.prefixed("APP_"):
            env.str("STR") == "foo"
            env.int("INT") == 42
            env("NOT_FOUND", "mydefault") == "mydefault"
        assert env.dump() == {
            "APP_STR": "foo",
            "APP_INT": 42,
            "APP_NOT_FOUND": "mydefault",
        }

class TestNestedPrefix:
    @pytest.fixture(autouse=True)
    def default_environ(self, set_env):
        set_env({"APP_STR": "foo", "APP_NESTED_INT": "42"})

    def test_nested_prefixed(self, env):
        with env.prefixed("APP_"):
            with env.prefixed("NESTED_"):
                assert env.int("INT") == 42
                assert env("NOT_FOUND", "mydefault") == "mydefault"
            assert env.str("STR") == "foo"
            assert env("NOT_FOUND", "mydefault") == "mydefault"

    def test_dump_with_nested_prefixed(self, env):
        with env.prefixed("APP_"):
            with env.prefixed("NESTED_"):
                env.int("INT") == 42
                env("NOT_FOUND", "mydefault") == "mydefault"
            env.str("STR") == "foo"
            env("NOT_FOUND", "mydefault") == "mydefault"
        assert env.dump() == {
            "APP_STR": "foo",
            "APP_NOT_FOUND": "mydefault",
            "APP_NESTED_INT": 42,
            "APP_NESTED_NOT_FOUND": "mydefault",
        }

class TestDjango:
    def test_dj_db_url(self, env, set_env):
        db_url = "postgresql://localhost:5432/mydb"
        set_env({"DATABASE_URL": db_url})
        res = env.dj_db_url("DATABASE_URL")
        assert res == dj_database_url.parse(db_url)

    def test_dj_db_url_passes_kwargs(self, env, set_env):
        db_url = "postgresql://localhost:5432/mydb"
        set_env({"DATABASE_URL": db_url})
        res = env.dj_db_url("DATABASE_URL", conn_max_age=600)
        assert res == dj_database_url.parse(db_url, conn_max_age=600)

    def test_dj_email_url(self, env, set_env):
        email_url = "smtp://user@domain.com:pass@smtp.example.com:465/?ssl=True"
        set_env({"EMAIL_URL": email_url})
        res = env.dj_email_url("EMAIL_URL")
        assert res == dj_email_url.parse(email_url)
<|endoftext|>


<commit_before>import socket
import sys
import threading
import uuid

import pubsub
import ssh
import termcast

class Server(object):
    def __init__(self, keyfile):
        self.publisher = pubsub.Publisher()
        self.keyfile = keyfile

    def listen(self):
        ssh_sock = self._open_socket(2200)
        termcast_sock = self._open_socket(2201)

        threading.Thread(
            target=lambda: self.wait_for_ssh_connection(ssh_sock)
        ).start()
        threading.Thread(
            target=lambda: self.wait_for_termcast_connection(termcast_sock)
        ).start()

    def wait_for_ssh_connection(self, sock):
        self._wait_for_connection(
            sock,
            lambda client: self.handle_ssh_connection(client)
        )

    def wait_for_termcast_connection(self, sock):
        self._wait_for_connection(
            sock,
            lambda client: self.handle_termcast_connection(client)
        )

    def handle_ssh_connection(self, client):
        self._handle_connection(
            client,
            lambda client, connection_id: ssh.Connection(client, connection_id, self.publisher, self.keyfile)
        )

    def handle_termcast_connection(self, client):
        self._handle_connection(
            client,
            lambda client, connection_id: termcast.Connection(client, connection_id, self.publisher)
        )

    def _wait_for_connection(self, sock, cb):
        while True:
            try:
                sock.listen(100)
                client, addr = sock.accept()
            except Exception as e:
                print('*** Listen/accept failed: ' + str(e))
                traceback.print_exc()
                continue

            threading.Thread(target=cb, args=(client,)).start()

    def _handle_connection(self, client, cb):
        connection_id = uuid.uuid4().hex
        connection = cb(client, connection_id)
        self.publisher.subscribe(connection)
        connection.run()
        self.publisher.unsubscribe(connection)

    def _open_socket(self, port):
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        sock.bind(('', port))
        return sock

if __name__ == '__main__':
    server = Server(sys.argv[1])
    server.listen()
<commit_msg>ignore SIGPIPE<commit_after>import signal
import socket
import sys
import threading
import uuid

import pubsub
import ssh
import termcast

class Server(object):
    def __init__(self, keyfile):
        self.publisher = pubsub.Publisher()
        self.keyfile = keyfile

    def listen(self):
        signal.signal(signal.SIGPIPE, signal.SIG_IGN)
        ssh_sock = self._open_socket(2200)
        termcast_sock = self._open_socket(2201)

        threading.Thread(
            target=lambda: self.wait_for_ssh_connection(ssh_sock)
        ).start()
        threading.Thread(
            target=lambda: self.wait_for_termcast_connection(termcast_sock)
        ).start()

    def wait_for_ssh_connection(self, sock):
        self._wait_for_connection(
            sock,
            lambda client: self.handle_ssh_connection(client)
        )

    def wait_for_termcast_connection(self, sock):
        self._wait_for_connection(
            sock,
            lambda client: self.handle_termcast_connection(client)
        )

    def handle_ssh_connection(self, client):
        self._handle_connection(
            client,
            lambda client, connection_id: ssh.Connection(client, connection_id, self.publisher, self.keyfile)
        )

    def handle_termcast_connection(self, client):
        self._handle_connection(
            client,
            lambda client, connection_id: termcast.Connection(client, connection_id, self.publisher)
        )

    def _wait_for_connection(self, sock, cb):
        while True:
            try:
                sock.listen(100)
                client, addr = sock.accept()
            except Exception as e:
                print('*** Listen/accept failed: ' + str(e))
                traceback.print_exc()
                continue

            threading.Thread(target=cb, args=(client,)).start()

    def _handle_connection(self, client, cb):
        connection_id = uuid.uuid4().hex
        connection = cb(client, connection_id)
        self.publisher.subscribe(connection)
        connection.run()
        self.publisher.unsubscribe(connection)

    def _open_socket(self, port):
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        sock.bind(('', port))
        return sock

if __name__ == '__main__':
    server = Server(sys.argv[1])
    server.listen()
<|endoftext|>


<commit_before>version = '0.4.4'

<commit_msg>version bump for 0.4.5.<commit_after>version = '0.4.5'

<|endoftext|>


<commit_before>from flask import session, render_template, Blueprint

from lexos.helpers import constants
from lexos.managers import session_manager
from lexos.models.dendro_model import DendrogramModel
from lexos.models.filemanager_model import FileManagerModel
from lexos.views.base_view import detect_active_docs

# this is a flask blue print
# it helps us to manage groups of views
# see here for more detail:
# http://exploreflask.com/en/latest/blueprints.html
# http://flask.pocoo.org/docs/0.12/blueprints/
dendrogram_blueprint = Blueprint('dendrogram', __name__)

@dendrogram_blueprint.route("/dendrogram", methods=['GET'])
def dendrogram():
    # Detect the number of active documents.
    num_active_docs = detect_active_docs()
    if 'analyoption' not in session:
        session['analyoption'] = constants.DEFAULT_ANALYZE_OPTIONS
    if 'hierarchyoption' not in session:
        session['hierarchyoption'] = constants.DEFAULT_HIERARCHICAL_OPTIONS
    labels = FileManagerModel().load_file_manager().get_active_labels()
    return render_template(
        'dendrogram.html',
        labels=labels,
        numActiveDocs=num_active_docs,
        itm="hierarchical")

@dendrogram_blueprint.route("/dendrogramDiv", methods=['POST'])
def dendrogram_div():
    """Send the dendrogram div that is generated by plotly to frontend

    :return: an html string of the dendrogram div
    """
    session_manager.cache_analysis_option()
    session_manager.cache_hierarchy_option()
    return DendrogramModel().get_dendrogram_div()
<commit_msg>remove the itm<commit_after>from flask import session, render_template, Blueprint

from lexos.helpers import constants
from lexos.managers import session_manager
from lexos.models.dendro_model import DendrogramModel
from lexos.models.filemanager_model import FileManagerModel
from lexos.views.base_view import detect_active_docs

# this is a flask blue print
# it helps us to manage groups of views
# see here for more detail:
# http://exploreflask.com/en/latest/blueprints.html
# http://flask.pocoo.org/docs/0.12/blueprints/
dendrogram_blueprint = Blueprint('dendrogram', __name__)

@dendrogram_blueprint.route("/dendrogram", methods=['GET'])
def dendrogram():
    # Detect the number of active documents.
    num_active_docs = detect_active_docs()
    if 'analyoption' not in session:
        session['analyoption'] = constants.DEFAULT_ANALYZE_OPTIONS
    if 'hierarchyoption' not in session:
        session['hierarchyoption'] = constants.DEFAULT_HIERARCHICAL_OPTIONS
    labels = FileManagerModel().load_file_manager().get_active_labels()
    return render_template(
        'dendrogram.html',
        labels=labels,
        numActiveDocs=num_active_docs)

@dendrogram_blueprint.route("/dendrogramDiv", methods=['POST'])
def dendrogram_div():
    """Send the dendrogram div that is generated by plotly to frontend

    :return: an html string of the dendrogram div
    """
    session_manager.cache_analysis_option()
    session_manager.cache_hierarchy_option()
    return DendrogramModel().get_dendrogram_div()
<|endoftext|>


<commit_before>import json
from urllib import parse

from elasticsearch import exceptions
from pyramid import httpexceptions
from pyramid import i18n
from pyramid import view
from statsd.defaults.env import statsd

from bouncer import util

_ = i18n.TranslationStringFactory(__package__)

#: The maximum length that the "netloc" (the www.example.com part in
#: http://www.example.com/example) can be in the pretty URL that is displayed
#: to the user before it gets truncated.
NETLOC_MAX_LENGTH = 20

@view.view_defaults(renderer="bouncer:templates/annotation.html.jinja2")
class AnnotationController(object):

    def __init__(self, request):
        self.request = request

    @view.view_config(route_name="annotation_with_url")
    @view.view_config(route_name="annotation_without_url")
    def annotation(self):
        settings = self.request.registry.settings

        try:
            document = util.elasticsearch_client(settings).get(
                index=settings["elasticsearch_index"],
                doc_type="annotation",
                id=self.request.matchdict["id"])
        except exceptions.NotFoundError:
            statsd.incr("views.annotation.404.annotation_not_found")
            raise httpexceptions.HTTPNotFound(_("Annotation not found"))

        try:
            annotation_id, document_uri = util.parse_document(document)
        except util.InvalidAnnotationError as exc:
            statsd.incr("views.annotation.422.{}".format(exc.reason))
            raise httpexceptions.HTTPUnprocessableEntity(str(exc))

        # Remove any existing #fragment identifier from the URI before we
        # append our own.
        document_uri = parse.urldefrag(document_uri)[0]

        if not (document_uri.startswith("http://") or
                document_uri.startswith("https://")):
            statsd.incr("views.annotation.422.not_an_http_or_https_document")
            raise httpexceptions.HTTPUnprocessableEntity(
                _("Sorry, but it looks like this annotation was made on a "
                  "document that is not publicly available. "
                  "To view its annotations, a document's address must start "
                  "with http:// or https://."))

        via_url = "{via_base_url}/{uri}#annotations:{id}".format(
            via_base_url=settings["via_base_url"],
            uri=document_uri,
            id=annotation_id)

        extension_url = "{uri}#annotations:{id}".format(
            uri=document_uri, id=annotation_id)

        parsed_url = parse.urlparse(document_uri)
        pretty_url = parsed_url.netloc[:NETLOC_MAX_LENGTH]
        if len(parsed_url.netloc) > NETLOC_MAX_LENGTH:
          pretty_url = pretty_url+ jinja2.Markup("…")

        statsd.incr("views.annotation.200.annotation_found")
        return {
            "data": json.dumps({
                # Warning: variable names change from python_style to
                # javaScriptStyle here!
                "chromeExtensionId": settings["chrome_extension_id"],
                "viaUrl": via_url,
                "extensionUrl": extension_url,
            }),
            "pretty_url": pretty_url
        }

@view.view_config(renderer="bouncer:templates/index.html.jinja2",
                  route_name="index")
def index(request):
    statsd.incr("views.index.302.redirected_to_hypothesis")
    raise httpexceptions.HTTPFound(
        location=request.registry.settings["hypothesis_url"])

@view.view_defaults(renderer='bouncer:templates/error.html.jinja2')
class ErrorController(object):

    def __init__(self, exc, request):
        self.exc = exc
        self.request = request

    @view.view_config(context=httpexceptions.HTTPError)
    @view.view_config(context=httpexceptions.HTTPServerError)
    def httperror(self):
        self.request.response.status_int = self.exc.status_int
        # If code raises an HTTPError or HTTPServerError we assume this was
        # deliberately raised and:
        # 1. Show the user an error page including specific error message
        # 2. _Do not_ report the error to Sentry.
        return {"message": str(self.exc)}

    @view.view_config(context=Exception)
    def error(self):
        # In debug mode re-raise exceptions so that they get printed in the
        # terminal.
        if self.request.registry.settings["debug"]:
            raise

        self.request.response.status_int = 500

        # If code raises a non-HTTPException exception we assume it was a bug
        # and:
        # 1. Show the user a generic error page
        # 2. Report the details of the error to Sentry.
        self.request.raven.captureException()
        return {"message": _("Sorry, but something went wrong with the link. "
                             "The issue has been reported and we'll try to "
                             "fix it.")}

def includeme(config):
    config.add_route("index", "/")
    config.add_route("annotation_with_url", "/{id}/*url")
    config.add_route("annotation_without_url", "/{id}")
    config.scan(__name__)
<commit_msg>Insert missing space around operator<commit_after>import json
from urllib import parse

from elasticsearch import exceptions
from pyramid import httpexceptions
from pyramid import i18n
from pyramid import view
from statsd.defaults.env import statsd

from bouncer import util

_ = i18n.TranslationStringFactory(__package__)

#: The maximum length that the "netloc" (the www.example.com part in
#: http://www.example.com/example) can be in the pretty URL that is displayed
#: to the user before it gets truncated.
NETLOC_MAX_LENGTH = 20

@view.view_defaults(renderer="bouncer:templates/annotation.html.jinja2")
class AnnotationController(object):

    def __init__(self, request):
        self.request = request

    @view.view_config(route_name="annotation_with_url")
    @view.view_config(route_name="annotation_without_url")
    def annotation(self):
        settings = self.request.registry.settings

        try:
            document = util.elasticsearch_client(settings).get(
                index=settings["elasticsearch_index"],
                doc_type="annotation",
                id=self.request.matchdict["id"])
        except exceptions.NotFoundError:
            statsd.incr("views.annotation.404.annotation_not_found")
            raise httpexceptions.HTTPNotFound(_("Annotation not found"))

        try:
            annotation_id, document_uri = util.parse_document(document)
        except util.InvalidAnnotationError as exc:
            statsd.incr("views.annotation.422.{}".format(exc.reason))
            raise httpexceptions.HTTPUnprocessableEntity(str(exc))

        # Remove any existing #fragment identifier from the URI before we
        # append our own.
        document_uri = parse.urldefrag(document_uri)[0]

        if not (document_uri.startswith("http://") or
                document_uri.startswith("https://")):
            statsd.incr("views.annotation.422.not_an_http_or_https_document")
            raise httpexceptions.HTTPUnprocessableEntity(
                _("Sorry, but it looks like this annotation was made on a "
                  "document that is not publicly available. "
                  "To view its annotations, a document's address must start "
                  "with http:// or https://."))

        via_url = "{via_base_url}/{uri}#annotations:{id}".format(
            via_base_url=settings["via_base_url"],
            uri=document_uri,
            id=annotation_id)

        extension_url = "{uri}#annotations:{id}".format(
            uri=document_uri, id=annotation_id)

        parsed_url = parse.urlparse(document_uri)
        pretty_url = parsed_url.netloc[:NETLOC_MAX_LENGTH]
        if len(parsed_url.netloc) > NETLOC_MAX_LENGTH:
          pretty_url = pretty_url + jinja2.Markup("…")

        statsd.incr("views.annotation.200.annotation_found")
        return {
            "data": json.dumps({
                # Warning: variable names change from python_style to
                # javaScriptStyle here!
                "chromeExtensionId": settings["chrome_extension_id"],
                "viaUrl": via_url,
                "extensionUrl": extension_url,
            }),
            "pretty_url": pretty_url
        }

@view.view_config(renderer="bouncer:templates/index.html.jinja2",
                  route_name="index")
def index(request):
    statsd.incr("views.index.302.redirected_to_hypothesis")
    raise httpexceptions.HTTPFound(
        location=request.registry.settings["hypothesis_url"])

@view.view_defaults(renderer='bouncer:templates/error.html.jinja2')
class ErrorController(object):

    def __init__(self, exc, request):
        self.exc = exc
        self.request = request

    @view.view_config(context=httpexceptions.HTTPError)
    @view.view_config(context=httpexceptions.HTTPServerError)
    def httperror(self):
        self.request.response.status_int = self.exc.status_int
        # If code raises an HTTPError or HTTPServerError we assume this was
        # deliberately raised and:
        # 1. Show the user an error page including specific error message
        # 2. _Do not_ report the error to Sentry.
        return {"message": str(self.exc)}

    @view.view_config(context=Exception)
    def error(self):
        # In debug mode re-raise exceptions so that they get printed in the
        # terminal.
        if self.request.registry.settings["debug"]:
            raise

        self.request.response.status_int = 500

        # If code raises a non-HTTPException exception we assume it was a bug
        # and:
        # 1. Show the user a generic error page
        # 2. Report the details of the error to Sentry.
        self.request.raven.captureException()
        return {"message": _("Sorry, but something went wrong with the link. "
                             "The issue has been reported and we'll try to "
                             "fix it.")}

def includeme(config):
    config.add_route("index", "/")
    config.add_route("annotation_with_url", "/{id}/*url")
    config.add_route("annotation_without_url", "/{id}")
    config.scan(__name__)
<|endoftext|>


<commit_before>import json
import numpy as np
import pandas as pd
import platform
import warnings
from os import environ
from rl.util import *

warnings.filterwarnings("ignore", module="matplotlib")

MPL_BACKEND = 'agg' if (
    environ.get('CI') or platform.system() == 'Darwin') else 'TkAgg'

STATS_COLS = [
    'fitness_score',
    'mean_rewards_per_epi_stats_mean',
    'mean_rewards_stats_mean',
    'epi_stats_mean',
    'solved_ratio_of_sessions',
    'num_of_sessions',
    'max_total_rewards_stats_mean',
    't_stats_mean',
    'trial_id'
]

EXPERIMENT_GRID_Y_COLS = [
    'fitness_score',
    'mean_rewards_stats_mean',
    'max_total_rewards_stats_mean',
    'epi_stats_mean'
]

# import matplotlib scoped to the class for gc in multiprocessing
def scoped_mpl_import():
    import matplotlib
    matplotlib.rcParams['backend'] = MPL_BACKEND

    import matplotlib.pyplot as plt
    plt.rcParams['toolbar'] = 'None'  # mute matplotlib toolbar

    import seaborn as sns
    sns.set(style="whitegrid", color_codes=True, font_scale=1.0,
            rc={'lines.linewidth': 1.0,
                'backend': matplotlib.rcParams['backend']})
    palette = sns.color_palette("Blues_d")
    palette.reverse()
    sns.set_palette(palette)

    return (matplotlib, plt, sns)

class Grapher(object):

    '''
    Grapher object that belongs to a Session
    to draw graphs from its data
    '''

    def __init__(self, session):
        if environ.get('CI'):
            return
        (_mpl, self.plt, _sns) = scoped_mpl_import()
        self.session = session
        self.graph_filename = self.session.graph_filename
        self.subgraphs = {}
        self.figure = self.plt.figure(facecolor='white', figsize=(8, 9))
        self.figure.suptitle(wrap_text(self.session.session_id))
        self.init_figure()

    def init_figure(self):
        # graph 1
        ax1 = self.figure.add_subplot(
            311,
            frame_on=False,
            title="\n\ntotal rewards per episode",
            ylabel='total rewards')
        p1, = ax1.plot([], [])
        self.subgraphs['total rewards'] = (ax1, p1)

        ax1e = ax1.twinx()
        ax1e.set_ylabel('exploration rate').set_color('r')
        ax1e.set_frame_on(False)
        ax1e.grid(False)
        p1e, = ax1e.plot([], [], 'r')
        self.subgraphs['e'] = (ax1e, p1e)

        # graph 2
        ax2 = self.figure.add_subplot(
            312,
            frame_on=False,
            title='mean rewards over last 100 episodes',
            ylabel='mean rewards')
        p2, = ax2.plot([], [], 'g')
        self.subgraphs['mean rewards'] = (ax2, p2)

        # graph 3
        ax3 = self.figure.add_subplot(
            313,
            frame_on=False,
            title='loss over time, episode',
            ylabel='loss')
        p3, = ax3.plot([], [])
        self.subgraphs['loss'] = (ax3, p3)

        self.plt.tight_layout()  # auto-fix spacing
        self.plt.ion()  # for live plot

    def plot(self):
        '''do live plotting'''
        if environ.get('CI'):
            return
        sys_vars = self.session.sys_vars
        ax1, p1 = self.subgraphs['total rewards']
        p1.set_ydata(sys_vars['total_rewards_history'])
        p1.set_xdata(np.arange(len(p1.get_ydata())))
        ax1.relim()
        ax1.autoscale_view(tight=True, scalex=True, scaley=True)

        ax1e, p1e = self.subgraphs['e']
        p1e.set_ydata(sys_vars['explore_history'])
        p1e.set_xdata(np.arange(len(p1e.get_ydata())))
        ax1e.relim()
        ax1e.autoscale_view(tight=True, scalex=True, scaley=True)

        ax2, p2 = self.subgraphs['mean rewards']
        p2.set_ydata(sys_vars['mean_rewards_history'])
        p2.set_xdata(np.arange(len(p2.get_ydata())))
        ax2.relim()
        ax2.autoscale_view(tight=True, scalex=True, scaley=True)

        ax3, p3 = self.subgraphs['loss']
        p3.set_ydata(sys_vars['loss'])
        p3.set_xdata(np.arange(len(p3.get_ydata())))
        ax3.relim()
        ax3.autoscale_view(tight=True, scalex=True, scaley=True)

        self.plt.draw()
        self.plt.pause(0.01)
        self.save()
        import gc
        gc.collect()

    def save(self):
        '''save graph to filename'''
        self.figure.savefig(self.graph_filename)

    def clear(self):
        if environ.get('CI'):
            return
        self.plt.close()
        del_self_attr(self)

def fitness_score(mean_rewards_per_epi, solved_ratio_of_sessions):
    '''
    calculate the fitness score for hyperparameter optimization
    +1 to ratio to account for partial solution where ratio = 0
    use square to isolate early-fail high mean_rewards/epi value
    '''
    return mean_rewards_per_epi * (1+solved_ratio_of_sessions)**2

def ideal_fitness_score(mean_rewards, epi, solved_epi_speedup):
    '''
    calculate the ideal fitness_score with perfect solved ratio
    for hyperparameter optimization to select
    '''
    ideal_mean_rewards_per_epi = mean_rewards / (epi/solved_epi_speedup)
    ideal_solved_ratio = 1
    ideal_fitness_score = fitness_score(
        ideal_mean_rewards_per_epi, ideal_solved_ratio)
    return ideal_fitness_score

def basic_stats(array):
    '''generate the basic stats for a numerical array'''
    if not len(array):
        return None
    return {
        'min': np.min(array).astype(float),
        'max': np.max(array).astype(float),
        'mean': np.mean(array).astype(float),
        'std': np.std(array).astype(float),
    }

def compose_data(trial):
    '''
    compose raw data from an trial object
    into useful summary and full metrics for analysis
    '''
    sys_vars_array = trial.data['sys_vars_array']

    # collect all data from sys_vars_array
    solved_sys_vars_array = list(filter(
        lambda sv: sv['solved'], sys_vars_array))
    errored_array = list(map(
        lambda sv: sv['errored'], sys_vars_array))
    mean_rewards_array = np.array(list(map(
        lambda sv: sv['mean_rewards'], sys_vars_array)))
    max_total_rewards_array = np.array(list(map(
        lambda sv: np.max(sv['total_rewards_history']), sys_vars_array)))
    epi_array = np.array(list(map(lambda sv: sv['epi'], sys_vars_array)))
    mean_rewards_per_epi_array = np.divide(mean_rewards_array, epi_array + 1)
    t_array = np.array(list(map(lambda sv: sv['t'], sys_vars_array)))
    time_taken_array = np.array(list(map(
        lambda sv: timestamp_elapse_to_seconds(sv['time_taken']),
        sys_vars_array)))
    solved_epi_array = np.array(list(map(
        lambda sv: sv['epi'], solved_sys_vars_array)))
    solved_t_array = np.array(list(map(
        lambda sv: sv['t'], solved_sys_vars_array)))
    solved_time_taken_array = np.array(list(map(
        lambda sv: timestamp_elapse_to_seconds(sv['time_taken']),
        solved_sys_vars_array)))

    # compose sys_vars stats
    stats = {
        'num_of_sessions': len(sys_vars_array),
        'solved_num_of_sessions': len(solved_sys_vars_array),
        'solved_ratio_of_sessions': float(len(
            solved_sys_vars_array)) / trial.times,
        'errored': any(errored_array),
        'mean_rewards_stats': basic_stats(mean_rewards_array),
        'mean_rewards_per_epi_stats': basic_stats(
            mean_rewards_per_epi_array),
        'max_total_rewards_stats': basic_stats(max_total_rewards_array),
        'epi_stats': basic_stats(epi_array),
        't_stats': basic_stats(t_array),
        'time_taken_stats': basic_stats(time_taken_array),
        'solved_epi_stats': basic_stats(solved_epi_array),
        'solved_t_stats': basic_stats(solved_t_array),
        'solved_time_taken_stats': basic_stats(solved_time_taken_array),
    }
    stats.update({
        'fitness_score': fitness_score(
            stats['mean_rewards_per_epi_stats']['mean'],
            stats['solved_ratio_of_sessions'])
    })

    # summary metrics
    metrics = {
        'fitness_score': stats['fitness_score'],
        'mean_rewards_per_epi_stats_mean': stats[
            'mean_rewards_per_epi_stats']['mean'],
        'mean_rewards_stats_mean': stats['mean_rewards_stats']['mean'],
        'epi_stats_mean': stats['epi_stats']['mean'],
        'solved_ratio_of_sessions': stats['solved_ratio_of_sessions'],
        'max_total_rewards_stats_mean': stats[
            'max_total_rewards_stats']['mean'],
        't_stats_mean': stats['t_stats']['mean'],
    }

    # param variables for independent vars of trials
    param_variables = {
        pv: trial.experiment_spec['param'][pv] for
        pv in trial.param_variables}
    param_variables = flat_cast_dict(param_variables)

    trial.data['metrics'].update(metrics)
    trial.data['param_variables'] = param_variables
    trial.data['stats'] = stats
    return trial.data

# order a unique df categorical data for plotting
def order_category(uniq_df):
    uniq_list = list(uniq_df)
    try:
        uniq_dict = {k: json.loads(k) for k in uniq_list}
        sorted_pair = sorted(uniq_dict.items(), key=lambda x: x[1])
        return [pair[0] for pair in sorted_pair]
    except:
        return list(sorted(uniq_list))

# plot the experiment data from data_df
# X are columns with name starting with 'variable_'
# Y cols are defined below
def plot_experiment(data_df, trial_id):
    if len(data_df) < 2:  # no multi selection
        return
    (_mpl, _plt, sns) = scoped_mpl_import()
    experiment_id = parse_experiment_id(trial_id)
    hue = 'solved_ratio_of_sessions'
    X_cols = list(filter(lambda c: c.startswith('variable_'), data_df.columns))
    col_size = len(X_cols)
    row_size = len(EXPERIMENT_GRID_Y_COLS)
    groups = data_df.groupby(hue)

    # for main grid plot
    sns_only = True
    big_fig, axes = sns.plt.subplots(
        row_size, col_size, figsize=(col_size*4, row_size*3),
        sharex='col', sharey='row')
    for ix, x in enumerate(X_cols):
        for iy, y in enumerate(EXPERIMENT_GRID_Y_COLS):
            big_ax = axes[iy] if col_size == 1 else axes[iy][ix]
            uniq_df = data_df[x].unique()
            if (data_df[x].dtype.name == 'category' or
                    len(uniq_df) <= 5):
                order = order_category(uniq_df)
                sns.swarmplot(
                    data=data_df, x=x, y=y, hue=hue, size=3,
                    order=order, ax=big_ax)
            else:
                sns_only = False
                big_ax.margins(0.05)
                big_ax.xaxis.grid(False)
                for _, group in groups:
                    big_ax.plot(group[x], group[y], label=hue,
                                marker='o', ms=3, linestyle='')
                    big_ax.set_xlabel(x)
                    big_ax.set_ylabel(y)

            big_ax.legend_ = None  # set common legend below
            # label only left and bottom axes
            if iy != row_size - 1:
                big_ax.set_xlabel('')
            if ix != 0:
                big_ax.set_ylabel('')

    big_fig.tight_layout()
    big_fig.suptitle(wrap_text(experiment_id))
    legend_labels = None if sns_only else sorted(data_df[hue].unique())
    legend_ms = 0.5 if sns_only else 1
    legend = sns.plt.legend(title='solved_ratio_of_sessions',
                            labels=legend_labels, markerscale=legend_ms,
                            fontsize=10, loc='center right',
                            bbox_to_anchor=(1.1+col_size*0.1, row_size+0.1))
    legend.get_title().set_fontsize('10')
    big_fig.subplots_adjust(top=0.96, right=0.9)

    filename = './data/{0}/{0}_analysis.png'.format(
        experiment_id)
    big_fig.savefig(filename)
    big_fig.clear()

    # use numerical, since contour only makes sense for ordered azes
    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
    numeric_X_cols = list(
        filter(lambda x: data_df[x].dtype in numerics, X_cols))
    with sns.axes_style('white', {'axes.linewidth': 0.2}):
        g = sns.pairplot(
            data_df, vars=numeric_X_cols, hue=hue,
            size=3, aspect=1, plot_kws={'s': 50, 'alpha': 0.5})
        g.fig.suptitle(wrap_text(experiment_id))
        g = g.add_legend()
        filename = './data/{0}/{0}_analysis_correlation.png'.format(
            experiment_id)
        g.savefig(filename)
        g.fig.clear()

    sns.plt.close()

def analyze_data(experiment_data_or_experiment_id):
    '''
    get all the data from all trials.run()
    or read from all data files matching the prefix of trial_id
    e.g. usage without running:
    experiment_id = 'DevCartPole-v0_DQN_LinearMemoryWithForgetting_BoltzmannPolicy_2017-01-15_142810'
    analyze_data(experiment_id)
    '''
    if isinstance(experiment_data_or_experiment_id, str):
        experiment_data = load_data_array_from_experiment_id(
            experiment_data_or_experiment_id)
    else:
        experiment_data = experiment_data_or_experiment_id

    stats_array, param_variables_array = [], []
    for data in experiment_data:
        stats = flatten_dict(data['stats'])
        stats.update({'trial_id': data['trial_id']})
        param_variables = data['param_variables']
        if stats['errored']:  # remove errored trials
            continue
        stats_array.append(stats)
        param_variables_array.append(param_variables)

    raw_stats_df = pd.DataFrame.from_dict(stats_array)
    stats_df = raw_stats_df[STATS_COLS]

    param_variables_df = pd.DataFrame.from_dict(param_variables_array)
    param_variables_df.columns = [
        'variable_'+c for c in param_variables_df.columns]

    data_df = pd.concat([stats_df, param_variables_df], axis=1)
    for c in data_df.columns:
        if data_df[c].dtype == object:  # guard
            data_df[c] = data_df[c].astype('category')

    data_df.sort_values(
        ['fitness_score'],
        inplace=True, ascending=False)

    trial_id = experiment_data[0]['trial_id']
    save_experiment_data(data_df, trial_id)
    plot_experiment(data_df, trial_id)
    return data_df
<commit_msg>fix codacy<commit_after>import json
import numpy as np
import pandas as pd
import platform
import warnings
from os import environ
from rl.util import *

warnings.filterwarnings("ignore", module="matplotlib")

MPL_BACKEND = 'agg' if (
    environ.get('CI') or platform.system() == 'Darwin') else 'TkAgg'

STATS_COLS = [
    'fitness_score',
    'mean_rewards_per_epi_stats_mean',
    'mean_rewards_stats_mean',
    'epi_stats_mean',
    'solved_ratio_of_sessions',
    'num_of_sessions',
    'max_total_rewards_stats_mean',
    't_stats_mean',
    'trial_id'
]

EXPERIMENT_GRID_Y_COLS = [
    'fitness_score',
    'mean_rewards_stats_mean',
    'max_total_rewards_stats_mean',
    'epi_stats_mean'
]

# import matplotlib scoped to the class for gc in multiprocessing
def scoped_mpl_import():
    import matplotlib
    matplotlib.rcParams['backend'] = MPL_BACKEND

    import matplotlib.pyplot as plt
    plt.rcParams['toolbar'] = 'None'  # mute matplotlib toolbar

    import seaborn as sns
    sns.set(style="whitegrid", color_codes=True, font_scale=1.0,
            rc={'lines.linewidth': 1.0,
                'backend': matplotlib.rcParams['backend']})
    palette = sns.color_palette("Blues_d")
    palette.reverse()
    sns.set_palette(palette)

    return (matplotlib, plt, sns)

class Grapher(object):

    '''
    Grapher object that belongs to a Session
    to draw graphs from its data
    '''

    def __init__(self, session):
        if environ.get('CI'):
            return
        (_mpl, self.plt, _sns) = scoped_mpl_import()
        self.session = session
        self.graph_filename = self.session.graph_filename
        self.subgraphs = {}
        self.figure = self.plt.figure(facecolor='white', figsize=(8, 9))
        self.figure.suptitle(wrap_text(self.session.session_id))
        self.init_figure()

    def init_figure(self):
        # graph 1
        ax1 = self.figure.add_subplot(
            311,
            frame_on=False,
            title="\n\ntotal rewards per episode",
            ylabel='total rewards')
        p1, = ax1.plot([], [])
        self.subgraphs['total rewards'] = (ax1, p1)

        ax1e = ax1.twinx()
        ax1e.set_ylabel('exploration rate').set_color('r')
        ax1e.set_frame_on(False)
        ax1e.grid(False)
        p1e, = ax1e.plot([], [], 'r')
        self.subgraphs['e'] = (ax1e, p1e)

        # graph 2
        ax2 = self.figure.add_subplot(
            312,
            frame_on=False,
            title='mean rewards over last 100 episodes',
            ylabel='mean rewards')
        p2, = ax2.plot([], [], 'g')
        self.subgraphs['mean rewards'] = (ax2, p2)

        # graph 3
        ax3 = self.figure.add_subplot(
            313,
            frame_on=False,
            title='loss over time, episode',
            ylabel='loss')
        p3, = ax3.plot([], [])
        self.subgraphs['loss'] = (ax3, p3)

        self.plt.tight_layout()  # auto-fix spacing
        self.plt.ion()  # for live plot

    def plot(self):
        '''do live plotting'''
        if environ.get('CI'):
            return
        sys_vars = self.session.sys_vars
        ax1, p1 = self.subgraphs['total rewards']
        p1.set_ydata(sys_vars['total_rewards_history'])
        p1.set_xdata(np.arange(len(p1.get_ydata())))
        ax1.relim()
        ax1.autoscale_view(tight=True, scalex=True, scaley=True)

        ax1e, p1e = self.subgraphs['e']
        p1e.set_ydata(sys_vars['explore_history'])
        p1e.set_xdata(np.arange(len(p1e.get_ydata())))
        ax1e.relim()
        ax1e.autoscale_view(tight=True, scalex=True, scaley=True)

        ax2, p2 = self.subgraphs['mean rewards']
        p2.set_ydata(sys_vars['mean_rewards_history'])
        p2.set_xdata(np.arange(len(p2.get_ydata())))
        ax2.relim()
        ax2.autoscale_view(tight=True, scalex=True, scaley=True)

        ax3, p3 = self.subgraphs['loss']
        p3.set_ydata(sys_vars['loss'])
        p3.set_xdata(np.arange(len(p3.get_ydata())))
        ax3.relim()
        ax3.autoscale_view(tight=True, scalex=True, scaley=True)

        self.plt.draw()
        self.plt.pause(0.01)
        self.save()
        import gc
        gc.collect()

    def save(self):
        '''save graph to filename'''
        self.figure.savefig(self.graph_filename)

    def clear(self):
        if environ.get('CI'):
            return
        self.plt.close()
        del_self_attr(self)

def fitness_score(mean_rewards_per_epi, solved_ratio_of_sessions):
    '''
    calculate the fitness score for hyperparameter optimization
    +1 to ratio to account for partial solution where ratio = 0
    use square to isolate early-fail high mean_rewards/epi value
    '''
    return mean_rewards_per_epi * (1+solved_ratio_of_sessions)**2

def ideal_fitness_score(mean_rewards, epi, solved_epi_speedup):
    '''
    calculate the ideal fitness_score with perfect solved ratio
    for hyperparameter optimization to select
    '''
    ideal_mean_rewards_per_epi = mean_rewards / (epi/solved_epi_speedup)
    ideal_solved_ratio = 1
    ideal_fitness_score = fitness_score(
        ideal_mean_rewards_per_epi, ideal_solved_ratio)
    return ideal_fitness_score

def basic_stats(array):
    '''generate the basic stats for a numerical array'''
    if not len(array):
        return None
    return {
        'min': np.min(array).astype(float),
        'max': np.max(array).astype(float),
        'mean': np.mean(array).astype(float),
        'std': np.std(array).astype(float),
    }

def compose_data(trial):
    '''
    compose raw data from an trial object
    into useful summary and full metrics for analysis
    '''
    sys_vars_array = trial.data['sys_vars_array']

    # collect all data from sys_vars_array
    solved_sys_vars_array = list(filter(
        lambda sv: sv['solved'], sys_vars_array))
    errored_array = list(map(
        lambda sv: sv['errored'], sys_vars_array))
    mean_rewards_array = np.array(list(map(
        lambda sv: sv['mean_rewards'], sys_vars_array)))
    max_total_rewards_array = np.array(list(map(
        lambda sv: np.max(sv['total_rewards_history']), sys_vars_array)))
    epi_array = np.array(list(map(lambda sv: sv['epi'], sys_vars_array)))
    mean_rewards_per_epi_array = np.divide(mean_rewards_array, epi_array + 1)
    t_array = np.array(list(map(lambda sv: sv['t'], sys_vars_array)))
    time_taken_array = np.array(list(map(
        lambda sv: timestamp_elapse_to_seconds(sv['time_taken']),
        sys_vars_array)))
    solved_epi_array = np.array(list(map(
        lambda sv: sv['epi'], solved_sys_vars_array)))
    solved_t_array = np.array(list(map(
        lambda sv: sv['t'], solved_sys_vars_array)))
    solved_time_taken_array = np.array(list(map(
        lambda sv: timestamp_elapse_to_seconds(sv['time_taken']),
        solved_sys_vars_array)))

    # compose sys_vars stats
    stats = {
        'num_of_sessions': len(sys_vars_array),
        'solved_num_of_sessions': len(solved_sys_vars_array),
        'solved_ratio_of_sessions': float(len(
            solved_sys_vars_array)) / trial.times,
        'errored': any(errored_array),
        'mean_rewards_stats': basic_stats(mean_rewards_array),
        'mean_rewards_per_epi_stats': basic_stats(
            mean_rewards_per_epi_array),
        'max_total_rewards_stats': basic_stats(max_total_rewards_array),
        'epi_stats': basic_stats(epi_array),
        't_stats': basic_stats(t_array),
        'time_taken_stats': basic_stats(time_taken_array),
        'solved_epi_stats': basic_stats(solved_epi_array),
        'solved_t_stats': basic_stats(solved_t_array),
        'solved_time_taken_stats': basic_stats(solved_time_taken_array),
    }
    stats.update({
        'fitness_score': fitness_score(
            stats['mean_rewards_per_epi_stats']['mean'],
            stats['solved_ratio_of_sessions'])
    })

    # summary metrics
    metrics = {
        'fitness_score': stats['fitness_score'],
        'mean_rewards_per_epi_stats_mean': stats[
            'mean_rewards_per_epi_stats']['mean'],
        'mean_rewards_stats_mean': stats['mean_rewards_stats']['mean'],
        'epi_stats_mean': stats['epi_stats']['mean'],
        'solved_ratio_of_sessions': stats['solved_ratio_of_sessions'],
        'max_total_rewards_stats_mean': stats[
            'max_total_rewards_stats']['mean'],
        't_stats_mean': stats['t_stats']['mean'],
    }

    # param variables for independent vars of trials
    param_variables = {
        pv: trial.experiment_spec['param'][pv] for
        pv in trial.param_variables}
    param_variables = flat_cast_dict(param_variables)

    trial.data['metrics'].update(metrics)
    trial.data['param_variables'] = param_variables
    trial.data['stats'] = stats
    return trial.data

# order a unique df categorical data for plotting
def order_category(uniq_df):
    uniq_list = list(uniq_df)
    try:
        uniq_dict = {k: json.loads(k) for k in uniq_list}
        sorted_pair = sorted(uniq_dict.items(), key=lambda x: x[1])
        return [pair[0] for pair in sorted_pair]
    except json.JSONDecodeError:
        return list(sorted(uniq_list))

# plot the experiment data from data_df
# X are columns with name starting with 'variable_'
# Y cols are defined below
def plot_experiment(data_df, trial_id):
    if len(data_df) < 2:  # no multi selection
        return
    (_mpl, _plt, sns) = scoped_mpl_import()
    experiment_id = parse_experiment_id(trial_id)
    hue = 'solved_ratio_of_sessions'
    X_cols = list(filter(lambda c: c.startswith('variable_'), data_df.columns))
    col_size = len(X_cols)
    row_size = len(EXPERIMENT_GRID_Y_COLS)
    groups = data_df.groupby(hue)

    # for main grid plot
    sns_only = True
    big_fig, axes = sns.plt.subplots(
        row_size, col_size, figsize=(col_size*4, row_size*3),
        sharex='col', sharey='row')
    for ix, x in enumerate(X_cols):
        for iy, y in enumerate(EXPERIMENT_GRID_Y_COLS):
            big_ax = axes[iy] if col_size == 1 else axes[iy][ix]
            uniq_df = data_df[x].unique()
            if (data_df[x].dtype.name == 'category' or
                    len(uniq_df) <= 5):
                order = order_category(uniq_df)
                sns.swarmplot(
                    data=data_df, x=x, y=y, hue=hue, size=3,
                    order=order, ax=big_ax)
            else:
                sns_only = False
                big_ax.margins(0.05)
                big_ax.xaxis.grid(False)
                for _, group in groups:
                    big_ax.plot(group[x], group[y], label=hue,
                                marker='o', ms=3, linestyle='')
                    big_ax.set_xlabel(x)
                    big_ax.set_ylabel(y)

            big_ax.legend_ = None  # set common legend below
            # label only left and bottom axes
            if iy != row_size - 1:
                big_ax.set_xlabel('')
            if ix != 0:
                big_ax.set_ylabel('')

    big_fig.tight_layout()
    big_fig.suptitle(wrap_text(experiment_id))
    legend_labels = None if sns_only else sorted(data_df[hue].unique())
    legend_ms = 0.5 if sns_only else 1
    legend = sns.plt.legend(title='solved_ratio_of_sessions',
                            labels=legend_labels, markerscale=legend_ms,
                            fontsize=10, loc='center right',
                            bbox_to_anchor=(1.1+col_size*0.1, row_size+0.1))
    legend.get_title().set_fontsize('10')
    big_fig.subplots_adjust(top=0.96, right=0.9)

    filename = './data/{0}/{0}_analysis.png'.format(
        experiment_id)
    big_fig.savefig(filename)
    big_fig.clear()

    # use numerical, since contour only makes sense for ordered azes
    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
    numeric_X_cols = list(
        filter(lambda x: data_df[x].dtype in numerics, X_cols))
    with sns.axes_style('white', {'axes.linewidth': 0.2}):
        g = sns.pairplot(
            data_df, vars=numeric_X_cols, hue=hue,
            size=3, aspect=1, plot_kws={'s': 50, 'alpha': 0.5})
        g.fig.suptitle(wrap_text(experiment_id))
        g = g.add_legend()
        filename = './data/{0}/{0}_analysis_correlation.png'.format(
            experiment_id)
        g.savefig(filename)
        g.fig.clear()

    sns.plt.close()

def analyze_data(experiment_data_or_experiment_id):
    '''
    get all the data from all trials.run()
    or read from all data files matching the prefix of trial_id
    e.g. usage without running:
    experiment_id = 'DevCartPole-v0_DQN_LinearMemoryWithForgetting_BoltzmannPolicy_2017-01-15_142810'
    analyze_data(experiment_id)
    '''
    if isinstance(experiment_data_or_experiment_id, str):
        experiment_data = load_data_array_from_experiment_id(
            experiment_data_or_experiment_id)
    else:
        experiment_data = experiment_data_or_experiment_id

    stats_array, param_variables_array = [], []
    for data in experiment_data:
        stats = flatten_dict(data['stats'])
        stats.update({'trial_id': data['trial_id']})
        param_variables = data['param_variables']
        if stats['errored']:  # remove errored trials
            continue
        stats_array.append(stats)
        param_variables_array.append(param_variables)

    raw_stats_df = pd.DataFrame.from_dict(stats_array)
    stats_df = raw_stats_df[STATS_COLS]

    param_variables_df = pd.DataFrame.from_dict(param_variables_array)
    param_variables_df.columns = [
        'variable_'+c for c in param_variables_df.columns]

    data_df = pd.concat([stats_df, param_variables_df], axis=1)
    for c in data_df.columns:
        if data_df[c].dtype == object:  # guard
            data_df[c] = data_df[c].astype('category')

    data_df.sort_values(
        ['fitness_score'],
        inplace=True, ascending=False)

    trial_id = experiment_data[0]['trial_id']
    save_experiment_data(data_df, trial_id)
    plot_experiment(data_df, trial_id)
    return data_df
<|endoftext|>


<commit_before># encoding: utf-8

from datetime import datetime

__author__ = "Tsuyoshi Hombashi"
__copyright__ = "Copyright 2016-{}, {}".format(datetime.now().year, __author__)
__license__ = "MIT License"
__version__ = "0.44.1"
__maintainer__ = __author__
__email__ = "tsuyoshi.hombashi@gmail.com"
<commit_msg>Bump version<commit_after># encoding: utf-8

from datetime import datetime

__author__ = "Tsuyoshi Hombashi"
__copyright__ = "Copyright 2016-{}, {}".format(datetime.now().year, __author__)
__license__ = "MIT License"
__version__ = "0.44.2"
__maintainer__ = __author__
__email__ = "tsuyoshi.hombashi@gmail.com"
<|endoftext|>


<commit_before>DRIVER_TEMPLATE = """\
import sys
import os
import inspect

sys.path.insert(0, os.getcwd())

from pydoop.mapreduce.pipes import run_task  # noqa: E402
import pydoop.pipes  # noqa: E402
import %(module)s  # noqa: E402

class ContextWriter(object):

    def __init__(self, context):
        self.context = context
        self.counters = {}

    def emit(self, k, v):
        self.context.emit(k, v)

    def count(self, what, howmany):
        counter = self.counters.setdefault(
            what, self.context.getCounter('%(module)s', what)
        )
        self.context.incrementCounter(counter, howmany)

    def status(self, msg):
        self.context.setStatus(msg)

    def progress(self):
        self.context.progress()

def setup_script_object(obj, fn_attr_name, user_fn, ctx):
    # Generic constructor for both map and reduce objects.
    #
    # Sets the 'writer' and 'conf' attributes.  Then, based on the arity
    # of the given user function (user_fn), sets the object attribute
    # (fn_attr_name, which should be either 'map' or 'reduce') to point
    # to either:
    #
    #   * obj.with_conf (when arity == 4)
    #   * obj.without_conf (when arity == 3)
    #
    # This way, when pipes calls the map/reduce function of the object
    # it actually gets either of the with_conf/without_conf functions
    # (which must be defined by the PydoopScriptMapper or
    # PydoopScriptReducer object passed into this function).
    #
    # Why all this?  The idea is to raise any decision about which
    # function to call out of the map/reduce functions, which get called
    # a number of times proportional to the amount of data to process.
    # On the other hand, the constructor only gets called once per task.
    if fn_attr_name not in ('map', 'reduce'):
        raise RuntimeError('Unexpected function attribute ' + fn_attr_name)
    obj.writer = ContextWriter(ctx)
    obj.conf = ctx.getJobConf()
    spec = inspect.getargspec(user_fn)
    if spec.varargs or len(spec.args) not in (3, 4):
        raise ValueError(
            user_fn +
            ' must take parameters key, value, writer, and optionally config'
        )
    if len(spec.args) == 3:
        setattr(obj, fn_attr_name, obj.without_conf)
    elif len(spec.args) == 4:
        setattr(obj, fn_attr_name, obj.with_conf)
    else:
        raise RuntimeError(
            'Unexpected number of %(map_fn)s arguments ' + len(spec.args))

class PydoopScriptMapper(pydoop.pipes.Mapper):
    def __init__(self, ctx):
        super(type(self), self).__init__(ctx)
        setup_script_object(self, 'map', %(module)s.%(map_fn)s, ctx)

    def without_conf(self, ctx):
        # old style map function, without the conf parameter
        writer = ContextWriter(ctx)
        %(module)s.%(map_fn)s(ctx.getInputKey(), ctx.getInputValue(), writer)

    def with_conf(self, ctx):
        # new style map function, without the conf parameter
        writer = ContextWriter(ctx)
        %(module)s.%(map_fn)s(
            ctx.getInputKey(), ctx.getInputValue(), writer, self.conf
        )

    def map(self, ctx):
        pass

class PydoopScriptReducer(pydoop.pipes.Reducer):

    def __init__(self, ctx):
        super(type(self), self).__init__(ctx)
        setup_script_object(self, 'reduce', %(module)s.%(reduce_fn)s, ctx)

    @staticmethod
    def iter(ctx):
        while ctx.nextValue():
            yield ctx.getInputValue()

    def without_conf(self, ctx):
        key = ctx.getInputKey()
        writer = ContextWriter(ctx)
        %(module)s.%(reduce_fn)s(key, PydoopScriptReducer.iter(ctx), writer)

    def with_conf(self, ctx):
        key = ctx.getInputKey()
        writer = ContextWriter(ctx)
        %(module)s.%(reduce_fn)s(
            key, PydoopScriptReducer.iter(ctx), writer, self.conf
        )

    def reduce(self, ctx):
        pass

class PydoopScriptCombiner(pydoop.pipes.Combiner):

    def __init__(self, ctx):
        super(type(self), self).__init__(ctx)
        setup_script_object(self, 'reduce', %(module)s.%(combine_fn)s, ctx)

    @staticmethod
    def iter(ctx):
        while ctx.nextValue():
            yield ctx.getInputValue()

    def without_conf(self, ctx):
        key = ctx.getInputKey()
        writer = ContextWriter(ctx)
        %(module)s.%(combine_fn)s(key, PydoopScriptCombiner.iter(ctx), writer)

    def with_conf(self, ctx):
        key = ctx.getInputKey()
        writer = ContextWriter(ctx)
        %(module)s.%(combine_fn)s(
            key, PydoopScriptReducer.iter(ctx), writer, self.conf
        )

    def reduce(self, ctx):
        pass

def main():
    run_task(pydoop.pipes.Factory(
        PydoopScriptMapper, PydoopScriptReducer,
        record_reader_class=None,
        record_writer_class=None,
        combiner_class=%(combiner_wp)s,
        partitioner_class=None))
"""
<commit_msg>pydoop script template: use new mapreduce API<commit_after>DRIVER_TEMPLATE = """\
import sys
import os
import inspect

sys.path.insert(0, os.getcwd())

import pydoop.mapreduce.api as api  # noqa: E402
import pydoop.mapreduce.pipes as pipes  # noqa: E402
import %(module)s  # noqa: E402

class ContextWriter(object):

    def __init__(self, context):
        self.context = context
        self.counters = {}

    def emit(self, k, v):
        self.context.emit(k, v)

    def count(self, what, howmany):
        counter = self.counters.setdefault(
            what, self.context.get_counter('%(module)s', what)
        )
        self.context.increment_counter(counter, howmany)

    def status(self, msg):
        self.context.set_status(msg)

    def progress(self):
        self.context.progress()

def setup_script_object(obj, fn_attr_name, user_fn, ctx):
    # Generic constructor for both map and reduce objects.
    #
    # Sets the 'writer' and 'conf' attributes.  Then, based on the arity
    # of the given user function (user_fn), sets the object attribute
    # (fn_attr_name, which should be either 'map' or 'reduce') to point
    # to either:
    #
    #   * obj.with_conf (when arity == 4)
    #   * obj.without_conf (when arity == 3)
    #
    # This way, when pipes calls the map/reduce function of the object
    # it actually gets either of the with_conf/without_conf functions
    # (which must be defined by the PydoopScriptMapper or
    # PydoopScriptReducer object passed into this function).
    #
    # Why all this?  The idea is to raise any decision about which
    # function to call out of the map/reduce functions, which get called
    # a number of times proportional to the amount of data to process.
    # On the other hand, the constructor only gets called once per task.
    if fn_attr_name not in ('map', 'reduce'):
        raise RuntimeError('Unexpected function attribute ' + fn_attr_name)
    obj.writer = ContextWriter(ctx)
    obj.conf = ctx.get_job_conf()
    spec = inspect.getargspec(user_fn)
    if spec.varargs or len(spec.args) not in (3, 4):
        raise ValueError(
            user_fn +
            ' must take parameters key, value, writer, and optionally config'
        )
    if len(spec.args) == 3:
        setattr(obj, fn_attr_name, obj.without_conf)
    elif len(spec.args) == 4:
        setattr(obj, fn_attr_name, obj.with_conf)
    else:
        raise RuntimeError(
            'Unexpected number of %(map_fn)s arguments ' + len(spec.args)
        )

class PydoopScriptMapper(api.Mapper):

    def __init__(self, ctx):
        super(PydoopScriptMapper, self).__init__(ctx)
        setup_script_object(self, 'map', %(module)s.%(map_fn)s, ctx)

    def without_conf(self, ctx):
        # old style map function, without the conf parameter
        writer = ContextWriter(ctx)
        %(module)s.%(map_fn)s(ctx.key, ctx.value, writer)

    def with_conf(self, ctx):
        # new style map function, without the conf parameter
        writer = ContextWriter(ctx)
        %(module)s.%(map_fn)s(ctx.key, ctx.value, writer, self.conf)

    def map(self, ctx):
        pass

class PydoopScriptReducer(api.Reducer):

    def __init__(self, ctx):
        super(PydoopScriptReducer, self).__init__(ctx)
        setup_script_object(self, 'reduce', %(module)s.%(reduce_fn)s, ctx)

    def without_conf(self, ctx):
        writer = ContextWriter(ctx)
        %(module)s.%(reduce_fn)s(ctx.key, ctx.values, writer)

    def with_conf(self, ctx):
        writer = ContextWriter(ctx)
        %(module)s.%(reduce_fn)s(ctx.key, ctx.values, writer, self.conf)

    def reduce(self, ctx):
        pass

class PydoopScriptCombiner(api.Reducer):

    def __init__(self, ctx):
        super(PydoopScriptCombiner, self).__init__(ctx)
        setup_script_object(self, 'reduce', %(module)s.%(combine_fn)s, ctx)

    def without_conf(self, ctx):
        writer = ContextWriter(ctx)
        %(module)s.%(combine_fn)s(ctx.key, ctx.values, writer)

    def with_conf(self, ctx):
        writer = ContextWriter(ctx)
        %(module)s.%(combine_fn)s(ctx.key, ctx.values, writer, self.conf)

    def reduce(self, ctx):
        pass

def main():
    pipes.run_task(pipes.Factory(
        PydoopScriptMapper, PydoopScriptReducer,
        record_reader_class=None,
        record_writer_class=None,
        combiner_class=%(combiner_wp)s,
        partitioner_class=None))
"""
<|endoftext|>


<commit_before><commit_msg>[FIX] event_sale: SyntaxError<commit_after><|endoftext|>