<commit_before># -*- coding: utf-8 -*-
import os

import pytest

from first import first
from mock import Mock, patch

import pipenv.utils
import pythonfinder.utils
from pipenv.exceptions import PipenvUsageError

# Pipfile format <-> requirements.txt format.
DEP_PIP_PAIRS = [
    ({"requests": "*"}, "requests"),
    ({"requests": {"extras": ["socks"], "version": "*"}}, "requests[socks]"),
    ({"django": ">1.10"}, "django>1.10"),
    ({"Django": ">1.10"}, "Django>1.10"),
    ({"requests": {"extras": ["socks"], "version": ">1.10"}}, "requests[socks]>1.10"),
    ({"requests": {"extras": ["socks"], "version": "==1.10"}}, "requests[socks]==1.10"),
    (
        {
            "pinax": {
                "git": "git://github.com/pinax/pinax.git",
                "ref": "1.4",
                "editable": True,
            }
        },
        "-e git+git://github.com/pinax/pinax.git@1.4#egg=pinax",
    ),
    (
        {"pinax": {"git": "git://github.com/pinax/pinax.git", "ref": "1.4"}},
        "git+git://github.com/pinax/pinax.git@1.4#egg=pinax",
    ),
    (  # Mercurial.
        {
            "MyProject": {
                "hg": "http://hg.myproject.org/MyProject",
                "ref": "da39a3ee5e6b",
            }
        },
        "hg+http://hg.myproject.org/MyProject@da39a3ee5e6b#egg=MyProject",
    ),
    (  # SVN.
        {
            "MyProject": {
                "svn": "svn://svn.myproject.org/svn/MyProject",
                "editable": True,
            }
        },
        "-e svn+svn://svn.myproject.org/svn/MyProject#egg=MyProject",
    ),
    (
        # Extras in url
        {
            "discord.py": {
                "file": "https://github.com/Rapptz/discord.py/archive/rewrite.zip",
                "extras": ["voice"],
            }
        },
        "https://github.com/Rapptz/discord.py/archive/rewrite.zip#egg=discord.py[voice]",
    ),
    (
        {
            "requests": {
                "git": "https://github.com/requests/requests.git",
                "ref": "master",
                "extras": ["security"],
                "editable": False,
            }
        },
        "git+https://github.com/requests/requests.git@master#egg=requests[security]",
    ),
]

@pytest.mark.utils
@pytest.mark.parametrize("deps, expected", DEP_PIP_PAIRS)
def test_convert_deps_to_pip(deps, expected):
    if expected.startswith("Django"):
        expected = expected.lower()
    assert pipenv.utils.convert_deps_to_pip(deps, r=False) == [expected]

@pytest.mark.utils
@pytest.mark.parametrize(
    "deps, expected",
    [
        # This one should be collapsed and treated as {'requests': '*'}.
        ({"requests": {}}, "requests"),
        # Hash value should be passed into the result.
        (
            {
                "FooProject": {
                    "version": "==1.2",
                    "hash": "sha256:2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824",
                }
            },
            "FooProject==1.2 --hash=sha256:2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824",
        ),
        (
            {
                "FooProject": {
                    "version": "==1.2",
                    "extras": ["stuff"],
                    "hash": "sha256:2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824",
                }
            },
            "FooProject[stuff]==1.2 --hash=sha256:2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824",
        ),
        (
            {
                "requests": {
                    "git": "https://github.com/requests/requests.git",
                    "ref": "master",
                    "extras": ["security"],
                }
            },
            "git+https://github.com/requests/requests.git@master#egg=requests[security]",
        ),
    ],
)
def test_convert_deps_to_pip_one_way(deps, expected):
    assert pipenv.utils.convert_deps_to_pip(deps, r=False) == [expected.lower()]

@pytest.mark.skipif(isinstance(u"", str), reason="don't need to test if unicode is str")
@pytest.mark.utils
def test_convert_deps_to_pip_unicode():
    deps = {u"django": u"==1.10"}
    deps = pipenv.utils.convert_deps_to_pip(deps, r=False)
    assert deps[0] == "django==1.10"

class TestUtils:
    """Test utility functions in pipenv"""

    @pytest.mark.utils
    @pytest.mark.parametrize(
        "version, specified_ver, expected",
        [
            ("*", "*", True),
            ("2.1.6", "==2.1.4", False),
            ("20160913", ">=20140815", True),
            (
                "1.4",
                {"svn": "svn://svn.myproj.org/svn/MyProj", "version": "==1.4"},
                True,
            ),
            ("2.13.0", {"extras": ["socks"], "version": "==2.12.4"}, False),
        ],
    )
    def test_is_required_version(self, version, specified_ver, expected):
        assert pipenv.utils.is_required_version(version, specified_ver) is expected

    @pytest.mark.utils
    @pytest.mark.parametrize(
        "entry, expected",
        [
            ({"git": "package.git", "ref": "v0.0.1"}, True),
            ({"hg": "https://package.com/package", "ref": "v1.2.3"}, True),
            ("*", False),
            ({"some_value": 5, "other_value": object()}, False),
            ("package", False),
            ("git+https://github.com/requests/requests.git#egg=requests", True),
            ("git+git@github.com:requests/requests.git#egg=requests", True),
            ("gitdb2", False),
        ],
    )
    @pytest.mark.vcs
    def test_is_vcs(self, entry, expected):
        from pipenv.vendor.requirementslib.utils import is_vcs
        assert is_vcs(entry) is expected

    @pytest.mark.utils
    def test_python_version_from_bad_path(self):
        assert pipenv.utils.python_version("/fake/path") is None

    @pytest.mark.utils
    def test_python_version_from_non_python(self):
        assert pipenv.utils.python_version("/dev/null") is None

    @pytest.mark.utils
    @pytest.mark.parametrize(
        "version_output, version",
        [
            ("Python 3.6.2", "3.6.2"),
            ("Python 3.6.2 :: Continuum Analytics, Inc.", "3.6.2"),
            ("Python 3.6.20 :: Continuum Analytics, Inc.", "3.6.20"),
            (
                "Python 3.5.3 (3f6eaa010fce78cc7973bdc1dfdb95970f08fed2, Jan 13 2018, 18:14:01)\n[PyPy 5.10.1 with GCC 4.2.1 Compatible Apple LLVM 9.0.0 (clang-900.0.39.2)]",
                "3.5.3",
            ),
        ],
    )
    # @patch(".vendor.pythonfinder.utils.get_python_version")
    def test_python_version_output_variants(
        self, monkeypatch, version_output, version
    ):
        def mock_version(path):
            return version_output.split()[1]
        monkeypatch.setattr("pipenv.vendor.pythonfinder.utils.get_python_version", mock_version)
        assert pipenv.utils.python_version("some/path") == version

    @pytest.mark.utils
    @pytest.mark.windows
    @pytest.mark.skipif(os.name != "nt", reason="Windows test only")
    def test_windows_shellquote(self):
        test_path = "C:\Program Files\Python36\python.exe"
        expected_path = '"C:\\\\Program Files\\\\Python36\\\\python.exe"'
        assert pipenv.utils.escape_grouped_arguments(test_path) == expected_path

    @pytest.mark.utils
    def test_is_valid_url(self):
        url = "https://github.com/kennethreitz/requests.git"
        not_url = "something_else"
        assert pipenv.utils.is_valid_url(url)
        assert pipenv.utils.is_valid_url(not_url) is False

    @pytest.mark.utils
    def test_download_file(self):
        url = "https://github.com/kennethreitz/pipenv/blob/master/README.md"
        output = "test_download.md"
        pipenv.utils.download_file(url, output)
        assert os.path.exists(output)
        os.remove(output)

    @pytest.mark.utils
    def test_new_line_end_of_toml_file(this):
        # toml file that needs clean up
        toml = """
[dev-packages]

"flake8" = ">=3.3.0,<4"
pytest = "*"
mock = "*"
sphinx = "<=1.5.5"
"-e ." = "*"
twine = "*"
"sphinx-click" = "*"
"pytest-xdist" = "*"
        """
        new_toml = pipenv.utils.cleanup_toml(toml)
        # testing if the end of the generated file contains a newline
        assert new_toml[-1] == "\n"

    @pytest.mark.utils
    @pytest.mark.parametrize(
        "input_path, expected",
        [
            (
                "c:\\Program Files\\Python36\\python.exe",
                "C:\\Program Files\\Python36\\python.exe",
            ),
            (
                "C:\\Program Files\\Python36\\python.exe",
                "C:\\Program Files\\Python36\\python.exe",
            ),
            ("\\\\host\\share\\file.zip", "\\\\host\\share\\file.zip"),
            ("artifacts\\file.zip", "artifacts\\file.zip"),
            (".\\artifacts\\file.zip", ".\\artifacts\\file.zip"),
            ("..\\otherproject\\file.zip", "..\\otherproject\\file.zip"),
        ],
    )
    @pytest.mark.skipif(os.name != "nt", reason="Windows file paths tested")
    def test_win_normalize_drive(self, input_path, expected):
        assert pipenv.utils.normalize_drive(input_path) == expected

    @pytest.mark.utils
    @pytest.mark.parametrize(
        "input_path, expected",
        [
            ("/usr/local/bin/python", "/usr/local/bin/python"),
            ("artifacts/file.zip", "artifacts/file.zip"),
            ("./artifacts/file.zip", "./artifacts/file.zip"),
            ("../otherproject/file.zip", "../otherproject/file.zip"),
        ],
    )
    @pytest.mark.skipif(os.name == "nt", reason="*nix file paths tested")
    def test_nix_normalize_drive(self, input_path, expected):
        assert pipenv.utils.normalize_drive(input_path) == expected

    @pytest.mark.utils
    @pytest.mark.parametrize(
        "sources, expected_args",
        [
            (
                [{"url": "https://test.example.com/simple", "verify_ssl": True}],
                ["-i", "https://test.example.com/simple"],
            ),
            (
                [{"url": "https://test.example.com/simple", "verify_ssl": False}],
                [
                    "-i",
                    "https://test.example.com/simple",
                    "--trusted-host",
                    "test.example.com",
                ],
            ),
            (
                [
                    {"url": "https://pypi.org/simple"},
                    {"url": "https://custom.example.com/simple"},
                ],
                [
                    "-i",
                    "https://pypi.org/simple",
                    "--extra-index-url",
                    "https://custom.example.com/simple",
                ],
            ),
            (
                [
                    {"url": "https://pypi.org/simple"},
                    {"url": "https://custom.example.com/simple", "verify_ssl": False},
                ],
                [
                    "-i",
                    "https://pypi.org/simple",
                    "--extra-index-url",
                    "https://custom.example.com/simple",
                    "--trusted-host",
                    "custom.example.com",
                ],
            ),
            (
                [
                    {"url": "https://pypi.org/simple"},
                    {
                        "url": "https://user:password@custom.example.com/simple",
                        "verify_ssl": False,
                    },
                ],
                [
                    "-i",
                    "https://pypi.org/simple",
                    "--extra-index-url",
                    "https://user:password@custom.example.com/simple",
                    "--trusted-host",
                    "custom.example.com",
                ],
            ),
            (
                [
                    {"url": "https://pypi.org/simple"},
                    {"url": "https://user:password@custom.example.com/simple"},
                ],
                [
                    "-i",
                    "https://pypi.org/simple",
                    "--extra-index-url",
                    "https://user:password@custom.example.com/simple",
                ],
            ),
            (
                [
                    {
                        "url": "https://user:password@custom.example.com/simple",
                        "verify_ssl": False,
                    },
                ],
                [
                    "-i",
                    "https://user:password@custom.example.com/simple",
                    "--trusted-host",
                    "custom.example.com",
                ],
            ),
        ],
    )
    def test_prepare_pip_source_args(self, sources, expected_args):
        assert (
            pipenv.utils.prepare_pip_source_args(sources, pip_args=None)
            == expected_args
        )

    def test_invalid_prepare_pip_source_args(self):
        sources = [{}]
        with pytest.raises(PipenvUsageError):
            pipenv.utils.prepare_pip_source_args(sources, pip_args=None)

    @pytest.mark.utils
    def test_parse_python_version(self):
        ver = pipenv.utils.parse_python_version("Python 3.6.5\n")
        assert ver == {"major": "3", "minor": "6", "micro": "5"}

    @pytest.mark.utils
    def test_parse_python_version_suffix(self):
        ver = pipenv.utils.parse_python_version("Python 3.6.5rc1\n")
        assert ver == {"major": "3", "minor": "6", "micro": "5"}

    @pytest.mark.utils
    def test_parse_python_version_270(self):
        ver = pipenv.utils.parse_python_version("Python 2.7\n")
        assert ver == {"major": "2", "minor": "7", "micro": "0"}

    @pytest.mark.utils
    def test_parse_python_version_270_garbage(self):
        ver = pipenv.utils.parse_python_version("Python 2.7+\n")
        assert ver == {"major": "2", "minor": "7", "micro": "0"}
<commit_msg>add unittests<commit_after># -*- coding: utf-8 -*-
import os

import pytest

from first import first
from mock import Mock, patch

import pipenv.utils
import pythonfinder.utils
from pipenv.exceptions import PipenvUsageError

# Pipfile format <-> requirements.txt format.
DEP_PIP_PAIRS = [
    ({"requests": "*"}, "requests"),
    ({"requests": {"extras": ["socks"], "version": "*"}}, "requests[socks]"),
    ({"django": ">1.10"}, "django>1.10"),
    ({"Django": ">1.10"}, "Django>1.10"),
    ({"requests": {"extras": ["socks"], "version": ">1.10"}}, "requests[socks]>1.10"),
    ({"requests": {"extras": ["socks"], "version": "==1.10"}}, "requests[socks]==1.10"),
    (
        {
            "pinax": {
                "git": "git://github.com/pinax/pinax.git",
                "ref": "1.4",
                "editable": True,
            }
        },
        "-e git+git://github.com/pinax/pinax.git@1.4#egg=pinax",
    ),
    (
        {"pinax": {"git": "git://github.com/pinax/pinax.git", "ref": "1.4"}},
        "git+git://github.com/pinax/pinax.git@1.4#egg=pinax",
    ),
    (  # Mercurial.
        {
            "MyProject": {
                "hg": "http://hg.myproject.org/MyProject",
                "ref": "da39a3ee5e6b",
            }
        },
        "hg+http://hg.myproject.org/MyProject@da39a3ee5e6b#egg=MyProject",
    ),
    (  # SVN.
        {
            "MyProject": {
                "svn": "svn://svn.myproject.org/svn/MyProject",
                "editable": True,
            }
        },
        "-e svn+svn://svn.myproject.org/svn/MyProject#egg=MyProject",
    ),
    (
        # Extras in url
        {
            "discord.py": {
                "file": "https://github.com/Rapptz/discord.py/archive/rewrite.zip",
                "extras": ["voice"],
            }
        },
        "https://github.com/Rapptz/discord.py/archive/rewrite.zip#egg=discord.py[voice]",
    ),
    (
        {
            "requests": {
                "git": "https://github.com/requests/requests.git",
                "ref": "master",
                "extras": ["security"],
                "editable": False,
            }
        },
        "git+https://github.com/requests/requests.git@master#egg=requests[security]",
    ),
]

@pytest.mark.utils
@pytest.mark.parametrize("deps, expected", DEP_PIP_PAIRS)
def test_convert_deps_to_pip(deps, expected):
    if expected.startswith("Django"):
        expected = expected.lower()
    assert pipenv.utils.convert_deps_to_pip(deps, r=False) == [expected]

@pytest.mark.utils
@pytest.mark.parametrize(
    "deps, expected",
    [
        # This one should be collapsed and treated as {'requests': '*'}.
        ({"requests": {}}, "requests"),
        # Hash value should be passed into the result.
        (
            {
                "FooProject": {
                    "version": "==1.2",
                    "hash": "sha256:2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824",
                }
            },
            "FooProject==1.2 --hash=sha256:2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824",
        ),
        (
            {
                "FooProject": {
                    "version": "==1.2",
                    "extras": ["stuff"],
                    "hash": "sha256:2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824",
                }
            },
            "FooProject[stuff]==1.2 --hash=sha256:2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824",
        ),
        (
            {
                "requests": {
                    "git": "https://github.com/requests/requests.git",
                    "ref": "master",
                    "extras": ["security"],
                }
            },
            "git+https://github.com/requests/requests.git@master#egg=requests[security]",
        ),
    ],
)
def test_convert_deps_to_pip_one_way(deps, expected):
    assert pipenv.utils.convert_deps_to_pip(deps, r=False) == [expected.lower()]

@pytest.mark.skipif(isinstance(u"", str), reason="don't need to test if unicode is str")
@pytest.mark.utils
def test_convert_deps_to_pip_unicode():
    deps = {u"django": u"==1.10"}
    deps = pipenv.utils.convert_deps_to_pip(deps, r=False)
    assert deps[0] == "django==1.10"

class TestUtils:
    """Test utility functions in pipenv"""

    @pytest.mark.utils
    @pytest.mark.parametrize(
        "version, specified_ver, expected",
        [
            ("*", "*", True),
            ("2.1.6", "==2.1.4", False),
            ("20160913", ">=20140815", True),
            (
                "1.4",
                {"svn": "svn://svn.myproj.org/svn/MyProj", "version": "==1.4"},
                True,
            ),
            ("2.13.0", {"extras": ["socks"], "version": "==2.12.4"}, False),
        ],
    )
    def test_is_required_version(self, version, specified_ver, expected):
        assert pipenv.utils.is_required_version(version, specified_ver) is expected

    @pytest.mark.utils
    @pytest.mark.parametrize(
        "entry, expected",
        [
            ({"git": "package.git", "ref": "v0.0.1"}, True),
            ({"hg": "https://package.com/package", "ref": "v1.2.3"}, True),
            ("*", False),
            ({"some_value": 5, "other_value": object()}, False),
            ("package", False),
            ("git+https://github.com/requests/requests.git#egg=requests", True),
            ("git+git@github.com:requests/requests.git#egg=requests", True),
            ("gitdb2", False),
        ],
    )
    @pytest.mark.vcs
    def test_is_vcs(self, entry, expected):
        from pipenv.vendor.requirementslib.utils import is_vcs
        assert is_vcs(entry) is expected

    @pytest.mark.utils
    def test_python_version_from_bad_path(self):
        assert pipenv.utils.python_version("/fake/path") is None

    @pytest.mark.utils
    def test_python_version_from_non_python(self):
        assert pipenv.utils.python_version("/dev/null") is None

    @pytest.mark.utils
    @pytest.mark.parametrize(
        "version_output, version",
        [
            ("Python 3.6.2", "3.6.2"),
            ("Python 3.6.2 :: Continuum Analytics, Inc.", "3.6.2"),
            ("Python 3.6.20 :: Continuum Analytics, Inc.", "3.6.20"),
            (
                "Python 3.5.3 (3f6eaa010fce78cc7973bdc1dfdb95970f08fed2, Jan 13 2018, 18:14:01)\n[PyPy 5.10.1 with GCC 4.2.1 Compatible Apple LLVM 9.0.0 (clang-900.0.39.2)]",
                "3.5.3",
            ),
        ],
    )
    # @patch(".vendor.pythonfinder.utils.get_python_version")
    def test_python_version_output_variants(
        self, monkeypatch, version_output, version
    ):
        def mock_version(path):
            return version_output.split()[1]
        monkeypatch.setattr("pipenv.vendor.pythonfinder.utils.get_python_version", mock_version)
        assert pipenv.utils.python_version("some/path") == version

    @pytest.mark.utils
    @pytest.mark.windows
    @pytest.mark.skipif(os.name != "nt", reason="Windows test only")
    def test_windows_shellquote(self):
        test_path = "C:\Program Files\Python36\python.exe"
        expected_path = '"C:\\\\Program Files\\\\Python36\\\\python.exe"'
        assert pipenv.utils.escape_grouped_arguments(test_path) == expected_path

    @pytest.mark.utils
    def test_is_valid_url(self):
        url = "https://github.com/kennethreitz/requests.git"
        not_url = "something_else"
        assert pipenv.utils.is_valid_url(url)
        assert pipenv.utils.is_valid_url(not_url) is False

    @pytest.mark.utils
    def test_download_file(self):
        url = "https://github.com/kennethreitz/pipenv/blob/master/README.md"
        output = "test_download.md"
        pipenv.utils.download_file(url, output)
        assert os.path.exists(output)
        os.remove(output)

    @pytest.mark.utils
    def test_new_line_end_of_toml_file(this):
        # toml file that needs clean up
        toml = """
[dev-packages]

"flake8" = ">=3.3.0,<4"
pytest = "*"
mock = "*"
sphinx = "<=1.5.5"
"-e ." = "*"
twine = "*"
"sphinx-click" = "*"
"pytest-xdist" = "*"
        """
        new_toml = pipenv.utils.cleanup_toml(toml)
        # testing if the end of the generated file contains a newline
        assert new_toml[-1] == "\n"

    @pytest.mark.utils
    @pytest.mark.parametrize(
        "input_path, expected",
        [
            (
                "c:\\Program Files\\Python36\\python.exe",
                "C:\\Program Files\\Python36\\python.exe",
            ),
            (
                "C:\\Program Files\\Python36\\python.exe",
                "C:\\Program Files\\Python36\\python.exe",
            ),
            ("\\\\host\\share\\file.zip", "\\\\host\\share\\file.zip"),
            ("artifacts\\file.zip", "artifacts\\file.zip"),
            (".\\artifacts\\file.zip", ".\\artifacts\\file.zip"),
            ("..\\otherproject\\file.zip", "..\\otherproject\\file.zip"),
        ],
    )
    @pytest.mark.skipif(os.name != "nt", reason="Windows file paths tested")
    def test_win_normalize_drive(self, input_path, expected):
        assert pipenv.utils.normalize_drive(input_path) == expected

    @pytest.mark.utils
    @pytest.mark.parametrize(
        "input_path, expected",
        [
            ("/usr/local/bin/python", "/usr/local/bin/python"),
            ("artifacts/file.zip", "artifacts/file.zip"),
            ("./artifacts/file.zip", "./artifacts/file.zip"),
            ("../otherproject/file.zip", "../otherproject/file.zip"),
        ],
    )
    @pytest.mark.skipif(os.name == "nt", reason="*nix file paths tested")
    def test_nix_normalize_drive(self, input_path, expected):
        assert pipenv.utils.normalize_drive(input_path) == expected

    @pytest.mark.utils
    @pytest.mark.parametrize(
        "sources, expected_args",
        [
            (
                [{"url": "https://test.example.com/simple", "verify_ssl": True}],
                ["-i", "https://test.example.com/simple"],
            ),
            (
                [{"url": "https://test.example.com/simple", "verify_ssl": False}],
                [
                    "-i",
                    "https://test.example.com/simple",
                    "--trusted-host",
                    "test.example.com",
                ],
            ),
            (
                [{"url": "https://test.example.com:12345/simple", "verify_ssl": False}],
                [
                    "-i",
                    "https://test.example.com:12345/simple",
                    "--trusted-host",
                    "test.example.com:12345",
                ],
            ),
            (
                [
                    {"url": "https://pypi.org/simple"},
                    {"url": "https://custom.example.com/simple"},
                ],
                [
                    "-i",
                    "https://pypi.org/simple",
                    "--extra-index-url",
                    "https://custom.example.com/simple",
                ],
            ),
            (
                [
                    {"url": "https://pypi.org/simple"},
                    {"url": "https://custom.example.com/simple", "verify_ssl": False},
                ],
                [
                    "-i",
                    "https://pypi.org/simple",
                    "--extra-index-url",
                    "https://custom.example.com/simple",
                    "--trusted-host",
                    "custom.example.com",
                ],
            ),
            (
                [
                    {"url": "https://pypi.org/simple"},
                    {"url": "https://custom.example.com:12345/simple", "verify_ssl": False},
                ],
                [
                    "-i",
                    "https://pypi.org/simple",
                    "--extra-index-url",
                    "https://custom.example.com:12345/simple",
                    "--trusted-host",
                    "custom.example.com:12345",
                ],
            ),
            (
                [
                    {"url": "https://pypi.org/simple"},
                    {
                        "url": "https://user:password@custom.example.com/simple",
                        "verify_ssl": False,
                    },
                ],
                [
                    "-i",
                    "https://pypi.org/simple",
                    "--extra-index-url",
                    "https://user:password@custom.example.com/simple",
                    "--trusted-host",
                    "custom.example.com",
                ],
            ),
            (
                [
                    {"url": "https://pypi.org/simple"},
                    {"url": "https://user:password@custom.example.com/simple"},
                ],
                [
                    "-i",
                    "https://pypi.org/simple",
                    "--extra-index-url",
                    "https://user:password@custom.example.com/simple",
                ],
            ),
            (
                [
                    {
                        "url": "https://user:password@custom.example.com/simple",
                        "verify_ssl": False,
                    },
                ],
                [
                    "-i",
                    "https://user:password@custom.example.com/simple",
                    "--trusted-host",
                    "custom.example.com",
                ],
            ),
        ],
    )
    def test_prepare_pip_source_args(self, sources, expected_args):
        assert (
            pipenv.utils.prepare_pip_source_args(sources, pip_args=None)
            == expected_args
        )

    def test_invalid_prepare_pip_source_args(self):
        sources = [{}]
        with pytest.raises(PipenvUsageError):
            pipenv.utils.prepare_pip_source_args(sources, pip_args=None)

    @pytest.mark.utils
    def test_parse_python_version(self):
        ver = pipenv.utils.parse_python_version("Python 3.6.5\n")
        assert ver == {"major": "3", "minor": "6", "micro": "5"}

    @pytest.mark.utils
    def test_parse_python_version_suffix(self):
        ver = pipenv.utils.parse_python_version("Python 3.6.5rc1\n")
        assert ver == {"major": "3", "minor": "6", "micro": "5"}

    @pytest.mark.utils
    def test_parse_python_version_270(self):
        ver = pipenv.utils.parse_python_version("Python 2.7\n")
        assert ver == {"major": "2", "minor": "7", "micro": "0"}

    @pytest.mark.utils
    def test_parse_python_version_270_garbage(self):
        ver = pipenv.utils.parse_python_version("Python 2.7+\n")
        assert ver == {"major": "2", "minor": "7", "micro": "0"}
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Generate a sitemap for osf.io"""
import boto3
import datetime
import gzip
import os
import shutil
import sys
import urlparse
import xml

import django
django.setup()
import logging

from framework import sentry
from framework.celery_tasks import app as celery_app
from osf.models import OSFUser, AbstractNode, PreprintService, PreprintProvider
from scripts import utils as script_utils
from website import settings
from website.app import init_app

logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)

class Progress(object):
    def __init__(self, bar_len=50):
        self.bar_len = bar_len

    def start(self, total, prefix):
        self.total = total
        self.count = 0
        self.prefix = prefix

    def increment(self, inc=1):
        self.count += inc
        filled_len = int(round(self.bar_len * self.count / float(self.total)))
        percents = round(100.0 * self.count / float(self.total), 1)
        bar = '=' * filled_len + '-' * (self.bar_len - filled_len)
        sys.stdout.flush()
        sys.stdout.write('{}[{}] {}{} ... {}\r'.format(self.prefix, bar, percents, '%', str(self.total)))

    def stop(self):
        # To preserve line, there is probably a better way to do this
        print('')

class Sitemap(object):
    def __init__(self):
        self.sitemap_count = 0
        self.errors = 0
        self.new_doc()
        self.sitemap_dir = os.path.join(settings.STATIC_FOLDER, 'sitemaps')
        if not os.path.exists(self.sitemap_dir):
            print('Creating sitemap directory at `{}`'.format(self.sitemap_dir))
            os.makedirs(self.sitemap_dir)
        if settings.SITEMAP_TO_S3:
            assert settings.SITEMAP_AWS_BUCKET, 'SITEMAP_AWS_BUCKET must be set for sitemap files to be sent to S3'
            assert settings.AWS_ACCESS_KEY_ID, 'AWS_ACCESS_KEY_ID must be set for sitemap files to be sent to S3'
            assert settings.AWS_SECRET_ACCESS_KEY, 'AWS_SECRET_ACCESS_KEY must be set for sitemap files to be sent to S3'
            self.s3 = boto3.resource(
                's3',
                aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
                aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
                region_name='us-east-1'
            )

    def new_doc(self):
        """Creates new sitemap document and resets the url_count."""
        self.doc = xml.dom.minidom.Document()
        self.urlset = self.doc.createElement('urlset')
        self.urlset.setAttribute('xmlns', 'http://www.sitemaps.org/schemas/sitemap/0.9')
        self.doc.appendChild(self.urlset)
        self.url_count = 0

    def add_tag(self, name, text):
        """Adds a tag to the current url"""
        tag = self.doc.createElement(name)
        self.url.appendChild(tag)
        tag_text = self.doc.createTextNode(text)
        tag.appendChild(tag_text)

    def add_url(self, config):
        """Adds a url to the current urlset"""
        if self.url_count >= settings.SITEMAP_URL_MAX:
            self.write_doc()
            self.new_doc()
        self.url = self.doc.createElement('url')
        self.urlset.appendChild(self.url)

        for k, v in config.iteritems():
            self.add_tag(k, v)
        self.url_count += 1

    def write_doc(self):
        """Writes and gzips each sitemap xml file"""
        file_name = 'sitemap_{}.xml'.format(str(self.sitemap_count))
        file_path = os.path.join(self.sitemap_dir, file_name)
        zip_file_name = file_name + '.gz'
        zip_file_path = file_path + '.gz'
        print('Writing and gzipping `{}`: url_count = {}'.format(file_path, str(self.url_count)))

        xml_str = self.doc.toprettyxml(indent="  ", encoding='utf-8')
        with open(file_path, 'wb') as f:
            f.write(xml_str)

        # Write zipped file
        with open(file_path, 'rb') as f_in, gzip.open(zip_file_path, 'wb') as f_out:
            shutil.copyfileobj(f_in, f_out)
        if settings.SITEMAP_TO_S3:
            self.ship_to_s3(file_name, file_path)
            self.ship_to_s3(zip_file_name, zip_file_path)
        self.sitemap_count += 1

    def ship_to_s3(self, name, path):
        data = open(path, 'rb')
        try:
            self.s3.Bucket(settings.SITEMAP_AWS_BUCKET).put_object(Key='sitemaps/{}'.format(name), Body=data)
        except Exception as e:
            logger.info('Error sending data to s3 via boto3')
            logger.exception(e)
            sentry.log_message('ERROR: Sitemaps could not be uploaded to s3, see `generate_sitemap` logs')
        data.close()

    def write_sitemap_index(self):
        """Writes the index file for all of the sitemap files"""
        doc = xml.dom.minidom.Document()
        sitemap_index = self.doc.createElement('sitemapindex')
        sitemap_index.setAttribute('xmlns', 'http://www.sitemaps.org/schemas/sitemap/0.9')
        doc.appendChild(sitemap_index)

        for f in range(self.sitemap_count):
            sitemap = doc.createElement('sitemap')
            sitemap_index.appendChild(sitemap)

            loc = doc.createElement('loc')
            sitemap.appendChild(loc)
            loc_text = self.doc.createTextNode(urlparse.urljoin(settings.DOMAIN, 'sitemaps/sitemap_{}.xml'.format(str(f))))
            loc.appendChild(loc_text)

            datemod = doc.createElement('lastmod')
            sitemap.appendChild(datemod)
            datemod_text = self.doc.createTextNode(datetime.datetime.now().strftime('%Y-%m-%d'))
            datemod.appendChild(datemod_text)

        print('Writing `sitemap_index.xml`')
        file_name = 'sitemap_index.xml'
        file_path = os.path.join(self.sitemap_dir, file_name)
        xml_str = doc.toprettyxml(indent="  ", encoding='utf-8')
        with open(file_path, 'wb') as f:
            f.write(xml_str)
        if settings.SITEMAP_TO_S3:
            self.ship_to_s3(file_name, file_path)

    def log_errors(self, obj, obj_id, error):
        if not self.errors:
            script_utils.add_file_logger(logger, __file__)
        self.errors += 1
        logger.info('Error on {}, {}:'.format(obj, obj_id))
        logger.exception(error)
        if self.errors == 1000:
            sentry.log_message('ERROR: generate_sitemap stopped execution after reaching 1000 errors. See logs for details.')
            raise Exception('Too many errors generating sitemap.')

    def generate(self):
        print('Generating Sitemap')

        # Progress bar
        progress = Progress()

        # Static urls
        progress.start(len(settings.SITEMAP_STATIC_URLS), 'STAT: ')
        for config in settings.SITEMAP_STATIC_URLS:
            config['loc'] = urlparse.urljoin(settings.DOMAIN, config['loc'])
            self.add_url(config)
            progress.increment()
        progress.stop()

        # User urls
        objs = OSFUser.objects.filter(is_active=True)
        progress.start(objs.count(), 'USER: ')
        for obj in objs.iterator():
            try:
                config = settings.SITEMAP_USER_CONFIG
                config['loc'] = urlparse.urljoin(settings.DOMAIN, obj.url)
                self.add_url(config)
            except Exception as e:
                self.log_errors(obj, obj._id, e)
            progress.increment()
        progress.stop()

        # AbstractNode urls (Nodes and Registrations, no colelctions)
        objs = AbstractNode.objects.filter(is_public=True, is_deleted=False, retraction_id__isnull=True).exclude(type="osf.collection")
        progress.start(objs.count(), 'NODE: ')
        for obj in objs.iterator():
            try:
                config = settings.SITEMAP_NODE_CONFIG
                config['loc'] = urlparse.urljoin(settings.DOMAIN, obj.url)
                config['lastmod'] = obj.date_modified.strftime('%Y-%m-%d')
                self.add_url(config)
            except Exception as e:
                self.log_errors(obj, obj._id, e)
            progress.increment()
        progress.stop()

        # Preprint urls
        objs = PreprintService.objects.filter(node__isnull=False, node__is_deleted=False, node__is_public=True, is_published=True)
        progress.start(objs.count() * 2, 'PREP: ')
        osf = PreprintProvider.load('osf')
        for obj in objs.iterator():
            try:
                preprint_date = obj.date_modified.strftime('%Y-%m-%d')
                config = settings.SITEMAP_PREPRINT_CONFIG
                preprint_url = obj.url
                if obj.provider == osf:
                    preprint_url = '/preprints/{}/'.format(obj._id)
                config['loc'] = urlparse.urljoin(settings.DOMAIN, preprint_url)
                config['lastmod'] = preprint_date
                self.add_url(config)

                # Preprint file urls
                try:
                    file_config = settings.SITEMAP_PREPRINT_FILE_CONFIG
                    file_config['loc'] = urlparse.urljoin(
                        settings.DOMAIN,
                        os.path.join(
                            'project',
                            obj.primary_file.node._id,  # Parent node id
                            'files',
                            'osfstorage',
                            obj.primary_file._id,  # Preprint file deep_url
                            '?action=download'
                        )
                    )
                    file_config['lastmod'] = preprint_date
                    self.add_url(file_config)
                except Exception as e:
                    self.log_errors(obj.primary_file, obj.primary_file._id, e)
            except Exception as e:
                self.log_errors(obj, obj._id, e)
            progress.increment(2)
        progress.stop()

        # Final write
        self.write_doc()
        # Create index file
        self.write_sitemap_index()

        # TODO: once the sitemap is validated add a ping to google with sitemap index file location
        # TODO: server side cursor query wrapper might be useful as the index gets larger.
        # Sitemap indexable limit check
        if self.sitemap_count > settings.SITEMAP_INDEX_MAX * .90: # 10% of urls remaining
            sentry.log_message('WARNING: Max sitemaps nearly reached.')
        print('Total url_count = {}'.format((self.sitemap_count - 1) * settings.SITEMAP_URL_MAX + self.url_count))
        print('Total sitemap_count = {}'.format(str(self.sitemap_count)))
        if self.errors:
            sentry.log_message('WARNING: Generate sitemap encountered errors. See logs for details.')
            print('Total errors = {}'.format(str(self.errors)))
        else:
            print('No errors')

@celery_app.task(name='scripts.generate_sitemap')
def main():
    init_app(routes=False)  # Sets the storage backends on all models
    Sitemap().generate()

if __name__ == '__main__':
    init_app(set_backends=True, routes=False)
    main()
<commit_msg>Optimize sitemap.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Generate a sitemap for osf.io"""
import boto3
import datetime
import gzip
import os
import shutil
import sys
import urlparse
import xml

import django
django.setup()
import logging

from framework import sentry
from framework.celery_tasks import app as celery_app
from osf.models import OSFUser, AbstractNode, PreprintService, PreprintProvider
from scripts import utils as script_utils
from website import settings
from website.app import init_app

logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)

class Progress(object):
    def __init__(self, bar_len=50):
        self.bar_len = bar_len

    def start(self, total, prefix):
        self.total = total
        self.count = 0
        self.prefix = prefix

    def increment(self, inc=1):
        self.count += inc
        filled_len = int(round(self.bar_len * self.count / float(self.total)))
        percents = round(100.0 * self.count / float(self.total), 1)
        bar = '=' * filled_len + '-' * (self.bar_len - filled_len)
        sys.stdout.flush()
        sys.stdout.write('{}[{}] {}{} ... {}\r'.format(self.prefix, bar, percents, '%', str(self.total)))

    def stop(self):
        # To preserve line, there is probably a better way to do this
        print('')

class Sitemap(object):
    def __init__(self):
        self.sitemap_count = 0
        self.errors = 0
        self.new_doc()
        self.sitemap_dir = os.path.join(settings.STATIC_FOLDER, 'sitemaps')
        if not os.path.exists(self.sitemap_dir):
            print('Creating sitemap directory at `{}`'.format(self.sitemap_dir))
            os.makedirs(self.sitemap_dir)
        if settings.SITEMAP_TO_S3:
            assert settings.SITEMAP_AWS_BUCKET, 'SITEMAP_AWS_BUCKET must be set for sitemap files to be sent to S3'
            assert settings.AWS_ACCESS_KEY_ID, 'AWS_ACCESS_KEY_ID must be set for sitemap files to be sent to S3'
            assert settings.AWS_SECRET_ACCESS_KEY, 'AWS_SECRET_ACCESS_KEY must be set for sitemap files to be sent to S3'
            self.s3 = boto3.resource(
                's3',
                aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
                aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
                region_name='us-east-1'
            )

    def new_doc(self):
        """Creates new sitemap document and resets the url_count."""
        self.doc = xml.dom.minidom.Document()
        self.urlset = self.doc.createElement('urlset')
        self.urlset.setAttribute('xmlns', 'http://www.sitemaps.org/schemas/sitemap/0.9')
        self.doc.appendChild(self.urlset)
        self.url_count = 0

    def add_tag(self, name, text):
        """Adds a tag to the current url"""
        tag = self.doc.createElement(name)
        self.url.appendChild(tag)
        tag_text = self.doc.createTextNode(text)
        tag.appendChild(tag_text)

    def add_url(self, config):
        """Adds a url to the current urlset"""
        if self.url_count >= settings.SITEMAP_URL_MAX:
            self.write_doc()
            self.new_doc()
        self.url = self.doc.createElement('url')
        self.urlset.appendChild(self.url)

        for k, v in config.iteritems():
            self.add_tag(k, v)
        self.url_count += 1

    def write_doc(self):
        """Writes and gzips each sitemap xml file"""
        file_name = 'sitemap_{}.xml'.format(str(self.sitemap_count))
        file_path = os.path.join(self.sitemap_dir, file_name)
        zip_file_name = file_name + '.gz'
        zip_file_path = file_path + '.gz'
        print('Writing and gzipping `{}`: url_count = {}'.format(file_path, str(self.url_count)))

        xml_str = self.doc.toprettyxml(indent="  ", encoding='utf-8')
        with open(file_path, 'wb') as f:
            f.write(xml_str)

        # Write zipped file
        with open(file_path, 'rb') as f_in, gzip.open(zip_file_path, 'wb') as f_out:
            shutil.copyfileobj(f_in, f_out)
        if settings.SITEMAP_TO_S3:
            self.ship_to_s3(file_name, file_path)
            self.ship_to_s3(zip_file_name, zip_file_path)
        self.sitemap_count += 1

    def ship_to_s3(self, name, path):
        data = open(path, 'rb')
        try:
            self.s3.Bucket(settings.SITEMAP_AWS_BUCKET).put_object(Key='sitemaps/{}'.format(name), Body=data)
        except Exception as e:
            logger.info('Error sending data to s3 via boto3')
            logger.exception(e)
            sentry.log_message('ERROR: Sitemaps could not be uploaded to s3, see `generate_sitemap` logs')
        data.close()

    def write_sitemap_index(self):
        """Writes the index file for all of the sitemap files"""
        doc = xml.dom.minidom.Document()
        sitemap_index = self.doc.createElement('sitemapindex')
        sitemap_index.setAttribute('xmlns', 'http://www.sitemaps.org/schemas/sitemap/0.9')
        doc.appendChild(sitemap_index)

        for f in range(self.sitemap_count):
            sitemap = doc.createElement('sitemap')
            sitemap_index.appendChild(sitemap)

            loc = doc.createElement('loc')
            sitemap.appendChild(loc)
            loc_text = self.doc.createTextNode(urlparse.urljoin(settings.DOMAIN, 'sitemaps/sitemap_{}.xml'.format(str(f))))
            loc.appendChild(loc_text)

            datemod = doc.createElement('lastmod')
            sitemap.appendChild(datemod)
            datemod_text = self.doc.createTextNode(datetime.datetime.now().strftime('%Y-%m-%d'))
            datemod.appendChild(datemod_text)

        print('Writing `sitemap_index.xml`')
        file_name = 'sitemap_index.xml'
        file_path = os.path.join(self.sitemap_dir, file_name)
        xml_str = doc.toprettyxml(indent="  ", encoding='utf-8')
        with open(file_path, 'wb') as f:
            f.write(xml_str)
        if settings.SITEMAP_TO_S3:
            self.ship_to_s3(file_name, file_path)

    def log_errors(self, obj, obj_id, error):
        if not self.errors:
            script_utils.add_file_logger(logger, __file__)
        self.errors += 1
        logger.info('Error on {}, {}:'.format(obj, obj_id))
        logger.exception(error)
        if self.errors == 1000:
            sentry.log_message('ERROR: generate_sitemap stopped execution after reaching 1000 errors. See logs for details.')
            raise Exception('Too many errors generating sitemap.')

    def generate(self):
        print('Generating Sitemap')

        # Progress bar
        progress = Progress()

        # Static urls
        progress.start(len(settings.SITEMAP_STATIC_URLS), 'STAT: ')
        for config in settings.SITEMAP_STATIC_URLS:
            config['loc'] = urlparse.urljoin(settings.DOMAIN, config['loc'])
            self.add_url(config)
            progress.increment()
        progress.stop()

        # User urls
        objs = OSFUser.objects.filter(is_active=True).values_list('guids___id', flat=True)
        progress.start(objs.count(), 'USER: ')
        for obj in objs:
            try:
                config = settings.SITEMAP_USER_CONFIG
                config['loc'] = urlparse.urljoin(settings.DOMAIN, '/{}/'.format(obj))
                self.add_url(config)
            except Exception as e:
                self.log_errors('USER', obj, e)
            progress.increment()
        progress.stop()

        # AbstractNode urls (Nodes and Registrations, no Collections)
        objs = (AbstractNode.objects
            .filter(is_public=True, is_deleted=False, retraction_id__isnull=True)
            .exclude(type="osf.collection")
            .values('guids___id', 'date_modified'))
        progress.start(objs.count(), 'NODE: ')
        for obj in objs:
            try:
                config = settings.SITEMAP_NODE_CONFIG
                config['loc'] = urlparse.urljoin(settings.DOMAIN, obj['guids___id'])
                config['lastmod'] = obj['date_modified'].strftime('%Y-%m-%d')
                self.add_url(config)
            except Exception as e:
                self.log_errors('NODE', obj['guids___id'], e)
            progress.increment()
        progress.stop()

        # Preprint urls
        objs = (PreprintService.objects
                    .filter(node__isnull=False, node__is_deleted=False, node__is_public=True, is_published=True)
                    .select_related('node', 'provider', 'node__preprint_file')
                    .include('guids', 'node__guids', 'node__preprint_file__guids'))
        progress.start(objs.count() * 2, 'PREP: ')
        osf = PreprintProvider.objects.get(_id='osf')
        for obj in objs:
            try:
                preprint_date = obj.date_modified.strftime('%Y-%m-%d')
                config = settings.SITEMAP_PREPRINT_CONFIG
                preprint_url = obj.url
                if obj.provider == osf:
                    preprint_url = '/preprints/{}/'.format(obj._id)
                config['loc'] = urlparse.urljoin(settings.DOMAIN, preprint_url)
                config['lastmod'] = preprint_date
                self.add_url(config)

                # Preprint file urls
                try:
                    file_config = settings.SITEMAP_PREPRINT_FILE_CONFIG
                    file_config['loc'] = urlparse.urljoin(
                        settings.DOMAIN,
                        os.path.join(
                            'project',
                            obj.node._id,   # Parent node id
                            'files',
                            'osfstorage',
                            obj.primary_file._id,  # Preprint file deep_url
                            '?action=download'
                        )
                    )
                    file_config['lastmod'] = preprint_date
                    self.add_url(file_config)
                except Exception as e:
                    self.log_errors(obj.primary_file, obj.primary_file._id, e)
            except Exception as e:
                self.log_errors(obj, obj._id, e)
            progress.increment(2)
        progress.stop()

        # Final write
        self.write_doc()
        # Create index file
        self.write_sitemap_index()

        # TODO: once the sitemap is validated add a ping to google with sitemap index file location
        # TODO: server side cursor query wrapper might be useful as the index gets larger.
        # Sitemap indexable limit check
        if self.sitemap_count > settings.SITEMAP_INDEX_MAX * .90: # 10% of urls remaining
            sentry.log_message('WARNING: Max sitemaps nearly reached.')
        print('Total url_count = {}'.format((self.sitemap_count - 1) * settings.SITEMAP_URL_MAX + self.url_count))
        print('Total sitemap_count = {}'.format(str(self.sitemap_count)))
        if self.errors:
            sentry.log_message('WARNING: Generate sitemap encountered errors. See logs for details.')
            print('Total errors = {}'.format(str(self.errors)))
        else:
            print('No errors')

@celery_app.task(name='scripts.generate_sitemap')
def main():
    init_app(routes=False)  # Sets the storage backends on all models
    Sitemap().generate()

if __name__ == '__main__':
    init_app(set_backends=True, routes=False)
    main()
<|endoftext|>


<commit_before>import os,os.path,sys,datetime,subprocess,string,urllib,zipfile
from time import strftime

#------------------------
#  2015-02-05  D. O'Hara - requires wget
#  2015-02-06  RSD - requires Python 3.2+, takes command line arguments
#  2015-02-18  RSD - Fix combining CSVs, don't re-download existing files
#  2015-12-10  Christophe Lambert -- converted script to python 2.7, and cleaned up command line arguments.
#------------------------

# This script will download and unzip SynPUF files from CMS.
#
# To run this script, you must have Python 2.7 installed on your system
# From the command line, type:
# python ppath/to/output
#
# This will download SynPUF files and extract them into path/to/output
#
# The SynPUF files are split into 20 sets of files.
#
# For more information about SynPUF see:
# https://www.cms.gov/Research-Statistics-Data-and-Systems/Downloadable-Public-Use-Files/SynPUFs/DE_Syn_PUF.html

# Read output directory from the command line
if len(sys.argv) < 3:
    print("usage: get_synpuf_files.py path/to/output/directory <SAMPLE> ... [SAMPLE]")
    print("where each SAMPLE is a number from 1 to 20, representing the 20 parts of the CMS data")
    quit();

SAMPLE_RANGE = []
for i in range(2,len(sys.argv)):
    try:
        x = int(sys.argv[i])
        if(x <1 or x > 20):
            raise ValueError('Invalid sample number')
        SAMPLE_RANGE.append(x)
    except ValueError:
        print("Invalid sample number: " + sys.argv[i] + ". Must be in range 1..20")
        quit()

OUTPUT_DIRECTORY    = sys.argv[1]
if not os.path.exists(OUTPUT_DIRECTORY): os.makedirs(OUTPUT_DIRECTORY)

#-----------------------------------
#-----------------------------------
def get_timestamp():
    return strftime("%Y-%m-%d %H:%M:%S")

#-----------------------------------
#- download and unzip all the files for a sample
#  combine the 3 beneficiary files into 1 file
#-----------------------------------
def download_synpuf_files(sample_directory, sample_number):
    print('-'*80)
    print(get_timestamp(),' download_synpuf_files starting: sample_number=',sample_number)

    # as of 2015-02-06, files come from different places
    url_www_cms_gov        = 'www.cms.gov/Research-Statistics-Data-and-Systems/Downloadable-Public-Use-Files/SynPUFs/Downloads'
    url_downloads_cms_gov  = 'downloads.cms.gov/files'

    synpuf_files = [
        [ url_www_cms_gov,          'DE1_0_2008_Beneficiary_Summary_File_Sample_~~.zip'         ],
        [ url_downloads_cms_gov,    'DE1_0_2008_to_2010_Carrier_Claims_Sample_~~A.zip'          ],
        [ url_downloads_cms_gov,    'DE1_0_2008_to_2010_Carrier_Claims_Sample_~~B.zip'          ],
        [ url_www_cms_gov,          'DE1_0_2008_to_2010_Inpatient_Claims_Sample_~~.zip'         ],
        [ url_www_cms_gov,          'DE1_0_2008_to_2010_Outpatient_Claims_Sample_~~.zip'        ],
        [ url_downloads_cms_gov,    'DE1_0_2008_to_2010_Prescription_Drug_Events_Sample_~~.zip' ],
        [ url_www_cms_gov,          'DE1_0_2009_Beneficiary_Summary_File_Sample_~~.zip'         ],
        [ url_www_cms_gov,          'DE1_0_2010_Beneficiary_Summary_File_Sample_~~.zip'         ]
    ]

    download_directory = os.path.join(sample_directory,"DE_{0}".format(sample_number))
    if not os.path.exists(download_directory): os.makedirs(download_directory)

    for base_url,sp_file in synpuf_files:
        sp_file = sp_file.replace('~~',str(sample_number))

        # The link on cms.gov website for the following file has .csv.zip in it, so change the variable sp_file.
        # Also, the link for cms.gov has 'https' whereas the link for 'downloads.cms.gov' has 'http', so the
        # file_url has been modified based on the base_url.
        if sp_file == 'DE1_0_2008_to_2010_Carrier_Claims_Sample_11A.zip':           # actual filename on CMS website has csv in it.
            sp_file = 'DE1_0_2008_to_2010_Carrier_Claims_Sample_11A.csv.zip'
        if base_url == url_downloads_cms_gov:                     #base urls have different protocols. one has http while other has https.
            file_url = 'http://{0}/{1}'.format(base_url, sp_file)
        elif base_url == url_www_cms_gov:
            file_url = 'https://{0}/{1}'.format(base_url, sp_file)

        if '.csv.zip' in sp_file:                   #downloaded file name shouldn't have .csv.zip.
            sp_file = sp_file.replace('.csv.zip', '.zip')

        file_local = os.path.join(download_directory,sp_file)
        # If the file already exists, let's not download it again
        # If a file is only partially downloaded, it will need to be deleted
        # before running this script again.
        if os.path.exists(file_local):
            print('..already exists: skipping', file_local)
            continue
        else:
            print('..downloading -> ', file_url)
            urllib.urlretrieve(file_url, filename=file_local)
            zipfile.ZipFile(file_local).extractall(download_directory)
    #---------------------------------------------------------------------------------------
    # some files in the zipped folder have Copy.csv in their names. The following code will
    # read all the files in the download folder and remove Copy from file name.
    #---------------------------------------------------------------------------------------
    for filename in os.listdir(download_directory):
        if ' - Copy.csv' in filename:
            filename1 = filename.replace(' - Copy.csv', '.csv')
            print ('..Renaming file ->', filename)
            o_filepath = os.path.join(download_directory, filename)     # old file path
            n_filepath = os.path.join(download_directory, filename1)    # new file path
            os.rename(o_filepath, n_filepath)   # rename the old file

    #-- combine the beneficiary files
    combine_beneficiary_files(download_directory, sample_number)

    print(get_timestamp(),' Done')

#-----------------------------------
#- combine 3 beneficiary files into 1, with the year prefixed
#-----------------------------------
def combine_beneficiary_files(output_directory, sample_number):
    print('-'*80)
    print(get_timestamp(),' combine_beneficiary_files starting: sample_number=',sample_number)

    output_bene_filename = os.path.join(output_directory ,
                        'DE1_0_comb_Beneficiary_Summary_File_Sample_{0}.csv'.format(sample_number))

    print('Writing to ->',output_bene_filename)
    total_recs_in=0
    total_recs_out=0

    with open(output_bene_filename, 'w') as f_out:
        for year in ['2008','2009','2010']:
            input_bene_filename = os.path.join(output_directory,
                            'DE1_0_{0}_Beneficiary_Summary_File_Sample_{1}.csv'.format(year,sample_number))
            print('Reading    ->',input_bene_filename)
            recs_in=0
            with open(input_bene_filename, 'r') as f_in:
                for line in f_in:
                    tyear = year
                    recs_in+=1
                    # We need to use the header line from the first
                    # file we encounter to serve as the header line for the
                    # combined file, but skip all other header lines in the
                    # remaining files
                    if recs_in == 1:
                        if total_recs_out == 0:
                            tyear = '"YEAR"'
                        else:
                            continue
                    if recs_in % 25000 == 0: print('Year-{0}: records read ={1}, total written={2}'.format(year,recs_in, total_recs_out))
                    f_out.write(tyear + ',' + line)
                    total_recs_out+=1
            print('Year-{0}: total records read ={1}'.format(year,recs_in))
            total_recs_in+=recs_in

    print(get_timestamp(),' Done: total records read ={0}, total records written={1}'.format(total_recs_in, total_recs_out))

#-----------------------------------
#-----------------------------------
if __name__ == '__main__':

    print(get_timestamp(),' Combine Beneficiary Year files...starting')

    print('OUTPUT_DIRECTORY         =', OUTPUT_DIRECTORY)
    print('SAMPLE_RANGE             =', SAMPLE_RANGE)

    #------
    # download from CMS
    #------
    for sample_number in SAMPLE_RANGE:
        download_synpuf_files(OUTPUT_DIRECTORY, sample_number)

    print(get_timestamp(),' Done')
<commit_msg>Python3 compatibility and "all" download option (#55)<commit_after>import os,os.path,sys,datetime,subprocess,string,urllib,zipfile
from time import strftime

if sys.version_info[0] >= 3:
    import urllib.request as ur

#------------------------
#  2015-02-05  D. O'Hara - requires wget
#  2015-02-06  RSD - requires Python 3.2+, takes command line arguments
#  2015-02-18  RSD - Fix combining CSVs, don't re-download existing files
#  2015-12-10  Christophe Lambert -- converted script to python 2.7, and cleaned up command line arguments.
#  2018-05-16  Mustafa Ascha - Added python3 compatibility for urllib import, and an 'all' option to download all files
#------------------------

# This script will download and unzip SynPUF files from CMS.
#
# To run this script, you must have Python 2.7 installed on your system
# From the command line, type:
# python ppath/to/output
#
# This will download SynPUF files and extract them into path/to/output
#
# The SynPUF files are split into 20 sets of files.
#
# For more information about SynPUF see:
# https://www.cms.gov/Research-Statistics-Data-and-Systems/Downloadable-Public-Use-Files/SynPUFs/DE_Syn_PUF.html

# Read output directory from the command line
if len(sys.argv) < 3 and not 'all' in sys.argv:
    print("usage: get_synpuf_files.py path/to/output/directory <SAMPLE> ... [SAMPLE]")
    print("where each SAMPLE is a number from 1 to 20, representing the 20 parts of the CMS data")
    quit();

SAMPLE_RANGE = []
for i in range(2,len(sys.argv)):
    try:
        x = int(sys.argv[i])
        if(x <1 or x > 20):
            raise ValueError('Invalid sample number')
        SAMPLE_RANGE.append(x)
    except ValueError:
        if 'all' in sys.argv[i]:
            for number in [x + 1 for x in range(20)]:
                SAMPLE_RANGE.append(number)
            break
        print("Invalid sample number: " + sys.argv[i] + ". Must be 'all' or in range 1..20")
        quit()

OUTPUT_DIRECTORY    = sys.argv[1]
if not os.path.exists(OUTPUT_DIRECTORY): os.makedirs(OUTPUT_DIRECTORY)

#-----------------------------------
#-----------------------------------
def get_timestamp():
    return strftime("%Y-%m-%d %H:%M:%S")

#-----------------------------------
#- download and unzip all the files for a sample
#  combine the 3 beneficiary files into 1 file
#-----------------------------------
def download_synpuf_files(sample_directory, sample_number):
    print('-'*80)
    print(get_timestamp(),' download_synpuf_files starting: sample_number=',sample_number)

    # as of 2015-02-06, files come from different places
    url_www_cms_gov        = 'www.cms.gov/Research-Statistics-Data-and-Systems/Downloadable-Public-Use-Files/SynPUFs/Downloads'
    url_downloads_cms_gov  = 'downloads.cms.gov/files'

    synpuf_files = [
        [ url_www_cms_gov,          'DE1_0_2008_Beneficiary_Summary_File_Sample_~~.zip'         ],
        [ url_downloads_cms_gov,    'DE1_0_2008_to_2010_Carrier_Claims_Sample_~~A.zip'          ],
        [ url_downloads_cms_gov,    'DE1_0_2008_to_2010_Carrier_Claims_Sample_~~B.zip'          ],
        [ url_www_cms_gov,          'DE1_0_2008_to_2010_Inpatient_Claims_Sample_~~.zip'         ],
        [ url_www_cms_gov,          'DE1_0_2008_to_2010_Outpatient_Claims_Sample_~~.zip'        ],
        [ url_downloads_cms_gov,    'DE1_0_2008_to_2010_Prescription_Drug_Events_Sample_~~.zip' ],
        [ url_www_cms_gov,          'DE1_0_2009_Beneficiary_Summary_File_Sample_~~.zip'         ],
        [ url_www_cms_gov,          'DE1_0_2010_Beneficiary_Summary_File_Sample_~~.zip'         ]
    ]

    download_directory = os.path.join(sample_directory,"DE_{0}".format(sample_number))
    if not os.path.exists(download_directory): os.makedirs(download_directory)

    for base_url,sp_file in synpuf_files:
        sp_file = sp_file.replace('~~',str(sample_number))

        # The link on cms.gov website for the following file has .csv.zip in it, so change the variable sp_file.
        # Also, the link for cms.gov has 'https' whereas the link for 'downloads.cms.gov' has 'http', so the
        # file_url has been modified based on the base_url.
        if sp_file == 'DE1_0_2008_to_2010_Carrier_Claims_Sample_11A.zip':           # actual filename on CMS website has csv in it.
            sp_file = 'DE1_0_2008_to_2010_Carrier_Claims_Sample_11A.csv.zip'
        if base_url == url_downloads_cms_gov:                     #base urls have different protocols. one has http while other has https.
            file_url = 'http://{0}/{1}'.format(base_url, sp_file)
        elif base_url == url_www_cms_gov:
            file_url = 'https://{0}/{1}'.format(base_url, sp_file)

        if '.csv.zip' in sp_file:                   #downloaded file name shouldn't have .csv.zip.
            sp_file = sp_file.replace('.csv.zip', '.zip')

        file_local = os.path.join(download_directory,sp_file)
        # If the file already exists, let's not download it again
        # If a file is only partially downloaded, it will need to be deleted
        # before running this script again.
        if os.path.exists(file_local):
            print('..already exists: skipping', file_local)
            continue
        else:
            print('..downloading -> ', file_url)
            if sys.version_info[0] >= 3:
                ur.urlretrieve(file_url, filename=file_local)
            else:
                urllib.urlretrieve(file_url, filename=file_local)
            zipfile.ZipFile(file_local).extractall(download_directory)
    #---------------------------------------------------------------------------------------
    # some files in the zipped folder have Copy.csv in their names. The following code will
    # read all the files in the download folder and remove Copy from file name.
    #---------------------------------------------------------------------------------------
    for filename in os.listdir(download_directory):
        if ' - Copy.csv' in filename:
            filename1 = filename.replace(' - Copy.csv', '.csv')
            print ('..Renaming file ->', filename)
            o_filepath = os.path.join(download_directory, filename)     # old file path
            n_filepath = os.path.join(download_directory, filename1)    # new file path
            os.rename(o_filepath, n_filepath)   # rename the old file

    #-- combine the beneficiary files
    combine_beneficiary_files(download_directory, sample_number)

    print(get_timestamp(),' Done')

#-----------------------------------
#- combine 3 beneficiary files into 1, with the year prefixed
#-----------------------------------
def combine_beneficiary_files(output_directory, sample_number):
    print('-'*80)
    print(get_timestamp(),' combine_beneficiary_files starting: sample_number=',sample_number)

    output_bene_filename = os.path.join(output_directory ,
                        'DE1_0_comb_Beneficiary_Summary_File_Sample_{0}.csv'.format(sample_number))

    print('Writing to ->',output_bene_filename)
    total_recs_in=0
    total_recs_out=0

    with open(output_bene_filename, 'w') as f_out:
        for year in ['2008','2009','2010']:
            input_bene_filename = os.path.join(output_directory,
                            'DE1_0_{0}_Beneficiary_Summary_File_Sample_{1}.csv'.format(year,sample_number))
            print('Reading    ->',input_bene_filename)
            recs_in=0
            with open(input_bene_filename, 'r') as f_in:
                for line in f_in:
                    tyear = year
                    recs_in+=1
                    # We need to use the header line from the first
                    # file we encounter to serve as the header line for the
                    # combined file, but skip all other header lines in the
                    # remaining files
                    if recs_in == 1:
                        if total_recs_out == 0:
                            tyear = '"YEAR"'
                        else:
                            continue
                    if recs_in % 25000 == 0: print('Year-{0}: records read ={1}, total written={2}'.format(year,recs_in, total_recs_out))
                    f_out.write(tyear + ',' + line)
                    total_recs_out+=1
            print('Year-{0}: total records read ={1}'.format(year,recs_in))
            total_recs_in+=recs_in

    print(get_timestamp(),' Done: total records read ={0}, total records written={1}'.format(total_recs_in, total_recs_out))

#-----------------------------------
#-----------------------------------
if __name__ == '__main__':

    print(get_timestamp(),' Combine Beneficiary Year files...starting')

    print('OUTPUT_DIRECTORY         =', OUTPUT_DIRECTORY)
    print('SAMPLE_RANGE             =', SAMPLE_RANGE)

    #------
    # download from CMS
    #------
    for sample_number in SAMPLE_RANGE:
        download_synpuf_files(OUTPUT_DIRECTORY, sample_number)

    print(get_timestamp(),' Done')
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import argparse
import gym

import babyai.utils as utils

# Parse arguments

parser = argparse.ArgumentParser()
parser.add_argument("--env", required=True,
                    help="name of the environment to be run (REQUIRED)")
parser.add_argument("--model", required=True,
                    help="name of the trained model (REQUIRED)")
parser.add_argument("--episodes", type=int, default=1000,
                    help="number of episodes to generate demonstrations for (default: 1000)")
parser.add_argument("--seed", type=int, default=1,
                    help="random seed (default: 1)")
parser.add_argument("--argmax", action="store_true", default=False,
                    help="action with highest probability is selected")
parser.add_argument("--save-interval", type=int, default=0,
                    help="interval between demonstrations saving (default: 0, 0 means only at the end)")
parser.add_argument("--filter-steps", type=int, default=0,
                    help="filter out demos with number of steps more than filter-steps (default: 0, No filtering)")
parser.add_argument("--valid", action="store_true", default=False,
                    help="generating demonstrations for validation set")

args = parser.parse_args()

# Set seed for all randomness sources

utils.seed(args.seed)

# Generate environment
env = gym.make(args.env)
env.seed(args.seed)

# Select Origin
origin = "agent" if not args.valid else "agent_valid"
assert not args.valid or args.seed == 0

# Define agent

agent = utils.load_agent(args, env)

# Load demonstrations
demos = utils.load_demos(args.env, origin, raise_not_found=False)
if demos is not None:
    utils.synthesize_demos(demos)
else:
    demos = []

done = False
obs = env.reset()

offset = 0

while True:
    # Run the expert for one episode

    done = False
    obs = env.reset()
    demo = []

    while not done:
        action = agent.get_action(obs)
        new_obs, reward, done, _ = env.step(action)
        agent.analyze_feedback(reward, done)

        demo.append((obs, action, reward, done))
        obs = new_obs
    if args.filter_steps is not 0:
        if len(demo) <= args.filter_steps and reward != 0:
            demos.append((demo, offset))
    if len(demos) >= args.episodes:
        break

    # Save demonstrations

    if args.save_interval > 0 and len(demos) < args.episodes and len(demos) % args.save_interval == 0:
        utils.save_demos(demos, args.env, origin)
        utils.synthesize_demos(demos)
    offset += 1

# Save demonstrations
utils.save_demos(demos, args.env, origin)
utils.synthesize_demos(demos)
<commit_msg>fix filter_steps=0 in make_agent_demos.py (#128)<commit_after>#!/usr/bin/env python3

import argparse
import gym

import babyai.utils as utils

# Parse arguments

parser = argparse.ArgumentParser()
parser.add_argument("--env", required=True,
                    help="name of the environment to be run (REQUIRED)")
parser.add_argument("--model", required=True,
                    help="name of the trained model (REQUIRED)")
parser.add_argument("--episodes", type=int, default=1000,
                    help="number of episodes to generate demonstrations for (default: 1000)")
parser.add_argument("--seed", type=int, default=1,
                    help="random seed (default: 1)")
parser.add_argument("--argmax", action="store_true", default=False,
                    help="action with highest probability is selected")
parser.add_argument("--save-interval", type=int, default=0,
                    help="interval between demonstrations saving (default: 0, 0 means only at the end)")
parser.add_argument("--filter-steps", type=int, default=0,
                    help="filter out demos with number of steps more than filter-steps (default: 0, No filtering)")
parser.add_argument("--valid", action="store_true", default=False,
                    help="generating demonstrations for validation set")

args = parser.parse_args()

# Set seed for all randomness sources

utils.seed(args.seed)

# Generate environment
env = gym.make(args.env)
env.seed(args.seed)

# Select Origin
origin = "agent" if not args.valid else "agent_valid"
assert not args.valid or args.seed == 0

# Define agent

agent = utils.load_agent(args, env)

# Load demonstrations
demos = utils.load_demos(args.env, origin, raise_not_found=False)
if demos is not None:
    utils.synthesize_demos(demos)
else:
    demos = []

offset = 0

while True:
    # Run the expert for one episode

    done = False
    obs = env.reset()
    demo = []

    while not done:
        action = agent.get_action(obs)
        new_obs, reward, done, _ = env.step(action)
        agent.analyze_feedback(reward, done)

        demo.append((obs, action, reward, done))
        obs = new_obs
    if reward > 0 and (args.filter_steps == 0 or len(demo) <= args.filter_steps):
            demos.append((demo, offset))
    if len(demos) >= args.episodes:
        break

    # Save demonstrations

    if args.save_interval > 0 and len(demos) < args.episodes and len(demos) % args.save_interval == 0:
        utils.save_demos(demos, args.env, origin)
        utils.synthesize_demos(demos)
    offset += 1

# Save demonstrations
utils.save_demos(demos, args.env, origin)
utils.synthesize_demos(demos)
<|endoftext|>


<commit_before>from ggame import App, RectangleAsset, ImageAsset, SoundAsset, CircleAsset
from ggame import LineStyle, Color, Sprite, Sound
from random import randint

SCREEN_WIDTH = 800
SCREEN_HEIGHT = 600
up=5
speed=3
blue=Color(0x87cefa, 1)
purple=Color(0x7b68ee, 1)
line=LineStyle(0,blue)
black = Color(0, 1)
bg_asset = RectangleAsset(SCREEN_WIDTH, SCREEN_HEIGHT, line, black)
bg = Sprite(bg_asset, (0,0))
rond=lambda x: 2*(round(x/2,-1))
length=1
x=20
y=20
z=4
snk=[(20,20)]
go= False
dir=0

class tail(Sprite):
    asset=RectangleAsset(20,20,line, purple)
    def __init__(self, position):
        super().__init__(tail.asset, position)
class ntail(Sprite):
    asset=RectangleAsset(20,20,line, black)
    def __init__(self, position):
        super().__init__(ntail.asset, position)
class dots(Sprite):
    asset=RectangleAsset(20,20,line, blue)
    def __init__(self, position):
        super().__init__(dots.asset, position)

dot=[(20*randint(0,39), 20*randint(0,29))]
for (h,k) in dot:
    dots((h,k))

tail((x,y))

def leftKey(event):
    global dir, go
    dir=0
    go=True
def rightKey(event):
    global dir, go
    dir=1
    go=True
def downKey(event):
    global dir, go
    dir=2
    go=True
def upKey(event):
    global dir, go
    dir=3
    go=True
def spaceKey(event):
    global go
    go=False

if x<SCREEN_WIDTH and x>=0 and y<SCREEN_HEIGHT and y>0:
    go=False
def step():
    global x,y,go,dir,z, dot, snk
    if go:
        z=z+1
        if z==5:
            if dir==0:
                x=x-20
                for (h,k) in dot:
                    for (c,d) in snk:
                        if x==c and y==d:
                            print("you lose")
                            ntail(snk[0])
                            x=5000000000
                    if x==h and y==k:
                        snk.append((x,y))
                        tail((x,y))
                        dot[0]=(20*randint(0,39), 20*randint(0,29))
                        for (z,q) in dot:
                            dots((z,q))
                    elif (x+20)>SCREEN_WIDTH or (y+20)>SCREEN_HEIGHT or x<0 or y<0:
                        print("you lose")
                        go=0
                        ntail(snk[0])
                        x=5000000000
                    else:
                        ntail(snk[0])
                        tail((x,y))
                        snk.append((x,y))
                        snk.remove(snk[0])
                        for (z,q) in dot:
                            dots((z,q))
            if dir ==1:
                x=x+20
                for (h,k) in dot:
                    for (c,d) in snk:
                        if x==c and y==d:
                            print("you lose")
                            ntail(snk[0])
                            x=5000000000
                    if x==h and y==k:
                        snk.append((x,y))
                        tail((x,y))
                        dot[0]=(20*randint(0,39), 20*randint(0,29))
                        for (z,q) in dot:
                            dots((z,q))
                    elif (x+20)>SCREEN_WIDTH or (y+20)>SCREEN_HEIGHT or x<0 or y<0:
                        print("you lose")
                        go=0
                        ntail(snk[0])
                        x=5000000000
                    else:
                        ntail(snk[0])
                        tail((x,y))
                        snk.append((x,y))
                        snk.remove(snk[0])
                        for (z,q) in dot:
                            dots((z,q))

            if dir ==2:
                y=y+20
                for (h,k) in dot:
                    for (c,d) in snk:
                        if x==c and y==d:
                            print("you lose")
                            ntail(snk[0])
                            x=5000000000
                    if x==h and y==k:
                        snk.append((x,y))
                        tail((x,y))
                        dot[0]=(20*randint(0,39), 20*randint(0,29))
                        for (z,q) in dot:
                            dots((z,q))
                    elif (x+20)>SCREEN_WIDTH or (y+20)>SCREEN_HEIGHT or x<0 or y<0:
                        print("you lose")
                        ntail(snk[0])
                        x=5000000000
                    else:
                        ntail(snk[0])
                        tail((x,y))
                        snk.append((x,y))
                        snk.remove(snk[0])
                        for (z,q) in dot:
                            dots((z,q))
            if dir ==3:
                y=y-20
                for (h,k) in dot:
                    for (c,d) in snk:
                        if x==c and y==d:
                            print("you lose")
                            ntail(snk[0])
                            x=5000000000
                    if x==h and y==k:
                        snk.append((x,y))
                        tail((x,y))
                        dot[0]=(20*randint(0,39), 20*randint(0,29))
                        for (z,q) in dot:
                            dots((z,q))
                    elif (x+20)>SCREEN_WIDTH or (y+20)>SCREEN_HEIGHT or x<0 or y<0:
                        print("you lose")
                        ntail(snk[0])
                        x=5000000000
                    else:
                        ntail(snk[0])
                        tail((x,y))
                        snk.append((x,y))
                        snk.remove(snk[0])
                        for (z,q) in dot:
                            dots((z,q))
            z=0

myapp = App(SCREEN_WIDTH, SCREEN_HEIGHT)
myapp.run(step)
myapp.listenKeyEvent('keydown', 'j', leftKey)
myapp.listenKeyEvent('keydown', 'i', upKey)
myapp.listenKeyEvent('keydown', 'k', downKey)
myapp.listenKeyEvent('keydown', 'l', rightKey)
myapp.listenKeyEvent('keyup', 'space', spaceKey)<commit_msg>Updated from Brython Server: 5/10/2017 1:29:14 PM<commit_after>from ggame import App, RectangleAsset, ImageAsset, SoundAsset, CircleAsset
from ggame import LineStyle, Color, Sprite, Sound
from random import randint

SCREEN_WIDTH = 800
SCREEN_HEIGHT = 600
up=5
speed=3
blue=Color(0x87cefa, 1)
purple=Color(0x7b68ee, 1)
line=LineStyle(0,blue)
black = Color(0, 1)
bg_asset = RectangleAsset(SCREEN_WIDTH, SCREEN_HEIGHT, line, black)
bg = Sprite(bg_asset, (0,0))
rond=lambda x: 2*(round(x/2,-1))
length=1
x=20
y=20
z=3
snk=[(20,20)]
go= False
dir=0

class tail(Sprite):
    asset=RectangleAsset(20,20,line, purple)
    def __init__(self, position):
        super().__init__(tail.asset, position)
class ntail(Sprite):
    asset=RectangleAsset(20,20,line, black)
    def __init__(self, position):
        super().__init__(ntail.asset, position)
class dots(Sprite):
    asset=RectangleAsset(20,20,line, blue)
    def __init__(self, position):
        super().__init__(dots.asset, position)

dot=[(20*randint(0,39), 20*randint(0,29))]
for (h,k) in dot:
    dots((h,k))

tail((x,y))

def leftKey(event):
    global dir, go
    dir=0
    go=True
def rightKey(event):
    global dir, go
    dir=1
    go=True
def downKey(event):
    global dir, go
    dir=2
    go=True
def upKey(event):
    global dir, go
    dir=3
    go=True
def spaceKey(event):
    global go
    if go:
        go=False

if x<SCREEN_WIDTH and x>=0 and y<SCREEN_HEIGHT and y>0:
    go=False
def step():
    global x,y,go,dir,z, dot, snk
    if go:
        z=z+1
        if z==4:
            if dir==0:
                x=x-20
                for (h,k) in dot:
                    for (c,d) in snk:
                        if x==c and y==d:
                            print("you lose")
                            ntail(snk[0])
                            x=5000000000
                            go=False
                    if x==h and y==k:
                        snk.append((x,y))
                        tail((x,y))
                        dot[0]=(20*randint(0,39), 20*randint(0,29))
                        for (z,q) in dot:
                            dots((z,q))
                    elif (x+20)>SCREEN_WIDTH or (y+20)>SCREEN_HEIGHT or x<0 or y<0:
                        print("you lose")
                        go=0
                        ntail(snk[0])
                        x=5000000000
                        go=False
                    else:
                        ntail(snk[0])
                        tail((x,y))
                        snk.append((x,y))
                        snk.remove(snk[0])
                        for (z,q) in dot:
                            dots((z,q))
            if dir ==1:
                x=x+20
                for (h,k) in dot:
                    for (c,d) in snk:
                        if x==c and y==d:
                            print("you lose")
                            ntail(snk[0])
                            x=5000000000
                            go=False
                    if x==h and y==k:
                        snk.append((x,y))
                        tail((x,y))
                        dot[0]=(20*randint(0,39), 20*randint(0,29))
                        for (z,q) in dot:
                            dots((z,q))
                    elif (x+20)>SCREEN_WIDTH or (y+20)>SCREEN_HEIGHT or x<0 or y<0:
                        print("you lose")
                        go=0
                        ntail(snk[0])
                        x=5000000000
                        go=False
                    else:
                        ntail(snk[0])
                        tail((x,y))
                        snk.append((x,y))
                        snk.remove(snk[0])
                        for (z,q) in dot:
                            dots((z,q))

            if dir ==2:
                y=y+20
                for (h,k) in dot:
                    for (c,d) in snk:
                        if x==c and y==d:
                            print("you lose")
                            ntail(snk[0])
                            x=5000000000
                            go=False
                    if x==h and y==k:
                        snk.append((x,y))
                        tail((x,y))
                        dot[0]=(20*randint(0,39), 20*randint(0,29))
                        for (z,q) in dot:
                            dots((z,q))
                    elif (x+20)>SCREEN_WIDTH or (y+20)>SCREEN_HEIGHT or x<0 or y<0:
                        print("you lose")
                        ntail(snk[0])
                        x=5000000000
                        go=False
                    else:
                        ntail(snk[0])
                        tail((x,y))
                        snk.append((x,y))
                        snk.remove(snk[0])
                        for (z,q) in dot:
                            dots((z,q))
            if dir ==3:
                y=y-20
                for (h,k) in dot:
                    for (c,d) in snk:
                        if x==c and y==d:
                            print("you lose")
                            ntail(snk[0])
                            x=5000000000
                            go=False
                    if x==h and y==k:
                        snk.append((x,y))
                        tail((x,y))
                        dot[0]=(20*randint(0,39), 20*randint(0,29))
                        for (z,q) in dot:
                            dots((z,q))
                    elif (x+20)>SCREEN_WIDTH or (y+20)>SCREEN_HEIGHT or x<0 or y<0:
                        print("you lose")
                        ntail(snk[0])
                        x=5000000000
                        go=False
                    else:
                        ntail(snk[0])
                        tail((x,y))
                        snk.append((x,y))
                        snk.remove(snk[0])
                        for (z,q) in dot:
                            dots((z,q))
            z=0

myapp = App(SCREEN_WIDTH, SCREEN_HEIGHT)
myapp.run(step)
myapp.listenKeyEvent('keydown', 'j', leftKey)
myapp.listenKeyEvent('keydown', 'i', upKey)
myapp.listenKeyEvent('keydown', 'k', downKey)
myapp.listenKeyEvent('keydown', 'l', rightKey)
myapp.listenKeyEvent('keyup', 'space', spaceKey)<|endoftext|>


<commit_before>"""
Django settings for ThetaTauMiami project.

For more information on this file, see
https://docs.djangoproject.com/en/1.6/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.6/ref/settings/
"""

import dj_database_url

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
BASE_DIR = os.path.dirname(os.path.dirname(__file__))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.6/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'q=asy$r^!q72tp@x2f6kxg83d)+@mg59u0v8#pby^bv(=n4__0'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = False

TEMPLATE_DEBUG = False

ALLOWED_HOSTS = ['localhost', 'aqueous-ravine-6084.herokuapp.com']

TEMPLATE_DIRS = (
    os.path.join(BASE_DIR, 'templates/'),
)

# Application definition

INSTALLED_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'marketing',
    'info',
)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'ThetaTauMiami.urls'

WSGI_APPLICATION = 'ThetaTauMiami.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.6/ref/settings/#databases

DATABASES = dict()

# Production setting
DATABASES['default'] = dj_database_url.config()

# Local setting
#DATABASES['default'] = { "ENGINE" : 'django.db.backends.sqlite', "NAME": "db.sqlite3" }

SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')

# Internationalization
# https://docs.djangoproject.com/en/1.6/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.6/howto/static-files/

STATIC_URL = '/static/'

STATICFILES_DIRS = (
    os.path.join(BASE_DIR, 'static/'),
)
<commit_msg>more stuff<commit_after>"""
Django settings for ThetaTauMiami project.

For more information on this file, see
https://docs.djangoproject.com/en/1.6/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.6/ref/settings/
"""

import dj_database_url

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
BASE_DIR = os.path.dirname(os.path.dirname(__file__))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.6/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'q=asy$r^!q72tp@x2f6kxg83d)+@mg59u0v8#pby^bv(=n4__0'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

TEMPLATE_DEBUG = True

ALLOWED_HOSTS = ['localhost', 'aqueous-ravine-6084.herokuapp.com']

TEMPLATE_DIRS = (
    os.path.join(BASE_DIR, 'templates/'),
)

# Application definition

INSTALLED_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'marketing',
    'info',
)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'ThetaTauMiami.urls'

WSGI_APPLICATION = 'ThetaTauMiami.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.6/ref/settings/#databases

DATABASES = dict()

# Production setting
DATABASES['default'] = dj_database_url.config()

# Local setting
#DATABASES['default'] = { "ENGINE" : 'django.db.backends.sqlite', "NAME": "db.sqlite3" }

SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')

# Internationalization
# https://docs.djangoproject.com/en/1.6/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.6/howto/static-files/

STATIC_URL = '/static/'

STATICFILES_DIRS = (
    os.path.join(BASE_DIR, 'static/'),
)
<|endoftext|>


<commit_before>import os
import time
import RPi.GPIO as GPIO
import csh_ldap as ldap
import colorhandler
import login

def main():
    global cshldap
    cshldap  = ldap.CSHLDAP(login.ldap_user, login.ldap_pass)
    get_ibutton()

def find_user(varID):
    ibutton = varID.strip()
    ibutton = "*" + ibutton[3:] + "01"

    try:
        member = cshldap.get_member_ibutton(ibutton)
        return member.homeDirectory
    except Exception as e:
        print(e)
        return None

def get_ibutton():
    base_dir = '/sys/devices/w1_bus_master1/w1_master_slaves'
    delete_dir = '/sys/devices/w1_bus_master1/w1_master_remove'
    startTime = time.time()

    while True:
        data = open(base_dir, "r")
        ibutton = data.read()
        ibutton = ibutton.strip()
        data.close()
        time.sleep(1)
        print(ibutton)
        if ibutton != "not found.":
            try:
                find_colors(find_user(ibutton))
            except Exception as e:
                print(e)

        d = open(delete_dir, "w")
        d.write(ibutton)

    d.close()

def find_colors(user_dir):
    if user_dir is not None:
        cfile = os.path.join(user_dir, ".colors")
        colors =  os.popen("ssh -i /home/sgreen/.ssh/id_rsa "
             + login.file_server + " cat " + cfile).read()
        print(colors)
        try:
            colorhandler.setColor(colors)
        except:
            print(error)

if __name__ == "__main__":
    main()
<commit_msg>debugging<commit_after>import os
import time
import RPi.GPIO as GPIO
import csh_ldap as ldap
import login

def main():
    global cshldap
    cshldap  = ldap.CSHLDAP(login.ldap_user, login.ldap_pass)
    get_ibutton()

def find_user(varID):
    ibutton = varID.strip()
    ibutton = "*" + ibutton[3:] + "01"

    try:
        member = cshldap.get_member_ibutton(ibutton)
        return member.homeDirectory
    except Exception as e:
        print(e)
        return None

def get_ibutton():
    base_dir = '/sys/devices/w1_bus_master1/w1_master_slaves'
    delete_dir = '/sys/devices/w1_bus_master1/w1_master_remove'
    startTime = time.time()

    while True:
        data = open(base_dir, "r")
        ibutton = data.read()
        ibutton = ibutton.strip()
        data.close()
        time.sleep(1)
        print(ibutton)
        if ibutton != "not found.":
            try:
                find_colors(find_user(ibutton))
            except Exception as e:
                print(e)

        d = open(delete_dir, "w")
        d.write(ibutton)

    d.close()

def find_colors(user_dir):
    if user_dir is not None:
        cfile = os.path.join(user_dir, ".colors")
        colors =  os.popen("ssh -i /home/sgreen/.ssh/id_rsa "
             + login.file_server + " cat " + cfile).read().split()
        for color in colors:
            print(color)
            os.system("curl -X POST -d 'color=" + color +
             "' localhost:80/set")
            time.sleep(2)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>def getVIPfee(month, level, typeid):
	#typeid:
	#1, 200Units
	#2, 500Units
	#3, Unlimited Units
	#month:

	fee = 0.1
	return fee

def getDeliveryfee(cmid):
	detail = u"同仓物流费用 6元"
	return (0.1, detail)<commit_msg>hello<commit_after># -*- coding: utf-8 -*-

def getVIPfee(month, level, typeid):
	#typeid:
	#1, 200Units
	#2, 500Units
	#3, Unlimited Units
	#month:

	fee = 0.1
	return fee

def getDeliveryfee(cmid):
	detail = u"同仓物流费用 6元"
	return (0.1, detail)<|endoftext|>


<commit_before>#! /usr/bin/env python2

import os
import sys
import argparse
import time
import snotes_persistence

config_dir = os.path.expanduser('~/.snotes')
data_file = "snotes.data"

def err_write(msg):
    sys.stderr.write(msg)

def init():
    if os.path.exists(config_dir):
        if not os.path.isdir(config_dir):
            err_write('{0} exists and is not a directory\n'.format(config_dir))
            sys.exit(1)
    else:
        os.makedirs(config_dir)
        os.chdir(config_dir)
        with open(data_file, 'w') as f:
            f.write('\n')
    os.chdir(config_dir)

def add(args):
    current_time = time.time()
    entry = snotes_persistence.Entry(current_time,
                                     current_time,
                                     args.tags,
                                     args.note)
    global journal
    journal.add_or_merge_entry(entry)
    journal.to_file(data_file)

def get(args):
    global journal
    entries = journal.get_entries(
        lambda entry:snotes_persistence.filter_tags_inclusive(args.tags, entry.tags),
        lambda entry:entry.update_timestamp
    )
    for entry in entries:
        print entry.value

init()
arg_parser = argparse.ArgumentParser()
action_parsers = arg_parser.add_subparsers()
get_action = action_parsers.add_parser('get')
get_action.add_argument('-t', '--tags', nargs='+')
get_action.set_defaults(func=get)
add_action = action_parsers.add_parser('add')
add_action.add_argument('-t', '--tags', nargs='+')
add_action.add_argument('-n', '--note', required=True)
add_action.set_defaults(func=add)
args = arg_parser.parse_args()
journal = snotes_persistence.Journal.from_file(data_file)
args.func(args)
<commit_msg>enabled notes to have spaces<commit_after>#! /usr/bin/env python2

import os
import sys
import argparse
import time
import snotes_persistence

config_dir = os.path.expanduser('~/.snotes')
data_file = "snotes.data"

def err_write(msg):
    sys.stderr.write(msg)

def init():
    if os.path.exists(config_dir):
        if not os.path.isdir(config_dir):
            err_write('{0} exists and is not a directory\n'.format(config_dir))
            sys.exit(1)
    else:
        os.makedirs(config_dir)
        os.chdir(config_dir)
        with open(data_file, 'w') as f:
            f.write('\n')
    os.chdir(config_dir)

def add(args):
    current_time = time.time()
    entry = snotes_persistence.Entry(current_time,
                                     current_time,
                                     args.tags,
                                     ' '.join(args.note))
    global journal
    journal.add_or_merge_entry(entry)
    journal.to_file(data_file)

def get(args):
    global journal
    entries = journal.get_entries(
        lambda entry:snotes_persistence.filter_tags_inclusive(args.tags, entry.tags),
        lambda entry:entry.update_timestamp
    )
    for entry in entries:
        print entry.value

init()
arg_parser = argparse.ArgumentParser()
action_parsers = arg_parser.add_subparsers()
get_action = action_parsers.add_parser('get')
get_action.add_argument('-t', '--tags', nargs='+')
get_action.set_defaults(func=get)
add_action = action_parsers.add_parser('add')
add_action.add_argument('-t', '--tags', nargs='+')
add_action.add_argument('-n', '--note', nargs='+')
add_action.set_defaults(func=add)
args = arg_parser.parse_args()
journal = snotes_persistence.Journal.from_file(data_file)
args.func(args)
<|endoftext|>