<commit_before>import os
from unittest import mock
import posixpath
import pytest

from mlflow.store.artifact.mlflow_artifacts_repo import MlflowArtifactsRepository
from mlflow.store.artifact.artifact_repository_registry import get_artifact_repository
from mlflow.exceptions import MlflowException
from mlflow.tracking._tracking_service.utils import _get_default_host_creds

@pytest.fixture(scope="module", autouse=True)
def set_tracking_uri():
    with mock.patch(
        "mlflow.store.artifact.mlflow_artifacts_repo.get_tracking_uri",
        return_value="http://localhost:5000/",
    ):
        yield

def test_artifact_uri_factory():
    repo = get_artifact_repository("mlflow-artifacts://test.com")
    assert isinstance(repo, MlflowArtifactsRepository)

base_url = "/api/2.0/mlflow-artifacts/artifacts"
base_path = "/my/artifact/path"
conditions = [
    (
        f"mlflow-artifacts://myhostname:4242{base_path}/hostport",
        f"http://myhostname:4242{base_url}{base_path}/hostport",
    ),
    (
        f"mlflow-artifacts://myhostname{base_path}/host",
        f"http://myhostname{base_url}{base_path}/host",
    ),
    (
        f"mlflow-artifacts:{base_path}/nohost",
        f"http://localhost:5000{base_url}{base_path}/nohost",
    ),
    (
        f"mlflow-artifacts://{base_path}/redundant",
        f"http://localhost:5000{base_url}{base_path}/redundant",
    ),
    ("mlflow-artifacts:/", f"http://localhost:5000{base_url}"),
]

@pytest.mark.parametrize("tracking_uri", ["http://localhost:5000", "http://localhost:5000/"])
@pytest.mark.parametrize(("artifact_uri", "resolved_uri"), conditions)
def test_mlflow_artifact_uri_formats_resolved(artifact_uri, resolved_uri, tracking_uri):

    assert MlflowArtifactsRepository.resolve_uri(artifact_uri, tracking_uri) == resolved_uri

def test_mlflow_artifact_uri_raises_with_invalid_tracking_uri():
    failing_conditions = [f"mlflow-artifacts://5000/{base_path}", "mlflow-artifacts://5000/"]

    for failing_condition in failing_conditions:
        with pytest.raises(
            MlflowException,
            match="The mlflow-artifacts uri was supplied with a port number: 5000, but no "
            "host was defined.",
        ):
            MlflowArtifactsRepository(failing_condition)

class MockResponse:
    def __init__(self, data, status_code):
        self.data = data
        self.status_code = status_code

    def json(self):
        return self.data

    def raise_for_status(self):
        if self.status_code >= 400:
            raise Exception("request failed")

class MockStreamResponse(MockResponse):
    def iter_content(self, chunk_size):  # pylint: disable=unused-argument
        yield self.data.encode("utf-8")

    def __enter__(self):
        return self

    def __exit__(self, *exc):
        pass

class FileObjectMatcher:
    def __init__(self, name, mode):
        self.name = name
        self.mode = mode

    def __eq__(self, other):
        return self.name == other.name and self.mode == other.mode

@pytest.fixture
def mlflow_artifact_repo():
    artifact_uri = "mlflow-artifacts:/api/2.0/mlflow-artifacts/artifacts"
    return MlflowArtifactsRepository(artifact_uri)

@pytest.fixture
def mlflow_artifact_repo_with_host():
    artifact_uri = "mlflow-artifacts://test.com:5000/api/2.0/mlflow-artifacts/artifacts"
    return MlflowArtifactsRepository(artifact_uri)

@pytest.mark.parametrize("artifact_path", [None, "dir", "path/to/artifacts/storage"])
def test_log_artifact(mlflow_artifact_repo, tmpdir, artifact_path):
    tmp_path = tmpdir.join("a.txt")
    tmp_path.write("0")
    with mock.patch(
        "mlflow.store.artifact.http_artifact_repo.http_request",
        return_value=MockResponse({}, 200),
    ) as mock_put:
        mlflow_artifact_repo.log_artifact(tmp_path, artifact_path)
        paths = (artifact_path, tmp_path.basename) if artifact_path else (tmp_path.basename,)
        mock_put.assert_called_once_with(
            mlflow_artifact_repo._host_creds,
            posixpath.join("/", *paths),
            "PUT",
            data=FileObjectMatcher(tmp_path, "rb"),
        )

    with mock.patch(
        "mlflow.store.artifact.http_artifact_repo.http_request",
        return_value=MockResponse({}, 400),
    ):
        with pytest.raises(Exception, match="request failed"):
            mlflow_artifact_repo.log_artifact(tmp_path, artifact_path)

@pytest.mark.parametrize("artifact_path", [None, "dir", "path/to/artifacts/storage"])
def test_log_artifact_with_host_and_port(mlflow_artifact_repo_with_host, tmpdir, artifact_path):
    tmp_path = tmpdir.join("a.txt")
    tmp_path.write("0")
    with mock.patch(
        "mlflow.store.artifact.http_artifact_repo.http_request",
        return_value=MockResponse({}, 200),
    ) as mock_put:
        mlflow_artifact_repo_with_host.log_artifact(tmp_path, artifact_path)
        paths = (artifact_path, tmp_path.basename) if artifact_path else (tmp_path.basename,)
        mock_put.assert_called_once_with(
            mlflow_artifact_repo_with_host._host_creds,
            posixpath.join("/", *paths),
            "PUT",
            data=FileObjectMatcher(tmp_path, "rb"),
        )

    with mock.patch(
        "mlflow.store.artifact.http_artifact_repo.http_request",
        return_value=MockResponse({}, 400),
    ):
        with pytest.raises(Exception, match="request failed"):
            mlflow_artifact_repo_with_host.log_artifact(tmp_path, artifact_path)

@pytest.mark.parametrize("artifact_path", [None, "dir", "path/to/artifacts/storage"])
def test_log_artifacts(mlflow_artifact_repo, tmpdir, artifact_path):
    tmp_path_a = tmpdir.join("a.txt")
    tmp_path_b = tmpdir.mkdir("dir").join("b.txt")
    tmp_path_a.write("0")
    tmp_path_b.write("1")

    with mock.patch.object(mlflow_artifact_repo, "log_artifact") as mock_log_artifact:
        mlflow_artifact_repo.log_artifacts(tmpdir, artifact_path)
        mock_log_artifact.assert_has_calls(
            [
                mock.call(tmp_path_a.strpath, artifact_path),
                mock.call(
                    tmp_path_b.strpath,
                    posixpath.join(artifact_path, "dir") if artifact_path else "dir",
                ),
            ],
        )

    with mock.patch(
        "mlflow.store.artifact.http_artifact_repo.http_request",
        return_value=MockResponse({}, 400),
    ):
        with pytest.raises(Exception, match="request failed"):
            mlflow_artifact_repo.log_artifacts(tmpdir, artifact_path)

def test_list_artifacts(mlflow_artifact_repo):
    with mock.patch(
        "mlflow.store.artifact.http_artifact_repo.http_request",
        return_value=MockResponse({}, 200),
    ) as mock_get:
        assert mlflow_artifact_repo.list_artifacts() == []
        endpoint = "/mlflow-artifacts/artifacts"
        url, _ = mlflow_artifact_repo.artifact_uri.split(endpoint, maxsplit=1)
        mock_get.assert_called_once_with(
            _get_default_host_creds(url),
            endpoint,
            "GET",
            params={"path": ""},
        )

    with mock.patch(
        "mlflow.store.artifact.http_artifact_repo.http_request",
        return_value=MockResponse(
            {
                "files": [
                    {"path": "1.txt", "is_dir": False, "file_size": 1},
                    {"path": "dir", "is_dir": True},
                ]
            },
            200,
        ),
    ):
        assert [a.path for a in mlflow_artifact_repo.list_artifacts()] == ["1.txt", "dir"]

    with mock.patch(
        "mlflow.store.artifact.http_artifact_repo.http_request",
        return_value=MockResponse(
            {
                "files": [
                    {"path": "1.txt", "is_dir": False, "file_size": 1},
                    {"path": "dir", "is_dir": True},
                ]
            },
            200,
        ),
    ):
        assert [a.path for a in mlflow_artifact_repo.list_artifacts(path="path")] == [
            "path/1.txt",
            "path/dir",
        ]

    with mock.patch(
        "mlflow.store.artifact.http_artifact_repo.http_request",
        return_value=MockResponse({}, 400),
    ):
        with pytest.raises(Exception, match="request failed"):
            mlflow_artifact_repo.list_artifacts()

def read_file(path):
    with open(path) as f:
        return f.read()

@pytest.mark.parametrize("remote_file_path", ["a.txt", "dir/b.xtx"])
def test_download_file(mlflow_artifact_repo, tmpdir, remote_file_path):
    with mock.patch(
        "mlflow.store.artifact.http_artifact_repo.http_request",
        return_value=MockStreamResponse("data", 200),
    ) as mock_get:
        tmp_path = tmpdir.join(posixpath.basename(remote_file_path))
        mlflow_artifact_repo._download_file(remote_file_path, tmp_path)
        mock_get.assert_called_once_with(
            mlflow_artifact_repo._host_creds,
            posixpath.join("/", remote_file_path),
            "GET",
            stream=True,
        )
        with open(tmp_path) as f:
            assert f.read() == "data"

    with mock.patch(
        "mlflow.store.artifact.http_artifact_repo.http_request",
        return_value=MockStreamResponse("data", 400),
    ):
        with pytest.raises(Exception, match="request failed"):
            mlflow_artifact_repo._download_file(remote_file_path, tmp_path)

def test_download_artifacts(mlflow_artifact_repo, tmpdir):
    # This test simulates downloading artifacts in the following structure:
    # ---------
    # - a.txt
    # - dir
    #   - b.txt
    # ---------
    side_effect = [
        # Response for `list_experiments("")` called by `_is_directory("")`
        MockResponse(
            {
                "files": [
                    {"path": "a.txt", "is_dir": False, "file_size": 6},
                    {"path": "dir", "is_dir": True},
                ]
            },
            200,
        ),
        # Response for `list_experiments("")`
        MockResponse(
            {
                "files": [
                    {"path": "a.txt", "is_dir": False, "file_size": 6},
                    {"path": "dir", "is_dir": True},
                ]
            },
            200,
        ),
        # Response for `_download_file("a.txt")`
        MockStreamResponse("data_a", 200),
        # Response for `list_experiments("dir")`
        MockResponse({"files": [{"path": "b.txt", "is_dir": False, "file_size": 1}]}, 200),
        # Response for `_download_file("dir/b.txt")`
        MockStreamResponse("data_b", 200),
    ]
    with mock.patch(
        "mlflow.store.artifact.http_artifact_repo.http_request",
        side_effect=side_effect,
    ):
        mlflow_artifact_repo.download_artifacts("", tmpdir)
        paths = [os.path.join(root, f) for root, _, files in os.walk(tmpdir) for f in files]
        assert [os.path.relpath(p, tmpdir) for p in paths] == [
            "a.txt",
            os.path.join("dir", "b.txt"),
        ]
        assert read_file(paths[0]) == "data_a"
        assert read_file(paths[1]) == "data_b"
<commit_msg>Fix flaky test_download_artifacts (#6801)<commit_after>import os
from unittest import mock
import posixpath
import pytest

from mlflow.store.artifact.mlflow_artifacts_repo import MlflowArtifactsRepository
from mlflow.store.artifact.artifact_repository_registry import get_artifact_repository
from mlflow.exceptions import MlflowException
from mlflow.tracking._tracking_service.utils import _get_default_host_creds

@pytest.fixture(scope="module", autouse=True)
def set_tracking_uri():
    with mock.patch(
        "mlflow.store.artifact.mlflow_artifacts_repo.get_tracking_uri",
        return_value="http://localhost:5000/",
    ):
        yield

def test_artifact_uri_factory():
    repo = get_artifact_repository("mlflow-artifacts://test.com")
    assert isinstance(repo, MlflowArtifactsRepository)

base_url = "/api/2.0/mlflow-artifacts/artifacts"
base_path = "/my/artifact/path"
conditions = [
    (
        f"mlflow-artifacts://myhostname:4242{base_path}/hostport",
        f"http://myhostname:4242{base_url}{base_path}/hostport",
    ),
    (
        f"mlflow-artifacts://myhostname{base_path}/host",
        f"http://myhostname{base_url}{base_path}/host",
    ),
    (
        f"mlflow-artifacts:{base_path}/nohost",
        f"http://localhost:5000{base_url}{base_path}/nohost",
    ),
    (
        f"mlflow-artifacts://{base_path}/redundant",
        f"http://localhost:5000{base_url}{base_path}/redundant",
    ),
    ("mlflow-artifacts:/", f"http://localhost:5000{base_url}"),
]

@pytest.mark.parametrize("tracking_uri", ["http://localhost:5000", "http://localhost:5000/"])
@pytest.mark.parametrize(("artifact_uri", "resolved_uri"), conditions)
def test_mlflow_artifact_uri_formats_resolved(artifact_uri, resolved_uri, tracking_uri):

    assert MlflowArtifactsRepository.resolve_uri(artifact_uri, tracking_uri) == resolved_uri

def test_mlflow_artifact_uri_raises_with_invalid_tracking_uri():
    failing_conditions = [f"mlflow-artifacts://5000/{base_path}", "mlflow-artifacts://5000/"]

    for failing_condition in failing_conditions:
        with pytest.raises(
            MlflowException,
            match="The mlflow-artifacts uri was supplied with a port number: 5000, but no "
            "host was defined.",
        ):
            MlflowArtifactsRepository(failing_condition)

class MockResponse:
    def __init__(self, data, status_code):
        self.data = data
        self.status_code = status_code

    def json(self):
        return self.data

    def raise_for_status(self):
        if self.status_code >= 400:
            raise Exception("request failed")

class MockStreamResponse(MockResponse):
    def iter_content(self, chunk_size):  # pylint: disable=unused-argument
        yield self.data.encode("utf-8")

    def __enter__(self):
        return self

    def __exit__(self, *exc):
        pass

class FileObjectMatcher:
    def __init__(self, name, mode):
        self.name = name
        self.mode = mode

    def __eq__(self, other):
        return self.name == other.name and self.mode == other.mode

@pytest.fixture
def mlflow_artifact_repo():
    artifact_uri = "mlflow-artifacts:/api/2.0/mlflow-artifacts/artifacts"
    return MlflowArtifactsRepository(artifact_uri)

@pytest.fixture
def mlflow_artifact_repo_with_host():
    artifact_uri = "mlflow-artifacts://test.com:5000/api/2.0/mlflow-artifacts/artifacts"
    return MlflowArtifactsRepository(artifact_uri)

@pytest.mark.parametrize("artifact_path", [None, "dir", "path/to/artifacts/storage"])
def test_log_artifact(mlflow_artifact_repo, tmpdir, artifact_path):
    tmp_path = tmpdir.join("a.txt")
    tmp_path.write("0")
    with mock.patch(
        "mlflow.store.artifact.http_artifact_repo.http_request",
        return_value=MockResponse({}, 200),
    ) as mock_put:
        mlflow_artifact_repo.log_artifact(tmp_path, artifact_path)
        paths = (artifact_path, tmp_path.basename) if artifact_path else (tmp_path.basename,)
        mock_put.assert_called_once_with(
            mlflow_artifact_repo._host_creds,
            posixpath.join("/", *paths),
            "PUT",
            data=FileObjectMatcher(tmp_path, "rb"),
        )

    with mock.patch(
        "mlflow.store.artifact.http_artifact_repo.http_request",
        return_value=MockResponse({}, 400),
    ):
        with pytest.raises(Exception, match="request failed"):
            mlflow_artifact_repo.log_artifact(tmp_path, artifact_path)

@pytest.mark.parametrize("artifact_path", [None, "dir", "path/to/artifacts/storage"])
def test_log_artifact_with_host_and_port(mlflow_artifact_repo_with_host, tmpdir, artifact_path):
    tmp_path = tmpdir.join("a.txt")
    tmp_path.write("0")
    with mock.patch(
        "mlflow.store.artifact.http_artifact_repo.http_request",
        return_value=MockResponse({}, 200),
    ) as mock_put:
        mlflow_artifact_repo_with_host.log_artifact(tmp_path, artifact_path)
        paths = (artifact_path, tmp_path.basename) if artifact_path else (tmp_path.basename,)
        mock_put.assert_called_once_with(
            mlflow_artifact_repo_with_host._host_creds,
            posixpath.join("/", *paths),
            "PUT",
            data=FileObjectMatcher(tmp_path, "rb"),
        )

    with mock.patch(
        "mlflow.store.artifact.http_artifact_repo.http_request",
        return_value=MockResponse({}, 400),
    ):
        with pytest.raises(Exception, match="request failed"):
            mlflow_artifact_repo_with_host.log_artifact(tmp_path, artifact_path)

@pytest.mark.parametrize("artifact_path", [None, "dir", "path/to/artifacts/storage"])
def test_log_artifacts(mlflow_artifact_repo, tmpdir, artifact_path):
    tmp_path_a = tmpdir.join("a.txt")
    tmp_path_b = tmpdir.mkdir("dir").join("b.txt")
    tmp_path_a.write("0")
    tmp_path_b.write("1")

    with mock.patch.object(mlflow_artifact_repo, "log_artifact") as mock_log_artifact:
        mlflow_artifact_repo.log_artifacts(tmpdir, artifact_path)
        mock_log_artifact.assert_has_calls(
            [
                mock.call(tmp_path_a.strpath, artifact_path),
                mock.call(
                    tmp_path_b.strpath,
                    posixpath.join(artifact_path, "dir") if artifact_path else "dir",
                ),
            ],
        )

    with mock.patch(
        "mlflow.store.artifact.http_artifact_repo.http_request",
        return_value=MockResponse({}, 400),
    ):
        with pytest.raises(Exception, match="request failed"):
            mlflow_artifact_repo.log_artifacts(tmpdir, artifact_path)

def test_list_artifacts(mlflow_artifact_repo):
    with mock.patch(
        "mlflow.store.artifact.http_artifact_repo.http_request",
        return_value=MockResponse({}, 200),
    ) as mock_get:
        assert mlflow_artifact_repo.list_artifacts() == []
        endpoint = "/mlflow-artifacts/artifacts"
        url, _ = mlflow_artifact_repo.artifact_uri.split(endpoint, maxsplit=1)
        mock_get.assert_called_once_with(
            _get_default_host_creds(url),
            endpoint,
            "GET",
            params={"path": ""},
        )

    with mock.patch(
        "mlflow.store.artifact.http_artifact_repo.http_request",
        return_value=MockResponse(
            {
                "files": [
                    {"path": "1.txt", "is_dir": False, "file_size": 1},
                    {"path": "dir", "is_dir": True},
                ]
            },
            200,
        ),
    ):
        assert [a.path for a in mlflow_artifact_repo.list_artifacts()] == ["1.txt", "dir"]

    with mock.patch(
        "mlflow.store.artifact.http_artifact_repo.http_request",
        return_value=MockResponse(
            {
                "files": [
                    {"path": "1.txt", "is_dir": False, "file_size": 1},
                    {"path": "dir", "is_dir": True},
                ]
            },
            200,
        ),
    ):
        assert [a.path for a in mlflow_artifact_repo.list_artifacts(path="path")] == [
            "path/1.txt",
            "path/dir",
        ]

    with mock.patch(
        "mlflow.store.artifact.http_artifact_repo.http_request",
        return_value=MockResponse({}, 400),
    ):
        with pytest.raises(Exception, match="request failed"):
            mlflow_artifact_repo.list_artifacts()

def read_file(path):
    with open(path) as f:
        return f.read()

@pytest.mark.parametrize("remote_file_path", ["a.txt", "dir/b.xtx"])
def test_download_file(mlflow_artifact_repo, tmpdir, remote_file_path):
    with mock.patch(
        "mlflow.store.artifact.http_artifact_repo.http_request",
        return_value=MockStreamResponse("data", 200),
    ) as mock_get:
        tmp_path = tmpdir.join(posixpath.basename(remote_file_path))
        mlflow_artifact_repo._download_file(remote_file_path, tmp_path)
        mock_get.assert_called_once_with(
            mlflow_artifact_repo._host_creds,
            posixpath.join("/", remote_file_path),
            "GET",
            stream=True,
        )
        with open(tmp_path) as f:
            assert f.read() == "data"

    with mock.patch(
        "mlflow.store.artifact.http_artifact_repo.http_request",
        return_value=MockStreamResponse("data", 400),
    ):
        with pytest.raises(Exception, match="request failed"):
            mlflow_artifact_repo._download_file(remote_file_path, tmp_path)

def test_download_artifacts(mlflow_artifact_repo, tmpdir):
    # This test simulates downloading artifacts in the following structure:
    # ---------
    # - a.txt
    # - dir
    #   - b.txt
    # ---------
    def http_request(_host_creds, endpoint, _method, **kwargs):
        # Responses for list_artifacts
        params = kwargs.get("params")
        if params:
            if params.get("path") == "":
                return MockResponse(
                    {
                        "files": [
                            {"path": "a.txt", "is_dir": False, "file_size": 1},
                            {"path": "dir", "is_dir": True},
                        ]
                    },
                    200,
                )
            elif params.get("path") == "dir":
                return MockResponse(
                    {
                        "files": [
                            {"path": "b.txt", "is_dir": False, "file_size": 1},
                        ]
                    },
                    200,
                )
            else:
                Exception("Unreachable")

        # Responses for _download_file
        if endpoint == "/a.txt":
            return MockStreamResponse("data_a", 200)
        elif endpoint == "/dir/b.txt":
            return MockStreamResponse("data_b", 200)
        else:
            raise Exception("Unreachable")

    with mock.patch("mlflow.store.artifact.http_artifact_repo.http_request", http_request):
        mlflow_artifact_repo.download_artifacts("", tmpdir)
        paths = [os.path.join(root, f) for root, _, files in os.walk(tmpdir) for f in files]
        assert [os.path.relpath(p, tmpdir) for p in paths] == [
            "a.txt",
            os.path.join("dir", "b.txt"),
        ]
        assert read_file(paths[0]) == "data_a"
        assert read_file(paths[1]) == "data_b"
<|endoftext|>


<commit_before>#!/usr/bin/python
# Copyright (C) 2009-2010 - Curtis Hovey <sinzui.is at verizon.net>
# This software is licensed under the MIT license (see the file COPYING).
"""Reformat a doctest to Launchpad style."""

__metaclass__ = type

import compiler
import os
import re
import sys
from difflib import unified_diff
from doctest import DocTestParser, Example
from optparse import OptionParser
from textwrap import wrap

from contrib.pyflakes.checker import Checker

__all__ = [
    'DoctestReviewer',
    ]

class DoctestReviewer:
    """Check and reformat doctests."""
    rule_pattern = re.compile(r'([=~-])+[ ]*$')
    moin_pattern = re.compile(r'^(=+)[ ](.+)[ ](=+[ ]*)$')

    SOURCE = 'source'
    WANT = 'want'
    NARRATIVE = 'narrative'

    def __init__(self, doctest, file_path, reporter=None):
        self.doctest = doctest
        self.file_path = file_path
        self.base_dir = os.path.dirname(file_path)
        self.file_name = os.path.basename(file_path)
        self.blocks = []
        self.block = []
        self.block_method = self.preserve_block
        self.code_lines = []
        self.example = None
        self.last_bad_indent = 0
        self.has_printed_filename = False
        self._reporter = reporter

    def get_parts(self):
        parser = DocTestParser()
        try:
            return parser.parse(self.doctest, self.file_path)
        except ValueError, error:
            self._print_message(str(error), 0)
            return []

    def _print_message(self, message, lineno):
        """Print the error message with the lineno.

        :param message: The message to print.
        :param lineno: The line number the message pertains to.
        """
        if self._reporter is None:
            if not self.has_printed_filename:
                print '%s:' % self.file_path
                self.has_printed_filename = True
            print '    % 4s: %s' % (lineno, message)
        else:
            self._reporter(
                int(lineno), message,
                base_dir=self.base_dir, file_name=self.file_name)

    def _is_formatted(self, text):
        """Return True if the text is pre-formatted, otherwise False.

        :param: text a string, or a list of strings.
        """
        if isinstance(text, list):
            text = text[0]
        return text.startswith(' ')

    def _walk(self, doctest_parts):
        """Walk the doctest parts; yield the line and kind.

        Yield the content of the line, and its kind (SOURCE, WANT, NARRATIVE).
        SOURCE and WANT lines are stripped of indentation, SOURCE is also
        stripped of the interpreter symbols.

        :param doctest_parts: The output of DocTestParser.parse.
        """
        for part in doctest_parts:
            if part == '':
                continue
            if isinstance(part, Example):
                self.example = part
                for line in part.source.splitlines():
                    kind = DoctestReviewer.SOURCE
                    yield line, kind
                for line in part.want.splitlines():
                    kind = DoctestReviewer.WANT
                    yield line, kind
            else:
                self.example = None
                kind = DoctestReviewer.NARRATIVE
                for line in part.splitlines():
                    yield line, kind

    def _apply(self, line_methods):
        """Call each line_method for each line in the doctest.

        :param line_methods: a list of methods that accept lineno, line,
            and kind as arguments. Each method must return the line for
            the next method to process.
        """
        self.blocks = []
        self.block = []
        lineno = 0
        previous_kind = DoctestReviewer.NARRATIVE
        for line, kind in self._walk(self.get_parts()):
            lineno += 1
            self._append_source(kind, line)
            if kind != previous_kind and kind != DoctestReviewer.WANT:
                # The WANT block must adjoin the preceding SOURCE block.
                self._store_block(previous_kind)
            for method in line_methods:
                line = method(lineno, line, kind, previous_kind)
                if line is None:
                    break
            if not line:
                continue
            self.block.append(line)
            previous_kind = kind
        # Capture the last block and a blank line.
        self.block.append('\n')
        self._store_block(previous_kind)

    def _append_source(self, kind, line):
        """Update the list of source code lines seen."""
        if kind == self.SOURCE:
            self.code_lines.append(line)
        else:
            self.code_lines.append('')

    def _store_block(self, kind):
        """Append the block to blocks, re-wrap unformatted narrative.

        :param kind: The block's kind (SOURCE, WANT, NARRATIVE)
        """
        if len(self.block) == 0:
            return
        block = self.block_method(kind, self.block, self.blocks)
        self.blocks.append('\n'.join(block))
        self.block = []

    def check(self):
        """Check the doctest for style and code issues.

        1. Check line lengths.
        2. Check that headings are not in Moin format.
        3. Check indentation.
        4. Check trailing whitespace.
        """
        self.code_lines = []
        line_checkers = [
            self.check_length,
            self.check_heading,
            self.check_indentation,
            self.check_trailing_whitespace,
            ]
        self._apply(line_checkers)
        code = '\n'.join(self.code_lines)
        self.check_source_code(code)

    def format(self):
        """Reformat doctest.

        1. Tests are reindented to 4 spaces.
        2. Simple narrative is rewrapped to 78 character width.
        3. Formatted (indented) narrative is preserved.
        4. Moin headings are converted to RSR =, == , and === levels.
        5. There is one blank line between blocks,
        6. Except for headers which have two leading blank lines.
        7. All trailing whitespace is removed.

        SOURCE and WANT long lines are not fixed--this is a human operation.
        """
        line_checkers = [
            self.fix_trailing_whitespace,
            self.fix_indentation,
            self.fix_heading,
            self.fix_narrative_paragraph,
            ]
        self.block_method = self.format_block
        self._apply(line_checkers)
        self.block_method = self.preserve_block
        return '\n\n'.join(self.blocks)

    def preserve_block(self, kind, block, blocks):
        """Do nothing to the block.

        :param kind: The block's kind (SOURCE, WANT, NARRATIVE)
        :param block: The list of lines that should remain together.
        :param blocks: The list of all collected blocks.
        """
        return block

    def format_block(self, kind, block, blocks):
        """Format paragraph blocks.

        :param kind: The block's kind (SOURCE, WANT, NARRATIVE)
        :param block: The list of lines that should remain together.
        :param blocks: The list of all collected blocks.
        """
        if kind != DoctestReviewer.NARRATIVE or self._is_formatted(block):
            return block
        try:
            rules = ('===', '---', '...')
            last_line = block[-1]
            is_heading = last_line[0:3] in rules and last_line[-3:] in rules
        except IndexError:
            is_heading = False
        if len(blocks) != 0 and is_heading:
            # Headings should have an extra leading blank line.
            block.insert(0, '')
        elif is_heading:
            # Do nothing. This is the first heading in the file.
            pass
        else:
            long_line = ' '.join(block).strip()
            block = wrap(long_line, 72)
        return block

    def is_code_comment(self, line):
        """Return True if the line is a code comment."""
        comment_pattern = re.compile(r'^\s+#')
        return comment_pattern.match(line) is not None

    def check_length(self, lineno, line, kind, previous_kind):
        """Check the length of the line.

        Each kind of line has a maximum length:

        * NARRATIVE: 78 characters.
        * SOURCE: 70 characters (discounting indentation and interpreter).
        * WANT: 74 characters (discounting indentation).
        """
        if kind == DoctestReviewer.NARRATIVE and self.is_code_comment(line):
            # comments follow WANT rules because they are in code.
            kind = DoctestReviewer.WANT
            line = line.lstrip()
        length = len(line)
        if kind == DoctestReviewer.NARRATIVE and length > 78:
            self._print_message('%s exceeds 78 characters.' % kind, lineno)
        elif kind == DoctestReviewer.WANT and length > 74:
            self._print_message('%s exceeds 78 characters.' % kind, lineno)
        elif kind == DoctestReviewer.SOURCE and length > 70:
            self._print_message('%s exceeds 78 characters.' % kind, lineno)
        else:
            # This line has a good length.
            pass
        return line

    def check_indentation(self, lineno, line, kind, previous_kind):
        """Check the indentation of the SOURCE or WANT line."""
        if kind == DoctestReviewer.NARRATIVE:
            return line
        if self.example.indent != 4:
            if self.last_bad_indent != lineno - 1:
                self._print_message('%s has bad indentation.' % kind, lineno)
            self.last_bad_indent = lineno
        return line

    def check_trailing_whitespace(self, lineno, line, kind, previous_kind):
        """Check for the presence of trailing whitespace in the line."""
        if line.endswith(' '):
            self._print_message('%s has trailing whitespace.' % kind, lineno)
        return line

    def check_heading(self, lineno, line, kind, previous_kind):
        """Check for narrative lines that use moin headers instead of RST."""
        if kind != DoctestReviewer.NARRATIVE:
            return line
        moin = self.moin_pattern.match(line)
        if moin is not None:
            self._print_message('%s uses a moin header.' % kind, lineno)
        return line

    def check_source_code(self, code):
        """Check for source code problems in the doctest using pyflakes.

        The most common problem found are unused imports. `UndefinedName`
        errors are suppressed because the test setup is not known.
        """
        if code == '':
            return
        try:
            tree = compiler.parse(code)
        except (SyntaxError, IndentationError), exc:
            (lineno, offset_, line) = exc[1][1:]
            if line.endswith("\n"):
                line = line[:-1]
            self._print_message(
                'Could not compile:\n          %s' % line, lineno)
        else:
            warnings = Checker(tree)
            for warning in warnings.messages:
                if 'undefined name ' in str(warning):
                    continue
                dummy, lineno, message = str(warning).split(':')
                self._print_message(message.strip(), lineno)

    def fix_trailing_whitespace(self, lineno, line, kind, previous_kind):
        """Return the line striped of trailing whitespace."""
        return line.rstrip()

    def fix_indentation(self, lineno, line, kind, previous_kind):
        """set the indentation to 4-spaces."""
        if kind == DoctestReviewer.NARRATIVE:
            return line
        elif kind == DoctestReviewer.WANT:
            return '    %s' % line
        else:
            if line.startswith(' '):
                # This is a continuation of DoctestReviewer.SOURCE.
                return '    ... %s' % line
            else:
                # This is a start of DoctestReviewer.SOURCE.
                return '    >>> %s' % line

    def fix_heading(self, lineno, line, kind, previous_kind):
        """Switch Moin headings to RST headings."""
        if kind != DoctestReviewer.NARRATIVE:
            return line
        moin = self.moin_pattern.match(line)
        if moin is None:
            return line
        heading_level = len(moin.group(1))
        heading = moin.group(2)
        rule_length = len(heading)
        if heading_level == 1:
            rule = '=' * rule_length
        elif heading_level == 2:
            rule = '-' * rule_length
        else:
            rule = '.' * rule_length
        # Force the heading on to the block of lines.
        self.block.append(heading)
        return rule

    def fix_narrative_paragraph(self, lineno, line, kind, previous_kind):
        """Break narrative into paragraphs."""
        if kind != DoctestReviewer.NARRATIVE or len(self.block) == 0:
            return line
        if line == '':
            # This is the start of a new paragraph in the narrative.
            self._store_block(previous_kind)
        if self._is_formatted(line) and not self._is_formatted(self.block):
            # This line starts a pre-formatted paragraph.
            self._store_block(previous_kind)
        return line

def get_option_parser():
    """Return the option parser for this program."""
    usage = "usage: %prog [options] doctest.txt"
    parser = OptionParser(usage=usage)
    parser.add_option(
        "-f", "--format", dest="is_format", action="store_true",
        help="Reformat the doctest.")
    parser.add_option(
        "-i", "--interactive", dest="is_interactive", action="store_true",
        help="Approve each change.")
    parser.set_defaults(
        is_format=False,
        is_interactive=False)
    return parser

def main(argv=None):
    """Run the operations requested from the command line."""
    if argv is None:
        argv = sys.argv
    parser = get_option_parser()
    (options, args) = parser.parse_args(args=argv[1:])
    if len(args) == 0:
        parser.error("A doctest must be specified.")

    for file_path in args:
        try:
            doctest_file = open(file_path)
            old_doctest = doctest_file.read()
        finally:
            doctest_file.close()
        reviewer = DoctestReviewer(old_doctest, file_path)

        if not options.is_format:
            reviewer.check()
            continue

        new_doctest = reviewer.format()
        if new_doctest != old_doctest:
            if options.is_interactive:
                diff = unified_diff(
                    old_doctest.splitlines(), new_doctest.splitlines())
                print '\n'.join(diff)
                print '\n'
                do_save = raw_input(
                    'Do you wish to save the changes? S(ave) or C(ancel)?')
            else:
                do_save = 'S'

            if do_save.upper() == 'S':
                try:
                    doctest_file = open(file_path, 'w')
                    doctest_file.write(new_doctest)
                finally:
                    doctest_file.close()
            reviewer = DoctestReviewer(new_doctest, file_path)
            reviewer.check()

if __name__ == '__main__':
    sys.exit(main())
<commit_msg>Update DocTestReviewer to use the new pyflakes.<commit_after>#!/usr/bin/python
# Copyright (C) 2009-2010 - Curtis Hovey <sinzui.is at verizon.net>
# This software is licensed under the MIT license (see the file COPYING).
"""Reformat a doctest to Launchpad style."""

__metaclass__ = type

import _ast
import os
import re
import sys
from difflib import unified_diff
from doctest import DocTestParser, Example
from optparse import OptionParser
from textwrap import wrap

from contrib.pyflakes.checker import Checker

__all__ = [
    'DoctestReviewer',
    ]

class DoctestReviewer:
    """Check and reformat doctests."""
    rule_pattern = re.compile(r'([=~-])+[ ]*$')
    moin_pattern = re.compile(r'^(=+)[ ](.+)[ ](=+[ ]*)$')

    SOURCE = 'source'
    WANT = 'want'
    NARRATIVE = 'narrative'

    def __init__(self, doctest, file_path, reporter=None):
        self.doctest = doctest
        self.file_path = file_path
        self.base_dir = os.path.dirname(file_path)
        self.file_name = os.path.basename(file_path)
        self.blocks = []
        self.block = []
        self.block_method = self.preserve_block
        self.code_lines = []
        self.example = None
        self.last_bad_indent = 0
        self.has_printed_filename = False
        self._reporter = reporter

    def get_parts(self):
        parser = DocTestParser()
        try:
            return parser.parse(self.doctest, self.file_path)
        except ValueError, error:
            self._print_message(str(error), 0)
            return []

    def _print_message(self, message, lineno):
        """Print the error message with the lineno.

        :param message: The message to print.
        :param lineno: The line number the message pertains to.
        """
        if self._reporter is None:
            if not self.has_printed_filename:
                print '%s:' % self.file_path
                self.has_printed_filename = True
            print '    % 4s: %s' % (lineno, message)
        else:
            self._reporter(
                int(lineno), message,
                base_dir=self.base_dir, file_name=self.file_name)

    def _is_formatted(self, text):
        """Return True if the text is pre-formatted, otherwise False.

        :param: text a string, or a list of strings.
        """
        if isinstance(text, list):
            text = text[0]
        return text.startswith(' ')

    def _walk(self, doctest_parts):
        """Walk the doctest parts; yield the line and kind.

        Yield the content of the line, and its kind (SOURCE, WANT, NARRATIVE).
        SOURCE and WANT lines are stripped of indentation, SOURCE is also
        stripped of the interpreter symbols.

        :param doctest_parts: The output of DocTestParser.parse.
        """
        for part in doctest_parts:
            if part == '':
                continue
            if isinstance(part, Example):
                self.example = part
                for line in part.source.splitlines():
                    kind = DoctestReviewer.SOURCE
                    yield line, kind
                for line in part.want.splitlines():
                    kind = DoctestReviewer.WANT
                    yield line, kind
            else:
                self.example = None
                kind = DoctestReviewer.NARRATIVE
                for line in part.splitlines():
                    yield line, kind

    def _apply(self, line_methods):
        """Call each line_method for each line in the doctest.

        :param line_methods: a list of methods that accept lineno, line,
            and kind as arguments. Each method must return the line for
            the next method to process.
        """
        self.blocks = []
        self.block = []
        lineno = 0
        previous_kind = DoctestReviewer.NARRATIVE
        for line, kind in self._walk(self.get_parts()):
            lineno += 1
            self._append_source(kind, line)
            if kind != previous_kind and kind != DoctestReviewer.WANT:
                # The WANT block must adjoin the preceding SOURCE block.
                self._store_block(previous_kind)
            for method in line_methods:
                line = method(lineno, line, kind, previous_kind)
                if line is None:
                    break
            if not line:
                continue
            self.block.append(line)
            previous_kind = kind
        # Capture the last block and a blank line.
        self.block.append('\n')
        self._store_block(previous_kind)

    def _append_source(self, kind, line):
        """Update the list of source code lines seen."""
        if kind == self.SOURCE:
            self.code_lines.append(line)
        else:
            self.code_lines.append('')

    def _store_block(self, kind):
        """Append the block to blocks, re-wrap unformatted narrative.

        :param kind: The block's kind (SOURCE, WANT, NARRATIVE)
        """
        if len(self.block) == 0:
            return
        block = self.block_method(kind, self.block, self.blocks)
        self.blocks.append('\n'.join(block))
        self.block = []

    def check(self):
        """Check the doctest for style and code issues.

        1. Check line lengths.
        2. Check that headings are not in Moin format.
        3. Check indentation.
        4. Check trailing whitespace.
        """
        self.code_lines = []
        line_checkers = [
            self.check_length,
            self.check_heading,
            self.check_indentation,
            self.check_trailing_whitespace,
            ]
        self._apply(line_checkers)
        code = '\n'.join(self.code_lines)
        self.check_source_code(code)

    def format(self):
        """Reformat doctest.

        1. Tests are reindented to 4 spaces.
        2. Simple narrative is rewrapped to 78 character width.
        3. Formatted (indented) narrative is preserved.
        4. Moin headings are converted to RSR =, == , and === levels.
        5. There is one blank line between blocks,
        6. Except for headers which have two leading blank lines.
        7. All trailing whitespace is removed.

        SOURCE and WANT long lines are not fixed--this is a human operation.
        """
        line_checkers = [
            self.fix_trailing_whitespace,
            self.fix_indentation,
            self.fix_heading,
            self.fix_narrative_paragraph,
            ]
        self.block_method = self.format_block
        self._apply(line_checkers)
        self.block_method = self.preserve_block
        return '\n\n'.join(self.blocks)

    def preserve_block(self, kind, block, blocks):
        """Do nothing to the block.

        :param kind: The block's kind (SOURCE, WANT, NARRATIVE)
        :param block: The list of lines that should remain together.
        :param blocks: The list of all collected blocks.
        """
        return block

    def format_block(self, kind, block, blocks):
        """Format paragraph blocks.

        :param kind: The block's kind (SOURCE, WANT, NARRATIVE)
        :param block: The list of lines that should remain together.
        :param blocks: The list of all collected blocks.
        """
        if kind != DoctestReviewer.NARRATIVE or self._is_formatted(block):
            return block
        try:
            rules = ('===', '---', '...')
            last_line = block[-1]
            is_heading = last_line[0:3] in rules and last_line[-3:] in rules
        except IndexError:
            is_heading = False
        if len(blocks) != 0 and is_heading:
            # Headings should have an extra leading blank line.
            block.insert(0, '')
        elif is_heading:
            # Do nothing. This is the first heading in the file.
            pass
        else:
            long_line = ' '.join(block).strip()
            block = wrap(long_line, 72)
        return block

    def is_code_comment(self, line):
        """Return True if the line is a code comment."""
        comment_pattern = re.compile(r'^\s+#')
        return comment_pattern.match(line) is not None

    def check_length(self, lineno, line, kind, previous_kind):
        """Check the length of the line.

        Each kind of line has a maximum length:

        * NARRATIVE: 78 characters.
        * SOURCE: 70 characters (discounting indentation and interpreter).
        * WANT: 74 characters (discounting indentation).
        """
        if kind == DoctestReviewer.NARRATIVE and self.is_code_comment(line):
            # comments follow WANT rules because they are in code.
            kind = DoctestReviewer.WANT
            line = line.lstrip()
        length = len(line)
        if kind == DoctestReviewer.NARRATIVE and length > 78:
            self._print_message('%s exceeds 78 characters.' % kind, lineno)
        elif kind == DoctestReviewer.WANT and length > 74:
            self._print_message('%s exceeds 78 characters.' % kind, lineno)
        elif kind == DoctestReviewer.SOURCE and length > 70:
            self._print_message('%s exceeds 78 characters.' % kind, lineno)
        else:
            # This line has a good length.
            pass
        return line

    def check_indentation(self, lineno, line, kind, previous_kind):
        """Check the indentation of the SOURCE or WANT line."""
        if kind == DoctestReviewer.NARRATIVE:
            return line
        if self.example.indent != 4:
            if self.last_bad_indent != lineno - 1:
                self._print_message('%s has bad indentation.' % kind, lineno)
            self.last_bad_indent = lineno
        return line

    def check_trailing_whitespace(self, lineno, line, kind, previous_kind):
        """Check for the presence of trailing whitespace in the line."""
        if line.endswith(' '):
            self._print_message('%s has trailing whitespace.' % kind, lineno)
        return line

    def check_heading(self, lineno, line, kind, previous_kind):
        """Check for narrative lines that use moin headers instead of RST."""
        if kind != DoctestReviewer.NARRATIVE:
            return line
        moin = self.moin_pattern.match(line)
        if moin is not None:
            self._print_message('%s uses a moin header.' % kind, lineno)
        return line

    def check_source_code(self, code):
        """Check for source code problems in the doctest using pyflakes.

        The most common problem found are unused imports. `UndefinedName`
        errors are suppressed because the test setup is not known.
        """
        if code == '':
            return
        try:
            tree = compile(
                code, self.file_path, "exec", _ast.PyCF_ONLY_AST)
        except (SyntaxError, IndentationError), exc:
            (lineno, offset_, line) = exc[1][1:]
            if line.endswith("\n"):
                line = line[:-1]
            self._print_message(
                'Could not compile:\n          %s' % line, lineno)
        else:
            warnings = Checker(tree)
            for warning in warnings.messages:
                if 'undefined name ' in str(warning):
                    continue
                dummy, lineno, message = str(warning).split(':')
                self._print_message(message.strip(), lineno)

    def fix_trailing_whitespace(self, lineno, line, kind, previous_kind):
        """Return the line striped of trailing whitespace."""
        return line.rstrip()

    def fix_indentation(self, lineno, line, kind, previous_kind):
        """set the indentation to 4-spaces."""
        if kind == DoctestReviewer.NARRATIVE:
            return line
        elif kind == DoctestReviewer.WANT:
            return '    %s' % line
        else:
            if line.startswith(' '):
                # This is a continuation of DoctestReviewer.SOURCE.
                return '    ... %s' % line
            else:
                # This is a start of DoctestReviewer.SOURCE.
                return '    >>> %s' % line

    def fix_heading(self, lineno, line, kind, previous_kind):
        """Switch Moin headings to RST headings."""
        if kind != DoctestReviewer.NARRATIVE:
            return line
        moin = self.moin_pattern.match(line)
        if moin is None:
            return line
        heading_level = len(moin.group(1))
        heading = moin.group(2)
        rule_length = len(heading)
        if heading_level == 1:
            rule = '=' * rule_length
        elif heading_level == 2:
            rule = '-' * rule_length
        else:
            rule = '.' * rule_length
        # Force the heading on to the block of lines.
        self.block.append(heading)
        return rule

    def fix_narrative_paragraph(self, lineno, line, kind, previous_kind):
        """Break narrative into paragraphs."""
        if kind != DoctestReviewer.NARRATIVE or len(self.block) == 0:
            return line
        if line == '':
            # This is the start of a new paragraph in the narrative.
            self._store_block(previous_kind)
        if self._is_formatted(line) and not self._is_formatted(self.block):
            # This line starts a pre-formatted paragraph.
            self._store_block(previous_kind)
        return line

def get_option_parser():
    """Return the option parser for this program."""
    usage = "usage: %prog [options] doctest.txt"
    parser = OptionParser(usage=usage)
    parser.add_option(
        "-f", "--format", dest="is_format", action="store_true",
        help="Reformat the doctest.")
    parser.add_option(
        "-i", "--interactive", dest="is_interactive", action="store_true",
        help="Approve each change.")
    parser.set_defaults(
        is_format=False,
        is_interactive=False)
    return parser

def main(argv=None):
    """Run the operations requested from the command line."""
    if argv is None:
        argv = sys.argv
    parser = get_option_parser()
    (options, args) = parser.parse_args(args=argv[1:])
    if len(args) == 0:
        parser.error("A doctest must be specified.")

    for file_path in args:
        try:
            doctest_file = open(file_path)
            old_doctest = doctest_file.read()
        finally:
            doctest_file.close()
        reviewer = DoctestReviewer(old_doctest, file_path)

        if not options.is_format:
            reviewer.check()
            continue

        new_doctest = reviewer.format()
        if new_doctest != old_doctest:
            if options.is_interactive:
                diff = unified_diff(
                    old_doctest.splitlines(), new_doctest.splitlines())
                print '\n'.join(diff)
                print '\n'
                do_save = raw_input(
                    'Do you wish to save the changes? S(ave) or C(ancel)?')
            else:
                do_save = 'S'

            if do_save.upper() == 'S':
                try:
                    doctest_file = open(file_path, 'w')
                    doctest_file.write(new_doctest)
                finally:
                    doctest_file.close()
            reviewer = DoctestReviewer(new_doctest, file_path)
            reviewer.check()

if __name__ == '__main__':
    sys.exit(main())
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function

import sys

import click

from marshmallow import ValidationError

from polyaxon_cli.logger import clean_outputs
from polyaxon_cli.managers.auth import AuthConfigManager
from polyaxon_cli.managers.project import ProjectManager
from polyaxon_cli.utils import constants
from polyaxon_cli.utils.clients import PolyaxonClients
from polyaxon_cli.utils.formatting import (
    Printer,
    dict_tabulate,
    get_experiments_with_metrics,
    get_meta_response,
    list_dicts_to_tabulate
)
from polyaxon_client.exceptions import PolyaxonHTTPError, PolyaxonShouldExitError
from polyaxon_schemas.project import ProjectConfig

def get_project_info(project):  # pylint:disable=redefined-outer-name
    parts = project.replace('.', '/').split('/')
    if len(parts) == 2:
        user, project_name = parts
    else:
        user = AuthConfigManager.get_value('username')
        project_name = project

    return user, project_name

def get_project_or_local(project=None):  # pylint:disable=redefined-outer-name
    if not project and not ProjectManager.is_initialized():
        Printer.print_error('Please provide a valid project, or init a new project. '
                            ' {}'.format(constants.INIT_COMMAND))
        sys.exit(1)

    if project:
        user, project_name = get_project_info(project)
    else:
        project = ProjectManager.get_config()
        user, project_name = project.user, project.name

    if not all([user, project_name]):
        Printer.print_error('Please provide a valid project, or init a new project.'
                            ' {}'.format(constants.INIT_COMMAND))
        sys.exit(1)
    return user, project_name

def get_project_details(project):  # pylint:disable=redefined-outer-name
    if project.description:
        Printer.print_header("Project description:")
        click.echo('{}\n'.format(project.description))

    response = project.to_light_dict(
        humanize_values=True,
        exclude_attrs=['uuid', 'experiment_groups', 'experiments', 'description'])

    Printer.print_header("Project info:")
    dict_tabulate(response)

@click.group()
@click.option('--project', '-p', type=str)
@click.pass_context
@clean_outputs
def project(ctx, project):  # pylint:disable=redefined-outer-name
    """Commands for projects."""
    if ctx.invoked_subcommand not in ['create', 'list']:
        ctx.obj = ctx.obj or {}
        ctx.obj['project'] = project

@project.command()
@click.option('--name', required=True, type=str,
              help='Name of the project, must be unique for the same user')
@click.option('--description', type=str, help='Description of the project,')
@click.option('--private', is_flag=True, help='Set the visibility of the project to private.')
@clean_outputs
def create(name, description, private):
    """Create a new project.

    Uses [Caching](/polyaxon_cli/introduction#Caching)

    Example:

    \b
    ```bash
    $ polyaxon project create --name=cats-vs-dogs --description="Image Classification with DL"
    ```
    """
    try:
        project_dict = dict(name=name, description=description, is_public=not private)
        project_config = ProjectConfig.from_dict(project_dict)
    except ValidationError:
        Printer.print_error('Project name should contain only alpha numerical, "-", and "_".')
        sys.exit(1)

    try:
        _project = PolyaxonClients().project.create_project(project_config)
    except (PolyaxonHTTPError, PolyaxonShouldExitError) as e:
        Printer.print_error('Could not create project `{}`.'.format(name))
        Printer.print_error('Error message `{}`.'.format(e))
        sys.exit(1)

    Printer.print_success("Project `{}` was created successfully.".format(_project.name))

@project.command()
@click.option('--page', type=int, help='To paginate through the list of projects.')
@clean_outputs
def list(page):  # pylint:disable=redefined-builtin
    """List projects.

    Uses [Caching](/polyaxon_cli/introduction#Caching)
    """
    user = AuthConfigManager.get_value('username')
    if not user:
        Printer.print_error('Please login first. `polyaxon login --help`')

    page = page or 1
    try:
        response = PolyaxonClients().project.list_projects(user, page=page)
    except (PolyaxonHTTPError, PolyaxonShouldExitError) as e:
        Printer.print_error('Could not get list of projects.')
        Printer.print_error('Error message `{}`.'.format(e))
        sys.exit(1)

    meta = get_meta_response(response)
    if meta:
        Printer.print_header('Projects for current user')
        Printer.print_header('Navigation:')
        dict_tabulate(meta)
    else:
        Printer.print_header('No projects found for current user')

    objects = list_dicts_to_tabulate([o.to_light_dict(humanize_values=True)
                                      for o in response['results']])
    if objects:
        Printer.print_header("Projects:")
        dict_tabulate(objects, is_list_dict=True)

@project.command()
@click.pass_context
@clean_outputs
def get(ctx):
    """Get info for current project, by project_name, or user/project_name.

    Uses [Caching](/polyaxon_cli/introduction#Caching)

    Examples:

    To get current project:

    \b
    ```bash
    $ polyaxon project get
    ```

    To get a project by name

    \b
    ```bash
    $ polyaxon project get user/project
    ```
    """
    user, project_name = get_project_or_local(ctx.obj['project'])

    try:
        response = PolyaxonClients().project.get_project(user, project_name)
    except (PolyaxonHTTPError, PolyaxonShouldExitError) as e:
        Printer.print_error('Could not get project `{}`.'.format(project_name))
        Printer.print_error('Error message `{}`.'.format(e))
        sys.exit(1)

    get_project_details(response)

@project.command()
@click.pass_context
@clean_outputs
def delete(ctx):
    """Delete project.

    Uses [Caching](/polyaxon_cli/introduction#Caching)
    """
    user, project_name = get_project_or_local(ctx.obj['project'])

    if not click.confirm("Are sure you want to delete project `{}/{}`".format(user, project_name)):
        click.echo('Existing without deleting project.')
        sys.exit(1)

    try:
        response = PolyaxonClients().project.delete_project(user, project_name)
        local_project = ProjectManager.get_config()
        if (user, project_name) == (local_project.user, local_project.name):
            # Purge caching
            ProjectManager.purge()
    except (PolyaxonHTTPError, PolyaxonShouldExitError) as e:
        Printer.print_error('Could not delete project `{}/{}`.'.format(user, project_name))
        Printer.print_error('Error message `{}`.'.format(e))
        sys.exit(1)

    if response.status_code == 204:
        Printer.print_success("Project `{}/{}` was delete successfully".format(user, project_name))

@project.command()
@click.option('--name', type=str,
              help='Name of the project, must be unique for the same user,')
@click.option('--description', type=str, help='Description of the project,')
@click.option('--private', type=bool, help='Set the visibility of the project to private/public.')
@click.pass_context
@clean_outputs
def update(ctx, name, description, private):
    """Update project.

    Uses [Caching](/polyaxon_cli/introduction#Caching)

    Example:

    \b
    ```bash
    $ polyaxon update foobar --description="Image Classification with DL using TensorFlow"
    ```

    \b
    ```bash
    $ polyaxon update mike1/foobar --description="Image Classification with DL using TensorFlow"
    ```
    """
    user, project_name = get_project_or_local(ctx.obj['project'])

    update_dict = {}
    if name:
        update_dict['name'] = name

    if description:
        update_dict['description'] = description

    if private is not None:
        update_dict['is_public'] = not private

    if not update_dict:
        Printer.print_warning('No argument was provided to update the project.')
        sys.exit(1)

    try:
        response = PolyaxonClients().project.update_project(user, project_name, update_dict)
    except (PolyaxonHTTPError, PolyaxonShouldExitError) as e:
        Printer.print_error('Could not update project `{}`.'.format(project_name))
        Printer.print_error('Error message `{}`.'.format(e))
        sys.exit(1)

    Printer.print_success("Project updated.")
    get_project_details(response)

@project.command()
@click.option('--page', type=int, help='To paginate through the list of groups.')
@click.pass_context
@clean_outputs
def groups(ctx, page):
    """List experiment groups for this project.

    Uses [Caching](/polyaxon_cli/introduction#Caching)
    """
    user, project_name = get_project_or_local(ctx.obj['project'])

    page = page or 1
    try:
        response = PolyaxonClients().project.list_experiment_groups(user, project_name, page=page)
    except (PolyaxonHTTPError, PolyaxonShouldExitError) as e:
        Printer.print_error(
            'Could not get experiment groups for project `{}`.'.format(project_name))
        Printer.print_error('Error message `{}`.'.format(e))
        sys.exit(1)

    meta = get_meta_response(response)
    if meta:
        Printer.print_header('Experiment groups for project `{}/{}`.'.format(user, project_name))
        Printer.print_header('Navigation:')
        dict_tabulate(meta)
    else:
        Printer.print_header('No experiment groups found for project `{}/{}`.'.format(
            user, project_name))

    objects = list_dicts_to_tabulate([o.to_light_dict(humanize_values=True)
                                      for o in response['results']])
    if objects:
        Printer.print_header("Experiment groups:")
        objects.pop('project', None)
        objects.pop('user', None)
        dict_tabulate(objects, is_list_dict=True)

@project.command()
@click.option('--page', type=int, help='To paginate through the list of experiments.')
@click.option('--metrics', '-m', is_flag=True, help='List experiments with their metrics.')
@click.pass_context
@clean_outputs
@clean_outputs
def experiments(ctx, page, metrics):
    """List experiments for this project.

    Uses [Caching](/polyaxon_cli/introduction#Caching)
    """
    user, project_name = get_project_or_local(ctx.obj['project'])

    page = page or 1
    try:
        response = PolyaxonClients().project.list_experiments(user, project_name, page=page)
    except (PolyaxonHTTPError, PolyaxonShouldExitError) as e:
        Printer.print_error('Could not get experiments for project `{}`.'.format(project_name))
        Printer.print_error('Error message `{}`.'.format(e))
        sys.exit(1)

    meta = get_meta_response(response)
    if meta:
        Printer.print_header('Experiments for project `{}/{}`.'.format(user, project_name))
        Printer.print_header('Navigation:')
        dict_tabulate(meta)
    else:
        Printer.print_header('No experiments found for project `{}/{}`.'.format(user, project_name))

    if metrics:
        objects = get_experiments_with_metrics(response)
    else:
        objects = [Printer.add_status_color(o.to_light_dict(humanize_values=True))
                   for o in response['results']]
    objects = list_dicts_to_tabulate(objects)
    if objects:
        Printer.print_header("Experiments:")
        objects.pop('project_name', None)
        dict_tabulate(objects, is_list_dict=True)

@project.command()
@click.pass_context
@clean_outputs
def clone(ctx):
    pass
<commit_msg>Add jobs and builds to cli<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function

import sys

import click

from marshmallow import ValidationError

from polyaxon_cli.logger import clean_outputs
from polyaxon_cli.managers.auth import AuthConfigManager
from polyaxon_cli.managers.project import ProjectManager
from polyaxon_cli.utils import constants
from polyaxon_cli.utils.clients import PolyaxonClients
from polyaxon_cli.utils.formatting import (
    Printer,
    dict_tabulate,
    get_experiments_with_metrics,
    get_meta_response,
    list_dicts_to_tabulate
)
from polyaxon_client.exceptions import PolyaxonHTTPError, PolyaxonShouldExitError
from polyaxon_schemas.project import ProjectConfig

def get_project_info(project):  # pylint:disable=redefined-outer-name
    parts = project.replace('.', '/').split('/')
    if len(parts) == 2:
        user, project_name = parts
    else:
        user = AuthConfigManager.get_value('username')
        project_name = project

    return user, project_name

def get_project_or_local(project=None):  # pylint:disable=redefined-outer-name
    if not project and not ProjectManager.is_initialized():
        Printer.print_error('Please provide a valid project, or init a new project. '
                            ' {}'.format(constants.INIT_COMMAND))
        sys.exit(1)

    if project:
        user, project_name = get_project_info(project)
    else:
        project = ProjectManager.get_config()
        user, project_name = project.user, project.name

    if not all([user, project_name]):
        Printer.print_error('Please provide a valid project, or init a new project.'
                            ' {}'.format(constants.INIT_COMMAND))
        sys.exit(1)
    return user, project_name

def get_project_details(project):  # pylint:disable=redefined-outer-name
    if project.description:
        Printer.print_header("Project description:")
        click.echo('{}\n'.format(project.description))

    response = project.to_light_dict(
        humanize_values=True,
        exclude_attrs=['uuid', 'experiment_groups', 'experiments', 'description'])

    Printer.print_header("Project info:")
    dict_tabulate(response)

@click.group()
@click.option('--project', '-p', type=str)
@click.pass_context
@clean_outputs
def project(ctx, project):  # pylint:disable=redefined-outer-name
    """Commands for projects."""
    if ctx.invoked_subcommand not in ['create', 'list']:
        ctx.obj = ctx.obj or {}
        ctx.obj['project'] = project

@project.command()
@click.option('--name', required=True, type=str,
              help='Name of the project, must be unique for the same user')
@click.option('--description', type=str, help='Description of the project,')
@click.option('--private', is_flag=True, help='Set the visibility of the project to private.')
@clean_outputs
def create(name, description, private):
    """Create a new project.

    Uses [Caching](/polyaxon_cli/introduction#Caching)

    Example:

    \b
    ```bash
    $ polyaxon project create --name=cats-vs-dogs --description="Image Classification with DL"
    ```
    """
    try:
        project_dict = dict(name=name, description=description, is_public=not private)
        project_config = ProjectConfig.from_dict(project_dict)
    except ValidationError:
        Printer.print_error('Project name should contain only alpha numerical, "-", and "_".')
        sys.exit(1)

    try:
        _project = PolyaxonClients().project.create_project(project_config)
    except (PolyaxonHTTPError, PolyaxonShouldExitError) as e:
        Printer.print_error('Could not create project `{}`.'.format(name))
        Printer.print_error('Error message `{}`.'.format(e))
        sys.exit(1)

    Printer.print_success("Project `{}` was created successfully.".format(_project.name))

@project.command()
@click.option('--page', type=int, help='To paginate through the list of projects.')
@clean_outputs
def list(page):  # pylint:disable=redefined-builtin
    """List projects.

    Uses [Caching](/polyaxon_cli/introduction#Caching)
    """
    user = AuthConfigManager.get_value('username')
    if not user:
        Printer.print_error('Please login first. `polyaxon login --help`')

    page = page or 1
    try:
        response = PolyaxonClients().project.list_projects(user, page=page)
    except (PolyaxonHTTPError, PolyaxonShouldExitError) as e:
        Printer.print_error('Could not get list of projects.')
        Printer.print_error('Error message `{}`.'.format(e))
        sys.exit(1)

    meta = get_meta_response(response)
    if meta:
        Printer.print_header('Projects for current user')
        Printer.print_header('Navigation:')
        dict_tabulate(meta)
    else:
        Printer.print_header('No projects found for current user')

    objects = list_dicts_to_tabulate([o.to_light_dict(humanize_values=True)
                                      for o in response['results']])
    if objects:
        Printer.print_header("Projects:")
        dict_tabulate(objects, is_list_dict=True)

@project.command()
@click.pass_context
@clean_outputs
def get(ctx):
    """Get info for current project, by project_name, or user/project_name.

    Uses [Caching](/polyaxon_cli/introduction#Caching)

    Examples:

    To get current project:

    \b
    ```bash
    $ polyaxon project get
    ```

    To get a project by name

    \b
    ```bash
    $ polyaxon project get user/project
    ```
    """
    user, project_name = get_project_or_local(ctx.obj['project'])

    try:
        response = PolyaxonClients().project.get_project(user, project_name)
    except (PolyaxonHTTPError, PolyaxonShouldExitError) as e:
        Printer.print_error('Could not get project `{}`.'.format(project_name))
        Printer.print_error('Error message `{}`.'.format(e))
        sys.exit(1)

    get_project_details(response)

@project.command()
@click.pass_context
@clean_outputs
def delete(ctx):
    """Delete project.

    Uses [Caching](/polyaxon_cli/introduction#Caching)
    """
    user, project_name = get_project_or_local(ctx.obj['project'])

    if not click.confirm("Are sure you want to delete project `{}/{}`".format(user, project_name)):
        click.echo('Existing without deleting project.')
        sys.exit(1)

    try:
        response = PolyaxonClients().project.delete_project(user, project_name)
        local_project = ProjectManager.get_config()
        if (user, project_name) == (local_project.user, local_project.name):
            # Purge caching
            ProjectManager.purge()
    except (PolyaxonHTTPError, PolyaxonShouldExitError) as e:
        Printer.print_error('Could not delete project `{}/{}`.'.format(user, project_name))
        Printer.print_error('Error message `{}`.'.format(e))
        sys.exit(1)

    if response.status_code == 204:
        Printer.print_success("Project `{}/{}` was delete successfully".format(user, project_name))

@project.command()
@click.option('--name', type=str,
              help='Name of the project, must be unique for the same user,')
@click.option('--description', type=str, help='Description of the project,')
@click.option('--private', type=bool, help='Set the visibility of the project to private/public.')
@click.pass_context
@clean_outputs
def update(ctx, name, description, private):
    """Update project.

    Uses [Caching](/polyaxon_cli/introduction#Caching)

    Example:

    \b
    ```bash
    $ polyaxon update foobar --description="Image Classification with DL using TensorFlow"
    ```

    \b
    ```bash
    $ polyaxon update mike1/foobar --description="Image Classification with DL using TensorFlow"
    ```
    """
    user, project_name = get_project_or_local(ctx.obj['project'])

    update_dict = {}
    if name:
        update_dict['name'] = name

    if description:
        update_dict['description'] = description

    if private is not None:
        update_dict['is_public'] = not private

    if not update_dict:
        Printer.print_warning('No argument was provided to update the project.')
        sys.exit(1)

    try:
        response = PolyaxonClients().project.update_project(user, project_name, update_dict)
    except (PolyaxonHTTPError, PolyaxonShouldExitError) as e:
        Printer.print_error('Could not update project `{}`.'.format(project_name))
        Printer.print_error('Error message `{}`.'.format(e))
        sys.exit(1)

    Printer.print_success("Project updated.")
    get_project_details(response)

@project.command()
@click.option('--page', type=int, help='To paginate through the list of groups.')
@click.pass_context
@clean_outputs
def groups(ctx, page):
    """List experiment groups for this project.

    Uses [Caching](/polyaxon_cli/introduction#Caching)
    """
    user, project_name = get_project_or_local(ctx.obj['project'])

    page = page or 1
    try:
        response = PolyaxonClients().project.list_experiment_groups(user, project_name, page=page)
    except (PolyaxonHTTPError, PolyaxonShouldExitError) as e:
        Printer.print_error(
            'Could not get experiment groups for project `{}`.'.format(project_name))
        Printer.print_error('Error message `{}`.'.format(e))
        sys.exit(1)

    meta = get_meta_response(response)
    if meta:
        Printer.print_header('Experiment groups for project `{}/{}`.'.format(user, project_name))
        Printer.print_header('Navigation:')
        dict_tabulate(meta)
    else:
        Printer.print_header('No experiment groups found for project `{}/{}`.'.format(
            user, project_name))

    objects = list_dicts_to_tabulate([o.to_light_dict(humanize_values=True)
                                      for o in response['results']])
    if objects:
        Printer.print_header("Experiment groups:")
        objects.pop('project', None)
        objects.pop('user', None)
        dict_tabulate(objects, is_list_dict=True)

@project.command()
@click.option('--page', type=int, help='To paginate through the list of experiments.')
@click.option('--metrics', '-m', is_flag=True, help='List experiments with their metrics.')
@click.pass_context
@clean_outputs
@clean_outputs
def experiments(ctx, page, metrics):
    """List experiments for this project.

    Uses [Caching](/polyaxon_cli/introduction#Caching)
    """
    user, project_name = get_project_or_local(ctx.obj['project'])

    page = page or 1
    try:
        response = PolyaxonClients().project.list_experiments(user, project_name, page=page)
    except (PolyaxonHTTPError, PolyaxonShouldExitError) as e:
        Printer.print_error('Could not get experiments for project `{}`.'.format(project_name))
        Printer.print_error('Error message `{}`.'.format(e))
        sys.exit(1)

    meta = get_meta_response(response)
    if meta:
        Printer.print_header('Experiments for project `{}/{}`.'.format(user, project_name))
        Printer.print_header('Navigation:')
        dict_tabulate(meta)
    else:
        Printer.print_header('No experiments found for project `{}/{}`.'.format(user, project_name))

    if metrics:
        objects = get_experiments_with_metrics(response)
    else:
        objects = [Printer.add_status_color(o.to_light_dict(humanize_values=True))
                   for o in response['results']]
    objects = list_dicts_to_tabulate(objects)
    if objects:
        Printer.print_header("Experiments:")
        objects.pop('project_name', None)
        dict_tabulate(objects, is_list_dict=True)

@project.command()
@click.option('--page', type=int, help='To paginate through the list of jobs.')
@click.pass_context
@clean_outputs
@clean_outputs
def jobs(ctx, page):
    """List jobs for this project.

    Uses [Caching](/polyaxon_cli/introduction#Caching)
    """
    user, project_name = get_project_or_local(ctx.obj['project'])

    page = page or 1
    try:
        response = PolyaxonClients().project.list_jobs(user, project_name, page=page)
    except (PolyaxonHTTPError, PolyaxonShouldExitError) as e:
        Printer.print_error('Could not get jobs for project `{}`.'.format(project_name))
        Printer.print_error('Error message `{}`.'.format(e))
        sys.exit(1)

    meta = get_meta_response(response)
    if meta:
        Printer.print_header('Jobs for project `{}/{}`.'.format(user, project_name))
        Printer.print_header('Navigation:')
        dict_tabulate(meta)
    else:
        Printer.print_header('No jobs found for project `{}/{}`.'.format(user, project_name))

    objects = [Printer.add_status_color(o.to_light_dict(humanize_values=True))
               for o in response['results']]
    objects = list_dicts_to_tabulate(objects)
    if objects:
        Printer.print_header("Jobs:")
        objects.pop('project_name', None)
        dict_tabulate(objects, is_list_dict=True)

@project.command()
@click.option('--page', type=int, help='To paginate through the list of build jobs.')
@click.pass_context
@clean_outputs
@clean_outputs
def builds(ctx, page):
    """List build jobs for this project.

    Uses [Caching](/polyaxon_cli/introduction#Caching)
    """
    user, project_name = get_project_or_local(ctx.obj['project'])

    page = page or 1
    try:
        response = PolyaxonClients().project.list_builds(user, project_name, page=page)
    except (PolyaxonHTTPError, PolyaxonShouldExitError) as e:
        Printer.print_error('Could not get builds for project `{}`.'.format(project_name))
        Printer.print_error('Error message `{}`.'.format(e))
        sys.exit(1)

    meta = get_meta_response(response)
    if meta:
        Printer.print_header('Builds for project `{}/{}`.'.format(user, project_name))
        Printer.print_header('Navigation:')
        dict_tabulate(meta)
    else:
        Printer.print_header('No builds found for project `{}/{}`.'.format(user, project_name))

    objects = [Printer.add_status_color(o.to_light_dict(humanize_values=True))
               for o in response['results']]
    objects = list_dicts_to_tabulate(objects)
    if objects:
        Printer.print_header("Builds:")
        objects.pop('project_name', None)
        dict_tabulate(objects, is_list_dict=True)

@project.command()
@click.pass_context
@clean_outputs
def clone(ctx):
    pass
<|endoftext|>


<commit_before>from __future__ import absolute_import

import codecs
import copy
import logging

from fluent.syntax import (
    ast,
    FluentParser,
    FluentSerializer
)

from pontoon.sync import SyncError
from pontoon.sync.formats.base import ParsedResource
from pontoon.sync.utils import create_parent_directory
from pontoon.sync.vcs.models import VCSTranslation

log = logging.getLogger(__name__)

parser = FluentParser()
serializer = FluentSerializer()
localizable_entries = (ast.Message, ast.Term)

class FTLEntity(VCSTranslation):
    """
    Represents entities in FTL (without its attributes).
    """
    def __init__(
        self, key, source_string, source_string_plural, strings, comments=None, order=None
    ):
        super(FTLEntity, self).__init__(
            key=key,
            source_string=source_string,
            source_string_plural=source_string_plural,
            strings=strings,
            comments=comments or [],
            fuzzy=False,
            order=order,
        )

    def __repr__(self):
        return '<FTLEntity {key}>'.format(key=self.key.encode('utf-8'))

class FTLResource(ParsedResource):
    def __init__(self, path, locale, source_resource=None):
        self.path = path
        self.locale = locale
        self.entities = {}
        self.source_resource = source_resource
        self.order = 0

        # Copy entities from the source_resource if it's available.
        if source_resource:
            for key, entity in source_resource.entities.items():
                self.entities[key] = FTLEntity(
                    entity.key,
                    '',
                    '',
                    {},
                    copy.copy(entity.comments),
                    entity.order
                )

        try:
            with codecs.open(path, 'r', 'utf-8') as resource:
                self.structure = parser.parse(resource.read())
        except IOError:
            # If the file doesn't exist, but we have a source resource,
            # we can keep going, we'll just not have any translations.
            if source_resource:
                return
            else:
                raise

        group_comment = []
        for obj in self.structure.body:
            if isinstance(obj, localizable_entries):
                key = obj.id.name

                # Bug 1521523: Term keys start with -
                if isinstance(obj, ast.Term):
                    key = '-' + key

                comment = [obj.comment.content] if obj.comment else []

                # Do not store comments in the string column
                obj.comment = None
                translation = serializer.serialize_entry(obj)

                self.entities[key] = FTLEntity(
                    key,
                    translation,
                    '',
                    {None: translation},
                    group_comment + comment,
                    self.order
                )
                self.order += 1

            elif isinstance(obj, ast.GroupComment):
                group_comment = [obj.content]

    @property
    def translations(self):
        return sorted(self.entities.values(), key=lambda e: e.order)

    def save(self, locale):
        """
        Load the source resource, modify it with changes made to this
        Resource instance, and save it over the locale-specific
        resource.
        """
        if not self.source_resource:
            raise SyncError(
                'Cannot save FTL resource {0}: No source resource given.'
                .format(self.path)
            )

        with codecs.open(self.source_resource.path, 'r', 'utf-8') as resource:
            structure = parser.parse(resource.read())

        entities = structure.body

        # Use list() to iterate over a copy, leaving original free to modify
        for obj in list(entities):
            if isinstance(obj, localizable_entries):
                index = entities.index(obj)
                entity = self.entities[obj.id.name]

                if entity.strings:
                    message = parser.parse_entry(entity.strings[None])
                    message.comment = obj.comment
                    entities[index] = message
                else:
                    del entities[index]

        create_parent_directory(self.path)

        with codecs.open(self.path, 'w+', 'utf-8') as f:
            log.debug('Saving file: %s', self.path)
            f.write(serializer.serialize(structure))

def parse(path, source_path=None, locale=None):
    if source_path is not None:
        source_resource = FTLResource(source_path, locale)
    else:
        source_resource = None
    return FTLResource(path, locale, source_resource)
<commit_msg>Fix bug 1521523: [FTL] Also hardcode - on serialization (#1176)<commit_after>from __future__ import absolute_import

import codecs
import copy
import logging

from fluent.syntax import (
    ast,
    FluentParser,
    FluentSerializer
)

from pontoon.sync import SyncError
from pontoon.sync.formats.base import ParsedResource
from pontoon.sync.utils import create_parent_directory
from pontoon.sync.vcs.models import VCSTranslation

log = logging.getLogger(__name__)

parser = FluentParser()
serializer = FluentSerializer()
localizable_entries = (ast.Message, ast.Term)

class FTLEntity(VCSTranslation):
    """
    Represents entities in FTL (without its attributes).
    """
    def __init__(
        self, key, source_string, source_string_plural, strings, comments=None, order=None
    ):
        super(FTLEntity, self).__init__(
            key=key,
            source_string=source_string,
            source_string_plural=source_string_plural,
            strings=strings,
            comments=comments or [],
            fuzzy=False,
            order=order,
        )

    def __repr__(self):
        return '<FTLEntity {key}>'.format(key=self.key.encode('utf-8'))

class FTLResource(ParsedResource):
    def __init__(self, path, locale, source_resource=None):
        self.path = path
        self.locale = locale
        self.entities = {}
        self.source_resource = source_resource
        self.order = 0

        # Copy entities from the source_resource if it's available.
        if source_resource:
            for key, entity in source_resource.entities.items():
                self.entities[key] = FTLEntity(
                    entity.key,
                    '',
                    '',
                    {},
                    copy.copy(entity.comments),
                    entity.order
                )

        try:
            with codecs.open(path, 'r', 'utf-8') as resource:
                self.structure = parser.parse(resource.read())
        except IOError:
            # If the file doesn't exist, but we have a source resource,
            # we can keep going, we'll just not have any translations.
            if source_resource:
                return
            else:
                raise

        group_comment = []
        for obj in self.structure.body:
            if isinstance(obj, localizable_entries):
                key = get_key(obj)
                comment = [obj.comment.content] if obj.comment else []

                # Do not store comments in the string column
                obj.comment = None
                translation = serializer.serialize_entry(obj)

                self.entities[key] = FTLEntity(
                    key,
                    translation,
                    '',
                    {None: translation},
                    group_comment + comment,
                    self.order
                )
                self.order += 1

            elif isinstance(obj, ast.GroupComment):
                group_comment = [obj.content]

    @property
    def translations(self):
        return sorted(self.entities.values(), key=lambda e: e.order)

    def save(self, locale):
        """
        Load the source resource, modify it with changes made to this
        Resource instance, and save it over the locale-specific
        resource.
        """
        if not self.source_resource:
            raise SyncError(
                'Cannot save FTL resource {0}: No source resource given.'
                .format(self.path)
            )

        with codecs.open(self.source_resource.path, 'r', 'utf-8') as resource:
            structure = parser.parse(resource.read())

        entities = structure.body

        # Use list() to iterate over a copy, leaving original free to modify
        for obj in list(entities):
            if isinstance(obj, localizable_entries):
                index = entities.index(obj)
                key = get_key(obj)
                entity = self.entities[key]

                if entity.strings:
                    message = parser.parse_entry(entity.strings[None])
                    message.comment = obj.comment
                    entities[index] = message
                else:
                    del entities[index]

        create_parent_directory(self.path)

        with codecs.open(self.path, 'w+', 'utf-8') as f:
            log.debug('Saving file: %s', self.path)
            f.write(serializer.serialize(structure))

def get_key(obj):
    """
    Get FTL Message/Term key as it appears in the file.
    In case of a Term, we need to prepend -. See bug 1521523.
    """
    key = obj.id.name

    if isinstance(obj, ast.Term):
        return '-' + key

    return key

def parse(path, source_path=None, locale=None):
    if source_path is not None:
        source_resource = FTLResource(source_path, locale)
    else:
        source_resource = None
    return FTLResource(path, locale, source_resource)
<|endoftext|>