<commit_before>import io
import json
import os
import tarfile
import uuid
from contextlib import contextmanager
from json import JSONDecodeError
from pathlib import Path
from random import randint
from time import sleep
from typing import Tuple

import docker
from django.conf import settings
from django.core.files import File
from docker.api.container import ContainerApiMixin
from docker.errors import ContainerError, APIError, NotFound
from docker.tls import TLSConfig
from docker.types import LogConfig
from requests import HTTPError

class DockerConnection:
    """
    Provides a client with a connection to a docker host, provisioned for
    running the container exec_image.
    """

    def __init__(
        self, *, job_id: uuid.UUID, exec_image: File, exec_image_sha256: str
    ):
        super().__init__()
        self._job_id = str(job_id)
        self._exec_image = exec_image
        self._exec_image_sha256 = exec_image_sha256

        client_kwargs = {"base_url": settings.CONTAINER_EXEC_DOCKER_BASE_URL}

        if settings.CONTAINER_EXEC_DOCKER_TLSVERIFY:
            tlsconfig = TLSConfig(
                verify=True,
                client_cert=(
                    settings.CONTAINER_EXEC_DOCKER_TLSCERT,
                    settings.CONTAINER_EXEC_DOCKER_TLSKEY,
                ),
                ca_cert=settings.CONTAINER_EXEC_DOCKER_TLSCACERT,
            )
            client_kwargs.update({"tls": tlsconfig})

        self._client = docker.DockerClient(**client_kwargs)

        self._run_kwargs = {
            "labels": {"job_id": self._job_id},
            "init": True,
            "network_disabled": True,
            "mem_limit": settings.CONTAINER_EXEC_MEMORY_LIMIT,
            # Set to the same as mem_limit to avoid using swap
            "memswap_limit": settings.CONTAINER_EXEC_MEMORY_LIMIT,
            "cpu_period": settings.CONTAINER_EXEC_CPU_PERIOD,
            "cpu_quota": settings.CONTAINER_EXEC_CPU_QUOTA,
            # Use the default weight
            "cpu_shares": 1024,
            "runtime": settings.CONTAINER_EXEC_DOCKER_RUNTIME,
            "cap_drop": ["all"],
            "security_opt": ["no-new-privileges"],
            "pids_limit": 64,
            "log_config": LogConfig(
                type=LogConfig.types.JSON, config={"max-size": "1g"}
            ),
        }

    @staticmethod
    def __retry_docker_obj_prune(*, obj, filters: dict):
        # Retry and exponential backoff of the prune command as only 1 prune
        # operation can occur at a time on a docker host
        num_retries = 0
        e = APIError
        while num_retries < 3:
            try:
                obj.prune(filters=filters)
                break
            except (APIError, HTTPError) as e:
                num_retries += 1
                sleep((2 ** num_retries) + (randint(0, 1000) / 1000))
        else:
            raise e

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        flt = {"label": f"job_id={self._job_id}"}

        try:
            for c in self._client.containers.list(filters=flt):
                c.stop()

            self.__retry_docker_obj_prune(
                obj=self._client.containers, filters=flt
            )
            self.__retry_docker_obj_prune(
                obj=self._client.volumes, filters=flt
            )
        except ConnectionError:
            raise RuntimeError("Could not connect to worker.")

    def _pull_images(self):
        if self._exec_image_sha256 not in [
            img.id for img in self._client.images.list()
        ]:
            with self._exec_image.open("rb") as f:
                self._client.images.load(f)

class Executor(DockerConnection):
    def __init__(
        self,
        *args,
        input_files: Tuple[File, ...],
        results_file: Path,
        **kwargs,
    ):
        super().__init__(*args, **kwargs)
        self._input_files = input_files
        self._results_file = results_file
        self._io_image = settings.CONTAINER_EXEC_IO_IMAGE

        self._input_volume = f"{self._job_id}-input"
        self._output_volume = f"{self._job_id}-output"

    def execute(self) -> dict:
        self._pull_images()
        self._create_io_volumes()
        self._provision_input_volume()
        self._chmod_volumes()
        self._execute_container()
        return self._get_result()

    def _pull_images(self):
        super()._pull_images()
        self._client.images.pull(repository=self._io_image)

    def _create_io_volumes(self):
        for volume in [self._input_volume, self._output_volume]:
            self._client.volumes.create(
                name=volume, labels=self._run_kwargs["labels"]
            )

    def _provision_input_volume(self):
        try:
            with cleanup(
                self._client.containers.run(
                    image=self._io_image,
                    volumes={
                        self._input_volume: {"bind": "/input/", "mode": "rw"}
                    },
                    name=f"{self._job_id}-writer",
                    detach=True,
                    tty=True,
                    **self._run_kwargs,
                )
            ) as writer:
                self._copy_input_files(writer=writer)
        except Exception as exc:
            raise RuntimeError(str(exc))

    def _copy_input_files(self, writer):
        for file in self._input_files:
            put_file(
                container=writer,
                src=file,
                dest=f"/input/{Path(file.name).name}",
            )

    def _chmod_volumes(self):
        """ Ensure that the i/o directories are writable """
        try:
            self._client.containers.run(
                image=self._io_image,
                volumes={
                    self._input_volume: {"bind": "/input/", "mode": "rw"},
                    self._output_volume: {"bind": "/output/", "mode": "rw"},
                },
                name=f"{self._job_id}-chmod-volumes",
                command=f"chmod -R 0777 /input/ /output/",
                remove=True,
                **self._run_kwargs,
            )
        except Exception as exc:
            raise RuntimeError(str(exc))

    def _execute_container(self):
        try:
            self._client.containers.run(
                image=self._exec_image_sha256,
                volumes={
                    self._input_volume: {"bind": "/input/", "mode": "rw"},
                    self._output_volume: {"bind": "/output/", "mode": "rw"},
                },
                name=f"{self._job_id}-executor",
                remove=True,
                **self._run_kwargs,
            )
        except ContainerError as exc:
            raise RuntimeError(exc.stderr.decode())

    def _get_result(self) -> dict:
        """
        Read and parse the created results file. Due to a bug in the docker
        client, copy the file to memory first rather than cat and read
        stdout.
        """
        try:
            with cleanup(
                self._client.containers.run(
                    image=self._io_image,
                    volumes={
                        self._output_volume: {"bind": "/output/", "mode": "ro"}
                    },
                    name=f"{self._job_id}-reader",
                    detach=True,
                    tty=True,
                    **self._run_kwargs,
                )
            ) as reader:
                result = get_file(container=reader, src=self._results_file)
        except NotFound:
            # The container exited without error, but no results file was
            # produced. This shouldn't happen, but does with poorly programmed
            # evaluation containers.
            raise RuntimeError(
                "The evaluation failed for an unknown reason as no results "
                "file was produced. Please contact the organisers for "
                "assistance."
            )
        except Exception as e:
            raise RuntimeError(str(e))

        try:
            result = json.loads(
                result.read().decode(),
                parse_constant=lambda x: None,  # Removes -inf, inf and NaN
            )
        except JSONDecodeError as exc:
            raise RuntimeError(exc.msg)

        return result

@contextmanager
def cleanup(container: ContainerApiMixin):
    """
    Cleans up a docker container which is running in detached mode

    :param container: An instance of a container
    :return:
    """
    try:
        yield container

    finally:
        container.stop()
        container.remove(force=True)

def put_file(*, container: ContainerApiMixin, src: File, dest: str) -> ():
    """
    Puts a file on the host into a container.
    This method will create an in memory tar archive, add the src file to this
    and upload it to the docker container where it will be unarchived at dest.

    :param container: The container to write to
    :param src: The path to the source file on the host
    :param dest: The path to the target file in the container
    :return:
    """
    tar_b = io.BytesIO()

    tarinfo = tarfile.TarInfo(name=os.path.basename(dest))
    tarinfo.size = src.size

    with tarfile.open(fileobj=tar_b, mode="w") as tar, src.open("rb") as f:
        tar.addfile(tarinfo, fileobj=f)

    tar_b.seek(0)
    container.put_archive(os.path.dirname(dest), tar_b)

def get_file(*, container: ContainerApiMixin, src: Path):
    tarstrm, info = container.get_archive(src)

    if info["size"] > 2E9:
        raise ValueError(f"File {src} is too big to be decompressed.")

    file_obj = io.BytesIO()
    for ts in tarstrm:
        file_obj.write(ts)

    file_obj.seek(0)
    tar = tarfile.open(mode="r", fileobj=file_obj)
    content = tar.extractfile(src.name)

    return content
<commit_msg>Adds `cleanup()`<commit_after>import io
import json
import os
import tarfile
import uuid
from contextlib import contextmanager
from json import JSONDecodeError
from pathlib import Path
from random import randint
from time import sleep
from typing import Tuple

import docker
from django.conf import settings
from django.core.files import File
from docker.api.container import ContainerApiMixin
from docker.errors import ContainerError, APIError, NotFound
from docker.tls import TLSConfig
from docker.types import LogConfig
from requests import HTTPError

class DockerConnection:
    """
    Provides a client with a connection to a docker host, provisioned for
    running the container exec_image.
    """

    def __init__(
        self, *, job_id: uuid.UUID, exec_image: File, exec_image_sha256: str
    ):
        super().__init__()
        self._job_id = str(job_id)
        self._exec_image = exec_image
        self._exec_image_sha256 = exec_image_sha256

        client_kwargs = {"base_url": settings.CONTAINER_EXEC_DOCKER_BASE_URL}

        if settings.CONTAINER_EXEC_DOCKER_TLSVERIFY:
            tlsconfig = TLSConfig(
                verify=True,
                client_cert=(
                    settings.CONTAINER_EXEC_DOCKER_TLSCERT,
                    settings.CONTAINER_EXEC_DOCKER_TLSKEY,
                ),
                ca_cert=settings.CONTAINER_EXEC_DOCKER_TLSCACERT,
            )
            client_kwargs.update({"tls": tlsconfig})

        self._client = docker.DockerClient(**client_kwargs)

        self._run_kwargs = {
            "labels": {"job_id": self._job_id},
            "init": True,
            "network_disabled": True,
            "mem_limit": settings.CONTAINER_EXEC_MEMORY_LIMIT,
            # Set to the same as mem_limit to avoid using swap
            "memswap_limit": settings.CONTAINER_EXEC_MEMORY_LIMIT,
            "cpu_period": settings.CONTAINER_EXEC_CPU_PERIOD,
            "cpu_quota": settings.CONTAINER_EXEC_CPU_QUOTA,
            # Use the default weight
            "cpu_shares": 1024,
            "runtime": settings.CONTAINER_EXEC_DOCKER_RUNTIME,
            "cap_drop": ["all"],
            "security_opt": ["no-new-privileges"],
            "pids_limit": 64,
            "log_config": LogConfig(
                type=LogConfig.types.JSON, config={"max-size": "1g"}
            ),
        }

    @staticmethod
    def __retry_docker_obj_prune(*, obj, filters: dict):
        # Retry and exponential backoff of the prune command as only 1 prune
        # operation can occur at a time on a docker host
        num_retries = 0
        e = APIError
        while num_retries < 3:
            try:
                obj.prune(filters=filters)
                break
            except (APIError, HTTPError) as e:
                num_retries += 1
                sleep((2 ** num_retries) + (randint(0, 1000) / 1000))
        else:
            raise e

    def cleanup(self):
        """
        Stops and prunes all containers and volumes associated with this job
        """
        flt = {"label": f"job_id={self._job_id}"}

        try:
            for c in self._client.containers.list(filters=flt):
                c.stop()

            self.__retry_docker_obj_prune(
                obj=self._client.containers, filters=flt
            )
            self.__retry_docker_obj_prune(
                obj=self._client.volumes, filters=flt
            )
        except ConnectionError:
            raise RuntimeError("Could not connect to worker.")

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.cleanup()

    def _pull_images(self):
        if self._exec_image_sha256 not in [
            img.id for img in self._client.images.list()
        ]:
            with self._exec_image.open("rb") as f:
                self._client.images.load(f)

class Executor(DockerConnection):
    def __init__(
        self,
        *args,
        input_files: Tuple[File, ...],
        results_file: Path,
        **kwargs,
    ):
        super().__init__(*args, **kwargs)
        self._input_files = input_files
        self._results_file = results_file
        self._io_image = settings.CONTAINER_EXEC_IO_IMAGE

        self._input_volume = f"{self._job_id}-input"
        self._output_volume = f"{self._job_id}-output"

    def execute(self) -> dict:
        self._pull_images()
        self._create_io_volumes()
        self._provision_input_volume()
        self._chmod_volumes()
        self._execute_container()
        return self._get_result()

    def _pull_images(self):
        super()._pull_images()
        self._client.images.pull(repository=self._io_image)

    def _create_io_volumes(self):
        for volume in [self._input_volume, self._output_volume]:
            self._client.volumes.create(
                name=volume, labels=self._run_kwargs["labels"]
            )

    def _provision_input_volume(self):
        try:
            with cleanup(
                self._client.containers.run(
                    image=self._io_image,
                    volumes={
                        self._input_volume: {"bind": "/input/", "mode": "rw"}
                    },
                    name=f"{self._job_id}-writer",
                    detach=True,
                    tty=True,
                    **self._run_kwargs,
                )
            ) as writer:
                self._copy_input_files(writer=writer)
        except Exception as exc:
            raise RuntimeError(str(exc))

    def _copy_input_files(self, writer):
        for file in self._input_files:
            put_file(
                container=writer,
                src=file,
                dest=f"/input/{Path(file.name).name}",
            )

    def _chmod_volumes(self):
        """ Ensure that the i/o directories are writable """
        try:
            self._client.containers.run(
                image=self._io_image,
                volumes={
                    self._input_volume: {"bind": "/input/", "mode": "rw"},
                    self._output_volume: {"bind": "/output/", "mode": "rw"},
                },
                name=f"{self._job_id}-chmod-volumes",
                command=f"chmod -R 0777 /input/ /output/",
                remove=True,
                **self._run_kwargs,
            )
        except Exception as exc:
            raise RuntimeError(str(exc))

    def _execute_container(self):
        try:
            self._client.containers.run(
                image=self._exec_image_sha256,
                volumes={
                    self._input_volume: {"bind": "/input/", "mode": "rw"},
                    self._output_volume: {"bind": "/output/", "mode": "rw"},
                },
                name=f"{self._job_id}-executor",
                remove=True,
                **self._run_kwargs,
            )
        except ContainerError as exc:
            raise RuntimeError(exc.stderr.decode())

    def _get_result(self) -> dict:
        """
        Read and parse the created results file. Due to a bug in the docker
        client, copy the file to memory first rather than cat and read
        stdout.
        """
        try:
            with cleanup(
                self._client.containers.run(
                    image=self._io_image,
                    volumes={
                        self._output_volume: {"bind": "/output/", "mode": "ro"}
                    },
                    name=f"{self._job_id}-reader",
                    detach=True,
                    tty=True,
                    **self._run_kwargs,
                )
            ) as reader:
                result = get_file(container=reader, src=self._results_file)
        except NotFound:
            # The container exited without error, but no results file was
            # produced. This shouldn't happen, but does with poorly programmed
            # evaluation containers.
            raise RuntimeError(
                "The evaluation failed for an unknown reason as no results "
                "file was produced. Please contact the organisers for "
                "assistance."
            )
        except Exception as e:
            raise RuntimeError(str(e))

        try:
            result = json.loads(
                result.read().decode(),
                parse_constant=lambda x: None,  # Removes -inf, inf and NaN
            )
        except JSONDecodeError as exc:
            raise RuntimeError(exc.msg)

        return result

@contextmanager
def cleanup(container: ContainerApiMixin):
    """
    Cleans up a docker container which is running in detached mode

    :param container: An instance of a container
    :return:
    """
    try:
        yield container

    finally:
        container.stop()
        container.remove(force=True)

def put_file(*, container: ContainerApiMixin, src: File, dest: str) -> ():
    """
    Puts a file on the host into a container.
    This method will create an in memory tar archive, add the src file to this
    and upload it to the docker container where it will be unarchived at dest.

    :param container: The container to write to
    :param src: The path to the source file on the host
    :param dest: The path to the target file in the container
    :return:
    """
    tar_b = io.BytesIO()

    tarinfo = tarfile.TarInfo(name=os.path.basename(dest))
    tarinfo.size = src.size

    with tarfile.open(fileobj=tar_b, mode="w") as tar, src.open("rb") as f:
        tar.addfile(tarinfo, fileobj=f)

    tar_b.seek(0)
    container.put_archive(os.path.dirname(dest), tar_b)

def get_file(*, container: ContainerApiMixin, src: Path):
    tarstrm, info = container.get_archive(src)

    if info["size"] > 2E9:
        raise ValueError(f"File {src} is too big to be decompressed.")

    file_obj = io.BytesIO()
    for ts in tarstrm:
        file_obj.write(ts)

    file_obj.seek(0)
    tar = tarfile.open(mode="r", fileobj=file_obj)
    content = tar.extractfile(src.name)

    return content
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""Generate static documentation."""

import os
import re

folder = os.path.join(os.path.dirname(os.getcwd()), 'site', 'doc')
loader = """
import time
from browser import doc, window, markdown
import header
header.show('../../')

def load():
    target = 'content'
    url = doc.query.getfirst('page', 'intro') + '.md'

    # fake query string to bypass browser cache
    qs = '?foo=%s' % time.time()
    try:
        src = open(url + qs).read()
    except IOError:
        doc[target].html = "Page %s not found" % url
        return False

    mk, scripts = markdown.mark(src)
    doc[target].html = mk
    for script in scripts:
        exec(script, globals())

language = "fr"

load()
"""

def repl(mo):
    return 'href="?page=%s">' % mo.groups()[0].replace('/', '%2F')

for lang in ['fr', 'en', 'es', 'pt']:
    with open(os.path.join(folder, lang, 'index.html')) as index_html_file:
        dyn = index_html_file.read()
    with open(os.path.join(folder, lang, 'static_index.html'), 'wb') as static:
        pattern = 'href="#" onClick="load\(\'(.+)\.md\',\'content\'\)">'
        res = re.sub(pattern, repl, dyn)
        script_pattern = '<script type="text/python3?">.*?</script>'
        res = re.sub(script_pattern,
                     '<script type="text/python3">%s</script>' % loader, res,
                     flags=re.S)
        res = res.replace(
            '<body onLoad="brython(1);load(\'intro.md\',\'content\')">',
            '<body onLoad="brython(1)">')
        static.write(res)
<commit_msg>Add Multiprocessing to make_static_doc.py if run standalone<commit_after># -*- coding: utf-8 -*-

"""Generate static documentation."""

import os
import re
from multiprocessing import Pool

list_of_languages = ('fr', 'en', 'es', 'pt')
folder = os.path.join(os.path.dirname(os.getcwd()), 'site', 'doc')
loader = """
import time
from browser import doc, window, markdown
import header
header.show('../../')

def load():
    target = 'content'
    url = doc.query.getfirst('page', 'intro') + '.md'

    # fake query string to bypass browser cache
    qs = '?foo=%s' % time.time()
    try:
        src = open(url + qs).read()
    except IOError:
        doc[target].html = "Page %s not found" % url
        return False

    mk, scripts = markdown.mark(src)
    doc[target].html = mk
    for script in scripts:
        exec(script, globals())

language = "fr"

load()
"""

def repl(mo):
    return 'href="?page=%s">' % mo.groups()[0].replace('/', '%2F')

def build_docs(lang):
    print("Process {} finished writing Docs on file: {}.".format(
        os.getpid(), os.path.join(folder, lang, 'static_index.html')))
    with open(os.path.join(folder, lang, 'index.html')) as index_html_file:
        dyn = index_html_file.read()
    with open(os.path.join(folder, lang, 'static_index.html'), 'wb') as static:
        pattern = 'href="#" onClick="load\(\'(.+)\.md\',\'content\'\)">'
        res = re.sub(pattern, repl, dyn)
        script_pattern = '<script type="text/python3?">.*?</script>'
        res = re.sub(script_pattern, '<script type="text/python3">%s</script>'
                     % loader, res, flags=re.S)
        res = res.replace(
            '<body onload="brython(1);load(\'intro.md\',\'content\')">',
            '<body onload="brython(1)">')
        static.write(res)
        return True

if __name__ in '__main__':
    Pool(len(list_of_languages)).map(build_docs, list_of_languages)
else:
    map(build_docs, list_of_languages)  # No multiprocess, its module
<|endoftext|>


<commit_before>import argparse
import logging
import os

import pandoc
from xdg import xdg_cache_home, xdg_config_home

from .editor import Editor
from .googleapi import GoogleApiService
from .pandoc import pandoc_to_task_lists, task_lists_to_pandoc

def main():
    args = parse_args()

    config_dir = f"{xdg_config_home()}/gtasks-md/{args.user}/"
    os.makedirs(os.path.dirname(config_dir), exist_ok=True)
    cache_dir = f"{xdg_cache_home()}/gtasks-md/{args.user}/"
    os.makedirs(os.path.dirname(cache_dir), exist_ok=True)

    logging.basicConfig(
        filename=f"{xdg_cache_home()}/gtasks-md/log.txt",
        format="%(asctime)s %(levelname)-8s %(message)s",
        encoding="utf-8",
        level=logging.INFO,
        datefmt="%Y-%m-%d %H:%M:%S",
    )

    match args.subcommand:
        case "auth":
            service = GoogleApiService(args.user)
            auth(service, args.credentials_file)
        case "edit":
            service = GoogleApiService(args.user)
            editor = Editor(args.editor)
            edit(service, editor)
        case "view":
            service = GoogleApiService(args.user)
            view(service)
        case None:
            print("Please run one of the subcommands.")

def parse_args():
    # https://stackoverflow.com/a/49977713
    parser = argparse.ArgumentParser(description="Google Tasks declarative management.")
    parser.add_argument(
        "--user",
        dest="user",
        default="default",
        help="Account for which the credentials are sourced. Should match desired Google account.",
        type=str,
    )

    subparsers = parser.add_subparsers(dest="subcommand")

    auth = subparsers.add_parser("auth", help="Authorize.")
    auth.add_argument(
        "credentials_file",
        help="Location of credential file.",
        type=str,
    )

    subparsers.add_parser("view", help="View Google Tasks.")

    edit_parser = subparsers.add_parser("edit", help="Edit Google Tasks.")
    edit_parser.add_argument(
        "--editor",
        dest="editor",
        default="",
        help="Editor to be used for editing tasks. Defaults to $EDITOR and then $VISUAL and then vim.",
        type=str,
    )

    return parser.parse_args()

def auth(service: GoogleApiService, file: str):
    with open(file, "r") as src_file:
        service.save_credentials(src_file.read())

def view(service: GoogleApiService):
    _, text = fetch_task_lists(service)
    print(text)

def edit(service: GoogleApiService, editor: Editor):
    old_task_lists, old_text = fetch_task_lists(service)
    new_text = editor.edit(old_text)
    print(new_text)
    new_task_lists = pandoc_to_task_lists(pandoc.read(new_text))
    service.reconcile(old_task_lists, new_task_lists)

def fetch_task_lists(service: GoogleApiService):
    task_lists = service.get_task_lists()
    doc = task_lists_to_pandoc(task_lists)
    return task_lists, pandoc.write(doc)

if __name__ == "__main__":
    main()
<commit_msg>Add reconcile subcommand<commit_after>import argparse
import logging
import os

import pandoc
from xdg import xdg_cache_home, xdg_config_home

from .editor import Editor
from .googleapi import GoogleApiService
from .pandoc import pandoc_to_task_lists, task_lists_to_pandoc

def main():
    args = parse_args()

    config_dir = f"{xdg_config_home()}/gtasks-md/{args.user}/"
    os.makedirs(os.path.dirname(config_dir), exist_ok=True)
    cache_dir = f"{xdg_cache_home()}/gtasks-md/{args.user}/"
    os.makedirs(os.path.dirname(cache_dir), exist_ok=True)

    logging.basicConfig(
        filename=f"{xdg_cache_home()}/gtasks-md/log.txt",
        format="%(asctime)s %(levelname)-8s %(message)s",
        encoding="utf-8",
        level=logging.INFO,
        datefmt="%Y-%m-%d %H:%M:%S",
    )

    match args.subcommand:
        case "auth":
            service = GoogleApiService(args.user)
            auth(service, args.credentials_file)
        case "edit":
            service = GoogleApiService(args.user)
            editor = Editor(args.editor)
            edit(service, editor)
        case "reconcile":
            service = GoogleApiService(args.user)
            reconcile(service, args.file_path)
        case "view":
            service = GoogleApiService(args.user)
            view(service)
        case None:
            print("Please run one of the subcommands.")

def parse_args():
    # https://stackoverflow.com/a/49977713
    parser = argparse.ArgumentParser(description="Google Tasks declarative management.")
    parser.add_argument(
        "--user",
        dest="user",
        default="default",
        help="Account for which the credentials are sourced. Should match desired Google account.",
        type=str,
    )

    subparsers = parser.add_subparsers(dest="subcommand")

    auth = subparsers.add_parser("auth", help="Authorize.")
    auth.add_argument(
        "credentials_file",
        help="Location of credential file.",
        type=str,
    )

    subparsers.add_parser("view", help="View Google Tasks.")

    edit_parser = subparsers.add_parser("edit", help="Edit Google Tasks.")
    edit_parser.add_argument(
        "--editor",
        dest="editor",
        default="",
        help="Editor to be used for editing tasks. Defaults to $EDITOR and then $VISUAL and then vim.",
        type=str,
    )

    reconcile_parser = subparsers.add_parser(
        "reconcile", help="Patch tasklists with an offline source."
    )
    reconcile_parser.add_argument(
        "file_path",
        help="Location of the source file.",
        type=str,
    )

    return parser.parse_args()

def auth(service: GoogleApiService, file: str):
    with open(file, "r") as src_file:
        service.save_credentials(src_file.read())

def view(service: GoogleApiService):
    _, text = fetch_task_lists(service)
    print(text)

def edit(service: GoogleApiService, editor: Editor):
    old_task_lists, old_text = fetch_task_lists(service)
    new_text = editor.edit(old_text)
    new_task_lists = pandoc_to_task_lists(pandoc.read(new_text))
    service.reconcile(old_task_lists, new_task_lists)

def reconcile(service: GoogleApiService, file_path: str):
    old_task_lists, _ = fetch_task_lists(service)
    with open(file_path, "r") as source:
        new_text = source.read()
        new_task_lists = pandoc_to_task_lists(pandoc.read(new_text))
        service.reconcile(old_task_lists, new_task_lists)

def fetch_task_lists(service: GoogleApiService):
    task_lists = service.get_task_lists()
    doc = task_lists_to_pandoc(task_lists)
    return task_lists, pandoc.write(doc)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!flask/bin/python
from app import app
app.run(debug=True)<commit_msg>changed path<commit_after>#!env/bin/python
from app import app
app.run(debug=True)<|endoftext|>


<commit_before>import numpy as np
import pdb
import math
import data_generators
import copy

def calc_iou(R, img_data, C, class_mapping):

	bboxes = img_data['bboxes']
	(width, height) = (img_data['width'], img_data['height'])
	# get image dimensions for resizing
	(resized_width, resized_height) = data_generators.get_new_img_size(width, height, C.im_size)

	gta = np.zeros((len(bboxes), 4))

	for bbox_num, bbox in enumerate(bboxes):
		# get the GT box coordinates, and resize to account for image resizing
		gta[bbox_num, 0] = int(round(bbox['x1'] * (resized_width / float(width))/C.rpn_stride))
		gta[bbox_num, 1] = int(round(bbox['x2'] * (resized_width / float(width))/C.rpn_stride))
		gta[bbox_num, 2] = int(round(bbox['y1'] * (resized_height / float(height))/C.rpn_stride))
		gta[bbox_num, 3] = int(round(bbox['y2'] * (resized_height / float(height))/C.rpn_stride))

	x_roi = []
	y_class_num = []
	y_class_regr_coords = []
	y_class_regr_label = []

	for ix in range(R.shape[0]):
		(x1, y1, x2, y2) = R[ix, :]
		x1 = int(round(x1))
		y1 = int(round(y1))
		x2 = int(round(x2))
		y2 = int(round(y2))

		best_iou = 0.0
		best_bbox = -1
		for bbox_num in range(len(bboxes)):
			curr_iou = data_generators.iou([gta[bbox_num, 0], gta[bbox_num, 2], gta[bbox_num, 1], gta[bbox_num, 3]], [x1, y1, x2, y2])
			if curr_iou > best_iou:
				best_iou = curr_iou
				best_bbox = bbox_num

		if best_iou < C.classifier_min_overlap:
				continue
		else:
			w = x2 - x1
			h = y2 - y1
			x_roi.append([x1, y1, w, h])

			if C.classifier_min_overlap <= best_iou < C.classifier_max_overlap:
				# hard negative example
				cls_name = 'bg'
			elif C.classifier_max_overlap <= best_iou:
				cls_name = bboxes[best_bbox]['class']
				cxg = (gta[best_bbox, 0] + gta[best_bbox, 1]) / 2.0
				cyg = (gta[best_bbox, 2] + gta[best_bbox, 3]) / 2.0

				cx = x1 + w / 2.0
				cy = y1 + h / 2.0

				tx = (cxg - cx) / float(w)
				ty = (cyg - cy) / float(h)
				tw = np.log((gta[best_bbox, 1] - gta[best_bbox, 0]) / float(w))
				th = np.log((gta[best_bbox, 3] - gta[best_bbox, 2]) / float(h))
			else:
				print('roi = {}'.format(best_iou))
				raise RuntimeError

		class_num = class_mapping[cls_name]
		class_label = len(class_mapping) * [0]
		class_label[class_num] = 1
		y_class_num.append(copy.deepcopy(class_label))
		coords = [0] * 4 * (len(class_mapping) - 1)
		labels = [0] * 4 * (len(class_mapping) - 1)
		if cls_name != 'bg':
			label_pos = 4 * class_num
			sx, sy, sw, sh = C.classifier_regr_std
			coords[label_pos:4+label_pos] = [sx*tx, sy*ty, sw*tw, sh*th]
			labels[label_pos:4+label_pos] = [1, 1, 1, 1]
			y_class_regr_coords.append(copy.deepcopy(coords))
			y_class_regr_label.append(copy.deepcopy(labels))
		else:
			y_class_regr_coords.append(copy.deepcopy(coords))
			y_class_regr_label.append(copy.deepcopy(labels))

	if len(x_roi) == 0:
		return None, None, None

	X = np.array(x_roi)
	Y1 = np.array(y_class_num)
	Y2 = np.concatenate([np.array(y_class_regr_label),np.array(y_class_regr_coords)],axis=1)

	return np.expand_dims(X, axis=0), np.expand_dims(Y1, axis=0), np.expand_dims(Y2, axis=0)

def apply_regr(x, y, w, h, tx, ty, tw, th):
	try:
		cx = x + w/2.
		cy = y + h/2.
		cx1 = tx * w + cx
		cy1 = ty * h + cy
		w1 = math.exp(tw) * w
		h1 = math.exp(th) * h
		x1 = cx1 - w1/2.
		y1 = cy1 - h1/2.
		x1 = int(round(x1))
		y1 = int(round(y1))
		w1 = int(round(w1))
		h1 = int(round(h1))

		return x1, y1, w1, h1

	except ValueError:
		return x, y, w, h
	except OverflowError:
		return x, y, w, h
	except Exception as e:
		print(e)
		return x, y, w, h

def apply_regr_np(X, T):
	try:
		x = X[0, :, :]
		y = X[1, :, :]
		w = X[2, :, :]
		h = X[3, :, :]

		tx = T[0, :, :]
		ty = T[1, :, :]
		tw = T[2, :, :]
		th = T[3, :, :]

		cx = x + w/2.
		cy = y + h/2.
		cx1 = tx * w + cx
		cy1 = ty * h + cy

		w1 = np.exp(tw) * w
		h1 = np.exp(th) * h
		x1 = cx1 - w1/2.
		y1 = cy1 - h1/2.

		x1 = np.round(x1)
		y1 = np.round(y1)
		w1 = np.round(w1)
		h1 = np.round(h1)
		return np.stack([x1, y1, w1, h1])
	except Exception as e:
		print(e)
		return X

def non_max_suppression_fast(boxes, probs, overlap_thresh=0.9, max_boxes=300):
	# code used from here: http://www.pyimagesearch.com/2015/02/16/faster-non-maximum-suppression-python/
	# if there are no boxes, return an empty list
	if len(boxes) == 0:
		return []

	# grab the coordinates of the bounding boxes
	x1 = boxes[:, 0]
	y1 = boxes[:, 1]
	x2 = boxes[:, 2]
	y2 = boxes[:, 3]

	np.testing.assert_array_less(x1, x2)
	np.testing.assert_array_less(y1, y2)

	# if the bounding boxes integers, convert them to floats --
	# this is important since we'll be doing a bunch of divisions
	if boxes.dtype.kind == "i":
		boxes = boxes.astype("float")

	# initialize the list of picked indexes
	pick = []

	# sort the bounding boxes
	idxs = np.argsort(probs)

	# keep looping while some indexes still remain in the indexes
	# list
	while len(idxs) > 0:
		# grab the last index in the indexes list and add the
		# index value to the list of picked indexes
		last = len(idxs) - 1
		i = idxs[last]
		pick.append(i)

		# find the intersection

		xx1_int = np.maximum(x1[i], x1[idxs[:last]])
		yy1_int = np.maximum(y1[i], y1[idxs[:last]])
		xx2_int = np.minimum(x2[i], x2[idxs[:last]])
		yy2_int = np.minimum(y2[i], y2[idxs[:last]])

		# find the union
		xx1_un = np.minimum(x1[i], x1[idxs[:last]])
		yy1_un = np.minimum(y1[i], y1[idxs[:last]])
		xx2_un = np.maximum(x2[i], x2[idxs[:last]])
		yy2_un = np.maximum(y2[i], y2[idxs[:last]])

		# compute the width and height of the bounding box
		ww_int = xx2_int - xx1_int
		hh_int = yy2_int - yy1_int

		ww_un = xx2_un - xx1_un
		hh_un = yy2_un - yy1_un

		ww_un = np.maximum(0, ww_un)
		hh_un = np.maximum(0, hh_un)

		# compute the ratio of overlap
		overlap = (ww_int*hh_int)/(ww_un*hh_un + 1e-9)

		# delete all indexes from the index list that have
		idxs = np.delete(idxs, np.concatenate(([last],
			np.where(overlap > overlap_thresh)[0])))

		if len(pick) >= max_boxes:
			break

	# return only the bounding boxes that were picked using the integer data type
	boxes = boxes[pick].astype("int")
	probs = probs[pick]
	return boxes, probs

import time
def rpn_to_roi(rpn_layer, regr_layer, C, dim_ordering, use_regr=True, max_boxes=300,overlap_thresh=0.9):

	regr_layer = regr_layer / C.std_scaling

	anchor_sizes = C.anchor_box_scales
	anchor_ratios = C.anchor_box_ratios

	assert rpn_layer.shape[0] == 1

	if dim_ordering == 'th':
		(rows,cols) = rpn_layer.shape[2:]

	elif dim_ordering == 'tf':
		(rows, cols) = rpn_layer.shape[1:3]

	curr_layer = 0
	if dim_ordering == 'tf':
		A = np.zeros((4, rpn_layer.shape[1], rpn_layer.shape[2], rpn_layer.shape[3]))
	elif dim_ordering == 'th':
		A = np.zeros((4, rpn_layer.shape[2], rpn_layer.shape[3], rpn_layer.shape[1]))

	for anchor_size in anchor_sizes:
		for anchor_ratio in anchor_ratios:

			anchor_x = (anchor_size * anchor_ratio[0])/C.rpn_stride
			anchor_y = (anchor_size * anchor_ratio[1])/C.rpn_stride
			if dim_ordering == 'th':
				regr = regr_layer[0, 4 * curr_layer:4 * curr_layer + 4, :, :]
			else:
				regr = regr_layer[0, :, :, 4 * curr_layer:4 * curr_layer + 4]
				regr = np.transpose(regr, (2, 0, 1))

			X, Y = np.meshgrid(np.arange(cols),np. arange(rows))

			A[0, :, :, curr_layer] = X - anchor_x/2
			A[1, :, :, curr_layer] = Y - anchor_y/2
			A[2, :, :, curr_layer] = anchor_x
			A[3, :, :, curr_layer] = anchor_y

			if use_regr:
				A[:, :, :, curr_layer] = apply_regr_np(A[:, :, :, curr_layer], regr)

			A[2, :, :, curr_layer] = np.maximum(1, A[2, :, :, curr_layer])
			A[3, :, :, curr_layer] = np.maximum(1, A[3, :, :, curr_layer])
			A[2, :, :, curr_layer] += A[0, :, :, curr_layer]
			A[3, :, :, curr_layer] += A[1, :, :, curr_layer]

			A[0, :, :, curr_layer] = np.maximum(0, A[0, :, :, curr_layer])
			A[1, :, :, curr_layer] = np.maximum(0, A[1, :, :, curr_layer])
			A[2, :, :, curr_layer] = np.minimum(cols-1, A[2, :, :, curr_layer])
			A[3, :, :, curr_layer] = np.minimum(rows-1, A[3, :, :, curr_layer])

			curr_layer += 1

	all_boxes = np.reshape(A.transpose((0, 3, 1,2)), (4, -1)).transpose((1, 0))
	all_probs = rpn_layer.transpose((0, 3, 1, 2)).reshape((-1))

	x1 = all_boxes[:, 0]
	y1 = all_boxes[:, 1]
	x2 = all_boxes[:, 2]
	y2 = all_boxes[:, 3]

	idxs = np.where((x1 - x2 >= 0) | (y1 - y2 >= 0))

	all_boxes = np.delete(all_boxes, idxs, 0)
	all_probs = np.delete(all_probs, idxs, 0)

	result = non_max_suppression_fast(all_boxes, all_probs, overlap_thresh=overlap_thresh, max_boxes=max_boxes)[0]

	return result<commit_msg>Add missing sign check for intersection<commit_after>import numpy as np
import pdb
import math
import data_generators
import copy

def calc_iou(R, img_data, C, class_mapping):

	bboxes = img_data['bboxes']
	(width, height) = (img_data['width'], img_data['height'])
	# get image dimensions for resizing
	(resized_width, resized_height) = data_generators.get_new_img_size(width, height, C.im_size)

	gta = np.zeros((len(bboxes), 4))

	for bbox_num, bbox in enumerate(bboxes):
		# get the GT box coordinates, and resize to account for image resizing
		gta[bbox_num, 0] = int(round(bbox['x1'] * (resized_width / float(width))/C.rpn_stride))
		gta[bbox_num, 1] = int(round(bbox['x2'] * (resized_width / float(width))/C.rpn_stride))
		gta[bbox_num, 2] = int(round(bbox['y1'] * (resized_height / float(height))/C.rpn_stride))
		gta[bbox_num, 3] = int(round(bbox['y2'] * (resized_height / float(height))/C.rpn_stride))

	x_roi = []
	y_class_num = []
	y_class_regr_coords = []
	y_class_regr_label = []

	for ix in range(R.shape[0]):
		(x1, y1, x2, y2) = R[ix, :]
		x1 = int(round(x1))
		y1 = int(round(y1))
		x2 = int(round(x2))
		y2 = int(round(y2))

		best_iou = 0.0
		best_bbox = -1
		for bbox_num in range(len(bboxes)):
			curr_iou = data_generators.iou([gta[bbox_num, 0], gta[bbox_num, 2], gta[bbox_num, 1], gta[bbox_num, 3]], [x1, y1, x2, y2])
			if curr_iou > best_iou:
				best_iou = curr_iou
				best_bbox = bbox_num

		if best_iou < C.classifier_min_overlap:
				continue
		else:
			w = x2 - x1
			h = y2 - y1
			x_roi.append([x1, y1, w, h])

			if C.classifier_min_overlap <= best_iou < C.classifier_max_overlap:
				# hard negative example
				cls_name = 'bg'
			elif C.classifier_max_overlap <= best_iou:
				cls_name = bboxes[best_bbox]['class']
				cxg = (gta[best_bbox, 0] + gta[best_bbox, 1]) / 2.0
				cyg = (gta[best_bbox, 2] + gta[best_bbox, 3]) / 2.0

				cx = x1 + w / 2.0
				cy = y1 + h / 2.0

				tx = (cxg - cx) / float(w)
				ty = (cyg - cy) / float(h)
				tw = np.log((gta[best_bbox, 1] - gta[best_bbox, 0]) / float(w))
				th = np.log((gta[best_bbox, 3] - gta[best_bbox, 2]) / float(h))
			else:
				print('roi = {}'.format(best_iou))
				raise RuntimeError

		class_num = class_mapping[cls_name]
		class_label = len(class_mapping) * [0]
		class_label[class_num] = 1
		y_class_num.append(copy.deepcopy(class_label))
		coords = [0] * 4 * (len(class_mapping) - 1)
		labels = [0] * 4 * (len(class_mapping) - 1)
		if cls_name != 'bg':
			label_pos = 4 * class_num
			sx, sy, sw, sh = C.classifier_regr_std
			coords[label_pos:4+label_pos] = [sx*tx, sy*ty, sw*tw, sh*th]
			labels[label_pos:4+label_pos] = [1, 1, 1, 1]
			y_class_regr_coords.append(copy.deepcopy(coords))
			y_class_regr_label.append(copy.deepcopy(labels))
		else:
			y_class_regr_coords.append(copy.deepcopy(coords))
			y_class_regr_label.append(copy.deepcopy(labels))

	if len(x_roi) == 0:
		return None, None, None

	X = np.array(x_roi)
	Y1 = np.array(y_class_num)
	Y2 = np.concatenate([np.array(y_class_regr_label),np.array(y_class_regr_coords)],axis=1)

	return np.expand_dims(X, axis=0), np.expand_dims(Y1, axis=0), np.expand_dims(Y2, axis=0)

def apply_regr(x, y, w, h, tx, ty, tw, th):
	try:
		cx = x + w/2.
		cy = y + h/2.
		cx1 = tx * w + cx
		cy1 = ty * h + cy
		w1 = math.exp(tw) * w
		h1 = math.exp(th) * h
		x1 = cx1 - w1/2.
		y1 = cy1 - h1/2.
		x1 = int(round(x1))
		y1 = int(round(y1))
		w1 = int(round(w1))
		h1 = int(round(h1))

		return x1, y1, w1, h1

	except ValueError:
		return x, y, w, h
	except OverflowError:
		return x, y, w, h
	except Exception as e:
		print(e)
		return x, y, w, h

def apply_regr_np(X, T):
	try:
		x = X[0, :, :]
		y = X[1, :, :]
		w = X[2, :, :]
		h = X[3, :, :]

		tx = T[0, :, :]
		ty = T[1, :, :]
		tw = T[2, :, :]
		th = T[3, :, :]

		cx = x + w/2.
		cy = y + h/2.
		cx1 = tx * w + cx
		cy1 = ty * h + cy

		w1 = np.exp(tw) * w
		h1 = np.exp(th) * h
		x1 = cx1 - w1/2.
		y1 = cy1 - h1/2.

		x1 = np.round(x1)
		y1 = np.round(y1)
		w1 = np.round(w1)
		h1 = np.round(h1)
		return np.stack([x1, y1, w1, h1])
	except Exception as e:
		print(e)
		return X

def non_max_suppression_fast(boxes, probs, overlap_thresh=0.9, max_boxes=300):
	# code used from here: http://www.pyimagesearch.com/2015/02/16/faster-non-maximum-suppression-python/
	# if there are no boxes, return an empty list
	if len(boxes) == 0:
		return []

	# grab the coordinates of the bounding boxes
	x1 = boxes[:, 0]
	y1 = boxes[:, 1]
	x2 = boxes[:, 2]
	y2 = boxes[:, 3]

	np.testing.assert_array_less(x1, x2)
	np.testing.assert_array_less(y1, y2)

	# if the bounding boxes integers, convert them to floats --
	# this is important since we'll be doing a bunch of divisions
	if boxes.dtype.kind == "i":
		boxes = boxes.astype("float")

	# initialize the list of picked indexes
	pick = []

	# sort the bounding boxes
	idxs = np.argsort(probs)

	# keep looping while some indexes still remain in the indexes
	# list
	while len(idxs) > 0:
		# grab the last index in the indexes list and add the
		# index value to the list of picked indexes
		last = len(idxs) - 1
		i = idxs[last]
		pick.append(i)

		# find the intersection

		xx1_int = np.maximum(x1[i], x1[idxs[:last]])
		yy1_int = np.maximum(y1[i], y1[idxs[:last]])
		xx2_int = np.minimum(x2[i], x2[idxs[:last]])
		yy2_int = np.minimum(y2[i], y2[idxs[:last]])

		# find the union
		xx1_un = np.minimum(x1[i], x1[idxs[:last]])
		yy1_un = np.minimum(y1[i], y1[idxs[:last]])
		xx2_un = np.maximum(x2[i], x2[idxs[:last]])
		yy2_un = np.maximum(y2[i], y2[idxs[:last]])

		# compute the width and height of the bounding box
		ww_int = np.maximum(0, xx2_int - xx1_int)
		hh_int = np.maximum(0, yy2_int - yy1_int)

		ww_un = np.maximum(0, xx2_un - xx1_un)
		hh_un = np.maximum(0, yy2_un - yy1_un)

		# compute the ratio of overlap
		overlap = (ww_int*hh_int)/(ww_un*hh_un + 1e-9)

		# delete all indexes from the index list that have
		idxs = np.delete(idxs, np.concatenate(([last],
			np.where(overlap > overlap_thresh)[0])))

		if len(pick) >= max_boxes:
			break

	# return only the bounding boxes that were picked using the integer data type
	boxes = boxes[pick].astype("int")
	probs = probs[pick]
	return boxes, probs

import time
def rpn_to_roi(rpn_layer, regr_layer, C, dim_ordering, use_regr=True, max_boxes=300,overlap_thresh=0.9):

	regr_layer = regr_layer / C.std_scaling

	anchor_sizes = C.anchor_box_scales
	anchor_ratios = C.anchor_box_ratios

	assert rpn_layer.shape[0] == 1

	if dim_ordering == 'th':
		(rows,cols) = rpn_layer.shape[2:]

	elif dim_ordering == 'tf':
		(rows, cols) = rpn_layer.shape[1:3]

	curr_layer = 0
	if dim_ordering == 'tf':
		A = np.zeros((4, rpn_layer.shape[1], rpn_layer.shape[2], rpn_layer.shape[3]))
	elif dim_ordering == 'th':
		A = np.zeros((4, rpn_layer.shape[2], rpn_layer.shape[3], rpn_layer.shape[1]))

	for anchor_size in anchor_sizes:
		for anchor_ratio in anchor_ratios:

			anchor_x = (anchor_size * anchor_ratio[0])/C.rpn_stride
			anchor_y = (anchor_size * anchor_ratio[1])/C.rpn_stride
			if dim_ordering == 'th':
				regr = regr_layer[0, 4 * curr_layer:4 * curr_layer + 4, :, :]
			else:
				regr = regr_layer[0, :, :, 4 * curr_layer:4 * curr_layer + 4]
				regr = np.transpose(regr, (2, 0, 1))

			X, Y = np.meshgrid(np.arange(cols),np. arange(rows))

			A[0, :, :, curr_layer] = X - anchor_x/2
			A[1, :, :, curr_layer] = Y - anchor_y/2
			A[2, :, :, curr_layer] = anchor_x
			A[3, :, :, curr_layer] = anchor_y

			if use_regr:
				A[:, :, :, curr_layer] = apply_regr_np(A[:, :, :, curr_layer], regr)

			A[2, :, :, curr_layer] = np.maximum(1, A[2, :, :, curr_layer])
			A[3, :, :, curr_layer] = np.maximum(1, A[3, :, :, curr_layer])
			A[2, :, :, curr_layer] += A[0, :, :, curr_layer]
			A[3, :, :, curr_layer] += A[1, :, :, curr_layer]

			A[0, :, :, curr_layer] = np.maximum(0, A[0, :, :, curr_layer])
			A[1, :, :, curr_layer] = np.maximum(0, A[1, :, :, curr_layer])
			A[2, :, :, curr_layer] = np.minimum(cols-1, A[2, :, :, curr_layer])
			A[3, :, :, curr_layer] = np.minimum(rows-1, A[3, :, :, curr_layer])

			curr_layer += 1

	all_boxes = np.reshape(A.transpose((0, 3, 1,2)), (4, -1)).transpose((1, 0))
	all_probs = rpn_layer.transpose((0, 3, 1, 2)).reshape((-1))

	x1 = all_boxes[:, 0]
	y1 = all_boxes[:, 1]
	x2 = all_boxes[:, 2]
	y2 = all_boxes[:, 3]

	idxs = np.where((x1 - x2 >= 0) | (y1 - y2 >= 0))

	all_boxes = np.delete(all_boxes, idxs, 0)
	all_probs = np.delete(all_probs, idxs, 0)

	result = non_max_suppression_fast(all_boxes, all_probs, overlap_thresh=overlap_thresh, max_boxes=max_boxes)[0]

	return result
<|endoftext|>


<commit_before>from numpy import arange, float64, int64, dtype, ones

import pytest

from bolt import array
from bolt.utils import allclose

def test_shape(sc):

    x = arange(2*3).reshape((2, 3))
    b = array(x, sc)
    assert b.shape == x.shape

    x = arange(2*3*4).reshape((2, 3, 4))
    b = array(x, sc)
    assert b.shape == x.shape

def test_size(sc):

    x = arange(2*3*4).reshape((2, 3, 4))
    b = array(x, sc, axes=(0,))
    assert b.size == x.size

def test_split(sc):

    x = arange(2*3*4).reshape((2, 3, 4))
    b = array(x, sc, axes=(0,))
    assert b.split == 1

    b = array(x, sc, axes=(0, 1))
    assert b.split == 2

def test_mask(sc):

    x = arange(2*3*4).reshape((2, 3, 4))
    b = array(x, sc, axes=(0,))
    assert b.mask == (1, 0, 0)

    b = array(x, sc, axes=(0, 1))
    assert b.mask == (1, 1, 0)

    b = array(x, sc, axes=(0, 1, 2))
    assert b.mask == (1, 1, 1)

def test_value_shape(sc):

    x = arange(2*3).reshape((2, 3))
    b = array(x, sc)
    assert b.values.shape == (3,)

    x = arange(2*3*4).reshape((2, 3, 4))
    b = array(x, sc, axes=(0,))
    assert b.values.shape == (3, 4)

def test_key_shape(sc):

    x = arange(2*3).reshape((2, 3))
    b = array(x, sc)
    assert b.keys.shape == (2,)

    x = arange(2*3*4).reshape((2, 3, 4))
    b = array(x, sc, axes=(0, 1))
    assert b.keys.shape == (2, 3)

def test_ndim(sc):

    x = arange(2**5).reshape(2, 2, 2, 2, 2)
    b = array(x, sc, axes=[0, 1, 2])

    assert b.keys.ndim == 3
    assert b.values.ndim == 2
    assert b.ndim == 5

def test_reshape_keys(sc):

    x = arange(2*3*4).reshape((2, 3, 4))

    b = array(x, sc, axes=(0, 1))
    c = b.keys.reshape((3, 2))
    assert c.keys.shape == (3, 2)
    assert allclose(c.toarray(), x.reshape((3, 2, 4)))

    b = array(x, sc, axes=(0,))
    c = b.keys.reshape((2, 1))
    assert allclose(c.toarray(), x.reshape((2, 1, 3, 4)))

    b = array(x, sc, axes=(0,))
    c = b.keys.reshape((2,))
    assert allclose(c.toarray(), x)

    b = array(x, sc, axes=(0, 1))
    c = b.keys.reshape((2, 3))
    assert allclose(c.toarray(), x)

def test_reshape_keys_errors(sc):

    x = arange(2*3*4).reshape((2, 3, 4))

    b = array(x, sc, axes=(0, 1))
    with pytest.raises(ValueError):
        b.keys.reshape((2, 3, 4))

def test_reshape_values(sc):

    x = arange(2*3*4).reshape((2, 3, 4))

    b = array(x, sc, axes=(0,))
    c = b.values.reshape((4, 3))
    assert c.values.shape == (4, 3)
    assert allclose(c.toarray(), x.reshape((2, 4, 3)))

    b = array(x, sc, axes=(0, 1))
    c = b.values.reshape((1, 4))
    assert allclose(c.toarray(), x.reshape((2, 3, 1, 4)))

    b = array(x, sc, axes=(0, 1))
    c = b.values.reshape((4,))
    assert allclose(c.toarray(), x)

    b = array(x, sc, axes=(0,))
    c = b.values.reshape((3, 4))
    assert allclose(c.toarray(), x)

def test_reshape_values_errors(sc):

    x = arange(2*3*4).reshape((2, 3, 4))

    b = array(x, sc, axes=(0, 1))
    with pytest.raises(ValueError):
        b.values.reshape((2, 3, 4))

def test_transpose_keys(sc):

    x = arange(2*3*4).reshape((2, 3, 4))

    b = array(x, sc, axes=(0, 1))
    c = b.keys.transpose((1, 0))
    assert c.keys.shape == (3, 2)
    assert allclose(c.toarray(), x.transpose((1, 0, 2)))

    b = array(x, sc, axes=(0,))
    c = b.keys.transpose((0,))
    assert allclose(c.toarray(), x)

    b = array(x, sc, axes=(0, 1))
    c = b.keys.transpose((0, 1))
    assert allclose(c.toarray(), x)

def test_transpose_keys_errors(sc):

    x = arange(2*3*4).reshape((2, 3, 4))

    b = array(x, sc, axes=(0, 1))
    with pytest.raises(ValueError):
        b.keys.transpose((0, 2))

    with pytest.raises(ValueError):
        b.keys.transpose((1, 1))

    with pytest.raises(ValueError):
        b.keys.transpose((0,))

def test_transpose_values(sc):

    x = arange(2*3*4).reshape((2, 3, 4))

    b = array(x, sc, axes=(0,))
    c = b.values.transpose((1, 0))
    assert c.values.shape == (4, 3)
    assert allclose(c.toarray(), x.transpose((0, 2, 1)))

    b = array(x, sc, axes=(0,))
    c = b.values.transpose((0, 1))
    assert allclose(c.toarray(), x)

    b = array(x, sc, axes=(0, 1))
    c = b.values.transpose((0,))
    assert allclose(c.toarray(), x.reshape((2, 3, 4)))

def test_traspose_values_errors(sc):

    x = arange(2*3*4).reshape((2, 3, 4))

    b = array(x, sc, axes=(0,))
    with pytest.raises(ValueError):
        b.values.transpose((0, 2))

    with pytest.raises(ValueError):
        b.values.transpose((1, 1))

    with pytest.raises(ValueError):
        b.values.transpose((0,))

def test_getitem_slice(sc):

    x = arange(2*3).reshape((2, 3))

    b = array(x, sc, axes=(0,))
    assert allclose(b[0:1, 0:1].toarray(), x[0:1, 0:1])
    assert allclose(b[0:2, 0:2].toarray(), x[0:2, 0:2])
    assert allclose(b[0:2, 0:3].toarray(), x[0:2, 0:3])
    assert allclose(b[0:2, 0:3:2].toarray(), x[0:2, 0:3:2])
    assert allclose(b[:2, :2].toarray(), x[:2, :2])
    assert allclose(b[1:, 1:].toarray(), x[1:, 1:])

    b = array(x, sc, axes=(0, 1))
    assert allclose(b[0:1, 0:1].toarray(), x[0:1, 0:1])
    assert allclose(b[0:2, 0:2].toarray(), x[0:2, 0:2])
    assert allclose(b[0:2, 0:3].toarray(), x[0:2, 0:3])
    assert allclose(b[0:2, 0:3:2].toarray(), x[0:2, 0:3:2])
    assert allclose(b[:2, :2].toarray(), x[:2, :2])
    assert allclose(b[1:, 1:].toarray(), x[1:, 1:])

def test_getitem_slice_ragged(sc):

    x = arange(10*10*3).reshape((10, 10, 3))

    b = array(x, sc, axes=(0,1))
    assert allclose(b[0:5:2, 0:2].toarray(), x[0:5:2, 0:2])
    assert allclose(b[0:5:3, 0:2].toarray(), x[0:5:3, 0:2])
    assert allclose(b[0:9:3, 0:2].toarray(), x[0:9:3, 0:2])

def test_getitem_int(sc):

    x = arange(2*3).reshape((2, 3))

    b = array(x, sc, axes=(0,))
    assert allclose(b[0, 0].toarray(), x[0, 0])
    assert allclose(b[0, 1].toarray(), x[0, 1])
    assert allclose(b[0, 0:1].toarray(), x[0, 0:1])
    assert allclose(b[1, 2].toarray(), x[1, 2])
    assert allclose(b[[1], [2]].toarray(), x[[1], [2]])

    b = array(x, sc, axes=(0, 1))
    assert allclose(b[0, 0].toarray(), x[0, 0])
    assert allclose(b[0, 1].toarray(), x[0, 1])
    assert allclose(b[0, 0:1].toarray(), x[0, 0:1])
    assert allclose(b[1, 2].toarray(), x[1, 2])
    assert allclose(b[[1], [2]].toarray(), x[[1], [2]])

def test_getitem_list(sc):

    x = arange(3*3*4).reshape((3, 3, 4))

    b = array(x, sc, axes=(0,))
    assert allclose(b[[0, 1], [0, 1], [0, 2]].toarray(), x[[0, 1], [0, 1], [0, 2]])
    assert allclose(b[[0, 1], [0, 2], [0, 3]].toarray(), x[[0, 1], [0, 2], [0, 3]])
    assert allclose(b[[0, 1, 2], [0, 2, 1], [0, 3, 1]].toarray(), x[[0, 1, 2], [0, 2, 1], [0, 3, 1]])

    b = array(x, sc, axes=(0,1))
    assert allclose(b[[0, 1], [0, 1], [0, 2]].toarray(), x[[0, 1], [0, 1], [0, 2]])
    assert allclose(b[[0, 1], [0, 2], [0, 3]].toarray(), x[[0, 1], [0, 2], [0, 3]])
    assert allclose(b[[0, 1, 2], [0, 2, 1], [0, 3, 1]].toarray(), x[[0, 1, 2], [0, 2, 1], [0, 3, 1]])

def test_getitem_list_array(sc):

    x = arange(3*3*4).reshape((3, 3, 4))

    rows = [[0, 0], [1, 1]]
    cols = [[0, 2], [0, 2]]
    dept = [[0, 3], [0, 3]]

    b = array(x, sc, axes=(0,))
    assert allclose(b[rows, cols, dept].toarray(), x[rows, cols, dept])

    b = array(x, sc, axes=(0,1))
    assert allclose(b[rows, cols, dept].toarray(), x[rows, cols, dept])

def test_swap(sc):

    a = arange(2**8).reshape(*(8*[2]))
    b = array(a, sc, axes=(0, 1, 2, 3))

    bs = b.swap([1, 2], [0, 3], size=(2, 2))
    at = a.transpose([0, 3, 4, 7, 1, 2, 5, 6])

    assert allclose(at, bs.toarray())

def test_dtype(sc):

    a = arange(2**8, dtype=int64)
    b = array(a, sc, dtype=int64)
    assert a.dtype == b.dtype
    assert b.dtype == dtype(int64)
    dtypes = b._rdd.map(lambda x:x[1].dtype).collect()
    for dt in dtypes:
        assert dt == dtype(int64)

    a = arange(2.0**8)
    b = array(a, sc)
    assert a.dtype == b.dtype
    assert b.dtype == dtype(float64)
    dtypes = b._rdd.map(lambda x:x[1].dtype).collect()
    for dt in dtypes:
        assert dt == dtype(float64)

    a = arange(2**8)
    b = array(a, sc)
    assert a.dtype == b.dtype
    assert b.dtype == dtype(int64)
    dtypes = b._rdd.map(lambda x:x[1].dtype).collect()
    for dt in dtypes:
        assert dt == dtype(int64)

    a = ones(2**8, dtype=bool)
    b = array(a, sc)
    assert a.dtype == b.dtype
    assert b.dtype == dtype(bool)
    dtypes = b._rdd.map(lambda x:x[1].dtype).collect()
    for dt in dtypes:
        assert dt == dtype(bool)

<commit_msg>adding a test for astype()<commit_after>from numpy import arange, float64, int64, dtype, ones

import pytest

from bolt import array
from bolt.utils import allclose

def test_shape(sc):

    x = arange(2*3).reshape((2, 3))
    b = array(x, sc)
    assert b.shape == x.shape

    x = arange(2*3*4).reshape((2, 3, 4))
    b = array(x, sc)
    assert b.shape == x.shape

def test_size(sc):

    x = arange(2*3*4).reshape((2, 3, 4))
    b = array(x, sc, axes=(0,))
    assert b.size == x.size

def test_split(sc):

    x = arange(2*3*4).reshape((2, 3, 4))
    b = array(x, sc, axes=(0,))
    assert b.split == 1

    b = array(x, sc, axes=(0, 1))
    assert b.split == 2

def test_mask(sc):

    x = arange(2*3*4).reshape((2, 3, 4))
    b = array(x, sc, axes=(0,))
    assert b.mask == (1, 0, 0)

    b = array(x, sc, axes=(0, 1))
    assert b.mask == (1, 1, 0)

    b = array(x, sc, axes=(0, 1, 2))
    assert b.mask == (1, 1, 1)

def test_value_shape(sc):

    x = arange(2*3).reshape((2, 3))
    b = array(x, sc)
    assert b.values.shape == (3,)

    x = arange(2*3*4).reshape((2, 3, 4))
    b = array(x, sc, axes=(0,))
    assert b.values.shape == (3, 4)

def test_key_shape(sc):

    x = arange(2*3).reshape((2, 3))
    b = array(x, sc)
    assert b.keys.shape == (2,)

    x = arange(2*3*4).reshape((2, 3, 4))
    b = array(x, sc, axes=(0, 1))
    assert b.keys.shape == (2, 3)

def test_ndim(sc):

    x = arange(2**5).reshape(2, 2, 2, 2, 2)
    b = array(x, sc, axes=[0, 1, 2])

    assert b.keys.ndim == 3
    assert b.values.ndim == 2
    assert b.ndim == 5

def test_reshape_keys(sc):

    x = arange(2*3*4).reshape((2, 3, 4))

    b = array(x, sc, axes=(0, 1))
    c = b.keys.reshape((3, 2))
    assert c.keys.shape == (3, 2)
    assert allclose(c.toarray(), x.reshape((3, 2, 4)))

    b = array(x, sc, axes=(0,))
    c = b.keys.reshape((2, 1))
    assert allclose(c.toarray(), x.reshape((2, 1, 3, 4)))

    b = array(x, sc, axes=(0,))
    c = b.keys.reshape((2,))
    assert allclose(c.toarray(), x)

    b = array(x, sc, axes=(0, 1))
    c = b.keys.reshape((2, 3))
    assert allclose(c.toarray(), x)

def test_reshape_keys_errors(sc):

    x = arange(2*3*4).reshape((2, 3, 4))

    b = array(x, sc, axes=(0, 1))
    with pytest.raises(ValueError):
        b.keys.reshape((2, 3, 4))

def test_reshape_values(sc):

    x = arange(2*3*4).reshape((2, 3, 4))

    b = array(x, sc, axes=(0,))
    c = b.values.reshape((4, 3))
    assert c.values.shape == (4, 3)
    assert allclose(c.toarray(), x.reshape((2, 4, 3)))

    b = array(x, sc, axes=(0, 1))
    c = b.values.reshape((1, 4))
    assert allclose(c.toarray(), x.reshape((2, 3, 1, 4)))

    b = array(x, sc, axes=(0, 1))
    c = b.values.reshape((4,))
    assert allclose(c.toarray(), x)

    b = array(x, sc, axes=(0,))
    c = b.values.reshape((3, 4))
    assert allclose(c.toarray(), x)

def test_reshape_values_errors(sc):

    x = arange(2*3*4).reshape((2, 3, 4))

    b = array(x, sc, axes=(0, 1))
    with pytest.raises(ValueError):
        b.values.reshape((2, 3, 4))

def test_transpose_keys(sc):

    x = arange(2*3*4).reshape((2, 3, 4))

    b = array(x, sc, axes=(0, 1))
    c = b.keys.transpose((1, 0))
    assert c.keys.shape == (3, 2)
    assert allclose(c.toarray(), x.transpose((1, 0, 2)))

    b = array(x, sc, axes=(0,))
    c = b.keys.transpose((0,))
    assert allclose(c.toarray(), x)

    b = array(x, sc, axes=(0, 1))
    c = b.keys.transpose((0, 1))
    assert allclose(c.toarray(), x)

def test_transpose_keys_errors(sc):

    x = arange(2*3*4).reshape((2, 3, 4))

    b = array(x, sc, axes=(0, 1))
    with pytest.raises(ValueError):
        b.keys.transpose((0, 2))

    with pytest.raises(ValueError):
        b.keys.transpose((1, 1))

    with pytest.raises(ValueError):
        b.keys.transpose((0,))

def test_transpose_values(sc):

    x = arange(2*3*4).reshape((2, 3, 4))

    b = array(x, sc, axes=(0,))
    c = b.values.transpose((1, 0))
    assert c.values.shape == (4, 3)
    assert allclose(c.toarray(), x.transpose((0, 2, 1)))

    b = array(x, sc, axes=(0,))
    c = b.values.transpose((0, 1))
    assert allclose(c.toarray(), x)

    b = array(x, sc, axes=(0, 1))
    c = b.values.transpose((0,))
    assert allclose(c.toarray(), x.reshape((2, 3, 4)))

def test_traspose_values_errors(sc):

    x = arange(2*3*4).reshape((2, 3, 4))

    b = array(x, sc, axes=(0,))
    with pytest.raises(ValueError):
        b.values.transpose((0, 2))

    with pytest.raises(ValueError):
        b.values.transpose((1, 1))

    with pytest.raises(ValueError):
        b.values.transpose((0,))

def test_getitem_slice(sc):

    x = arange(2*3).reshape((2, 3))

    b = array(x, sc, axes=(0,))
    assert allclose(b[0:1, 0:1].toarray(), x[0:1, 0:1])
    assert allclose(b[0:2, 0:2].toarray(), x[0:2, 0:2])
    assert allclose(b[0:2, 0:3].toarray(), x[0:2, 0:3])
    assert allclose(b[0:2, 0:3:2].toarray(), x[0:2, 0:3:2])
    assert allclose(b[:2, :2].toarray(), x[:2, :2])
    assert allclose(b[1:, 1:].toarray(), x[1:, 1:])

    b = array(x, sc, axes=(0, 1))
    assert allclose(b[0:1, 0:1].toarray(), x[0:1, 0:1])
    assert allclose(b[0:2, 0:2].toarray(), x[0:2, 0:2])
    assert allclose(b[0:2, 0:3].toarray(), x[0:2, 0:3])
    assert allclose(b[0:2, 0:3:2].toarray(), x[0:2, 0:3:2])
    assert allclose(b[:2, :2].toarray(), x[:2, :2])
    assert allclose(b[1:, 1:].toarray(), x[1:, 1:])

def test_getitem_slice_ragged(sc):

    x = arange(10*10*3).reshape((10, 10, 3))

    b = array(x, sc, axes=(0,1))
    assert allclose(b[0:5:2, 0:2].toarray(), x[0:5:2, 0:2])
    assert allclose(b[0:5:3, 0:2].toarray(), x[0:5:3, 0:2])
    assert allclose(b[0:9:3, 0:2].toarray(), x[0:9:3, 0:2])

def test_getitem_int(sc):

    x = arange(2*3).reshape((2, 3))

    b = array(x, sc, axes=(0,))
    assert allclose(b[0, 0].toarray(), x[0, 0])
    assert allclose(b[0, 1].toarray(), x[0, 1])
    assert allclose(b[0, 0:1].toarray(), x[0, 0:1])
    assert allclose(b[1, 2].toarray(), x[1, 2])
    assert allclose(b[[1], [2]].toarray(), x[[1], [2]])

    b = array(x, sc, axes=(0, 1))
    assert allclose(b[0, 0].toarray(), x[0, 0])
    assert allclose(b[0, 1].toarray(), x[0, 1])
    assert allclose(b[0, 0:1].toarray(), x[0, 0:1])
    assert allclose(b[1, 2].toarray(), x[1, 2])
    assert allclose(b[[1], [2]].toarray(), x[[1], [2]])

def test_getitem_list(sc):

    x = arange(3*3*4).reshape((3, 3, 4))

    b = array(x, sc, axes=(0,))
    assert allclose(b[[0, 1], [0, 1], [0, 2]].toarray(), x[[0, 1], [0, 1], [0, 2]])
    assert allclose(b[[0, 1], [0, 2], [0, 3]].toarray(), x[[0, 1], [0, 2], [0, 3]])
    assert allclose(b[[0, 1, 2], [0, 2, 1], [0, 3, 1]].toarray(), x[[0, 1, 2], [0, 2, 1], [0, 3, 1]])

    b = array(x, sc, axes=(0,1))
    assert allclose(b[[0, 1], [0, 1], [0, 2]].toarray(), x[[0, 1], [0, 1], [0, 2]])
    assert allclose(b[[0, 1], [0, 2], [0, 3]].toarray(), x[[0, 1], [0, 2], [0, 3]])
    assert allclose(b[[0, 1, 2], [0, 2, 1], [0, 3, 1]].toarray(), x[[0, 1, 2], [0, 2, 1], [0, 3, 1]])

def test_getitem_list_array(sc):

    x = arange(3*3*4).reshape((3, 3, 4))

    rows = [[0, 0], [1, 1]]
    cols = [[0, 2], [0, 2]]
    dept = [[0, 3], [0, 3]]

    b = array(x, sc, axes=(0,))
    assert allclose(b[rows, cols, dept].toarray(), x[rows, cols, dept])

    b = array(x, sc, axes=(0,1))
    assert allclose(b[rows, cols, dept].toarray(), x[rows, cols, dept])

def test_swap(sc):

    a = arange(2**8).reshape(*(8*[2]))
    b = array(a, sc, axes=(0, 1, 2, 3))

    bs = b.swap([1, 2], [0, 3], size=(2, 2))
    at = a.transpose([0, 3, 4, 7, 1, 2, 5, 6])

    assert allclose(at, bs.toarray())

def test_dtype(sc):

    a = arange(2**8, dtype=int64)
    b = array(a, sc, dtype=int64)
    assert a.dtype == b.dtype
    assert b.dtype == dtype(int64)
    dtypes = b._rdd.map(lambda x:x[1].dtype).collect()
    for dt in dtypes:
        assert dt == dtype(int64)

    a = arange(2.0**8)
    b = array(a, sc)
    assert a.dtype == b.dtype
    assert b.dtype == dtype(float64)
    dtypes = b._rdd.map(lambda x:x[1].dtype).collect()
    for dt in dtypes:
        assert dt == dtype(float64)

    a = arange(2**8)
    b = array(a, sc)
    assert a.dtype == b.dtype
    assert b.dtype == dtype(int64)
    dtypes = b._rdd.map(lambda x:x[1].dtype).collect()
    for dt in dtypes:
        assert dt == dtype(int64)

    a = ones(2**8, dtype=bool)
    b = array(a, sc)
    assert a.dtype == b.dtype
    assert b.dtype == dtype(bool)
    dtypes = b._rdd.map(lambda x:x[1].dtype).collect()
    for dt in dtypes:
        assert dt == dtype(bool)

def test_astype(sc):

    a = ones(2**8, dtype=int64)
    b = array(a, sc, dtype=int64)
    c = b.astype(bool)

    assert c.dtype == dtype(bool)

    dtypes = c._rdd.map(lambda x:x[1].dtype).collect()
    for dt in dtypes:
        assert dt == dtype(bool)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Copyright © 2015,2016 Mathieu Duponchelle <mathieu.duponchelle@opencreed.com>
# Copyright © 2015,2016 Collabora Ltd
#
# This library is free software; you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by the Free
# Software Foundation; either version 2.1 of the License, or (at your option)
# any later version.
#
# This library is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this library.  If not, see <http://www.gnu.org/licenses/>.

"""
Banana banana
"""

import os
import sys
import subprocess

HERE = os.path.dirname(__file__)

with open(os.path.join(HERE, '..', 'VERSION.txt'), 'r') as _:
    VERSION = _.read().strip()

THEME_VERSION = "0.8.1"

def _check_submodule_status(root, submodules):
    """check submodule status
    Has three return values:
    'missing' - submodules are absent
    'unclean' - submodules have unstaged changes
    'clean' - all submodules are up to date
    """

    if hasattr(sys, "frozen"):
        # frozen via py2exe or similar, don't bother
        return 'clean'

    if not os.path.exists(os.path.join(root, '.git')):
        # not in git, assume clean
        return 'clean'

    for submodule in submodules:
        if not os.path.exists(submodule):
            return 'missing'

    # Popen can't handle unicode cwd on Windows Python 2
    if sys.platform == 'win32' and sys.version_info[0] < 3 \
            and not isinstance(root, bytes):
        root = root.encode(sys.getfilesystemencoding() or 'ascii')
    # check with git submodule status
    proc = subprocess.Popen('git submodule status',
                            stdout=subprocess.PIPE,
                            stderr=subprocess.PIPE,
                            shell=True,
                            cwd=root)
    status, _ = proc.communicate()
    status = status.decode("ascii", "replace")

    for line in status.splitlines():
        if line.startswith('-'):
            return 'missing'
        elif line.startswith('+'):
            return 'unclean'

    return 'clean'

def _update_submodules(repo_dir):
    """update submodules in a repo"""
    subprocess.check_call("git submodule init", cwd=repo_dir, shell=True)
    subprocess.check_call(
        "git submodule update --recursive", cwd=repo_dir, shell=True)

UNCLEAN_SUBMODULES_MSG =\
    """
    WARNING:

    The submodules are not clean.  This means that either you have
    changed the code in the submodules, or something has gone quite wrong
    with your git repo.  I will continue to try to build, but if
    something goes wrong with the installation, this is probably the
    cause.  You may want to commit any submodule changes you have made.
    """

def require_clean_submodules(repo_root, submodules):
    """Check on git submodules before distutils can do anything
    Since distutils cannot be trusted to update the tree
    after everything has been set in motion,
    this is not a distutils command.
    """
    # PACKAGERS: Add a return here to skip checks for git submodules

    # don't do anything if nothing is actually supposed to happen
    for do_nothing in (
            '-h', '--help', '--help-commands', 'clean', 'submodule'):
        if do_nothing in sys.argv:
            return

    status = _check_submodule_status(repo_root, submodules)

    if status == "missing":
        print "checking out submodules for the first time"
        _update_submodules(repo_root)
    elif status == "unclean":
        print UNCLEAN_SUBMODULES_MSG

def symlink(source, link_name):
    """
    Method to allow creating symlinks on Windows
    """
    os_symlink = getattr(os, "symlink", None)
    if callable(os_symlink):
        os_symlink(source, link_name)
    else:
        import ctypes
        csl = ctypes.windll.kernel32.CreateSymbolicLinkW
        csl.argtypes = (ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_uint32)
        csl.restype = ctypes.c_ubyte
        flags = 1 if os.path.isdir(source) else 0
        if csl(link_name, source, flags) == 0:
            raise ctypes.WinError()
<commit_msg>Bump theme version<commit_after># -*- coding: utf-8 -*-
#
# Copyright © 2015,2016 Mathieu Duponchelle <mathieu.duponchelle@opencreed.com>
# Copyright © 2015,2016 Collabora Ltd
#
# This library is free software; you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by the Free
# Software Foundation; either version 2.1 of the License, or (at your option)
# any later version.
#
# This library is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this library.  If not, see <http://www.gnu.org/licenses/>.

"""
Banana banana
"""

import os
import sys
import subprocess

HERE = os.path.dirname(__file__)

with open(os.path.join(HERE, '..', 'VERSION.txt'), 'r') as _:
    VERSION = _.read().strip()

THEME_VERSION = "0.8.2"

def _check_submodule_status(root, submodules):
    """check submodule status
    Has three return values:
    'missing' - submodules are absent
    'unclean' - submodules have unstaged changes
    'clean' - all submodules are up to date
    """

    if hasattr(sys, "frozen"):
        # frozen via py2exe or similar, don't bother
        return 'clean'

    if not os.path.exists(os.path.join(root, '.git')):
        # not in git, assume clean
        return 'clean'

    for submodule in submodules:
        if not os.path.exists(submodule):
            return 'missing'

    # Popen can't handle unicode cwd on Windows Python 2
    if sys.platform == 'win32' and sys.version_info[0] < 3 \
            and not isinstance(root, bytes):
        root = root.encode(sys.getfilesystemencoding() or 'ascii')
    # check with git submodule status
    proc = subprocess.Popen('git submodule status',
                            stdout=subprocess.PIPE,
                            stderr=subprocess.PIPE,
                            shell=True,
                            cwd=root)
    status, _ = proc.communicate()
    status = status.decode("ascii", "replace")

    for line in status.splitlines():
        if line.startswith('-'):
            return 'missing'
        elif line.startswith('+'):
            return 'unclean'

    return 'clean'

def _update_submodules(repo_dir):
    """update submodules in a repo"""
    subprocess.check_call("git submodule init", cwd=repo_dir, shell=True)
    subprocess.check_call(
        "git submodule update --recursive", cwd=repo_dir, shell=True)

UNCLEAN_SUBMODULES_MSG =\
    """
    WARNING:

    The submodules are not clean.  This means that either you have
    changed the code in the submodules, or something has gone quite wrong
    with your git repo.  I will continue to try to build, but if
    something goes wrong with the installation, this is probably the
    cause.  You may want to commit any submodule changes you have made.
    """

def require_clean_submodules(repo_root, submodules):
    """Check on git submodules before distutils can do anything
    Since distutils cannot be trusted to update the tree
    after everything has been set in motion,
    this is not a distutils command.
    """
    # PACKAGERS: Add a return here to skip checks for git submodules

    # don't do anything if nothing is actually supposed to happen
    for do_nothing in (
            '-h', '--help', '--help-commands', 'clean', 'submodule'):
        if do_nothing in sys.argv:
            return

    status = _check_submodule_status(repo_root, submodules)

    if status == "missing":
        print "checking out submodules for the first time"
        _update_submodules(repo_root)
    elif status == "unclean":
        print UNCLEAN_SUBMODULES_MSG

def symlink(source, link_name):
    """
    Method to allow creating symlinks on Windows
    """
    os_symlink = getattr(os, "symlink", None)
    if callable(os_symlink):
        os_symlink(source, link_name)
    else:
        import ctypes
        csl = ctypes.windll.kernel32.CreateSymbolicLinkW
        csl.argtypes = (ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_uint32)
        csl.restype = ctypes.c_ubyte
        flags = 1 if os.path.isdir(source) else 0
        if csl(link_name, source, flags) == 0:
            raise ctypes.WinError()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Copyright (c) 2018~2999 - Cologler <skyoflw@gmail.com>
# ----------
# a comparer like csharp
# ----------

from abc import abstractmethod

class IEqualityComparer:
    '''
    the comparer interface.
    '''
    @abstractmethod
    def hash(self, obj):
        '''
        get the hash from object.
        '''
        raise NotImplementedError

    @abstractmethod
    def eq(self, obj1, obj2):
        '''
        compare two value is equals or not.
        '''
        raise NotImplementedError

class ObjectWrapper:
    '''
    wrap a object with given equality comparer.
    '''
    __slots__ = ('_comparer', '_obj', '_hashcode')

    def __init__(self, comparer: IEqualityComparer, obj):
        self._comparer = comparer
        self._obj = obj
        self._hashcode = None

    def unwrap(self):
        '''
        get the origin object.
        '''
        return self._obj

    def __hash__(self):
        if self._hashcode is None:
            self._hashcode = self._comparer.hash(self._obj)
        return self._hashcode

    def __eq__(self, other):
        '''
        note: user should ensure other is instance of Wrap.
        '''
        return self._comparer.eq(self._obj, other.unwrap())

class ObjectComparer(IEqualityComparer):
    '''
    the default comparer implemention for object.
    '''
    def hash(self, obj):
        return hash(obj)

    def eq(self, obj1, obj2):
        return obj1 == obj2

class IgnoreCaseStringComparer(IEqualityComparer):
    def hash(self, obj: str):
        return hash(obj.upper())

    def eq(self, obj1: str, obj2: str):
        return obj1.upper() == obj2.upper()

class StringComparer:
    IgnoreCaseComparer = IgnoreCaseStringComparer()
<commit_msg>cache str.upper()<commit_after># -*- coding: utf-8 -*-
#
# Copyright (c) 2018~2999 - Cologler <skyoflw@gmail.com>
# ----------
# a comparer like csharp
# ----------

from abc import abstractmethod

class IEqualityComparer:
    '''
    the comparer interface.
    '''
    @abstractmethod
    def hash(self, obj):
        '''
        get the hash from object.
        '''
        raise NotImplementedError

    @abstractmethod
    def eq(self, cmpvalue_1, cmpvalue_2):
        '''
        compare two value is equals or not.
        '''
        raise NotImplementedError

    def get_cmpvalue(self, obj):
        '''
        get the cmpvalue use for eq compare.
        this is help to cache some value like `str.upper()`.
        '''
        return obj

class ObjectWrapper:
    '''
    wrap a object with given equality comparer.
    '''
    __slots__ = ('_comparer', '_obj', '_hashcode', '_cmpvalue')

    def __init__(self, comparer: IEqualityComparer, obj):
        self._comparer = comparer
        self._obj = obj
        self._hashcode = None
        self._cmpvalue = None

    def unwrap(self):
        '''
        get the origin object.
        '''
        return self._obj

    def __hash__(self):
        if self._hashcode is None:
            self._hashcode = self._comparer.hash(self._obj)
        return self._hashcode

    def get_cmpvalue(self):
        if self._cmpvalue is None:
            self._cmpvalue = (self._comparer.get_cmpvalue(self._obj), )
        return self._cmpvalue

    def __eq__(self, other):
        '''
        note: user should ensure other is instance of Wrap.
        '''
        return self._comparer.eq(self.get_cmpvalue(), other.get_cmpvalue())

class ObjectComparer(IEqualityComparer):
    '''
    the default comparer implemention for object.
    '''
    def hash(self, obj):
        return hash(obj)

    def eq(self, obj1, obj2):
        return obj1 == obj2

class IgnoreCaseStringComparer(IEqualityComparer):
    def hash(self, obj: str):
        return hash(obj.upper())

    def eq(self, cmpvalue_1, cmpvalue_2):
        return cmpvalue_1 == cmpvalue_1

    def get_cmpvalue(self, obj: str):
        return obj.upper()

class StringComparer:
    IgnoreCaseComparer = IgnoreCaseStringComparer()
<|endoftext|>