<commit_before># Requriments - transformers, tokenizers
# Right now, the Smiles Tokenizer uses an exiesting vocab file from rxnfp that is fairly comprehensive and from the USPTO dataset.
# The vocab may be expanded in the near future

import collections
import logging
import os
import re
import numpy as np
import pkg_resources
from typing import List
from transformers import BertTokenizer

# export
SMI_REGEX_PATTERN = r"(\[[^\]]+]|Br?|Cl?|N|O|S|P|F|I|b|c|n|o|s|p|\(|\)|\.|=|#|-|\+|\\|\/|:|~|@|\?|>>?|\*|\$|\%[0-9]{2}|[0-9])"

def get_default_tokenizer():
    default_vocab_path = (
        pkg_resources.resource_filename(
            "chemberta",
            "tokenizers/vocab.txt"
        )
    )
    return SmilesTokenizer(default_vocab_path)

class SmilesTokenizer(BertTokenizer):
    r"""
    Constructs a SmilesTokenizer.
    Mostly copied from https://github.com/huggingface/transformers
    Args:
        vocab_file: Path to a SMILES character per line vocabulary file
    """

    def __init__(
            self,
            vocab_file='',
            # unk_token="[UNK]",
            # sep_token="[SEP]",
            # pad_token="[PAD]",
            # cls_token="[CLS]",
            # mask_token="[MASK]",
            **kwargs
    ):
        """Constructs a BertTokenizer.
        Args:
            **vocab_file**: Path to a SMILES character per line vocabulary file
        """
        super().__init__(vocab_file, **kwargs)
        # take into account special tokens in max length
        self.max_len_single_sentence = self.max_len - 2
        self.max_len_sentences_pair = self.max_len - 3

        if not os.path.isfile(vocab_file):
            raise ValueError(
                "Can't find a vocab file at path '{}'.".format(vocab_file)
            )
        self.vocab = load_vocab(vocab_file)
        self.highest_unused_index = max(
            [
                i for i, v in enumerate(self.vocab.keys())
                if v.startswith("[unused")
            ]
        )
        self.ids_to_tokens = collections.OrderedDict(
            [(ids, tok) for tok, ids in self.vocab.items()]
        )
        self.basic_tokenizer = BasicSmilesTokenizer()
        self.init_kwargs["max_len"] = self.max_len

    @property
    def vocab_size(self):
        return len(self.vocab)

    @property
    def vocab_list(self):
        return list(self.vocab.keys())

    def _tokenize(self, text):
        split_tokens = [token for token in self.basic_tokenizer.tokenize(text)]
        return split_tokens

    def _convert_token_to_id(self, token):
        """ Converts a token (str/unicode) in an id using the vocab. """
        return self.vocab.get(token, self.vocab.get(self.unk_token))

    def _convert_id_to_token(self, index):
        """Converts an index (integer) in a token (string/unicode) using the vocab."""
        return self.ids_to_tokens.get(index, self.unk_token)

    def convert_tokens_to_string(self, tokens):
        """ Converts a sequence of tokens (string) in a single string. """
        out_string = " ".join(tokens).replace(" ##", "").strip()
        return out_string

    def add_special_tokens_ids_single_sequence(self, token_ids):
        """
        Adds special tokens to the a sequence for sequence classification tasks.
        A BERT sequence has the following format: [CLS] X [SEP]
        """
        return [self.cls_token_id] + token_ids + [self.sep_token_id]

    def add_special_tokens_single_sequence(self, tokens):
        """
        Adds special tokens to the a sequence for sequence classification tasks.
        A BERT sequence has the following format: [CLS] X [SEP]
        """
        return [self.cls_token] + tokens + [self.sep_token]

    def add_special_tokens_sequence_pair(self, token_0, token_1):
        """
        Adds special tokens to a sequence pair for sequence classification tasks.
        A BERT sequence pair has the following format: [CLS] A [SEP] B [SEP]
        """
        sep = [self.sep_token]
        cls = [self.cls_token]
        return cls + token_0 + sep + token_1 + sep

    def add_special_tokens_ids_sequence_pair(self, token_ids_0, token_ids_1):
        """
        Adds special tokens to a sequence pair for sequence classification tasks.
        A BERT sequence pair has the following format: [CLS] A [SEP] B [SEP]
        """
        sep = [self.sep_token_id]
        cls = [self.cls_token_id]
        return cls + token_ids_0 + sep + token_ids_1 + sep

    def add_padding_tokens(self, token_ids, length, right=True):
        """
        Adds padding tokens to return a sequence of length max_length.
        By  default padding tokens are added to the right of the sequence.
        """
        padding = [self.pad_token_id] * (length - len(token_ids))
        if right:
            return token_ids + padding
        else:
            return padding + token_ids

    def save_vocabulary(self, vocab_path):
        """Save the tokenizer vocabulary to a file."""
        index = 0
        vocab_file = vocab_path
        with open(vocab_file, "w", encoding="utf-8") as writer:
            for token, token_index in sorted(
                    self.vocab.items(), key=lambda kv: kv[1]
            ):
                if index != token_index:
                    logger.warning(
                        "Saving vocabulary to {}: vocabulary indices are not consecutive."
                        " Please check that the vocabulary is not corrupted!".
                            format(vocab_file)
                    )
                    index = token_index
                writer.write(token + u"\n")
                index += 1
        return (vocab_file,)

class BasicSmilesTokenizer(object):
    """Run basic SMILES tokenization"""

    def __init__(self, regex_pattern=SMI_REGEX_PATTERN):
        """ Constructs a BasicSMILESTokenizer.
        Args:
            **regex**: SMILES token regex
        """
        self.regex_pattern = regex_pattern
        self.regex = re.compile(self.regex_pattern)

    def tokenize(self, text):
        """ Basic Tokenization of a SMILES.
        """
        tokens = [token for token in self.regex.findall(text)]
        return tokens

def load_vocab(vocab_file):
    """Loads a vocabulary file into a dictionary."""
    vocab = collections.OrderedDict()
    with open(vocab_file, "r", encoding="utf-8") as reader:
        tokens = reader.readlines()
    for index, token in enumerate(tokens):
        token = token.rstrip("\n")
        vocab[token] = index
    return vocab<commit_msg>commenting source of tokenizer<commit_after># Requriments - transformers, tokenizers
# Right now, the Smiles Tokenizer uses an exiesting vocab file from rxnfp that is fairly comprehensive and from the USPTO dataset.
# The vocab may be expanded in the near future

import collections
import logging
import os
import re
import numpy as np
import pkg_resources
from typing import List
from transformers import BertTokenizer

# export
SMI_REGEX_PATTERN = r"(\[[^\]]+]|Br?|Cl?|N|O|S|P|F|I|b|c|n|o|s|p|\(|\)|\.|=|#|-|\+|\\|\/|:|~|@|\?|>>?|\*|\$|\%[0-9]{2}|[0-9])"

def get_default_tokenizer():
    default_vocab_path = (
        pkg_resources.resource_filename(
            "chemberta",
            "tokenizers/vocab.txt"
        )
    )
    return SmilesTokenizer(default_vocab_path)

class SmilesTokenizer(BertTokenizer):
    r"""
    Constructs a SmilesTokenizer.
    Bulk of code is from https://github.com/huggingface/transformers and https://github.com/rxn4chemistry/rxnfp
    Args:
        vocab_file: Path to a SMILES character per line vocabulary file
    """

    def __init__(
            self,
            vocab_file='',
            # unk_token="[UNK]",
            # sep_token="[SEP]",
            # pad_token="[PAD]",
            # cls_token="[CLS]",
            # mask_token="[MASK]",
            **kwargs
    ):
        """Constructs a BertTokenizer.
        Args:
            **vocab_file**: Path to a SMILES character per line vocabulary file
        """
        super().__init__(vocab_file, **kwargs)
        # take into account special tokens in max length
        self.max_len_single_sentence = self.max_len - 2
        self.max_len_sentences_pair = self.max_len - 3

        if not os.path.isfile(vocab_file):
            raise ValueError(
                "Can't find a vocab file at path '{}'.".format(vocab_file)
            )
        self.vocab = load_vocab(vocab_file)
        self.highest_unused_index = max(
            [
                i for i, v in enumerate(self.vocab.keys())
                if v.startswith("[unused")
            ]
        )
        self.ids_to_tokens = collections.OrderedDict(
            [(ids, tok) for tok, ids in self.vocab.items()]
        )
        self.basic_tokenizer = BasicSmilesTokenizer()
        self.init_kwargs["max_len"] = self.max_len

    @property
    def vocab_size(self):
        return len(self.vocab)

    @property
    def vocab_list(self):
        return list(self.vocab.keys())

    def _tokenize(self, text):
        split_tokens = [token for token in self.basic_tokenizer.tokenize(text)]
        return split_tokens

    def _convert_token_to_id(self, token):
        """ Converts a token (str/unicode) in an id using the vocab. """
        return self.vocab.get(token, self.vocab.get(self.unk_token))

    def _convert_id_to_token(self, index):
        """Converts an index (integer) in a token (string/unicode) using the vocab."""
        return self.ids_to_tokens.get(index, self.unk_token)

    def convert_tokens_to_string(self, tokens):
        """ Converts a sequence of tokens (string) in a single string. """
        out_string = " ".join(tokens).replace(" ##", "").strip()
        return out_string

    def add_special_tokens_ids_single_sequence(self, token_ids):
        """
        Adds special tokens to the a sequence for sequence classification tasks.
        A BERT sequence has the following format: [CLS] X [SEP]
        """
        return [self.cls_token_id] + token_ids + [self.sep_token_id]

    def add_special_tokens_single_sequence(self, tokens):
        """
        Adds special tokens to the a sequence for sequence classification tasks.
        A BERT sequence has the following format: [CLS] X [SEP]
        """
        return [self.cls_token] + tokens + [self.sep_token]

    def add_special_tokens_sequence_pair(self, token_0, token_1):
        """
        Adds special tokens to a sequence pair for sequence classification tasks.
        A BERT sequence pair has the following format: [CLS] A [SEP] B [SEP]
        """
        sep = [self.sep_token]
        cls = [self.cls_token]
        return cls + token_0 + sep + token_1 + sep

    def add_special_tokens_ids_sequence_pair(self, token_ids_0, token_ids_1):
        """
        Adds special tokens to a sequence pair for sequence classification tasks.
        A BERT sequence pair has the following format: [CLS] A [SEP] B [SEP]
        """
        sep = [self.sep_token_id]
        cls = [self.cls_token_id]
        return cls + token_ids_0 + sep + token_ids_1 + sep

    def add_padding_tokens(self, token_ids, length, right=True):
        """
        Adds padding tokens to return a sequence of length max_length.
        By  default padding tokens are added to the right of the sequence.
        """
        padding = [self.pad_token_id] * (length - len(token_ids))
        if right:
            return token_ids + padding
        else:
            return padding + token_ids

    def save_vocabulary(self, vocab_path):
        """Save the tokenizer vocabulary to a file."""
        index = 0
        vocab_file = vocab_path
        with open(vocab_file, "w", encoding="utf-8") as writer:
            for token, token_index in sorted(
                    self.vocab.items(), key=lambda kv: kv[1]
            ):
                if index != token_index:
                    logger.warning(
                        "Saving vocabulary to {}: vocabulary indices are not consecutive."
                        " Please check that the vocabulary is not corrupted!".
                            format(vocab_file)
                    )
                    index = token_index
                writer.write(token + u"\n")
                index += 1
        return (vocab_file,)

class BasicSmilesTokenizer(object):
    """Run basic SMILES tokenization"""

    def __init__(self, regex_pattern=SMI_REGEX_PATTERN):
        """ Constructs a BasicSMILESTokenizer.
        Args:
            **regex**: SMILES token regex
        """
        self.regex_pattern = regex_pattern
        self.regex = re.compile(self.regex_pattern)

    def tokenize(self, text):
        """ Basic Tokenization of a SMILES.
        """
        tokens = [token for token in self.regex.findall(text)]
        return tokens

def load_vocab(vocab_file):
    """Loads a vocabulary file into a dictionary."""
    vocab = collections.OrderedDict()
    with open(vocab_file, "r", encoding="utf-8") as reader:
        tokens = reader.readlines()
    for index, token in enumerate(tokens):
        token = token.rstrip("\n")
        vocab[token] = index
    return vocab<|endoftext|>


<commit_before>import io
import json
import os
import tarfile
import uuid
from contextlib import contextmanager
from json import JSONDecodeError
from pathlib import Path
from random import randint
from time import sleep
from typing import Tuple

import docker
from django.conf import settings
from django.core.files import File
from docker.api.container import ContainerApiMixin
from docker.errors import ContainerError, APIError
from docker.tls import TLSConfig
from docker.types import LogConfig
from requests import HTTPError

class Executor(object):
    def __init__(
        self,
        *,
        job_id: uuid.UUID,
        input_files: Tuple[File, ...],
        exec_image: File,
        exec_image_sha256: str,
        results_file: Path,
    ):
        super().__init__()
        self._job_id = str(job_id)
        self._input_files = input_files
        self._exec_image = exec_image
        self._exec_image_sha256 = exec_image_sha256
        self._io_image = settings.CONTAINER_EXEC_IO_IMAGE
        self._results_file = results_file

        client_kwargs = {"base_url": settings.CONTAINER_EXEC_DOCKER_BASE_URL}

        if settings.CONTAINER_EXEC_DOCKER_TLSVERIFY:
            tlsconfig = TLSConfig(
                verify=True,
                client_cert=(
                    settings.CONTAINER_EXEC_DOCKER_TLSCERT,
                    settings.CONTAINER_EXEC_DOCKER_TLSKEY,
                ),
                ca_cert=settings.CONTAINER_EXEC_DOCKER_TLSCACERT,
            )
            client_kwargs.update({"tls": tlsconfig})

        self._client = docker.DockerClient(**client_kwargs)

        self._input_volume = f"{self._job_id}-input"
        self._output_volume = f"{self._job_id}-output"

        self._run_kwargs = {
            "labels": {"job_id": self._job_id},
            "network_disabled": True,
            "mem_limit": settings.CONTAINER_EXEC_MEMORY_LIMIT,
            # Set to the same as mem_limit to avoid using swap
            "memswap_limit": settings.CONTAINER_EXEC_MEMORY_LIMIT,
            "cpu_period": settings.CONTAINER_EXEC_CPU_PERIOD,
            "cpu_quota": settings.CONTAINER_EXEC_CPU_QUOTA,
            # Use the default weight
            "cpu_shares": 1024,
            "runtime": settings.CONTAINER_EXEC_DOCKER_RUNTIME,
            "cap_drop": ["all"],
            "security_opt": ["no-new-privileges"],
            "pids_limit": 64,
            "log_config": LogConfig(
                type=LogConfig.types.JSON, config={"max-size": "1g"}
            ),
        }

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        flt = {"label": f"job_id={self._job_id}"}

        for container in self._client.containers.list(filters=flt):
            container.stop()

        self.__retry_docker_obj_prune(obj=self._client.containers, filters=flt)
        self.__retry_docker_obj_prune(obj=self._client.volumes, filters=flt)

    @staticmethod
    def __retry_docker_obj_prune(*, obj, filters: dict):
        # Retry and exponential backoff of the prune command as only 1 prune
        # operation can occur at a time on a docker host
        num_retries = 0
        e = APIError
        while num_retries < 3:
            try:
                obj.prune(filters=filters)
                break
            except (APIError, HTTPError) as e:
                num_retries += 1
                sleep((2 ** num_retries) + (randint(0, 1000) / 1000))
        else:
            raise e

    def execute(self) -> dict:
        self._pull_images()
        self._create_io_volumes()
        self._provision_input_volume()
        self._chmod_output()
        self._execute_container()
        return self._get_result()

    def _pull_images(self):
        self._client.images.pull(repository=self._io_image)

        if self._exec_image_sha256 not in [
            img.id for img in self._client.images.list()
        ]:
            with self._exec_image.open("rb") as f:
                self._client.images.load(f)

    def _create_io_volumes(self):
        for volume in [self._input_volume, self._output_volume]:
            self._client.volumes.create(
                name=volume, labels=self._run_kwargs["labels"]
            )

    def _provision_input_volume(self):
        try:
            with cleanup(
                self._client.containers.run(
                    image=self._io_image,
                    volumes={
                        self._input_volume: {"bind": "/input/", "mode": "rw"}
                    },
                    name=f"{self._job_id}-writer",
                    detach=True,
                    tty=True,
                    **self._run_kwargs,
                )
            ) as writer:
                self._copy_input_files(writer=writer)
        except Exception as exc:
            raise RuntimeError(str(exc))

    def _copy_input_files(self, writer):
        for file in self._input_files:
            put_file(
                container=writer,
                src=file,
                dest=f"/input/{Path(file.name).name}",
            )

    def _chmod_output(self):
        """ Ensure that the output is writable """
        try:
            self._client.containers.run(
                image=self._io_image,
                volumes={
                    self._output_volume: {"bind": "/output/", "mode": "rw"}
                },
                name=f"{self._job_id}-chmod-output",
                command="chmod 777 /output/",
                remove=True,
                **self._run_kwargs,
            )
        except Exception as exc:
            raise RuntimeError(str(exc))

    def _execute_container(self):
        try:
            self._client.containers.run(
                image=self._exec_image_sha256,
                volumes={
                    self._input_volume: {"bind": "/input/", "mode": "rw"},
                    self._output_volume: {"bind": "/output/", "mode": "rw"},
                },
                name=f"{self._job_id}-executor",
                remove=True,
                **self._run_kwargs,
            )
        except ContainerError as exc:
            raise RuntimeError(exc.stderr.decode())

    def _get_result(self) -> dict:
        """
        Read and parse the created results file. Due to a bug in the docker
        client, copy the file to memory first rather than cat and read
        stdout.
        """
        try:
            with cleanup(
                self._client.containers.run(
                    image=self._io_image,
                    volumes={
                        self._output_volume: {"bind": "/output/", "mode": "ro"}
                    },
                    name=f"{self._job_id}-reader",
                    detach=True,
                    tty=True,
                    **self._run_kwargs,
                )
            ) as reader:
                result = get_file(container=reader, src=self._results_file)
        except Exception as e:
            raise RuntimeError(str(e))

        try:
            result = json.loads(
                result.read().decode(),
                parse_constant=lambda x: None,  # Removes -inf, inf and NaN
            )
        except JSONDecodeError as exc:
            raise RuntimeError(exc.msg)

        return result

@contextmanager
def cleanup(container: ContainerApiMixin):
    """
    Cleans up a docker container which is running in detached mode

    :param container: An instance of a container
    :return:
    """
    try:
        yield container

    finally:
        container.stop()
        container.remove(force=True)

def put_file(*, container: ContainerApiMixin, src: File, dest: str) -> ():
    """
    Puts a file on the host into a container.
    This method will create an in memory tar archive, add the src file to this
    and upload it to the docker container where it will be unarchived at dest.

    :param container: The container to write to
    :param src: The path to the source file on the host
    :param dest: The path to the target file in the container
    :return:
    """
    tar_b = io.BytesIO()

    tarinfo = tarfile.TarInfo(name=os.path.basename(dest))
    tarinfo.size = src.size

    with tarfile.open(fileobj=tar_b, mode="w") as tar, src.open("rb") as f:
        tar.addfile(tarinfo, fileobj=f)

    tar_b.seek(0)
    container.put_archive(os.path.dirname(dest), tar_b)

def get_file(*, container: ContainerApiMixin, src: Path):
    tarstrm, info = container.get_archive(src)

    if info["size"] > 2E9:
        raise ValueError(f"File {src} is too big to be decompressed.")

    file_obj = io.BytesIO()
    for ts in tarstrm:
        file_obj.write(ts)

    file_obj.seek(0)
    tar = tarfile.open(mode="r", fileobj=file_obj)
    content = tar.extractfile(src.name)

    return content
<commit_msg>Chmod io volumes<commit_after>import io
import json
import os
import tarfile
import uuid
from contextlib import contextmanager
from json import JSONDecodeError
from pathlib import Path
from random import randint
from time import sleep
from typing import Tuple

import docker
from django.conf import settings
from django.core.files import File
from docker.api.container import ContainerApiMixin
from docker.errors import ContainerError, APIError
from docker.tls import TLSConfig
from docker.types import LogConfig
from requests import HTTPError

class Executor(object):
    def __init__(
        self,
        *,
        job_id: uuid.UUID,
        input_files: Tuple[File, ...],
        exec_image: File,
        exec_image_sha256: str,
        results_file: Path,
    ):
        super().__init__()
        self._job_id = str(job_id)
        self._input_files = input_files
        self._exec_image = exec_image
        self._exec_image_sha256 = exec_image_sha256
        self._io_image = settings.CONTAINER_EXEC_IO_IMAGE
        self._results_file = results_file

        client_kwargs = {"base_url": settings.CONTAINER_EXEC_DOCKER_BASE_URL}

        if settings.CONTAINER_EXEC_DOCKER_TLSVERIFY:
            tlsconfig = TLSConfig(
                verify=True,
                client_cert=(
                    settings.CONTAINER_EXEC_DOCKER_TLSCERT,
                    settings.CONTAINER_EXEC_DOCKER_TLSKEY,
                ),
                ca_cert=settings.CONTAINER_EXEC_DOCKER_TLSCACERT,
            )
            client_kwargs.update({"tls": tlsconfig})

        self._client = docker.DockerClient(**client_kwargs)

        self._input_volume = f"{self._job_id}-input"
        self._output_volume = f"{self._job_id}-output"

        self._run_kwargs = {
            "labels": {"job_id": self._job_id},
            "network_disabled": True,
            "mem_limit": settings.CONTAINER_EXEC_MEMORY_LIMIT,
            # Set to the same as mem_limit to avoid using swap
            "memswap_limit": settings.CONTAINER_EXEC_MEMORY_LIMIT,
            "cpu_period": settings.CONTAINER_EXEC_CPU_PERIOD,
            "cpu_quota": settings.CONTAINER_EXEC_CPU_QUOTA,
            # Use the default weight
            "cpu_shares": 1024,
            "runtime": settings.CONTAINER_EXEC_DOCKER_RUNTIME,
            "cap_drop": ["all"],
            "security_opt": ["no-new-privileges"],
            "pids_limit": 64,
            "log_config": LogConfig(
                type=LogConfig.types.JSON, config={"max-size": "1g"}
            ),
        }

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        flt = {"label": f"job_id={self._job_id}"}

        for container in self._client.containers.list(filters=flt):
            container.stop()

        self.__retry_docker_obj_prune(obj=self._client.containers, filters=flt)
        self.__retry_docker_obj_prune(obj=self._client.volumes, filters=flt)

    @staticmethod
    def __retry_docker_obj_prune(*, obj, filters: dict):
        # Retry and exponential backoff of the prune command as only 1 prune
        # operation can occur at a time on a docker host
        num_retries = 0
        e = APIError
        while num_retries < 3:
            try:
                obj.prune(filters=filters)
                break
            except (APIError, HTTPError) as e:
                num_retries += 1
                sleep((2 ** num_retries) + (randint(0, 1000) / 1000))
        else:
            raise e

    def execute(self) -> dict:
        self._pull_images()
        self._create_io_volumes()
        self._provision_input_volume()
        self._chmod_volumes()
        self._execute_container()
        return self._get_result()

    def _pull_images(self):
        self._client.images.pull(repository=self._io_image)

        if self._exec_image_sha256 not in [
            img.id for img in self._client.images.list()
        ]:
            with self._exec_image.open("rb") as f:
                self._client.images.load(f)

    def _create_io_volumes(self):
        for volume in [self._input_volume, self._output_volume]:
            self._client.volumes.create(
                name=volume, labels=self._run_kwargs["labels"]
            )

    def _provision_input_volume(self):
        try:
            with cleanup(
                self._client.containers.run(
                    image=self._io_image,
                    volumes={
                        self._input_volume: {"bind": "/input/", "mode": "rw"}
                    },
                    name=f"{self._job_id}-writer",
                    detach=True,
                    tty=True,
                    **self._run_kwargs,
                )
            ) as writer:
                self._copy_input_files(writer=writer)
        except Exception as exc:
            raise RuntimeError(str(exc))

    def _copy_input_files(self, writer):
        for file in self._input_files:
            put_file(
                container=writer,
                src=file,
                dest=f"/input/{Path(file.name).name}",
            )

    def _chmod_volumes(self):
        """ Ensure that the i/o directories are writable """
        try:
            self._client.containers.run(
                image=self._io_image,
                volumes={
                    self._input_volume: {"bind": "/input/", "mode": "rw"},
                    self._output_volume: {"bind": "/output/", "mode": "rw"},
                },
                name=f"{self._job_id}-chmod-volumes",
                command=f"chmod -R 0777 /input/ /output/",
                remove=True,
                **self._run_kwargs,
            )
        except Exception as exc:
            raise RuntimeError(str(exc))

    def _execute_container(self):
        try:
            self._client.containers.run(
                image=self._exec_image_sha256,
                volumes={
                    self._input_volume: {"bind": "/input/", "mode": "rw"},
                    self._output_volume: {"bind": "/output/", "mode": "rw"},
                },
                name=f"{self._job_id}-executor",
                remove=True,
                **self._run_kwargs,
            )
        except ContainerError as exc:
            raise RuntimeError(exc.stderr.decode())

    def _get_result(self) -> dict:
        """
        Read and parse the created results file. Due to a bug in the docker
        client, copy the file to memory first rather than cat and read
        stdout.
        """
        try:
            with cleanup(
                self._client.containers.run(
                    image=self._io_image,
                    volumes={
                        self._output_volume: {"bind": "/output/", "mode": "ro"}
                    },
                    name=f"{self._job_id}-reader",
                    detach=True,
                    tty=True,
                    **self._run_kwargs,
                )
            ) as reader:
                result = get_file(container=reader, src=self._results_file)
        except Exception as e:
            raise RuntimeError(str(e))

        try:
            result = json.loads(
                result.read().decode(),
                parse_constant=lambda x: None,  # Removes -inf, inf and NaN
            )
        except JSONDecodeError as exc:
            raise RuntimeError(exc.msg)

        return result

@contextmanager
def cleanup(container: ContainerApiMixin):
    """
    Cleans up a docker container which is running in detached mode

    :param container: An instance of a container
    :return:
    """
    try:
        yield container

    finally:
        container.stop()
        container.remove(force=True)

def put_file(*, container: ContainerApiMixin, src: File, dest: str) -> ():
    """
    Puts a file on the host into a container.
    This method will create an in memory tar archive, add the src file to this
    and upload it to the docker container where it will be unarchived at dest.

    :param container: The container to write to
    :param src: The path to the source file on the host
    :param dest: The path to the target file in the container
    :return:
    """
    tar_b = io.BytesIO()

    tarinfo = tarfile.TarInfo(name=os.path.basename(dest))
    tarinfo.size = src.size

    with tarfile.open(fileobj=tar_b, mode="w") as tar, src.open("rb") as f:
        tar.addfile(tarinfo, fileobj=f)

    tar_b.seek(0)
    container.put_archive(os.path.dirname(dest), tar_b)

def get_file(*, container: ContainerApiMixin, src: Path):
    tarstrm, info = container.get_archive(src)

    if info["size"] > 2E9:
        raise ValueError(f"File {src} is too big to be decompressed.")

    file_obj = io.BytesIO()
    for ts in tarstrm:
        file_obj.write(ts)

    file_obj.seek(0)
    tar = tarfile.open(mode="r", fileobj=file_obj)
    content = tar.extractfile(src.name)

    return content
<|endoftext|>


<commit_before>#!/usr/bin/env python
# coding:utf-8
# vi:tabstop=4:shiftwidth=4:expandtab:sts=4

import Queue
import thread

class AsyncIterate:

    def __init__(self,it):
        self.it=it
        self.instance=None
        self.quitflag=False
        self.q=Queue.Queue(2)

    def __copy__(self):
        return AsyncIterate(self.it)

    def __call__(self,*args,**kwargs):
        assert self.instance is None
        self.instance=self.it(*args,**kwargs)
        if self.instance is not None:
            thread.start_new_thread(self.backend,())
            data=self.q.get()
            while data is not None:
                try:
                    yield data
                except GeneratorExit:
                    #print 'GeneratorExit'
                    self.quitflag=True
                    #self.instance.close()
                    while self.q.get() is not None:
                        pass
                    break
                data=self.q.get()
            #print 'end'
            self.instance=None
            self.quitflag=False

    def backend(self):
        try:
            while True:
                data=self.instance.next()
                if self.quitflag:
                    self.instance.close()
                if data is not None:
                    self.q.put(data)
        except (GeneratorExit,StopIteration):
            self.q.put(None)

if __name__=='__main__':
    def myit(n):
        for x in range(n):
            yield x

    iterator=AsyncIterate(myit)

    it=iterator(5)
    for t in it:
        print t
        it.close()
        break
    it=iterator(5)
    for t in it:
        print t
<commit_msg>exit on error<commit_after>#!/usr/bin/env python
# coding:utf-8
# vi:tabstop=4:shiftwidth=4:expandtab:sts=4

import os
import sys
import Queue
import thread
import traceback

class AsyncIterate:

    def __init__(self,it):
        self.it=it
        self.instance=None
        self.quitflag=False
        self.q=Queue.Queue(2)

    def __copy__(self):
        return AsyncIterate(self.it)

    def __call__(self,*args,**kwargs):
        assert self.instance is None
        self.instance=self.it(*args,**kwargs)
        if self.instance is not None:
            thread.start_new_thread(self.backend,())
            data=self.q.get()
            while data is not None:
                try:
                    yield data
                except GeneratorExit:
                    #print 'GeneratorExit'
                    self.quitflag=True
                    #self.instance.close()
                    while self.q.get() is not None:
                        pass
                    break
                data=self.q.get()
            #print 'end'
            self.instance=None
            self.quitflag=False

    def backend(self):
        try:
            while True:
                data=self.instance.next()
                if self.quitflag:
                    self.instance.close()
                if data is not None:
                    self.q.put(data)
        except (GeneratorExit,StopIteration):
            self.q.put(None)
        except:
            print "Unexpected error:", sys.exc_info()[0]
            traceback.print_exc()
            os._exit(1)

if __name__=='__main__':
    def myit(n):
        for x in range(n):
            yield x

    iterator=AsyncIterate(myit)

    it=iterator(5)
    for t in it:
        print t
        it.close()
        break
    it=iterator(5)
    for t in it:
        print t
<|endoftext|>


<commit_before>__version__= '0.1'
import re
import sys
import pandas as pd
from fractions import Fraction, gcd

import periodic
from periodic import element
from .chemical_formula import ChemicalFormula
from .mixture import Mixture
from .reaction import Reaction
from .utils import LCD, GCD, LCM
from .half_rxn_balancer import HalfReactionBalancer

molecular_formula = dict(glucose='C6H12O6',
                         ammonia='NH3',
                         ammonium='NH4+',
                         oxygen='O2',
                         sucrose='C12H22O11',
                         NAD='C21H26N7O14P2',
                         ADP='C10H12N5O10P2-3',
                         Pi='H2O4P-',
                         carbon_dioxide = 'CO2',
                         proton='H+',
                         NADH='C21H27N7O14P2',
                         ATP='C10H12N5O13P3-4',
                         water='H2O',
                         acetate='CH3COO-',
                         alanine='CH3CHNH2COO-',
                         benzoate='C6H5COO-',
                         citrate='C6H5O7-3',
                         ethanol='CH3CH2OH',
                         formate='HCOO-',
                         glutamate='COOHCH2CH2CHNH2COO-',
                         glycerol='CH2OHCHOHCH2OH',
                         glycine='CH2NH2COOH',
                         lactate='CH3CHOHCOO-',
                         methane='CH4',
                         methanol='CH3OH',
                         palmitate='C16H31O2-',
                         propionate='CH3CH2COO-',
                         pyruvate='CH3COCOO-',
                         succinate='C4H4O4-2',
                         nitrate='NO3-',
                         sulfate='SO4-2',
                         ironIII='Fe+3',
                         ironII='Fe+2',
                         dinitrogen='N2')
mu0m = dict(Pi=-1052.8, # Equilibrator mu0m (millimolar concentrations)
           ATP=-2295.1,
           ADP=-1422.5,
           glucose=-446.8,
           ammonia=-26.5,
           ammonium=-79.0,
           water=-157.6,
           carbon_dioxide=-403.1,
           proton=-17.1)

deltaGEE = dict(
                glucose=41.35,
                acetate=27.40,
                alanine=31.37,
                benzoate=27.34,
                citrate=33.08,
                ethanol=31.18,
                formate=39.19,
                ammonia=-26.57,
                ammonium=-79.37,
                glutamate=30.93,
                glycerol=38.88,
                glycine=39.80,
                lactate=32.29,
                methane=23.53,
                methanol=36.84,
                palmitate=27.26,

                propionate=27.63,
                pyruvate=35.09,
                succinate=29.09,
                oxygen=-78.72,
                nitrate=-72.20,
                sulfate=20.85,
                iron=-74.27,
                pyruvate_to_cell_using_ammonium=18.8
               )
# def make_half_reactions(molecules, deltaGEE):
#     half_rxn = HalfReactionBalancer()
#     rxns = {}
#     for molecule in molecules:
#         rxns[molecule] = half_rxn.balance_half_reaction(oxidized_form='CO2',reduced_form=molecules[molecule])
#         if molecule in deltaGEE:
#             rxns[molecule].setDeltaGEE(deltaGEE[molecule])
#     return rxns

# half_reactions = make_half_reactions( molecular_formula, deltaGEE )
<commit_msg>Added mu0ph7 and bicarbonate<commit_after>__version__= '0.1'
import re
import sys
import pandas as pd
from fractions import Fraction, gcd

import periodic
from periodic import element
from .chemical_formula import ChemicalFormula
from .mixture import Mixture
from .reaction import Reaction
from .utils import LCD, GCD, LCM
from .half_rxn_balancer import HalfReactionBalancer

molecular_formula = dict(glucose='C6H12O6',
                         ammonia='NH3',
                         ammonium='NH4+',
                         bicarbonate='HCO3-',
                         oxygen='O2',
                         sucrose='C12H22O11',
                         NAD='C21H26N7O14P2',
                         ADP='C10H12N5O10P2-3',
                         Pi='H2O4P-',
                         carbon_dioxide = 'CO2',
                         proton='H+',
                         NADH='C21H27N7O14P2',
                         ATP='C10H12N5O13P3-4',
                         water='H2O',
                         acetate='CH3COO-',
                         alanine='CH3CHNH2COO-',
                         benzoate='C6H5COO-',
                         citrate='C6H5O7-3',
                         ethanol='CH3CH2OH',
                         formate='HCOO-',
                         glutamate='COOHCH2CH2CHNH2COO-',
                         glycerol='CH2OHCHOHCH2OH',
                         glycine='CH2NH2COOH',
                         lactate='CH3CHOHCOO-',
                         methane='CH4',
                         methanol='CH3OH',
                         palmitate='C16H31O2-',
                         propionate='CH3CH2COO-',
                         pyruvate='CH3COCOO-',
                         succinate='C4H4O4-2',
                         nitrate='NO3-',
                         sulfate='SO4-2',
                         ironIII='Fe+3',
                         ironII='Fe+2',
                         hydrogen='H2',
                         dinitrogen='N2')

mu0m = {'ADP': -1422.5, # Equilibrator mu0'm (ph7, 1 mM concentrations)
 'ATP': -2295.1,
 'Pi': -1052.8,
 'ammonia': -26.5,
 'ammonium': -79.0,
 'carbon_dioxide': -403.1,
 'glucose': -446.8,
 'proton': -17.1,
 'water': -157.6}

mu0ph7 = dict(Pi=-1052.8, # Equilibrator mu0' (ph7, 1 M concentrations)
              ATP=-2295.8,
              ADP=-1423.6,
              glucose=-429.7,
              ammonia=89.6,
              ammonium=79.0,
              water=-157.6,
              carbon_dioxide=-386.0,
              bicarbonate=-546.8,
              proton=0,
              hydrogen=98.7
)

deltaGEE = dict(
                glucose=41.35,
                acetate=27.40,
                alanine=31.37,
                benzoate=27.34,
                citrate=33.08,
                ethanol=31.18,
                formate=39.19,
                ammonia=-26.57,
                ammonium=-79.37,
                glutamate=30.93,
                glycerol=38.88,
                glycine=39.80,
                lactate=32.29,
                methane=23.53,
                methanol=36.84,
                palmitate=27.26,

                propionate=27.63,
                pyruvate=35.09,
                succinate=29.09,
                oxygen=-78.72,
                nitrate=-72.20,
                sulfate=20.85,
                iron=-74.27,
                pyruvate_to_cell_using_ammonium=18.8
               )
# def make_half_reactions(molecules, deltaGEE):
#     half_rxn = HalfReactionBalancer()
#     rxns = {}
#     for molecule in molecules:
#         rxns[molecule] = half_rxn.balance_half_reaction(oxidized_form='CO2',reduced_form=molecules[molecule])
#         if molecule in deltaGEE:
#             rxns[molecule].setDeltaGEE(deltaGEE[molecule])
#     return rxns

# half_reactions = make_half_reactions( molecular_formula, deltaGEE )
<|endoftext|>


<commit_before>import autonetkit.anm
import autonetkit.ank as ank
import math
import itertools
import autonetkit.ank_pika
import autonetkit.config
settings = autonetkit.config.settings
import autonetkit.log as log
import autonetkit.load.graphml as graphml

__all__ = ['build']

rabbitmq_server = settings['Rabbitmq']['server']
pika_channel = autonetkit.ank_pika.AnkPika(rabbitmq_server)

def build(input_filename):
    #TODO: move this out of main console wrapper
    anm = autonetkit.anm.AbstractNetworkModel()
    input_graph = graphml.load_graphml(input_filename)

    G_in = anm.add_overlay("input", input_graph)

    import autonetkit.plugins.graph_product as graph_product
    graph_product.expand(G_in) # apply graph products if relevant

    if len(ank.unique_attr(G_in, "asn")) > 1:
        # Multiple ASNs set, use label format device.asn
        anm.set_node_label(".",  ['label', 'pop', 'asn'])

# set syntax for routers according to platform
    G_in.update(G_in.nodes("is_router", platform = "junosphere"), syntax="junos")
    G_in.update(G_in.nodes("is_router", platform = "dynagen"), syntax="ios")
    G_in.update(G_in.nodes("is_router", platform = "netkit"), syntax="quagga")
    G_in.update(G_in.nodes("is_router", platform = "cisco"), syntax="ios")

    G_graphics = anm.add_overlay("graphics") # plotting data
    G_graphics.add_nodes_from(G_in, retain=['x', 'y', 'device_type', 'device_subtype', 'pop', 'asn'])

    build_phy(anm)
    build_ip(anm)
    build_ospf(anm)
    build_isis(anm)
    build_bgp(anm)
    return anm

def build_bgp(anm):
    # eBGP
    G_phy = anm['phy']
    G_in = anm['input']
    G_bgp = anm.add_overlay("bgp", directed = True)
    G_bgp.add_nodes_from(G_in.nodes("is_router"))
    ebgp_edges = [edge for edge in G_in.edges() if not edge.attr_equal("asn")]
    G_bgp.add_edges_from(ebgp_edges, bidirectional = True, type = 'ebgp')

# now iBGP
    if len(G_phy) < 5:
# full mesh
        for asn, devices in G_phy.groupby("asn").items():
            routers = [d for d in devices if d.is_router]
            ibgp_edges = [ (s, t) for s in routers for t in routers if s!=t]
            G_bgp.add_edges_from(ibgp_edges, type = 'ibgp')
    else:
        import autonetkit.plugins.route_reflectors as route_reflectors
        route_reflectors.allocate(G_phy, G_bgp)

#TODO: probably want to use l3 connectivity graph for allocating route reflectors

    ebgp_nodes = [d for d in G_bgp if any(edge.type == 'ebgp' for edge in d.edges())]
    G_bgp.update(ebgp_nodes, ebgp=True)

def build_ip(anm):
    import autonetkit.plugins.ip as ip
    G_ip = anm.add_overlay("ip")
    G_in = anm['input']
    G_graphics = anm['graphics']
    G_phy = anm['phy']

    G_ip.add_nodes_from(G_in)
    G_ip.add_edges_from(G_in.edges(type="physical"))

    ank.aggregate_nodes(G_ip, G_ip.nodes("is_switch"), retain = "edge_id")
#TODO: add function to update edge properties: can overload node update?

    edges_to_split = [edge for edge in G_ip.edges() if edge.attr_both("is_l3device")]
    split_created_nodes = list(ank.split(G_ip, edges_to_split, retain='edge_id'))
    for node in split_created_nodes:
        node.overlay.graphics.x = ank.neigh_average(G_ip, node, "x", G_graphics)
        node.overlay.graphics.y = ank.neigh_average(G_ip, node, "y", G_graphics)
        node.overlay.graphics.asn = ank.neigh_most_frequent(G_ip, node, "asn", G_phy) # arbitrary choice
#TODO: could choose largest ASN if tie break

    switch_nodes = G_ip.nodes("is_switch")# regenerate due to aggregated
    G_ip.update(switch_nodes, collision_domain=True) # switches are part of collision domain
    G_ip.update(split_created_nodes, collision_domain=True)
# Assign collision domain to a host if all neighbours from same host
    for node in split_created_nodes:
        if ank.neigh_equal(G_ip, node, "host", G_phy):
            node.host = ank.neigh_attr(G_ip, node, "host", G_phy).next() # first attribute

# set collision domain IPs
#TODO: trim next line
    collision_domain_id = itertools.count(0)
    for node in G_ip.nodes("collision_domain"):
        graphics_node = G_graphics.node(node)
        graphics_node.device_type = "collision_domain"
        cd_id = collision_domain_id.next()
        node.cd_id = cd_id
#TODO: Use this label
        if not node.is_switch:
            label = "_".join(sorted(ank.neigh_attr(G_ip, node, "label", G_phy)))
            cd_label = "cd_%s" % label # switches keep their names
            node.label = cd_label
            node.cd_id = cd_label
            graphics_node.label = cd_label

    ip.allocate_ips(G_ip)
    ank.save(G_ip)

def build_phy(anm):
    G_in = anm['input']
    G_phy = anm['phy']
    G_phy.add_nodes_from(G_in, retain=['label', 'device_type', 'device_subtype', 'asn', 'platform', 'host', 'syntax'])
    if G_in.data.Creator == "Topology Zoo Toolset":
        ank.copy_attr_from(G_in, G_phy, "Network") # Copy Network from Zoo
    G_phy.add_edges_from(G_in.edges(type="physical"))

def build_ospf(anm):
    G_in = anm['input']
    G_ospf = anm.add_overlay("ospf")
    G_ospf.add_nodes_from(G_in.nodes("is_router"), retain=['asn'])
    #update_pika(anm)
    G_ospf.add_nodes_from(G_in.nodes("is_switch"), retain=['asn'])
    #update_pika(anm)
    G_ospf.add_edges_from(G_in.edges(), retain = ['edge_id'])

    #update_pika(anm)
    ank.aggregate_nodes(G_ospf, G_ospf.nodes("is_switch"), retain = "edge_id")
    #update_pika(anm)
    ank.explode_nodes(G_ospf, G_ospf.nodes("is_switch"), retain= "edge_id")
    #update_pika(anm)

    #update_pika(anm)
    G_ospf.remove_edges_from([link for link in G_ospf.edges() if link.src.asn != link.dst.asn])
    for link in G_ospf.edges():
        link.area = 0
        link.cost = 1

    #update_pika(anm)

def ip_to_net_ent_title_ios(ip):
    """ Converts an IP address into an OSI Network Entity Title
    suitable for use in IS-IS on IOS.

    >>> ip_to_net_ent_title_ios(IPAddress("192.168.19.1"))
    '49.1921.6801.9001.00'
    """
    try:
        ip_words = ip.words
    except AttributeError:
        import netaddr # try to cast to IP Address
        ip = netaddr.IPAddress(ip)
        ip_words = ip.words

    log.debug("Converting IP to OSI ENT format")
    area_id = "49"
    ip_octets = "".join("%03d" % int(octet) for octet in ip_words) # single string, padded if needed
    return ".".join([area_id, ip_octets[0:4], ip_octets[4:8], ip_octets[8:12], "00"])

def build_isis(anm):
    G_in = anm['input']
    G_ip = anm['ip']
    G_isis = anm.add_overlay("isis")
    #G_isis.add_nodes_from(G_in.nodes("is_router", igp = "isis"), retain=['asn'])
#TODO: filter only igp=isis nodes, set the igp as a default in build_network
    G_isis.add_nodes_from(G_in.nodes("is_router"), retain=['asn'])
    G_isis.add_nodes_from(G_in.nodes("is_switch"), retain=['asn'])
    G_isis.add_edges_from(G_in.edges(), retain = ['edge_id'])
# Merge and explode switches
    ank.aggregate_nodes(G_isis, G_isis.nodes("is_switch"), retain = "edge_id")
    ank.explode_nodes(G_isis, G_isis.nodes("is_switch"), retain = "edge_id")

    G_isis.remove_edges_from([link for link in G_isis.edges() if link.src.asn != link.dst.asn])

    for node in G_isis:
        ip_node = G_ip.node(node)
        node.net = ip_to_net_ent_title_ios(ip_node.loopback)

def update_pika(anm):
    log.debug("Sending anm to pika")
    body = autonetkit.ank_json.dumps(anm, None)
    pika_channel.publish_compressed("www", "client", body)
<commit_msg>handle non .graphml<commit_after>import autonetkit
import autonetkit.anm
import autonetkit.ank as ank
import math
import itertools
import autonetkit.ank_pika
import autonetkit.config
settings = autonetkit.config.settings
import autonetkit.log as log
import autonetkit.load.graphml as graphml
import autonetkit.exception

__all__ = ['build']

rabbitmq_server = settings['Rabbitmq']['server']
pika_channel = autonetkit.ank_pika.AnkPika(rabbitmq_server)

def build(input_filename):
    #TODO: move this out of main console wrapper
    anm = autonetkit.anm.AbstractNetworkModel()
    try:
        input_graph = graphml.load_graphml(input_filename)
    except autonetkit.exception.AnkIncorrectFileFormat:
# try a different reader
        try:
            import autonetkit.load.bug as bug
        except ImportError:
            return # module not present (development module)
        input_graph = bug.load(input_filename)
        import pprint
        pprint.pprint(input_graph.nodes(data=True))

    G_in = anm.add_overlay("input", input_graph)

    import autonetkit.plugins.graph_product as graph_product
    graph_product.expand(G_in) # apply graph products if relevant

    if len(ank.unique_attr(G_in, "asn")) > 1:
        # Multiple ASNs set, use label format device.asn
        anm.set_node_label(".",  ['label', 'pop', 'asn'])

# set syntax for routers according to platform
    G_in.update(G_in.nodes("is_router", platform = "junosphere"), syntax="junos")
    G_in.update(G_in.nodes("is_router", platform = "dynagen"), syntax="ios")
    G_in.update(G_in.nodes("is_router", platform = "netkit"), syntax="quagga")
    G_in.update(G_in.nodes("is_router", platform = "cisco"), syntax="ios")

    G_graphics = anm.add_overlay("graphics") # plotting data
    G_graphics.add_nodes_from(G_in, retain=['x', 'y', 'device_type', 'device_subtype', 'pop', 'asn'])

    build_phy(anm)
    build_ip(anm)
    build_ospf(anm)
    build_isis(anm)
    build_bgp(anm)
    return anm

def build_bgp(anm):
    # eBGP
    G_phy = anm['phy']
    G_in = anm['input']
    G_bgp = anm.add_overlay("bgp", directed = True)
    G_bgp.add_nodes_from(G_in.nodes("is_router"))
    ebgp_edges = [edge for edge in G_in.edges() if not edge.attr_equal("asn")]
    G_bgp.add_edges_from(ebgp_edges, bidirectional = True, type = 'ebgp')

# now iBGP
    if len(G_phy) < 5:
# full mesh
        for asn, devices in G_phy.groupby("asn").items():
            routers = [d for d in devices if d.is_router]
            ibgp_edges = [ (s, t) for s in routers for t in routers if s!=t]
            G_bgp.add_edges_from(ibgp_edges, type = 'ibgp')
    else:
        import autonetkit.plugins.route_reflectors as route_reflectors
        route_reflectors.allocate(G_phy, G_bgp)

#TODO: probably want to use l3 connectivity graph for allocating route reflectors

    ebgp_nodes = [d for d in G_bgp if any(edge.type == 'ebgp' for edge in d.edges())]
    G_bgp.update(ebgp_nodes, ebgp=True)

def build_ip(anm):
    import autonetkit.plugins.ip as ip
    G_ip = anm.add_overlay("ip")
    G_in = anm['input']
    G_graphics = anm['graphics']
    G_phy = anm['phy']

    G_ip.add_nodes_from(G_in)
    G_ip.add_edges_from(G_in.edges(type="physical"))

    ank.aggregate_nodes(G_ip, G_ip.nodes("is_switch"), retain = "edge_id")
#TODO: add function to update edge properties: can overload node update?

    edges_to_split = [edge for edge in G_ip.edges() if edge.attr_both("is_l3device")]
    split_created_nodes = list(ank.split(G_ip, edges_to_split, retain='edge_id'))
    for node in split_created_nodes:
        node.overlay.graphics.x = ank.neigh_average(G_ip, node, "x", G_graphics)
        node.overlay.graphics.y = ank.neigh_average(G_ip, node, "y", G_graphics)
        node.overlay.graphics.asn = ank.neigh_most_frequent(G_ip, node, "asn", G_phy) # arbitrary choice
#TODO: could choose largest ASN if tie break

    switch_nodes = G_ip.nodes("is_switch")# regenerate due to aggregated
    G_ip.update(switch_nodes, collision_domain=True) # switches are part of collision domain
    G_ip.update(split_created_nodes, collision_domain=True)
# Assign collision domain to a host if all neighbours from same host
    for node in split_created_nodes:
        if ank.neigh_equal(G_ip, node, "host", G_phy):
            node.host = ank.neigh_attr(G_ip, node, "host", G_phy).next() # first attribute

# set collision domain IPs
#TODO: trim next line
    collision_domain_id = itertools.count(0)
    for node in G_ip.nodes("collision_domain"):
        graphics_node = G_graphics.node(node)
        graphics_node.device_type = "collision_domain"
        cd_id = collision_domain_id.next()
        node.cd_id = cd_id
#TODO: Use this label
        if not node.is_switch:
            label = "_".join(sorted(ank.neigh_attr(G_ip, node, "label", G_phy)))
            cd_label = "cd_%s" % label # switches keep their names
            node.label = cd_label
            node.cd_id = cd_label
            graphics_node.label = cd_label

    ip.allocate_ips(G_ip)
    ank.save(G_ip)

def build_phy(anm):
    G_in = anm['input']
    G_phy = anm['phy']
    G_phy.add_nodes_from(G_in, retain=['label', 'device_type', 'device_subtype', 'asn', 'platform', 'host', 'syntax'])
    if G_in.data.Creator == "Topology Zoo Toolset":
        ank.copy_attr_from(G_in, G_phy, "Network") # Copy Network from Zoo
    G_phy.add_edges_from(G_in.edges(type="physical"))

def build_ospf(anm):
    G_in = anm['input']
    G_ospf = anm.add_overlay("ospf")
    G_ospf.add_nodes_from(G_in.nodes("is_router"), retain=['asn'])
    #update_pika(anm)
    G_ospf.add_nodes_from(G_in.nodes("is_switch"), retain=['asn'])
    #update_pika(anm)
    G_ospf.add_edges_from(G_in.edges(), retain = ['edge_id'])

    #update_pika(anm)
    ank.aggregate_nodes(G_ospf, G_ospf.nodes("is_switch"), retain = "edge_id")
    #update_pika(anm)
    ank.explode_nodes(G_ospf, G_ospf.nodes("is_switch"), retain= "edge_id")
    #update_pika(anm)

    #update_pika(anm)
    G_ospf.remove_edges_from([link for link in G_ospf.edges() if link.src.asn != link.dst.asn])
    for link in G_ospf.edges():
        link.area = 0
        link.cost = 1

    #update_pika(anm)

def ip_to_net_ent_title_ios(ip):
    """ Converts an IP address into an OSI Network Entity Title
    suitable for use in IS-IS on IOS.

    >>> ip_to_net_ent_title_ios(IPAddress("192.168.19.1"))
    '49.1921.6801.9001.00'
    """
    try:
        ip_words = ip.words
    except AttributeError:
        import netaddr # try to cast to IP Address
        ip = netaddr.IPAddress(ip)
        ip_words = ip.words

    log.debug("Converting IP to OSI ENT format")
    area_id = "49"
    ip_octets = "".join("%03d" % int(octet) for octet in ip_words) # single string, padded if needed
    return ".".join([area_id, ip_octets[0:4], ip_octets[4:8], ip_octets[8:12], "00"])

def build_isis(anm):
    G_in = anm['input']
    G_ip = anm['ip']
    G_isis = anm.add_overlay("isis")
    #G_isis.add_nodes_from(G_in.nodes("is_router", igp = "isis"), retain=['asn'])
#TODO: filter only igp=isis nodes, set the igp as a default in build_network
    G_isis.add_nodes_from(G_in.nodes("is_router"), retain=['asn'])
    G_isis.add_nodes_from(G_in.nodes("is_switch"), retain=['asn'])
    G_isis.add_edges_from(G_in.edges(), retain = ['edge_id'])
# Merge and explode switches
    ank.aggregate_nodes(G_isis, G_isis.nodes("is_switch"), retain = "edge_id")
    ank.explode_nodes(G_isis, G_isis.nodes("is_switch"), retain = "edge_id")

    G_isis.remove_edges_from([link for link in G_isis.edges() if link.src.asn != link.dst.asn])

    for node in G_isis:
        ip_node = G_ip.node(node)
        node.net = ip_to_net_ent_title_ios(ip_node.loopback)

def update_pika(anm):
    log.debug("Sending anm to pika")
    body = autonetkit.ank_json.dumps(anm, None)
    pika_channel.publish_compressed("www", "client", body)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    Author Matthieu Dietrich. Copyright 2012 Camptocamp SA
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from . import account_validate_move
# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:<commit_msg>[FIX] flake8 PEP8 account_move_validation_improvement<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    Author Matthieu Dietrich. Copyright 2012 Camptocamp SA
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from . import account_validate_move
<|endoftext|>


<commit_before>import re
import pandas as pd
import numpy as np
from pint import UnitRegistry
from .utils import LCD
from fractions import Fraction
from . import Mixture, ChemicalFormula
class Reaction:
    u = UnitRegistry()
    u.define('electron-equivalent = [] = e-eq = e-equiv = eeq')
    gas_constant = 8.3144598*u.joule/u.mole/u.kelvin
    def __init__(self, rxn, deltaGEE=None ):
        self.rxn = self.parse_reaction( rxn )
        self.setDeltaGEE( deltaGEE )
    def setDeltaGEE( self, deltaGEE):
        self.deltaGEE = deltaGEE

    def get_deltaGEE_from_potentials( self, mu0 ):
        deltaGEE = 0
        for species in self.get_species():
            try:
                deltaGEE +=  mu0[str(species)]*self.get_stoichiometry_of_species( species )
            except KeyError:
                print("Species {} not in mu0".format( species ))
                deltaGEE += 0
        return deltaGEE

    def hasDeltaGEE( self ):
        return self.deltaGEE is not None
    def getDeltaGEE( self ):
        return self.deltaGEE

    # def setDeltaG0( self, deltaG0, temperature=273):
    #     self.deltaGEE = deltaG0 + gas_constant*temperature*u.kelvin*np.ln(10**-7)
    # def getDeltaG0( self, temperature=273):
    #     u = self.u
    #     return self.deltaGEE - self.gas_constant*temperature*u.kelvin*np.ln(10**-7)

    def __mul__(self, other):
        if type(other) in [int, float, Fraction]:
            return self.multiply_constant( other )
        elif type(other) is pd.Series:
            return self.multiply_factor( other )
        else:
            raise NotImplemented('unsupported operand type(s) for *: "{}" and "{}"'.format())
    def __rmul__(self, other):
        if type(other) in [int, float, Fraction]:
            return self.multiply_constant( other )
        elif type(other) is pd.Series:
            raise NotImplemented

    def multiply_constant( self, constant ):
        return self.multiply_factor(pd.Series(Fraction(constant),index=self.get_species(),dtype='object'))
    def multiply_factor( self, factor ):
        rxn = Reaction( str(self ) )
        rxn.rxn['reactant'] = rxn.rxn['reactant'].multiply_factor( factor )
        rxn.rxn['product'] = rxn.rxn['product'].multiply_factor( factor )
        return rxn

    def add_reaction( self, rxn ):
        new_reactants = Mixture(str(self.rxn['reactant'])).add_to_mixture(rxn.rxn['reactant']).\
            subtract_from_mixture(self.rxn['product']).subtract_from_mixture(rxn.rxn['product'])
        new_products = Mixture( str(self.rxn['product'])).add_to_mixture(rxn.rxn['product']).\
            subtract_from_mixture(self.rxn['reactant']).subtract_from_mixture(rxn.rxn['reactant'])
        if self.hasDeltaGEE() and rxn.hasDeltaGEE():
            newDeltaGEE = self.getDeltaGEE() + rxn.getDeltaGEE()
        else:
            newDeltaGEE = None
        return Reaction(str(new_reactants) + ' ==> ' + str(new_products), newDeltaGEE)
    def subtract_reaction( self, rxn ):
        new_reactants = Mixture(str(self.rxn['reactant'])).add_to_mixture(rxn.rxn['product']).\
            subtract_from_mixture(self.rxn['product']).subtract_from_mixture(rxn.rxn['reactant'])
        new_products = Mixture( str(self.rxn['product'])).add_to_mixture(rxn.rxn['reactant']).\
            subtract_from_mixture(self.rxn['reactant']).subtract_from_mixture( rxn.rxn['product'])
        if self.hasDeltaGEE() and rxn.hasDeltaGEE():
            newDeltaGEE = self.getDeltaGEE() - rxn.getDeltaGEE()
        else:
            newDeltaGEE = None
        return Reaction(str(new_reactants) + ' ==> ' + str(new_products), newDeltaGEE)

    def get_charge( self ):
        return self.rxn['product'].get_charge() - self.rxn['reactant'].get_charge()
    def get_reactants( self ):
        return self.rxn['reactant'].get_species()

    def get_products( self ):
        return self.rxn['product'].get_species()

    def get_stoichiometry_of_species( self, species ):
        return self.rxn['product'].get_stoichiometry_of_species( species ) - self.rxn['reactant'].get_stoichiometry_of_species(species)

    def get_species( self ):
        return self.rxn['reactant'].get_species() + self.rxn['product'].get_species()

    def parse_reaction( self, rxn ):
        if type(rxn) is Reaction:
            rxn = str(Reaction)
        pattern = re.compile(r'==>')
        mixtures = pattern.split(rxn)
        if len(mixtures) == 2:
            reactant = Mixture( mixtures[0].strip())
            product = Mixture( mixtures[1].strip())
            return dict(reactant=reactant, product=product)
    def normalize( self ):
        species = self.get_species()
        lcd = LCD([self.get_stoichiometry_of_species( s ) for s in species])
        return self * lcd
    def deltaGEE_to_latex( self ):
        if self.deltaGEE:
            return r"$\Delta G^{0'} = %0.2f \frac{kJ}{\mathrm{e}^- equiv}$" % self.deltaGEE
        else:
            return ''
    def to_latex( self, deltaGEE=False ):
        if deltaGEE:
            return r'${} \rightarrow {}$ | {}'.format(self.rxn['reactant'].to_latex(), self.rxn['product'].to_latex(), self.deltaGEE_to_latex())
        else:
            return r'${} \rightarrow {}$'.format(self.rxn['reactant'].to_latex(), self.rxn['product'].to_latex())

    def __repr__( self ):
        return str(self.rxn['reactant']) + ' ==> ' + str(self.rxn['product'])
    def __str__( self ):
        return self.__repr__()

    def to_dict( self ):
        rxn = {}
        for species in self.get_species():
            rxn[species] = self.get_stoichiometry_of_species( species )
        return rxn

    def to_series( self ):
        return pd.Series(self.to_dict(), index=self.get_species())

    def get_balance( self ):
        return pd.concat( [-self.rxn['reactant'].get_chemical_composition(),
                           self.rxn['product'].get_chemical_composition()],
                          axis=1).fillna(0).sum(axis=1)

    def get_chemical_composition( self ):
        return pd.concat( [self.rxn['reactant'].get_chemical_composition(),
                           self.rxn['product'].get_chemical_composition()],
                        axis=1).fillna(0)
    __add__ = add_reaction
    __sub__ = subtract_reaction
<commit_msg>Fixed to_dict and to_series to use strings as species instead of objects<commit_after>import re
import pandas as pd
import numpy as np
from pint import UnitRegistry
from .utils import LCD
from fractions import Fraction
from . import Mixture, ChemicalFormula
class Reaction:
    u = UnitRegistry()
    u.define('electron-equivalent = [] = e-eq = e-equiv = eeq')
    gas_constant = 8.3144598*u.joule/u.mole/u.kelvin
    def __init__(self, rxn, deltaGEE=None ):
        self.rxn = self.parse_reaction( rxn )
        self.setDeltaGEE( deltaGEE )
    def setDeltaGEE( self, deltaGEE):
        self.deltaGEE = deltaGEE

    def get_deltaGEE_from_potentials( self, mu0 ):
        deltaGEE = 0
        for species in self.get_species():
            try:
                deltaGEE +=  mu0[str(species)]*self.get_stoichiometry_of_species( species )
            except KeyError:
                print("Species {} not in mu0".format( species ))
                deltaGEE += 0
        return deltaGEE

    def hasDeltaGEE( self ):
        return self.deltaGEE is not None
    def getDeltaGEE( self ):
        return self.deltaGEE

    # def setDeltaG0( self, deltaG0, temperature=273):
    #     self.deltaGEE = deltaG0 + gas_constant*temperature*u.kelvin*np.ln(10**-7)
    # def getDeltaG0( self, temperature=273):
    #     u = self.u
    #     return self.deltaGEE - self.gas_constant*temperature*u.kelvin*np.ln(10**-7)

    def __mul__(self, other):
        if type(other) in [int, float, Fraction]:
            return self.multiply_constant( other )
        elif type(other) is pd.Series:
            return self.multiply_factor( other )
        else:
            raise NotImplemented('unsupported operand type(s) for *: "{}" and "{}"'.format())
    def __rmul__(self, other):
        if type(other) in [int, float, Fraction]:
            return self.multiply_constant( other )
        elif type(other) is pd.Series:
            raise NotImplemented

    def multiply_constant( self, constant ):
        return self.multiply_factor(pd.Series(Fraction(constant),index=self.get_species(),dtype='object'))
    def multiply_factor( self, factor ):
        rxn = Reaction( str(self ) )
        rxn.rxn['reactant'] = rxn.rxn['reactant'].multiply_factor( factor )
        rxn.rxn['product'] = rxn.rxn['product'].multiply_factor( factor )
        return rxn

    def add_reaction( self, rxn ):
        new_reactants = Mixture(str(self.rxn['reactant'])).add_to_mixture(rxn.rxn['reactant']).\
            subtract_from_mixture(self.rxn['product']).subtract_from_mixture(rxn.rxn['product'])
        new_products = Mixture( str(self.rxn['product'])).add_to_mixture(rxn.rxn['product']).\
            subtract_from_mixture(self.rxn['reactant']).subtract_from_mixture(rxn.rxn['reactant'])
        if self.hasDeltaGEE() and rxn.hasDeltaGEE():
            newDeltaGEE = self.getDeltaGEE() + rxn.getDeltaGEE()
        else:
            newDeltaGEE = None
        return Reaction(str(new_reactants) + ' ==> ' + str(new_products), newDeltaGEE)
    def subtract_reaction( self, rxn ):
        new_reactants = Mixture(str(self.rxn['reactant'])).add_to_mixture(rxn.rxn['product']).\
            subtract_from_mixture(self.rxn['product']).subtract_from_mixture(rxn.rxn['reactant'])
        new_products = Mixture( str(self.rxn['product'])).add_to_mixture(rxn.rxn['reactant']).\
            subtract_from_mixture(self.rxn['reactant']).subtract_from_mixture( rxn.rxn['product'])
        if self.hasDeltaGEE() and rxn.hasDeltaGEE():
            newDeltaGEE = self.getDeltaGEE() - rxn.getDeltaGEE()
        else:
            newDeltaGEE = None
        return Reaction(str(new_reactants) + ' ==> ' + str(new_products), newDeltaGEE)

    def get_charge( self ):
        return self.rxn['product'].get_charge() - self.rxn['reactant'].get_charge()
    def get_reactants( self ):
        return self.rxn['reactant'].get_species()

    def get_products( self ):
        return self.rxn['product'].get_species()

    def get_stoichiometry_of_species( self, species ):
        return self.rxn['product'].get_stoichiometry_of_species( species ) - self.rxn['reactant'].get_stoichiometry_of_species(species)

    def get_species( self ):
        return self.rxn['reactant'].get_species() + self.rxn['product'].get_species()

    def parse_reaction( self, rxn ):
        if type(rxn) is Reaction:
            rxn = str(Reaction)
        pattern = re.compile(r'==>')
        mixtures = pattern.split(rxn)
        if len(mixtures) == 2:
            reactant = Mixture( mixtures[0].strip())
            product = Mixture( mixtures[1].strip())
            return dict(reactant=reactant, product=product)
    def normalize( self ):
        species = self.get_species()
        lcd = LCD([self.get_stoichiometry_of_species( s ) for s in species])
        return self * lcd
    def deltaGEE_to_latex( self ):
        if self.deltaGEE:
            return r"$%0.2f \frac{kJ}{\mathrm{e^{-}\ equiv}}$" % self.deltaGEE
        else:
            return ''
    def to_latex( self, deltaGEE=False ):
        if deltaGEE:
            return r'${}$ | $\rightarrow$ | ${}$ | {}'.format(self.rxn['reactant'].to_latex(), self.rxn['product'].to_latex(), self.deltaGEE_to_latex())
        else:
            return r'${} \rightarrow {}$'.format(self.rxn['reactant'].to_latex(), self.rxn['product'].to_latex())

    def __repr__( self ):
        return str(self.rxn['reactant']) + ' ==> ' + str(self.rxn['product'])
    def __str__( self ):
        return self.__repr__()

    def to_dict( self ):
        rxn = {}
        for species in self.get_species():
            rxn[str(species)] = self.get_stoichiometry_of_species( species )
        return rxn

    def to_series( self ):
        return pd.Series(self.to_dict(), index=[str(s) for s in self.get_species()])

    def get_balance( self ):
        return pd.concat( [-self.rxn['reactant'].get_chemical_composition(),
                           self.rxn['product'].get_chemical_composition()],
                          axis=1).fillna(0).sum(axis=1)

    def get_chemical_composition( self ):
        return pd.concat( [self.rxn['reactant'].get_chemical_composition(),
                           self.rxn['product'].get_chemical_composition()],
                        axis=1).fillna(0)
    __add__ = add_reaction
    __sub__ = subtract_reaction
<|endoftext|>


<commit_before>from django.views import generic

from autostew_web_users.models import SteamUser

class SteamUserListView(generic.ListView):
    model = SteamUser

    def get_queryset(self):
        name = self.request.GET.get("name")
        if name is not None and name != '':
            exact_search = self.model.objects.filter(display_name__exact=name)
            contains_search = self.model.objects.filter(display_name__icontains=name)
            if len(exact_search) == 1:
                object_list = self.center_around_user_searched_for(exact_search[0])
            elif len(contains_search) == 1:
                object_list = self.center_around_user_searched_for(contains_search[0])
            else:
                object_list = self.model.objects.filter(display_name__icontains=name)
        else:
            object_list = self.model.objects.all()
        return object_list

    def center_around_user_searched_for(self, user_searched_for, result_length=50):
        all_users = self.model.objects.all()
        l = list(all_users)  # automatically ordered by elo_rating
        index = l.index(user_searched_for)
        lower_index = index-(result_length / 2) if index-(result_length / 2) >= 0 else 0
        upper_index = index+(result_length / 2) if index+(result_length / 2) < len(l) else len(l)-1
        object_list = self.model.objects.all()[lower_index:upper_index]
        return object_list

    def get_context_data(self, **kwargs):
        context = super(SteamUserListView, self).get_context_data(**kwargs)
        context['page'] = self.request.GET.get('page', '1')
        return context
<commit_msg>again debug but in problematic place<commit_after>import sys
from django.views import generic

from autostew_web_users.models import SteamUser

class SteamUserListView(generic.ListView):
    model = SteamUser

    def get_queryset(self):

        sys.stderr.write('\nin get_queryset' + '\n')

        name = self.request.GET.get("name")
        if name is not None and name != '':
            exact_search = self.model.objects.filter(display_name__exact=name)
            contains_search = self.model.objects.filter(display_name__icontains=name)
            if len(exact_search) == 1:
                object_list = self.center_around_user_searched_for(exact_search[0])
            elif len(contains_search) == 1:
                object_list = self.center_around_user_searched_for(contains_search[0])
            else:
                object_list = self.model.objects.filter(display_name__icontains=name)
        else:
            object_list = self.model.objects.all()
        return object_list

    def center_around_user_searched_for(self, user_searched_for, result_length=50):
        all_users = self.model.objects.all()
        l = list(all_users)  # automatically ordered by elo_rating
        index = l.index(user_searched_for)
        lower_index = index-(result_length / 2) if index-(result_length / 2) >= 0 else 0
        upper_index = index+(result_length / 2) if index+(result_length / 2) < len(l) else len(l)-1
        object_list = self.model.objects.all()[lower_index:upper_index]
        return object_list

    def get_context_data(self, **kwargs):
        sys.stderr.write('\n in get_context_data' + '\n')
        context = super(SteamUserListView, self).get_context_data(**kwargs)
        context['page'] = self.request.GET.get('page', '1')
        return context
<|endoftext|>