<commit_before>import configparser
import logging
import math
import os
import re
import time

import fairseq
import numpy as np
import torch
from fairseq.models.transformer import TransformerModel
from fairseq.sequence_generator import SequenceGenerator
from mmt import textencoder
from mmt.alignment import make_alignment
from mmt.tuning import Tuner, TuningOptions

# Regular expression to match strong punctuation split points; splitting points are sequences of
# strong punctuation marks in several alphabets, optionally separated by spaces.
#
# Western: '.?!'
# Chinese/Japanese: '。？！'
# Korean: '.?!'
# Arabic: '.؟!'
#
_STOPS_RE = re.compile(r'( [\s\?\!\.。？！؟]+ )')

class Translation(object):
    def __init__(self, text, alignment=None, score=None):
        self.text = text
        self.alignment = alignment
        self.score = score

    @classmethod
    def split(cls, text):
        # greedy approach for splitting the text into sentences according to strong punctuation
        substrings = _STOPS_RE.split(text.strip())
        if len(substrings) % 2 == 1:
            substrings.append('')

        # list of pairs: (string, separator)
        string_pairs = [[substrings[i], substrings[i + 1]] for i in range(0, len(substrings), 2)]

        text_splits = []
        for e in string_pairs:
            if len(text_splits) == 0:
                text_splits.append(e)
            else:
                starting_char = e[0][0]
                if starting_char.isalpha() and (starting_char.isupper() or not starting_char.islower()):
                    text_splits.append(e)
                else:
                    text_splits[-1].extend(e)

        return [''.join(e).strip() for e in text_splits]

    @classmethod
    def concatenate(cls, sources, translations, separator=' '):
        source_lengths = [len(s.split()) for s in sources]
        translation_lengths = [len(t.text.split()) for t in translations]

        log_scores = [math.log(t.score) for t in translations]
        avg_score = math.exp(sum([s * w for s, w in zip(log_scores, translation_lengths)]) / sum(translation_lengths))

        result = Translation(text=separator.join([t.text for t in translations]),
                             alignment=translations[0].alignment, score=avg_score)

        # concatenate alignment with index shifting
        source_len_offset = 0
        translation_len_offset = 0
        for i in range(1, len(translations)):
            source_len_offset += source_lengths[i - 1]
            translation_len_offset += translation_lengths[i - 1]

            for al in translations[i].alignment:
                new_al = (al[0] + source_len_offset, al[1] + translation_len_offset)
                result.alignment.append(new_al)

        return result

class Suggestion(object):
    def __init__(self, source_lang, target_lang, segment, translation, score):
        self.source_lang = source_lang
        self.target_lang = target_lang
        self.segment = segment
        self.translation = translation
        self.score = score

class ModelConfig(object):
    __custom_values = {'True': True, 'False': False, 'None': None}

    @classmethod
    def load(cls, model_path):
        config = configparser.ConfigParser()
        config.read(os.path.join(model_path, 'model.conf'))
        return cls(model_path, config)

    def __init__(self, base_path, config_parser):
        self._base_path = base_path
        self._config = config_parser

    def _parse(self, value):
        if value in self.__custom_values:
            value = self.__custom_values[value]
        else:
            try:
                number = float(value)
                value = number if '.' in value else int(value)
            except ValueError:
                pass  # value is a string

        return value

    @property
    def tuning(self):
        ops = TuningOptions()

        if self._config.has_section('settings'):
            for name, value in self._config.items('settings'):
                if not hasattr(ops, name):
                    raise ValueError('Invalid option "%s"' % name)
                setattr(ops, name, self._parse(value))

        return ops

    @property
    def checkpoints(self):
        def normalize_lang(lang):
            if '-' in lang:
                a, b = lang.split('-')
                return '%s-%s' % (a.lower(), b.upper())
            else:
                return lang.lower()

        result = []

        for name, value in self._config.items('models'):
            if not os.path.isabs(value):
                value = os.path.join(self._base_path, value)

            source, target = name.split('__')
            name = '%s__%s' % (normalize_lang(source), normalize_lang(target))

            result.append((name, value))

        return result

class MMTDecoder(object):
    @classmethod
    def _create_model(cls, checkpoints, device, beam_size, use_fp16):
        model = TransformerModel.build_model(checkpoints.args, checkpoints.task)

        # Custom make_generation_fast_
        eval_fn, train_fn = model.eval, model.train
        model.eval = lambda: None

        model.make_generation_fast_(
            beamable_mm_beam_size=None if beam_size == 0 else beam_size,
            need_attn=True,  # --print-alignment
        )

        model.eval, model.train = eval_fn, train_fn

        if device is not None:
            torch.cuda.set_device(device)
            model = model.cuda(device)

        if use_fp16:
            model.half()

        return model

    @classmethod
    def _create_translator(cls, checkpoints, beam_size):
        return SequenceGenerator(
            checkpoints.task.target_dictionary, beam_size=beam_size,
            max_len_a=0, max_len_b=1, min_len=1,
            stop_early=True, normalize_scores=True,
            len_penalty=1, unk_penalty=0,
            sampling=False, sampling_topk=-1, temperature=1,
            diverse_beam_groups=1, diverse_beam_strength=0.5
        )

    @classmethod
    def _create_tuner(cls, checkpoints, model, tuning_ops, device):
        return Tuner(checkpoints.args, checkpoints.task, model, tuning_ops=tuning_ops, device=device)

    def __init__(self, checkpoints, device=None, beam_size=5, use_fp16=False, tuning_ops=None):
        torch.manual_seed(checkpoints.args.seed)

        self._checkpoints = checkpoints
        self._device = device
        self._model = self._create_model(checkpoints, device=device, beam_size=beam_size, use_fp16=use_fp16)
        self._translator = self._create_translator(checkpoints, beam_size)
        self._tuner = self._create_tuner(checkpoints, self._model, tuning_ops, device)
        self._max_positions = fairseq.utils.resolve_max_positions(
            checkpoints.task.max_positions(),
            self._model.max_positions(),
        )

        self._logger = logging.getLogger('Transformer')

        self._nn_needs_reset = True
        self._checkpoint = None

    # - High level functions -------------------------------------------------------------------------------------------

    def test(self):
        test_batch, _, _ = self._make_batch([])

        begin = time.time()
        self._translator.max_len_b = 1
        self._translator.generate([self._model], test_batch)
        test_time = time.time() - begin

        self._logger.info('test_time = %.3f' % test_time)

    def translate(self, source_lang, target_lang, text, suggestions=None,
                  tuning_epochs=None, tuning_learning_rate=None, forced_translation=None):
        # (1) Reset model (if necessary)
        begin = time.time()
        self._reset_model(source_lang, target_lang)
        reset_time = time.time() - begin

        # (2) Tune engine if suggestions provided
        begin = time.time()
        if suggestions is not None and len(suggestions) > 0:
            self._tune(suggestions, epochs=tuning_epochs, learning_rate=tuning_learning_rate)
        tune_time = time.time() - begin

        # (3) Translate and compute word alignment
        begin = time.time()
        if forced_translation is not None:
            result = self._force_decode(target_lang, text, forced_translation)
        else:
            # split the input text into sentences according to strong punctuation
            text_splits = Translation.split(text)

            translations = self._decode(source_lang, target_lang, text_splits)

            # concatenate Translations of all splits into one Translation
            result = Translation.concatenate(text_splits, translations)

        decode_time = time.time() - begin

        self._logger.info('reset_time = %.3f, tune_time = %.3f, decode_time = %.3f'
                          % (reset_time, tune_time, decode_time))

        return result

    # - Low level functions --------------------------------------------------------------------------------------------

    def _reset_model(self, source_lang, target_lang):
        checkpoint = self._checkpoints.load(source_lang, target_lang)

        if self._nn_needs_reset or checkpoint != self._checkpoint:
            self._model.load_state_dict(checkpoint.state, strict=True)
            self._checkpoint = checkpoint
            self._nn_needs_reset = False

    def _tune(self, suggestions, epochs=None, learning_rate=None):
        # Set tuning parameters
        if epochs is None or learning_rate is None:
            _epochs, _learning_rate = self._tuner.estimate_tuning_parameters(suggestions)

            epochs = epochs if epochs is not None else _epochs
            learning_rate = learning_rate if learning_rate is not None else _learning_rate

        # Run tuning
        if learning_rate > 0. and epochs > 0:
            sub_dict = self._checkpoint.subword_dictionary

            if self._checkpoint.multilingual_target:
                src_samples = [sub_dict.language_tag(e.target_lang) + ' ' + e.segment for e in suggestions]
            else:
                src_samples = [e.segment for e in suggestions]
            tgt_samples = [e.translation for e in suggestions]

            dataset = self._tuner.dataset(src_samples, tgt_samples, sub_dict)
            self._tuner.tune(dataset, num_iterations=epochs, lr=learning_rate)
            self._nn_needs_reset = True

    def _decode(self, source_lang, target_lang, batch):
        prefix_lang = target_lang if self._checkpoint.multilingual_target else None
        batch, input_indexes, sentence_len = self._make_batch(batch, prefix_lang=prefix_lang)

        # Compute translation
        self._translator.max_len_b = self._checkpoint.decode_length(source_lang, target_lang, sentence_len)
        translations = self._translator.generate([self._model], batch)

        # Decode translation
        sub_dict = self._checkpoint.subword_dictionary

        results = []

        for i, hypo in enumerate(translations):
            hypo = hypo[0]  # (top-1 best nbest)
            hypo_score = math.exp(hypo['score'])
            hypo_tokens = hypo['tokens']
            hypo_indexes = sub_dict.indexes_of(hypo_tokens)
            hypo_str = sub_dict.string(hypo_tokens)
            hypo_attention = np.asarray(hypo['attention'].data.cpu())

            # Make alignment
            if len(hypo_indexes) > 0:
                hypo_alignment = make_alignment(input_indexes[i], hypo_indexes, hypo_attention,
                                                prefix_lang=prefix_lang is not None)
            else:
                hypo_alignment = []

            results.append(Translation(hypo_str, alignment=hypo_alignment, score=hypo_score))

        return results

    def _force_decode(self, target_lang, segment, translation):
        sub_dict = self._checkpoint.subword_dictionary

        if self._checkpoint.multilingual_target:
            segment = sub_dict.language_tag(target_lang) + ' ' + segment

        dataset = self._tuner.dataset([segment], [translation], sub_dict)

        src_tokens = dataset.src.tokens[0]
        src_indexes = sub_dict.indexes_of(src_tokens)
        src_tokens = src_tokens.reshape((1, -1))
        src_lengths = torch.tensor(dataset.src.sizes)
        tgt_tokens = dataset.tgt.tokens[0]
        tgt_indexes = sub_dict.indexes_of(tgt_tokens)
        tgt_tokens = torch.cat((tgt_tokens[-1:], tgt_tokens[:-1]))
        tgt_tokens = tgt_tokens.reshape((1, -1))

        if self._device is not None:
            src_tokens = src_tokens.cuda(self._device)
            src_lengths = src_lengths.cuda(self._device)
            tgt_tokens = tgt_tokens.cuda(self._device)

        self._model.eval()
        _, attn = self._model(src_tokens, src_lengths, tgt_tokens)
        if type(attn) is dict:
            attn = attn['attn']
        attn = attn[0]
        attn = attn.transpose(0, 1).cpu()

        hypo_alignment = make_alignment(src_indexes, tgt_indexes, attn.data.numpy(),
                                        prefix_lang=self._checkpoint.multilingual_target)
        return Translation(translation, alignment=hypo_alignment)

    def _make_batch(self, lines, prefix_lang=None):
        # Add language prefix if multilingual target
        if prefix_lang is not None:
            lines = [self._checkpoint.subword_dictionary.language_tag(prefix_lang) + ' ' + text for text in lines]

        # Prepare batch
        if len(lines) > 0:
            sub_dict = self._checkpoint.subword_dictionary
            tokens = [
                sub_dict.encode_line(text, line_tokenizer=sub_dict.tokenize, add_if_not_exist=False).long()
                for text in lines
            ]
            input_indexes = [sub_dict.indexes_of(el) for el in tokens]
            lengths = torch.LongTensor([t.numel() for t in tokens])
        else:
            input_indexes = [[]]
            tokens = torch.LongTensor([[textencoder.EOS_ID]])
            lengths = torch.LongTensor([1])

        max_length = torch.max(lengths)

        # Apply padding
        if len(lines) > 1:
            tokens = [torch.nn.functional.pad(el, (max_length - el.size(0), 0), value=sub_dict.pad()) for el in tokens]

        # Reshape tokens tensor
        if len(lines) > 1:
            tokens = torch.cat(tokens)
        else:
            tokens = tokens[0]

        tokens = tokens.reshape([max(1, len(lines)), max_length])

        if self._device is not None:
            tokens = tokens.cuda(self._device)
            lengths = lengths.cuda(self._device)

        batch = {'net_input': {
            'src_tokens': tokens,
            'src_lengths': lengths
        }}

        return batch, input_indexes, max_length
<commit_msg>Removed sentence splitting in python decoder<commit_after>import configparser
import logging
import math
import os
import time

import fairseq
import numpy as np
import torch
from fairseq.models.transformer import TransformerModel
from fairseq.sequence_generator import SequenceGenerator
from mmt import textencoder
from mmt.alignment import make_alignment
from mmt.tuning import Tuner, TuningOptions

class Translation(object):
    def __init__(self, text, alignment=None, score=None):
        self.text = text
        self.alignment = alignment
        self.score = score

class Suggestion(object):
    def __init__(self, source_lang, target_lang, segment, translation, score):
        self.source_lang = source_lang
        self.target_lang = target_lang
        self.segment = segment
        self.translation = translation
        self.score = score

class ModelConfig(object):
    __custom_values = {'True': True, 'False': False, 'None': None}

    @classmethod
    def load(cls, model_path):
        config = configparser.ConfigParser()
        config.read(os.path.join(model_path, 'model.conf'))
        return cls(model_path, config)

    def __init__(self, base_path, config_parser):
        self._base_path = base_path
        self._config = config_parser

    def _parse(self, value):
        if value in self.__custom_values:
            value = self.__custom_values[value]
        else:
            try:
                number = float(value)
                value = number if '.' in value else int(value)
            except ValueError:
                pass  # value is a string

        return value

    @property
    def tuning(self):
        ops = TuningOptions()

        if self._config.has_section('settings'):
            for name, value in self._config.items('settings'):
                if not hasattr(ops, name):
                    raise ValueError('Invalid option "%s"' % name)
                setattr(ops, name, self._parse(value))

        return ops

    @property
    def checkpoints(self):
        def normalize_lang(lang):
            if '-' in lang:
                a, b = lang.split('-')
                return '%s-%s' % (a.lower(), b.upper())
            else:
                return lang.lower()

        result = []

        for name, value in self._config.items('models'):
            if not os.path.isabs(value):
                value = os.path.join(self._base_path, value)

            source, target = name.split('__')
            name = '%s__%s' % (normalize_lang(source), normalize_lang(target))

            result.append((name, value))

        return result

class MMTDecoder(object):
    @classmethod
    def _create_model(cls, checkpoints, device, beam_size, use_fp16):
        model = TransformerModel.build_model(checkpoints.args, checkpoints.task)

        # Custom make_generation_fast_
        eval_fn, train_fn = model.eval, model.train
        model.eval = lambda: None

        model.make_generation_fast_(
            beamable_mm_beam_size=None if beam_size == 0 else beam_size,
            need_attn=True,  # --print-alignment
        )

        model.eval, model.train = eval_fn, train_fn

        if device is not None:
            torch.cuda.set_device(device)
            model = model.cuda(device)

        if use_fp16:
            model.half()

        return model

    @classmethod
    def _create_translator(cls, checkpoints, beam_size):
        return SequenceGenerator(
            checkpoints.task.target_dictionary, beam_size=beam_size,
            max_len_a=0, max_len_b=1, min_len=1,
            stop_early=True, normalize_scores=True,
            len_penalty=1, unk_penalty=0,
            sampling=False, sampling_topk=-1, temperature=1,
            diverse_beam_groups=1, diverse_beam_strength=0.5
        )

    @classmethod
    def _create_tuner(cls, checkpoints, model, tuning_ops, device):
        return Tuner(checkpoints.args, checkpoints.task, model, tuning_ops=tuning_ops, device=device)

    def __init__(self, checkpoints, device=None, beam_size=5, use_fp16=False, tuning_ops=None):
        torch.manual_seed(checkpoints.args.seed)

        self._checkpoints = checkpoints
        self._device = device
        self._model = self._create_model(checkpoints, device=device, beam_size=beam_size, use_fp16=use_fp16)
        self._translator = self._create_translator(checkpoints, beam_size)
        self._tuner = self._create_tuner(checkpoints, self._model, tuning_ops, device)
        self._max_positions = fairseq.utils.resolve_max_positions(
            checkpoints.task.max_positions(),
            self._model.max_positions(),
        )

        self._logger = logging.getLogger('Transformer')

        self._nn_needs_reset = True
        self._checkpoint = None

    # - High level functions -------------------------------------------------------------------------------------------

    def test(self):
        test_batch, _, _ = self._make_batch([])

        begin = time.time()
        self._translator.max_len_b = 1
        self._translator.generate([self._model], test_batch)
        test_time = time.time() - begin

        self._logger.info('test_time = %.3f' % test_time)

    def translate(self, source_lang, target_lang, text, suggestions=None,
                  tuning_epochs=None, tuning_learning_rate=None, forced_translation=None):
        # (1) Reset model (if necessary)
        begin = time.time()
        self._reset_model(source_lang, target_lang)
        reset_time = time.time() - begin

        # (2) Tune engine if suggestions provided
        begin = time.time()
        if suggestions is not None and len(suggestions) > 0:
            self._tune(suggestions, epochs=tuning_epochs, learning_rate=tuning_learning_rate)
        tune_time = time.time() - begin

        # (3) Translate and compute word alignment
        begin = time.time()
        if forced_translation is not None:
            result = self._force_decode(target_lang, text, forced_translation)
        else:
            result = self._decode(source_lang, target_lang, [text])[0]

        decode_time = time.time() - begin

        self._logger.info('reset_time = %.3f, tune_time = %.3f, decode_time = %.3f'
                          % (reset_time, tune_time, decode_time))

        return result

    # - Low level functions --------------------------------------------------------------------------------------------

    def _reset_model(self, source_lang, target_lang):
        checkpoint = self._checkpoints.load(source_lang, target_lang)

        if self._nn_needs_reset or checkpoint != self._checkpoint:
            self._model.load_state_dict(checkpoint.state, strict=True)
            self._checkpoint = checkpoint
            self._nn_needs_reset = False

    def _tune(self, suggestions, epochs=None, learning_rate=None):
        # Set tuning parameters
        if epochs is None or learning_rate is None:
            _epochs, _learning_rate = self._tuner.estimate_tuning_parameters(suggestions)

            epochs = epochs if epochs is not None else _epochs
            learning_rate = learning_rate if learning_rate is not None else _learning_rate

        # Run tuning
        if learning_rate > 0. and epochs > 0:
            sub_dict = self._checkpoint.subword_dictionary

            if self._checkpoint.multilingual_target:
                src_samples = [sub_dict.language_tag(e.target_lang) + ' ' + e.segment for e in suggestions]
            else:
                src_samples = [e.segment for e in suggestions]
            tgt_samples = [e.translation for e in suggestions]

            dataset = self._tuner.dataset(src_samples, tgt_samples, sub_dict)
            self._tuner.tune(dataset, num_iterations=epochs, lr=learning_rate)
            self._nn_needs_reset = True

    def _decode(self, source_lang, target_lang, batch):
        prefix_lang = target_lang if self._checkpoint.multilingual_target else None
        batch, input_indexes, sentence_len = self._make_batch(batch, prefix_lang=prefix_lang)

        # Compute translation
        self._translator.max_len_b = self._checkpoint.decode_length(source_lang, target_lang, sentence_len)
        translations = self._translator.generate([self._model], batch)

        # Decode translation
        sub_dict = self._checkpoint.subword_dictionary

        results = []

        for i, hypo in enumerate(translations):
            hypo = hypo[0]  # (top-1 best nbest)
            hypo_score = math.exp(hypo['score'])
            hypo_tokens = hypo['tokens']
            hypo_indexes = sub_dict.indexes_of(hypo_tokens)
            hypo_str = sub_dict.string(hypo_tokens)
            hypo_attention = np.asarray(hypo['attention'].data.cpu())

            # Make alignment
            if len(hypo_indexes) > 0:
                hypo_alignment = make_alignment(input_indexes[i], hypo_indexes, hypo_attention,
                                                prefix_lang=prefix_lang is not None)
            else:
                hypo_alignment = []

            results.append(Translation(hypo_str, alignment=hypo_alignment, score=hypo_score))

        return results

    def _force_decode(self, target_lang, segment, translation):
        sub_dict = self._checkpoint.subword_dictionary

        if self._checkpoint.multilingual_target:
            segment = sub_dict.language_tag(target_lang) + ' ' + segment

        dataset = self._tuner.dataset([segment], [translation], sub_dict)

        src_tokens = dataset.src.tokens[0]
        src_indexes = sub_dict.indexes_of(src_tokens)
        src_tokens = src_tokens.reshape((1, -1))
        src_lengths = torch.tensor(dataset.src.sizes)
        tgt_tokens = dataset.tgt.tokens[0]
        tgt_indexes = sub_dict.indexes_of(tgt_tokens)
        tgt_tokens = torch.cat((tgt_tokens[-1:], tgt_tokens[:-1]))
        tgt_tokens = tgt_tokens.reshape((1, -1))

        if self._device is not None:
            src_tokens = src_tokens.cuda(self._device)
            src_lengths = src_lengths.cuda(self._device)
            tgt_tokens = tgt_tokens.cuda(self._device)

        self._model.eval()
        _, attn = self._model(src_tokens, src_lengths, tgt_tokens)
        if type(attn) is dict:
            attn = attn['attn']
        attn = attn[0]
        attn = attn.transpose(0, 1).cpu()

        hypo_alignment = make_alignment(src_indexes, tgt_indexes, attn.data.numpy(),
                                        prefix_lang=self._checkpoint.multilingual_target)
        return Translation(translation, alignment=hypo_alignment)

    def _make_batch(self, lines, prefix_lang=None):
        # Add language prefix if multilingual target
        if prefix_lang is not None:
            lines = [self._checkpoint.subword_dictionary.language_tag(prefix_lang) + ' ' + text for text in lines]

        # Prepare batch
        if len(lines) > 0:
            sub_dict = self._checkpoint.subword_dictionary
            tokens = [
                sub_dict.encode_line(text, line_tokenizer=sub_dict.tokenize, add_if_not_exist=False).long()
                for text in lines
            ]
            input_indexes = [sub_dict.indexes_of(el) for el in tokens]
            lengths = torch.LongTensor([t.numel() for t in tokens])
        else:
            input_indexes = [[]]
            tokens = torch.LongTensor([[textencoder.EOS_ID]])
            lengths = torch.LongTensor([1])

        max_length = torch.max(lengths)

        # Apply padding
        if len(lines) > 1:
            tokens = [torch.nn.functional.pad(el, (max_length - el.size(0), 0), value=sub_dict.pad()) for el in tokens]

        # Reshape tokens tensor
        if len(lines) > 1:
            tokens = torch.cat(tokens)
        else:
            tokens = tokens[0]

        tokens = tokens.reshape([max(1, len(lines)), max_length])

        if self._device is not None:
            tokens = tokens.cuda(self._device)
            lengths = lengths.cuda(self._device)

        batch = {'net_input': {
            'src_tokens': tokens,
            'src_lengths': lengths
        }}

        return batch, input_indexes, max_length
<|endoftext|>


<commit_before>"""A configuration structure which supports cascading layers.

In Vivarium it allows base configurations to be overridden by component level
configurations which are in turn overridden by model level configuration
which can be overridden by user supplied overrides. From the perspective
of normal client code the cascading is hidden and configuration values
are presented as attributes of the configuration object the values of
which are the value of that key in the outermost layer of configuration
where it appears.

For example:
>>> config = ConfigTree(layers=['inner_layer', 'middle_layer', 'outer_layer', 'user_overrides'])
>>> config.read_dict({'section_a': {'item1': 'value1', 'item2': 'value2'}, 'section_b': {'item1': 'value3'}}, layer='inner_layer')
>>> config.read_dict({'section_a': {'item1': 'value4'}, 'section_b': {'item1': 'value5'}}, layer='middle_layer')
>>> config.read_dict({'section_b': {'item1': 'value6'}}, layer='outer_layer')
>>> config.section_a.item1
'value4'
>>> config.section_a.item2
'value2'
>>> config.section_b.item1
'value6'
>>> config.section_b.item1 = 'value7'
>>> config.section_b.item1
'value7'
"""
from typing import Mapping, Union
import yaml

class ConfigNode:
    """A single configuration value which may have different variants for different layers.
    Each ConfigNode also records the source (if any is reported) from which the value is derived.
    """
    def __init__(self, layers=None):
        if not layers:
            self._layers = ['base']
        else:
            self._layers = layers
        self._values = {}
        self._frozen = False
        self._accessed = False

    def freeze(self):
        """Causes the node to become read only. This is useful for loading and then
        freezing configurations that should not be modified at runtime.
        """
        self._frozen = True

    def get_value_with_source(self, layer=None):
        """Returns a tuple of the value's source and the value at the specified
        layer. If no layer is specified then the outer layer is used.

        Parameters
        ----------
        layer : str
            Name of the layer to use. If None then the outermost where the value
            exists will be used.

        Raises
        ------
        KeyError
            If the value is not set for the specified layer
        """
        if layer:
            return self._values[layer]

        for layer in reversed(self._layers):
            if layer in self._values:
                return self._values[layer]
        raise KeyError(layer)

    def is_empty(self):
        return not self._values

    def get_value(self, layer=None):
        """Returns the value at the specified layer.

        Parameters
        ----------
        layer : str
            Name of the layer to use. If None then the outermost where the value
            exists will be used.

        Raises
        ------
        KeyError
            If the value is not set for the specified layer
        """
        self._accessed = True
        return self.get_value_with_source(layer)[1]

    def has_been_accessed(self):
        """Returns whether this node has been accessed.

        Returns
        -------
        bool
            Whether this node has been accessed
        """
        return self._accessed

    def metadata(self):
        """Returns all values and associated metadata for this node as a dict.
        The value which would be selected if the node's value was requested
        is indicated by the `default` flag.
        """
        result = []
        for layer in self._layers:
            if layer in self._values:
                result.append({
                    'layer': layer,
                    'value': self._values[layer][1],
                    'source': self._values[layer][0],
                    'default': layer == self._layers[-1]
                })
        return result

    def set_value(self, value, layer=None, source=None):
        """Set a value for a particular layer with optional metadata about source.

        Parameters
        ----------
        value : str
            Data to store in the node.
        layer : str
            Name of the layer to use. If None then the outermost where the value
            exists will be used.
        source : str
            Metadata indicating the source of this value (e.g. a file path)

        Raises
        ------
        TypeError
            If the node is frozen
        KeyError
            If the named layer does not exist
        """
        if self._frozen:
            raise TypeError('Frozen ConfigNode does not support assignment')

        if not layer:
            layer = self._layers[-1]
        self._values[layer] = (source, value)

    def drop_layer(self, layer):
        """Removes the named layer and the value associated with it from the node.

        Parameters
        ----------
        layer : str
            Name of the layer to drop.

        Raises
        ------
        TypeError
            If the node is frozen
        KeyError
            If the named layer does not exist
        """
        if self._frozen:
            raise TypeError('Frozen ConfigNode does not support modification')
        self.reset_layer(layer)
        self._layers.remove(layer)

    def reset_layer(self, layer):
        """Removes any value and metadata associated with the named layer.

        Parameters
        ----------
        layer : str
            Name of the layer to reset.

        Raises
        ------
        TypeError
            If the node is frozen
        KeyError
            If the named layer does not exist
        """
        if self._frozen:
            raise TypeError('Frozen ConfigNode does not support modification')
        if layer in self._values:
            del self._values[layer]

    def __repr__(self):
        return 'ConfigNode(layers={}, values={}, frozen={}, accessed={})'.format(
            self._layers, self._values, self._frozen, self._accessed)

    def __str__(self):
        return '\n'.join(reversed(['{}: {}\n    source: {}'.format(layer, value[1], value[0])
                                   for layer, value in self._values.items()]))

class ConfigTree:
    """A container for configuration information. Each configuration value is
    exposed as an attribute the value of which is determined by the outermost
    layer which has the key defined.
    """
    def __init__(self, data=None, layers=None):
        """
        Parameters
        ----------
        data : dict, str, or ConfigTree, optional
            A dictionary containing initial values
        layers : list
            A list of layer names. The order in which layers defined determines
            how they cascade. Later layers override the values from earlier ones.
        """
        if not layers:
            self.__dict__['_layers'] = ['base']
        else:
            self.__dict__['_layers'] = layers
        self.__dict__['_children'] = {}
        self.__dict__['_frozen'] = False

        if data:
            self.update(data, layer=self._layers[0], source='initial data')

    def freeze(self):
        """Causes the ConfigTree to become read only.

        This is useful for loading and then freezing configurations that should not be modified at runtime.
        """
        self.__dict__['_frozen'] = True
        for child in self._children.values():
            child.freeze()

    def __setattr__(self, name, value):
        """Set a configuration value on the outermost layer."""
        self.set_with_metadata(name, value, layer=None, source=None)

    def __setitem__(self, name, value):
        """Set a configuration value on the outermost layer."""
        self.set_with_metadata(name, value, layer=None, source=None)

    def __getattr__(self, name):
        """Get a configuration value from the outermost layer in which it appears."""
        try:
            return self.get_from_layer(name)
        except KeyError:
            raise AttributeError(name)

    def __getitem__(self, name):
        """Get a configuration value from the outermost layer in which it appears."""
        return self.get_from_layer(name)

    def __delattr__(self, name):
        if name in self._children:
            del self._children[name]

    def __delitem__(self, name):
        if name in self._children:
            del self._children[name]

    def __contains__(self, name):
        """Test if a configuration key exists in any layer."""
        return name in self._children

    def __iter__(self):
        """Dictionary-like iteration."""
        return iter(self._children)

    def items(self):
        """Return a list of all (child_name, child) pairs."""
        return self._children.items()

    def keys(self):
        return self._children.keys()

    def get_from_layer(self, name, layer=None):
        """Get a configuration value from the named layer.
        Parameters
        ----------
        name : str
            The name of the value to retrieve
        layer: str
            The name of the layer to retrieve the value from. If it is not supplied
            then the outermost layer in which the key is defined will be used.
        """
        if name not in self._children:
            if self._frozen:
                raise KeyError(name)
            self._children[name] = ConfigTree(layers=self._layers)
        child = self._children[name]
        if isinstance(child, ConfigNode):
            return child.get_value(layer)
        else:
            return child

    def set_with_metadata(self, name, value, layer=None, source=None):
        """Set a value in the named layer with the given source.
        Parameters
        ----------
        name : str
            The name of the value
        value
            The value to store
        layer : str, optional
            The name of the layer to store the value in. If none is supplied
            then the value will be stored in the outermost layer.
        source : str, optional
            The source to attribute the value to.

        Raises
        ------
        TypeError
            if the ConfigTree is frozen
        """

        if self._frozen:
            raise TypeError('Frozen ConfigTree does not support assignment')

        if isinstance(value, dict):
            if name not in self._children or not isinstance(self._children[name], ConfigTree):
                self._children[name] = ConfigTree(layers=list(self._layers))
            self._children[name].read_dict(value, layer, source)
        else:
            if name not in self._children or not isinstance(self._children[name], ConfigNode):
                self._children[name] = ConfigNode(list(self._layers))
            child = self._children[name]
            child.set_value(value, layer, source)

    def update(self, data: Union[Mapping, str, bytes], layer: str=None, source: str=None):
        """Adds additional data into the ConfigTree.

        Parameters
        ----------
        data :
            source data
        layer :
            layer to load data into. If none is supplied the outermost one is used
        source :
            Source to attribute the values to

        See Also
        --------
        read_dict
        """
        if isinstance(data, dict):
            self.read_dict(data, layer, source)
        elif isinstance(data, ConfigTree):
            # TODO: set this to parse the other config tree including layer and source info.  Maybe.
            self.read_dict(data.to_dict(), layer, source)
        elif isinstance(data, str):
            if data.endswith('.yaml'):
                source = source if source else data
                self.load(data, layer, source)
            else:
                self.loads(data, layer, source)
        elif data is None:
            pass
        else:
            raise ValueError(f"Update must be called with dictionary, string, or ConfigTree. "
                             f"You passed in {type(data)}")

    def read_dict(self, data_dict, layer=None, source=None):
        """Load a dictionary into the ConfigTree. If the dict contains nested dicts
        then the values will be added recursively. See module docstring for example code.

        Parameters
        ----------
        data_dict : dict
            source data
        layer : str
            layer to load data into. If none is supplied the outermost one is used
        source : str
            Source to attribute the values to
        """
        for k, v in data_dict.items():
            self.set_with_metadata(k, v, layer, source)

    def loads(self, data_string, layer=None, source=None):
        """Load data from a yaml formatted string.

        Parameters
        ----------
        data_string : str
            yaml formatted string. The root element of the document should be
            an associative array
        layer : str
            layer to load data into. If none is supplied the outermost one is used
        source : str
            Source to attribute the values to
        """
        data_dict = yaml.load(data_string)
        self.read_dict(data_dict, layer, source)

    def load(self, f, layer=None, source=None):
        """Load data from a yaml formatted file.

        Parameters
        ----------
        f : str or file like object
            If f is a string then it is interpreted as a path to the file to load
            If it is a file like object then data is read directly from it.
        layer : str
            layer to load data into. If none is supplied the outermost one is used
        source : str
            Source to attribute the values to
        """
        if hasattr(f, 'read'):
            self.loads(f.read(), layer=layer, source=source)
        else:
            with open(f) as f:
                self.loads(f.read(), layer=layer, source=source)

    def to_dict(self):
        result = {}
        for k, v in self._children.items():
            if isinstance(v, ConfigNode):
                result[k] = v.get_value()
            else:
                result[k] = v.to_dict()
        return result

    def metadata(self, name):
        """Return value and metadata associated with the named value

        Parameters
        ----------
        name : str
            name to retrieve. If the name contains '.'s it will be retrieved recursively

        Raises
        ------
        KeyError
            if name is not defined in the ConfigTree
        """
        if name in self._children:
            return self._children[name].metadata()
        else:
            head, _, tail = name.partition('.')
            if head in self._children:
                return self._children[head].metadata(key=tail)
            else:
                raise KeyError(name)

    def reset_layer(self, layer, preserve_keys=()):
        """Removes any value and metadata associated with the named layer.

        Parameters
        ----------
        layer : str
            Name of the layer to reset.
        preserve_keys : list or tuple
            A list of keys to skip while removing data

        Raises
        ------
        TypeError
            If the node is frozen
        KeyError
            If the named layer does not exist
        """
        self._reset_layer(layer, [k.split('.') for k in preserve_keys], prefix=[])

    def _reset_layer(self, layer, preserve_keys, prefix):
        if self._frozen:
            raise TypeError('Frozen ConfigTree does not support modification')
        deletable = []
        for key, child in self._children.items():
            if prefix + [key] not in preserve_keys:
                if isinstance(child, ConfigTree):
                    child._reset_layer(layer, preserve_keys, prefix + [key])
                else:
                    child.reset_layer(layer)
                    if child.is_empty():
                        deletable.append(key)
        for key in deletable:
            del self._children[key]

    def drop_layer(self, layer):
        """Removes the named layer and the value associated with it from the node.

        Parameters
        ----------
        layer : str
            Name of the layer to drop.

        Raises
        ------
        TypeError
            If the node is frozen
        KeyError
            If the named layer does not exist
        """
        if self._frozen:
            raise TypeError('Frozen ConfigTree does not support modification')
        for child in self._children.values():
            child.drop_layer(layer)
        self._layers.remove(layer)

    def unused_keys(self):
        """Lists all keys which are present in the ConfigTree but which have not been accessed.
        """
        unused = set()
        for k, c in self._children.items():
            if isinstance(c, ConfigNode):
                if not c.has_been_accessed():
                    unused.add(k)
            else:
                for ck in c.unused_keys():
                    unused.add(k+'.'+ck)
        return unused

    def __len__(self):
        return len(self._children)

    def __dir__(self):
        return list(self._children.keys()) + dir(super(ConfigTree, self))

    def __repr__(self):
        return 'ConfigTree(children={}, frozen={})'.format(
            ' '.join([repr(c) for c in self._children.values()]), self._frozen)

    def __str__(self):
        return '\n'.join(['{}:\n    {}'.format(name, str(c).replace('\n', '\n    '))
                          for name, c in self._children.items()])
<commit_msg>Add keys method<commit_after>"""A configuration structure which supports cascading layers.

In Vivarium it allows base configurations to be overridden by component level
configurations which are in turn overridden by model level configuration
which can be overridden by user supplied overrides. From the perspective
of normal client code the cascading is hidden and configuration values
are presented as attributes of the configuration object the values of
which are the value of that key in the outermost layer of configuration
where it appears.

For example:
>>> config = ConfigTree(layers=['inner_layer', 'middle_layer', 'outer_layer', 'user_overrides'])
>>> config.read_dict({'section_a': {'item1': 'value1', 'item2': 'value2'}, 'section_b': {'item1': 'value3'}}, layer='inner_layer')
>>> config.read_dict({'section_a': {'item1': 'value4'}, 'section_b': {'item1': 'value5'}}, layer='middle_layer')
>>> config.read_dict({'section_b': {'item1': 'value6'}}, layer='outer_layer')
>>> config.section_a.item1
'value4'
>>> config.section_a.item2
'value2'
>>> config.section_b.item1
'value6'
>>> config.section_b.item1 = 'value7'
>>> config.section_b.item1
'value7'
"""
from typing import Mapping, Union
import yaml

class ConfigNode:
    """A single configuration value which may have different variants for different layers.
    Each ConfigNode also records the source (if any is reported) from which the value is derived.
    """
    def __init__(self, layers=None):
        if not layers:
            self._layers = ['base']
        else:
            self._layers = layers
        self._values = {}
        self._frozen = False
        self._accessed = False

    def freeze(self):
        """Causes the node to become read only. This is useful for loading and then
        freezing configurations that should not be modified at runtime.
        """
        self._frozen = True

    def get_value_with_source(self, layer=None):
        """Returns a tuple of the value's source and the value at the specified
        layer. If no layer is specified then the outer layer is used.

        Parameters
        ----------
        layer : str
            Name of the layer to use. If None then the outermost where the value
            exists will be used.

        Raises
        ------
        KeyError
            If the value is not set for the specified layer
        """
        if layer:
            return self._values[layer]

        for layer in reversed(self._layers):
            if layer in self._values:
                return self._values[layer]
        raise KeyError(layer)

    def is_empty(self):
        return not self._values

    def get_value(self, layer=None):
        """Returns the value at the specified layer.

        Parameters
        ----------
        layer : str
            Name of the layer to use. If None then the outermost where the value
            exists will be used.

        Raises
        ------
        KeyError
            If the value is not set for the specified layer
        """
        self._accessed = True
        return self.get_value_with_source(layer)[1]

    def has_been_accessed(self):
        """Returns whether this node has been accessed.

        Returns
        -------
        bool
            Whether this node has been accessed
        """
        return self._accessed

    def metadata(self):
        """Returns all values and associated metadata for this node as a dict.
        The value which would be selected if the node's value was requested
        is indicated by the `default` flag.
        """
        result = []
        for layer in self._layers:
            if layer in self._values:
                result.append({
                    'layer': layer,
                    'value': self._values[layer][1],
                    'source': self._values[layer][0],
                    'default': layer == self._layers[-1]
                })
        return result

    def set_value(self, value, layer=None, source=None):
        """Set a value for a particular layer with optional metadata about source.

        Parameters
        ----------
        value : str
            Data to store in the node.
        layer : str
            Name of the layer to use. If None then the outermost where the value
            exists will be used.
        source : str
            Metadata indicating the source of this value (e.g. a file path)

        Raises
        ------
        TypeError
            If the node is frozen
        KeyError
            If the named layer does not exist
        """
        if self._frozen:
            raise TypeError('Frozen ConfigNode does not support assignment')

        if not layer:
            layer = self._layers[-1]
        self._values[layer] = (source, value)

    def drop_layer(self, layer):
        """Removes the named layer and the value associated with it from the node.

        Parameters
        ----------
        layer : str
            Name of the layer to drop.

        Raises
        ------
        TypeError
            If the node is frozen
        KeyError
            If the named layer does not exist
        """
        if self._frozen:
            raise TypeError('Frozen ConfigNode does not support modification')
        self.reset_layer(layer)
        self._layers.remove(layer)

    def reset_layer(self, layer):
        """Removes any value and metadata associated with the named layer.

        Parameters
        ----------
        layer : str
            Name of the layer to reset.

        Raises
        ------
        TypeError
            If the node is frozen
        KeyError
            If the named layer does not exist
        """
        if self._frozen:
            raise TypeError('Frozen ConfigNode does not support modification')
        if layer in self._values:
            del self._values[layer]

    def __repr__(self):
        return 'ConfigNode(layers={}, values={}, frozen={}, accessed={})'.format(
            self._layers, self._values, self._frozen, self._accessed)

    def __str__(self):
        return '\n'.join(reversed(['{}: {}\n    source: {}'.format(layer, value[1], value[0])
                                   for layer, value in self._values.items()]))

class ConfigTree:
    """A container for configuration information. Each configuration value is
    exposed as an attribute the value of which is determined by the outermost
    layer which has the key defined.
    """
    def __init__(self, data=None, layers=None):
        """
        Parameters
        ----------
        data : dict, str, or ConfigTree, optional
            A dictionary containing initial values
        layers : list
            A list of layer names. The order in which layers defined determines
            how they cascade. Later layers override the values from earlier ones.
        """
        if not layers:
            self.__dict__['_layers'] = ['base']
        else:
            self.__dict__['_layers'] = layers
        self.__dict__['_children'] = {}
        self.__dict__['_frozen'] = False

        if data:
            self.update(data, layer=self._layers[0], source='initial data')

    def freeze(self):
        """Causes the ConfigTree to become read only.

        This is useful for loading and then freezing configurations that should not be modified at runtime.
        """
        self.__dict__['_frozen'] = True
        for child in self._children.values():
            child.freeze()

    def __setattr__(self, name, value):
        """Set a configuration value on the outermost layer."""
        self.set_with_metadata(name, value, layer=None, source=None)

    def __setitem__(self, name, value):
        """Set a configuration value on the outermost layer."""
        self.set_with_metadata(name, value, layer=None, source=None)

    def __getattr__(self, name):
        """Get a configuration value from the outermost layer in which it appears."""
        try:
            return self.get_from_layer(name)
        except KeyError:
            raise AttributeError(name)

    def __getitem__(self, name):
        """Get a configuration value from the outermost layer in which it appears."""
        return self.get_from_layer(name)

    def __delattr__(self, name):
        if name in self._children:
            del self._children[name]

    def __delitem__(self, name):
        if name in self._children:
            del self._children[name]

    def __contains__(self, name):
        """Test if a configuration key exists in any layer."""
        return name in self._children

    def __iter__(self):
        """Dictionary-like iteration."""
        return iter(self._children)

    def items(self):
        """Return a list of all (child_name, child) pairs."""
        return self._children.items()

    def keys(self):
        return self._children.keys()

    def get_from_layer(self, name, layer=None):
        """Get a configuration value from the named layer.
        Parameters
        ----------
        name : str
            The name of the value to retrieve
        layer: str
            The name of the layer to retrieve the value from. If it is not supplied
            then the outermost layer in which the key is defined will be used.
        """
        if name not in self._children:
            if self._frozen:
                raise KeyError(name)
            self._children[name] = ConfigTree(layers=self._layers)
        child = self._children[name]
        if isinstance(child, ConfigNode):
            return child.get_value(layer)
        else:
            return child

    def set_with_metadata(self, name, value, layer=None, source=None):
        """Set a value in the named layer with the given source.
        Parameters
        ----------
        name : str
            The name of the value
        value
            The value to store
        layer : str, optional
            The name of the layer to store the value in. If none is supplied
            then the value will be stored in the outermost layer.
        source : str, optional
            The source to attribute the value to.

        Raises
        ------
        TypeError
            if the ConfigTree is frozen
        """

        if self._frozen:
            raise TypeError('Frozen ConfigTree does not support assignment')

        if isinstance(value, dict):
            if name not in self._children or not isinstance(self._children[name], ConfigTree):
                self._children[name] = ConfigTree(layers=list(self._layers))
            self._children[name].read_dict(value, layer, source)
        else:
            if name not in self._children or not isinstance(self._children[name], ConfigNode):
                self._children[name] = ConfigNode(list(self._layers))
            child = self._children[name]
            child.set_value(value, layer, source)

    def update(self, data: Union[Mapping, str, bytes], layer: str=None, source: str=None):
        """Adds additional data into the ConfigTree.

        Parameters
        ----------
        data :
            source data
        layer :
            layer to load data into. If none is supplied the outermost one is used
        source :
            Source to attribute the values to

        See Also
        --------
        read_dict
        """
        if isinstance(data, dict):
            self.read_dict(data, layer, source)
        elif isinstance(data, ConfigTree):
            # TODO: set this to parse the other config tree including layer and source info.  Maybe.
            self.read_dict(data.to_dict(), layer, source)
        elif isinstance(data, str):
            if data.endswith('.yaml'):
                source = source if source else data
                self.load(data, layer, source)
            else:
                self.loads(data, layer, source)
        elif data is None:
            pass
        else:
            raise ValueError(f"Update must be called with dictionary, string, or ConfigTree. "
                             f"You passed in {type(data)}")

    def read_dict(self, data_dict, layer=None, source=None):
        """Load a dictionary into the ConfigTree. If the dict contains nested dicts
        then the values will be added recursively. See module docstring for example code.

        Parameters
        ----------
        data_dict : dict
            source data
        layer : str
            layer to load data into. If none is supplied the outermost one is used
        source : str
            Source to attribute the values to
        """
        for k, v in data_dict.items():
            self.set_with_metadata(k, v, layer, source)

    def loads(self, data_string, layer=None, source=None):
        """Load data from a yaml formatted string.

        Parameters
        ----------
        data_string : str
            yaml formatted string. The root element of the document should be
            an associative array
        layer : str
            layer to load data into. If none is supplied the outermost one is used
        source : str
            Source to attribute the values to
        """
        data_dict = yaml.load(data_string)
        self.read_dict(data_dict, layer, source)

    def load(self, f, layer=None, source=None):
        """Load data from a yaml formatted file.

        Parameters
        ----------
        f : str or file like object
            If f is a string then it is interpreted as a path to the file to load
            If it is a file like object then data is read directly from it.
        layer : str
            layer to load data into. If none is supplied the outermost one is used
        source : str
            Source to attribute the values to
        """
        if hasattr(f, 'read'):
            self.loads(f.read(), layer=layer, source=source)
        else:
            with open(f) as f:
                self.loads(f.read(), layer=layer, source=source)

    def to_dict(self):
        result = {}
        for k, v in self._children.items():
            if isinstance(v, ConfigNode):
                result[k] = v.get_value()
            else:
                result[k] = v.to_dict()
        return result

    def metadata(self, name):
        """Return value and metadata associated with the named value

        Parameters
        ----------
        name : str
            name to retrieve. If the name contains '.'s it will be retrieved recursively

        Raises
        ------
        KeyError
            if name is not defined in the ConfigTree
        """
        if name in self._children:
            return self._children[name].metadata()
        else:
            head, _, tail = name.partition('.')
            if head in self._children:
                return self._children[head].metadata(key=tail)
            else:
                raise KeyError(name)

    def reset_layer(self, layer, preserve_keys=()):
        """Removes any value and metadata associated with the named layer.

        Parameters
        ----------
        layer : str
            Name of the layer to reset.
        preserve_keys : list or tuple
            A list of keys to skip while removing data

        Raises
        ------
        TypeError
            If the node is frozen
        KeyError
            If the named layer does not exist
        """
        self._reset_layer(layer, [k.split('.') for k in preserve_keys], prefix=[])

    def _reset_layer(self, layer, preserve_keys, prefix):
        if self._frozen:
            raise TypeError('Frozen ConfigTree does not support modification')
        deletable = []
        for key, child in self._children.items():
            if prefix + [key] not in preserve_keys:
                if isinstance(child, ConfigTree):
                    child._reset_layer(layer, preserve_keys, prefix + [key])
                else:
                    child.reset_layer(layer)
                    if child.is_empty():
                        deletable.append(key)
        for key in deletable:
            del self._children[key]

    def drop_layer(self, layer):
        """Removes the named layer and the value associated with it from the node.

        Parameters
        ----------
        layer : str
            Name of the layer to drop.

        Raises
        ------
        TypeError
            If the node is frozen
        KeyError
            If the named layer does not exist
        """
        if self._frozen:
            raise TypeError('Frozen ConfigTree does not support modification')
        for child in self._children.values():
            child.drop_layer(layer)
        self._layers.remove(layer)

    def unused_keys(self):
        """Lists all keys which are present in the ConfigTree but which have not been accessed."""
        unused = set()
        for k, c in self._children.items():
            if isinstance(c, ConfigNode):
                if not c.has_been_accessed():
                    unused.add(k)
            else:
                for ck in c.unused_keys():
                    unused.add(k+'.'+ck)
        return unused

    def __len__(self):
        return len(self._children)

    def __dir__(self):
        return list(self._children.keys()) + dir(super(ConfigTree, self))

    def __repr__(self):
        return 'ConfigTree(children={}, frozen={})'.format(
            ' '.join([repr(c) for c in self._children.values()]), self._frozen)

    def __str__(self):
        return '\n'.join(['{}:\n    {}'.format(name, str(c).replace('\n', '\n    '))
                          for name, c in self._children.items()])
<|endoftext|>


<commit_before>import redisdl
import unittest
import json
import os.path

class RedisdlTest(unittest.TestCase):
    def test_roundtrip(self):
        path = os.path.join(os.path.dirname(__file__), 'fixtures', 'dump.json')
        with open(path) as f:
            dump = f.read()

        redisdl.loads(dump)

        redump = redisdl.dumps()

        expected = json.loads(dump)
        actual = json.loads(redump)

        self.assertEqual(expected, actual)
<commit_msg>Kill whitespace<commit_after>import redisdl
import unittest
import json
import os.path

class RedisdlTest(unittest.TestCase):
    def test_roundtrip(self):
        path = os.path.join(os.path.dirname(__file__), 'fixtures', 'dump.json')
        with open(path) as f:
            dump = f.read()

        redisdl.loads(dump)

        redump = redisdl.dumps()

        expected = json.loads(dump)
        actual = json.loads(redump)

        self.assertEqual(expected, actual)
<|endoftext|>


<commit_before>from __future__ import print_function, absolute_import, division
import numpy as np
import pandas as pd
# from mousestyles.path_features import path_features
from path_features import angle_between

def find_edge_points(path_obj):
    r"""
    Returns a dictionary of the edge points of the path object.

    The dictionary containts min and max of each of the x, y coordinates in the path.

    Parameters
    ----------
    path_obj : pandas.DataFrame
        CX and CY must be contained.
        The length must be greater than 2.

    Returns
    -------
    edge points : dictionary
        contains edge points the path which are
        'xmin', 'xmax', 'ymin', and 'ymax'

    Examples
    --------
    >>> movement = data.load_movement(1,2,1)
    >>> sep = path_index(movement, 1, 1)
    >>> path = movement[sep[2][0]:sep[2][1]+1]
    >>> edges = find_edge_points(path)
    """
    if not isinstance(path_obj, pd.core.frame.DataFrame):
        raise TypeError("path_obj must be pandas DataFrame")

    if not set(path_obj.keys()).issuperset(['x', 'y']):
        raise ValueError("the keys of path_obj must contain 'x', 'y'")

    if len(path_obj) <= 1:
        raise ValueError("path_obj must contain at least 2 rows")

    return( {'xmin' : np.min(path_obj.x) , 'xmax' : np.max(path_obj.x) , \
             'ymin' : np.min(path_obj.y) , 'ymax' : np.max(path_obj.y)} )

def compute_area_rectangle(edge_points):
    r"""
    Returns the area of the rectangle spanned by the path.

    The rectangle, constructed by edge_points which is supposed
    to have xmin, xmax, ymin, and ymax, has 4 edges which are
    1) (xmin, ymin)
    2) (xmin, ymax)
    3) (xmax, ymin)
    4) (xmax, ymax)

    The path which has such edge points is by design completely
    covered by such rectangle.

    Parameters
    ----------
    edge_points : dictionary
        key must be xmin, xmax, ymin, ymax.
        Expecting the output of find_edge_points.

    Returns
    -------
    area : numpy float object
        area of the rectangle spanned by the path.

    Examples
    --------
    >>> movement = data.load_movement(1,2,1)
    >>> sep = path_index(movement, 1, 1)
    >>> path = movement[sep[2][0]:sep[2][1]+1]
    >>> edges = find_edge_points(path)
    >>> area_rec = compute_area_rectangle(edges)
    """
    if not isinstance(edge_points, dict):
        raise TypeError("edge_points must be a dictionary")

    if set(edge_points.keys()) != {'xmax', 'xmin', 'ymax', 'ymin'}:
        raise ValueError("the keys of edge_points must be 'xmax', 'xmin', 'ymax', and 'ymin'")

    return( (edge_points['xmax']-edge_points['xmin']) * \
           (edge_points['ymax']-edge_points['ymin']) )

def find_center(edge_points):
    r"""
    Returns the center point of the path.

    The center point is defined by the intersection point of
    2 diagonal lines of the rectangle which is constructed by
    edge_points. That is, for the elements of edge_points,
    xmin, xmax, ymin, and ymax, the center point is expressed by

    x_center = (xmin + xmax)/2,
    y_center = (ymin + ymax)/2.

    Parameters
    ----------
    edge_points : dictionary
        its key must be 'xmin', 'xmax', 'ymin', 'ymax'.
        Expecting the output of find_edge_points.

    Returns
    -------
    center point of the path : dictinoary
        defined by the intersection point of 2 diagonal lines of the
        rectangle spanned by the path.

    Examples
    --------
    >>> movement = data.load_movement(1,2,1)
    >>> sep = path_index(movement, 1, 1)
    >>> path = movement[sep[2][0]:sep[2][1]+1]
    >>> edges = find_edge_points(path)
    >>> cent = find_center(edges)
    """
    if not isinstance(edge_points, dict):
        raise TypeError("edge_points must be a dictionary")

    if set(edge_points.keys()) != {'xmax', 'xmin', 'ymax', 'ymin'}:
        raise ValueError("the keys of edge_points must be \
           'xmax', 'xmin', 'ymax', and 'ymin'")

    return( { 'x' : (edge_points['xmin'] + edge_points['xmax'])/2 , \
             'y' : (edge_points['ymin'] + edge_points['ymax'])/2 } )

def compute_radius_and_center_angle(path_obj, center):
    r"""
    Returns radius and center angles of the path.

    Radius of a path are defined by the length of each vector which
    connects the center point and each point in the path. The number
    of radius in a path is by design equal to the number of points in
    the path.

    For vectors connecting the center and each point in a path,
    center angles of the path are the angles between the 2 adjacent vectors.
    The number of the center angles in the path is by design equal to
    the number of points in the path minus 1.

    Parameters
    ----------
    path_obj : pandas.DataFrame
        CX and CY must be contained.
        The length must be greater than 2.

    center : dictionary
        its key must be 'x' and 'y'.
        Expecting the output of find_center.

    Returns
    -------
    dictionary having the following elements.
    radius : list
        each element is the distance between center point and
        each point in the path. The length equals to the length
        of the path_obj.

    center_anlges : list
        each element is the center angle generated by 2 adjacent
        radius. The length equals to the length of the radius minus 1.

    Examples
    --------
    >>> movement = data.load_movement(1,2,1)
    >>> sep = path_index(movement, 1, 1)
    >>> path = movement[sep[2][0]:sep[2][1]+1]
    >>> edges = find_edge_points(path)
    >>> cent = find_center(edges)
    >>> r_and_ca = compute_radius_and_center_angle(path, cent)
    """
    if not isinstance(path_obj, pd.core.frame.DataFrame):
        raise TypeError("path_obj must be pandas DataFrame")

    if not set(path_obj.keys()).issuperset(['x', 'y']):
        raise ValueError("the keys of path_obj must contain 'x', 'y'")

    if len(path_obj) <= 1:
        raise ValueError("path_obj must contain at least 2 rows")

    if not isinstance(center, dict):
        raise TypeError("center must be a dictionary")

    if set(center.keys()) != {'x', 'y'}:
        raise ValueError("the keys of center must be 'x', and 'y'")

    indices = path_obj.index[:-1]
    vectors = [ path_obj.loc[i,'x':'y'] - [center['x'],center['y']] for i in indices]
    # center_angles = [oath_features.angle_between(v1,v2) for v1,v2 in zip(vectors[1:], vectors[:len(vectors)])]
    center_angles = [angle_between(list(v1),list(v2)) for v1,v2 in zip(vectors[1:], vectors[:-1])]
    radius = [np.linalg.norm(v) for v in vectors]
    return({'radius': radius, 'center_angles': center_angles})

def compute_area_covered(r_and_theta):
    r"""
    Returns the area spanned by the path.

    For 2 adjacent points in a path and its center, the area of the triangle
    constructed by such 3 points can be calculated by the 2 radius and the center
    angle: let r1 and r2 be 2 adjacent radius, and theta be the angle between such 2 vectors.
    The area of the triangle is then calculated by
    r1 * r2 * sin(theta) / 2.

    This function sums up such areas of the triangles over all pairs of 2 adjacent points
    in the path.

    Parameters
    ----------
    r_and_theta : dictionary
        its key must be radius and center_angles.
        Expecting the output of compute_area_radius_and_center_angle.

    Returns
    -------
    area : numpy float
        area computed by radius and center angles in the path.

    Examples
    --------
    >>> movement = data.load_movement(1,2,1)
    >>> sep = path_index(movement, 1, 1)
    >>> path = movement[sep[2][0]:sep[2][1]+1]
    >>> edges = find_edge_points(path)
    >>> cent = find_center(edges)
    >>> r_and_ca = compute_radius_and_center_angle(path, cent)
    >>> area = compute_area_covered(r_and_ca)
    """

    if not isinstance(r_and_theta, dict):
        raise TypeError("r_and_theta must be a dictionary")

    if set(r_and_theta.keys()) != {'radius', 'center_angles'}:
        raise ValueError("the keys of r_and_theta must be 'radius' and 'center_angles'")

    if not isinstance(r_and_theta['radius'], list):
        raise TypeError("r_and_theta['radius'] must be a list")

    if not isinstance(r_and_theta['center_angles'], list):
        raise TypeError("r_and_theta['center_angles'] must be a list")

    if len(r_and_theta['radius']) != len(r_and_theta['center_angles']) + 1:
        raise ValueError("length of r_and_theta['radius'] must be \
            the length of r_and_theta['center_angles'] plus 1")

    zipped = zip(r_and_theta['radius'][1:], r_and_theta['radius'][:-1], \
                 r_and_theta['center_angles'])
    areas = [ v1 * v2 * np.sin(theta) / 2 for v1,v2,theta in zipped]
    return(sum(areas))

def compute_start_end_distance(path_obj):
    r'''
    Returns the distance between the start and end in a path.

    Parameters
    ----------
    path_obj : pandas.DataFrame
        CX and CY must be contained.
        The length must be greater than 2.

    Returns
    -------
    start-end distance : numpy float
        the distance between the start and end points in a path.

    Examples
    --------
    >>> movement = data.load_movement(1,2,1)
    >>> sep = path_index(movement, 1, 1)
    >>> path = movement[sep[2][0]:sep[2][1]+1]
    >>> p_dist = compute_start_end_distance(path)
    '''
    if not isinstance(path_obj, pd.core.frame.DataFrame):
        raise TypeError("path_obj must be pandas DataFrame")

    if not set(path_obj.keys()).issuperset(['x', 'y']):
        raise ValueError("the keys of path_obj must contain 'x', 'y'")

    if len(path_obj) <= 1:
        raise ValueError("path_obj must contain at least 2 rows")

    initial = path_obj.loc[path_obj.index[0], 'x':'y']
    end = path_obj.loc[path_obj.index[-1], 'x':'y']
    return(np.sqrt( (end['x'] - initial['x'])**2 + (end['y'] - initial['y'])**2 ))
<commit_msg>BUG, STY: modified alanna's point and styles<commit_after>from __future__ import print_function, absolute_import, division
import numpy as np
import pandas as pd
# from mousestyles.path_features import path_features
from path_features import angle_between

def find_edge_points(path_obj):
    r"""
    Returns a dictionary of the edge points of the path object.

    The dictionary containts min and max of each of the x, y coordinates
    in the path.

    Parameters
    ----------
    path_obj : pandas.DataFrame
        CX and CY must be contained.
        The length must be greater than 2.

    Returns
    -------
    edge points : dictionary
        contains edge points the path which are
        'xmin', 'xmax', 'ymin', and 'ymax'

    Examples
    --------
    >>> movement = data.load_movement(1,2,1)
    >>> sep = path_index(movement, 1, 1)
    >>> path = movement[sep[2][0]:sep[2][1]+1]
    >>> edges = find_edge_points(path)
    """
    if not isinstance(path_obj, pd.core.frame.DataFrame):
        raise TypeError("path_obj must be pandas DataFrame")

    if not set(path_obj.keys()).issuperset(['x', 'y']):
        raise ValueError("the keys of path_obj must contain 'x', 'y'")

    if len(path_obj) <= 1:
        raise ValueError("path_obj must contain at least 2 rows")

    return({'xmin': np.min(path_obj.x), 'xmax': np.max(path_obj.x),
            'ymin': np.min(path_obj.y), 'ymax': np.max(path_obj.y)})

def compute_area_rectangle(edge_points):
    r"""
    Returns the area of the rectangle spanned by the path.

    The rectangle, constructed by edge_points which is supposed
    to have xmin, xmax, ymin, and ymax, has 4 edges which are
    1) (xmin, ymin)
    2) (xmin, ymax)
    3) (xmax, ymin)
    4) (xmax, ymax)

    The path which has such edge points is by design completely
    covered by such rectangle.

    Parameters
    ----------
    edge_points : dictionary
        key must be xmin, xmax, ymin, ymax.
        Expecting the output of find_edge_points.

    Returns
    -------
    area : numpy float object
        area of the rectangle spanned by the path.

    Examples
    --------
    >>> movement = data.load_movement(1,2,1)
    >>> sep = path_index(movement, 1, 1)
    >>> path = movement[sep[2][0]:sep[2][1]+1]
    >>> edges = find_edge_points(path)
    >>> area_rec = compute_area_rectangle(edges)
    """
    if not isinstance(edge_points, dict):
        raise TypeError("edge_points must be a dictionary")

    if set(edge_points.keys()) != {'xmax', 'xmin', 'ymax', 'ymin'}:
        raise ValueError("the keys of edge_points must be 'xmax', 'xmin',
                         'ymax', and 'ymin'")

    return((edge_points['xmax']-edge_points['xmin']) *
           (edge_points['ymax']-edge_points['ymin']))

def find_center(edge_points):
    r"""
    Returns the center point of the path.

    The center point is defined by the intersection point of
    2 diagonal lines of the rectangle which is constructed by
    edge_points. That is, for the elements of edge_points,
    xmin, xmax, ymin, and ymax, the center point is expressed by

    x_center = (xmin + xmax)/2,
    y_center = (ymin + ymax)/2.

    Parameters
    ----------
    edge_points : dictionary
        its key must be 'xmin', 'xmax', 'ymin', 'ymax'.
        Expecting the output of find_edge_points.

    Returns
    -------
    center point of the path : dictionary
        defined by the intersection point of 2 diagonal lines of the
        rectangle spanned by the path.

    Examples
    --------
    >>> movement = data.load_movement(1,2,1)
    >>> sep = path_index(movement, 1, 1)
    >>> path = movement[sep[2][0]:sep[2][1]+1]
    >>> edges = find_edge_points(path)
    >>> cent = find_center(edges)
    """
    if not isinstance(edge_points, dict):
        raise TypeError("edge_points must be a dictionary")

    if set(edge_points.keys()) != {'xmax', 'xmin', 'ymax', 'ymin'}:
        raise ValueError("the keys of edge_points must be \
           'xmax', 'xmin', 'ymax', and 'ymin'")

    return({'x': (edge_points['xmin'] + edge_points['xmax'])/2,
            'y': (edge_points['ymin'] + edge_points['ymax'])/2})

def compute_radius_and_center_angle(path_obj, center):
    r"""
    Returns radius and center angles of the path.

    Radius of a path are defined by the length of each vector which
    connects the center point and each point in the path. The number
    of radius in a path is by design equal to the number of points in
    the path.

    For vectors connecting the center and each point in a path,
    center angles of the path are the angles between the 2 adjacent vectors.
    The number of the center angles in the path is by design equal to
    the number of points in the path minus 1.

    Parameters
    ----------
    path_obj : pandas.DataFrame
        CX and CY must be contained.
        The length must be greater than 2.

    center : dictionary
        its key must be 'x' and 'y'.
        Expecting the output of find_center.

    Returns
    -------
    dictionary having the following elements.
    radius : list
        each element is the distance between center point and
        each point in the path. The length equals to the length
        of the path_obj.

    center_anlges : list
        each element is the center angle generated by 2 adjacent
        radius. The length equals to the length of the radius minus 1.

    Examples
    --------
    >>> movement = data.load_movement(1,2,1)
    >>> sep = path_index(movement, 1, 1)
    >>> path = movement[sep[2][0]:sep[2][1]+1]
    >>> edges = find_edge_points(path)
    >>> cent = find_center(edges)
    >>> r_and_ca = compute_radius_and_center_angle(path, cent)
    """
    if not isinstance(path_obj, pd.core.frame.DataFrame):
        raise TypeError("path_obj must be pandas DataFrame")

    if not set(path_obj.keys()).issuperset(['x', 'y']):
        raise ValueError("the keys of path_obj must contain 'x', 'y'")

    if len(path_obj) <= 1:
        raise ValueError("path_obj must contain at least 2 rows")

    if not isinstance(center, dict):
        raise TypeError("center must be a dictionary")

    if set(center.keys()) != {'x', 'y'}:
        raise ValueError("the keys of center must be 'x', and 'y'")

    indices = path_obj.index[:-1]
    vectors = [path_obj.loc[i, 'x':'y'] -
               [center['x'], center['y']] for i in indices]
    center_angles = [path_features.angle_between(list(v1), list(v2)) for
                     v1, v2 in zip(vectors[1:], vectors[:len(vectors)])]
    radius = [np.linalg.norm(v) for v in vectors]
    return({'radius': radius, 'center_angles': center_angles})

def compute_area_covered(r_and_theta):
    r"""
    Returns the area spanned by the path.

    For 2 adjacent points in a path and its center,
    the area of the triangle constructed by such 3 points
    can be calculated by the 2 radius and the center angle:
    let r1 and r2 be 2 adjacent radius, and theta be
    the angle between such 2 vectors. The area of the triangle is
    then calculated by
    r1 * r2 * sin(theta) / 2.

    This function sums up such areas of the triangles over
    all pairs of 2 adjacent points in the path.

    Parameters
    ----------
    r_and_theta : dictionary
        its key must be radius and center_angles.
        Expecting the output of compute_area_radius_and_center_angle.

    Returns
    -------
    area : numpy float
        area computed by radius and center angles in the path.

    Examples
    --------
    >>> movement = data.load_movement(1,2,1)
    >>> sep = path_index(movement, 1, 1)
    >>> path = movement[sep[2][0]:sep[2][1]+1]
    >>> edges = find_edge_points(path)
    >>> cent = find_center(edges)
    >>> r_and_ca = compute_radius_and_center_angle(path, cent)
    >>> area = compute_area_covered(r_and_ca)
    """

    if not isinstance(r_and_theta, dict):
        raise TypeError("r_and_theta must be a dictionary")

    if set(r_and_theta.keys()) != {'radius', 'center_angles'}:
        raise ValueError("the keys of r_and_theta must be 'radius' and
                         'center_angles'")

    if not isinstance(r_and_theta['radius'], list):
        raise TypeError("r_and_theta['radius'] must be a list")

    if not isinstance(r_and_theta['center_angles'], list):
        raise TypeError("r_and_theta['center_angles'] must be a list")

    if len(r_and_theta['radius']) != len(r_and_theta['center_angles']) + 1:
        raise ValueError("length of r_and_theta['radius'] must be
                         the length of r_and_theta['center_angles'] plus 1")

    zipped = zip(r_and_theta['radius'][1:], r_and_theta['radius'][:-1],
                 r_and_theta['center_angles'])
    areas = [v1 * v2 * np.sin(theta) / 2 for v1, v2, theta in zipped]
    return(sum(areas))

def compute_start_end_distance(path_obj):
    r'''
    Returns the distance between the start and end in a path.

    Parameters
    ----------
    path_obj : pandas.DataFrame
        CX and CY must be contained.
        The length must be greater than 2.

    Returns
    -------
    start-end distance : numpy float
        the distance between the start and end points in a path.

    Examples
    --------
    >>> movement = data.load_movement(1,2,1)
    >>> sep = path_index(movement, 1, 1)
    >>> path = movement[sep[2][0]:sep[2][1]+1]
    >>> p_dist = compute_start_end_distance(path)
    '''
    if not isinstance(path_obj, pd.core.frame.DataFrame):
        raise TypeError("path_obj must be pandas DataFrame")

    if not set(path_obj.keys()).issuperset(['x', 'y']):
        raise ValueError("the keys of path_obj must contain 'x', 'y'")

    if len(path_obj) <= 1:
        raise ValueError("path_obj must contain at least 2 rows")

    initial = path_obj.loc[path_obj.index[0], 'x':'y']
    end = path_obj.loc[path_obj.index[-1], 'x':'y']
    return(np.sqrt((end['x'] - initial['x'])**2 +
           (end['y'] - initial['y'])**2))
<|endoftext|>


<commit_before>

#
# Magic commands
#

def load_experiment(magics, path):
    """Loads an experiment to iCE."""
    import pprint
    pprint.pprint(magics)
    pprint.pprint(path)

#
# Help command
#

MAGICS_HELP = {
    'load_exp': {
        'usage': '<The path of the experiment>',
        'description': load_experiment.__doc__
    }
}

def print_help(magics, definition):
    # Definition is defined :)
    if definition != '':
        if MAGICS_HELP.has_key(definition):
            print '%s\n Usage: %s %s' % (
                MAGICS_HELP[definition]['description'],
                definition, MAGICS_HELP[definition]['usage']
            )
        else:
            help(str(definition))
        return

    # General help
    print 'All commands explained:'
    print '\n'.join(
        [
            ' * %s: %s' % (key, h['description'])
            for key, h in MAGICS_HELP.items()
        ]
    )
<commit_msg>dict.has_key is removed in Python 3.x<commit_after>

#
# Magic commands
#

def load_experiment(magics, path):
    """Loads an experiment to iCE."""
    import pprint
    pprint.pprint(magics)
    pprint.pprint(path)

#
# Help command
#

MAGICS_HELP = {
    'load_exp': {
        'usage': '<The path of the experiment>',
        'description': load_experiment.__doc__
    }
}

def print_help(magics, definition):
    # Definition is defined :)
    if definition != '':
        if definition in MAGICS_HELP:
            print '%s\n Usage: %s %s' % (
                MAGICS_HELP[definition]['description'],
                definition, MAGICS_HELP[definition]['usage']
            )
        else:
            help(str(definition))
        return

    # General help
    print 'All commands explained:'
    print '\n'.join(
        [
            ' * %s: %s' % (key, h['description'])
            for key, h in MAGICS_HELP.items()
        ]
    )
<|endoftext|>


<commit_before>def modules():
	return ["python_inv"]

def derive(module, parameters):
	assert module == r"python_inv"
	if parameters.keys() != {r"\width"}:
		raise ValueError("Invalid parameters")
	return "ilang", r"""
module \impl
	wire width {width:d} input 1 \i
	wire width {width:d} output 2 \o
	cell $neg $0
		parameter \A_SIGNED 1'0
		parameter \A_WIDTH 32'{width:b}
		parameter \Y_WIDTH 32'{width:b}
		connect \A \i
		connect \Y \o
	end
end
module \python_inv
	wire width {width:d} input 1 \i
	wire width {width:d} output 2 \o
	cell \impl $0
		connect \i \i
		connect \o \o
	end
end
""".format(width=parameters[r"\width"])

# ----------------------------------------------------------------------------

import json
import argparse
import sys, socket, os, subprocess
try:
	import msvcrt, win32pipe, win32file
except ImportError:
	msvcrt = win32pipe = win32file = None

def map_parameter(parameter):
	if parameter["type"] == "unsigned":
		return int(parameter["value"], 2)
	if parameter["type"] == "signed":
		width = len(parameter["value"])
		value = int(parameter["value"], 2)
		if value & (1 << (width - 1)):
			value = -((1 << width) - value)
		return value
	if parameter["type"] == "string":
		return parameter["value"]
	if parameter["type"] == "real":
		return float(parameter["value"])

def call(input_json):
	input = json.loads(input_json)
	if input["method"] == "modules":
		return json.dumps({"modules": modules()})
	if input["method"] == "derive":
		try:
			frontend, source = derive(input["module"],
				{name: map_parameter(value) for name, value in input["parameters"].items()})
			return json.dumps({"frontend": frontend, "source": source})
		except ValueError as e:
			return json.dumps({"error": str(e)})

def main():
	parser = argparse.ArgumentParser()
	modes = parser.add_subparsers(dest="mode")
	mode_stdio = modes.add_parser("stdio")
	if os.name == "posix":
		mode_path = modes.add_parser("unix-socket")
	if os.name == "nt":
		mode_path = modes.add_parser("named-pipe")
	mode_path.add_argument("path")
	args = parser.parse_args()

	if args.mode == "stdio":
		while True:
			input = sys.stdin.readline()
			if not input: break
			sys.stdout.write(call(input) + "\n")
			sys.stdout.flush()

	if args.mode == "unix-socket":
		sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
		sock.bind(args.path)
		try:
			ys_proc = subprocess.Popen(["../../yosys", "-ql", "unix.log", "-p", "connect_rpc -path {}; read_verilog design.v; hierarchy -top top; flatten; select -assert-count 1 t:$neg".format(args.path)])
			sock.listen(1)
			conn, addr = sock.accept()
			file = conn.makefile("rw")
			while True:
				input = file.readline()
				if not input: break
				file.write(call(input) + "\n")
				file.flush()
			ys_proc.wait(timeout=10)
			if ys_proc.returncode:
				raise subprocess.CalledProcessError(ys_proc.returncode, ys_proc.args)
		finally:
			ys_proc.kill()
			sock.close()
			os.unlink(args.path)

	if args.mode == "named-pipe":
		pipe = win32pipe.CreateNamedPipe(args.path, win32pipe.PIPE_ACCESS_DUPLEX,
		    win32pipe.PIPE_TYPE_BYTE|win32pipe.PIPE_READMODE_BYTE|win32pipe.PIPE_WAIT,
		    1, 4096, 4096, 0, None)
		win32pipe.ConnectNamedPipe(pipe, None)
		try:
			while True:
				input = b""
				while not input.endswith(b"\n"):
					result, data = win32file.ReadFile(pipe, 4096)
					assert result == 0
					input += data
					assert not b"\n" in input or input.endswith(b"\n")
				output = (call(input.decode("utf-8")) + "\n").encode("utf-8")
				length = len(output)
				while length > 0:
					result, done = win32file.WriteFile(pipe, output)
					assert result == 0
					length -= done
		except win32file.error as e:
			if e.args[0] == 109: # ERROR_BROKEN_PIPE
				pass
			else:
				raise

if __name__ == "__main__":
	main()
<commit_msg>rpc test: make frontend listen before launching yosys & introduce safeguard if yosys errors<commit_after>def modules():
	return ["python_inv"]

def derive(module, parameters):
	assert module == r"python_inv"
	if parameters.keys() != {r"\width"}:
		raise ValueError("Invalid parameters")
	return "ilang", r"""
module \impl
	wire width {width:d} input 1 \i
	wire width {width:d} output 2 \o
	cell $neg $0
		parameter \A_SIGNED 1'0
		parameter \A_WIDTH 32'{width:b}
		parameter \Y_WIDTH 32'{width:b}
		connect \A \i
		connect \Y \o
	end
end
module \python_inv
	wire width {width:d} input 1 \i
	wire width {width:d} output 2 \o
	cell \impl $0
		connect \i \i
		connect \o \o
	end
end
""".format(width=parameters[r"\width"])

# ----------------------------------------------------------------------------

import json
import argparse
import sys, socket, os, subprocess
try:
	import msvcrt, win32pipe, win32file
except ImportError:
	msvcrt = win32pipe = win32file = None

def map_parameter(parameter):
	if parameter["type"] == "unsigned":
		return int(parameter["value"], 2)
	if parameter["type"] == "signed":
		width = len(parameter["value"])
		value = int(parameter["value"], 2)
		if value & (1 << (width - 1)):
			value = -((1 << width) - value)
		return value
	if parameter["type"] == "string":
		return parameter["value"]
	if parameter["type"] == "real":
		return float(parameter["value"])

def call(input_json):
	input = json.loads(input_json)
	if input["method"] == "modules":
		return json.dumps({"modules": modules()})
	if input["method"] == "derive":
		try:
			frontend, source = derive(input["module"],
				{name: map_parameter(value) for name, value in input["parameters"].items()})
			return json.dumps({"frontend": frontend, "source": source})
		except ValueError as e:
			return json.dumps({"error": str(e)})

def main():
	parser = argparse.ArgumentParser()
	modes = parser.add_subparsers(dest="mode")
	mode_stdio = modes.add_parser("stdio")
	if os.name == "posix":
		mode_path = modes.add_parser("unix-socket")
	if os.name == "nt":
		mode_path = modes.add_parser("named-pipe")
	mode_path.add_argument("path")
	args = parser.parse_args()

	if args.mode == "stdio":
		while True:
			input = sys.stdin.readline()
			if not input: break
			sys.stdout.write(call(input) + "\n")
			sys.stdout.flush()

	if args.mode == "unix-socket":
		sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
		sock.settimeout(30)
		sock.bind(args.path)
		try:
			sock.listen(1)
			ys_proc = subprocess.Popen(["../../yosys", "-ql", "unix.log", "-p", "connect_rpc -path {}; read_verilog design.v; hierarchy -top top; flatten; select -assert-count 1 t:$neg".format(args.path)])
			conn, addr = sock.accept()
			file = conn.makefile("rw")
			while True:
				input = file.readline()
				if not input: break
				file.write(call(input) + "\n")
				file.flush()
			ys_proc.wait(timeout=10)
			if ys_proc.returncode:
				raise subprocess.CalledProcessError(ys_proc.returncode, ys_proc.args)
		finally:
			ys_proc.kill()
			sock.close()
			os.unlink(args.path)

	if args.mode == "named-pipe":
		pipe = win32pipe.CreateNamedPipe(args.path, win32pipe.PIPE_ACCESS_DUPLEX,
		    win32pipe.PIPE_TYPE_BYTE|win32pipe.PIPE_READMODE_BYTE|win32pipe.PIPE_WAIT,
		    1, 4096, 4096, 0, None)
		win32pipe.ConnectNamedPipe(pipe, None)
		try:
			while True:
				input = b""
				while not input.endswith(b"\n"):
					result, data = win32file.ReadFile(pipe, 4096)
					assert result == 0
					input += data
					assert not b"\n" in input or input.endswith(b"\n")
				output = (call(input.decode("utf-8")) + "\n").encode("utf-8")
				length = len(output)
				while length > 0:
					result, done = win32file.WriteFile(pipe, output)
					assert result == 0
					length -= done
		except win32file.error as e:
			if e.args[0] == 109: # ERROR_BROKEN_PIPE
				pass
			else:
				raise

if __name__ == "__main__":
	main()
<|endoftext|>