<commit_before># coding: utf-8

# python lib:
import json
import random
from collections import Iterable

# thirdparties:
import unicodecsv

# pynames:
from pynames.relations import GENDER, LANGUAGE, LANGUAGE_FORMS_LANGTH
from pynames.names import Name
from pynames.base import BaseGenerator
from pynames import exceptions

class Template(object):

    __slots__ = ('name', 'probability', 'genders', 'template', 'native_language', 'languages')

    def __init__(self, name, native_language, languages, data):
        self.name = name
        self.native_language = native_language
        self.languages = frozenset(languages)
        self.probability = data['probability']
        self.genders = frozenset(data['genders'])
        self.template = data['template']

    def exists_for(self, genders):
        return genders & self.genders

    def get_names_number(self, tables):
        number = 1
        for slug in self.template:
            number *= len(tables[slug])
        return number

    @classmethod
    def merge_forms(cls, left, right):
        if not isinstance(left, basestring):
            if not isinstance(right, basestring):
                if len(left) != len(right):
                    raise exceptions.NotEqualFormsLengths(left=left, right=right)
                return [l+r for l, r in zip(left, right)]
            else:
                return [l+right for l in left]
        else:
            if not isinstance(right, basestring):
                return [left+r for r in right]
            else:
                return left + right

    def get_name(self, tables):
        languages = dict(
            (lang, u'') for lang in self.languages
        )
        for slug in self.template:
            record = random.choice(tables[slug])
            languages = {
                lang: self.merge_forms(forms, record['languages'][lang])
                for lang, forms in languages.iteritems()
            }

        genders = dict(
            (gender, languages)
            for gender in self.genders
        )

        return Name(self.native_language, {'genders': genders})

    def __eq__(self, other):
        return (
            self.native_language == other.native_language
            and self.languages == other.languages
            and self.probability == other.probability
            and self.genders == other.genders
            and self.template == other.template
        )

    def __hash__(self):
        return hash((self.native_language, self.languages, self.probability, self.genders, ';'.join(self.template)))

    def __repr__(self):
        return "<pynames.from_tables_generator.Template: %s=%s>" % (self.name, self.template)

class FromTablesGenerator(BaseGenerator):

    SOURCE = None

    def __init__(self):
        super(FromTablesGenerator, self).__init__()
        self.templates_choices = {}
        self.templates = []
        self.tables = {}
        self.source_loader(self.SOURCE)

    def source_loader(self, source):
        if source is None:
            error_msg = 'FromTablesGenerator: you must make subclass of FromTablesGenerator and define attribute SOURCE in it.'
            raise NotImplementedError(error_msg)

        with open(source) as f:
            data = json.load(f)
            self.native_language = data['native_language']
            self.languages = set(data['languages'])
            self.full_forms_for_languages = set(data.get('full_forms_for_languages', set()))
            self.templates = [
                Template(template_name, self.native_language, self.languages, template_data)
                for template_name, template_data in data['templates'].items()
            ]
            self.tables = data['tables']

    @staticmethod
    def _get_templates_cache_key(genders):
        return 't:%s' % '_'.join(genders)

    def _get_templates_slice(self, genders):
        key = self._get_templates_cache_key(genders)
        genders = frozenset(genders)
        if key not in self.templates_choices:
            self.templates_choices[key] = [template
                                           for template in self.templates
                                           if template.exists_for(genders)]
        return self.templates_choices[key]

    def _get_names_number_for_template(self):
        pass

    def get_names_number(self, genders=GENDER.ALL):
        templates = self._get_templates_slice(genders)
        number = sum([template.get_names_number(self.tables) for template in templates])
        return number

    def get_name(self, genders=GENDER.ALL):
        templates = self._get_templates_slice(genders)
        definition_number = sum([template.probability for template in templates])

        choice = random.randint(1, definition_number)

        for template in templates:
            if choice > template.probability:
                choice -= template.probability
                continue
            return template.get_name(self.tables)

        raise exceptions.WrongTemplateStructureError(genders=genders, soruce=self.SOURCE)

    def get_name_simple(self, gender=GENDER.MALE, language=LANGUAGE.NATIVE):
        name = self.get_name(genders=[gender])
        return name.get_for(gender, language)

    def test_names_consistency(self, test):
        for table_name, table in self.tables.iteritems():
            for record in table:
                test.assertEqual(set(record['languages'].keys()) & self.languages, self.languages)

        for template in self.templates:
            last_table = template.template[-1]
            for language in self.full_forms_for_languages:
                for record in self.tables[last_table]:
                    test.assertTrue(isinstance(record['languages'][language], list))
                    test.assertEqual(len(record['languages'][language]), LANGUAGE_FORMS_LANGTH[language])

class FromCSVTablesGenerator(FromTablesGenerator):

    """Variation of :py:calss:`FromTablesGenerator` that accepts path to 3 csv files as SOURCE.

    Read docs of :py:meth:`source_loader` for more details.

    """

    def source_loader(self, source_paths):
        """Load source from 3 csv files.

        First file should contain global settings:

        * ``native_lagnauge,languages`` header on first row
        * appropriate values on following rows

        Example::

            native_lagnauge,languages
            ru,ru
              ,en

        Second file should contain templates:

        * ``template_name,probability,genders,template`` header on first row
        * appropriate values on following rows (separate values with semicolon ";" in template column)

        Example::

            template_name,probability,genders,template
            male_1,5,m,prefixes;male_suffixes
            baby_1,1,m;f,prefixes;descriptive

        Third file should contain tables with values for template slugs in all languages:

        * first row should contain slugs with language code after colon for each
        * appropriate values on following rows. Multiple forms may be specified using semicolon as separator

        Example::

            prefixes:ru,prefixes:en,male_suffixes:ru,male_suffixes:en,descriptive:ru,descriptive:en
            Бж,Bzh,пра,pra,быстряк;быстряку,fasty
            дон;дону,don,Иван;Ивану,Ivan,Иванов;Иванову,Ivanov

        Note: you may use slugs without ":lang_code" suffix in csv header of tables file. Such headers will be treated as headers for native language

        If tables are missing for some slug then it is automatically created with values equeal to slug itself.
        So you may use some slugs without specifying tables data for them. Example for apostrophe and space:

            male_1,5,m,prefixes;';male_suffixes
            male_full,5,m,first_name; ;last_name

        """
        if not isinstance(source_paths, Iterable) or len(source_paths) < 3:
            raise TypeError('FromCSVTablesGenerator.source_loader accepts list of 3 paths as argument. Got `%s` instead' % source_paths)
        self.native_language = ''
        self.languages = []
        self.templates = []
        self.tables = {}

        with open(source_paths[0]) as settings_file:
            reader = unicodecsv.DictReader(settings_file, encoding='utf-8')
            for row in reader:
                new_native_language = row.get('native_language', '').strip()
                if new_native_language and not self.native_language:
                    self.native_language = new_native_language
                elif self.native_language and new_native_language and self.native_language != new_native_language:
                    raise exceptions.WrongCSVData(
                        'Wrong settings csv file. Native language is already set to "%(native_language)s" but new value "%(new_value)s" is present on some row',
                        native_language=self.native_language,
                        new_value=new_native_language
                    )

                new_language = row.get('languages', '').strip()
                if new_language:
                    self.languages.append(new_language)
        self.languages = set(self.languages)

        template_slugs = []

        with open(source_paths[1]) as templates_file:
            reader = unicodecsv.DictReader(templates_file, encoding='utf-8')
            for row in reader:
                template_data = {
                    'probability': float(row['probability']),
                    'genders': row['genders'].replace(' ', '').split(';'),
                    'template': row['template'].split(';'),
                }
                self.templates.append(
                    Template(row['template_name'], self.native_language, self.languages, template_data)
                )
                template_slugs.extend(template_data['template'])

        template_slugs = set(template_slugs)

        with open(source_paths[2]) as tables_file:
            reader = unicodecsv.DictReader(tables_file, encoding='utf-8')
            slugs = set([fieldname.split(':')[0] for fieldname in reader.fieldnames])
            for slug in slugs:
                self.tables[slug] = []
            for row in reader:
                for slug in slugs:
                    table_item = {}
                    for language in self.languages:
                        value = row.get('%s:%s' % (slug, language), '')
                        if not value and language == self.native_language:
                            value = row.get(slug, '')
                        if value:
                            if value.find(';') > 0:
                                value = value.split(';')
                            table_item.setdefault('languages', {})[language] = value
                        elif table_item:
                            # some language already present but current is missing
                            raise exceptions.WrongCSVData(
                                'Missing language "%(language)s" for table "%(slug)s" with partial datum "%(table_item)s"',
                                language=language, slug=slug, table_item=table_item,
                            )
                    if table_item:
                        self.tables[slug].append(table_item)

        for slug in template_slugs:
            if not self.tables.get(slug, ''):
                table_item = {'languages': {}}
                for language in self.languages:
                    table_item['languages'][language] = slug

                self.tables.setdefault(slug, []).append(table_item)

        self.full_forms_for_languages = set()
<commit_msg>source loading may now be skipped on generator init<commit_after># coding: utf-8

# python lib:
import json
import random
from collections import Iterable

# thirdparties:
import unicodecsv

# pynames:
from pynames.relations import GENDER, LANGUAGE, LANGUAGE_FORMS_LANGTH
from pynames.names import Name
from pynames.base import BaseGenerator
from pynames import exceptions

class Template(object):

    __slots__ = ('name', 'probability', 'genders', 'template', 'native_language', 'languages')

    def __init__(self, name, native_language, languages, data):
        self.name = name
        self.native_language = native_language
        self.languages = frozenset(languages)
        self.probability = data['probability']
        self.genders = frozenset(data['genders'])
        self.template = data['template']

    def exists_for(self, genders):
        return genders & self.genders

    def get_names_number(self, tables):
        number = 1
        for slug in self.template:
            number *= len(tables[slug])
        return number

    @classmethod
    def merge_forms(cls, left, right):
        if not isinstance(left, basestring):
            if not isinstance(right, basestring):
                if len(left) != len(right):
                    raise exceptions.NotEqualFormsLengths(left=left, right=right)
                return [l+r for l, r in zip(left, right)]
            else:
                return [l+right for l in left]
        else:
            if not isinstance(right, basestring):
                return [left+r for r in right]
            else:
                return left + right

    def get_name(self, tables):
        languages = dict(
            (lang, u'') for lang in self.languages
        )
        for slug in self.template:
            record = random.choice(tables[slug])
            languages = {
                lang: self.merge_forms(forms, record['languages'][lang])
                for lang, forms in languages.iteritems()
            }

        genders = dict(
            (gender, languages)
            for gender in self.genders
        )

        return Name(self.native_language, {'genders': genders})

    def __eq__(self, other):
        return (
            self.native_language == other.native_language
            and self.languages == other.languages
            and self.probability == other.probability
            and self.genders == other.genders
            and self.template == other.template
        )

    def __hash__(self):
        return hash((self.native_language, self.languages, self.probability, self.genders, ';'.join(self.template)))

    def __repr__(self):
        return "<pynames.from_tables_generator.Template: %s=%s>" % (self.name, self.template)

class FromTablesGenerator(BaseGenerator):

    SOURCE = None
    AUTOLOAD = True  # you may want to set ``AUTOLOAD = False`` when subclassing table-based generator to skip soruce loading on init

    def __init__(self):
        super(FromTablesGenerator, self).__init__()
        self.templates_choices = {}
        self.templates = []
        self.tables = {}
        if self.AUTOLOAD:
            self.source_loader(self.SOURCE)

    def source_loader(self, source):
        if source is None:
            error_msg = 'FromTablesGenerator: you must make subclass of FromTablesGenerator and define attribute SOURCE in it.'
            raise NotImplementedError(error_msg)

        with open(source) as f:
            data = json.load(f)
            self.native_language = data['native_language']
            self.languages = set(data['languages'])
            self.full_forms_for_languages = set(data.get('full_forms_for_languages', set()))
            self.templates = [
                Template(template_name, self.native_language, self.languages, template_data)
                for template_name, template_data in data['templates'].items()
            ]
            self.tables = data['tables']

    @staticmethod
    def _get_templates_cache_key(genders):
        return 't:%s' % '_'.join(genders)

    def _get_templates_slice(self, genders):
        key = self._get_templates_cache_key(genders)
        genders = frozenset(genders)
        if key not in self.templates_choices:
            self.templates_choices[key] = [template
                                           for template in self.templates
                                           if template.exists_for(genders)]
        return self.templates_choices[key]

    def _get_names_number_for_template(self):
        pass

    def get_names_number(self, genders=GENDER.ALL):
        templates = self._get_templates_slice(genders)
        number = sum([template.get_names_number(self.tables) for template in templates])
        return number

    def get_name(self, genders=GENDER.ALL):
        templates = self._get_templates_slice(genders)
        definition_number = sum([template.probability for template in templates])

        choice = random.randint(1, definition_number)

        for template in templates:
            if choice > template.probability:
                choice -= template.probability
                continue
            return template.get_name(self.tables)

        raise exceptions.WrongTemplateStructureError(genders=genders, soruce=self.SOURCE)

    def get_name_simple(self, gender=GENDER.MALE, language=LANGUAGE.NATIVE):
        name = self.get_name(genders=[gender])
        return name.get_for(gender, language)

    def test_names_consistency(self, test):
        for table_name, table in self.tables.iteritems():
            for record in table:
                test.assertEqual(set(record['languages'].keys()) & self.languages, self.languages)

        for template in self.templates:
            last_table = template.template[-1]
            for language in self.full_forms_for_languages:
                for record in self.tables[last_table]:
                    test.assertTrue(isinstance(record['languages'][language], list))
                    test.assertEqual(len(record['languages'][language]), LANGUAGE_FORMS_LANGTH[language])

class FromCSVTablesGenerator(FromTablesGenerator):

    """Variation of :py:calss:`FromTablesGenerator` that accepts path to 3 csv files as SOURCE.

    Read docs of :py:meth:`source_loader` for more details.

    """

    def source_loader(self, source_paths):
        """Load source from 3 csv files.

        First file should contain global settings:

        * ``native_lagnauge,languages`` header on first row
        * appropriate values on following rows

        Example::

            native_lagnauge,languages
            ru,ru
              ,en

        Second file should contain templates:

        * ``template_name,probability,genders,template`` header on first row
        * appropriate values on following rows (separate values with semicolon ";" in template column)

        Example::

            template_name,probability,genders,template
            male_1,5,m,prefixes;male_suffixes
            baby_1,1,m;f,prefixes;descriptive

        Third file should contain tables with values for template slugs in all languages:

        * first row should contain slugs with language code after colon for each
        * appropriate values on following rows. Multiple forms may be specified using semicolon as separator

        Example::

            prefixes:ru,prefixes:en,male_suffixes:ru,male_suffixes:en,descriptive:ru,descriptive:en
            Бж,Bzh,пра,pra,быстряк;быстряку,fasty
            дон;дону,don,Иван;Ивану,Ivan,Иванов;Иванову,Ivanov

        Note: you may use slugs without ":lang_code" suffix in csv header of tables file. Such headers will be treated as headers for native language

        If tables are missing for some slug then it is automatically created with values equeal to slug itself.
        So you may use some slugs without specifying tables data for them. Example for apostrophe and space:

            male_1,5,m,prefixes;';male_suffixes
            male_full,5,m,first_name; ;last_name

        """
        if not isinstance(source_paths, Iterable) or len(source_paths) < 3:
            raise TypeError('FromCSVTablesGenerator.source_loader accepts list of 3 paths as argument. Got `%s` instead' % source_paths)
        self.native_language = ''
        self.languages = []
        self.templates = []
        self.tables = {}

        with open(source_paths[0]) as settings_file:
            reader = unicodecsv.DictReader(settings_file, encoding='utf-8')
            for row in reader:
                new_native_language = row.get('native_language', '').strip()
                if new_native_language and not self.native_language:
                    self.native_language = new_native_language
                elif self.native_language and new_native_language and self.native_language != new_native_language:
                    raise exceptions.WrongCSVData(
                        'Wrong settings csv file. Native language is already set to "%(native_language)s" but new value "%(new_value)s" is present on some row',
                        native_language=self.native_language,
                        new_value=new_native_language
                    )

                new_language = row.get('languages', '').strip()
                if new_language:
                    self.languages.append(new_language)
        self.languages = set(self.languages)

        template_slugs = []

        with open(source_paths[1]) as templates_file:
            reader = unicodecsv.DictReader(templates_file, encoding='utf-8')
            for row in reader:
                template_data = {
                    'probability': float(row['probability']),
                    'genders': row['genders'].replace(' ', '').split(';'),
                    'template': row['template'].split(';'),
                }
                self.templates.append(
                    Template(row['template_name'], self.native_language, self.languages, template_data)
                )
                template_slugs.extend(template_data['template'])

        template_slugs = set(template_slugs)

        with open(source_paths[2]) as tables_file:
            reader = unicodecsv.DictReader(tables_file, encoding='utf-8')
            slugs = set([fieldname.split(':')[0] for fieldname in reader.fieldnames])
            for slug in slugs:
                self.tables[slug] = []
            for row in reader:
                for slug in slugs:
                    table_item = {}
                    for language in self.languages:
                        value = row.get('%s:%s' % (slug, language), '')
                        if not value and language == self.native_language:
                            value = row.get(slug, '')
                        if value:
                            if value.find(';') > 0:
                                value = value.split(';')
                            table_item.setdefault('languages', {})[language] = value
                        elif table_item:
                            # some language already present but current is missing
                            raise exceptions.WrongCSVData(
                                'Missing language "%(language)s" for table "%(slug)s" with partial datum "%(table_item)s"',
                                language=language, slug=slug, table_item=table_item,
                            )
                    if table_item:
                        self.tables[slug].append(table_item)

        for slug in template_slugs:
            if not self.tables.get(slug, ''):
                table_item = {'languages': {}}
                for language in self.languages:
                    table_item['languages'][language] = slug

                self.tables.setdefault(slug, []).append(table_item)

        self.full_forms_for_languages = set()
<|endoftext|>


<commit_before>import asyncio
import sys
from pathlib import Path
import textwrap
from typing import Text

import pytest
from _pytest.capture import CaptureFixture
from _pytest.monkeypatch import MonkeyPatch

import rasa.shared.utils.io
import rasa.utils.io
from rasa.core.agent import Agent
from rasa.shared.core.events import UserUttered
from rasa.core.test import (
    EvaluationStore,
    WronglyClassifiedUserUtterance,
    WronglyPredictedAction,
)
from rasa.shared.core.trackers import DialogueStateTracker
from rasa.shared.core.training_data.story_writer.yaml_story_writer import (
    YAMLStoryWriter,
)
import rasa.model
import rasa.cli.utils
from rasa.nlu.test import NO_ENTITY
import rasa.core
from rasa.shared.nlu.constants import (
    ENTITY_ATTRIBUTE_VALUE,
    ENTITY_ATTRIBUTE_START,
    ENTITY_ATTRIBUTE_END,
    ENTITY_ATTRIBUTE_TYPE,
    ENTITY_ATTRIBUTE_TEXT,
)

def monkeypatch_get_latest_model(tmp_path: Path, monkeypatch: MonkeyPatch) -> None:
    latest_model = tmp_path / "my_test_model.tar.gz"
    monkeypatch.setattr(rasa.model, "get_latest_model", lambda: str(latest_model))

def test_get_sanitized_model_directory_when_not_passing_model(
    capsys: CaptureFixture, tmp_path: Path, monkeypatch: MonkeyPatch
):
    from rasa.model_testing import _get_sanitized_model_directory

    monkeypatch_get_latest_model(tmp_path, monkeypatch)

    # Create a fake model on disk so that `is_file` returns `True`
    latest_model = Path(rasa.model.get_latest_model())
    latest_model.touch()

    # Input: default model file
    # => Should return containing directory
    new_modeldir = _get_sanitized_model_directory(str(latest_model))
    captured = capsys.readouterr()
    assert not captured.out
    assert new_modeldir == str(latest_model.parent)

def test_get_sanitized_model_directory_when_passing_model_file_explicitly(
    capsys: CaptureFixture, tmp_path: Path, monkeypatch: MonkeyPatch
):
    from rasa.model_testing import _get_sanitized_model_directory

    monkeypatch_get_latest_model(tmp_path, monkeypatch)

    other_model = tmp_path / "my_test_model1.tar.gz"
    assert str(other_model) != rasa.model.get_latest_model()
    other_model.touch()

    # Input: some file
    # => Should return containing directory and print a warning
    new_modeldir = _get_sanitized_model_directory(str(other_model))
    captured = capsys.readouterr()
    assert captured.out
    assert new_modeldir == str(other_model.parent)

def test_get_sanitized_model_directory_when_passing_other_input(
    capsys: CaptureFixture, tmp_path: Path, monkeypatch: MonkeyPatch
):
    from rasa.model_testing import _get_sanitized_model_directory

    monkeypatch_get_latest_model(tmp_path, monkeypatch)

    # Input: anything that is not an existing file
    # => Should return input
    modeldir = "random_dir"
    assert not Path(modeldir).is_file()
    new_modeldir = _get_sanitized_model_directory(modeldir)
    captured = capsys.readouterr()
    assert not captured.out
    assert new_modeldir == modeldir

@pytest.mark.parametrize(
    "targets,predictions,expected_precision,expected_fscore,expected_accuracy",
    [
        (
            ["no_entity", "location", "no_entity", "location", "no_entity"],
            ["no_entity", "location", "no_entity", "no_entity", "person"],
            1.0,
            0.6666666666666666,
            3 / 5,
        ),
        (
            ["no_entity", "no_entity", "no_entity", "no_entity", "person"],
            ["no_entity", "no_entity", "no_entity", "no_entity", "no_entity"],
            0.0,
            0.0,
            4 / 5,
        ),
    ],
)
def test_get_evaluation_metrics(
    targets, predictions, expected_precision, expected_fscore, expected_accuracy
):
    from rasa.model_testing import get_evaluation_metrics

    report, precision, f1, accuracy = get_evaluation_metrics(
        targets, predictions, True, exclude_label=NO_ENTITY
    )

    assert f1 == expected_fscore
    assert precision == expected_precision
    assert accuracy == expected_accuracy
    assert NO_ENTITY not in report

@pytest.mark.parametrize(
    "targets,exclude_label,expected",
    [
        (
            ["no_entity", "location", "location", "location", "person"],
            NO_ENTITY,
            ["location", "person"],
        ),
        (
            ["no_entity", "location", "location", "location", "person"],
            None,
            ["no_entity", "location", "person"],
        ),
        (["no_entity"], NO_ENTITY, []),
        (["location", "location", "location"], NO_ENTITY, ["location"]),
        ([], None, []),
    ],
)
def test_get_label_set(targets, exclude_label, expected):
    from rasa.model_testing import get_unique_labels

    actual = get_unique_labels(targets, exclude_label)
    assert set(expected) == set(actual)

async def test_interpreter_passed_to_agent(
    monkeypatch: MonkeyPatch, trained_rasa_model: Text
):
    from rasa.core.interpreter import RasaNLUInterpreter

    agent = Agent.load(trained_rasa_model)
    assert isinstance(agent.interpreter, RasaNLUInterpreter)

def test_e2e_warning_if_no_nlu_model(
    monkeypatch: MonkeyPatch, trained_core_model: Text, capsys: CaptureFixture
):
    from rasa.model_testing import test_core

    # Patching is bit more complicated as we have a module `train` and function
    # with the same name 😬
    monkeypatch.setattr(
        sys.modules["rasa.core.test"], "test", asyncio.coroutine(lambda *_, **__: True)
    )

    test_core(trained_core_model, additional_arguments={"e2e": True})

    assert "No NLU model found. Using default" in capsys.readouterr().out

def test_write_classification_errors():
    evaluation = EvaluationStore(
        action_predictions=["utter_goodbye"],
        action_targets=["utter_greet"],
        intent_predictions=["goodbye"],
        intent_targets=["greet"],
        entity_predictions=None,
        entity_targets=None,
    )
    events = [
        WronglyClassifiedUserUtterance(
            UserUttered("Hello", {"name": "goodbye"}), evaluation
        ),
        WronglyPredictedAction("utter_greet", "", "utter_goodbye"),
    ]
    tracker = DialogueStateTracker.from_events("default", events)
    dump = YAMLStoryWriter().dumps(tracker.as_story().story_steps)
    assert (
        dump.strip()
        == textwrap.dedent(
            """
        version: "2.0"
        stories:
        - story: default
          steps:
          - intent: greet  # predicted: goodbye: Hello
          - action: utter_greet  # predicted: utter_goodbye

    """
        ).strip()
    )

def test_log_failed_stories(tmp_path: Path):
    path = str(tmp_path / "stories.yml")
    rasa.core.test._log_stories([], path, "Some text")

    dump = rasa.shared.utils.io.read_file(path)

    assert dump.startswith("#")
    assert len(dump.split("\n")) == 1

@pytest.mark.parametrize(
    "entity_predictions,entity_targets",
    [
        (
            [{"text": "hi, how are you", "start": 4, "end": 7, "entity": "aa"}],
            [
                {"text": "hi, how are you", "start": 0, "end": 2, "entity": "bb"},
                {"text": "hi, how are you", "start": 4, "end": 7, "entity": "aa"},
            ],
        ),
        (
            [
                {"text": "hi, how are you", "start": 0, "end": 2, "entity": "bb"},
                {"text": "hi, how are you", "start": 4, "end": 7, "entity": "aa"},
            ],
            [
                {"text": "hi, how are you", "start": 0, "end": 2, "entity": "bb"},
                {"text": "hi, how are you", "start": 4, "end": 7, "entity": "aa"},
            ],
        ),
        (
            [
                {"text": "hi, how are you", "start": 0, "end": 2, "entity": "bb"},
                {"text": "hi, how are you", "start": 4, "end": 7, "entity": "aa"},
            ],
            [{"text": "hi, how are you", "start": 4, "end": 7, "entity": "aa"}],
        ),
        (
            [
                {
                    "text": "Tanja is currently in Munich, but she lives in Berlin",
                    "start": 0,
                    "end": 5,
                    "entity": "person",
                },
                {
                    "text": "Tanja is currently in Munich, but she lives in Berlin",
                    "start": 22,
                    "end": 28,
                    "entity": "city",
                },
                {
                    "text": "Tanja is currently in Munich, but she lives in Berlin",
                    "start": 47,
                    "end": 53,
                    "entity": "city",
                },
            ],
            [
                {
                    "text": "Tanja is currently in Munich, but she lives in Berlin",
                    "start": 22,
                    "end": 28,
                    "entity": "city",
                }
            ],
        ),
        (
            [
                {
                    "text": "Tanja is currently in Munich, but she lives in Berlin",
                    "start": 0,
                    "end": 5,
                    "entity": "person",
                },
                {
                    "text": "Tanja is currently in Munich, but she lives in Berlin",
                    "start": 47,
                    "end": 53,
                    "entity": "city",
                },
            ],
            [
                {
                    "text": "Tanja is currently in Munich, but she lives in Berlin",
                    "start": 22,
                    "end": 28,
                    "entity": "city",
                },
                {
                    "text": "Tanja is currently in Munich, but she lives in Berlin",
                    "start": 47,
                    "end": 53,
                    "entity": "city",
                },
            ],
        ),
        (
            [
                {
                    "text": "Tanja is currently in Munich, but she lives in Berlin",
                    "start": 47,
                    "end": 53,
                    "entity": "city",
                }
            ],
            [
                {
                    "text": "Tanja is currently in Munich, but she lives in Berlin",
                    "start": 0,
                    "end": 5,
                    "entity": "person",
                },
                {
                    "text": "Tanja is currently in Munich, but she lives in Berlin",
                    "start": 22,
                    "end": 28,
                    "entity": "city",
                },
                {
                    "text": "Tanja is currently in Munich, but she lives in Berlin",
                    "start": 47,
                    "end": 53,
                    "entity": "city",
                },
            ],
        ),
    ],
)
def test_evaluation_store_serialise(entity_predictions, entity_targets):
    from rasa.shared.nlu.training_data.formats.readerwriter import TrainingDataWriter

    store = EvaluationStore(
        entity_predictions=entity_predictions, entity_targets=entity_targets
    )

    targets, predictions = store.serialise()

    assert len(targets) == len(predictions)

    i_pred = 0
    i_target = 0
    for i, prediction in enumerate(predictions):
        target = targets[i]
        if prediction != "None" and target != "None":
            predicted = entity_predictions[i_pred]
            assert prediction == TrainingDataWriter.generate_entity(
                predicted.get("text"), predicted
            )
            assert predicted.get("start") == entity_targets[i_target].get("start")
            assert predicted.get("end") == entity_targets[i_target].get("end")

        if prediction != "None":
            i_pred += 1
        if target != "None":
            i_target += 1

async def test_test_does_not_use_rules(tmp_path: Path, default_agent: Agent):
    from rasa.core.test import _create_data_generator

    test_file = tmp_path / "test.yml"
    test_name = "my test story"
    tests = f"""
stories:
- story: {test_name}
  steps:
  - intent: greet
  - action: utter_greet

rules:
- rule: rule which is ignored
  steps:
  - intent: greet
  - action: utter_greet
    """

    test_file.write_text(tests)

    generator = await _create_data_generator(str(test_file), default_agent)
    test_trackers = generator.generate_story_trackers()
    assert len(test_trackers) == 1
    assert test_trackers[0].sender_id == test_name

def test_duplicated_entity_predictions_tolerated():
    """Same entity extracted multiple times shouldn't be flagged as prediction error.

    This can happen when multiple entity extractors extract the same entity but a test
    story only lists the entity once. For completeness, the other case (entity listed
    twice in test story and extracted once) is also tested here because it should work
    the same way.
    """
    entity = {
        ENTITY_ATTRIBUTE_TEXT: "Algeria",
        ENTITY_ATTRIBUTE_START: 0,
        ENTITY_ATTRIBUTE_END: 7,
        ENTITY_ATTRIBUTE_VALUE: "Algeria",
        ENTITY_ATTRIBUTE_TYPE: "country",
    }
    evaluation_with_duplicated_prediction = EvaluationStore(
        entity_predictions=[entity, entity], entity_targets=[entity]
    )
    assert not evaluation_with_duplicated_prediction.check_prediction_target_mismatch()

    evaluation_with_duplicated_target = EvaluationStore(
        entity_predictions=[entity], entity_targets=[entity, entity]
    )
    assert not evaluation_with_duplicated_target.check_prediction_target_mismatch()

def test_differently_ordered_entity_predictions_tolerated():
    """Order in which entities were extracted shouldn't matter.

    Let's have an utterance like this: "[Researcher](job_name) from [Germany](country)."
    and imagine we use different entity extractors for the two entities. Then, the order
    in which entities are extracted from the utterance depends on the order in which the
    extractors are listed in the NLU pipeline. However, the expected order is given by
    where the entities are found in the utterance, i.e. "Researcher" comes before
    "Germany". Hence, it's reasonable for the expected and extracted order to not match
    and it shouldn't be flagged as a prediction error.

    """
    entity1 = {
        ENTITY_ATTRIBUTE_TEXT: "Algeria and Albania",
        ENTITY_ATTRIBUTE_START: 0,
        ENTITY_ATTRIBUTE_END: 7,
        ENTITY_ATTRIBUTE_VALUE: "Algeria",
        ENTITY_ATTRIBUTE_TYPE: "country",
    }
    entity2 = {
        ENTITY_ATTRIBUTE_TEXT: "Algeria and Albania",
        ENTITY_ATTRIBUTE_START: 12,
        ENTITY_ATTRIBUTE_END: 19,
        ENTITY_ATTRIBUTE_VALUE: "Albania",
        ENTITY_ATTRIBUTE_TYPE: "country",
    }
    evaluation = EvaluationStore(
        entity_predictions=[entity1, entity2], entity_targets=[entity2, entity1]
    )
    assert not evaluation.check_prediction_target_mismatch()
<commit_msg>Update tests/test_model_testing.py<commit_after>import asyncio
import sys
from pathlib import Path
import textwrap
from typing import Text

import pytest
from _pytest.capture import CaptureFixture
from _pytest.monkeypatch import MonkeyPatch

import rasa.shared.utils.io
import rasa.utils.io
from rasa.core.agent import Agent
from rasa.shared.core.events import UserUttered
from rasa.core.test import (
    EvaluationStore,
    WronglyClassifiedUserUtterance,
    WronglyPredictedAction,
)
from rasa.shared.core.trackers import DialogueStateTracker
from rasa.shared.core.training_data.story_writer.yaml_story_writer import (
    YAMLStoryWriter,
)
import rasa.model
import rasa.cli.utils
from rasa.nlu.test import NO_ENTITY
import rasa.core
from rasa.shared.nlu.constants import (
    ENTITY_ATTRIBUTE_VALUE,
    ENTITY_ATTRIBUTE_START,
    ENTITY_ATTRIBUTE_END,
    ENTITY_ATTRIBUTE_TYPE,
    ENTITY_ATTRIBUTE_TEXT,
)

def monkeypatch_get_latest_model(tmp_path: Path, monkeypatch: MonkeyPatch) -> None:
    latest_model = tmp_path / "my_test_model.tar.gz"
    monkeypatch.setattr(rasa.model, "get_latest_model", lambda: str(latest_model))

def test_get_sanitized_model_directory_when_not_passing_model(
    capsys: CaptureFixture, tmp_path: Path, monkeypatch: MonkeyPatch
):
    from rasa.model_testing import _get_sanitized_model_directory

    monkeypatch_get_latest_model(tmp_path, monkeypatch)

    # Create a fake model on disk so that `is_file` returns `True`
    latest_model = Path(rasa.model.get_latest_model())
    latest_model.touch()

    # Input: default model file
    # => Should return containing directory
    new_modeldir = _get_sanitized_model_directory(str(latest_model))
    captured = capsys.readouterr()
    assert not captured.out
    assert new_modeldir == str(latest_model.parent)

def test_get_sanitized_model_directory_when_passing_model_file_explicitly(
    capsys: CaptureFixture, tmp_path: Path, monkeypatch: MonkeyPatch
):
    from rasa.model_testing import _get_sanitized_model_directory

    monkeypatch_get_latest_model(tmp_path, monkeypatch)

    other_model = tmp_path / "my_test_model1.tar.gz"
    assert str(other_model) != rasa.model.get_latest_model()
    other_model.touch()

    # Input: some file
    # => Should return containing directory and print a warning
    new_modeldir = _get_sanitized_model_directory(str(other_model))
    captured = capsys.readouterr()
    assert captured.out
    assert new_modeldir == str(other_model.parent)

def test_get_sanitized_model_directory_when_passing_other_input(
    capsys: CaptureFixture, tmp_path: Path, monkeypatch: MonkeyPatch
):
    from rasa.model_testing import _get_sanitized_model_directory

    monkeypatch_get_latest_model(tmp_path, monkeypatch)

    # Input: anything that is not an existing file
    # => Should return input
    modeldir = "random_dir"
    assert not Path(modeldir).is_file()
    new_modeldir = _get_sanitized_model_directory(modeldir)
    captured = capsys.readouterr()
    assert not captured.out
    assert new_modeldir == modeldir

@pytest.mark.parametrize(
    "targets,predictions,expected_precision,expected_fscore,expected_accuracy",
    [
        (
            ["no_entity", "location", "no_entity", "location", "no_entity"],
            ["no_entity", "location", "no_entity", "no_entity", "person"],
            1.0,
            0.6666666666666666,
            3 / 5,
        ),
        (
            ["no_entity", "no_entity", "no_entity", "no_entity", "person"],
            ["no_entity", "no_entity", "no_entity", "no_entity", "no_entity"],
            0.0,
            0.0,
            4 / 5,
        ),
    ],
)
def test_get_evaluation_metrics(
    targets, predictions, expected_precision, expected_fscore, expected_accuracy
):
    from rasa.model_testing import get_evaluation_metrics

    report, precision, f1, accuracy = get_evaluation_metrics(
        targets, predictions, True, exclude_label=NO_ENTITY
    )

    assert f1 == expected_fscore
    assert precision == expected_precision
    assert accuracy == expected_accuracy
    assert NO_ENTITY not in report

@pytest.mark.parametrize(
    "targets,exclude_label,expected",
    [
        (
            ["no_entity", "location", "location", "location", "person"],
            NO_ENTITY,
            ["location", "person"],
        ),
        (
            ["no_entity", "location", "location", "location", "person"],
            None,
            ["no_entity", "location", "person"],
        ),
        (["no_entity"], NO_ENTITY, []),
        (["location", "location", "location"], NO_ENTITY, ["location"]),
        ([], None, []),
    ],
)
def test_get_label_set(targets, exclude_label, expected):
    from rasa.model_testing import get_unique_labels

    actual = get_unique_labels(targets, exclude_label)
    assert set(expected) == set(actual)

async def test_interpreter_passed_to_agent(
    monkeypatch: MonkeyPatch, trained_rasa_model: Text
):
    from rasa.core.interpreter import RasaNLUInterpreter

    agent = Agent.load(trained_rasa_model)
    assert isinstance(agent.interpreter, RasaNLUInterpreter)

def test_e2e_warning_if_no_nlu_model(
    monkeypatch: MonkeyPatch, trained_core_model: Text, capsys: CaptureFixture
):
    from rasa.model_testing import test_core

    # Patching is bit more complicated as we have a module `train` and function
    # with the same name 😬
    monkeypatch.setattr(
        sys.modules["rasa.core.test"], "test", asyncio.coroutine(lambda *_, **__: True)
    )

    test_core(trained_core_model, additional_arguments={"e2e": True})

    assert "No NLU model found. Using default" in capsys.readouterr().out

def test_write_classification_errors():
    evaluation = EvaluationStore(
        action_predictions=["utter_goodbye"],
        action_targets=["utter_greet"],
        intent_predictions=["goodbye"],
        intent_targets=["greet"],
        entity_predictions=None,
        entity_targets=None,
    )
    events = [
        WronglyClassifiedUserUtterance(
            UserUttered("Hello", {"name": "goodbye"}), evaluation
        ),
        WronglyPredictedAction("utter_greet", "", "utter_goodbye"),
    ]
    tracker = DialogueStateTracker.from_events("default", events)
    dump = YAMLStoryWriter().dumps(tracker.as_story().story_steps)
    assert (
        dump.strip()
        == textwrap.dedent(
            """
        version: "2.0"
        stories:
        - story: default
          steps:
          - intent: greet  # predicted: goodbye: Hello
          - action: utter_greet  # predicted: utter_goodbye

    """
        ).strip()
    )

def test_log_failed_stories(tmp_path: Path):
    path = str(tmp_path / "stories.yml")
    rasa.core.test._log_stories([], path, "Some text")

    dump = rasa.shared.utils.io.read_file(path)

    assert dump.startswith("#")
    assert len(dump.split("\n")) == 1

@pytest.mark.parametrize(
    "entity_predictions,entity_targets",
    [
        (
            [{"text": "hi, how are you", "start": 4, "end": 7, "entity": "aa"}],
            [
                {"text": "hi, how are you", "start": 0, "end": 2, "entity": "bb"},
                {"text": "hi, how are you", "start": 4, "end": 7, "entity": "aa"},
            ],
        ),
        (
            [
                {"text": "hi, how are you", "start": 0, "end": 2, "entity": "bb"},
                {"text": "hi, how are you", "start": 4, "end": 7, "entity": "aa"},
            ],
            [
                {"text": "hi, how are you", "start": 0, "end": 2, "entity": "bb"},
                {"text": "hi, how are you", "start": 4, "end": 7, "entity": "aa"},
            ],
        ),
        (
            [
                {"text": "hi, how are you", "start": 0, "end": 2, "entity": "bb"},
                {"text": "hi, how are you", "start": 4, "end": 7, "entity": "aa"},
            ],
            [{"text": "hi, how are you", "start": 4, "end": 7, "entity": "aa"}],
        ),
        (
            [
                {
                    "text": "Tanja is currently in Munich, but she lives in Berlin",
                    "start": 0,
                    "end": 5,
                    "entity": "person",
                },
                {
                    "text": "Tanja is currently in Munich, but she lives in Berlin",
                    "start": 22,
                    "end": 28,
                    "entity": "city",
                },
                {
                    "text": "Tanja is currently in Munich, but she lives in Berlin",
                    "start": 47,
                    "end": 53,
                    "entity": "city",
                },
            ],
            [
                {
                    "text": "Tanja is currently in Munich, but she lives in Berlin",
                    "start": 22,
                    "end": 28,
                    "entity": "city",
                }
            ],
        ),
        (
            [
                {
                    "text": "Tanja is currently in Munich, but she lives in Berlin",
                    "start": 0,
                    "end": 5,
                    "entity": "person",
                },
                {
                    "text": "Tanja is currently in Munich, but she lives in Berlin",
                    "start": 47,
                    "end": 53,
                    "entity": "city",
                },
            ],
            [
                {
                    "text": "Tanja is currently in Munich, but she lives in Berlin",
                    "start": 22,
                    "end": 28,
                    "entity": "city",
                },
                {
                    "text": "Tanja is currently in Munich, but she lives in Berlin",
                    "start": 47,
                    "end": 53,
                    "entity": "city",
                },
            ],
        ),
        (
            [
                {
                    "text": "Tanja is currently in Munich, but she lives in Berlin",
                    "start": 47,
                    "end": 53,
                    "entity": "city",
                }
            ],
            [
                {
                    "text": "Tanja is currently in Munich, but she lives in Berlin",
                    "start": 0,
                    "end": 5,
                    "entity": "person",
                },
                {
                    "text": "Tanja is currently in Munich, but she lives in Berlin",
                    "start": 22,
                    "end": 28,
                    "entity": "city",
                },
                {
                    "text": "Tanja is currently in Munich, but she lives in Berlin",
                    "start": 47,
                    "end": 53,
                    "entity": "city",
                },
            ],
        ),
    ],
)
def test_evaluation_store_serialise(entity_predictions, entity_targets):
    from rasa.shared.nlu.training_data.formats.readerwriter import TrainingDataWriter

    store = EvaluationStore(
        entity_predictions=entity_predictions, entity_targets=entity_targets
    )

    targets, predictions = store.serialise()

    assert len(targets) == len(predictions)

    i_pred = 0
    i_target = 0
    for i, prediction in enumerate(predictions):
        target = targets[i]
        if prediction != "None" and target != "None":
            predicted = entity_predictions[i_pred]
            assert prediction == TrainingDataWriter.generate_entity(
                predicted.get("text"), predicted
            )
            assert predicted.get("start") == entity_targets[i_target].get("start")
            assert predicted.get("end") == entity_targets[i_target].get("end")

        if prediction != "None":
            i_pred += 1
        if target != "None":
            i_target += 1

async def test_test_does_not_use_rules(tmp_path: Path, default_agent: Agent):
    from rasa.core.test import _create_data_generator

    test_file = tmp_path / "test.yml"
    test_name = "my test story"
    tests = f"""
stories:
- story: {test_name}
  steps:
  - intent: greet
  - action: utter_greet

rules:
- rule: rule which is ignored
  steps:
  - intent: greet
  - action: utter_greet
    """

    test_file.write_text(tests)

    generator = await _create_data_generator(str(test_file), default_agent)
    test_trackers = generator.generate_story_trackers()
    assert len(test_trackers) == 1
    assert test_trackers[0].sender_id == test_name

def test_duplicated_entity_predictions_tolerated():
    """Same entity extracted multiple times shouldn't be flagged as prediction error.

    This can happen when multiple entity extractors extract the same entity but a test
    story only lists the entity once. For completeness, the other case (entity listed
    twice in test story and extracted once) is also tested here because it should work
    the same way.
    """
    entity = {
        ENTITY_ATTRIBUTE_TEXT: "Algeria",
        ENTITY_ATTRIBUTE_START: 0,
        ENTITY_ATTRIBUTE_END: 7,
        ENTITY_ATTRIBUTE_VALUE: "Algeria",
        ENTITY_ATTRIBUTE_TYPE: "country",
    }
    evaluation_with_duplicated_prediction = EvaluationStore(
        entity_predictions=[entity, entity], entity_targets=[entity]
    )
    assert not evaluation_with_duplicated_prediction.check_prediction_target_mismatch()

    evaluation_with_duplicated_target = EvaluationStore(
        entity_predictions=[entity], entity_targets=[entity, entity]
    )
    assert not evaluation_with_duplicated_target.check_prediction_target_mismatch()

def test_differently_ordered_entity_predictions_tolerated():
    """The order in which entities were extracted shouldn't matter.

    Let's have an utterance like this: "[Researcher](job_name) from [Germany](country)."
    and imagine we use different entity extractors for the two entities. Then, the order
    in which entities are extracted from the utterance depends on the order in which the
    extractors are listed in the NLU pipeline. However, the expected order is given by
    where the entities are found in the utterance, i.e. "Researcher" comes before
    "Germany". Hence, it's reasonable for the expected and extracted order to not match
    and it shouldn't be flagged as a prediction error.

    """
    entity1 = {
        ENTITY_ATTRIBUTE_TEXT: "Algeria and Albania",
        ENTITY_ATTRIBUTE_START: 0,
        ENTITY_ATTRIBUTE_END: 7,
        ENTITY_ATTRIBUTE_VALUE: "Algeria",
        ENTITY_ATTRIBUTE_TYPE: "country",
    }
    entity2 = {
        ENTITY_ATTRIBUTE_TEXT: "Algeria and Albania",
        ENTITY_ATTRIBUTE_START: 12,
        ENTITY_ATTRIBUTE_END: 19,
        ENTITY_ATTRIBUTE_VALUE: "Albania",
        ENTITY_ATTRIBUTE_TYPE: "country",
    }
    evaluation = EvaluationStore(
        entity_predictions=[entity1, entity2], entity_targets=[entity2, entity1]
    )
    assert not evaluation.check_prediction_target_mismatch()
<|endoftext|>


<commit_before>import datetime

from django import forms

class FormLimitedDateTimeField(forms.DateTimeField):
    def __init__(self, upper_limit=None, lower_limit=None, input_formats=None, *args, **kwargs):
        self.upper_limit = upper_limit
        self.lower_limit = lower_limit

        if self.upper_limit and not isinstance(self.upper_limit, (datetime.datetime, datetime.date)):
            self.error(u'Invalid upper_limit argument.')
        if self.lower_limit and not isinstance(self.lower_limit, (datetime.datetime, datetime.date)):
            self.error(u'Invalid lower_limit argument.')

        super(FormLimitedDateTimeField, self).__init__(input_formats=None, *args, **kwargs)

    def clean(self, value):
        super(FormLimitedDateTimeField, self).clean(value)

        try:
            birthdate = datetime.datetime.strptime(value, '%Y-%m-%dT%H:%M:%S')
        except ValueError:
            birthdate = datetime.datetime.strptime(value, '%Y-%m-%d')

        if self.upper_limit:
            tmp_value = birthdate
            tmp_upper_limit = self.upper_limit
            if not isinstance(birthdate, datetime.datetime) or not isinstance(self.upper_limit, datetime.datetime):
                if isinstance(self.upper_limit, datetime.datetime):
                    tmp_upper_limit = self.upper_limit.date()
                elif isinstance(tmp_value, datetime.datetime):
                    tmp_value = birthdate.date()
            if tmp_value > tmp_upper_limit:
                self.error(u'Value is out of bounds.')

        if self.lower_limit:
            tmp_value = birthdate
            tmp_lower_limit = self.lower_limit
            if not isinstance(birthdate, datetime.datetime) or not isinstance(self.lower_limit, datetime.datetime):
                if isinstance(self.lower_limit, datetime.datetime):
                    tmp_lower_limit = self.lower_limit.date()
                elif isinstance(tmp_value, datetime.datetime):
                    tmp_value = birthdate.date()
            if tmp_value < tmp_lower_limit:
                self.error(u'Value is out of bounds.')<commit_msg>Fixed error reporting and clean() method.<commit_after>import datetime

from django import forms

from piplmesh.account import models

class LimitedDateTimeField(forms.DateTimeField):
    def __init__(self, upper_limit=None, lower_limit=None, input_formats=None, *args, **kwargs):
        self.upper_limit = upper_limit
        self.lower_limit = lower_limit

        if self.upper_limit and not isinstance(self.upper_limit, (datetime.datetime, datetime.date)):
            raise ValidationError(models.error_messages['syntax_upper'])
        if self.lower_limit and not isinstance(self.lower_limit, (datetime.datetime, datetime.date)):
            raise ValidationError(models.error_messages['syntax_lower'])

        super(FormLimitedDateTimeField, self).__init__(input_formats=None, *args, **kwargs)

    def clean(self, value):
        value = super(LimitedDateTimeField, self).clean(value)

        def error():
            raise ValidationError(models.error_messages['bounds'])

        models.limit_date(value, self.lower_limit, self.upper_limit, error)

        return value<|endoftext|>


<commit_before>"""
This config file tries to mimic the production environment more closely than the
normal dev.py. It assumes you're running a local instance of MySQL 5.1 and that
you're running memcached. You'll want to use this to test caching and database
migrations.

Assumptions:
* MySQL 5.1 (version important -- askbot breaks on 5.5)

Dir structure:
/envroot/
        /mitx # The location of this repo
        /log  # Where we're going to write log files

"""
from dev import *

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.mysql',
        'NAME': 'wwc',
        'USER': 'root',
        'PASSWORD': '',
        'HOST': '127.0.0.1',
        'PORT': '3306',
    }
}

CACHES = {
   'default': {
       'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
       'LOCATION': '127.0.0.1:11211',
   }
   'general': {
       'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
       'LOCATION': '127.0.0.1:11211',
       'KEY_PREFIX' : 'general',
       'VERSION' : 5,
   }
}

SESSION_ENGINE = 'django.contrib.sessions.backends.cache'
<commit_msg>fixed typo in devplus env<commit_after>"""
This config file tries to mimic the production environment more closely than the
normal dev.py. It assumes you're running a local instance of MySQL 5.1 and that
you're running memcached. You'll want to use this to test caching and database
migrations.

Assumptions:
* MySQL 5.1 (version important -- askbot breaks on 5.5)

Dir structure:
/envroot/
        /mitx # The location of this repo
        /log  # Where we're going to write log files

"""
from dev import *

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.mysql',
        'NAME': 'wwc',
        'USER': 'root',
        'PASSWORD': '',
        'HOST': '127.0.0.1',
        'PORT': '3306',
    }
}

CACHES = {
   'default': {
       'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
       'LOCATION': '127.0.0.1:11211',
   },
   'general': {
       'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
       'LOCATION': '127.0.0.1:11211',
       'KEY_PREFIX' : 'general',
       'VERSION' : 5,
   }
}

SESSION_ENGINE = 'django.contrib.sessions.backends.cache'
<|endoftext|>


<commit_before>"""Metadata"""
__version__ = '0.3.0'
__author__ = "Simon van Heeringen"
<commit_msg>version bump<commit_after>"""Metadata"""
__version__ = '0.3.1'
__author__ = "Simon van Heeringen"
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import re
import redis

import alias.config

email_re = re.compile(r'.*@.*\..*')

cfg = alias.config.AliasConfig()

user_db = redis.StrictRedis(host='localhost', port=6379, db=cfg.user_db)
email_db = redis.StrictRedis(host='localhost', port=6379, db=cfg.email_db)
nym_db = redis.StrictRedis(host='localhost', port=6379, db=cfg.nym_db)
loc_db = redis.StrictRedis(host='localhost', port=6379, db=cfg.loc_db)
url_db = redis.StrictRedis(host='localhost', port=6379, db=cfg.url_db)
name_db = redis.StrictRedis(host='localhost', port=6379, db=cfg.name_db)
desc_db = redis.StrictRedis(host='localhost', port=6379, db=cfg.desc_db)
image_db = redis.StrictRedis(host='localhost', port=6379, db=cfg.image_db)

def get_email_directory():
    return sorted(set([i[:2] for i in cfg.email_db.keys('*@*.*')]))

def get_emails_by_key(key):
    return sorted(cfg.email_db.keys(key + '*@*.*'))

def get_usernames_by_email(address):
    ids = cfg.email_db.lrange(address, 0, -1)
    return sorted(set([cfg.user_db.hget(i, 'key') for i in ids]))

def load_new_targets(targets):
    # Make sure the key_id is available before adding data.
    if cfg.user_db.get('key_id') is None:
        cfg.user_db.set('key_id', 1)

    # Add usernames and email addresses.
    count = 0
    for target in targets.split(','):
        target = target.lower().strip()
        if target == '':
            continue

        if email_re.search(target) is None:
            add_new_target(target, 'user')
        else:
            add_new_target(target, 'email')

        count += 1

    return count

def get_users_with_data(key=None):
    '''
    Return a list of all users with data associated with the username or email
    address.
    '''
    if key is None:
        keys = cfg.user_db.keys('*:data')
    else:
        keys = cfg.user_db.keys(key + '*:data')

    users = []
    for key in keys:
        u, j = key.split(':')
        user_id = cfg.user_db.get(u)
        user = cfg.user_db.hget(user_id, 'key')
        users.append(user)

    return users

def add_new_target(target, key_type):
    '''
    Adds a new target to the database.
    '''
    key_id = cfg.user_db.get('key_id')
    cfg.user_db.incr('key_id')

    data = {'key': target, 'type': key_type,
            'gravatar': None, 'twitter': None}

    cfg.user_db.hmset('id:' + key_id, data)
    cfg.user_db.set(target, 'id:' + key_id)

def get_user_data(username):
    user_id = cfg.user_db.get(username)
    data = {}

    data['emails'] = cfg.email_db.lrange(user_id, 0, -1)
    data['nyms'] = cfg.nym_db.lrange(user_id, 0, -1)
    data['urls'] = cfg.url_db.lrange(user_id, 0, -1)
    data['locs'] = cfg.loc_db.lrange(user_id, 0, -1)
    data['names'] = cfg.name_db.lrange(user_id, 0, -1)
    data['descs'] = cfg.desc_db.lrange(user_id, 0, -1)
    data['images'] = cfg.image_db.lrange(user_id, 0, -1)

    return data
<commit_msg>Moved db declaration to db.py file.<commit_after># -*- coding: utf-8 -*-

import re
import redis

import alias.config

email_re = re.compile(r'.*@.*\..*')

cfg = alias.config.AliasConfig()

user_db = redis.StrictRedis(host='localhost', port=6379, db=cfg.user_db)
email_db = redis.StrictRedis(host='localhost', port=6379, db=cfg.email_db)
nym_db = redis.StrictRedis(host='localhost', port=6379, db=cfg.nym_db)
loc_db = redis.StrictRedis(host='localhost', port=6379, db=cfg.loc_db)
url_db = redis.StrictRedis(host='localhost', port=6379, db=cfg.url_db)
name_db = redis.StrictRedis(host='localhost', port=6379, db=cfg.name_db)
desc_db = redis.StrictRedis(host='localhost', port=6379, db=cfg.desc_db)
image_db = redis.StrictRedis(host='localhost', port=6379, db=cfg.image_db)

def get_email_directory():
    return sorted(set([i[:2] for i in email_db.keys('*@*.*')]))

def get_emails_by_key(key):
    return sorted(email_db.keys(key + '*@*.*'))

def get_usernames_by_email(address):
    ids = email_db.lrange(address, 0, -1)
    return sorted(set([user_db.hget(i, 'key') for i in ids]))

def load_new_targets(targets):
    # Make sure the key_id is available before adding data.
    if user_db.get('key_id') is None:
        user_db.set('key_id', 1)

    # Add usernames and email addresses.
    count = 0
    for target in targets.split(','):
        target = target.lower().strip()
        if target == '':
            continue

        if email_re.search(target) is None:
            add_new_target(target, 'user')
        else:
            add_new_target(target, 'email')

        count += 1

    return count

def get_users_with_data(key=None):
    '''
    Return a list of all users with data associated with the username or email
    address.
    '''
    if key is None:
        keys = user_db.keys('*:data')
    else:
        keys = user_db.keys(key + '*:data')

    users = []
    for key in keys:
        u, j = key.split(':')
        user_id = user_db.get(u)
        user = user_db.hget(user_id, 'key')
        users.append(user)

    return users

def add_new_target(target, key_type):
    '''
    Adds a new target to the database.
    '''
    key_id = user_db.get('key_id')
    user_db.incr('key_id')

    data = {'key': target, 'type': key_type,
            'gravatar': None, 'twitter': None}

    user_db.hmset('id:' + key_id, data)
    user_db.set(target, 'id:' + key_id)

def get_user_data(username):
    user_id = user_db.get(username)
    data = {}

    data['emails'] = email_db.lrange(user_id, 0, -1)
    data['nyms'] = nym_db.lrange(user_id, 0, -1)
    data['urls'] = url_db.lrange(user_id, 0, -1)
    data['locs'] = loc_db.lrange(user_id, 0, -1)
    data['names'] = name_db.lrange(user_id, 0, -1)
    data['descs'] = desc_db.lrange(user_id, 0, -1)
    data['images'] = image_db.lrange(user_id, 0, -1)

    return data
<|endoftext|>


<commit_before>from requests import get
from json import loads, load

class Miner:
    def __init__(self, domain, stat):
        self.mine(domain, stat)

    @staticmethod
    def mine_revisions(commit, current_revision, domain):
        revisions = ['/files']

        for revision in revisions:
            url = domain + 'changes/' + str(commit) + '/revisions/' + str(current_revision) + revision
            request = get(url)

            if request.status_code == 200:
                response = loads(request.text[5:])
            else:
                print('Error')

    @staticmethod
    def mine_details(commit, domain):
        details = ['/detail?O=10004']

        for detail in details:
            url = domain + "changes/" + str(commit) + detail
            request = get(url)

            if request.status_code == 200:
                response = loads(request.text[5:])
            else:
                print("Error")

    def mine_review(self, commit, current_revision, domain):
        self.mine_details(commit, domain)
        self.mine_revisions(commit, current_revision, domain)

    @staticmethod
    def search_vccs_in_comments(base_url, keywords_json, commit, domain):
        # URL to comments of a commit
        url = base_url + "/comments"

        # Request the JSON file of this URL
        request = get(url)

        # If request was successful, look for keywords in comments
        if request.status_code == 200:
            response = loads(request.text[5:])
            keys = response.keys()

            for key in keys:
                for comment in response[key]:
                    for keywords in keywords_json['keywords']:
                        if keywords['keyword'] in comment['message']:
                            if keywords['depends_on']:
                                for dependency in keywords['depends_on']:
                                    if dependency in comment['message']:
                                        print("  Found keyword: " + "\033[1m" + keywords['keyword'] + "\033[0m" +
                                              " and the dependency: " + "\033[1m" + dependency + "\033[0m" +
                                              " in comment\n" + "    Domain: " + domain + "\n" + "    Commit: " +
                                              str(commit) + "\n" + "    File: " + key + "\n" + "    Comment ID: " +
                                              comment['id'])
                            else:
                                print("  Found keyword: " + "\033[1m" + keywords['keyword'] + "\033[0m" +
                                      " in comment\n" + "    Domain: " + domain + "\n" + "    Commit: " +
                                      str(commit) + "\n" + "    File: " + key + "\n" + "    Comment ID: " +
                                      comment['id'])

    @staticmethod
    def search_vccs_in_messages(base_url, keywords_json, commit, domain):
        # URL to details of a commit
        url = base_url + "/detail?O=10004"

        # Request the JSON file of this URL
        request = get(url)

        # If request was successful, look for keywords in messages
        if request.status_code == 200:
            response = loads(request.text[5:])

            for message in response['messages']:
                for keywords in keywords_json['keywords']:
                    if keywords['keyword'] in message['message']:
                        if keywords['depends_on']:
                            for dependency in keywords['depends_on']:
                                if dependency in message['message']:
                                    subject = response['subject']

                                    print("  Found keyword: " + "\033[1m" + keywords['keyword'] + "\033[0m" +
                                          " in message\n" + "    Domain: " + domain + "\n" + "    Commit: " +
                                          str(commit) + "\n" + "    Subject: " + subject + "\n" + "    Message ID: " +
                                          message['id'])
        else:
            print("Error")

    def search_vccs(self, commit, domain):
        # Base URL for requests
        base_url = domain + "changes/" + str(commit)

        with open('data/keywords.json') as keywords_file:
            keywords_json = load(keywords_file)

        self.search_vccs_in_messages(base_url, keywords_json, commit, domain)
        self.search_vccs_in_comments(base_url, keywords_json, commit, domain)

    def mine(self, domain, status):
        k = 0

        while k <= 9999:
            url = domain + "changes/?q=status:" + status + "&n=100&O=81&S=" + str(k)
            request = get(url)

            if request.status_code == 200:
                reviews = loads(request.text[5:])

                for review in reviews:
                    self.search_vccs(review['_number'], domain)

                print("")

            k += 100
<commit_msg>Gerrit-miner 0.2.1<commit_after>from requests import get
from json import loads, load

class Miner:
    def __init__(self, domain, stat):
        self.mine(domain, stat)

    @staticmethod
    def mine_revisions(commit, current_revision, domain):
        revisions = ['/files']

        for revision in revisions:
            url = domain + 'changes/' + str(commit) + '/revisions/' + str(current_revision) + revision
            request = get(url)

            if request.status_code == 200:
                response = loads(request.text[5:])
            else:
                print('Error')

    @staticmethod
    def mine_details(commit, domain):
        details = ['/detail?O=10004']

        for detail in details:
            url = domain + "changes/" + str(commit) + detail
            request = get(url)

            if request.status_code == 200:
                response = loads(request.text[5:])
            else:
                print("Error")

    def mine_review(self, commit, current_revision, domain):
        self.mine_details(commit, domain)
        self.mine_revisions(commit, current_revision, domain)

    @staticmethod
    def search_vccs_in_comments(base_url, keywords_json, commit, domain):
        # URL to comments of a commit
        url = base_url + "/comments"

        # Request the JSON file of this URL
        request = get(url)

        # If request was successful, look for keywords in comments
        if request.status_code == 200:
            response = loads(request.text[5:])
            keys = response.keys()

            for key in keys:
                for comment in response[key]:
                    for keywords in keywords_json['keywords']:
                        if keywords['keyword'] in comment['message']:
                            if keywords['depends_on']:
                                for dependency in keywords['depends_on']:
                                    if dependency in comment['message']:
                                        print("  Found keyword: " + "\033[1m" + keywords['keyword'] + "\033[0m" +
                                              " and the dependency: " + "\033[1m" + dependency + "\033[0m" +
                                              " in comment\n" + "    Vulnerability: " + "\033[1m" +
                                              str(keywords['vulnerabilities']) + "\n" + "\033[0m" + "    Domain: " +
                                              domain + "\n" + "    Commit: " + str(commit) + "\n" + "    File: " + key +
                                              "\n" + "    Comment ID: " + comment['id'])
                            else:
                                print("  Found keyword: " + "\033[1m" + keywords['keyword'] + "\033[0m" +
                                      " in comment\n" + "    Vulnerability: " + "\033[1m" +
                                      str(keywords['vulnerabilities']) + "\n" + "\033[0m" + "    Domain: " + domain +
                                      "\n" + "    Commit: " + str(commit) + "\n" + "    File: " + key + "\n" +
                                      "    Comment ID: " + comment['id'])

    @staticmethod
    def search_vccs_in_messages(base_url, keywords_json, commit, domain):
        # URL to details of a commit
        url = base_url + "/detail?O=10004"

        # Request the JSON file of this URL
        request = get(url)

        # If request was successful, look for keywords in messages
        if request.status_code == 200:
            response = loads(request.text[5:])

            for message in response['messages']:
                for keywords in keywords_json['keywords']:
                    if keywords['keyword'] in message['message']:
                        if keywords['depends_on']:
                            for dependency in keywords['depends_on']:
                                if dependency in message['message']:
                                    subject = response['subject']

                                    print("  Found keyword: " + "\033[1m" + keywords['keyword'] + "\033[0m" +
                                          " in message\n" + "    Vulnerability: " + "\033[1m" +
                                          str(keywords['vulnerabilities']) + "\033[0m" + "\n" + "    Domain: " + domain + "\n" +
                                          "    Commit: " + str(commit) + "\n" + "    Subject: " + subject + "\n" +
                                          "    Message ID: " + message['id'])
        else:
            print("Error")

    def search_vccs(self, commit, domain):
        # Base URL for requests
        base_url = domain + "changes/" + str(commit)

        with open('data/keywords.json') as keywords_file:
            keywords_json = load(keywords_file)

        self.search_vccs_in_messages(base_url, keywords_json, commit, domain)
        self.search_vccs_in_comments(base_url, keywords_json, commit, domain)

    def mine(self, domain, status):
        i = 0

        while i <= 9999:
            url = domain + "changes/?q=status:" + status + "&n=100&O=81&S=" + str(i)
            request = get(url)

            if request.status_code == 200:
                reviews = loads(request.text[5:])

                for review in reviews:
                    self.search_vccs(review['_number'], domain)

                print("")

            i += 100
<|endoftext|>


<commit_before># This code is heavily based on contrib.admin's autodiscover mechanism.

import logging
import sys

__name__ = 'appsearch'
__author__ = 'Pivotal Energy Solutions'
__version_info__ = (1, 0, '0rc2')
__version__ = '.'.join(map(str, __version_info__))
__date__ = '2014/07/22 4:47:00 PM'
__credits__ = ['Tim Valenta', "Steven Klass"]
__license__ = 'See the file LICENSE.txt for licensing information.'

log = logging.getLogger(__name__)

def autodiscover():
    """
    Auto-discover INSTALLED_APPS search.py modules and fail silently when
    not present. This forces an import on them to register any appsearch bits
    they may want.
    """

    import copy
    from importlib import import_module

    from django.conf import settings
    from django.utils.module_loading import module_has_submodule
    from appsearch.registry import search

    for app in settings.INSTALLED_APPS:
        mod = import_module(app)
        # Attempt to import the app's appsearch module.
        try:
            before_import_registry = copy.copy(search._registry)
            import_module('%s.search' % app)
        except:
            # Reset the model registry to the state before the last import as
            # this import will have to reoccur on the next request and this
            # could raise NotRegistered and AlreadyRegistered exceptions
            # (see #8245).
            search._registry = before_import_registry

            # Decide whether to bubble up this error. If the app just
            # doesn't have an admin module, we can ignore the error
            # attempting to import it, otherwise we want it to bubble up.
            if module_has_submodule(mod, 'search'):
                exc_type, e, traceback = sys.exc_info()
                raise exc_type, e.message, traceback
<commit_msg>Release<commit_after># This code is heavily based on contrib.admin's autodiscover mechanism.

import logging
import sys

__name__ = 'appsearch'
__author__ = 'Pivotal Energy Solutions'
__version_info__ = (1, 0, 0)
__version__ = '.'.join(map(str, __version_info__))
__date__ = '2014/07/22 4:47:00 PM'
__credits__ = ['Tim Valenta', "Steven Klass"]
__license__ = 'See the file LICENSE.txt for licensing information.'

log = logging.getLogger(__name__)

def autodiscover():
    """
    Auto-discover INSTALLED_APPS search.py modules and fail silently when
    not present. This forces an import on them to register any appsearch bits
    they may want.
    """

    import copy
    from importlib import import_module

    from django.conf import settings
    from django.utils.module_loading import module_has_submodule
    from appsearch.registry import search

    for app in settings.INSTALLED_APPS:
        mod = import_module(app)
        # Attempt to import the app's appsearch module.
        try:
            before_import_registry = copy.copy(search._registry)
            import_module('%s.search' % app)
        except:
            # Reset the model registry to the state before the last import as
            # this import will have to reoccur on the next request and this
            # could raise NotRegistered and AlreadyRegistered exceptions
            # (see #8245).
            search._registry = before_import_registry

            # Decide whether to bubble up this error. If the app just
            # doesn't have an admin module, we can ignore the error
            # attempting to import it, otherwise we want it to bubble up.
            if module_has_submodule(mod, 'search'):
                exc_type, e, traceback = sys.exc_info()
                raise exc_type, e.message, traceback
<|endoftext|>


<commit_before>from raco import RACompiler
from raco.language import MyriaAlgebra
from raco.myrialang import compile_to_json
from raco.viz import plan_to_dot
from google.appengine.ext.webapp import template
import os.path

import webapp2

defaultquery = """A(x,z) :- R(x,y),S(y,z),T(z,x)"""

def programplan(query,target):
    dlog = RACompiler()

    dlog.fromDatalog(query)
    return dlog.logicalplan

def format_rule(expressions):
    return "\n".join(["%s = %s" % e for e in expressions])

class MainPage(webapp2.RequestHandler):
    def get(self,query=defaultquery):

        dlog = RACompiler()
        dlog.fromDatalog(query)
        plan = format_rule(dlog.logicalplan)
        dlog.optimize(target=MyriaAlgebra, eliminate_common_subexpressions=False)
        myria_plan = format_rule(dlog.physicalplan)

        self.response.headers['Content-Type'] = 'text/html'

        path = os.path.join(os.path.dirname(__file__), 'templates/editor.html')

        self.response.out.write(template.render(path, locals()))

class Plan(webapp2.RequestHandler):
    def get(self):
        query = self.request.get("query")
        dlog = RACompiler()
        dlog.fromDatalog(query)
        plan = format_rule(dlog.logicalplan)

        self.response.headers['Content-Type'] = 'text/plain'
        self.response.write(plan)

class Optimize(webapp2.RequestHandler):
    def get(self):
        query = self.request.get("query")

        dlog = RACompiler()
        dlog.fromDatalog(query)

        dlog.optimize(target=MyriaAlgebra, eliminate_common_subexpressions=False)

        optimized = format_rule(dlog.physicalplan)

        self.response.headers['Content-Type'] = 'text/plain'
        self.response.write(optimized)

class Compile(webapp2.RequestHandler):
    def get(self):
        query = self.request.get("query")

        dlog = RACompiler()
        dlog.fromDatalog(query)
        # Cache logical plan
        cached_logicalplan = str(dlog.logicalplan)

        # Generate physical plan
        dlog.optimize(target=MyriaAlgebra, eliminate_common_subexpressions=False)

        compiled = compile_to_json(query, cached_logicalplan, dlog.physicalplan)

        self.response.headers['Content-Type'] = 'application/json'
        self.response.write(compiled)

class Dot(webapp2.RequestHandler):
    def get(self):
        query = self.request.get("query")
        svg_type = self.request.get("type")

        dlog = RACompiler()
        dlog.fromDatalog(query)

        if svg_type is None or len(svg_type) == 0 or svg_type.lower() == "ra":
            plan = dlog.logicalplan
        elif svg_type.lower() == "myria":
            dlog.optimize(target=MyriaAlgebra, eliminate_common_subexpressions=False)
            plan = dlog.physicalplan
        else:
            self.abort(400, detail="argument type expected 'ra' or 'myria'")

        self.response.headers['Content-Type'] = 'text/plain'
        self.response.write(plan_to_dot(plan))

app = webapp2.WSGIApplication([
   ('/', MainPage),
   ('/plan',Plan),
   ('/optimize',Optimize),
   ('/compile',Compile),
   ('/dot',Dot)
  ],
  debug=True
)

"""
TODO:
Debug conditions: A(x,z) :- R(x,p1,y),R(y,p2,z),R(z,p3,w)
Multiple rules
Recursion
Show graph visually
Protobuf
Show parse errors (with link to error)
"""
<commit_msg>appengine: manually ram the stack trace into the web response<commit_after>from raco import RACompiler
from raco.language import MyriaAlgebra
from raco.myrialang import compile_to_json
from raco.viz import plan_to_dot
from google.appengine.ext.webapp import template
import os.path
import traceback
import webapp2

defaultquery = """A(x,z) :- R(x,y),S(y,z),T(z,x)"""

def programplan(query,target):
    dlog = RACompiler()

    dlog.fromDatalog(query)
    return dlog.logicalplan

def format_rule(expressions):
    return "\n".join(["%s = %s" % e for e in expressions])

class RequestHandlerBase(webapp2.RequestHandler):
    def handle_exception(self, exception, debug_mode):
        if isinstance(exception, webapp2.HTTPException):
            self.response.set_status(exception.code)
            self.response.out.write(str(exception))
        else:
            self.response.set_status(500)
            self.response.out.write(traceback.format_exc(exception))
        self.response.headers['Content-Type'] = 'text/plain'

class MainPage(RequestHandlerBase):
    def get(self,query=defaultquery):

        dlog = RACompiler()
        dlog.fromDatalog(query)
        plan = format_rule(dlog.logicalplan)
        dlog.optimize(target=MyriaAlgebra, eliminate_common_subexpressions=False)
        myria_plan = format_rule(dlog.physicalplan)

        self.response.headers['Content-Type'] = 'text/html'

        path = os.path.join(os.path.dirname(__file__), 'templates/editor.html')

        self.response.out.write(template.render(path, locals()))

class Plan(RequestHandlerBase):
    def get(self):
        query = self.request.get("query")
        dlog = RACompiler()
        dlog.fromDatalog(query)
        plan = format_rule(dlog.logicalplan)

        self.response.headers['Content-Type'] = 'text/plain'
        self.response.write(plan)

class Optimize(RequestHandlerBase):
    def get(self):
        query = self.request.get("query")

        dlog = RACompiler()
        dlog.fromDatalog(query)

        dlog.optimize(target=MyriaAlgebra, eliminate_common_subexpressions=False)

        optimized = format_rule(dlog.physicalplan)

        self.response.headers['Content-Type'] = 'text/plain'
        self.response.write(optimized)

class Compile(RequestHandlerBase):
    def get(self):
        query = self.request.get("query")

        dlog = RACompiler()
        dlog.fromDatalog(query)
        # Cache logical plan
        cached_logicalplan = str(dlog.logicalplan)

        # Generate physical plan
        dlog.optimize(target=MyriaAlgebra, eliminate_common_subexpressions=False)

        compiled = compile_to_json(query, cached_logicalplan, dlog.physicalplan)

        self.response.headers['Content-Type'] = 'application/json'
        self.response.write(compiled)

class Dot(RequestHandlerBase):
    def get(self):
        query = self.request.get("query")
        svg_type = self.request.get("type")

        dlog = RACompiler()
        dlog.fromDatalog(query)

        if svg_type is None or len(svg_type) == 0 or svg_type.lower() == "ra":
            plan = dlog.logicalplan
        elif svg_type.lower() == "myria":
            dlog.optimize(target=MyriaAlgebra, eliminate_common_subexpressions=False)
            plan = dlog.physicalplan
        else:
            self.abort(400, detail="argument type expected 'ra' or 'myria'")

        self.response.headers['Content-Type'] = 'text/plain'
        self.response.write(plan_to_dot(plan))

app = webapp2.WSGIApplication([
   ('/', MainPage),
   ('/plan',Plan),
   ('/optimize',Optimize),
   ('/compile',Compile),
   ('/dot',Dot)
  ],
  debug=True
)

"""
TODO:
Debug conditions: A(x,z) :- R(x,p1,y),R(y,p2,z),R(z,p3,w)
Multiple rules
Recursion
Show graph visually
Protobuf
Show parse errors (with link to error)
"""
<|endoftext|>