<commit_before>import pyproctor
from moto import mock_s3
from pyshelf.app import app
import pyshelf.configure as configure
import boto
from boto.s3.key import Key
import yaml
import tests.metadata_utils as meta_utils
import tests.permission_utils as utils
from tests.route_tester.tester import Tester

class FunctionalTestBase(pyproctor.TestBase):
    RESPONSE_404 = {
        "message": "Resource not found",
        "code": "resource_not_found"
    }

    RESPONSE_403 = {
        "code": "forbidden",
        "message": "Forbidden"
    }

    RESPONSE_INVALID_NAME = {
        "message": "The artifact name provided is not allowable. Please remove leading underscores.",
        "code": "invalid_artifact_name"
    }

    RESPONSE_DUPLICATE = {
        "code": "duplicate_artifact",
        "message": "Artifact by name test already exists in current directory"
    }

    def setUp(self):
        self.app = app
        self.configure_moto()
        self.test_client = app.test_client()
        self._route_tester = None

    @classmethod
    def setUpClass(cls):
        config = {
            "buckets": {
                "test": {
                    "accessKey": "test",
                    "secretKey": "test"
                },
                "bucket2": {
                    "accessKey": "test",
                    "secretKey": "test"
                }
            },
            "elasticSearchHost": "localhost:9200"
        }
        configure.logger(app.logger, "DEBUG")
        app.config.update(config)

    def configure_moto(self):
        self.moto_s3 = mock_s3()
        self.moto_s3.start()
        self.boto_connection = boto.connect_s3()
        self.boto_connection.create_bucket("test")
        self.boto_connection.create_bucket("bucket2")
        self.test_bucket = self.boto_connection.get_bucket("test")
        self.configure_artifacts()
        self.create_auth_key()

    def configure_artifacts(self):
        key = Key(self.test_bucket, "test")
        key.set_contents_from_string("hello world")
        nested_key = Key(self.test_bucket, "/dir/dir2/dir3/nest-test")
        nested_key.set_contents_from_string("hello world")
        # Metadata for artifacts
        meta_key = Key(self.test_bucket, "_metadata_test.yaml")
        meta_key.set_contents_from_string(yaml.dump(meta_utils.get_meta()))
        nest_meta_key = Key(self.test_bucket, "/dir/dir2/dir3/_metadata_nest-test.yaml")
        nest_meta_key.set_contents_from_string("")
        artifact_list = Key(self.test_bucket, "/dir/dir2/dir3/dir4/test5")
        artifact_list.set_contents_from_string("")

    def create_auth_key(self):
        self.auth = utils.auth_header()
        key_name = "_keys/{}".format(self.auth["Authorization"])
        auth_key = Key(self.test_bucket, key_name)
        auth_key.set_contents_from_string(utils.get_permissions_func_test())
        auth_bucket2 = Key(self.boto_connection.get_bucket("bucket2"), key_name)
        auth_bucket2.set_contents_from_string(utils.get_permissions_func_test())

    @property
    def route_tester(self):
        if not self._route_tester:
            self._route_tester = Tester(self, self.test_client)

        return self._route_tester

    def tearDown(self):
        self.moto_s3.stop()

    def response_500(self, message=None):
        if not message:
            message = "Internal server error"

        return {
            "message": message,
            "code": "internal_server_error"
        }
<commit_msg>Add setup and teardown for elasticsearch func tests<commit_after>import pyproctor
from moto import mock_s3
from pyshelf.app import app
import pyshelf.configure as configure
import boto
from boto.s3.key import Key
import yaml
import tests.metadata_utils as meta_utils
import tests.permission_utils as utils
from tests.route_tester.tester import Tester
from elasticsearch_dsl import Index
from elasticsearch import Elasticsearch
from pyshelf.search.metadata import Metadata

class FunctionalTestBase(pyproctor.TestBase):
    RESPONSE_404 = {
        "message": "Resource not found",
        "code": "resource_not_found"
    }

    RESPONSE_403 = {
        "code": "forbidden",
        "message": "Forbidden"
    }

    RESPONSE_INVALID_NAME = {
        "message": "The artifact name provided is not allowable. Please remove leading underscores.",
        "code": "invalid_artifact_name"
    }

    RESPONSE_DUPLICATE = {
        "code": "duplicate_artifact",
        "message": "Artifact by name test already exists in current directory"
    }

    def setUp(self):
        self.app = app
        self.configure_moto()
        self.test_client = app.test_client()
        self._route_tester = None

    @classmethod
    def setUpClass(cls):
        config = {
            "buckets": {
                "test": {
                    "accessKey": "test",
                    "secretKey": "test"
                },
                "bucket2": {
                    "accessKey": "test",
                    "secretKey": "test"
                }
            },
            "elasticSearchHost": "localhost:9200"
        }
        configure.logger(app.logger, "DEBUG")
        app.config.update(config)
        es = Elasticsearch(config.get("elasticSearchHost"))
        Metadata.init(using=es)
        Metadata._doc_type.refresh(using=es)

    @classmethod
    def tearDownClass(cls):
        es = Elasticsearch(app.config.get("elasticSearchHost"))
        meta_index = Index("metadata", using=es)
        meta_index.delete()

    def configure_moto(self):
        self.moto_s3 = mock_s3()
        self.moto_s3.start()
        self.boto_connection = boto.connect_s3()
        self.boto_connection.create_bucket("test")
        self.boto_connection.create_bucket("bucket2")
        self.test_bucket = self.boto_connection.get_bucket("test")
        self.configure_artifacts()
        self.create_auth_key()

    def configure_artifacts(self):
        key = Key(self.test_bucket, "test")
        key.set_contents_from_string("hello world")
        nested_key = Key(self.test_bucket, "/dir/dir2/dir3/nest-test")
        nested_key.set_contents_from_string("hello world")
        # Metadata for artifacts
        meta_key = Key(self.test_bucket, "_metadata_test.yaml")
        meta_key.set_contents_from_string(yaml.dump(meta_utils.get_meta()))
        nest_meta_key = Key(self.test_bucket, "/dir/dir2/dir3/_metadata_nest-test.yaml")
        nest_meta_key.set_contents_from_string("")
        artifact_list = Key(self.test_bucket, "/dir/dir2/dir3/dir4/test5")
        artifact_list.set_contents_from_string("")

    def create_auth_key(self):
        self.auth = utils.auth_header()
        key_name = "_keys/{}".format(self.auth["Authorization"])
        auth_key = Key(self.test_bucket, key_name)
        auth_key.set_contents_from_string(utils.get_permissions_func_test())
        auth_bucket2 = Key(self.boto_connection.get_bucket("bucket2"), key_name)
        auth_bucket2.set_contents_from_string(utils.get_permissions_func_test())

    @property
    def route_tester(self):
        if not self._route_tester:
            self._route_tester = Tester(self, self.test_client)

        return self._route_tester

    def tearDown(self):
        self.moto_s3.stop()

    def response_500(self, message=None):
        if not message:
            message = "Internal server error"

        return {
            "message": message,
            "code": "internal_server_error"
        }
<|endoftext|>


<commit_before># Copyright 2020 Google LLC. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
from typing import Any, Dict, List, Optional, Tuple

from ast_parser.core import constants
from ast_parser.lib import file_utils

import yaml

def __attr_validate_required_values(
    yaml_path: str,
    yaml_entry: Dict[str, Any],
    tag: str,
    attr: str
) -> Optional[str]:
    """Ensure required-value YAML keys have the correct values

    Some .drift-data.yml keys only allow a predefined set of values.

    Arguments:
        yaml_path: A path to a .drift-data.yml file
        yaml_entry: The YAML entry to validate
        tag: The region tag corresponding to the specified YAML entry
        attr: The attribute of the YAML entry to validate

    Returns:
        An error message if an attribute is invalid; None otherwise
    """
    if not attr or attr not in constants.REQUIRED_KEY_VALUES:
        return None

    actual = yaml_entry[attr]
    expected = constants.REQUIRED_KEY_VALUES[attr]
    if actual != expected:
        return (
            f'Invalid {attr} value in file {yaml_path} '
            f'for tag {tag}: {actual}, expected {expected} '
            ' (or omission)')

    return None

def __attr_validate_additions(
    yaml_path: str,
    yaml_entry: Dict[str, Any],
    tag: str,
    attr: str,
    grep_tags: List[str]
) -> Optional[str]:
    """Ensure additions attributes have correct values

    This function ensures that additions attributes are lists, and that
    the child tags they refer to are detected in the codebase.

    Arguments:
        yaml_path: A path to a .drift-data.yml file
        yaml_entry: The YAML entry to validate
        tag: The region tag corresponding to the specified YAML entry
        attr: The attribute of the YAML entry to validate
        grep_tags: A list of tags existing (not necessarily parsed out of)
                   the source code

    Returns:
        An error message if the additions attribute is invalid; None otherwise
    """
    if attr != 'additions':
        return None

    # Additions field must be an array
    if not isinstance(yaml_entry[attr], list):
        return (f'Additions key for {tag} in '
                f'{yaml_path} is not a list!')

    # Added tags must be correctly parsed from the codebase
    if any(t not in grep_tags for t in yaml_entry[attr]):
        return (f'Yaml file {yaml_path} contains region '
                f'tag not used in source files: {tag}')

    return None

def __attr_validate_manually_specified_tests(
    yaml_path: str,
    yaml_entry: Dict[str, Any],
    tag: str,
    attr: str,
    grep_tags: List[str]
) -> List[str]:
    """Ensure manually-specified tests are correct

    This function ensures that manually-specified
    tests refer to files that actually exist.

    Arguments:
        yaml_path: A path to a .drift-data.yml file
        yaml_entry: The YAML entry to validate
        tag: The region tag corresponding to the specified YAML entry
        attr: The attribute of the YAML entry to validate
        grep_tags: A list of tags existing (not necessarily parsed out of)
                   the source code

    Returns:
        An error message if the manually-specified tests are invalid; None
        otherwise
    """
    errors = []
    yaml_dirname = os.path.dirname(yaml_path)

    for test_path in yaml_entry.keys():
        if test_path in constants.RESERVED_YAML_KEYS:
            continue  # Skip non-filepaths

        if not os.path.isabs(test_path):
            test_path = os.path.join(yaml_dirname, test_path)

        if not os.path.exists(test_path):
            errors.append(f'Test file {test_path} used '
                          f'in {yaml_path} not found!')

    return errors

def _validate_attrs(
    yaml_paths: List[str],
    grep_tags: List[str]
) -> Tuple[bool, List[str]]:
    """Validate attributes in a list of .drift-data.yml files

    This method verifies that attributes within
    a .drift-data.yml file are used appropriately.

    Args:
        yaml_paths: A list of .drift-data.yml filepaths to validate
        grep_tags: A list of region tags (parsed *and* unparsed) that exist
                   in the target directory

    Returns:
        A 2-tuple containing the following:
         - A boolean including file validity (True if all files passed
           attribute validation, False otherwise)
         - A list of validation error messages (if any) that were raised
    """

    errors = []

    for yaml_path in yaml_paths:
        with open(yaml_path, 'r') as file:
            yaml_contents = '\n'.join(file.readlines())
            parsed_yaml = yaml.safe_load(yaml_contents)

            for tag in parsed_yaml.keys():
                yaml_entry = parsed_yaml[tag]
                yaml_keys = list(yaml_entry.keys())

                if not yaml_keys:
                    continue

                yaml_attr = yaml_keys[0]

                # Validate keys that require specific values
                required_value_error = __attr_validate_required_values(
                    yaml_path,
                    yaml_entry,
                    tag,
                    yaml_attr
                )
                if required_value_error:
                    errors.append(required_value_error)

                # Validate additions field
                additions_error = __attr_validate_additions(
                    yaml_path,
                    yaml_entry,
                    tag,
                    yaml_attr,
                    grep_tags
                )
                if additions_error:
                    errors.append(additions_error)

                # Validate manually-specified tests
                manual_errors = __attr_validate_manually_specified_tests(
                    yaml_path,
                    yaml_entry,
                    tag,
                    yaml_attr,
                    grep_tags
                )
                if manual_errors:
                    errors += manual_errors

    is_valid = not errors
    return (is_valid, errors)

def _validate_region_tags(
    yaml_paths: List[str],
    grep_tags: List[str],
    source_tags: List[str],
) -> Tuple[bool, List[str]]:
    """Validate region-tag keys in a list of .drift-data.yml files

    This method verifies that region tag keys within a .drift-data.yml
    file are used *exactly* once in the source code.

    Args:
        yaml_paths: A list of .drift-data.yml filepaths to validate
        grep_tags: A list of region tags (parsed *and* unparsed) that exist
                   in the target directory
        source_tags: A list of successfully-parsed region tags in the target
                     directory

    Returns:
        A 2-tuple containing the following:
         - A boolean including file validity (True if all files passed region
           tag validation, False otherwise)
         - A list of validation error messages (if any) that were raised
    """

    seen_region_tags = set()
    output = []
    is_valid = True

    for yaml_path in yaml_paths:
        with open(yaml_path, 'r') as file:
            yaml_contents = '\n'.join(file.readlines())
            parsed_yaml = yaml.safe_load(yaml_contents)

            for tag in parsed_yaml.keys():
                yaml_entry = parsed_yaml[tag]
                tag_should_be_in_source = not (
                    'tested' in yaml_entry and yaml_entry['tested'] is False)

                # Verify mentioned region tags are used in the
                # source code (via parsing and/or grep results)
                if tag not in grep_tags:
                    output.append(
                        f'Yaml file {yaml_path} contains region '
                        f'tag not used in source files: {tag}')
                    is_valid = False
                elif tag_should_be_in_source and tag not in source_tags:
                    output.append(
                        f'Yaml file {yaml_path} contains '
                        f'unparsed region tag: {tag}')
                    output.append(
                        '  Remove it, or label it with "tested: false".')
                    is_valid = False
                elif not tag_should_be_in_source and tag in source_tags:
                    output.append(f'Parsed tag {tag} in file'
                                  f'{yaml_path} marked untested!')
                    is_valid = False

                # Verify region tags are present at most once
                if tag in seen_region_tags:
                    output.append(f'Region tag {tag} is used multiple '
                                  'times in .drift-data.yml files!')
                    is_valid = False
                else:
                    seen_region_tags.add(tag)

    return (is_valid, output)

def validate_yaml_syntax(
    root_dir: str,
    grep_tags: List[str],
    source_tags: List[str]
) -> Tuple[bool, List[str]]:
    """Validate a given directory's .drift-data.yml files

    This method validates that a given directory's .drift-data.yml files are
    semantically correct, and match up with the directory's source code.

    Args:
        yaml_paths: A list of .drift-data.yml filepaths to validate
        grep_tags: A list of region tags (parsed *and* unparsed) that exist
                   in the target directory
        source_tags: A list of successfully-parsed region tags in the target
                     directory

    Returns:
        A 2-tuple containing the following:
         - A boolean including file validity (True if all .drift-data.yml
           files passed validation, False otherwise)
         - A list of validation error messages (if any) that were raised
    """
    yaml_paths = file_utils.get_drift_yaml_files(root_dir)

    (tags_are_valid, tags_output) = (
        _validate_region_tags(yaml_paths, grep_tags, source_tags))
    (attrs_are_valid, attrs_output) = _validate_attrs(yaml_paths, grep_tags)

    output = tags_output + attrs_output
    is_valid = tags_are_valid and attrs_are_valid

    return (is_valid, output)
<commit_msg>Single underscore for private methods<commit_after># Copyright 2020 Google LLC. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
from typing import Any, Dict, List, Optional, Tuple

from ast_parser.core import constants
from ast_parser.lib import file_utils

import yaml

def _attr_validate_required_values(
    yaml_path: str,
    yaml_entry: Dict[str, Any],
    tag: str,
    attr: str
) -> Optional[str]:
    """Ensure required-value YAML keys have the correct values

    Some .drift-data.yml keys only allow a predefined set of values.

    Arguments:
        yaml_path: A path to a .drift-data.yml file
        yaml_entry: The YAML entry to validate
        tag: The region tag corresponding to the specified YAML entry
        attr: The attribute of the YAML entry to validate

    Returns:
        An error message if an attribute is invalid; None otherwise
    """
    if not attr or attr not in constants.REQUIRED_KEY_VALUES:
        return None

    actual = yaml_entry[attr]
    expected = constants.REQUIRED_KEY_VALUES[attr]
    if actual != expected:
        return (
            f'Invalid {attr} value in file {yaml_path} '
            f'for tag {tag}: {actual}, expected {expected} '
            ' (or omission)')

    return None

def _attr_validate_additions(
    yaml_path: str,
    yaml_entry: Dict[str, Any],
    tag: str,
    attr: str,
    grep_tags: List[str]
) -> Optional[str]:
    """Ensure additions attributes have correct values

    This function ensures that additions attributes are lists, and that
    the child tags they refer to are detected in the codebase.

    Arguments:
        yaml_path: A path to a .drift-data.yml file
        yaml_entry: The YAML entry to validate
        tag: The region tag corresponding to the specified YAML entry
        attr: The attribute of the YAML entry to validate
        grep_tags: A list of tags existing (not necessarily parsed out of)
                   the source code

    Returns:
        An error message if the additions attribute is invalid; None otherwise
    """
    if attr != 'additions':
        return None

    # Additions field must be an array
    if not isinstance(yaml_entry[attr], list):
        return (f'Additions key for {tag} in '
                f'{yaml_path} is not a list!')

    # Added tags must be correctly parsed from the codebase
    if any(t not in grep_tags for t in yaml_entry[attr]):
        return (f'Yaml file {yaml_path} contains region '
                f'tag not used in source files: {tag}')

    return None

def _attr_validate_manually_specified_tests(
    yaml_path: str,
    yaml_entry: Dict[str, Any],
    tag: str,
    attr: str,
    grep_tags: List[str]
) -> List[str]:
    """Ensure manually-specified tests are correct

    This function ensures that manually-specified
    tests refer to files that actually exist.

    Arguments:
        yaml_path: A path to a .drift-data.yml file
        yaml_entry: The YAML entry to validate
        tag: The region tag corresponding to the specified YAML entry
        attr: The attribute of the YAML entry to validate
        grep_tags: A list of tags existing (not necessarily parsed out of)
                   the source code

    Returns:
        An error message if the manually-specified tests are invalid; None
        otherwise
    """
    errors = []
    yaml_dirname = os.path.dirname(yaml_path)

    for test_path in yaml_entry.keys():
        if test_path in constants.RESERVED_YAML_KEYS:
            continue  # Skip non-filepaths

        if not os.path.isabs(test_path):
            test_path = os.path.join(yaml_dirname, test_path)

        if not os.path.exists(test_path):
            errors.append(f'Test file {test_path} used '
                          f'in {yaml_path} not found!')

    return errors

def _validate_attrs(
    yaml_paths: List[str],
    grep_tags: List[str]
) -> Tuple[bool, List[str]]:
    """Validate attributes in a list of .drift-data.yml files

    This method verifies that attributes within
    a .drift-data.yml file are used appropriately.

    Args:
        yaml_paths: A list of .drift-data.yml filepaths to validate
        grep_tags: A list of region tags (parsed *and* unparsed) that exist
                   in the target directory

    Returns:
        A 2-tuple containing the following:
         - A boolean including file validity (True if all files passed
           attribute validation, False otherwise)
         - A list of validation error messages (if any) that were raised
    """

    errors = []

    for yaml_path in yaml_paths:
        with open(yaml_path, 'r') as file:
            yaml_contents = '\n'.join(file.readlines())
            parsed_yaml = yaml.safe_load(yaml_contents)

            for tag in parsed_yaml.keys():
                yaml_entry = parsed_yaml[tag]
                yaml_keys = list(yaml_entry.keys())

                if not yaml_keys:
                    continue

                yaml_attr = yaml_keys[0]

                # Validate keys that require specific values
                required_value_error = __attr_validate_required_values(
                    yaml_path,
                    yaml_entry,
                    tag,
                    yaml_attr
                )
                if required_value_error:
                    errors.append(required_value_error)

                # Validate additions field
                additions_error = __attr_validate_additions(
                    yaml_path,
                    yaml_entry,
                    tag,
                    yaml_attr,
                    grep_tags
                )
                if additions_error:
                    errors.append(additions_error)

                # Validate manually-specified tests
                manual_errors = __attr_validate_manually_specified_tests(
                    yaml_path,
                    yaml_entry,
                    tag,
                    yaml_attr,
                    grep_tags
                )
                if manual_errors:
                    errors += manual_errors

    is_valid = not errors
    return (is_valid, errors)

def _validate_region_tags(
    yaml_paths: List[str],
    grep_tags: List[str],
    source_tags: List[str],
) -> Tuple[bool, List[str]]:
    """Validate region-tag keys in a list of .drift-data.yml files

    This method verifies that region tag keys within a .drift-data.yml
    file are used *exactly* once in the source code.

    Args:
        yaml_paths: A list of .drift-data.yml filepaths to validate
        grep_tags: A list of region tags (parsed *and* unparsed) that exist
                   in the target directory
        source_tags: A list of successfully-parsed region tags in the target
                     directory

    Returns:
        A 2-tuple containing the following:
         - A boolean including file validity (True if all files passed region
           tag validation, False otherwise)
         - A list of validation error messages (if any) that were raised
    """

    seen_region_tags = set()
    output = []
    is_valid = True

    for yaml_path in yaml_paths:
        with open(yaml_path, 'r') as file:
            yaml_contents = '\n'.join(file.readlines())
            parsed_yaml = yaml.safe_load(yaml_contents)

            for tag in parsed_yaml.keys():
                yaml_entry = parsed_yaml[tag]
                tag_should_be_in_source = not (
                    'tested' in yaml_entry and yaml_entry['tested'] is False)

                # Verify mentioned region tags are used in the
                # source code (via parsing and/or grep results)
                if tag not in grep_tags:
                    output.append(
                        f'Yaml file {yaml_path} contains region '
                        f'tag not used in source files: {tag}')
                    is_valid = False
                elif tag_should_be_in_source and tag not in source_tags:
                    output.append(
                        f'Yaml file {yaml_path} contains '
                        f'unparsed region tag: {tag}')
                    output.append(
                        '  Remove it, or label it with "tested: false".')
                    is_valid = False
                elif not tag_should_be_in_source and tag in source_tags:
                    output.append(f'Parsed tag {tag} in file'
                                  f'{yaml_path} marked untested!')
                    is_valid = False

                # Verify region tags are present at most once
                if tag in seen_region_tags:
                    output.append(f'Region tag {tag} is used multiple '
                                  'times in .drift-data.yml files!')
                    is_valid = False
                else:
                    seen_region_tags.add(tag)

    return (is_valid, output)

def validate_yaml_syntax(
    root_dir: str,
    grep_tags: List[str],
    source_tags: List[str]
) -> Tuple[bool, List[str]]:
    """Validate a given directory's .drift-data.yml files

    This method validates that a given directory's .drift-data.yml files are
    semantically correct, and match up with the directory's source code.

    Args:
        yaml_paths: A list of .drift-data.yml filepaths to validate
        grep_tags: A list of region tags (parsed *and* unparsed) that exist
                   in the target directory
        source_tags: A list of successfully-parsed region tags in the target
                     directory

    Returns:
        A 2-tuple containing the following:
         - A boolean including file validity (True if all .drift-data.yml
           files passed validation, False otherwise)
         - A list of validation error messages (if any) that were raised
    """
    yaml_paths = file_utils.get_drift_yaml_files(root_dir)

    (tags_are_valid, tags_output) = (
        _validate_region_tags(yaml_paths, grep_tags, source_tags))
    (attrs_are_valid, attrs_output) = _validate_attrs(yaml_paths, grep_tags)

    output = tags_output + attrs_output
    is_valid = tags_are_valid and attrs_are_valid

    return (is_valid, output)
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

import sys
import calendar
from datetime import datetime

from preggy import expect
from tornado.testing import gen_test
from tornado.httpclient import HTTPError
from ujson import dumps

from holmes.models import Review
from tests.base import ApiTestCase
from tests.fixtures import DomainFactory, PageFactory, ReviewFactory

class TestReviewHandler(ApiTestCase):

    @gen_test
    def test_invalid_review_returns_404(self):
        domain = yield DomainFactory.create()
        page = yield PageFactory.create(domain=domain)

        url = self.get_url('/page/%s/review/invalid' % page.uuid)

        try:
            yield self.http_client.fetch(
                url,
                method='GET'
            )
        except HTTPError:
            err = sys.exc_info()[1]
            expect(err).not_to_be_null()
            expect(err.code).to_equal(404)
            expect(err.response.reason).to_be_like("Review with uuid of invalid not found!")
        else:
            assert False, "Should not have got this far"

    @gen_test
    def test_can_get_review(self):
        domain = yield DomainFactory.create()
        page = yield PageFactory.create(domain=domain)
        review = yield ReviewFactory.create(page=page)

        url = self.get_url(
            '/page/%s/review/%s' % (
                page.uuid,
                review.uuid
            )
        )

        response = yield self.http_client.fetch(url)

        expect(response.code).to_equal(200)

        dt = calendar.timegm(datetime.now().utctimetuple())

        expected = {
            "domain": domain.name,
            "pageId": str(page.uuid),
            "reviewId": str(review.uuid),
            "isComplete": False,
            "facts": [],
            "violations": [],
            "createdAt": dt
        }

        expect(response.body).to_be_like(dumps(expected))
<commit_msg>Review json done<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

import sys
import calendar
from datetime import datetime

from preggy import expect
from tornado.testing import gen_test
from tornado.httpclient import HTTPError
from ujson import loads

from tests.base import ApiTestCase
from tests.fixtures import DomainFactory, PageFactory, ReviewFactory

class TestReviewHandler(ApiTestCase):

    @gen_test
    def test_invalid_review_returns_404(self):
        domain = yield DomainFactory.create()
        page = yield PageFactory.create(domain=domain)

        url = self.get_url('/page/%s/review/invalid' % page.uuid)

        try:
            yield self.http_client.fetch(
                url,
                method='GET'
            )
        except HTTPError:
            err = sys.exc_info()[1]
            expect(err).not_to_be_null()
            expect(err.code).to_equal(404)
            expect(err.response.reason).to_be_like("Review with uuid of invalid not found!")
        else:
            assert False, "Should not have got this far"

    @gen_test
    def test_can_get_review(self):
        domain = yield DomainFactory.create()
        page = yield PageFactory.create(domain=domain)
        review = yield ReviewFactory.create(page=page)

        review.add_fact("fact", "kb", "value")
        review.add_violation("violation", "title", "description", 100)
        yield review.save()

        url = self.get_url(
            '/page/%s/review/%s' % (
                page.uuid,
                review.uuid
            )
        )

        response = yield self.http_client.fetch(url)

        expect(response.code).to_equal(200)

        dt = calendar.timegm(datetime.now().utctimetuple())

        expected = {
            "domain": domain.name,
            "pageId": str(page.uuid),
            "reviewId": str(review.uuid),
            "isComplete": False,
            'facts': [
                {'key': 'fact', 'unit': 'value', 'value': 'kb'}
            ],
            'violations': [
                {u'points': 100, u'description': u'description', u'key': u'violation', u'title': u'title'}
            ],
            "createdAt": dt
        }

        expect(loads(response.body)).to_be_like(expected)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# coding=utf-8

"""Integration tests."""

import json
from unittest import TestCase

from bson import ObjectId

from .schema import (
    User, Article, Email, Reference, UserReferenceNoAutoSave,
    UserReferenceAutoSave, UserReferenceDisabledIDCheck
)
from .fixtures import (
    user, user_dict, article, article_dict, article_dict_epoch,
    email, email_dict_id, email_dict_email, reference, reference_dict
)
from ..connection_case import DBConBase

try:
    str = unicode
except NameError:
    pass

class ToJSONNormalIntegrationTest(TestCase):
    """Good JSON Encoder Normal Data test."""

    def setUp(self):
        """Setup the class."""
        self.maxDiff = None
        self.user_cls = User
        self.user = user
        self.article_cls = Article
        self.article = article
        self.user_dict = user_dict
        self.article_dict = article_dict
        self.article_dict_epoch = article_dict_epoch

    def test_encode_user_data(self):
        """User model should be encoded properly."""
        result = json.loads(self.user.to_json())
        self.assertDictEqual(self.user_dict, result)

    def test_encode_article_data(self):
        """Article model should be encoded properly."""
        result = json.loads(self.article.to_json())
        self.assertDictEqual(self.article_dict, result)

    def test_encode_article_data_epoch_flag(self):
        """Article model should be encoded properly (Epoch flag is on)."""
        result = json.loads(self.article.to_json(epoch_mode=True))
        self.assertDictEqual(self.article_dict_epoch, result)

    def test_decode_user_data(self):
        """The decoded user data should be self.user."""
        user = self.user_cls.from_json(json.dumps(self.user_dict))
        self.assertIs(type(user), self.user_cls)
        self.assertDictEqual(self.user.to_mongo(), user.to_mongo())

    def test_decode_article_data(self):
        """The decoded user data should be self.user."""
        article = self.article_cls.from_json(json.dumps(self.article_dict))
        self.assertIs(type(article), self.article_cls)
        self.assertDictEqual(self.article.to_mongo(), article.to_mongo())

    def test_decode_article_data_epoch(self):
        """The decoded user data should be self.article."""
        article = self.article_cls.from_json(
            json.dumps(self.article_dict_epoch)
        )
        self.assertIs(type(article), self.article_cls)
        self.assertDictEqual(self.article.to_mongo(), article.to_mongo())

class FollowReferenceTest(DBConBase):
    """Good JSON follow reference encoder/decoder test."""

    def setUp(self):
        """Setup function."""
        self.maxDiff = None
        self.reference_cls = Reference
        self.reference = reference
        self.reference_dict = reference_dict

    def test_encode_follow_reference_data(self):
        """reference data should follow ReferenceField."""
        result = json.loads(self.reference.to_json(follow_reference=True))
        self.assertDictEqual(self.reference_dict, result)

    def test_decode_reference(self):
        """The decoded reference data should be self.reference."""
        result = self.reference_cls.from_json(
            json.dumps(self.reference_dict)
        )
        self.assertIs(type(result), self.reference_cls)
        self.assertEqual(result.id, self.reference.id)
        self.assertEqual(result.name, self.reference.name)
        self.assertListEqual(self.reference.references, result.references)

    def test_actual_data_store(self):
        """Actually data store."""
        for ref in self.reference.references:
            ref.user.save()
            ref.save()
        self.reference.save(cascade=True)
        result = json.loads(
            self.reference_cls.objects(
                id=self.reference.id
            ).get().to_json(follow_reference=True)
        )
        print(self.reference_dict)
        self.assertDictEqual(self.reference_dict, result)

class PrimaryKeyNotOidTest(TestCase):
    """Good JSON encoder/decoder email as primary key test."""

    def setUp(self):
        """Setup the class."""
        self.email = email
        self.data_id = email_dict_id
        self.data_email = email_dict_email

    def test_encode(self):
        """Email document should be encoded properly."""
        result = json.loads(self.email.to_json())
        self.assertDictEqual(self.data_id, result)

    def test_decode_id(self):
        """
        Email document should be decoded from json properly.

        This test is in the case that email is at "id" field.
        """
        result = Email.from_json(json.dumps(self.data_id)).to_mongo()
        self.assertDictEqual(self.email.to_mongo(), result)

    def test_decode_email(self):
        """
        Email document should be decoded from json properly.

        This test is in the case that email is at "email" field.
        """
        result = Email.from_json(json.dumps(self.data_email)).to_mongo()
        self.assertDictEqual(self.email.to_mongo(), result)

class FollowReferenceFieldTest(DBConBase):
    """Follow reference field integration tests."""

    def setUp(self):
        """Setup."""
        self.RefDoc = User
        self.Doc = UserReferenceNoAutoSave
        self.AutoSaveDoc = UserReferenceAutoSave
        self.DocNoIDCheck = UserReferenceDisabledIDCheck
        self.ref_doc = self.RefDoc(name="Test")
        self.doc = self.Doc()
        self.non_id_check_doc = self.DocNoIDCheck()
        self.autosave_doc = self.AutoSaveDoc()
        self.data = {
            u"id": str(ObjectId()),
            u"ref": {
                u"id": str(ObjectId()),
                u"name": self.ref_doc.name,
                u"address": []
            }
        }

    def tearDown(self):
        """Teardown class."""
        self.Doc.drop_collection()
        self.AutoSaveDoc.drop_collection()
        self.DocNoIDCheck.drop_collection()

    def test_serialization_with_save(self):
        """The serializer should follow the referenced doc."""
        self.ref_doc.save()
        self.doc.ref = self.ref_doc
        self.doc.save()
        result = json.loads(self.doc.to_json())
        self.assertDictEqual({
            u"id": str(self.doc.pk),
            u"ref": {
                u"id": str(self.ref_doc.pk),
                u"name": self.ref_doc.name,
                u"address": []
            }
        }, result)

    def test_serialization_without_save(self):
        """The serializer should follow the referenced doc (no ID check)."""
        self.non_id_check_doc.ref = self.ref_doc
        result = json.loads(self.non_id_check_doc.to_json())
        self.assertDictEqual({
            u"ref": {
                u"name": self.ref_doc.name,
                u"address": []
            }
        }, result)

    def test_deserialization_without_autosave(self):
        """The deserializer should work (no autosave)."""
        result = self.Doc.from_json(json.dumps(self.data))
        self.assertIsInstance(result, self.Doc)

    def test_deserialization_with_autosave(self):
        """The deserializer should work (autosave)."""
        result = self.AutoSaveDoc.from_json(json.dumps(self.data))
        self.assertDictEqual(self.data, json.loads(result.to_json()))
<commit_msg>Changed test_deserialization_without_autosave to evaluate the data value<commit_after>#!/usr/bin/env python
# coding=utf-8

"""Integration tests."""

import json
from unittest import TestCase

from bson import ObjectId

from .schema import (
    User, Article, Email, Reference, UserReferenceNoAutoSave,
    UserReferenceAutoSave, UserReferenceDisabledIDCheck
)
from .fixtures import (
    user, user_dict, article, article_dict, article_dict_epoch,
    email, email_dict_id, email_dict_email, reference, reference_dict
)
from ..connection_case import DBConBase

try:
    str = unicode
except NameError:
    pass

class ToJSONNormalIntegrationTest(TestCase):
    """Good JSON Encoder Normal Data test."""

    def setUp(self):
        """Setup the class."""
        self.maxDiff = None
        self.user_cls = User
        self.user = user
        self.article_cls = Article
        self.article = article
        self.user_dict = user_dict
        self.article_dict = article_dict
        self.article_dict_epoch = article_dict_epoch

    def test_encode_user_data(self):
        """User model should be encoded properly."""
        result = json.loads(self.user.to_json())
        self.assertDictEqual(self.user_dict, result)

    def test_encode_article_data(self):
        """Article model should be encoded properly."""
        result = json.loads(self.article.to_json())
        self.assertDictEqual(self.article_dict, result)

    def test_encode_article_data_epoch_flag(self):
        """Article model should be encoded properly (Epoch flag is on)."""
        result = json.loads(self.article.to_json(epoch_mode=True))
        self.assertDictEqual(self.article_dict_epoch, result)

    def test_decode_user_data(self):
        """The decoded user data should be self.user."""
        user = self.user_cls.from_json(json.dumps(self.user_dict))
        self.assertIs(type(user), self.user_cls)
        self.assertDictEqual(self.user.to_mongo(), user.to_mongo())

    def test_decode_article_data(self):
        """The decoded user data should be self.user."""
        article = self.article_cls.from_json(json.dumps(self.article_dict))
        self.assertIs(type(article), self.article_cls)
        self.assertDictEqual(self.article.to_mongo(), article.to_mongo())

    def test_decode_article_data_epoch(self):
        """The decoded user data should be self.article."""
        article = self.article_cls.from_json(
            json.dumps(self.article_dict_epoch)
        )
        self.assertIs(type(article), self.article_cls)
        self.assertDictEqual(self.article.to_mongo(), article.to_mongo())

class FollowReferenceTest(DBConBase):
    """Good JSON follow reference encoder/decoder test."""

    def setUp(self):
        """Setup function."""
        self.maxDiff = None
        self.reference_cls = Reference
        self.reference = reference
        self.reference_dict = reference_dict

    def test_encode_follow_reference_data(self):
        """reference data should follow ReferenceField."""
        result = json.loads(self.reference.to_json(follow_reference=True))
        self.assertDictEqual(self.reference_dict, result)

    def test_decode_reference(self):
        """The decoded reference data should be self.reference."""
        result = self.reference_cls.from_json(
            json.dumps(self.reference_dict)
        )
        self.assertIs(type(result), self.reference_cls)
        self.assertEqual(result.id, self.reference.id)
        self.assertEqual(result.name, self.reference.name)
        self.assertListEqual(self.reference.references, result.references)

    def test_actual_data_store(self):
        """Actually data store."""
        for ref in self.reference.references:
            ref.user.save()
            ref.save()
        self.reference.save(cascade=True)
        result = json.loads(
            self.reference_cls.objects(
                id=self.reference.id
            ).get().to_json(follow_reference=True)
        )
        print(self.reference_dict)
        self.assertDictEqual(self.reference_dict, result)

class PrimaryKeyNotOidTest(TestCase):
    """Good JSON encoder/decoder email as primary key test."""

    def setUp(self):
        """Setup the class."""
        self.email = email
        self.data_id = email_dict_id
        self.data_email = email_dict_email

    def test_encode(self):
        """Email document should be encoded properly."""
        result = json.loads(self.email.to_json())
        self.assertDictEqual(self.data_id, result)

    def test_decode_id(self):
        """
        Email document should be decoded from json properly.

        This test is in the case that email is at "id" field.
        """
        result = Email.from_json(json.dumps(self.data_id)).to_mongo()
        self.assertDictEqual(self.email.to_mongo(), result)

    def test_decode_email(self):
        """
        Email document should be decoded from json properly.

        This test is in the case that email is at "email" field.
        """
        result = Email.from_json(json.dumps(self.data_email)).to_mongo()
        self.assertDictEqual(self.email.to_mongo(), result)

class FollowReferenceFieldTest(DBConBase):
    """Follow reference field integration tests."""

    def setUp(self):
        """Setup."""
        self.RefDoc = User
        self.Doc = UserReferenceNoAutoSave
        self.AutoSaveDoc = UserReferenceAutoSave
        self.DocNoIDCheck = UserReferenceDisabledIDCheck
        self.ref_doc = self.RefDoc(name="Test")
        self.doc = self.Doc()
        self.non_id_check_doc = self.DocNoIDCheck()
        self.autosave_doc = self.AutoSaveDoc()
        self.data = {
            u"id": str(ObjectId()),
            u"ref": {
                u"id": str(ObjectId()),
                u"name": self.ref_doc.name,
                u"address": []
            }
        }

    def tearDown(self):
        """Teardown class."""
        self.Doc.drop_collection()
        self.AutoSaveDoc.drop_collection()
        self.DocNoIDCheck.drop_collection()

    def test_serialization_with_save(self):
        """The serializer should follow the referenced doc."""
        self.ref_doc.save()
        self.doc.ref = self.ref_doc
        self.doc.save()
        result = json.loads(self.doc.to_json())
        self.assertDictEqual({
            u"id": str(self.doc.pk),
            u"ref": {
                u"id": str(self.ref_doc.pk),
                u"name": self.ref_doc.name,
                u"address": []
            }
        }, result)

    def test_serialization_without_save(self):
        """The serializer should follow the referenced doc (no ID check)."""
        self.non_id_check_doc.ref = self.ref_doc
        result = json.loads(self.non_id_check_doc.to_json())
        self.assertDictEqual({
            u"ref": {
                u"name": self.ref_doc.name,
                u"address": []
            }
        }, result)

    def test_deserialization_without_autosave(self):
        """The deserializer should work (no autosave)."""
        result = self.Doc.from_json(json.dumps(self.data))
        self.assertDictEqual(self.data, json.loads(result.to_json()))

    def test_deserialization_with_autosave(self):
        """The deserializer should work (autosave)."""
        result = self.AutoSaveDoc.from_json(json.dumps(self.data))
        self.assertDictEqual(self.data, json.loads(result.to_json()))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from hamcrest import assert_that
from hamcrest import has_entries
from hamcrest import not_none

from pyworkplace.message import Message

RECIPIENT_ID = 10000000
URL = ''

def test_send_message():

    params = {
        'version': 'v2.8',
        'access_token': 'this is my token',
    }
    message = Message(**params)

    args = {
        'recipient_id': RECIPIENT_ID,
        'message': 'Hola Mundo',
    }
    assert_that(
        message.send_message(**args),
        not_none(),
    )

    body = {
        'message': 'Hola Mundo',
        'recipient': {'id': RECIPIENT_ID},
        'notification_type': 'REGULAR',
    }
    response = {
        'url': 'https://graph.facebook.com/v2.8/me/messages',
        'params': {'access_token': params['access_token']},
        'json': body,
        'headers': {'Content-type': 'application/json'},
        'method': 'post',
    }
    assert_that(
        response,
        has_entries(message.response),
    )

def test_send_text_message():
    params = {
        'version': 'v2.8',
        'access_token': 'this is my token',
    }
    message = Message(**params)
    args = {
        'recipient_id': RECIPIENT_ID,
        'message': 'o/ World',
    }
    assert_that(
        message.send_text_message(**args),
        not_none(),
    )

    body = {
        'message': {'text': 'o/ World'},
        'recipient': {'id': RECIPIENT_ID},
        'notification_type': 'REGULAR',
    }
    response = {
        'url': 'https://graph.facebook.com/v2.8/me/messages',
        'params': {'access_token': params['access_token']},
        'json': body,
        'headers': {'Content-type': 'application/json'},
        'method': 'post',
    }
    assert_that(
        response,
        has_entries(message.response),
    )
<commit_msg>!#34 Add: test send_quick_replies single<commit_after># -*- coding: utf-8 -*-
from hamcrest import assert_that
from hamcrest import has_entries
from hamcrest import not_none

from pyworkplace.message import Message

RECIPIENT_ID = 10000000
URL = ''

def test_send_message():

    params = {
        'version': 'v2.8',
        'access_token': 'this is my token',
    }
    message = Message(**params)

    args = {
        'recipient_id': RECIPIENT_ID,
        'message': 'Hola Mundo',
    }
    assert_that(
        message.send_message(**args),
        not_none(),
    )

    body = {
        'message': 'Hola Mundo',
        'recipient': {'id': RECIPIENT_ID},
        'notification_type': 'REGULAR',
    }
    response = {
        'url': 'https://graph.facebook.com/v2.8/me/messages',
        'params': {'access_token': params['access_token']},
        'json': body,
        'headers': {'Content-type': 'application/json'},
        'method': 'post',
    }
    assert_that(
        response,
        has_entries(message.response),
    )

def test_send_text_message():
    params = {
        'version': 'v2.8',
        'access_token': 'this is my token',
    }
    message = Message(**params)
    args = {
        'recipient_id': RECIPIENT_ID,
        'message': 'o/ World',
    }
    assert_that(
        message.send_text_message(**args),
        not_none(),
    )

    body = {
        'message': {'text': 'o/ World'},
        'recipient': {'id': RECIPIENT_ID},
        'notification_type': 'REGULAR',
    }
    response = {
        'url': 'https://graph.facebook.com/v2.8/me/messages',
        'params': {'access_token': params['access_token']},
        'json': body,
        'headers': {'Content-type': 'application/json'},
        'method': 'post',
    }
    assert_that(
        response,
        has_entries(message.response),
    )

def test_send_quick_replies_message():
    params = {
        'version': 'v2.8',
        'access_token': 'this is my token',
    }
    message = Message(**params)
    args = {
        'recipient_id': RECIPIENT_ID,
        'message': 'o/ World',
        'quick_replies': [
            {
                'content_type': 'text',
                'title': 'Search',
                'payload': '<POSTBACK_PAYLOAD>',
                'image_url': 'http://example.com/img/red.png',
            },
        ],
    }
    assert_that(
        message.send_quick_replies_message(**args),
        not_none(),
    )

    body = {
        'message': {
            'text': 'o/ World',
            'quick_replies': [
                {
                    'content_type': 'text',
                    'title': 'Search',
                    'payload': '<POSTBACK_PAYLOAD>',
                    'image_url': 'http://example.com/img/red.png',
                },
            ],
        },
        'recipient': {'id': RECIPIENT_ID},
        'notification_type': 'REGULAR',

    }
    response = {
        'url': 'https://graph.facebook.com/v2.8/me/messages',
        'params': {'access_token': params['access_token']},
        'json': body,
        'headers': {'Content-type': 'application/json'},
        'method': 'post',
    }
    assert_that(
        response,
        has_entries(message.response),
    )
<|endoftext|>


<commit_before>import nose.tools as nt
import numpy as np
import theano
import theano.tensor as T

import treeano
import treeano.nodes as tn

from .. import utils

fX = theano.config.floatX

def test_dense_node_serialization():
    tn.check_serialization(tn.DenseNode("a"))
    tn.check_serialization(tn.DenseNode("a", num_units=100))

def test_dense_combine_node_serialization():
    tn.check_serialization(tn.DenseCombineNode("a", []))
    tn.check_serialization(tn.DenseCombineNode("a", [], num_units=100))

def test_dense_node():
    network = tn.SequentialNode(
        "seq",
        [tn.InputNode("in", shape=(3, 4, 5)),
         tn.DenseNode("fc1", num_units=6),
         tn.DenseNode("fc2", num_units=7),
         tn.DenseNode("fc3", num_units=8)]
    ).build()
    x = np.random.randn(3, 4, 5).astype(fX)
    fn = network.function(["in"], ["fc3"])
    res = fn(x)[0]
    nt.assert_equal(res.shape, (3, 8))

def test_dense_combine_node():
    network = tn.SequentialNode(
        "seq",
        [tn.InputNode("in", shape=(3, 4, 5)),
         tn.DenseCombineNode("fc1", [tn.IdentityNode("i1")], num_units=6),
         tn.DenseCombineNode("fc2", [tn.IdentityNode("i2")], num_units=7),
         tn.DenseCombineNode("fc3", [tn.IdentityNode("i3")], num_units=8)]
    ).build()
    x = np.random.randn(3, 4, 5).astype(fX)
    fn = network.function(["in"], ["fc3"])
    res = fn(x)[0]
    nt.assert_equal(res.shape, (3, 8))

def test_dense_node_and_dense_combine_node():
    # testing that dense node and dense combine node with identity child
    # return the same thing
    network1 = tn.HyperparameterNode(
        "hp",
        tn.SequentialNode(
            "seq",
            [tn.InputNode("in", shape=(3, 4, 5)),
             tn.DenseNode("fc1", num_units=6),
             tn.DenseNode("fc2", num_units=7),
             tn.DenseNode("fc3", num_units=8)]
        ),
        shared_initializations=[utils.OnesInitialization()]
    ).build()
    network2 = tn.HyperparameterNode(
        "hp",
        tn.SequentialNode(
            "seq",
            [tn.InputNode("in", shape=(3, 4, 5)),
             tn.DenseCombineNode("fc1", [tn.IdentityNode("i1")], num_units=6),
             tn.DenseCombineNode("fc2", [tn.IdentityNode("i2")], num_units=7),
             tn.DenseCombineNode("fc3", [tn.IdentityNode("i3")], num_units=8)]
        ),
        shared_initializations=[utils.OnesInitialization()]
    ).build()
    x = np.random.randn(3, 4, 5).astype(fX)
    fn1 = network1.function(["in"], ["fc3"])
    fn2 = network2.function(["in"], ["fc3"])
    np.testing.assert_allclose(fn1(x), fn2(x))
<commit_msg>additional test for dense combine node<commit_after>import nose.tools as nt
import numpy as np
import theano
import theano.tensor as T

import treeano
import treeano.nodes as tn

from .. import utils

fX = theano.config.floatX

def test_dense_node_serialization():
    tn.check_serialization(tn.DenseNode("a"))
    tn.check_serialization(tn.DenseNode("a", num_units=100))

def test_dense_combine_node_serialization():
    tn.check_serialization(tn.DenseCombineNode("a", []))
    tn.check_serialization(tn.DenseCombineNode("a", [], num_units=100))

def test_dense_node():
    network = tn.SequentialNode(
        "seq",
        [tn.InputNode("in", shape=(3, 4, 5)),
         tn.DenseNode("fc1", num_units=6),
         tn.DenseNode("fc2", num_units=7),
         tn.DenseNode("fc3", num_units=8)]
    ).build()
    x = np.random.randn(3, 4, 5).astype(fX)
    fn = network.function(["in"], ["fc3"])
    res = fn(x)[0]
    nt.assert_equal(res.shape, (3, 8))

def test_dense_combine_node():
    network = tn.SequentialNode(
        "seq",
        [tn.InputNode("in", shape=(3, 4, 5)),
         tn.DenseCombineNode("fc1", [tn.IdentityNode("i1")], num_units=6),
         tn.DenseCombineNode("fc2", [tn.IdentityNode("i2")], num_units=7),
         tn.DenseCombineNode("fc3", [tn.IdentityNode("i3")], num_units=8)]
    ).build()
    x = np.random.randn(3, 4, 5).astype(fX)
    fn = network.function(["in"], ["fc3"])
    res = fn(x)[0]
    nt.assert_equal(res.shape, (3, 8))

def test_dense_node_and_dense_combine_node1():
    # testing that dense node and dense combine node with identity child
    # return the same thing
    network1 = tn.HyperparameterNode(
        "hp",
        tn.SequentialNode(
            "seq",
            [tn.InputNode("in", shape=(3, 4, 5)),
             tn.DenseNode("fc1", num_units=6),
             tn.DenseNode("fc2", num_units=7),
             tn.DenseNode("fc3", num_units=8)]
        ),
        shared_initializations=[utils.OnesInitialization()]
    ).build()
    network2 = tn.HyperparameterNode(
        "hp",
        tn.SequentialNode(
            "seq",
            [tn.InputNode("in", shape=(3, 4, 5)),
             tn.DenseCombineNode("fc1", [tn.IdentityNode("i1")], num_units=6),
             tn.DenseCombineNode("fc2", [tn.IdentityNode("i2")], num_units=7),
             tn.DenseCombineNode("fc3", [tn.IdentityNode("i3")], num_units=8)]
        ),
        shared_initializations=[utils.OnesInitialization()]
    ).build()
    x = np.random.randn(3, 4, 5).astype(fX)
    fn1 = network1.function(["in"], ["fc3"])
    fn2 = network2.function(["in"], ["fc3"])
    np.testing.assert_allclose(fn1(x), fn2(x))

def test_dense_node_and_dense_combine_node2():
    # testing that summing the output of 2 dense nodes is the same as
    # applying a dense combine node with 2 identities (+ bias)
    # and the same as multiplying the output of 1 dense node by 2
    network0 = tn.HyperparameterNode(
        "hp",
        tn.SequentialNode(
            "seq",
            [tn.InputNode("in", shape=(3, 4, 5)),
             tn.DenseNode("dense1", num_units=6),
             tn.toy.MultiplyConstantNode("mul", value=2)]
        ),
        shared_initializations=[utils.OnesInitialization()]
    ).build()
    network1 = tn.HyperparameterNode(
        "hp",
        tn.SequentialNode(
            "seq",
            [tn.InputNode("in", shape=(3, 4, 5)),
             tn.ElementwiseSumNode(
                 "sum",
                 [tn.DenseNode("dense1", num_units=6),
                  tn.DenseNode("dense2", num_units=6)])]
        ),
        shared_initializations=[utils.OnesInitialization()]
    ).build()
    network2 = tn.HyperparameterNode(
        "hp",
        tn.SequentialNode(
            "seq",
            [tn.InputNode("in", shape=(3, 4, 5)),
             tn.DenseCombineNode(
                 "fc",
                 [tn.IdentityNode("i1"),
                  tn.IdentityNode("i2")],
                 num_units=6),
             tn.AddBiasNode("bias")]
        ),
        shared_initializations=[utils.OnesInitialization()]
    ).build()
    x = np.random.randn(3, 4, 5).astype(fX)
    fn0 = network0.function(["in"], ["hp"])
    fn1 = network1.function(["in"], ["hp"])
    fn2 = network2.function(["in"], ["hp"])
    np.testing.assert_allclose(fn0(x), fn1(x))
    np.testing.assert_allclose(fn0(x), fn2(x))
<|endoftext|>


<commit_before>import unittest

from netjsonconfig import OpenWrt
from netjsonconfig.backends.openwrt.timezones import timezones
from netjsonconfig.utils import _TabsMixin

class TestContext(unittest.TestCase, _TabsMixin):
    """
    tests for configuration variables feature
    """
    def test_config(self):
        config = {
            "general": {
                "hostname": "{{ name }}",
                "description": "{{ desc }}"
            }
        }
        context = {
            "name": "test-context-name",
            "desc": "test.context.desc"
        }
        o = OpenWrt(config, context=context)
        output = o.render()
        self.assertIn("option hostname 'test-context-name'", output)
        self.assertIn("option description 'test.context.desc'", output)

    def test_template(self):
        config = {"general": {"hostname": "{{ name }}"}}
        template = {"general": {"description": "{{ desc }}"}}
        context = {
            "name": "test-context-name",
            "desc": "test.context.desc"
        }
        o = OpenWrt(config, context=context, templates=[template])
        output = o.render()
        self.assertIn("option hostname 'test-context-name'", output)
        self.assertIn("option description 'test.context.desc'", output)

    def test_evaluation_order(self):
        config = {
            "general": {
                "timezone": "{{ tz }}",
            }
        }
        context = {"tz": "Europe/Amsterdam"}
        o = OpenWrt(config, context=context)
        line = "option timezone '{Europe/Amsterdam}'".format(**timezones)
        output = o.render()
        self.assertIn(line, output)

    def test_no_variables_found(self):
        config = {
            "general": {
                "hostname": "host",
            }
        }
        o = OpenWrt(config, context={"a": "b"})
        output = o.render()
        self.assertIn("option hostname 'host'", output)
<commit_msg>Corrected text_context: test_no_variables_found<commit_after>import unittest

from netjsonconfig import OpenWrt
from netjsonconfig.backends.openwrt.timezones import timezones
from netjsonconfig.utils import _TabsMixin

class TestContext(unittest.TestCase, _TabsMixin):
    """
    tests for configuration variables feature
    """
    def test_config(self):
        config = {
            "general": {
                "hostname": "{{ name }}",
                "description": "{{ desc }}"
            }
        }
        context = {
            "name": "test-context-name",
            "desc": "test.context.desc"
        }
        o = OpenWrt(config, context=context)
        output = o.render()
        self.assertIn("option hostname 'test-context-name'", output)
        self.assertIn("option description 'test.context.desc'", output)

    def test_template(self):
        config = {"general": {"hostname": "{{ name }}"}}
        template = {"general": {"description": "{{ desc }}"}}
        context = {
            "name": "test-context-name",
            "desc": "test.context.desc"
        }
        o = OpenWrt(config, context=context, templates=[template])
        output = o.render()
        self.assertIn("option hostname 'test-context-name'", output)
        self.assertIn("option description 'test.context.desc'", output)

    def test_evaluation_order(self):
        config = {
            "general": {
                "timezone": "{{ tz }}",
            }
        }
        context = {"tz": "Europe/Amsterdam"}
        o = OpenWrt(config, context=context)
        line = "option timezone '{Europe/Amsterdam}'".format(**timezones)
        output = o.render()
        self.assertIn(line, output)

    def test_no_variables_found(self):
        config = {
            "general": {
                "description": "{{ desc }}",
            }
        }
        o = OpenWrt(config, context={"a": "b"})
        output = o.render()
        self.assertIn("option description '{{ desc }}'", output)
<|endoftext|>


<commit_before>"""
Tests BoundColumn attributes and methods.
"""
from contextlib2 import ExitStack
from unittest import TestCase

from pandas import date_range, DataFrame
from pandas.util.testing import assert_frame_equal

from zipline.pipeline import Pipeline
from zipline.pipeline.data.testing import TestingDataSet as TDS
from zipline.utils.test_utils import chrange, temp_pipeline_engine

class LatestTestCase(TestCase):

    @classmethod
    def setUpClass(cls):
        cls._stack = stack = ExitStack()
        cls.calendar = cal = date_range('2014', '2015', freq='D', tz='UTC')
        cls.sids = list(range(5))
        cls.engine = stack.enter_context(
            temp_pipeline_engine(
                cal,
                cls.sids,
                random_seed=100,
                symbols=chrange('A', 'E'),
            ),
        )
        cls.assets = cls.engine._finder.retrieve_all(cls.sids)

    @classmethod
    def tearDownClass(cls):
        cls._stack.close()

    def expected_latest(self, column, slice_):
        loader = self.engine.get_loader(column)
        return DataFrame(
            loader.values(column.dtype, self.calendar, self.sids)[slice_],
            index=self.calendar[slice_],
            columns=self.sids,
        )

    def test_latest(self):
        columns = TDS.columns
        pipe = Pipeline(
            columns={c.name: c.latest for c in columns},
        )

        cal_slice = slice(20, 40)
        dates_to_test = self.calendar[cal_slice]
        result = self.engine.run_pipeline(
            pipe,
            dates_to_test[0],
            dates_to_test[-1],
        )
        for column in columns:
            float_result = result[column.name].unstack()
            expected_float_result = self.expected_latest(column, cal_slice)
            assert_frame_equal(float_result, expected_float_result)
<commit_msg>BUG: Check against assets, not sids in pipeline output.<commit_after>"""
Tests BoundColumn attributes and methods.
"""
from contextlib2 import ExitStack
from unittest import TestCase

from pandas import date_range, DataFrame
from pandas.util.testing import assert_frame_equal

from zipline.pipeline import Pipeline
from zipline.pipeline.data.testing import TestingDataSet as TDS
from zipline.utils.test_utils import chrange, temp_pipeline_engine

class LatestTestCase(TestCase):

    @classmethod
    def setUpClass(cls):
        cls._stack = stack = ExitStack()
        cls.calendar = cal = date_range('2014', '2015', freq='D', tz='UTC')
        cls.sids = list(range(5))
        cls.engine = stack.enter_context(
            temp_pipeline_engine(
                cal,
                cls.sids,
                random_seed=100,
                symbols=chrange('A', 'E'),
            ),
        )
        cls.assets = cls.engine._finder.retrieve_all(cls.sids)

    @classmethod
    def tearDownClass(cls):
        cls._stack.close()

    def expected_latest(self, column, slice_):
        loader = self.engine.get_loader(column)
        return DataFrame(
            loader.values(column.dtype, self.calendar, self.sids)[slice_],
            index=self.calendar[slice_],
            columns=self.assets,
        )

    def test_latest(self):
        columns = TDS.columns
        pipe = Pipeline(
            columns={c.name: c.latest for c in columns},
        )

        cal_slice = slice(20, 40)
        dates_to_test = self.calendar[cal_slice]
        result = self.engine.run_pipeline(
            pipe,
            dates_to_test[0],
            dates_to_test[-1],
        )
        for column in columns:
            float_result = result[column.name].unstack()
            expected_float_result = self.expected_latest(column, cal_slice)
            assert_frame_equal(float_result, expected_float_result)
<|endoftext|>


<commit_before>from __future__ import absolute_import, division, print_function
import os

import attr
import pytest

import environ

@environ.config(prefix="APP")
class AppConfig(object):
    host = environ.var("127.0.0.1")
    port = environ.var(5000, converter=int)

def test_default():
    cfg = AppConfig.from_environ()
    assert cfg.host == "127.0.0.1"
    assert cfg.port == 5000

def test_env():
    env = {"APP_HOST": "0.0.0.0"}
    cfg = AppConfig.from_environ(environ=env)
    assert cfg.host == "0.0.0.0"
    assert cfg.port == 5000
<commit_msg>Fix formatting, remove unused vars<commit_after>from __future__ import absolute_import, division, print_function

import environ

@environ.config(prefix="APP")
class AppConfig(object):
    host = environ.var("127.0.0.1")
    port = environ.var(5000, converter=int)

def test_default():
    cfg = AppConfig.from_environ()
    assert cfg.host == "127.0.0.1"
    assert cfg.port == 5000

def test_env():
    env = {"APP_HOST": "0.0.0.0"}
    cfg = AppConfig.from_environ(environ=env)
    assert cfg.host == "0.0.0.0"
    assert cfg.port == 5000
<|endoftext|>


<commit_before>"""
FS Nav unittests for utility: count
"""

from glob import glob
import unittest

from click.testing import CliRunner

from fsnav import cmdl

class TestCount(unittest.TestCase):

    def setUp(self):
        self.paths = glob('*')
        self.runner = CliRunner()

    def test_unique(self):

        # Count a list of unique paths
        result = self.runner.invoke(cmdl.count.main, self.paths)
        self.assertEqual(0, result.exit_code)
        self.assertEqual(str(len(self.paths)), result.output.strip())

    def test_non_unique(self):

        # Count a list of paths containing duplicates
        paths = self.paths
        paths.append(paths[0])
        paths.append(paths[1])
        result = self.runner.invoke(cmdl.count.main, paths)
        self.assertEqual(0, result.exit_code)
        self.assertEqual(str(len(set(paths))), result.output.strip())

    def test_non_existent(self):

        # Try counting a path that doesn't exist
        result = self.runner.invoke(cmdl.count.main, ['.-1-2-3--I_DONT_EXIST-_-'])
        self.assertEqual(0, result.exit_code)
        self.assertEqual('0', result.output.strip())

    def test_home_dir(self):

        # Home directory path like ~/
        result = self.runner.invoke(cmdl.count.main, ['~/'])
        self.assertEqual(0, result.exit_code)
        self.assertEqual('1', result.output.strip())

    def test_wildcard(self):

        # Wildcard path like *
        result = self.runner.invoke(cmdl.count.main, ['*'])
        self.assertEqual(0, result.exit_code)
        self.assertEqual(str(len(self.paths)), result.output.strip())
<commit_msg>+ tests for --license & --version<commit_after>"""
FS Nav unittests for utility: count
"""

from glob import glob
import unittest

from click.testing import CliRunner

import fsnav
from fsnav import cmdl

class TestCount(unittest.TestCase):

    def setUp(self):
        self.paths = glob('*')
        self.runner = CliRunner()

    def test_unique(self):

        # Count a list of unique paths
        result = self.runner.invoke(cmdl.count.main, self.paths)
        self.assertEqual(0, result.exit_code)
        self.assertEqual(str(len(self.paths)), result.output.strip())

    def test_non_unique(self):

        # Count a list of paths containing duplicates
        paths = self.paths
        paths.append(paths[0])
        paths.append(paths[1])
        result = self.runner.invoke(cmdl.count.main, paths)
        self.assertEqual(0, result.exit_code)
        self.assertEqual(str(len(set(paths))), result.output.strip())

    def test_non_existent(self):

        # Try counting a path that doesn't exist
        result = self.runner.invoke(cmdl.count.main, ['.-1-2-3--I_DONT_EXIST-_-'])
        self.assertEqual(0, result.exit_code)
        self.assertEqual('0', result.output.strip())

    def test_home_dir(self):

        # Home directory path like ~/
        result = self.runner.invoke(cmdl.count.main, ['~/'])
        self.assertEqual(0, result.exit_code)
        self.assertEqual('1', result.output.strip())

    def test_wildcard(self):

        # Wildcard path like *
        result = self.runner.invoke(cmdl.count.main, ['*'])
        self.assertEqual(0, result.exit_code)
        self.assertEqual(str(len(self.paths)), result.output.strip())

    def test_license(self):

        # count --license
        result = self.runner.invoke(cmdl.count.main, ['--license'])
        self.assertEqual(result.output.strip(), fsnav.__license__.strip())

    def test_version(self):

        # count --version
        result = self.runner.invoke(cmdl.count.main, ['--version'])
        self.assertEqual(result.output.strip(), fsnav.__version__.strip())<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

#   Copyright (c) 2010-2014, MIT Probabilistic Computing Project
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
import os
import exceptions
import pytest

import crosscat.LocalEngine

import bayeslite
import bayeslite.crosscat
from bayeslite.sqlite3_util import sqlite3_quote_name as quote

root = os.path.dirname(os.path.abspath(__file__))
DUMMY_CODEBOOK = os.path.join(root, 'dummy_codebook.csv')
DUMMY_DATA = os.path.join(root, 'dummy.csv')

def test_codebook_valmap():
    with bayeslite.bayesdb_open() as bdb:
        cc = crosscat.LocalEngine.LocalEngine(seed=0)
        ccme = bayeslite.crosscat.CrosscatMetamodel(cc)
        bayeslite.bayesdb_register_metamodel(bdb, ccme)

        bayeslite.bayesdb_read_csv(bdb, 'dummy', file(DUMMY_DATA),
            header = True,
            create = True)

        bayeslite.bayesdb_load_codebook_csv_file(bdb, 'dummy', DUMMY_CODEBOOK)

        bdb.execute("""
        CREATE GENERATOR dummy_cc FOR dummy
            USING crosscat(
                GUESS(*),
                kerberos IGNORE,
                age NUMERICAL,
                city CATEGORICAL
            );
        """)

        bdb.execute("""
        INITIALIZE 10 MODELS FOR dummy_cc;
        """)

        bdb.execute("""
        ANALYZE dummy_cc FOR 20 ITERATIONS WAIT;
        """)

        bdb.execute("""
        SIMULATE age FROM dummy_cc GIVEN city = RIO LIMIT 5
        """)

        bdb.sql_execute("""
        INSERT INTO dummy (kerberos, age, city) VALUES
            ('jackie', 18, 'LA'), ('rocker', 22, 'DC');
        """)

        bdb.execute("""
        ANALYZE dummy_cc FOR 20 ITERATIONS WAIT;
        """)

        with pytest.raises(exceptions.KeyError):
            bdb.execute("""
            SIMULATE age FROM dummy_cc GIVEN city = LA LIMIT 5;
            """)
<commit_msg>added docmentation to the test<commit_after># -*- coding: utf-8 -*-

#   Copyright (c) 2010-2014, MIT Probabilistic Computing Project
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
import os
import exceptions
import pytest

import crosscat.LocalEngine

import bayeslite
import bayeslite.crosscat
from bayeslite.sqlite3_util import sqlite3_quote_name as quote

root = os.path.dirname(os.path.abspath(__file__))
DUMMY_CODEBOOK = os.path.join(root, 'dummy_codebook.csv')
DUMMY_DATA = os.path.join(root, 'dummy.csv')

"""
A categorical column in crosscat can only take on a fixed number of values
v1, v2, ..., v3. In this test, we have a categorical column called
`city` which takes on values `RIO, LA, SF, DC` as specified in the codebook
value map.

    INITIALIE dummy table with only RIO and SF appearing in dataset
    ANALYZE dummy_cc
    INSERT rows with `city` names `LA` and `DC`
    ANALYZE dummy_cc
    SIMULATE specifying `city` = `LA` (throws KeyError)
"""

def test_codebook_valmap():
    with bayeslite.bayesdb_open() as bdb:
        cc = crosscat.LocalEngine.LocalEngine(seed=0)
        ccme = bayeslite.crosscat.CrosscatMetamodel(cc)
        bayeslite.bayesdb_register_metamodel(bdb, ccme)

        bayeslite.bayesdb_read_csv(bdb, 'dummy', file(DUMMY_DATA),
            header = True,
            create = True)

        bayeslite.bayesdb_load_codebook_csv_file(bdb, 'dummy', DUMMY_CODEBOOK)

        bdb.execute("""
        CREATE GENERATOR dummy_cc FOR dummy
            USING crosscat(
                GUESS(*),
                kerberos IGNORE,
                age NUMERICAL,
                city CATEGORICAL
            );
        """)

        bdb.execute("""
        INITIALIZE 10 MODELS FOR dummy_cc;
        """)

        bdb.execute("""
        ANALYZE dummy_cc FOR 20 ITERATIONS WAIT;
        """)

        bdb.execute("""
        SIMULATE age FROM dummy_cc GIVEN city = RIO LIMIT 5
        """)

        bdb.sql_execute("""
        INSERT INTO dummy (kerberos, age, city) VALUES
            ('jackie', 18, 'LA'), ('rocker', 22, 'DC');
        """)

        bdb.execute("""
        ANALYZE dummy_cc FOR 20 ITERATIONS WAIT;
        """)

        with pytest.raises(exceptions.KeyError):
            bdb.execute("""
            SIMULATE age FROM dummy_cc GIVEN city = LA LIMIT 5;
            """)
<|endoftext|>