<commit_before>########################################################################
#
# (C) 2015, Brian Coca <bcoca@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#
########################################################################

from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import datetime
import os
import tarfile
import tempfile
import yaml
from distutils.version import LooseVersion
from shutil import rmtree

import ansible.constants as C
from ansible.errors import AnsibleError
from ansible.module_utils.urls import open_url
from ansible.playbook.role.requirement import RoleRequirement
from ansible.galaxy.api import GalaxyAPI

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()

class GalaxyRole(object):

    SUPPORTED_SCMS = set(['git', 'hg'])
    META_MAIN = os.path.join('meta', 'main.yml')
    META_INSTALL = os.path.join('meta', '.galaxy_install_info')
    ROLE_DIRS = ('defaults','files','handlers','meta','tasks','templates','vars','tests')

    def __init__(self, galaxy, name, src=None, version=None, scm=None, path=None):

        self._metadata = None
        self._install_info = None
        self._validate_certs = not galaxy.options.ignore_certs

        display.debug('Validate TLS certificates: %s' % self._validate_certs)

        self.options = galaxy.options
        self.galaxy  = galaxy

        self.name = name
        self.version = version
        self.src = src or name
        self.scm = scm

        if path is not None:
            if self.name not in path:
                path = os.path.join(path, self.name)
            self.path = path
        else:
            for role_path_dir in galaxy.roles_paths:
                role_path = os.path.join(role_path_dir, self.name)
                if os.path.exists(role_path):
                    self.path = role_path
                    break
            else:
                # use the first path by default
                self.path = os.path.join(galaxy.roles_paths[0], self.name)
                # create list of possible paths
                self.paths = [x for x in galaxy.roles_paths]
                self.paths = [os.path.join(x, self.name) for x in self.paths]

    def __eq__(self, other):
        return self.name == other.name

    @property
    def metadata(self):
        """
        Returns role metadata
        """
        if self._metadata is None:
            meta_path = os.path.join(self.path, self.META_MAIN)
            if os.path.isfile(meta_path):
                try:
                    f = open(meta_path, 'r')
                    self._metadata = yaml.safe_load(f)
                except:
                    display.vvvvv("Unable to load metadata for %s" % self.name)
                    return False
                finally:
                    f.close()

        return self._metadata

    @property
    def install_info(self):
        """
        Returns role install info
        """
        if self._install_info is None:

            info_path = os.path.join(self.path, self.META_INSTALL)
            if os.path.isfile(info_path):
                try:
                    f = open(info_path, 'r')
                    self._install_info = yaml.safe_load(f)
                except:
                    display.vvvvv("Unable to load Galaxy install info for %s" % self.name)
                    return False
                finally:
                    f.close()
        return self._install_info

    def _write_galaxy_install_info(self):
        """
        Writes a YAML-formatted file to the role's meta/ directory
        (named .galaxy_install_info) which contains some information
        we can use later for commands like 'list' and 'info'.
        """

        info = dict(
            version=self.version,
            install_date=datetime.datetime.utcnow().strftime("%c"),
        )
        if not os.path.exists(os.path.join(self.path, 'meta')):
            os.makedirs(os.path.join(self.path, 'meta'))
        info_path = os.path.join(self.path, self.META_INSTALL)
        with open(info_path, 'w+') as f:
            try:
                self._install_info = yaml.safe_dump(info, f)
            except:
                return False

        return True

    def remove(self):
        """
        Removes the specified role from the roles path.
        There is a sanity check to make sure there's a meta/main.yml file at this
        path so the user doesn't blow away random directories.
        """
        if self.metadata:
            try:
                rmtree(self.path)
                return True
            except:
                pass

        return False

    def fetch(self, role_data):
        """
        Downloads the archived role from github to a temp location
        """
        if role_data:

            # first grab the file and save it to a temp location
            if "github_user" in role_data and "github_repo" in role_data:
                archive_url = 'https://github.com/%s/%s/archive/%s.tar.gz' % (role_data["github_user"], role_data["github_repo"], self.version)
            else:
                archive_url = self.src

            display.display("- downloading role from %s" % archive_url)

            try:
                url_file = open_url(archive_url, validate_certs=self._validate_certs)
                temp_file = tempfile.NamedTemporaryFile(delete=False)
                data = url_file.read()
                while data:
                    temp_file.write(data)
                    data = url_file.read()
                temp_file.close()
                return temp_file.name
            except Exception as e:
                display.error("failed to download the file: %s" % str(e))

        return False

    def install(self):
        # the file is a tar, so open it that way and extract it
        # to the specified (or default) roles directory
        local_file = False

        if self.scm:
            # create tar file from scm url
            tmp_file = RoleRequirement.scm_archive_role(**self.spec)
        elif self.src:
            if  os.path.isfile(self.src):
                # installing a local tar.gz
                local_file = True
                tmp_file = self.src
            elif '://' in self.src:
                role_data = self.src
                tmp_file = self.fetch(role_data)
            else:
                api = GalaxyAPI(self.galaxy)
                role_data = api.lookup_role_by_name(self.src)
                if not role_data:
                    raise AnsibleError("- sorry, %s was not found on %s." % (self.src, api.api_server))

                if role_data.get('role_type') == 'CON':
                    # Container Enabled
                    display.warning("%s is a Container Enabled role and should only be installed using "
                                    "Ansible Container" % self.name)

                if role_data.get('role_type') == 'APP':
                    # Container Role
                    display.warning("%s is a Container App role and should only be installed using Ansible "
                                    "Container" % self.name)

                role_versions = api.fetch_role_related('versions', role_data['id'])
                if not self.version:
                    # convert the version names to LooseVersion objects
                    # and sort them to get the latest version. If there
                    # are no versions in the list, we'll grab the head
                    # of the master branch
                    if len(role_versions) > 0:
                        loose_versions = [LooseVersion(a.get('name',None)) for a in role_versions]
                        loose_versions.sort()
                        self.version = str(loose_versions[-1])
                    elif role_data.get('github_branch', None):
                        self.version = role_data['github_branch']
                    else:
                        self.version = 'master'
                elif self.version != 'master':
                    if role_versions and str(self.version) not in [a.get('name', None) for a in role_versions]:
                        raise AnsibleError("- the specified version (%s) of %s was not found in the list of available versions (%s)." % (self.version, self.name, role_versions))

                tmp_file = self.fetch(role_data)

        else:
           raise AnsibleError("No valid role data found")

        if tmp_file:

            display.debug("installing from %s" % tmp_file)

            if not tarfile.is_tarfile(tmp_file):
                raise AnsibleError("the file downloaded was not a tar.gz")
            else:
                if tmp_file.endswith('.gz'):
                    role_tar_file = tarfile.open(tmp_file, "r:gz")
                else:
                    role_tar_file = tarfile.open(tmp_file, "r")
                # verify the role's meta file
                meta_file = None
                members = role_tar_file.getmembers()
                # next find the metadata file
                for member in members:
                    if self.META_MAIN in member.name:
                        meta_file = member
                        break
                if not meta_file:
                    raise AnsibleError("this role does not appear to have a meta/main.yml file.")
                else:
                    try:
                        self._metadata = yaml.safe_load(role_tar_file.extractfile(meta_file))
                    except:
                        raise AnsibleError("this role does not appear to have a valid meta/main.yml file.")

                # we strip off the top-level directory for all of the files contained within
                # the tar file here, since the default is 'github_repo-target', and change it
                # to the specified role's name
                installed = False
                while not installed:
                    display.display("- extracting %s to %s" % (self.name, self.path))
                    try:
                        if os.path.exists(self.path):
                            if not os.path.isdir(self.path):
                                raise AnsibleError("the specified roles path exists and is not a directory.")
                            elif not getattr(self.options, "force", False):
                                raise AnsibleError("the specified role %s appears to already exist. Use --force to replace it." % self.name)
                            else:
                                # using --force, remove the old path
                                if not self.remove():
                                    raise AnsibleError("%s doesn't appear to contain a role.\n  please remove this directory manually if you really want to put the role here." % self.path)
                        else:
                            os.makedirs(self.path)

                        # now we do the actual extraction to the path
                        for member in members:
                            # we only extract files, and remove any relative path
                            # bits that might be in the file for security purposes
                            # and drop the leading directory, as mentioned above
                            if member.isreg() or member.issym():
                                parts = member.name.split(os.sep)[1:]
                                final_parts = []
                                for part in parts:
                                    if part != '..' and '~' not in part and '$' not in part:
                                        final_parts.append(part)
                                member.name = os.path.join(*final_parts)
                                role_tar_file.extract(member, self.path)

                        # write out the install info file for later use
                        self._write_galaxy_install_info()
                        installed = True
                    except OSError as e:
                        error = True
                        if e[0] == 13 and len(self.paths) > 1:
                            current = self.paths.index(self.path)
                            nextidx = current + 1
                            if len(self.paths) >= current:
                                self.path = self.paths[nextidx]
                                error = False
                        if error:
                            raise AnsibleError("Could not update files in %s: %s" % (self.path, str(e)))

                # return the parsed yaml metadata
                display.display("- %s was installed successfully" % self.name)
                if not local_file:
                    try:
                        os.unlink(tmp_file)
                    except (OSError,IOError) as e:
                        display.warning("Unable to remove tmp file (%s): %s" % (tmp_file, str(e)))
                return True

        return False

    @property
    def spec(self):
        """
        Returns role spec info
        {
           'scm': 'git',
           'src': 'http://git.example.com/repos/repo.git',
           'version': 'v1.0',
           'name': 'repo'
        }
        """
        return dict(scm=self.scm, src=self.src, version=self.version, name=self.name)
<commit_msg>For container enabled role, display warning only when not ANSIBLE_CONTAINER (#18717)<commit_after>########################################################################
#
# (C) 2015, Brian Coca <bcoca@ansible.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#
########################################################################

from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import datetime
import os
import tarfile
import tempfile
import yaml
from distutils.version import LooseVersion
from shutil import rmtree

import ansible.constants as C
from ansible.errors import AnsibleError
from ansible.module_utils.urls import open_url
from ansible.playbook.role.requirement import RoleRequirement
from ansible.galaxy.api import GalaxyAPI

try:
    from __main__ import display
except ImportError:
    from ansible.utils.display import Display
    display = Display()

class GalaxyRole(object):

    SUPPORTED_SCMS = set(['git', 'hg'])
    META_MAIN = os.path.join('meta', 'main.yml')
    META_INSTALL = os.path.join('meta', '.galaxy_install_info')
    ROLE_DIRS = ('defaults','files','handlers','meta','tasks','templates','vars','tests')

    def __init__(self, galaxy, name, src=None, version=None, scm=None, path=None):

        self._metadata = None
        self._install_info = None
        self._validate_certs = not galaxy.options.ignore_certs

        display.debug('Validate TLS certificates: %s' % self._validate_certs)

        self.options = galaxy.options
        self.galaxy  = galaxy

        self.name = name
        self.version = version
        self.src = src or name
        self.scm = scm

        if path is not None:
            if self.name not in path:
                path = os.path.join(path, self.name)
            self.path = path
        else:
            for role_path_dir in galaxy.roles_paths:
                role_path = os.path.join(role_path_dir, self.name)
                if os.path.exists(role_path):
                    self.path = role_path
                    break
            else:
                # use the first path by default
                self.path = os.path.join(galaxy.roles_paths[0], self.name)
                # create list of possible paths
                self.paths = [x for x in galaxy.roles_paths]
                self.paths = [os.path.join(x, self.name) for x in self.paths]

    def __eq__(self, other):
        return self.name == other.name

    @property
    def metadata(self):
        """
        Returns role metadata
        """
        if self._metadata is None:
            meta_path = os.path.join(self.path, self.META_MAIN)
            if os.path.isfile(meta_path):
                try:
                    f = open(meta_path, 'r')
                    self._metadata = yaml.safe_load(f)
                except:
                    display.vvvvv("Unable to load metadata for %s" % self.name)
                    return False
                finally:
                    f.close()

        return self._metadata

    @property
    def install_info(self):
        """
        Returns role install info
        """
        if self._install_info is None:

            info_path = os.path.join(self.path, self.META_INSTALL)
            if os.path.isfile(info_path):
                try:
                    f = open(info_path, 'r')
                    self._install_info = yaml.safe_load(f)
                except:
                    display.vvvvv("Unable to load Galaxy install info for %s" % self.name)
                    return False
                finally:
                    f.close()
        return self._install_info

    def _write_galaxy_install_info(self):
        """
        Writes a YAML-formatted file to the role's meta/ directory
        (named .galaxy_install_info) which contains some information
        we can use later for commands like 'list' and 'info'.
        """

        info = dict(
            version=self.version,
            install_date=datetime.datetime.utcnow().strftime("%c"),
        )
        if not os.path.exists(os.path.join(self.path, 'meta')):
            os.makedirs(os.path.join(self.path, 'meta'))
        info_path = os.path.join(self.path, self.META_INSTALL)
        with open(info_path, 'w+') as f:
            try:
                self._install_info = yaml.safe_dump(info, f)
            except:
                return False

        return True

    def remove(self):
        """
        Removes the specified role from the roles path.
        There is a sanity check to make sure there's a meta/main.yml file at this
        path so the user doesn't blow away random directories.
        """
        if self.metadata:
            try:
                rmtree(self.path)
                return True
            except:
                pass

        return False

    def fetch(self, role_data):
        """
        Downloads the archived role from github to a temp location
        """
        if role_data:

            # first grab the file and save it to a temp location
            if "github_user" in role_data and "github_repo" in role_data:
                archive_url = 'https://github.com/%s/%s/archive/%s.tar.gz' % (role_data["github_user"], role_data["github_repo"], self.version)
            else:
                archive_url = self.src

            display.display("- downloading role from %s" % archive_url)

            try:
                url_file = open_url(archive_url, validate_certs=self._validate_certs)
                temp_file = tempfile.NamedTemporaryFile(delete=False)
                data = url_file.read()
                while data:
                    temp_file.write(data)
                    data = url_file.read()
                temp_file.close()
                return temp_file.name
            except Exception as e:
                display.error("failed to download the file: %s" % str(e))

        return False

    def install(self):
        # the file is a tar, so open it that way and extract it
        # to the specified (or default) roles directory
        local_file = False

        if self.scm:
            # create tar file from scm url
            tmp_file = RoleRequirement.scm_archive_role(**self.spec)
        elif self.src:
            if  os.path.isfile(self.src):
                # installing a local tar.gz
                local_file = True
                tmp_file = self.src
            elif '://' in self.src:
                role_data = self.src
                tmp_file = self.fetch(role_data)
            else:
                api = GalaxyAPI(self.galaxy)
                role_data = api.lookup_role_by_name(self.src)
                if not role_data:
                    raise AnsibleError("- sorry, %s was not found on %s." % (self.src, api.api_server))

                if role_data.get('role_type') == 'CON' and not os.environ.get('ANSIBLE_CONTAINER'):
                    # Container Enabled, running outside of a container
                    display.warning("%s is a Container Enabled role and should only be installed using "
                                    "Ansible Container" % self.name)

                if role_data.get('role_type') == 'APP':
                    # Container Role
                    display.warning("%s is a Container App role and should only be installed using Ansible "
                                    "Container" % self.name)

                role_versions = api.fetch_role_related('versions', role_data['id'])
                if not self.version:
                    # convert the version names to LooseVersion objects
                    # and sort them to get the latest version. If there
                    # are no versions in the list, we'll grab the head
                    # of the master branch
                    if len(role_versions) > 0:
                        loose_versions = [LooseVersion(a.get('name',None)) for a in role_versions]
                        loose_versions.sort()
                        self.version = str(loose_versions[-1])
                    elif role_data.get('github_branch', None):
                        self.version = role_data['github_branch']
                    else:
                        self.version = 'master'
                elif self.version != 'master':
                    if role_versions and str(self.version) not in [a.get('name', None) for a in role_versions]:
                        raise AnsibleError("- the specified version (%s) of %s was not found in the list of available versions (%s)." % (self.version, self.name, role_versions))

                tmp_file = self.fetch(role_data)

        else:
           raise AnsibleError("No valid role data found")

        if tmp_file:

            display.debug("installing from %s" % tmp_file)

            if not tarfile.is_tarfile(tmp_file):
                raise AnsibleError("the file downloaded was not a tar.gz")
            else:
                if tmp_file.endswith('.gz'):
                    role_tar_file = tarfile.open(tmp_file, "r:gz")
                else:
                    role_tar_file = tarfile.open(tmp_file, "r")
                # verify the role's meta file
                meta_file = None
                members = role_tar_file.getmembers()
                # next find the metadata file
                for member in members:
                    if self.META_MAIN in member.name:
                        meta_file = member
                        break
                if not meta_file:
                    raise AnsibleError("this role does not appear to have a meta/main.yml file.")
                else:
                    try:
                        self._metadata = yaml.safe_load(role_tar_file.extractfile(meta_file))
                    except:
                        raise AnsibleError("this role does not appear to have a valid meta/main.yml file.")

                # we strip off the top-level directory for all of the files contained within
                # the tar file here, since the default is 'github_repo-target', and change it
                # to the specified role's name
                installed = False
                while not installed:
                    display.display("- extracting %s to %s" % (self.name, self.path))
                    try:
                        if os.path.exists(self.path):
                            if not os.path.isdir(self.path):
                                raise AnsibleError("the specified roles path exists and is not a directory.")
                            elif not getattr(self.options, "force", False):
                                raise AnsibleError("the specified role %s appears to already exist. Use --force to replace it." % self.name)
                            else:
                                # using --force, remove the old path
                                if not self.remove():
                                    raise AnsibleError("%s doesn't appear to contain a role.\n  please remove this directory manually if you really want to put the role here." % self.path)
                        else:
                            os.makedirs(self.path)

                        # now we do the actual extraction to the path
                        for member in members:
                            # we only extract files, and remove any relative path
                            # bits that might be in the file for security purposes
                            # and drop the leading directory, as mentioned above
                            if member.isreg() or member.issym():
                                parts = member.name.split(os.sep)[1:]
                                final_parts = []
                                for part in parts:
                                    if part != '..' and '~' not in part and '$' not in part:
                                        final_parts.append(part)
                                member.name = os.path.join(*final_parts)
                                role_tar_file.extract(member, self.path)

                        # write out the install info file for later use
                        self._write_galaxy_install_info()
                        installed = True
                    except OSError as e:
                        error = True
                        if e[0] == 13 and len(self.paths) > 1:
                            current = self.paths.index(self.path)
                            nextidx = current + 1
                            if len(self.paths) >= current:
                                self.path = self.paths[nextidx]
                                error = False
                        if error:
                            raise AnsibleError("Could not update files in %s: %s" % (self.path, str(e)))

                # return the parsed yaml metadata
                display.display("- %s was installed successfully" % self.name)
                if not local_file:
                    try:
                        os.unlink(tmp_file)
                    except (OSError,IOError) as e:
                        display.warning("Unable to remove tmp file (%s): %s" % (tmp_file, str(e)))
                return True

        return False

    @property
    def spec(self):
        """
        Returns role spec info
        {
           'scm': 'git',
           'src': 'http://git.example.com/repos/repo.git',
           'version': 'v1.0',
           'name': 'repo'
        }
        """
        return dict(scm=self.scm, src=self.src, version=self.version, name=self.name)
<|endoftext|>


<commit_before>import pytest
from textwrap import dedent
from sourcer import Grammar

def test_simple_words():
    g = Grammar(r'''
        ignore Space = @/[ \t]+/
        Word = @/[_a-zA-Z][_a-zA-Z0-9]*/
        start = Word*
    ''')

    result = g.parse('foo bar baz')
    assert result == ['foo', 'bar', 'baz']

def test_arithmetic_expressions():
    g = Grammar(r'''
        ignored Space = @/\s+/

        Int = @/\d+/ |> `int`
        Parens = '(' >> Expr << ')'

        Expr = OperatorPrecedence(
            Int | Parens,
            Prefix('+' | '-'),
            RightAssoc('^'),
            LeftAssoc('*' | '/'),
            LeftAssoc('+' | '-'),
        )
        start = Expr
    ''')

    # Define short names for the constructors.
    I, P = g.Infix, g.Prefix

    result = g.parse('1 + 2')
    assert result == I(1, '+', 2)

    result = g.parse('11 * (22 + 33) - 44 / 55')
    assert result == I(I(11, '*', I(22, '+', 33)), '-', I(44, '/', 55))

    result = g.parse('123 ^ 456')
    assert result == I(123, '^', 456)

    result = g.parse('12 * 34 ^ 56 ^ 78 - 90')
    assert result == I(I(12, '*', I(34, '^', I(56, '^', 78))), '-', 90)

    result = g.parse('---123')
    assert result == P('-', P('-', P('-', 123)))

    result = g.parse('+-12--34++56')
    assert result == I(I(P('+', P('-', 12)), '-', P('-', 34)), '+', P('+', 56))

def test_simple_json_grammar():
    g = Grammar(r'''
        `from ast import literal_eval`

        start = Value

        Value = Object | Array | String | Number | Keyword

        Object = "{" >> (Member // ",") << "}" |> `dict`

        Member = [String << ":", Value]

        Array = "[" >> (Value // ",") << "]"

        String = @/"(?:[^\\"]|\\.)*"/ |> `literal_eval`

        Number = @/-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?/ |> `float`

        Keyword = "true" >> `True` | "false" >> `False` | "null" >> `None`

        ignored Space = @/\s+/
    ''')

    result = g.parse('{"foo": "bar", "baz": true}')
    assert result == {'foo': 'bar', 'baz': True}

    result = g.parse('[12, -34, {"56": 78, "foo": null}]')
    assert result == [12, -34, {'56': 78, 'foo': None}]

def test_many_nested_parentheses():
    g = Grammar(r'start = ["(", start?, ")"]')

    depth = 1001
    text = ('(' * depth) + (')' * depth)
    result = g.parse(text)

    count = 0
    while result:
        assert isinstance(result, list)
        assert len(result) == 3
        assert result[0] == '('
        assert result[2] == ')'
        result = result[1]
        count += 1

    assert count == depth

def test_python_expressions():
    g = Grammar(r'''
        ```
        import ast
        ```

        Expr = (
            @/\d+/ |> `int`
            | @/true/ >> `True`
            | @/false/ >> `False`
            | @/null/ >> `None`
            | @/"([^"\\]|\\.)*"/ |> `ast.literal_eval`
        )
        Space = @/\s+/

        Start = Expr / Space
    ''')
    result = g.parse(r'12 null 34 false 56 "hello:\n\tworld" 78 true')
    assert result == [12, None, 34, False, 56, 'hello:\n\tworld', 78, True]

def test_postfix_operators():
    g = Grammar(r'''
        ignore Space = @/[ \t]+/
        Atom = @/[a-zA-Z]+/

        class ArgList {
            args: "(" >> (Expr / ",") << ")"
        }

        Expr = OperatorPrecedence(
            Atom,
            Postfix(ArgList),
            Postfix("?" | "*" | "+" | "!"),
            LeftAssoc("|"),
        )

        start = Expr
    ''')

    result = g.parse('foo(bar+, baz! | fiz)?')
    assert result == g.Postfix(g.Postfix('foo', g.ArgList([
        g.Postfix('bar', '+'),
        g.Infix(g.Postfix('baz', '!'), '|', 'fiz'),
    ])), '?')

def test_where_expressions():
    g = Grammar(r'''
        start = (Odd | Even)+

        class Odd {
            value: Int where `lambda x: x % 2`
        }

        class Even {
            value: Int where `lambda x: x % 2 == 0`
        }

        Int = @/\d+/ |> `int`

        ignore Space = @/[ \t]+/
    ''')
    result = g.parse('11 22 33 44')
    assert result == [g.Odd(11), g.Even(22), g.Odd(33), g.Even(44)]

def test_expect_and_expect_not_expressions():
    g = Grammar(r'''
        ignore Space = @/[ \t]+/
        Word = @/[_a-zA-Z][_a-zA-Z0-9]*/

        class Angry {
            value: Word << Expect("!")
        }

        class Calm {
            value: Word << ExpectNot("!" | "?")
        }

        class Confused {
            value: Word << Expect("?")
        }

        start = (Angry | Calm | Confused) // ("!" | "?" | "." | ";")
    ''')
    result = g.parse('foo! bar? baz? fiz; buz.')
    assert result == [
        g.Angry('foo'),
        g.Confused('bar'),
        g.Confused('baz'),
        g.Calm('fiz'),
        g.Calm('buz'),
    ]

def test_simple_data_dependent_class():
    g = Grammar(r'''
        class Element {
            open_tag: "<" >> Word << ">"
            content: Item*
            close_tag: "</" >> Word << ">" where `lambda x: x == open_tag`
        }

        class Text {
            content: @/[^<]+/
        }

        Item = Element | Text

        Word = @/[_a-zA-Z][_a-zA-Z0-9]*/

        start = Item+
    ''')
    result = g.parse('foo <bar>baz <zim>zam</zim> fiz</bar> buz')
    assert result == [
        g.Text('foo '),
        g.Element('bar', [
            g.Text('baz '),
            g.Element('zim', [g.Text('zam')], 'zim'),
            g.Text(' fiz'),
        ], 'bar'),
        g.Text(' buz'),
    ]

def test_simple_rule_with_parameter():
    g = Grammar(r'''
        ignored Space = @/[ \t]+/
        Name = @/[_a-zA-Z][_a-zA-Z0-9]*/
        Pair(x) = "(" >> [x << ",", x] << ")"

        class Range {
            start: Name << "to"
            stop: Name
        }

        start = Pair(Range | "ok")
    ''')
    result = g.parse('(foo to bar, fiz to buz)')
    assert result == [g.Range('foo', 'bar'), g.Range('fiz', 'buz')]

    result = g.parse('(ok, ok)')
    assert result == ['ok', 'ok']

def test_simple_class_with_parameters():
    g = Grammar(r'''
        ignored Space = @/[ \t]+/

        Name = @/[_a-zA-Z][_a-zA-Z0-9]*/
        Int = @/\d+/ |> `int`

        class Pair(A, B) {
            first: A << "&"
            second: B
        }

        Names = "[" >> (Name / ",") << "]"
        Ints = "[" >> (Int / ",") << "]"
        Start = Pair(Names, Ints)
    ''')
    result = g.parse('[foo, bar, baz] & [11, 22, 33]')
    assert result == g.Pair(['foo', 'bar', 'baz'], [11, 22, 33])

def test_simplified_indentation():
    g = Grammar(r'''
        ignore Space = @/[ \t]+/

        Indent = @/\n[ \t]*/

        MatchIndent(i) =>
            Indent where `lambda x: x == i`

        IncreaseIndent(i) =>
            Indent where `lambda x: len(x) > len(i)`

        Body(current_indent) =>
            let i = IncreaseIndent(current_indent) in
            Statement(i) // MatchIndent(i)

        Statement(current_indent) =>
            If(current_indent) | Print

        class If(current_indent) {
            test: "if" >> Name
            body: Body(current_indent)
        }

        class Print {
            name: "print" >> Name
        }

        Name = @/[a-zA-Z]+/
        Newline = @/[\r\n]+/

        Start = Opt(Newline) >> (Statement('') // Newline)
    ''')

    result = g.parse('print ok\nprint bye')
    assert result == [g.Print('ok'), g.Print('bye')]

    result = g.parse('if foo\n  print bar')
    assert result == [g.If('foo', [g.Print('bar')])]

    result = g.parse(dedent('''
        print ok
        if foo
            if bar
                print baz
                print fiz
            print buz
        print zim
    '''))
    assert result == [
        g.Print('ok'),
        g.If('foo', [
            g.If('bar', [
                g.Print('baz'),
                g.Print('fiz'),
            ]),
            g.Print('buz'),
        ]),
        g.Print('zim'),
    ]

def test_that_shadowing_is_not_allowed():
    with pytest.raises(Exception):
        Grammar(r'''
            Foo(Bar) => List(Bar)
            Bar = @/[a-z]+/
            Bar = @/\d+/
            Start = Foo(Baz) | Bar
        ''')
<commit_msg>Remove old test.<commit_after>import pytest
from textwrap import dedent
from sourcer import Grammar

def test_simple_words():
    g = Grammar(r'''
        ignore Space = @/[ \t]+/
        Word = @/[_a-zA-Z][_a-zA-Z0-9]*/
        start = Word*
    ''')

    result = g.parse('foo bar baz')
    assert result == ['foo', 'bar', 'baz']

def test_arithmetic_expressions():
    g = Grammar(r'''
        ignored Space = @/\s+/

        Int = @/\d+/ |> `int`
        Parens = '(' >> Expr << ')'

        Expr = OperatorPrecedence(
            Int | Parens,
            Prefix('+' | '-'),
            RightAssoc('^'),
            LeftAssoc('*' | '/'),
            LeftAssoc('+' | '-'),
        )
        start = Expr
    ''')

    # Define short names for the constructors.
    I, P = g.Infix, g.Prefix

    result = g.parse('1 + 2')
    assert result == I(1, '+', 2)

    result = g.parse('11 * (22 + 33) - 44 / 55')
    assert result == I(I(11, '*', I(22, '+', 33)), '-', I(44, '/', 55))

    result = g.parse('123 ^ 456')
    assert result == I(123, '^', 456)

    result = g.parse('12 * 34 ^ 56 ^ 78 - 90')
    assert result == I(I(12, '*', I(34, '^', I(56, '^', 78))), '-', 90)

    result = g.parse('---123')
    assert result == P('-', P('-', P('-', 123)))

    result = g.parse('+-12--34++56')
    assert result == I(I(P('+', P('-', 12)), '-', P('-', 34)), '+', P('+', 56))

def test_simple_json_grammar():
    g = Grammar(r'''
        `from ast import literal_eval`

        start = Value

        Value = Object | Array | String | Number | Keyword

        Object = "{" >> (Member // ",") << "}" |> `dict`

        Member = [String << ":", Value]

        Array = "[" >> (Value // ",") << "]"

        String = @/"(?:[^\\"]|\\.)*"/ |> `literal_eval`

        Number = @/-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?/ |> `float`

        Keyword = "true" >> `True` | "false" >> `False` | "null" >> `None`

        ignored Space = @/\s+/
    ''')

    result = g.parse('{"foo": "bar", "baz": true}')
    assert result == {'foo': 'bar', 'baz': True}

    result = g.parse('[12, -34, {"56": 78, "foo": null}]')
    assert result == [12, -34, {'56': 78, 'foo': None}]

def test_many_nested_parentheses():
    g = Grammar(r'start = ["(", start?, ")"]')

    depth = 1001
    text = ('(' * depth) + (')' * depth)
    result = g.parse(text)

    count = 0
    while result:
        assert isinstance(result, list)
        assert len(result) == 3
        assert result[0] == '('
        assert result[2] == ')'
        result = result[1]
        count += 1

    assert count == depth

def test_python_expressions():
    g = Grammar(r'''
        ```
        import ast
        ```

        Expr = (
            @/\d+/ |> `int`
            | @/true/ >> `True`
            | @/false/ >> `False`
            | @/null/ >> `None`
            | @/"([^"\\]|\\.)*"/ |> `ast.literal_eval`
        )
        Space = @/\s+/

        Start = Expr / Space
    ''')
    result = g.parse(r'12 null 34 false 56 "hello:\n\tworld" 78 true')
    assert result == [12, None, 34, False, 56, 'hello:\n\tworld', 78, True]

def test_postfix_operators():
    g = Grammar(r'''
        ignore Space = @/[ \t]+/
        Atom = @/[a-zA-Z]+/

        class ArgList {
            args: "(" >> (Expr / ",") << ")"
        }

        Expr = OperatorPrecedence(
            Atom,
            Postfix(ArgList),
            Postfix("?" | "*" | "+" | "!"),
            LeftAssoc("|"),
        )

        start = Expr
    ''')

    result = g.parse('foo(bar+, baz! | fiz)?')
    assert result == g.Postfix(g.Postfix('foo', g.ArgList([
        g.Postfix('bar', '+'),
        g.Infix(g.Postfix('baz', '!'), '|', 'fiz'),
    ])), '?')

def test_where_expressions():
    g = Grammar(r'''
        start = (Odd | Even)+

        class Odd {
            value: Int where `lambda x: x % 2`
        }

        class Even {
            value: Int where `lambda x: x % 2 == 0`
        }

        Int = @/\d+/ |> `int`

        ignore Space = @/[ \t]+/
    ''')
    result = g.parse('11 22 33 44')
    assert result == [g.Odd(11), g.Even(22), g.Odd(33), g.Even(44)]

def test_expect_and_expect_not_expressions():
    g = Grammar(r'''
        ignore Space = @/[ \t]+/
        Word = @/[_a-zA-Z][_a-zA-Z0-9]*/

        class Angry {
            value: Word << Expect("!")
        }

        class Calm {
            value: Word << ExpectNot("!" | "?")
        }

        class Confused {
            value: Word << Expect("?")
        }

        start = (Angry | Calm | Confused) // ("!" | "?" | "." | ";")
    ''')
    result = g.parse('foo! bar? baz? fiz; buz.')
    assert result == [
        g.Angry('foo'),
        g.Confused('bar'),
        g.Confused('baz'),
        g.Calm('fiz'),
        g.Calm('buz'),
    ]

def test_simple_data_dependent_class():
    g = Grammar(r'''
        class Element {
            open_tag: "<" >> Word << ">"
            content: Item*
            close_tag: "</" >> Word << ">" where `lambda x: x == open_tag`
        }

        class Text {
            content: @/[^<]+/
        }

        Item = Element | Text

        Word = @/[_a-zA-Z][_a-zA-Z0-9]*/

        start = Item+
    ''')
    result = g.parse('foo <bar>baz <zim>zam</zim> fiz</bar> buz')
    assert result == [
        g.Text('foo '),
        g.Element('bar', [
            g.Text('baz '),
            g.Element('zim', [g.Text('zam')], 'zim'),
            g.Text(' fiz'),
        ], 'bar'),
        g.Text(' buz'),
    ]

def test_simple_rule_with_parameter():
    g = Grammar(r'''
        ignored Space = @/[ \t]+/
        Name = @/[_a-zA-Z][_a-zA-Z0-9]*/
        Pair(x) = "(" >> [x << ",", x] << ")"

        class Range {
            start: Name << "to"
            stop: Name
        }

        start = Pair(Range | "ok")
    ''')
    result = g.parse('(foo to bar, fiz to buz)')
    assert result == [g.Range('foo', 'bar'), g.Range('fiz', 'buz')]

    result = g.parse('(ok, ok)')
    assert result == ['ok', 'ok']

def test_simple_class_with_parameters():
    g = Grammar(r'''
        ignored Space = @/[ \t]+/

        Name = @/[_a-zA-Z][_a-zA-Z0-9]*/
        Int = @/\d+/ |> `int`

        class Pair(A, B) {
            first: A << "&"
            second: B
        }

        Names = "[" >> (Name / ",") << "]"
        Ints = "[" >> (Int / ",") << "]"
        Start = Pair(Names, Ints)
    ''')
    result = g.parse('[foo, bar, baz] & [11, 22, 33]')
    assert result == g.Pair(['foo', 'bar', 'baz'], [11, 22, 33])

def test_simplified_indentation():
    g = Grammar(r'''
        ignore Space = @/[ \t]+/

        Indent = @/\n[ \t]*/

        MatchIndent(i) =>
            Indent where `lambda x: x == i`

        IncreaseIndent(i) =>
            Indent where `lambda x: len(x) > len(i)`

        Body(current_indent) =>
            let i = IncreaseIndent(current_indent) in
            Statement(i) // MatchIndent(i)

        Statement(current_indent) =>
            If(current_indent) | Print

        class If(current_indent) {
            test: "if" >> Name
            body: Body(current_indent)
        }

        class Print {
            name: "print" >> Name
        }

        Name = @/[a-zA-Z]+/
        Newline = @/[\r\n]+/

        Start = Opt(Newline) >> (Statement('') // Newline)
    ''')

    result = g.parse('print ok\nprint bye')
    assert result == [g.Print('ok'), g.Print('bye')]

    result = g.parse('if foo\n  print bar')
    assert result == [g.If('foo', [g.Print('bar')])]

    result = g.parse(dedent('''
        print ok
        if foo
            if bar
                print baz
                print fiz
            print buz
        print zim
    '''))
    assert result == [
        g.Print('ok'),
        g.If('foo', [
            g.If('bar', [
                g.Print('baz'),
                g.Print('fiz'),
            ]),
            g.Print('buz'),
        ]),
        g.Print('zim'),
    ]
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
test_text_data_cards
----------------------------------

Tests for `text_data_cards` module.
"""

import pytest

from text_data_cards import text_data_cards

# DataCard
@pytest.fixture()
def tc():
    return text_data_cards.DataCard('(I3, F5.4, F8.5, I2, F8.5, F8.5, A8, '
                                    'A5, A5)',
                                    ['IP', 'SKIN', 'RESIS', 'IX', 'REACT',
                                     'DIAM', 'T', 'FIXED', 'RIGHT'],
                                    fixed_fields=(7, 8))

@pytest.fixture()
def tt_nomatch():
    #      I3 F5.4 F8.5    I2F8.5    F8.5    A8      A4
    #      1231234512345678121234567812345678123456781234512345
    tt = ['  3  0.0   .1357 0   .3959    1.18TESTTEXTFIXEDWRONG']
    return tt

@pytest.fixture()
def tt_match():
    #      I3 F5.4 F8.5    I2F8.5    F8.5    A8      A4
    #      1231234512345678121234567812345678123456781234512345
    tt = ['  3  0.0   .1357 0   .3959    1.18TESTTEXTFIXEDRIGHT']
    return tt

def test_DataCard_nomatch(tc, tt_nomatch):
    assert tc.match(tt_nomatch) is False
    with pytest.raises(ValueError):
        tc.read(tt_nomatch)

def test_DataCard_match(tc, tt_match):
    assert tc.match(tt_match) is True
    tc.read(tt_match)
    assert tc.data['IP'] == 3
    assert tc.data['SKIN'] == 0.0
    assert tc.data['RESIS'] == 0.1357
    assert tc.data['IX'] == 0
    assert tc.data['REACT'] == 0.3959
    assert tc.data['DIAM'] == 1.18
    assert tc.data['T'] == 'TESTTEXT'
    assert tc.data['FIXED'] == 'FIXED'
    assert tc.data['RIGHT'] == 'RIGHT'

# DataCardFixedText
@pytest.fixture()
def tc_fixed_text():
    return text_data_cards.DataCardFixedText('SOME FIXED TEXT')

@pytest.fixture()
def tt_fixed_text_nomatch():
    return ['SOME WRONG TEXT']

@pytest.fixture()
def tt_fixed_text_match():
    return ['SOME FIXED TEXT']

def test_DataCardFixedText_match(tc_fixed_text, tt_fixed_text_match):
    assert tc_fixed_text.match(tt_fixed_text_match) is True

def test_DataCardFixedText_nomatch(tc_fixed_text, tt_fixed_text_nomatch):
    assert tc_fixed_text.match(tt_fixed_text_nomatch) is False

# DataCardStack
# DataCardRepeat
#
<commit_msg>Added tests for remaining DataCard classes.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
test_text_data_cards
----------------------------------

Tests for `text_data_cards` module.
"""

import pytest

from text_data_cards import text_data_cards

# DataCard
@pytest.fixture()
def tc():
    return text_data_cards.DataCard('(I3, F5.4, F8.5, I2, F8.5, F8.5, A8, '
                                    'A5, A5)',
                                    ['IP', 'SKIN', 'RESIS', 'IX', 'REACT',
                                     'DIAM', 'T', 'FIXED', 'RIGHT'],
                                    fixed_fields=(7, 8))

@pytest.fixture()
def tt_nomatch():
    #      I3 F5.4 F8.5    I2F8.5    F8.5    A8      A4
    #      1231234512345678121234567812345678123456781234512345
    tt = ['  3  0.0   .1357 0   .3959    1.18TESTTEXTFIXEDWRONG']
    return tt

@pytest.fixture()
def tt_match():
    #      I3 F5.4 F8.5    I2F8.5    F8.5    A8      A4
    #      1231234512345678121234567812345678123456781234512345
    tt = ['  3  0.0   .1357 0   .3959    1.18TESTTEXTFIXEDRIGHT']
    return tt

def test_DataCard_nomatch(tc, tt_nomatch):
    assert tc.match(tt_nomatch) is False
    with pytest.raises(ValueError):
        tc.read(tt_nomatch)

def test_DataCard_match(tc, tt_match):
    assert tc.match(tt_match) is True
    tc.read(tt_match)
    assert tc.data['IP'] == 3
    assert tc.data['SKIN'] == 0.0
    assert tc.data['RESIS'] == 0.1357
    assert tc.data['IX'] == 0
    assert tc.data['REACT'] == 0.3959
    assert tc.data['DIAM'] == 1.18
    assert tc.data['T'] == 'TESTTEXT'
    assert tc.data['FIXED'] == 'FIXED'
    assert tc.data['RIGHT'] == 'RIGHT'

# DataCardFixedText
@pytest.fixture()
def tc_fixed_text():
    return text_data_cards.DataCardFixedText('SOME FIXED TEXT')

@pytest.fixture()
def tt_fixed_text_nomatch():
    return ['SOME WRONG TEXT']

@pytest.fixture()
def tt_fixed_text_match():
    return ['SOME FIXED TEXT']

def test_DataCardFixedText_match(tc_fixed_text, tt_fixed_text_match):
    assert tc_fixed_text.match(tt_fixed_text_match) is True

def test_DataCardFixedText_nomatch(tc_fixed_text, tt_fixed_text_nomatch):
    assert tc_fixed_text.match(tt_fixed_text_nomatch) is False

# DataCardStack
@pytest.fixture()
def tc_stack():
    return text_data_cards.DataCardStack([tc(), tc_fixed_text()])

@pytest.fixture()
def tt_stack_match():
    return tt_match() + tt_fixed_text_match()

@pytest.fixture()
def tt_stack_nomatch1():
    return tt_nomatch() + tt_fixed_text_match()

@pytest.fixture()
def tt_stack_nomatch2():
    return tt_match() + tt_fixed_text_nomatch()

def test_DataCardStack_match(tc_stack, tt_stack_match):
    test_DataCard_match(tc_stack, tt_stack_match)

def test_DataCardStack_nomatch1(tc_stack, tt_stack_nomatch1):
    test_DataCard_nomatch(tc_stack, tt_stack_nomatch1)

def test_DataCardStack_nomatch2(tc_stack, tt_stack_nomatch2):
    test_DataCard_nomatch(tc_stack, tt_stack_nomatch2)

# DataCardRepeat
@pytest.fixture()
def tc_repeat():
    return text_data_cards.DataCardRepeat(tc(), tc_fixed_text())

@pytest.fixture()
def tt_repeat_match():
    return tt_match() + tt_match() + tt_fixed_text_match()

@pytest.fixture()
def tt_repeat_nomatch1():
    return tt_nomatch() + tt_match() + tt_fixed_text_match()

@pytest.fixture()
def tt_repeat_nomatch2():
    return tt_match() + tt_nomatch() + tt_fixed_text_match()

@pytest.fixture()
def tt_repeat_nomatch3():
    return tt_match() + tt_match() + tt_fixed_text_nomatch()

def test_DataCardRepeat_match(tc_repeat, tt_repeat_match):
    assert tc_repeat.match(tt_repeat_match) is True
    tc_repeat.read(tt_repeat_match)
    for d in tc_repeat.data:
        assert d['IP'] == 3
        assert d['SKIN'] == 0.0
        assert d['RESIS'] == 0.1357
        assert d['IX'] == 0
        assert d['REACT'] == 0.3959
        assert d['DIAM'] == 1.18
        assert d['T'] == 'TESTTEXT'
        assert d['FIXED'] == 'FIXED'
        assert d['RIGHT'] == 'RIGHT'

def test_DataCardRepeat_nomatch1(tc_repeat, tt_repeat_nomatch1):
    test_DataCard_nomatch(tc_repeat, tt_repeat_nomatch1)

def test_DataCardRepeat_nomatch2(tc_repeat, tt_repeat_nomatch2):
    test_DataCard_nomatch(tc_repeat, tt_repeat_nomatch2)

def test_DataCardFixedText_nomatch3(tc_repeat, tt_repeat_nomatch2):
    test_DataCard_nomatch(tc_repeat, tt_repeat_nomatch2)
<|endoftext|>


<commit_before>#!/usr/bin/env python
'''
PEXPECT LICENSE

    This license is approved by the OSI and FSF as GPL-compatible.
        http://opensource.org/licenses/isc-license.txt

    Copyright (c) 2012, Noah Spurrier <noah@noah.org>
    PERMISSION TO USE, COPY, MODIFY, AND/OR DISTRIBUTE THIS SOFTWARE FOR ANY
    PURPOSE WITH OR WITHOUT FEE IS HEREBY GRANTED, PROVIDED THAT THE ABOVE
    COPYRIGHT NOTICE AND THIS PERMISSION NOTICE APPEAR IN ALL COPIES.
    THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
    WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
    MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
    ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
    WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
    ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

'''
from __future__ import with_statement  # bring 'with' stmt to py25
import pexpect
import unittest
import PexpectTestCase
import sys

class Exp_TimeoutTestCase(PexpectTestCase.PexpectTestCase):
    def test_matches_exp_timeout (self):
        '''This tests that we can raise and catch TIMEOUT.
        '''
        try:
            raise pexpect.TIMEOUT("TIMEOUT match test")
        except pexpect.TIMEOUT:
            pass
            #print "Correctly caught TIMEOUT when raising TIMEOUT."
        else:
            self.fail('TIMEOUT not caught by an except TIMEOUT clause.')

    def test_pattern_printout (self):
        '''Verify that a TIMEOUT returns the proper patterns it is trying to match against.
        Make sure it is returning the pattern from the correct call.'''
        try:
            p = pexpect.spawn('cat')
            p.sendline('Hello')
            p.expect('Hello')
            p.expect('Goodbye',timeout=5)
        except pexpect.TIMEOUT:
            assert p.match_index == None
        else:
            self.fail("Did not generate a TIMEOUT exception.")

    def test_exp_timeout_notThrown (self):
        '''Verify that a TIMEOUT is not thrown when we match what we expect.'''
        try:
            p = pexpect.spawn('cat')
            p.sendline('Hello')
            p.expect('Hello')
        except pexpect.TIMEOUT:
            self.fail("TIMEOUT caught when it shouldn't be raised because we match the proper pattern.")

    def test_stacktraceMunging (self):
        '''Verify that the stack trace returned with a TIMEOUT instance does not contain references to pexpect.'''
        try:
            p = pexpect.spawn('cat')
            p.sendline('Hello')
            p.expect('Goodbye',timeout=5)
        except pexpect.TIMEOUT:
            err = sys.exc_info()[1]
            if err.get_trace().count("pexpect.py") != 0:
                self.fail("The TIMEOUT get_trace() referenced pexpect.py. "
                    "It should only reference the caller.\n" + err.get_trace())

    def test_correctStackTrace (self):
        '''Verify that the stack trace returned with a TIMEOUT instance correctly handles function calls.'''
        def nestedFunction (spawnInstance):
            spawnInstance.expect("junk", timeout=3)

        try:
            p = pexpect.spawn('cat')
            p.sendline('Hello')
            nestedFunction(p)
        except pexpect.TIMEOUT:
            err = sys.exc_info()[1]
            if err.get_trace().count("nestedFunction") == 0:
                self.fail("The TIMEOUT get_trace() did not show the call "
                    "to the nestedFunction function.\n" + str(err) + "\n"
                    + err.get_trace())

if __name__ == '__main__':
    unittest.main()

suite = unittest.makeSuite(Exp_TimeoutTestCase,'test')
<commit_msg>Correct test for stack trace munging<commit_after>#!/usr/bin/env python
'''
PEXPECT LICENSE

    This license is approved by the OSI and FSF as GPL-compatible.
        http://opensource.org/licenses/isc-license.txt

    Copyright (c) 2012, Noah Spurrier <noah@noah.org>
    PERMISSION TO USE, COPY, MODIFY, AND/OR DISTRIBUTE THIS SOFTWARE FOR ANY
    PURPOSE WITH OR WITHOUT FEE IS HEREBY GRANTED, PROVIDED THAT THE ABOVE
    COPYRIGHT NOTICE AND THIS PERMISSION NOTICE APPEAR IN ALL COPIES.
    THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
    WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
    MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
    ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
    WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
    ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

'''
from __future__ import with_statement  # bring 'with' stmt to py25
import pexpect
import unittest
import PexpectTestCase
import sys

class Exp_TimeoutTestCase(PexpectTestCase.PexpectTestCase):
    def test_matches_exp_timeout (self):
        '''This tests that we can raise and catch TIMEOUT.
        '''
        try:
            raise pexpect.TIMEOUT("TIMEOUT match test")
        except pexpect.TIMEOUT:
            pass
            #print "Correctly caught TIMEOUT when raising TIMEOUT."
        else:
            self.fail('TIMEOUT not caught by an except TIMEOUT clause.')

    def test_pattern_printout (self):
        '''Verify that a TIMEOUT returns the proper patterns it is trying to match against.
        Make sure it is returning the pattern from the correct call.'''
        try:
            p = pexpect.spawn('cat')
            p.sendline('Hello')
            p.expect('Hello')
            p.expect('Goodbye',timeout=5)
        except pexpect.TIMEOUT:
            assert p.match_index == None
        else:
            self.fail("Did not generate a TIMEOUT exception.")

    def test_exp_timeout_notThrown (self):
        '''Verify that a TIMEOUT is not thrown when we match what we expect.'''
        try:
            p = pexpect.spawn('cat')
            p.sendline('Hello')
            p.expect('Hello')
        except pexpect.TIMEOUT:
            self.fail("TIMEOUT caught when it shouldn't be raised because we match the proper pattern.")

    def test_stacktraceMunging (self):
        '''Verify that the stack trace returned with a TIMEOUT instance does not contain references to pexpect.'''
        try:
            p = pexpect.spawn('cat')
            p.sendline('Hello')
            p.expect('Goodbye',timeout=5)
        except pexpect.TIMEOUT:
            err = sys.exc_info()[1]
            if err.get_trace().count("pexpect/__init__.py") != 0:
                self.fail("The TIMEOUT get_trace() referenced pexpect.py. "
                    "It should only reference the caller.\n" + err.get_trace())

    def test_correctStackTrace (self):
        '''Verify that the stack trace returned with a TIMEOUT instance correctly handles function calls.'''
        def nestedFunction (spawnInstance):
            spawnInstance.expect("junk", timeout=3)

        try:
            p = pexpect.spawn('cat')
            p.sendline('Hello')
            nestedFunction(p)
        except pexpect.TIMEOUT:
            err = sys.exc_info()[1]
            if err.get_trace().count("nestedFunction") == 0:
                self.fail("The TIMEOUT get_trace() did not show the call "
                    "to the nestedFunction function.\n" + str(err) + "\n"
                    + err.get_trace())

if __name__ == '__main__':
    unittest.main()

suite = unittest.makeSuite(Exp_TimeoutTestCase,'test')
<|endoftext|>


<commit_before>import unittest

import numpy as np
import sympy as sym

from .. boundary_value_problems import SymbolicBoundaryValueProblem
from .. orthogonal_polynomials import OrthogonalPolynomialSolver
from .. import solutions

class SolowModel(unittest.TestCase):

    @staticmethod
    def steady_state(g, n, s, alpha, delta, sigma):
        """Steady state value for capital (per unit effective labor)."""
        return (s / (g + n + delta))**(1 / (1 - alpha))

    def setUp(self):
        """Set up a Solow model to solve."""
        # define some variables
        t, k, c = sym.symbols('t, k, c')

        # define some parameters
        alpha, sigma = sym.symbols('alpha, sigma')
        rho, theta = sym.symbols('rho, theta')
        g, n, s, delta = sym.symbols('g, n, s, delta')

        # intensive output has the Cobb-Douglas form
        y = k**alpha

        # define the equation of motion for capital
        k_dot = s * y - (g + n + delta) * k
        rhs = {k: k_dot}

        # set some randomly generated parameters
        self.params = {'g': np.random.uniform(),
                       's': np.random.uniform(),
                       'n': np.random.uniform(),
                       'alpha': np.random.uniform(),
                       'sigma': 1.0,
                       'delta': np.random.uniform()}

        # specify some boundary conditions
        kstar = self.steady_state(**self.params)
        if np.random.uniform() < 0.5:
            k0 = 0.5 * kstar
        else:
            k0 = 2.0 * kstar
        bcs = {'lower': [k - k0], 'upper': None}

        # set the model instance
        self.model = SymbolicBoundaryValueProblem(dependent_vars=[k],
                                                  independent_var=t,
                                                  rhs=rhs,
                                                  boundary_conditions=bcs,
                                                  params=self.params)

        # set the solver instance
        self.solver = OrthogonalPolynomialSolver(self.model)

        # set the domain
        self.domain = [0, 100]

        # set an initial guess
        ts = np.linspace(self.domain[0], self.domain[1], 1000)
        ks = kstar - (kstar - k0) * np.exp(-ts)
        initial_guess = np.polynomial.Chebyshev.fit(ts, ks, 50, self.domain)
        self.initial_coefs = {k: initial_guess.coef}

    def test_chebyshev_collocation(self):
        """Test collocation solver using Chebyshev polynomials for basis."""
        # compute the solution
        self.solver.solve(kind="Chebyshev",
                          coefs_dict=self.initial_coefs,
                          domain=self.domain)
        solution = solutions.Solution(self.solver)

        # check that solver terminated successfully
        self.assertTrue(solution.solver.result.success, msg="Solver failed!")

        # compute the residuals
        solution.interpolation_knots = np.linspace(self.domain[0],
                                                   self.domain[1],
                                                   1000)
        residuals = solution.residuals.values

        # check that residuals are close to zero on average
        mesg = "Chebyshev residuals:\n{}\n\nDictionary of model params: {}"
        self.assertTrue(np.mean(residuals) < 1e-6,
                        msg=mesg.format(residuals, self.params))

    def test_legendre_collocation(self):
        """Test collocation solver using Legendre polynomials for basis."""
        self.solver.solve(kind="Legendre",
                          coefs_dict=self.initial_coefs,
                          domain=self.domain)
        solution = solutions.Solution(self.solver)

        # check that solver terminated successfully
        self.assertTrue(solution.solver.result.success, msg="Solver failed!")

        # compute the residuals
        solution.interpolation_knots = np.linspace(self.domain[0],
                                                   self.domain[1],
                                                   1000)
        residuals = solution.residuals.values

        # check that residuals are all close to zero
        mesg = "Legendre residuals:\n{}\n\nDictionary of model params: {}"
        self.assertTrue(np.mean(residuals) < 1e-6,
                        msg=mesg.format(residuals, self.params))
<commit_msg>Added some additional tests.<commit_after>import unittest

import numpy as np
import sympy as sym

from .. boundary_value_problems import SymbolicBoundaryValueProblem
from .. orthogonal_polynomials import OrthogonalPolynomialSolver
from .. import solutions

class SolowModel(unittest.TestCase):

    @staticmethod
    def analytic_solution(k0, t, g, n, s, alpha, delta, sigma=1.0):
        """Analytic solution for model with Cobb-Douglas production."""
        # lambda governs the speed of convergence
        lmbda = (g + n + delta) * (1 - alpha)

        # analytic solution for Solow model at time t
        ks = (((s / (g + n + delta)) * (1 - np.exp(-lmbda * t)) +
               k0**(1 - alpha) * np.exp(-lmbda * t))**(1 / (1 - alpha)))
        return ks

    @staticmethod
    def steady_state(g, n, s, alpha, delta, sigma=1.0):
        """Steady state value for capital (per unit effective labor)."""
        return (s / (g + n + delta))**(1 / (1 - alpha))

    def setUp(self):
        """Set up a Solow model to solve."""
        # define some variables
        t, k, c = sym.symbols('t, k, c')

        # define some parameters
        alpha, sigma = sym.symbols('alpha, sigma')
        rho, theta = sym.symbols('rho, theta')
        g, n, s, delta = sym.symbols('g, n, s, delta')

        # intensive output has the Cobb-Douglas form
        y = k**alpha

        # define the equation of motion for capital
        k_dot = s * y - (g + n + delta) * k
        rhs = {k: k_dot}

        # set some randomly generated parameters
        self.params = {'g': np.random.uniform(),
                       's': np.random.uniform(),
                       'n': np.random.uniform(),
                       'alpha': np.random.uniform(),
                       'sigma': 1.0,
                       'delta': np.random.uniform()}

        # specify some boundary conditions
        kstar = self.steady_state(**self.params)
        if np.random.uniform() < 0.5:
            self.k0 = 0.5 * kstar
        else:
            self.k0 = 2.0 * kstar
        bcs = {'lower': [k - self.k0], 'upper': None}

        # set the model instance
        self.model = SymbolicBoundaryValueProblem(dependent_vars=[k],
                                                  independent_var=t,
                                                  rhs=rhs,
                                                  boundary_conditions=bcs,
                                                  params=self.params)

        # set the solver instance
        self.solver = OrthogonalPolynomialSolver(self.model)

        # set the domain
        self.domain = [0, 100]

        # set an initial guess
        ts = np.linspace(self.domain[0], self.domain[1], 1000)
        ks = kstar - (kstar - self.k0) * np.exp(-ts)
        initial_guess = np.polynomial.Chebyshev.fit(ts, ks, 50, self.domain)
        self.initial_coefs = {k: initial_guess.coef}

    def test_chebyshev_collocation(self):
        """Test collocation solver using Chebyshev polynomials for basis."""
        # compute the solution
        self.solver.solve(kind="Chebyshev",
                          coefs_dict=self.initial_coefs,
                          domain=self.domain)
        solution = solutions.Solution(self.solver)

        # check that solver terminated successfully
        self.assertTrue(solution.solver.result.success, msg="Solver failed!")

        # compute the residuals
        solution.interpolation_knots = np.linspace(self.domain[0],
                                                   self.domain[1],
                                                   1000)
        residuals = solution.residuals.values

        # check that residuals are close to zero on average
        mesg = "Chebyshev residuals:\n{}\n\nDictionary of model params: {}"
        self.assertTrue(np.mean(residuals) < 1e-6,
                        msg=mesg.format(residuals, self.params))

        # check that the numerical and analytic solutions are close
        numeric_soln = solution.solution.ix[:, 0]
        analytic_soln = self.analytic_solution(self.k0,
                                               solution.interpolation_knots,
                                               **self.params)
        self.assertTrue(np.mean(numeric_soln - analytic_soln) < 1e-6)

    def test_legendre_collocation(self):
        """Test collocation solver using Legendre polynomials for basis."""
        self.solver.solve(kind="Legendre",
                          coefs_dict=self.initial_coefs,
                          domain=self.domain)
        solution = solutions.Solution(self.solver)

        # check that solver terminated successfully
        self.assertTrue(solution.solver.result.success, msg="Solver failed!")

        # compute the residuals
        solution.interpolation_knots = np.linspace(self.domain[0],
                                                   self.domain[1],
                                                   1000)
        residuals = solution.residuals.values

        # check that residuals are all close to zero
        mesg = "Legendre residuals:\n{}\n\nDictionary of model params: {}"
        self.assertTrue(np.mean(residuals) < 1e-6,
                        msg=mesg.format(residuals, self.params))

        # check that the numerical and analytic solutions are close
        numeric_soln = solution.solution.ix[:, 0]
        analytic_soln = self.analytic_solution(self.k0,
                                               solution.interpolation_knots,
                                               **self.params)
        self.assertTrue(np.mean(numeric_soln - analytic_soln) < 1e-6)
<|endoftext|>


<commit_before>import pytest
import os
from slack_today_i_did.bot_file import TodayIDidBot

MOCK_PERSON = 'dave'
MOCK_CHANNEL = '#durp'
MOCK_START_SESSION_TEXT = 'start-session'
MOCK_TEST_FILE = '.testdata_sessions'

def test_session(mocker):
    mocked_channel_message = mocker.patch.object(TodayIDidBot, 'send_channel_message')
    mocker.patch.object(TodayIDidBot, 'was_directed_at_me', return_value=True)
    mocker.patch.object(TodayIDidBot, 'user_name_from_id', return_value=MOCK_PERSON)
    mocker.patch.object(TodayIDidBot, 'connected_user', return_value=MOCK_PERSON)

    bot = TodayIDidBot('', rollbar_token='', elm_repo=None)

    bot.parse_direct_message({
        'user': MOCK_PERSON,
        'channel': MOCK_CHANNEL,
        'text': MOCK_START_SESSION_TEXT
        })

    assert 'Started a session for you' in mocked_channel_message.call_args[0]
    assert mocked_channel_message.call_count == 1
<commit_msg>set last sender<commit_after>import pytest
import os
from slack_today_i_did.bot_file import TodayIDidBot

MOCK_PERSON = 'dave'
MOCK_CHANNEL = '#durp'
MOCK_START_SESSION_TEXT = 'start-session'
MOCK_TEST_FILE = '.testdata_sessions'

def test_session(mocker):
    mocked_channel_message = mocker.patch.object(TodayIDidBot, 'send_channel_message')
    mocker.patch.object(TodayIDidBot, 'was_directed_at_me', return_value=True)
    mocker.patch.object(TodayIDidBot, 'user_name_from_id', return_value=MOCK_PERSON)
    mocker.patch.object(TodayIDidBot, 'connected_user', return_value='TodayIDidBot')

    bot = TodayIDidBot('', rollbar_token='', elm_repo=None)
    bot._last_sender = MOCK_PERSON

    bot.parse_direct_message({
        'user': MOCK_PERSON,
        'channel': MOCK_CHANNEL,
        'text': MOCK_START_SESSION_TEXT
        })

    assert MOCK_CHANNEL == mocked_channel_message.call_args[0]
    assert 'Started a session for you' in mocked_channel_message.call_args[1]
    assert mocked_channel_message.call_count == 1
<|endoftext|>


<commit_before>from ..utils import Scraper
from bs4 import BeautifulSoup
from collections import OrderedDict
import json
import os
import requests
import datetime
now = datetime.datetime.now()

class UTMCalendar:

    host1 = 'http://m.utm.utoronto.ca/importantDates.php?mode=full&session={0}5&header='
    host2 = 'http://m.utm.utoronto.ca/importantDates.php?mode=full&session={0}9&header='
    sessionLinks = [host1, host2]

    @staticmethod
    def scrape(location='.', year=None):
        year = year or now.year

        calendar = OrderedDict()
        Scraper.logger.info('UTMCalendar initialized.')
        for link in UTMCalendar.sessionLinks:
            html = Scraper.get(link.format(year))
            soup = BeautifulSoup(html, 'html.parser')
            content = soup.find('div', class_='content')
            dates = content.find_all('div', class_='title')
            i = 0
            currentDate = dates[i]
            while(i<len(dates)):
                date = dates[i].text
                events = []
                while (currentDate == dates[i]):
                    info = dates[i].find_next('div', class_='info')
                    description = info.text
                    eventStartEnd = date.split('-')
                    eventStart = eventStartEnd[0].strip()
                    if len(eventStartEnd)>1:
                        eventEnd = eventStartEnd[1].strip()
                    else:
                        eventEnd = eventStart

                    events.append(OrderedDict([
                            ('end_date', eventEnd),
                            ('campus', 'UTM'),
                            ('description', description)
                        ]))
                    i+=1
                    if(i>=len(dates)):
                        break;
                calendar[date] = OrderedDict([
                        ('date', eventStart),
                        ('session', "Summer"),
                        ('events', events)
                    ])
                if(i<len(dates)):
                    currentDate = dates[i]

        for date, info in calendar.items():
            Scraper.save_json(info, location, date)

        Scraper.logger.info('UTMCalendar completed.')
        return calendar<commit_msg>fixed important dates session bug<commit_after>from ..utils import Scraper
from bs4 import BeautifulSoup
from collections import OrderedDict
import json
import os
import requests
import datetime
now = datetime.datetime.now()

class UTMCalendar:
    '''Scraper for Important dates from UTM calendar found at https://www.utm.utoronto.ca/registrar/important-dates
        '''

    summerLink = 'http://m.utm.utoronto.ca/importantDates.php?mode=full&session={0}5&header='
    fallLink = 'http://m.utm.utoronto.ca/importantDates.php?mode=full&session={0}9&header='
    sessionLinks = [summerLink, fallLink]
    currentSession = "Summer"
    @staticmethod
    def scrape(location='.', year=None): #scrapes most current sessions by default

        year = year or now.year

        calendar = OrderedDict()
        Scraper.logger.info('UTMCalendar initialized.')
        for link in UTMCalendar.sessionLinks:
            html = Scraper.get(link.format(year))
            soup = BeautifulSoup(html, 'html.parser')
            content = soup.find('div', class_='content')
            dates = content.find_all('div', class_='title')
            i = 0
            currentDate = dates[i]
            while(i<len(dates)):
                date = dates[i].text
                events = []
                while (currentDate == dates[i]):
                    info = dates[i].find_next('div', class_='info')
                    description = info.text
                    eventStartEnd = date.split('-') #splits event dates over a period
                    eventStart = eventStartEnd[0].strip()
                    if len(eventStartEnd)>1:
                        eventEnd = eventStartEnd[1].strip()
                    else:
                        eventEnd = eventStart

                    events.append(OrderedDict([
                            ('end_date', eventEnd),
                            ('campus', 'UTM'),
                            ('description', description)
                        ]))
                    i+=1
                    if(i>=len(dates)):
                        break;
                calendar[date] = OrderedDict([
                        ('date', eventStart),
                        ('session', UTMCalendar.currentSession),
                        ('events', events)
                    ])
                if(i<len(dates)):
                    currentDate = dates[i]
            UTMCalendar.currentSession = "Fall/Winter"

        for date, info in calendar.items():
            Scraper.save_json(info, location, date)

        Scraper.logger.info('UTMCalendar completed.')
        return calendar<|endoftext|>


<commit_before>import os
import unittest

from oct.core.turrets_manager import TurretsManager
from oct.results.models import set_database, db, Turret, Result

BASE_DIR = os.path.dirname(os.path.realpath(__file__))

class ReportTest(unittest.TestCase):

    def setUp(self):
        set_database(os.path.join(BASE_DIR, 'fixtures', 'avaible_results.sqlite'), db, {})
        if not Result.table_exists() and not Turret.table_exists():
            db.create_tables([Result, Turret])
        self.manager = TurretsManager(0)

    def test_bad_message(self):
        """Turrets manager should correctly handle bad messages
        """
        res = self.manager.process_message({'bad': 'message'})
        self.assertFalse(res)

    def test_is_started(self):
        """Turrets manager should correctly handle turrets if tests are already started
        """
        res = self.manager.process_message({
            'cannons': 10,
            'rampup': 0,
            'turret': 'navigation',
            'script': 'v_user.py',
            'uuid': 'f26bda4a-e906-4e6c-b55a-5c1a0d00b7a6',
            'status': 'ready'}, is_started=True)
        self.assertTrue(res)

    def test_update(self):
        """Turrets manager should correctly handle message if no uuid is provided
        """
        res = self.manager.update({'not': 'present'})
        self.assertFalse(res)

    def tearDown(self):
        for turret in Turret.select():
            turret.delete_instance()
<commit_msg>improve test turrets manager<commit_after>import os
import unittest

from oct.core.turrets_manager import TurretsManager
from oct.results.models import set_database, db, Turret, Result

BASE_DIR = os.path.dirname(os.path.realpath(__file__))

class ReportTest(unittest.TestCase):

    def setUp(self):
        set_database(os.path.join(BASE_DIR, 'fixtures', 'avaible_results.sqlite'), db, {})
        if not Result.table_exists() and not Turret.table_exists():
            db.create_tables([Result, Turret])
        self.manager = TurretsManager(0)

    def test_bad_message(self):
        """Turrets manager should correctly handle bad messages
        """
        res = self.manager.process_message({'bad': 'message'})
        self.assertFalse(res)

    def test_is_started(self):
        """Turrets manager should correctly handle turrets if tests are already started
        """
        res = self.manager.process_message({
            'cannons': 10,
            'rampup': 0,
            'turret': 'navigation',
            'script': 'v_user.py',
            'uuid': 'f26bda4a-e906-4e6c-b55a-5c1a0d00b7a6',
            'status': 'ready'}, is_started=True)
        self.assertTrue(res)

    def test_update(self):
        """Turrets manager should correctly handle message if no uuid is provided
        """
        res = self.manager.update({'not': 'present'})
        self.assertFalse(res)

    def test_master_false(self):
        """Turrets manager should act correctly in case of master set to false"""
        self.manager.master = False
        self.manager.process_message("bad")
        self.manager.publish("bad")
        self.manager.master = True

    def tearDown(self):
        for turret in Turret.select():
            turret.delete_instance()
<|endoftext|>


<commit_before>#
# Copyright 2016 The BigDL Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from bigdl.nn.layer import *
from bigdl.nn.criterion import *
from bigdl.util.common import *
from bigdl.models.ml_pipeline.dl_classifier import *
from pyspark.sql.types import *
from pyspark.context import SparkContext
import numpy as np
import pytest
from numpy.testing import assert_allclose

class TestDLClassifer():
    def setup_method(self, method):
        """ setup any state tied to the execution of the given method in a
        class.  setup_method is invoked for every test method of a class.
        """
        sparkConf = create_spark_conf().setMaster("local[1]").setAppName("test model")
        self.sc = get_spark_context(sparkConf)
        self.sqlContext = SQLContext(self.sc)
        init_engine()

    def teardown_method(self, method):
        """ teardown any state that was previously setup with a setup_method
        call.
        """
        self.sc.stop()

    def test_all_set_get_methods(self):
        """ run tests for all the set and get methods involved in DLEstimator, DLModel,
            DLClassifier, DLClassifierModel
        """

        '''
        use linear model and MSE criterion to test the DLEstimator and DLModel
        '''
        linear_model = Sequential().add(Linear(2, 2))
        mse_criterion = MSECriterion()

        '''
        initialize a DLEstimator to test setter, getter for
        batchSize, maxEpoch, learningRate in DLEstimator
        '''
        estimator = DLEstimator(model=linear_model, criterion=mse_criterion,
                                feature_size=[2], label_size=[2])

        # check the set and get methods for batchSize
        assert estimator.setBatchSize(30).getBatchSize() == 30
        # check the set and get methods for maxEpoch
        assert estimator.setMaxEpoch(40).getMaxEpoch() == 40
        # check the set and get methods for learningRate
        assert estimator.setLearningRate(1e-4).getLearningRate() == 1e-4

        '''
        initialize a DLModel to test setter, getter for featureSize, batchSize in DLModel
        '''
        dl_model = DLClassifierModel(model=linear_model, featureSize=[1])

        # check the set and get methods for featureSize in DLModel
        assert dl_model.setFeatureSize([2]).getFeatureSize() == [2]
        # check the set and get methods for batchSize in DLModel
        assert dl_model.setBatchSize((20)).getBatchSize() == 20

        '''
        use linear model and ClassNLL criterion to test the DLClassifier and DLClassifierModel
        '''
        linear_model = Sequential().add(Linear(2, 2))
        classNLL_criterion = ClassNLLCriterion()

        '''
        initialize a DLClassifier to test setter, getter for
        batchSize, maxEpoch, learningRate in DLClassifier
        '''
        classifier = DLClassifier(model=linear_model, criterion=classNLL_criterion,
                                  feature_size=[2])

        # check the set and get methods for batchSize
        assert classifier.setBatchSize(20).getBatchSize() == 20
        # check the set and get methods for maxEpoch
        assert classifier.setMaxEpoch(50).getMaxEpoch() == 50
        # check the set and get methods for learningRate
        assert classifier.setLearningRate(1e-5).getLearningRate() == 1e-5
        '''
        initialize a DLClassifierModel to test setter, getter
        for featureSize, batchSize in DLClassifierModel
        '''
        dl_classifier_model = DLClassifierModel(model=linear_model, featureSize=[1])

        # check the set and get methods for featureSize
        assert dl_classifier_model.setFeatureSize([2]).getFeatureSize() == [2]

        # check the set and get methods for batchSize
        assert dl_classifier_model.setBatchSize((20)).getBatchSize() == 20

    def test_dlestimator_fit_dlmodel_transform(self):
        model = Sequential().add(Linear(2, 2))
        criterion = MSECriterion()
        estimator = DLEstimator(model, criterion, [2], [2]).setBatchSize(4)\
            .setLearningRate(0.01).setMaxEpoch(1000)

        data = self.sc.parallelize([
            ((2.0, 1.0), (1.0, 2.0)),
            ((1.0, 2.0), (2.0, 1.0)),
            ((2.0, 1.0), (1.0, 2.0)),
            ((1.0, 2.0), (2.0, 1.0))])

        schema = StructType([
            StructField("features", ArrayType(DoubleType(), False), False),
            StructField("label", ArrayType(DoubleType(), False), False)])
        df = self.sqlContext.createDataFrame(data, schema)
        dlModel = estimator.fit(df)

        dlModel.transform(df).registerTempTable("dlModelDF")  # Compatible with spark 1.6
        results = self.sqlContext.table("dlModelDF")

        count = results.rdd.count()
        data = results.rdd.collect()

        for i in range(count):
            row_label = data[i][1]
            row_prediction = data[i][2]
            assert_allclose(row_label[0], row_prediction[0], atol=0, rtol=1e-1)
            assert_allclose(row_label[1], row_prediction[1], atol=0, rtol=1e-1)

    def test_dlclassifier_fit_dlclassifiermodel_transform(self):
        model = Sequential().add(Linear(2, 2))
        criterion = ClassNLLCriterion()
        classifier = DLClassifier(model, criterion, [2]).setBatchSize(4)\
            .setLearningRate(0.01).setMaxEpoch(1000)
        data = self.sc.parallelize([
            ((2.0, 1.0), 1.0),
            ((1.0, 2.0), 2.0),
            ((2.0, 1.0), 1.0),
            ((1.0, 2.0), 2.0)])

        schema = StructType([
            StructField("features", ArrayType(DoubleType(), False), False),
            StructField("label", DoubleType(), False)])
        df = self.sqlContext.createDataFrame(data, schema)
        dlClassifierModel = classifier.fit(df)

        dlClassifierModel.transform(df).registerTempTable("dlClassifierModelDF")
        results = self.sqlContext.table("dlClassifierModelDF")

        count = results.rdd.count()
        data = results.rdd.collect()

        for i in range(count):
            row_label = data[i][1]
            row_prediction = data[i][2]
            assert row_label == row_prediction

if __name__ == "__main__":
    pytest.main()
<commit_msg>fix python DataFrame (#1883)<commit_after>#
# Copyright 2016 The BigDL Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from bigdl.nn.layer import *
from bigdl.nn.criterion import *
from bigdl.util.common import *
from bigdl.models.ml_pipeline.dl_classifier import *
from pyspark.sql.types import *
from pyspark.context import SparkContext
import numpy as np
import pytest
from numpy.testing import assert_allclose

class TestDLClassifer():
    def setup_method(self, method):
        """ setup any state tied to the execution of the given method in a
        class.  setup_method is invoked for every test method of a class.
        """
        sparkConf = create_spark_conf().setMaster("local[1]").setAppName("test model")
        self.sc = get_spark_context(sparkConf)
        self.sqlContext = SQLContext(self.sc)
        init_engine()

    def teardown_method(self, method):
        """ teardown any state that was previously setup with a setup_method
        call.
        """
        self.sc.stop()

    def test_all_set_get_methods(self):
        """ run tests for all the set and get methods involved in DLEstimator, DLModel,
            DLClassifier, DLClassifierModel
        """

        '''
        use linear model and MSE criterion to test the DLEstimator and DLModel
        '''
        linear_model = Sequential().add(Linear(2, 2))
        mse_criterion = MSECriterion()

        '''
        initialize a DLEstimator to test setter, getter for
        batchSize, maxEpoch, learningRate in DLEstimator
        '''
        estimator = DLEstimator(model=linear_model, criterion=mse_criterion,
                                feature_size=[2], label_size=[2])

        # check the set and get methods for batchSize
        assert estimator.setBatchSize(30).getBatchSize() == 30
        # check the set and get methods for maxEpoch
        assert estimator.setMaxEpoch(40).getMaxEpoch() == 40
        # check the set and get methods for learningRate
        assert estimator.setLearningRate(1e-4).getLearningRate() == 1e-4

        '''
        initialize a DLModel to test setter, getter for featureSize, batchSize in DLModel
        '''
        dl_model = DLClassifierModel(model=linear_model, featureSize=[1])

        # check the set and get methods for featureSize in DLModel
        assert dl_model.setFeatureSize([2]).getFeatureSize() == [2]
        # check the set and get methods for batchSize in DLModel
        assert dl_model.setBatchSize((20)).getBatchSize() == 20

        '''
        use linear model and ClassNLL criterion to test the DLClassifier and DLClassifierModel
        '''
        linear_model = Sequential().add(Linear(2, 2))
        classNLL_criterion = ClassNLLCriterion()

        '''
        initialize a DLClassifier to test setter, getter for
        batchSize, maxEpoch, learningRate in DLClassifier
        '''
        classifier = DLClassifier(model=linear_model, criterion=classNLL_criterion,
                                  feature_size=[2])

        # check the set and get methods for batchSize
        assert classifier.setBatchSize(20).getBatchSize() == 20
        # check the set and get methods for maxEpoch
        assert classifier.setMaxEpoch(50).getMaxEpoch() == 50
        # check the set and get methods for learningRate
        assert classifier.setLearningRate(1e-5).getLearningRate() == 1e-5
        '''
        initialize a DLClassifierModel to test setter, getter
        for featureSize, batchSize in DLClassifierModel
        '''
        dl_classifier_model = DLClassifierModel(model=linear_model, featureSize=[1])

        # check the set and get methods for featureSize
        assert dl_classifier_model.setFeatureSize([2]).getFeatureSize() == [2]

        # check the set and get methods for batchSize
        assert dl_classifier_model.setBatchSize((20)).getBatchSize() == 20

    def test_dlestimator_fit_dlmodel_transform(self):
        model = Sequential().add(Linear(2, 2))
        criterion = MSECriterion()
        estimator = DLEstimator(model, criterion, [2], [2]).setBatchSize(4)\
            .setLearningRate(0.01).setMaxEpoch(1000)

        data = self.sc.parallelize([
            ((2.0, 1.0), (1.0, 2.0)),
            ((1.0, 2.0), (2.0, 1.0)),
            ((2.0, 1.0), (1.0, 2.0)),
            ((1.0, 2.0), (2.0, 1.0))])

        schema = StructType([
            StructField("features", ArrayType(DoubleType(), False), False),
            StructField("label", ArrayType(DoubleType(), False), False)])
        df = self.sqlContext.createDataFrame(data, schema)
        dlModel = estimator.fit(df)

        res = dlModel.transform(df)
        assert type(res).__name__ == 'DataFrame'
        res.registerTempTable("dlModelDF")  # Compatible with spark 1.6
        results = self.sqlContext.table("dlModelDF")

        count = results.rdd.count()
        data = results.rdd.collect()

        for i in range(count):
            row_label = data[i][1]
            row_prediction = data[i][2]
            assert_allclose(row_label[0], row_prediction[0], atol=0, rtol=1e-1)
            assert_allclose(row_label[1], row_prediction[1], atol=0, rtol=1e-1)

    def test_dlclassifier_fit_dlclassifiermodel_transform(self):
        model = Sequential().add(Linear(2, 2))
        criterion = ClassNLLCriterion()
        classifier = DLClassifier(model, criterion, [2]).setBatchSize(4)\
            .setLearningRate(0.01).setMaxEpoch(1000)
        data = self.sc.parallelize([
            ((2.0, 1.0), 1.0),
            ((1.0, 2.0), 2.0),
            ((2.0, 1.0), 1.0),
            ((1.0, 2.0), 2.0)])

        schema = StructType([
            StructField("features", ArrayType(DoubleType(), False), False),
            StructField("label", DoubleType(), False)])
        df = self.sqlContext.createDataFrame(data, schema)
        dlClassifierModel = classifier.fit(df)

        res = dlClassifierModel.transform(df)
        assert type(res).__name__ == 'DataFrame'
        res.registerTempTable("dlClassifierModelDF")
        results = self.sqlContext.table("dlClassifierModelDF")

        count = results.rdd.count()
        data = results.rdd.collect()

        for i in range(count):
            row_label = data[i][1]
            row_prediction = data[i][2]
            assert row_label == row_prediction

if __name__ == "__main__":
    pytest.main()
<|endoftext|>


<commit_before>from Queue import Queue, Empty

try:
    from subprocess import check_output, CalledProcessError
except ImportError:
    print "Unable to import subprocess.check_output, are you using python 2.6?")
    print "Using self defined check_output."

    def check_output(*popenargs, **kwargs):
        import subprocess
        if 'stdout' in kwargs:
            raise ValueError('stdout argument not allowed, it will be overridden.')
        process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs)
        output, unused_err = process.communicate()
        retcode = process.poll()
        if retcode:
            cmd = kwargs.get("args")
            if cmd is None:
                cmd = popenargs[0]
            raise subprocess.CalledProcessError(retcode, cmd)
        return output

class SubprocessExecution:
    __io_q = Queue()
    __process = None

    def main_execution_function(self, shell_command, wait_cmd=True, logger=None):
        """
        :rtype: stdout, stderr
        """
        log_string = 'Executing system the system external command: ' + shell_command
        if logger is not None:
            logger.info(log_string)
        else:
            print log_string

        return_code = 0
        stdout = stderr = ''
        try:
            stdout = check_output(shell_command, shell=True)
        except CalledProcessError as e:
            return_code = e.returncode
            stderr = e.output

        return return_code, stdout, 'stderr: ' + stderr

    def print_output(self, communicates_message):
        for message in communicates_message:
            if message != '':
                print message
<commit_msg>remove stray ).<commit_after>from Queue import Queue, Empty

try:
    from subprocess import check_output, CalledProcessError
except ImportError:
    print "Unable to import subprocess.check_output, are you using python 2.6?"
    print "Using self defined check_output."

    def check_output(*popenargs, **kwargs):
        import subprocess
        if 'stdout' in kwargs:
            raise ValueError('stdout argument not allowed, it will be overridden.')
        process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs)
        output, unused_err = process.communicate()
        retcode = process.poll()
        if retcode:
            cmd = kwargs.get("args")
            if cmd is None:
                cmd = popenargs[0]
            raise subprocess.CalledProcessError(retcode, cmd)
        return output

class SubprocessExecution:
    __io_q = Queue()
    __process = None

    def main_execution_function(self, shell_command, wait_cmd=True, logger=None):
        """
        :rtype: stdout, stderr
        """
        log_string = 'Executing system the system external command: ' + shell_command
        if logger is not None:
            logger.info(log_string)
        else:
            print log_string

        return_code = 0
        stdout = stderr = ''
        try:
            stdout = check_output(shell_command, shell=True)
        except CalledProcessError as e:
            return_code = e.returncode
            stderr = e.output

        return return_code, stdout, 'stderr: ' + stderr

    def print_output(self, communicates_message):
        for message in communicates_message:
            if message != '':
                print message
<|endoftext|>