<commit_before>from datetime import date

class EnumValue:
    def __init__(self, name, value=None, comment=None):
        self.name = name
        self.value = value
        self.comment = comment

class Enum:
    """c-style enumeration class"""

    def __init__(self, name, prefix=""):

        # enum values
        self.values = []
        self.name = name

        self.prefix = prefix

    def add_value(self, name, value=None, comment=None):
        """assuees that the user adds the values in the correct order"""

        self.values.append(EnumValue(name, value=value, comment=comment))

class Variable:
    """c-style variable"""

    def __init__(self,
                 name,
                 primitive,
                 qualifiers=None,
                 array=None,
                 comment=None):
        self.name = name
        self.primitive = primitive
        self.comment = comment
        self.array = array
        self.qualifiers = qualifiers

    def declaration(self):
        """Return a declaration string."""
        if isinstance(self.qualifiers, (list, tuple)):
            qual = " ".join(self.qualifiers) + " "
        elif self.qualifiers is not None:
            qual = str(self.qualifiers) + " "
        else:
            qual = ""

        return '{qual}{prim} {name}{array}'.format(
            qual=qual,
            prim=self.primitive,
            name=self.name,
            array='[{a}]'.format(a=self.array) if self.array else '')

class Struct:
    """c-style struct class"""

    def __init__(self, name, ref_name=None, comment=None):
        self.name = name  # definition name of this struct e.g. Struct_t
        self.ref_name = ref_name  # reference name of this struct e.g. the_struct
        self.variables = []
        self.comment = comment

    def add_variable(self, variable):
        """Add another variable to struct"""
        if not isinstance(variable, (Variable, Struct)):
            raise TypeError("variable must be 'Variable' or 'Struct'")

        self.variables.append(variable)

    def declaration(self):
        """Return a declaration string."""
        if self.ref_name is None:
            raise ValueError('no ref_name supplied for Struct "{name}"'.format(
                name=self.name))

        return '{name} {ref};'.format(name=self.name, ref=self.ref_name)

class Function:
    """c-style function"""

    def __init__(self, name, return_type='void'):
        self.name = name
        self.return_type = return_type

        self.variables = []

    def add_argument(self, var):
        """Add an argument to function"""
        if not isinstance(var, Variable):
            raise TypeError("variable must be of type 'Variable'")

        self.variables.append(var)

    def prototype(self):
        """function prototype string"""
        prot = '{ret} {nm}({funcs})'.format(
            ret=self.return_type,
            nm=self.name,
            funcs=', '.join([v.declaration() for v in self.variables])
            if self.variables else 'void')

        return prot

    def call(self, *arg):
        """call a function"""
        if not len(arg) == len(self.variables):
            print(arg)
            raise ValueError(
                "number of arguments must match number of variables")

        call_ = '{name}({args});'.format(
            name=self.name, args=', '.join([str(a) for a in arg]))

        return call_

class CodeWriter:

    CPP = "__cplusplus"

    VERSION = "1.1"

    def __init__(self, lf="\n", indent=4):

        self.line_feed = lf
        if isinstance(self.indent, int):
            self.indent = ' ' * indent
        else:
            self.indent = indent

        # initialize values
        self.commenting = False  # switch for bulk commenting
        self.defs = []  # define levels
        self.switch = []  # switch levels
        self.tabs = 0
        self.text = ''  # code

    def tab_in(self):
        """increase tab level"""
        self.tabs += 1

    def tab_out(self):
        """decrease tab level"""
        if self.tabs > 0:
            self.tabs -= 1

    def reset_tabs(self):
        self.tabs = 0

    def start_comment(self):
        """start a bulk comment"""
        self.add_line('/*')
        self.commenting = True

    def end_comment(self):
        """end a bulk comment"""
        self.commenting = False
        self.add_line('*/')

    def add_autogen_comment(self, source=None):
        """add the auto-gen comment (user can point to the source file if required)"""
        self.start_comment()
        self.add_line(
            "This file was autogenerated using the C-Snake v{version} script".
            format(version=self.VERSION))
        self.add_line(
            "This file should not be edited directly, any changes will be overwritten next time the script is run"
        )
        if source:
            self.add_line(
                "Make any changes to the file '{src}'".format(src=str(source)))
        self.add_line(
            "Source code for C-Snake available at https://github.com/SchrodingersGat/C-Snake"
        )
        self.end_comment()

    def add_license_comment(self, license_, authors, intro=None):
        """Add the license comment."""
        self.start_comment()

        if intro:
            for line in intro.splitlines():
                if line == '':
                    self.add_line()
                else:
                    self.add_line(line)

        year = date.today().year
        if authors:
            for author in authors:
                if author.get('email', None):
                    email = ' <{0}>'.format(author['email'])
                else:
                    email = ''
                self.add_line("Copyright © {year} {name}{email}".format(
                    year=year,
                    name=author['name'],
                    email=email))
        self.add_line()

        if not isinstance(license_, str):
            raise TypeError('license_ must be a string.')
        for line in license_.splitlines():
            self.add_line(line)

        self.end_comment()

    def open_brace(self):
        """open-brace and tab"""
        self.add_line('{')
        self.tab_in()

    def close_brace(self, new_line=True):
        """close-brace and tab-out"""
        self.tab_out()
        self.add(self.indent * self.tabs + '}')
        if new_line:
            self.add_line('')

    def define(self, name, value=None, comment=None):
        """add a define"""
        line = "#define " + name
        if value:
            line += ' ' + str(value)

        self.add_line(line, comment=comment, ignore_tabs=True)

    def start_if_def(self, define, invert=False, comment=None):
        """start an #ifdef block (preprocessor)"""
        self.defs.append(define)
        if invert:
            self.add_line(
                "#ifndef " + define, comment=comment, ignore_tabs=True)
        else:
            self.add_line(
                "#ifdef " + define, comment=comment, ignore_tabs=True)

    def end_if_def(self):
        """end an #ifdef block"""

        if self.defs:
            self.add_line("#endif ", comment=self.defs.pop(), ignore_tabs=True)
        else:
            self.add_line("#endif", ignore_tabs=True)

    def cpp_entry(self):
        """add an 'extern' switch for CPP compilers"""
        self.start_if_def(self.CPP, "Play nice with C++ compilers")
        self.add_line('extern "C" {', ignore_tabs=True)
        self.end_if_def()

    def cpp_exit(self):
        self.start_if_def(self.CPP, "Done playing nice with C++ compilers")
        self.add_line('}', ignore_tabs=True)
        self.end_if_def()

    def start_switch(self, switch):
        """start a switch statement"""
        self.switch.append(switch)
        self.add_line('switch ({sw})'.format(sw=switch))
        self.open_brace()

    def end_switch(self):
        """end a switch statement"""
        self.tab_out()
        self.add('}')
        if self.switch:
            self.add(' // ~switch ({sw})'.format(sw=self.switch.pop()))
        self.add_line()

    def add_case(self, case, comment=None):
        """add a case statement"""
        self.add_line('case {case}:'.format(case=case), comment=comment)
        self.tab_in()

    def add_default(self, comment=None):
        """add a default case statement"""
        self.add_line('default:', comment=comment)
        self.tab_in()

    def break_from_case(self):
        """break from a case"""
        self.add_line('break;')
        self.tab_out()

    def return_from_case(self, value=None):
        """return from a case"""
        self.add_line(
            'return{val};'.format(val=' ' + str(value) if value else ''))
        self.tab_out()

    def add(self, text):
        """add raw text"""
        self.text += text

    def add_line(self, text=None, comment=None, ignore_tabs=False):
        """add a line of (formatted) text"""

        # empty line
        if not text and not comment and not self.commenting:
            self.add(self.line_feed)
            return

        if not ignore_tabs and not self.commenting:
            self.add(self.indent * self.tabs)

        if self.commenting:
            self.add("* ")

        # add the text (if appropriate)
        if text:
            self.add(text)
        # add a trailing comment (if appropriate)
        if comment:
            if text:
                self.add(' ')  # add a space after the text
            self.add('//' + comment)

        self.add(self.line_feed)

    def include(self, file, comment=None):
        """add a c-style include"""
        self.add_line(
            "#include {file}".format(file=file),
            comment=comment,
            ignore_tabs=True)

    def add_enum(self, enum):
        """add a constructed enumeration"""
        if not isinstance(enum, Enum):
            raise TypeError('enum must be of type "Enum"')

        self.add_line("typedef enum")
        self.open_brace()

        for i, v in enumerate(enum.values):
            line = enum.prefix + v.name
            if v.value:
                line += " = " + str(v.value)

            if i < (len(enum.values) - 1):
                line += ","

            self.add_line(line, comment=v.comment)

        self.close_brace(new_line=False)
        self.add(' ' + enum.name + ';')
        self.add_line()

    def add_variable(self, var):
        """add a variable definition"""
        if not isinstance(var, Variable):
            raise TypeError("variable must be of type 'Variable'")

        self.add_line(var.declaration() + ";", comment=var.comment)

    def add_struct(self, struct):
        """add a struct"""
        if not isinstance(struct, Struct):
            raise TypeError("struct must be of type 'Struct'")

        self.add_line("typedef struct")
        self.open_brace()

        for var in struct.variables:
            if isinstance(var, Variable):  # variables within the struct
                self.add_variable(var)
            elif isinstance(var, Struct):  # other structs within the struct
                if var.ref_name is None:
                    raise ValueError('no ref_name provided for struct {name}'.
                                     format(name=var.name))
                self.add_line(var.declaration(), comment=var.comment)

        self.close_brace(new_line=False)
        self.add(' ' + struct.name + ';')
        self.add_line()

    def add_function_prototype(self, func, comment=None):
        """add a function prototype"""
        if not isinstance(func, Function):
            raise TypeError("func must be of type 'Function'")

        self.add_line(func.prototype() + ';', comment=comment)

    def add_function_definition(self, func, comment=None):
        """add a function definition"""
        if not isinstance(func, Function):
            raise TypeError("func must be of type 'Function'")

        self.add_line(func.prototype(), comment=comment)

    def call_function(self, func, *arg):
        """enter a function"""
        if not isinstance(func, Function):
            raise TypeError("func must be of type 'Function'")

        self.add_line(func.call(*arg))

    def write_to_file(self, file):
        """write code to file"""
        with open(file, 'w') as the_file:
            the_file.write(self.text)
<commit_msg>Added a docstring for EnumValue.<commit_after>from datetime import date

class EnumValue:
    """Singular value of an C-style enumeration"""

    def __init__(self, name, value=None, comment=None):
        self.name = name
        self.value = value
        self.comment = comment

class Enum:
    """c-style enumeration class"""

    def __init__(self, name, prefix=""):

        # enum values
        self.values = []
        self.name = name

        self.prefix = prefix

    def add_value(self, name, value=None, comment=None):
        """assuees that the user adds the values in the correct order"""

        self.values.append(EnumValue(name, value=value, comment=comment))

class Variable:
    """c-style variable"""

    def __init__(self,
                 name,
                 primitive,
                 qualifiers=None,
                 array=None,
                 comment=None):
        self.name = name
        self.primitive = primitive
        self.comment = comment
        self.array = array
        self.qualifiers = qualifiers

    def declaration(self):
        """Return a declaration string."""
        if isinstance(self.qualifiers, (list, tuple)):
            qual = " ".join(self.qualifiers) + " "
        elif self.qualifiers is not None:
            qual = str(self.qualifiers) + " "
        else:
            qual = ""

        return '{qual}{prim} {name}{array}'.format(
            qual=qual,
            prim=self.primitive,
            name=self.name,
            array='[{a}]'.format(a=self.array) if self.array else '')

class Struct:
    """c-style struct class"""

    def __init__(self, name, ref_name=None, comment=None):
        self.name = name  # definition name of this struct e.g. Struct_t
        self.ref_name = ref_name  # reference name of this struct e.g. the_struct
        self.variables = []
        self.comment = comment

    def add_variable(self, variable):
        """Add another variable to struct"""
        if not isinstance(variable, (Variable, Struct)):
            raise TypeError("variable must be 'Variable' or 'Struct'")

        self.variables.append(variable)

    def declaration(self):
        """Return a declaration string."""
        if self.ref_name is None:
            raise ValueError('no ref_name supplied for Struct "{name}"'.format(
                name=self.name))

        return '{name} {ref};'.format(name=self.name, ref=self.ref_name)

class Function:
    """c-style function"""

    def __init__(self, name, return_type='void'):
        self.name = name
        self.return_type = return_type

        self.variables = []

    def add_argument(self, var):
        """Add an argument to function"""
        if not isinstance(var, Variable):
            raise TypeError("variable must be of type 'Variable'")

        self.variables.append(var)

    def prototype(self):
        """function prototype string"""
        prot = '{ret} {nm}({funcs})'.format(
            ret=self.return_type,
            nm=self.name,
            funcs=', '.join([v.declaration() for v in self.variables])
            if self.variables else 'void')

        return prot

    def call(self, *arg):
        """call a function"""
        if not len(arg) == len(self.variables):
            print(arg)
            raise ValueError(
                "number of arguments must match number of variables")

        call_ = '{name}({args});'.format(
            name=self.name, args=', '.join([str(a) for a in arg]))

        return call_

class CodeWriter:

    CPP = "__cplusplus"

    VERSION = "1.1"

    def __init__(self, lf="\n", indent=4):

        self.line_feed = lf
        if isinstance(self.indent, int):
            self.indent = ' ' * indent
        else:
            self.indent = indent

        # initialize values
        self.commenting = False  # switch for bulk commenting
        self.defs = []  # define levels
        self.switch = []  # switch levels
        self.tabs = 0
        self.text = ''  # code

    def tab_in(self):
        """increase tab level"""
        self.tabs += 1

    def tab_out(self):
        """decrease tab level"""
        if self.tabs > 0:
            self.tabs -= 1

    def reset_tabs(self):
        self.tabs = 0

    def start_comment(self):
        """start a bulk comment"""
        self.add_line('/*')
        self.commenting = True

    def end_comment(self):
        """end a bulk comment"""
        self.commenting = False
        self.add_line('*/')

    def add_autogen_comment(self, source=None):
        """add the auto-gen comment (user can point to the source file if required)"""
        self.start_comment()
        self.add_line(
            "This file was autogenerated using the C-Snake v{version} script".
            format(version=self.VERSION))
        self.add_line(
            "This file should not be edited directly, any changes will be overwritten next time the script is run"
        )
        if source:
            self.add_line(
                "Make any changes to the file '{src}'".format(src=str(source)))
        self.add_line(
            "Source code for C-Snake available at https://github.com/SchrodingersGat/C-Snake"
        )
        self.end_comment()

    def add_license_comment(self, license_, authors, intro=None):
        """Add the license comment."""
        self.start_comment()

        if intro:
            for line in intro.splitlines():
                if line == '':
                    self.add_line()
                else:
                    self.add_line(line)

        year = date.today().year
        if authors:
            for author in authors:
                if author.get('email', None):
                    email = ' <{0}>'.format(author['email'])
                else:
                    email = ''
                self.add_line("Copyright © {year} {name}{email}".format(
                    year=year,
                    name=author['name'],
                    email=email))
        self.add_line()

        if not isinstance(license_, str):
            raise TypeError('license_ must be a string.')
        for line in license_.splitlines():
            self.add_line(line)

        self.end_comment()

    def open_brace(self):
        """open-brace and tab"""
        self.add_line('{')
        self.tab_in()

    def close_brace(self, new_line=True):
        """close-brace and tab-out"""
        self.tab_out()
        self.add(self.indent * self.tabs + '}')
        if new_line:
            self.add_line('')

    def define(self, name, value=None, comment=None):
        """add a define"""
        line = "#define " + name
        if value:
            line += ' ' + str(value)

        self.add_line(line, comment=comment, ignore_tabs=True)

    def start_if_def(self, define, invert=False, comment=None):
        """start an #ifdef block (preprocessor)"""
        self.defs.append(define)
        if invert:
            self.add_line(
                "#ifndef " + define, comment=comment, ignore_tabs=True)
        else:
            self.add_line(
                "#ifdef " + define, comment=comment, ignore_tabs=True)

    def end_if_def(self):
        """end an #ifdef block"""

        if self.defs:
            self.add_line("#endif ", comment=self.defs.pop(), ignore_tabs=True)
        else:
            self.add_line("#endif", ignore_tabs=True)

    def cpp_entry(self):
        """add an 'extern' switch for CPP compilers"""
        self.start_if_def(self.CPP, "Play nice with C++ compilers")
        self.add_line('extern "C" {', ignore_tabs=True)
        self.end_if_def()

    def cpp_exit(self):
        self.start_if_def(self.CPP, "Done playing nice with C++ compilers")
        self.add_line('}', ignore_tabs=True)
        self.end_if_def()

    def start_switch(self, switch):
        """start a switch statement"""
        self.switch.append(switch)
        self.add_line('switch ({sw})'.format(sw=switch))
        self.open_brace()

    def end_switch(self):
        """end a switch statement"""
        self.tab_out()
        self.add('}')
        if self.switch:
            self.add(' // ~switch ({sw})'.format(sw=self.switch.pop()))
        self.add_line()

    def add_case(self, case, comment=None):
        """add a case statement"""
        self.add_line('case {case}:'.format(case=case), comment=comment)
        self.tab_in()

    def add_default(self, comment=None):
        """add a default case statement"""
        self.add_line('default:', comment=comment)
        self.tab_in()

    def break_from_case(self):
        """break from a case"""
        self.add_line('break;')
        self.tab_out()

    def return_from_case(self, value=None):
        """return from a case"""
        self.add_line(
            'return{val};'.format(val=' ' + str(value) if value else ''))
        self.tab_out()

    def add(self, text):
        """add raw text"""
        self.text += text

    def add_line(self, text=None, comment=None, ignore_tabs=False):
        """add a line of (formatted) text"""

        # empty line
        if not text and not comment and not self.commenting:
            self.add(self.line_feed)
            return

        if not ignore_tabs and not self.commenting:
            self.add(self.indent * self.tabs)

        if self.commenting:
            self.add("* ")

        # add the text (if appropriate)
        if text:
            self.add(text)
        # add a trailing comment (if appropriate)
        if comment:
            if text:
                self.add(' ')  # add a space after the text
            self.add('//' + comment)

        self.add(self.line_feed)

    def include(self, file, comment=None):
        """add a c-style include"""
        self.add_line(
            "#include {file}".format(file=file),
            comment=comment,
            ignore_tabs=True)

    def add_enum(self, enum):
        """add a constructed enumeration"""
        if not isinstance(enum, Enum):
            raise TypeError('enum must be of type "Enum"')

        self.add_line("typedef enum")
        self.open_brace()

        for i, v in enumerate(enum.values):
            line = enum.prefix + v.name
            if v.value:
                line += " = " + str(v.value)

            if i < (len(enum.values) - 1):
                line += ","

            self.add_line(line, comment=v.comment)

        self.close_brace(new_line=False)
        self.add(' ' + enum.name + ';')
        self.add_line()

    def add_variable(self, var):
        """add a variable definition"""
        if not isinstance(var, Variable):
            raise TypeError("variable must be of type 'Variable'")

        self.add_line(var.declaration() + ";", comment=var.comment)

    def add_struct(self, struct):
        """add a struct"""
        if not isinstance(struct, Struct):
            raise TypeError("struct must be of type 'Struct'")

        self.add_line("typedef struct")
        self.open_brace()

        for var in struct.variables:
            if isinstance(var, Variable):  # variables within the struct
                self.add_variable(var)
            elif isinstance(var, Struct):  # other structs within the struct
                if var.ref_name is None:
                    raise ValueError('no ref_name provided for struct {name}'.
                                     format(name=var.name))
                self.add_line(var.declaration(), comment=var.comment)

        self.close_brace(new_line=False)
        self.add(' ' + struct.name + ';')
        self.add_line()

    def add_function_prototype(self, func, comment=None):
        """add a function prototype"""
        if not isinstance(func, Function):
            raise TypeError("func must be of type 'Function'")

        self.add_line(func.prototype() + ';', comment=comment)

    def add_function_definition(self, func, comment=None):
        """add a function definition"""
        if not isinstance(func, Function):
            raise TypeError("func must be of type 'Function'")

        self.add_line(func.prototype(), comment=comment)

    def call_function(self, func, *arg):
        """enter a function"""
        if not isinstance(func, Function):
            raise TypeError("func must be of type 'Function'")

        self.add_line(func.call(*arg))

    def write_to_file(self, file):
        """write code to file"""
        with open(file, 'w') as the_file:
            the_file.write(self.text)
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""Runs integration tests."""

import argparse
import os
import platform
import sys
import re

script_dir = os.path.dirname(os.path.realpath(__file__))
docker_dir = os.path.join(script_dir, 'bamboos', 'docker')
sys.path.insert(0, docker_dir)
from environment import docker

def parse_valgrind_log_error_count(log_file):
    """
    Parses valgrind memcheck file and returns the identified error count.
    """
    with open(log_file, 'r') as f:
        regex = re.compile("ERROR SUMMARY:\s(\d+)\serrors")
        for line in f:
            match = re.search(regex, line)
            if match:
                return int(match.groups()[0])
        raise SystemExit("Invalid Valgrind memcheck report file: "+log_file)

parser = argparse.ArgumentParser(
    formatter_class=argparse.ArgumentDefaultsHelpFormatter,
    description='Run Common Tests.')

parser.add_argument(
    '--gdb',
    action='store_true',
    default=False,
    help='run tests in GDB')

parser.add_argument(
    '--valgrind',
    action='store_true',
    default=False,
    help='run tests under Valgrind',
    dest='valgrind')

parser.add_argument(
    '--callgrind',
    action='store_true',
    default=False,
    help='run tests under Valgrind callgrind tool',
    dest='callgrind')

parser.add_argument(
    '--image', '-i',
    action='store',
    default='onedata/builder:2102-5',
    help='docker image to use as a test master',
    dest='image')

parser.add_argument(
    '--release',
    action='store',
    default='release',
    help='release directory to run tests from',
    dest='release')

parser.add_argument(
    '--suite',
    action='append',
    default=[],
    help='name of the test suite',
    dest='suites')

[args, pass_args] = parser.parse_known_args()
script_dir = os.path.dirname(os.path.realpath(__file__))
base_test_dir = os.path.join(os.path.realpath(args.release), 'test',
                             'integration')
test_dirs = map(lambda suite: os.path.join(base_test_dir, suite), args.suites)
if args.valgrind:
    if len(test_dirs) != 1:
        raise SystemExit('Valgrind test run requires specification of a single '
                         'test case suite, e.g. \'--suite ceph_helper_test\'')
    if args.gdb:
        raise SystemExit('GDB and Valgrind cannot be used simultanously for '
                         'tests')

if not test_dirs:
    test_dirs = [base_test_dir]

command = '''
import os, subprocess, sys, stat

if {shed_privileges}:
    os.environ['HOME'] = '/tmp'
    docker_gid = os.stat('/var/run/docker.sock').st_gid
    os.chmod('/etc/resolv.conf', 0o666)
    os.setgroups([docker_gid])
    os.setregid({gid}, {gid})
    os.setreuid({uid}, {uid})

if {gdb}:
    command = ['gdb', 'python', '-silent', '-ex', """run -c "
import pytest
pytest.main({args} + ['{test_dirs}'])" """]
elif {valgrind}:
    command = ['valgrind'] \\
            + ['--gen-suppressions=all'] \\
            + ['--suppressions=valgrind.supp'] \\
            + ['--track-origins=yes'] \\
            + ['--log-file=valgrind-{suite}.txt'] \\
            + ['--show-leak-kinds=definite'] \\
            + ['--leak-check=full'] \\
            + ['py.test'] + {args} + ['{test_dirs}']
elif {callgrind}:
    command = ['valgrind'] \\
            + ['--tool=callgrind'] \\
            + ['py.test'] + {args} + ['{test_dirs}']
else:
    command = ['py.test'] + {args} + ['{test_dirs}']

ret = subprocess.call(command)
sys.exit(ret)
'''
command = command.format(
    args=pass_args,
    uid=os.geteuid(),
    gid=os.getegid(),
    test_dirs="', '".join(test_dirs),
    shed_privileges=(platform.system() == 'Linux'),
    gdb=args.gdb,
    valgrind=args.valgrind,
    callgrind=args.callgrind,
    suite=(args.suites[0] if args.valgrind else "', '".join(test_dirs)))

docker.run(tty=True,
           rm=True,
           interactive=True,
           workdir=script_dir,
           reflect=[(script_dir, 'rw'),
                    ('/var/run/docker.sock', 'rw')],
           image=args.image,
           envs={'BASE_TEST_DIR': base_test_dir},
           run_params=['--privileged'] if (args.gdb or args.valgrind) else [],
           command=['python', '-c', command])

# If exit code != 0 then bamboo always fails build.
# If it is 0 then result is based on test report.
ret = 0
# If Valgrind was enabled, parse the memcheck report
# and return error if any errors were identified
if args.valgrind:
    ret = parse_valgrind_log_error_count("valgrind-"+args.suites[0]+".txt")

sys.exit(ret)
<commit_msg>VFS-8258 Added cpuset option to ct_run script<commit_after>#!/usr/bin/env python

"""Runs integration tests."""

import argparse
import os
import platform
import sys
import re

script_dir = os.path.dirname(os.path.realpath(__file__))
docker_dir = os.path.join(script_dir, 'bamboos', 'docker')
sys.path.insert(0, docker_dir)
from environment import docker

def parse_valgrind_log_error_count(log_file):
    """
    Parses valgrind memcheck file and returns the identified error count.
    """
    with open(log_file, 'r') as f:
        regex = re.compile("ERROR SUMMARY:\s(\d+)\serrors")
        for line in f:
            match = re.search(regex, line)
            if match:
                return int(match.groups()[0])
        raise SystemExit("Invalid Valgrind memcheck report file: "+log_file)

parser = argparse.ArgumentParser(
    formatter_class=argparse.ArgumentDefaultsHelpFormatter,
    description='Run Common Tests.')

parser.add_argument(
    '--gdb',
    action='store_true',
    default=False,
    help='run tests in GDB')

parser.add_argument(
    '--valgrind',
    action='store_true',
    default=False,
    help='run tests under Valgrind',
    dest='valgrind')

parser.add_argument(
    '--callgrind',
    action='store_true',
    default=False,
    help='run tests under Valgrind callgrind tool',
    dest='callgrind')

parser.add_argument(
    '--image', '-i',
    action='store',
    default='onedata/builder:2102-5',
    help='docker image to use as a test master',
    dest='image')

parser.add_argument(
    '--release',
    action='store',
    default='release',
    help='release directory to run tests from',
    dest='release')

parser.add_argument(
    '--suite',
    action='append',
    default=[],
    help='name of the test suite',
    dest='suites')

parser.add_argument(
    '--cpuset-cpus',
    action='store',
    default=None,
    help='CPUs in which to allow execution (0-3, 0,1)',
    dest='cpuset_cpus')

[args, pass_args] = parser.parse_known_args()
script_dir = os.path.dirname(os.path.realpath(__file__))
base_test_dir = os.path.join(os.path.realpath(args.release), 'test',
                             'integration')
test_dirs = map(lambda suite: os.path.join(base_test_dir, suite), args.suites)
if args.valgrind:
    if len(test_dirs) != 1:
        raise SystemExit('Valgrind test run requires specification of a single '
                         'test case suite, e.g. \'--suite ceph_helper_test\'')
    if args.gdb:
        raise SystemExit('GDB and Valgrind cannot be used simultanously for '
                         'tests')

if not test_dirs:
    test_dirs = [base_test_dir]

command = '''
import os, subprocess, sys, stat

if {shed_privileges}:
    os.environ['HOME'] = '/tmp'
    docker_gid = os.stat('/var/run/docker.sock').st_gid
    os.chmod('/etc/resolv.conf', 0o666)
    os.setgroups([docker_gid])
    os.setregid({gid}, {gid})
    os.setreuid({uid}, {uid})

if {gdb}:
    command = ['gdb', 'python', '-silent', '-ex', """run -c "
import pytest
pytest.main({args} + ['{test_dirs}'])" """]
elif {valgrind}:
    command = ['valgrind'] \\
            + ['--gen-suppressions=all'] \\
            + ['--suppressions=valgrind.supp'] \\
            + ['--track-origins=yes'] \\
            + ['--log-file=valgrind-{suite}.txt'] \\
            + ['--show-leak-kinds=definite'] \\
            + ['--leak-check=full'] \\
            + ['py.test'] + {args} + ['{test_dirs}']
elif {callgrind}:
    command = ['valgrind'] \\
            + ['--tool=callgrind'] \\
            + ['py.test'] + {args} + ['{test_dirs}']
else:
    command = ['py.test'] + {args} + ['{test_dirs}']

ret = subprocess.call(command)
sys.exit(ret)
'''
command = command.format(
    args=pass_args,
    uid=os.geteuid(),
    gid=os.getegid(),
    test_dirs="', '".join(test_dirs),
    shed_privileges=(platform.system() == 'Linux'),
    gdb=args.gdb,
    valgrind=args.valgrind,
    callgrind=args.callgrind,
    suite=(args.suites[0] if args.valgrind else "', '".join(test_dirs)))

docker.run(tty=True,
           rm=True,
           interactive=True,
           workdir=script_dir,
           reflect=[(script_dir, 'rw'),
                    ('/var/run/docker.sock', 'rw')],
           image=args.image,
           envs={'BASE_TEST_DIR': base_test_dir},
           run_params=['--privileged'] if (args.gdb or args.valgrind) else [],
           cpuset_cpus=args.cpuset_cpus,
           command=['python', '-c', command])

# If exit code != 0 then bamboo always fails build.
# If it is 0 then result is based on test report.
ret = 0
# If Valgrind was enabled, parse the memcheck report
# and return error if any errors were identified
if args.valgrind:
    ret = parse_valgrind_log_error_count("valgrind-"+args.suites[0]+".txt")

sys.exit(ret)
<|endoftext|>


<commit_before>from cum import config, output, sanity
from natsort import humansorted
from shutil import copyfile
from sqlalchemy import (
    Boolean,
    Column,
    create_engine,
    ForeignKey,
    Integer,
    String,
    Table
)
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relationship, sessionmaker
from sqlalchemy.orm.exc import NoResultFound
import sqlalchemy.engine.url
from urllib.parse import urlparse
import click
import os

Base = declarative_base()

group_table = Table(
    'group_association', Base.metadata,
    Column('chapter_id', Integer, ForeignKey('chapters.id')),
    Column('group_id', Integer, ForeignKey('groups.id'))
)

class Series(Base):
    __tablename__ = 'series'

    id = Column(Integer, primary_key=True)

    name = Column(String)
    alias = Column(String, unique=True)
    url = Column(String, unique=True)
    following = Column(Boolean, default=True)
    directory = Column(String)

    chapters = relationship("Chapter", backref="series")

    def __init__(self, series):
        self.name = series.name
        self.alias = series.alias
        self.url = series.url
        self.directory = series.directory

    @property
    def ordered_chapters(self):
        return humansorted(self.chapters, key=lambda x: x.chapter)

    @staticmethod
    def alias_lookup(alias):
        """Returns a DB object for a series by alias name. Prints an error if
        an invalid alias is specified.
        """
        try:
            s = (session.query(Series)
                 .filter_by(alias=alias, following=True)
                 .one())
        except NoResultFound:
            output.error('Could not find alias "{}"'.format(alias))
            exit(1)
        else:
            return s

class Chapter(Base):
    __tablename__ = 'chapters'

    id = Column(Integer, primary_key=True)
    series_id = Column(Integer, ForeignKey('series.id'))

    # Downloaded has the value -1 for ignored chapters, 0 for new chapters and
    # 1 for downloaded chapters.
    downloaded = Column(Integer, default=0)
    chapter = Column(String)
    url = Column(String, unique=True)
    title = Column(String)

    groups = relationship('Group', secondary=group_table, backref='chapters')

    def __init__(self, chapter, series):
        self.series = series
        self.chapter = chapter.chapter
        self.title = chapter.title
        self.url = chapter.url

        self.groups = []
        for group in chapter.groups:
            try:
                g = session.query(Group).filter(Group.name == group).one()
            except NoResultFound:
                g = Group(group)
                session.add(g)
                session.commit()
            self.groups.append(g)

    @property
    def group_tag(self):
        """Return a joined string of chapter's groups enclosed in brackets."""
        return ''.join(['[{}]'.format(x.name) for x in self.groups])

    @property
    def status(self):
        """Return the chapter's downloaded status as a one character flag."""
        if self.downloaded == 0:
            return 'n'
        elif self.downloaded == -1:
            return 'i'
        else:
            return ' '

    @staticmethod
    def find_new(alias=None):
        """Return a list of new chapters as Chapter objects and applies human
        sorting to it. Accepts an optional 'alias' argument, which will filter
        the query.
        """
        query = session.query(Chapter).join(Series).filter(Series.following)
        if alias:
            query = query.filter(Series.alias == alias)
        query = query.filter(Chapter.downloaded == 0).all()
        return humansorted([x.to_object() for x in query],
                           key=lambda x: x.chapter)

    def to_object(self):
        """Turns a database entry into a chapter object for the respective
        site by parsing the URL.
        """
        parse = urlparse(self.url)
        kwargs = {'name': self.series.name,
                  'alias': self.series.alias,
                  'chapter': self.chapter,
                  'url': self.url,
                  'groups': self.groups,
                  'directory': self.series.directory}
        if parse.netloc == 'bato.to':
            from cum.scrapers.batoto import BatotoChapter
            return BatotoChapter(**kwargs)
        elif parse.netloc == 'dynasty-scans.com':
            from cum.scrapers.dynastyscans import DynastyScansChapter
            return DynastyScansChapter(**kwargs)
        elif parse.netloc == 'manga.madokami.com':
            from cum.scrapers.madokami import MadokamiChapter
            return MadokamiChapter(**kwargs)
        else:
            return None

class Group(Base):
    __tablename__ = 'groups'

    id = Column(Integer, primary_key=True)
    name = Column(String, unique=True)

    def __init__(self, name):
        self.name = name

    def __str__(self):
        return self.name

def backup_database():
    """Backs up the database file to a file called cum.db.bak."""
    db_path = os.path.join(cum_dir, 'cum.db')
    backup_path = os.path.join(cum_dir, 'cum.db.bak')
    copyfile(db_path, backup_path)

def initialize():
    global db_path, engine, session
    db_path = os.path.join(config.cum_dir, 'cum.db')
    db_url = sqlalchemy.engine.url.URL('sqlite', database=db_path)
    engine = create_engine(db_url)
    if not os.path.exists(db_path):
        Base.metadata.create_all(bind=engine)
    Session = sessionmaker(bind=engine)
    session = Session()

def test_database():
    """Runs a database sanity test."""
    sanity_tester = sanity.DatabaseSanity(Base, engine)
    sanity_tester.test()
    if sanity_tester.errors:
        for error in sanity_tester.errors:
            err_target, err_msg = str(error).split(' ', 1)
            message = ' '.join([click.style(err_target, bold=True), err_msg])
            output.warning(message)
        output.error('Database has failed sanity check; '
                     'run `cum repair-db` to repair database')
        exit(1)
<commit_msg>Fix backup_database() to use the new config.cum_dir.<commit_after>from cum import config, output, sanity
from natsort import humansorted
from shutil import copyfile
from sqlalchemy import (
    Boolean,
    Column,
    create_engine,
    ForeignKey,
    Integer,
    String,
    Table
)
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relationship, sessionmaker
from sqlalchemy.orm.exc import NoResultFound
import sqlalchemy.engine.url
from urllib.parse import urlparse
import click
import os

Base = declarative_base()

group_table = Table(
    'group_association', Base.metadata,
    Column('chapter_id', Integer, ForeignKey('chapters.id')),
    Column('group_id', Integer, ForeignKey('groups.id'))
)

class Series(Base):
    __tablename__ = 'series'

    id = Column(Integer, primary_key=True)

    name = Column(String)
    alias = Column(String, unique=True)
    url = Column(String, unique=True)
    following = Column(Boolean, default=True)
    directory = Column(String)

    chapters = relationship("Chapter", backref="series")

    def __init__(self, series):
        self.name = series.name
        self.alias = series.alias
        self.url = series.url
        self.directory = series.directory

    @property
    def ordered_chapters(self):
        return humansorted(self.chapters, key=lambda x: x.chapter)

    @staticmethod
    def alias_lookup(alias):
        """Returns a DB object for a series by alias name. Prints an error if
        an invalid alias is specified.
        """
        try:
            s = (session.query(Series)
                 .filter_by(alias=alias, following=True)
                 .one())
        except NoResultFound:
            output.error('Could not find alias "{}"'.format(alias))
            exit(1)
        else:
            return s

class Chapter(Base):
    __tablename__ = 'chapters'

    id = Column(Integer, primary_key=True)
    series_id = Column(Integer, ForeignKey('series.id'))

    # Downloaded has the value -1 for ignored chapters, 0 for new chapters and
    # 1 for downloaded chapters.
    downloaded = Column(Integer, default=0)
    chapter = Column(String)
    url = Column(String, unique=True)
    title = Column(String)

    groups = relationship('Group', secondary=group_table, backref='chapters')

    def __init__(self, chapter, series):
        self.series = series
        self.chapter = chapter.chapter
        self.title = chapter.title
        self.url = chapter.url

        self.groups = []
        for group in chapter.groups:
            try:
                g = session.query(Group).filter(Group.name == group).one()
            except NoResultFound:
                g = Group(group)
                session.add(g)
                session.commit()
            self.groups.append(g)

    @property
    def group_tag(self):
        """Return a joined string of chapter's groups enclosed in brackets."""
        return ''.join(['[{}]'.format(x.name) for x in self.groups])

    @property
    def status(self):
        """Return the chapter's downloaded status as a one character flag."""
        if self.downloaded == 0:
            return 'n'
        elif self.downloaded == -1:
            return 'i'
        else:
            return ' '

    @staticmethod
    def find_new(alias=None):
        """Return a list of new chapters as Chapter objects and applies human
        sorting to it. Accepts an optional 'alias' argument, which will filter
        the query.
        """
        query = session.query(Chapter).join(Series).filter(Series.following)
        if alias:
            query = query.filter(Series.alias == alias)
        query = query.filter(Chapter.downloaded == 0).all()
        return humansorted([x.to_object() for x in query],
                           key=lambda x: x.chapter)

    def to_object(self):
        """Turns a database entry into a chapter object for the respective
        site by parsing the URL.
        """
        parse = urlparse(self.url)
        kwargs = {'name': self.series.name,
                  'alias': self.series.alias,
                  'chapter': self.chapter,
                  'url': self.url,
                  'groups': self.groups,
                  'directory': self.series.directory}
        if parse.netloc == 'bato.to':
            from cum.scrapers.batoto import BatotoChapter
            return BatotoChapter(**kwargs)
        elif parse.netloc == 'dynasty-scans.com':
            from cum.scrapers.dynastyscans import DynastyScansChapter
            return DynastyScansChapter(**kwargs)
        elif parse.netloc == 'manga.madokami.com':
            from cum.scrapers.madokami import MadokamiChapter
            return MadokamiChapter(**kwargs)
        else:
            return None

class Group(Base):
    __tablename__ = 'groups'

    id = Column(Integer, primary_key=True)
    name = Column(String, unique=True)

    def __init__(self, name):
        self.name = name

    def __str__(self):
        return self.name

def backup_database():
    """Backs up the database file to a file called cum.db.bak."""
    db_path = os.path.join(config.cum_dir, 'cum.db')
    backup_path = os.path.join(config.cum_dir, 'cum.db.bak')
    copyfile(db_path, backup_path)

def initialize():
    global db_path, engine, session
    db_path = os.path.join(config.cum_dir, 'cum.db')
    db_url = sqlalchemy.engine.url.URL('sqlite', database=db_path)
    engine = create_engine(db_url)
    if not os.path.exists(db_path):
        Base.metadata.create_all(bind=engine)
    Session = sessionmaker(bind=engine)
    session = Session()

def test_database():
    """Runs a database sanity test."""
    sanity_tester = sanity.DatabaseSanity(Base, engine)
    sanity_tester.test()
    if sanity_tester.errors:
        for error in sanity_tester.errors:
            err_target, err_msg = str(error).split(' ', 1)
            message = ' '.join([click.style(err_target, bold=True), err_msg])
            output.warning(message)
        output.error('Database has failed sanity check; '
                     'run `cum repair-db` to repair database')
        exit(1)
<|endoftext|>


<commit_before>"""HC Corpus Preprocessor which inherits from BasePreprocessor."""

import sys
import regex as re
import unicodedata
from base_preprocessor import BasePreprocessor
from nltk.tokenize import sent_tokenize

class HcCorpusPreprocessor(BasePreprocessor):
    """Hc-Corpus Preprocessor which preprocesses the Hc-Corpus."""

    def __init__(
        self,
        corpus_fname,
        corpus_dir_path='.',
        encoding='utf-8',
        need_preprocessing=False
    ):
        """Constructor which initializes the BasePreprocessor constructor."""
        super(HcCorpusPreprocessor, self).__init__(
            corpus_fname,
            corpus_dir_path=corpus_dir_path,
            encoding=encoding,
            need_preprocessing=need_preprocessing
        )

    def _extract_corpus_data(self, data):
        """Extract 4th column of corpus which contains the body."""
        tab_split_list = data.split('\t')
        return ". ".join(
            tab_split_list[x] for x in range(
                len(tab_split_list)
            ) if x % 4 == 0 and x != 0
        )

    def _clean_word(self, word):
        """
        Preprocess words after tokenizing words from sentences.

        * Remove apostrophes ['s, s'].
        * Remove website urls [wordpress.com, blogspot.com].
        * Bring to lowercase.
        * Remove punctuations.
        """
        return re.sub(
            pattern=ur"(([a-zA-Z0-9]*).(com|net|org|in).?)|(\p{P}+)",
            repl='',
            string=word.lower().strip()
        )

    def _tokenize_sentences(self, data):
        """
        Sentence tokenize corpus.

        * Sentence Tokenize the corpus using NLTK.
        * Remove punctuations [ except space ] from each individual sentences.
        """
        # tbl = dict.fromkeys(
        #     i for i in xrange(sys.maxunicode)
        #     if unicodedata.category(unichr(i)).startswith('P')
        # )
        # return (
        #     sentence.translate(tbl) for sentence in sent_tokenize(data)
        # )
        return (
            sentence for sentence in sent_tokenize(data)
        )

    def _tokenize_words(self, sentence):
        """Tokenize Words from sentences."""
        return sentence.split()

BasePreprocessor.register(HcCorpusPreprocessor)
<commit_msg>Modified preprocessor to discard characters after punctuation such as i'm to i<commit_after>"""HC Corpus Preprocessor which inherits from BasePreprocessor."""

import sys
import regex as re
import unicodedata
from base_preprocessor import BasePreprocessor
from nltk.tokenize import sent_tokenize

class HcCorpusPreprocessor(BasePreprocessor):
    """Hc-Corpus Preprocessor which preprocesses the Hc-Corpus."""

    def __init__(
        self,
        corpus_fname,
        corpus_dir_path='.',
        encoding='utf-8',
        need_preprocessing=False
    ):
        """Constructor which initializes the BasePreprocessor constructor."""
        super(HcCorpusPreprocessor, self).__init__(
            corpus_fname,
            corpus_dir_path=corpus_dir_path,
            encoding=encoding,
            need_preprocessing=need_preprocessing
        )

    def _extract_corpus_data(self, data):
        """Extract 4th column of corpus which contains the body."""
        line_split_list=data.split('\n')
        corpus_data = []
        for i in range(len(line_split_list)):
            tab_split_list=line_split_list[i].split('\t')
            for j in range(len(tab_split_list)):
                if j % 4 == 0 and j != 0:
                    corpus_data.append(tab_split_list[j].strip())
        return ".".join(corpus_data)

    def _clean_word(self, word):
        """
        Preprocess words after tokenizing words from sentences.

        * Remove apostrophes ['s, s'].
        * Remove website urls [wordpress.com, blogspot.com].
        * Bring to lowercase.
        * Remove punctuations.
        """
        return re.sub(
            pattern=ur"((\p{P}+[a-zA-Z]*)|(\p{S}))",
            repl='',
            string=word.lower()
        )

    def _tokenize_sentences(self, data):
        """
        Sentence tokenize corpus.

        * Sentence Tokenize the corpus using NLTK.
        * Remove punctuations [ except space ] from each individual sentences.
        """
        # tbl = dict.fromkeys(
        #     i for i in xrange(sys.maxunicode)
        #     if unicodedata.category(unichr(i)).startswith('P')
        # )
        # return (
        #     sentence.translate(tbl) for sentence in sent_tokenize(data)
        # )
        return (
            sentence for sentence in sent_tokenize(data)
        )

    def _tokenize_words(self, sentence):
        """Tokenize Words from sentences."""
        return sentence.split()

BasePreprocessor.register(HcCorpusPreprocessor)
<|endoftext|>


<commit_before>import numpy as np
from elephas.ml import adapter
import pytest

def test_to_data_frame(spark_context):
    features = np.ones((2, 10))
    labels = np.asarray([[2.0], [1.0]])

    data_frame = adapter.to_data_frame(
        spark_context, features, labels, categorical=False)
    assert data_frame.count() == 2

def test_to_data_frame_cat(spark_context):
    features = np.ones((2, 10))
    labels = np.asarray([[0, 0, 1.0], [0, 1.0, 0]])

    data_frame = adapter.to_data_frame(
        spark_context, features, labels, categorical=True)
    assert data_frame.count() == 2

def test_from_data_frame(spark_context):
    features = np.ones((2, 10))
    labels = np.asarray([[2.0], [1.0]]).reshape((2,))

    data_frame = adapter.to_data_frame(
        spark_context, features, labels, categorical=False)

    x, y = adapter.from_data_frame(data_frame, categorical=False)
    assert features.shape == x.shape
    assert labels.shape == y.shape

def test_from_data_frame_cat(spark_context):
    features = np.ones((2, 10))
    labels = np.asarray([[0, 0, 1.0], [0, 1.0, 0]])

    data_frame = adapter.to_data_frame(
        spark_context, features, labels, categorical=True)

    x, y = adapter.from_data_frame(data_frame, categorical=True, nb_classes=3)
    assert features.shape == x.shape
    assert labels.shape == y.shape

def test_df_to_simple_rdd(spark_context):
    features = np.ones((2, 10))
    labels = np.asarray([[2.0], [1.0]]).reshape((2,))

    data_frame = adapter.to_data_frame(
        spark_context, features, labels, categorical=False)

    rdd = adapter.df_to_simple_rdd(data_frame, False)
    assert rdd.count() == 2
<commit_msg>delete import<commit_after>import numpy as np
from elephas.ml import adapter

def test_to_data_frame(spark_context):
    features = np.ones((2, 10))
    labels = np.asarray([[2.0], [1.0]])

    data_frame = adapter.to_data_frame(
        spark_context, features, labels, categorical=False)
    assert data_frame.count() == 2

def test_to_data_frame_cat(spark_context):
    features = np.ones((2, 10))
    labels = np.asarray([[0, 0, 1.0], [0, 1.0, 0]])

    data_frame = adapter.to_data_frame(
        spark_context, features, labels, categorical=True)
    assert data_frame.count() == 2

def test_from_data_frame(spark_context):
    features = np.ones((2, 10))
    labels = np.asarray([[2.0], [1.0]]).reshape((2,))

    data_frame = adapter.to_data_frame(
        spark_context, features, labels, categorical=False)

    x, y = adapter.from_data_frame(data_frame, categorical=False)
    assert features.shape == x.shape
    assert labels.shape == y.shape

def test_from_data_frame_cat(spark_context):
    features = np.ones((2, 10))
    labels = np.asarray([[0, 0, 1.0], [0, 1.0, 0]])

    data_frame = adapter.to_data_frame(
        spark_context, features, labels, categorical=True)

    x, y = adapter.from_data_frame(data_frame, categorical=True, nb_classes=3)
    assert features.shape == x.shape
    assert labels.shape == y.shape

def test_df_to_simple_rdd(spark_context):
    features = np.ones((2, 10))
    labels = np.asarray([[2.0], [1.0]]).reshape((2,))

    data_frame = adapter.to_data_frame(
        spark_context, features, labels, categorical=False)

    rdd = adapter.df_to_simple_rdd(data_frame, False)
    assert rdd.count() == 2
<|endoftext|>


<commit_before>import os
from typing import Text, List
from unittest.mock import Mock

import pytest
from _pytest.monkeypatch import MonkeyPatch

import rasa.utils.io as io_utils
from rasa.importers import autoconfig
from rasa.importers.rasa import RasaFileImporter
from rasa.nlu.config import RasaNLUModelConfig
from rasa.nlu import config, load_data
from rasa.nlu import components
from rasa.nlu.components import ComponentBuilder
from rasa.nlu.constants import TRAINABLE_EXTRACTORS
from rasa.nlu.model import Trainer
from tests.nlu.utilities import write_file_config

def test_blank_config(blank_config):
    file_config = {}
    f = write_file_config(file_config)
    final_config = config.load(f.name)

    assert final_config.as_dict() == blank_config.as_dict()

def test_invalid_config_json(tmp_path):
    file_config = """pipeline: [pretrained_embeddings_spacy"""  # invalid yaml

    f = tmp_path / "tmp_config_file.json"
    f.write_text(file_config)

    with pytest.raises(config.InvalidConfigError):
        config.load(str(f))

def test_invalid_many_tokenizers_in_config():
    nlu_config = {
        "pipeline": [{"name": "WhitespaceTokenizer"}, {"name": "SpacyTokenizer"}]
    }

    with pytest.raises(config.InvalidConfigError) as execinfo:
        Trainer(config.RasaNLUModelConfig(nlu_config))
    assert "More than one tokenizer is used" in str(execinfo.value)

@pytest.mark.parametrize(
    "_config",
    [
        {"pipeline": [{"name": "WhitespaceTokenizer"}, {"name": "SpacyFeaturizer"}]},
        pytest.param(
            {
                "pipeline": [
                    {"name": "WhitespaceTokenizer"},
                    {"name": "ConveRTFeaturizer"},
                ]
            },
        ),
        pytest.param(
            {
                "pipeline": [
                    {"name": "ConveRTTokenizer"},
                    {"name": "LanguageModelFeaturizer"},
                ]
            },
        ),
    ],
)
@pytest.mark.skip_on_windows
def test_missing_required_component(_config):
    with pytest.raises(config.InvalidConfigError) as execinfo:
        Trainer(config.RasaNLUModelConfig(_config))
    assert "Add required components to the pipeline" in str(execinfo.value)

@pytest.mark.parametrize(
    "pipeline_config", [{"pipeline": [{"name": "CountVectorsFeaturizer"}]}]
)
def test_missing_property(pipeline_config):
    with pytest.raises(config.InvalidConfigError) as execinfo:
        Trainer(config.RasaNLUModelConfig(pipeline_config))
    assert "Add required components to the pipeline" in str(execinfo.value)

def test_default_config_file():
    final_config = config.RasaNLUModelConfig()
    assert len(final_config) > 1

def test_set_attr_on_component():
    _config = RasaNLUModelConfig(
        {
            "language": "en",
            "pipeline": [
                {"name": "SpacyNLP"},
                {"name": "SpacyTokenizer"},
                {"name": "SpacyFeaturizer"},
                {"name": "DIETClassifier"},
            ],
        }
    )
    idx_classifier = _config.component_names.index("DIETClassifier")
    idx_tokenizer = _config.component_names.index("SpacyTokenizer")

    _config.set_component_attr(idx_classifier, epochs=10)

    assert _config.for_component(idx_tokenizer) == {"name": "SpacyTokenizer"}
    assert _config.for_component(idx_classifier) == {
        "name": "DIETClassifier",
        "epochs": 10,
    }

def test_override_defaults_supervised_embeddings_pipeline():
    builder = ComponentBuilder()

    _config = RasaNLUModelConfig(
        {
            "language": "en",
            "pipeline": [
                {"name": "SpacyNLP"},
                {"name": "SpacyTokenizer"},
                {"name": "SpacyFeaturizer", "pooling": "max"},
                {
                    "name": "DIETClassifier",
                    "epochs": 10,
                    "hidden_layers_sizes": {"text": [256, 128]},
                },
            ],
        }
    )

    idx_featurizer = _config.component_names.index("SpacyFeaturizer")
    idx_classifier = _config.component_names.index("DIETClassifier")

    component1 = builder.create_component(
        _config.for_component(idx_featurizer), _config
    )
    assert component1.component_config["pooling"] == "max"

    component2 = builder.create_component(
        _config.for_component(idx_classifier), _config
    )
    assert component2.component_config["epochs"] == 10
    assert (
        component2.defaults["hidden_layers_sizes"].keys()
        == component2.component_config["hidden_layers_sizes"].keys()
    )

def config_files_in(config_directory: Text):
    return [
        os.path.join(config_directory, f)
        for f in os.listdir(config_directory)
        if os.path.isfile(os.path.join(config_directory, f))
    ]

@pytest.mark.parametrize(
    "config_file",
    config_files_in("data/configs_for_docs") + config_files_in("docker/configs"),
)
async def test_train_docker_and_docs_configs(
    config_file: Text, monkeypatch: MonkeyPatch
):
    monkeypatch.setattr(autoconfig, "_dump_config", Mock())
    importer = RasaFileImporter(config_file=config_file)
    imported_config = await importer.get_config()

    loaded_config = config.load(imported_config)

    assert len(loaded_config.component_names) > 1
    assert loaded_config.language == imported_config["language"]

@pytest.mark.parametrize(
    "config_path, data_path, expected_warning_excerpts",
    [
        (
            "data/test_config/config_supervised_embeddings.yml",
            "data/examples/rasa",
            ["add a 'ResponseSelector'"],
        ),
        (
            "data/test_config/config_spacy_entity_extractor.yml",
            "data/test/md_converted_to_json.json",
            [f"add one of {TRAINABLE_EXTRACTORS}"],
        ),
        (
            "data/test_config/config_crf_no_regex.yml",
            "data/test/duplicate_intents_markdown/demo-rasa-intents-2.md",
            ["training data with regexes", "include a 'RegexFeaturizer'"],
        ),
        (
            "data/test_config/config_crf_no_regex.yml",
            "data/test/lookup_tables/lookup_table.json",
            ["training data consisting of lookup tables", "add a 'RegexFeaturizer'"],
        ),
        (
            "data/test_config/config_spacy_entity_extractor.yml",
            "data/test/lookup_tables/lookup_table.json",
            [
                "add a 'DIETClassifier' or a 'CRFEntityExtractor' with the 'pattern' feature"
            ],
        ),
        (
            "data/test_config/config_crf_no_pattern_feature.yml",
            "data/test/lookup_tables/lookup_table.md",
            "your NLU pipeline's 'CRFEntityExtractor' does not include the 'pattern' feature",
        ),
        (
            "data/test_config/config_crf_no_synonyms.yml",
            "data/test/markdown_single_sections/synonyms_only.md",
            ["add an 'EntitySynonymMapper'"],
        ),
        (
            "data/test_config/config_embedding_intent_response_selector.yml",
            "data/test/demo-rasa-composite-entities.md",
            ["include either 'DIETClassifier' or 'CRFEntityExtractor'"],
        ),
    ],
)
def test_validate_required_components_from_data(
    config_path: Text, data_path: Text, expected_warning_excerpts: List[Text]
):
    loaded_config = config.load(config_path)
    trainer = Trainer(loaded_config)
    training_data = load_data(data_path)
    with pytest.warns(UserWarning) as record:
        components.validate_required_components_from_data(
            trainer.pipeline, training_data
        )
    assert len(record) == 1
    assert all(
        [excerpt in record[0].message.args[0]] for excerpt in expected_warning_excerpts
    )
<commit_msg>remove unused import<commit_after>import os
from typing import Text, List
from unittest.mock import Mock

import pytest
from _pytest.monkeypatch import MonkeyPatch

from rasa.importers import autoconfig
from rasa.importers.rasa import RasaFileImporter
from rasa.nlu.config import RasaNLUModelConfig
from rasa.nlu import config, load_data
from rasa.nlu import components
from rasa.nlu.components import ComponentBuilder
from rasa.nlu.constants import TRAINABLE_EXTRACTORS
from rasa.nlu.model import Trainer
from tests.nlu.utilities import write_file_config

def test_blank_config(blank_config):
    file_config = {}
    f = write_file_config(file_config)
    final_config = config.load(f.name)

    assert final_config.as_dict() == blank_config.as_dict()

def test_invalid_config_json(tmp_path):
    file_config = """pipeline: [pretrained_embeddings_spacy"""  # invalid yaml

    f = tmp_path / "tmp_config_file.json"
    f.write_text(file_config)

    with pytest.raises(config.InvalidConfigError):
        config.load(str(f))

def test_invalid_many_tokenizers_in_config():
    nlu_config = {
        "pipeline": [{"name": "WhitespaceTokenizer"}, {"name": "SpacyTokenizer"}]
    }

    with pytest.raises(config.InvalidConfigError) as execinfo:
        Trainer(config.RasaNLUModelConfig(nlu_config))
    assert "More than one tokenizer is used" in str(execinfo.value)

@pytest.mark.parametrize(
    "_config",
    [
        {"pipeline": [{"name": "WhitespaceTokenizer"}, {"name": "SpacyFeaturizer"}]},
        pytest.param(
            {
                "pipeline": [
                    {"name": "WhitespaceTokenizer"},
                    {"name": "ConveRTFeaturizer"},
                ]
            },
        ),
        pytest.param(
            {
                "pipeline": [
                    {"name": "ConveRTTokenizer"},
                    {"name": "LanguageModelFeaturizer"},
                ]
            },
        ),
    ],
)
@pytest.mark.skip_on_windows
def test_missing_required_component(_config):
    with pytest.raises(config.InvalidConfigError) as execinfo:
        Trainer(config.RasaNLUModelConfig(_config))
    assert "Add required components to the pipeline" in str(execinfo.value)

@pytest.mark.parametrize(
    "pipeline_config", [{"pipeline": [{"name": "CountVectorsFeaturizer"}]}]
)
def test_missing_property(pipeline_config):
    with pytest.raises(config.InvalidConfigError) as execinfo:
        Trainer(config.RasaNLUModelConfig(pipeline_config))
    assert "Add required components to the pipeline" in str(execinfo.value)

def test_default_config_file():
    final_config = config.RasaNLUModelConfig()
    assert len(final_config) > 1

def test_set_attr_on_component():
    _config = RasaNLUModelConfig(
        {
            "language": "en",
            "pipeline": [
                {"name": "SpacyNLP"},
                {"name": "SpacyTokenizer"},
                {"name": "SpacyFeaturizer"},
                {"name": "DIETClassifier"},
            ],
        }
    )
    idx_classifier = _config.component_names.index("DIETClassifier")
    idx_tokenizer = _config.component_names.index("SpacyTokenizer")

    _config.set_component_attr(idx_classifier, epochs=10)

    assert _config.for_component(idx_tokenizer) == {"name": "SpacyTokenizer"}
    assert _config.for_component(idx_classifier) == {
        "name": "DIETClassifier",
        "epochs": 10,
    }

def test_override_defaults_supervised_embeddings_pipeline():
    builder = ComponentBuilder()

    _config = RasaNLUModelConfig(
        {
            "language": "en",
            "pipeline": [
                {"name": "SpacyNLP"},
                {"name": "SpacyTokenizer"},
                {"name": "SpacyFeaturizer", "pooling": "max"},
                {
                    "name": "DIETClassifier",
                    "epochs": 10,
                    "hidden_layers_sizes": {"text": [256, 128]},
                },
            ],
        }
    )

    idx_featurizer = _config.component_names.index("SpacyFeaturizer")
    idx_classifier = _config.component_names.index("DIETClassifier")

    component1 = builder.create_component(
        _config.for_component(idx_featurizer), _config
    )
    assert component1.component_config["pooling"] == "max"

    component2 = builder.create_component(
        _config.for_component(idx_classifier), _config
    )
    assert component2.component_config["epochs"] == 10
    assert (
        component2.defaults["hidden_layers_sizes"].keys()
        == component2.component_config["hidden_layers_sizes"].keys()
    )

def config_files_in(config_directory: Text):
    return [
        os.path.join(config_directory, f)
        for f in os.listdir(config_directory)
        if os.path.isfile(os.path.join(config_directory, f))
    ]

@pytest.mark.parametrize(
    "config_file",
    config_files_in("data/configs_for_docs") + config_files_in("docker/configs"),
)
async def test_train_docker_and_docs_configs(
    config_file: Text, monkeypatch: MonkeyPatch
):
    monkeypatch.setattr(autoconfig, "_dump_config", Mock())
    importer = RasaFileImporter(config_file=config_file)
    imported_config = await importer.get_config()

    loaded_config = config.load(imported_config)

    assert len(loaded_config.component_names) > 1
    assert loaded_config.language == imported_config["language"]

@pytest.mark.parametrize(
    "config_path, data_path, expected_warning_excerpts",
    [
        (
            "data/test_config/config_supervised_embeddings.yml",
            "data/examples/rasa",
            ["add a 'ResponseSelector'"],
        ),
        (
            "data/test_config/config_spacy_entity_extractor.yml",
            "data/test/md_converted_to_json.json",
            [f"add one of {TRAINABLE_EXTRACTORS}"],
        ),
        (
            "data/test_config/config_crf_no_regex.yml",
            "data/test/duplicate_intents_markdown/demo-rasa-intents-2.md",
            ["training data with regexes", "include a 'RegexFeaturizer'"],
        ),
        (
            "data/test_config/config_crf_no_regex.yml",
            "data/test/lookup_tables/lookup_table.json",
            ["training data consisting of lookup tables", "add a 'RegexFeaturizer'"],
        ),
        (
            "data/test_config/config_spacy_entity_extractor.yml",
            "data/test/lookup_tables/lookup_table.json",
            [
                "add a 'DIETClassifier' or a 'CRFEntityExtractor' with the 'pattern' feature"
            ],
        ),
        (
            "data/test_config/config_crf_no_pattern_feature.yml",
            "data/test/lookup_tables/lookup_table.md",
            "your NLU pipeline's 'CRFEntityExtractor' does not include the 'pattern' feature",
        ),
        (
            "data/test_config/config_crf_no_synonyms.yml",
            "data/test/markdown_single_sections/synonyms_only.md",
            ["add an 'EntitySynonymMapper'"],
        ),
        (
            "data/test_config/config_embedding_intent_response_selector.yml",
            "data/test/demo-rasa-composite-entities.md",
            ["include either 'DIETClassifier' or 'CRFEntityExtractor'"],
        ),
    ],
)
def test_validate_required_components_from_data(
    config_path: Text, data_path: Text, expected_warning_excerpts: List[Text]
):
    loaded_config = config.load(config_path)
    trainer = Trainer(loaded_config)
    training_data = load_data(data_path)
    with pytest.warns(UserWarning) as record:
        components.validate_required_components_from_data(
            trainer.pipeline, training_data
        )
    assert len(record) == 1
    assert all(
        [excerpt in record[0].message.args[0]] for excerpt in expected_warning_excerpts
    )
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Author: Bo Maryniuk <bo@suse.de>
#
# Copyright 2017 SUSE LLC
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Ansible Support
===============

This module can have an optional minion-level
configuration in /etc/salt/minion.d/ as follows:

  ansible_timeout: 1200

The timeout is how many seconds Salt should wait for
any Ansible module to respond.
"""

from __future__ import absolute_import, print_function, unicode_literals

import fnmatch
import importlib
import json
import logging
import os
import subprocess
import sys

import salt.utils.decorators.path
import salt.utils.json
import salt.utils.platform
import salt.utils.timed_subprocess
import salt.utils.yaml
from salt.exceptions import CommandExecutionError, LoaderError
from salt.ext import six
from salt.utils.decorators import depends

try:
    import ansible
    import ansible.constants  # pylint: disable=no-name-in-module
    import ansible.modules  # pylint: disable=no-name-in-module
except ImportError:
    ansible = None

# Function alias to make sure not to shadow built-in's
__func_alias__ = {
    'list_': 'list'
}

__virtualname__ = "ansible"

log = logging.getLogger(__name__)

class AnsibleModuleResolver(object):
    """
    This class is to resolve all available modules in Ansible.
    """

    def __init__(self, opts):
        self.opts = opts
        self._modules_map = {}

    def _get_modules_map(self, path=None):
        """
        Get installed Ansible modules
        :return:
        """
        paths = {}
        root = ansible.modules.__path__[0]
        if not path:
            path = root
        for p_el in os.listdir(path):
            p_el_path = os.path.join(path, p_el)
            if os.path.islink(p_el_path):
                continue
            if os.path.isdir(p_el_path):
                paths.update(self._get_modules_map(p_el_path))
            else:
                if (
                    any(p_el.startswith(elm) for elm in ["__", "."])
                    or not p_el.endswith(".py")
                    or p_el in ansible.constants.IGNORE_FILES
                ):
                    continue
                p_el_path = p_el_path.replace(root, "").split(".")[0]
                als_name = (
                    p_el_path.replace(".", "").replace("/", "", 1).replace("/", ".")
                )
                paths[als_name] = p_el_path

        return paths

    def load_module(self, module):
        """
        Introspect Ansible module.

        :param module:
        :return:
        """
        m_ref = self._modules_map.get(module)
        if m_ref is None:
            raise LoaderError('Module "{0}" was not found'.format(module))
        mod = importlib.import_module(
            "ansible.modules{0}".format(
                ".".join([elm.split(".")[0] for elm in m_ref.split(os.path.sep)])
            )
        )

        return mod

    def get_modules_list(self, pattern=None):
        """
        Return module map references.
        :return:
        """
        if pattern and "*" not in pattern:
            pattern = "*{0}*".format(pattern)
        modules = []
        for m_name, m_path in self._modules_map.items():
            m_path = m_path.split(".")[0]
            m_name = ".".join([elm for elm in m_path.split(os.path.sep) if elm])
            if pattern and fnmatch.fnmatch(m_name, pattern) or not pattern:
                modules.append(m_name)
        return sorted(modules)

    def resolve(self):
        log.debug("Resolving Ansible modules")
        self._modules_map = self._get_modules_map()
        return self

    def install(self):
        log.debug("Installing Ansible modules")
        return self

class AnsibleModuleCaller(object):
    DEFAULT_TIMEOUT = 1200  # seconds (20 minutes)
    OPT_TIMEOUT_KEY = "ansible_timeout"

    def __init__(self, resolver):
        self._resolver = resolver
        self.timeout = self._resolver.opts.get(
            self.OPT_TIMEOUT_KEY, self.DEFAULT_TIMEOUT
        )

    def call(self, module, *args, **kwargs):
        """
        Call an Ansible module by invoking it.
        :param module: the name of the module.
        :param args: Arguments to the module
        :param kwargs: keywords to the module
        :return:
        """

        module = self._resolver.load_module(module)
        if not hasattr(module, "main"):
            raise CommandExecutionError(
                "This module is not callable "
                '(see "ansible.help {0}")'.format(
                    module.__name__.replace("ansible.modules.", "")
                )
            )
        if args:
            kwargs["_raw_params"] = " ".join(args)
        js_args = str(
            '{{"ANSIBLE_MODULE_ARGS": {args}}}'
        )  # future lint: disable=blacklisted-function
        js_args = js_args.format(args=salt.utils.json.dumps(kwargs))

        proc_out = salt.utils.timed_subprocess.TimedProc(
            ["echo", "{0}".format(js_args)],
            stdout=subprocess.PIPE,
            timeout=self.timeout,
        )
        proc_out.run()
        if six.PY2:
            proc_out_stdout = proc_out.stdout
        else:
            proc_out_stdout = proc_out.stdout.decode()
        proc_exc = salt.utils.timed_subprocess.TimedProc(
            [sys.executable, module.__file__],
            stdin=proc_out_stdout,
            stdout=subprocess.PIPE,
            timeout=self.timeout
        )
        proc_exc.run()

        try:
            out = salt.utils.json.loads(proc_exc.stdout)
        except ValueError as ex:
            out = {
                "Error": (
                    proc_exc.stderr and (proc_exc.stderr + ".") or six.text_type(ex)
                )
            }
            if proc_exc.stdout:
                out["Given JSON output"] = proc_exc.stdout
            return out

        if "invocation" in out:
            del out["invocation"]

        out["timeout"] = self.timeout

        return out

_resolver = None
_caller = None

def _set_callables(modules):
    """
    Set all Ansible modules callables
    :return:
    """

    def _set_function(cmd_name, doc):
        """
        Create a Salt function for the Ansible module.
        """

        def _cmd(*args, **kw):
            """
            Call an Ansible module as a function from the Salt.
            """
            kwargs = {}
            if kw.get("__pub_arg"):
                for _kw in kw.get("__pub_arg", []):
                    if isinstance(_kw, dict):
                        kwargs = _kw
                        break

            return _caller.call(cmd_name, *args, **kwargs)

        _cmd.__doc__ = doc
        return _cmd

    for mod in modules:
        setattr(sys.modules[__name__], mod, _set_function(mod, "Available"))

def __virtual__():
    """
    Ansible module caller.
    :return:
    """
    if salt.utils.platform.is_windows():
        return False, "The ansiblegate module isn't supported on Windows"
    ret = ansible is not None
    msg = not ret and "Ansible is not installed on this system" or None
    if ret:
        global _resolver
        global _caller
        _resolver = AnsibleModuleResolver(__opts__).resolve().install()
        _caller = AnsibleModuleCaller(_resolver)
        _set_callables(list())
    return __virtualname__

@depends("ansible")
def help(module=None, *args):
    """
    Display help on Ansible standard module.

    :param module:
    :return:
    """
    if not module:
        raise CommandExecutionError(
            "Please tell me what module you want to have helped with. "
            'Or call "ansible.list" to know what is available.'
        )
    try:
        module = _resolver.load_module(module)
    except (ImportError, LoaderError) as err:
        raise CommandExecutionError(
            'Module "{0}" is currently not functional on your system.'.format(module)
        )

    doc = {}
    ret = {}
    for docset in module.DOCUMENTATION.split("---"):
        try:
            docset = salt.utils.yaml.safe_load(docset)
            if docset:
                doc.update(docset)
        except Exception as err:  # pylint: disable=broad-except
            log.error("Error parsing doc section: %s", err)
    if not args:
        if "description" in doc:
            description = doc.get("description") or ""
            del doc["description"]
            ret["Description"] = description
        ret[
            'Available sections on module "{}"'.format(
                module.__name__.replace("ansible.modules.", "")
            )
        ] = list(doc)
    else:
        for arg in args:
            info = doc.get(arg)
            if info is not None:
                ret[arg] = info

    return ret

@depends("ansible")
def list_(pattern=None):
    """
    Lists available modules.
    :return:
    """
    return _resolver.get_modules_list(pattern=pattern)

@salt.utils.decorators.path.which("ansible-playbook")
def playbooks(
    playbook,
    rundir=None,
    check=False,
    diff=False,
    extra_vars=None,
    flush_cache=False,
    forks=5,
    inventory=None,
    limit=None,
    list_hosts=False,
    list_tags=False,
    list_tasks=False,
    module_path=None,
    skip_tags=None,
    start_at_task=None,
    syntax_check=False,
    tags=None,
    playbook_kwargs=None,
):
    """
    Run Ansible Playbooks

    :param playbook: Which playbook to run.
    :param rundir: Directory to run `ansible-playbook` in. (Default: None)
    :param check: don't make any changes; instead, try to predict some
                  of the changes that may occur (Default: False)
    :param diff: when changing (small) files and templates, show the
                 differences in those files; works great with --check
                 (default: False)
    :param extra_vars: set additional variables as key=value or YAML/JSON, if
                       filename prepend with @, (default: None)
    :param flush_cache: clear the fact cache for every host in inventory
                        (default: False)
    :param forks: specify number of parallel processes to use
                  (Default: 5)
    :param inventory: specify inventory host path or comma separated host
                      list. (Default: None) (Ansible's default is /etc/ansible/hosts)
    :param limit: further limit selected hosts to an additional pattern (Default: None)
    :param list_hosts: outputs a list of matching hosts; does not execute anything else
                       (Default: False)
    :param list_tags: list all available tags (Default: False)
    :param list_tasks: list all tasks that would be executed (Default: False)
    :param module_path: prepend colon-separated path(s) to module library. (Default: None)
    :param skip_tags: only run plays and tasks whose tags do not match these
                      values (Default: False)
    :param start_at_task: start the playbook at the task matching this name (Default: None)
    :param: syntax_check: perform a syntax check on the playbook, but do not execute it
                          (Default: False)
    :param tags: only run plays and tasks tagged with these values (Default: None)

    :return: Playbook return

    CLI Example:

    .. code-block:: bash

        salt 'ansiblehost'  ansible.playbook playbook=/srv/playbooks/play.yml
    """
    command = ["ansible-playbook", playbook]
    if check:
        command.append("--check")
    if diff:
        command.append("--diff")
    if isinstance(extra_vars, dict):
        command.append("--extra-vars='{0}'".format(json.dumps(extra_vars)))
    elif isinstance(extra_vars, six.text_type) and extra_vars.startswith("@"):
        command.append("--extra-vars={0}".format(extra_vars))
    if flush_cache:
        command.append("--flush-cache")
    if inventory:
        command.append("--inventory={0}".format(inventory))
    if limit:
        command.append("--limit={0}".format(limit))
    if list_hosts:
        command.append("--list-hosts")
    if list_tags:
        command.append("--list-tags")
    if list_tasks:
        command.append("--list-tasks")
    if module_path:
        command.append("--module-path={0}".format(module_path))
    if skip_tags:
        command.append("--skip-tags={0}".format(skip_tags))
    if start_at_task:
        command.append("--start-at-task={0}".format(start_at_task))
    if syntax_check:
        command.append("--syntax-check")
    if tags:
        command.append("--tags={0}".format(tags))
    if playbook_kwargs:
        for key, value in six.iteritems(playbook_kwargs):
            key = key.replace("_", "-")
            if value is True:
                command.append("--{0}".format(key))
            elif isinstance(value, six.text_type):
                command.append("--{0}={1}".format(key, value))
            elif isinstance(value, dict):
                command.append("--{0}={1}".format(key, json.dumps(value)))
    command.append("--forks={0}".format(forks))
    cmd_kwargs = {
        "env": {"ANSIBLE_STDOUT_CALLBACK": "json", "ANSIBLE_RETRY_FILES_ENABLED": "0"},
        "cwd": rundir,
        "cmd": " ".join(command),
    }
    ret = __salt__["cmd.run_all"](**cmd_kwargs)
    log.debug("Ansible Playbook Return: %s", ret)
    retdata = json.loads(ret["stdout"])
    if ret["retcode"]:
        __context__["retcode"] = ret["retcode"]
    return retdata
<commit_msg>Use of salt.utils.stringutils module to handle strings<commit_after># -*- coding: utf-8 -*-
#
# Author: Bo Maryniuk <bo@suse.de>
#
# Copyright 2017 SUSE LLC
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Ansible Support
===============

This module can have an optional minion-level
configuration in /etc/salt/minion.d/ as follows:

  ansible_timeout: 1200

The timeout is how many seconds Salt should wait for
any Ansible module to respond.
"""

from __future__ import absolute_import, print_function, unicode_literals

import fnmatch
import importlib
import json
import logging
import os
import subprocess
import sys

import salt.utils.decorators.path
import salt.utils.json
import salt.utils.platform
import salt.utils.timed_subprocess
import salt.utils.yaml
import salt.utils.stringutils
from salt.exceptions import CommandExecutionError, LoaderError
from salt.ext import six
from salt.utils.decorators import depends

try:
    import ansible
    import ansible.constants  # pylint: disable=no-name-in-module
    import ansible.modules  # pylint: disable=no-name-in-module
except ImportError:
    ansible = None

# Function alias to make sure not to shadow built-in's
__func_alias__ = {
    'list_': 'list'
}

__virtualname__ = "ansible"

log = logging.getLogger(__name__)

class AnsibleModuleResolver(object):
    """
    This class is to resolve all available modules in Ansible.
    """

    def __init__(self, opts):
        self.opts = opts
        self._modules_map = {}

    def _get_modules_map(self, path=None):
        """
        Get installed Ansible modules
        :return:
        """
        paths = {}
        root = ansible.modules.__path__[0]
        if not path:
            path = root
        for p_el in os.listdir(path):
            p_el_path = os.path.join(path, p_el)
            if os.path.islink(p_el_path):
                continue
            if os.path.isdir(p_el_path):
                paths.update(self._get_modules_map(p_el_path))
            else:
                if (
                    any(p_el.startswith(elm) for elm in ["__", "."])
                    or not p_el.endswith(".py")
                    or p_el in ansible.constants.IGNORE_FILES
                ):
                    continue
                p_el_path = p_el_path.replace(root, "").split(".")[0]
                als_name = (
                    p_el_path.replace(".", "").replace("/", "", 1).replace("/", ".")
                )
                paths[als_name] = p_el_path

        return paths

    def load_module(self, module):
        """
        Introspect Ansible module.

        :param module:
        :return:
        """
        m_ref = self._modules_map.get(module)
        if m_ref is None:
            raise LoaderError('Module "{0}" was not found'.format(module))
        mod = importlib.import_module(
            "ansible.modules{0}".format(
                ".".join([elm.split(".")[0] for elm in m_ref.split(os.path.sep)])
            )
        )

        return mod

    def get_modules_list(self, pattern=None):
        """
        Return module map references.
        :return:
        """
        if pattern and "*" not in pattern:
            pattern = "*{0}*".format(pattern)
        modules = []
        for m_name, m_path in self._modules_map.items():
            m_path = m_path.split(".")[0]
            m_name = ".".join([elm for elm in m_path.split(os.path.sep) if elm])
            if pattern and fnmatch.fnmatch(m_name, pattern) or not pattern:
                modules.append(m_name)
        return sorted(modules)

    def resolve(self):
        log.debug("Resolving Ansible modules")
        self._modules_map = self._get_modules_map()
        return self

    def install(self):
        log.debug("Installing Ansible modules")
        return self

class AnsibleModuleCaller(object):
    DEFAULT_TIMEOUT = 1200  # seconds (20 minutes)
    OPT_TIMEOUT_KEY = "ansible_timeout"

    def __init__(self, resolver):
        self._resolver = resolver
        self.timeout = self._resolver.opts.get(
            self.OPT_TIMEOUT_KEY, self.DEFAULT_TIMEOUT
        )

    def call(self, module, *args, **kwargs):
        """
        Call an Ansible module by invoking it.
        :param module: the name of the module.
        :param args: Arguments to the module
        :param kwargs: keywords to the module
        :return:
        """

        module = self._resolver.load_module(module)
        if not hasattr(module, "main"):
            raise CommandExecutionError(
                "This module is not callable "
                '(see "ansible.help {0}")'.format(
                    module.__name__.replace("ansible.modules.", "")
                )
            )
        if args:
            kwargs["_raw_params"] = " ".join(args)
        js_args = str(
            '{{"ANSIBLE_MODULE_ARGS": {args}}}'
        )  # future lint: disable=blacklisted-function
        js_args = js_args.format(args=salt.utils.json.dumps(kwargs))

        proc_out = salt.utils.timed_subprocess.TimedProc(
            ["echo", "{0}".format(js_args)],
            stdout=subprocess.PIPE,
            timeout=self.timeout,
        )
        proc_out.run()
        proc_out_stdout = salt.utils.stringutils.to_str(proc_out.stdout)
        proc_exc = salt.utils.timed_subprocess.TimedProc(
            [sys.executable, module.__file__],
            stdin=proc_out_stdout,
            stdout=subprocess.PIPE,
            timeout=self.timeout
        )
        proc_exc.run()

        try:
            out = salt.utils.json.loads(proc_exc.stdout)
        except ValueError as ex:
            out = {
                "Error": (
                    proc_exc.stderr and (proc_exc.stderr + ".") or six.text_type(ex)
                )
            }
            if proc_exc.stdout:
                out["Given JSON output"] = proc_exc.stdout
            return out

        if "invocation" in out:
            del out["invocation"]

        out["timeout"] = self.timeout

        return out

_resolver = None
_caller = None

def _set_callables(modules):
    """
    Set all Ansible modules callables
    :return:
    """

    def _set_function(cmd_name, doc):
        """
        Create a Salt function for the Ansible module.
        """

        def _cmd(*args, **kw):
            """
            Call an Ansible module as a function from the Salt.
            """
            kwargs = {}
            if kw.get("__pub_arg"):
                for _kw in kw.get("__pub_arg", []):
                    if isinstance(_kw, dict):
                        kwargs = _kw
                        break

            return _caller.call(cmd_name, *args, **kwargs)

        _cmd.__doc__ = doc
        return _cmd

    for mod in modules:
        setattr(sys.modules[__name__], mod, _set_function(mod, "Available"))

def __virtual__():
    """
    Ansible module caller.
    :return:
    """
    if salt.utils.platform.is_windows():
        return False, "The ansiblegate module isn't supported on Windows"
    ret = ansible is not None
    msg = not ret and "Ansible is not installed on this system" or None
    if ret:
        global _resolver
        global _caller
        _resolver = AnsibleModuleResolver(__opts__).resolve().install()
        _caller = AnsibleModuleCaller(_resolver)
        _set_callables(list())
    return __virtualname__

@depends("ansible")
def help(module=None, *args):
    """
    Display help on Ansible standard module.

    :param module:
    :return:
    """
    if not module:
        raise CommandExecutionError(
            "Please tell me what module you want to have helped with. "
            'Or call "ansible.list" to know what is available.'
        )
    try:
        module = _resolver.load_module(module)
    except (ImportError, LoaderError) as err:
        raise CommandExecutionError(
            'Module "{0}" is currently not functional on your system.'.format(module)
        )

    doc = {}
    ret = {}
    for docset in module.DOCUMENTATION.split("---"):
        try:
            docset = salt.utils.yaml.safe_load(docset)
            if docset:
                doc.update(docset)
        except Exception as err:  # pylint: disable=broad-except
            log.error("Error parsing doc section: %s", err)
    if not args:
        if "description" in doc:
            description = doc.get("description") or ""
            del doc["description"]
            ret["Description"] = description
        ret[
            'Available sections on module "{}"'.format(
                module.__name__.replace("ansible.modules.", "")
            )
        ] = list(doc)
    else:
        for arg in args:
            info = doc.get(arg)
            if info is not None:
                ret[arg] = info

    return ret

@depends("ansible")
def list_(pattern=None):
    """
    Lists available modules.
    :return:
    """
    return _resolver.get_modules_list(pattern=pattern)

@salt.utils.decorators.path.which("ansible-playbook")
def playbooks(
    playbook,
    rundir=None,
    check=False,
    diff=False,
    extra_vars=None,
    flush_cache=False,
    forks=5,
    inventory=None,
    limit=None,
    list_hosts=False,
    list_tags=False,
    list_tasks=False,
    module_path=None,
    skip_tags=None,
    start_at_task=None,
    syntax_check=False,
    tags=None,
    playbook_kwargs=None,
):
    """
    Run Ansible Playbooks

    :param playbook: Which playbook to run.
    :param rundir: Directory to run `ansible-playbook` in. (Default: None)
    :param check: don't make any changes; instead, try to predict some
                  of the changes that may occur (Default: False)
    :param diff: when changing (small) files and templates, show the
                 differences in those files; works great with --check
                 (default: False)
    :param extra_vars: set additional variables as key=value or YAML/JSON, if
                       filename prepend with @, (default: None)
    :param flush_cache: clear the fact cache for every host in inventory
                        (default: False)
    :param forks: specify number of parallel processes to use
                  (Default: 5)
    :param inventory: specify inventory host path or comma separated host
                      list. (Default: None) (Ansible's default is /etc/ansible/hosts)
    :param limit: further limit selected hosts to an additional pattern (Default: None)
    :param list_hosts: outputs a list of matching hosts; does not execute anything else
                       (Default: False)
    :param list_tags: list all available tags (Default: False)
    :param list_tasks: list all tasks that would be executed (Default: False)
    :param module_path: prepend colon-separated path(s) to module library. (Default: None)
    :param skip_tags: only run plays and tasks whose tags do not match these
                      values (Default: False)
    :param start_at_task: start the playbook at the task matching this name (Default: None)
    :param: syntax_check: perform a syntax check on the playbook, but do not execute it
                          (Default: False)
    :param tags: only run plays and tasks tagged with these values (Default: None)

    :return: Playbook return

    CLI Example:

    .. code-block:: bash

        salt 'ansiblehost'  ansible.playbook playbook=/srv/playbooks/play.yml
    """
    command = ["ansible-playbook", playbook]
    if check:
        command.append("--check")
    if diff:
        command.append("--diff")
    if isinstance(extra_vars, dict):
        command.append("--extra-vars='{0}'".format(json.dumps(extra_vars)))
    elif isinstance(extra_vars, six.text_type) and extra_vars.startswith("@"):
        command.append("--extra-vars={0}".format(extra_vars))
    if flush_cache:
        command.append("--flush-cache")
    if inventory:
        command.append("--inventory={0}".format(inventory))
    if limit:
        command.append("--limit={0}".format(limit))
    if list_hosts:
        command.append("--list-hosts")
    if list_tags:
        command.append("--list-tags")
    if list_tasks:
        command.append("--list-tasks")
    if module_path:
        command.append("--module-path={0}".format(module_path))
    if skip_tags:
        command.append("--skip-tags={0}".format(skip_tags))
    if start_at_task:
        command.append("--start-at-task={0}".format(start_at_task))
    if syntax_check:
        command.append("--syntax-check")
    if tags:
        command.append("--tags={0}".format(tags))
    if playbook_kwargs:
        for key, value in six.iteritems(playbook_kwargs):
            key = key.replace("_", "-")
            if value is True:
                command.append("--{0}".format(key))
            elif isinstance(value, six.text_type):
                command.append("--{0}={1}".format(key, value))
            elif isinstance(value, dict):
                command.append("--{0}={1}".format(key, json.dumps(value)))
    command.append("--forks={0}".format(forks))
    cmd_kwargs = {
        "env": {"ANSIBLE_STDOUT_CALLBACK": "json", "ANSIBLE_RETRY_FILES_ENABLED": "0"},
        "cwd": rundir,
        "cmd": " ".join(command),
    }
    ret = __salt__["cmd.run_all"](**cmd_kwargs)
    log.debug("Ansible Playbook Return: %s", ret)
    retdata = json.loads(ret["stdout"])
    if ret["retcode"]:
        __context__["retcode"] = ret["retcode"]
    return retdata
<|endoftext|>