<commit_before>from sympy.utilities.pytest import raises
from sympy.core import symbols, pi, S
from sympy.matrices import Identity, MatrixSymbol, ImmutableMatrix
from sympy.matrices.expressions import MatPow
from sympy.matrices.expressions.matexpr import ShapeError

n, m, l, k = symbols('n m l k', integer=True)
A = MatrixSymbol('A', n, m)
B = MatrixSymbol('B', m, l)
C = MatrixSymbol('C', n, n)
D = MatrixSymbol('D', n, n)
E = MatrixSymbol('E', m, n)

def test_entry():
    from sympy.concrete import Sum
    assert MatPow(A, 1)[0, 0] == A[0, 0]
    assert MatPow(C, 0)[0, 0] == 1
    assert MatPow(C, 0)[0, 1] == 0
    assert isinstance(MatPow(C, 2)[0, 0], Sum)

def test_as_explicit():
    A = ImmutableMatrix([[1, 2], [3, 4]])
    assert MatPow(A, 0).as_explicit() == ImmutableMatrix(Identity(2))
    assert MatPow(A, 1).as_explicit() == A
    assert MatPow(A, 2).as_explicit() == A**2
    assert MatPow(A, -1).as_explicit() == A.inv()
    assert MatPow(A, -2).as_explicit() == (A.inv())**2
    # less expensive than testing on a 2x2
    A = ImmutableMatrix([4]);
    assert MatPow(A, S.Half).as_explicit() == A**S.Half

def test_as_explicit_nonsquare():
    A = ImmutableMatrix([[1, 2, 3], [4, 5, 6]])
    assert MatPow(A, 1).as_explicit() == A
    raises(ShapeError, lambda: MatPow(A, 0).as_explicit())
    raises(ShapeError, lambda: MatPow(A, 2).as_explicit())
    raises(ShapeError, lambda: MatPow(A, -1).as_explicit())
    raises(ValueError, lambda: MatPow(A, pi).as_explicit())
<commit_msg>MatPow: doit tests<commit_after>from sympy.utilities.pytest import raises
from sympy.core import symbols, pi, S
from sympy.matrices import Identity, MatrixSymbol, ImmutableMatrix
from sympy.matrices.expressions import MatPow, MatAdd, MatMul
from sympy.matrices.expressions.matexpr import ShapeError

n, m, l, k = symbols('n m l k', integer=True)
A = MatrixSymbol('A', n, m)
B = MatrixSymbol('B', m, l)
C = MatrixSymbol('C', n, n)
D = MatrixSymbol('D', n, n)
E = MatrixSymbol('E', m, n)

def test_entry():
    from sympy.concrete import Sum
    assert MatPow(A, 1)[0, 0] == A[0, 0]
    assert MatPow(C, 0)[0, 0] == 1
    assert MatPow(C, 0)[0, 1] == 0
    assert isinstance(MatPow(C, 2)[0, 0], Sum)

def test_as_explicit():
    A = ImmutableMatrix([[1, 2], [3, 4]])
    assert MatPow(A, 0).as_explicit() == ImmutableMatrix(Identity(2))
    assert MatPow(A, 1).as_explicit() == A
    assert MatPow(A, 2).as_explicit() == A**2
    assert MatPow(A, -1).as_explicit() == A.inv()
    assert MatPow(A, -2).as_explicit() == (A.inv())**2
    # less expensive than testing on a 2x2
    A = ImmutableMatrix([4]);
    assert MatPow(A, S.Half).as_explicit() == A**S.Half

def test_as_explicit_nonsquare():
    A = ImmutableMatrix([[1, 2, 3], [4, 5, 6]])
    assert MatPow(A, 1).as_explicit() == A
    raises(ShapeError, lambda: MatPow(A, 0).as_explicit())
    raises(ShapeError, lambda: MatPow(A, 2).as_explicit())
    raises(ShapeError, lambda: MatPow(A, -1).as_explicit())
    raises(ValueError, lambda: MatPow(A, pi).as_explicit())

def test_doit_nonsquare_MatrixSymbol():
    assert MatPow(A, 1).doit() == A
    for r in [0, 2, -1, pi]:
        assert MatPow(A, r).doit() == MatPow(A, r)

def test_doit_square_MatrixSymbol_symsize():
    assert MatPow(C, 0).doit() == Identity(n)
    assert MatPow(C, 1).doit() == C
    for r in [2, -1, pi]:
        assert MatPow(C, r).doit() == MatPow(C, r)

def test_doit_with_MatrixBase():
    X = ImmutableMatrix([[1, 2], [3, 4]])
    assert MatPow(X, 0).doit() == ImmutableMatrix(Identity(2))
    assert MatPow(X, 1).doit() == X
    assert MatPow(X, 2).doit() == X**2
    assert MatPow(X, -1).doit() == X.inv()
    assert MatPow(X, -2).doit() == (X.inv())**2
    # less expensive than testing on a 2x2
    assert MatPow(ImmutableMatrix([4]), S.Half).doit() == ImmutableMatrix([2])

def test_doit_nonsquare():
    X = ImmutableMatrix([[1, 2, 3], [4, 5, 6]])
    assert MatPow(X, 1).doit() == X
    raises(ShapeError, lambda: MatPow(X, 0).doit())
    raises(ShapeError, lambda: MatPow(X, 2).doit())
    raises(ShapeError, lambda: MatPow(X, -1).doit())
    raises(ShapeError, lambda: MatPow(X, pi).doit())

def test_doit_nested_MatrixExpr():
    X = ImmutableMatrix([[1, 2], [3, 4]])
    Y = ImmutableMatrix([[2, 3], [4, 5]])
    assert MatPow(MatMul(X, Y), 2).doit() == (X*Y)**2
    assert MatPow(MatAdd(X, Y), 2).doit() == (X + Y)**2
<|endoftext|>


<commit_before>__version__ = '0.1.0a'

import sysadmintoolkit
import signal
import sys
import readline
import docutils.core
import tempfile
import collections

global plugin_instance

plugin_instance = None
modes_graph = collections.OrderedDict()
modes_graph['root'] = {'next_modes': ['config', 'operator'], 'desc': 'Display system properties, most commands require superuser privilege'}
modes_graph['config'] = {'next_modes': [], 'desc': 'Change system configuration'}
modes_graph['operator'] = {'next_modes': [], 'desc': 'View basic system properties'}

def get_plugin(logger, config):
    '''
    '''
    global plugin_instance, modes_graph

    if plugin_instance is None:
        plugin_instance = CommandPrompt(logger, config, modes_graph)

    return plugin_instance

class CommandPrompt(sysadmintoolkit.plugin.Plugin):
    '''
    Description
    -----------

    Provides basic commands to the sysadmin-toolkit CLI.

    Configuration
    -------------

    *plugin-dir*
      Directory where plugins reside. Plugins must be with the .py extension.

      Default: /etc/sysadmin-toolkit/plugin.d/

    *script-dir*
      Directory where scripts reside. The commandprompt plugin doesn't do anything with this value.

      Other plugins would use it for indirection if needed.

      Default: /etc/sysadmin-toolkit/scripts.d/

    '''
    def __init__(self, logger, config, modes_graph):
        super(CommandPrompt, self).__init__('commandprompt', logger, config, version=__version__)

        self.modes_graph = modes_graph

        self.add_command(sysadmintoolkit.command.LabelHelp('debug', self, 'Debug plugins'))
        self.add_command(sysadmintoolkit.command.ExecCommand('debug commandprompt', self, self.debug))

        # use_help = sysadmintoolkit.command.LabelHelp('use', self, 'Restrict the command to a specific plugin')
        # use_help._Label__is_reserved = True
        # self.add_command(use_help)

        # use_cmd = sysadmintoolkit.command.ExecCommand('use <plugin> <*>', self, self.cmd_input_with_scope)
        # use_cmd._Label__is_reserved = True
        # self.add_command(use_cmd)

        switchmode_help = sysadmintoolkit.command.LabelHelp('switchmode', self, 'Change the command prompt mode')
        switchmode_help._Label__is_reserved = True
        self.add_command(switchmode_help)

        switchmode_cmd = sysadmintoolkit.command.ExecCommand('switchmode <mode>', self, self.change_mode)
        switchmode_cmd._Label__is_reserved = True
        self.add_command(switchmode_cmd)

        help_help = sysadmintoolkit.command.LabelHelp('help', self, 'Displays plugin help page')
        help_help._Label__is_reserved = True
        self.add_command(help_help)

        help_cmd = sysadmintoolkit.command.ExecCommand('help <plugin>', self, self.show_plugin_help)
        help_cmd._Label__is_reserved = True
        self.add_command(help_cmd)

        exit_cmd = sysadmintoolkit.command.ExecCommand('exit', self, self.exit_last_commandprompt_level)
        exit_cmd._Label__is_reserved = True
        self.add_command(exit_cmd)

        quit_cmd = sysadmintoolkit.command.ExecCommand('quit', self, self.exit_all_commandprompt_levels)
        quit_cmd._Label__is_reserved = True
        self.add_command(quit_cmd)

        # set_cmd = sysadmintoolkit.command.ExecCommand('set', self, self.change_global_config)
        # set_cmd._Label__is_reserved = True
        # self.add_command(set_cmd)

        self.add_dynamic_keyword_fn('<plugin>', self.get_plugins)
        self.add_dynamic_keyword_fn('<mode>', self.get_modes)

        def sigint_handler(signum=None, frame=None):
            global plugin_instance
            plugin_instance.logger.info('Received signal %s' % signum)
            print '\n>> Use quit or exit to leave the program'
            sys.stdout.write('%s%s' % (plugin_instance.cmdstack[-1].prompt, readline.get_line_buffer()))
            sys.stdout.flush()

        signal.signal(signal.SIGINT, sigint_handler)

    def get_plugin_documentation(self, plugin):
        pluginname = plugin.get_name()

        plugin_doc_header = []

        plugin_doc_header.append(sysadmintoolkit.utils.trim_docstring(plugin.get_doc_header()))
        plugin_doc_header.append('')

        plugin_method_map = collections.OrderedDict()

        for mode in plugin.label_map:
            modename = mode

            if mode is '':
                modename = 'default'

            for label in plugin.label_map[mode]:
                command = plugin.label_map[mode][label]
                if not isinstance(command, sysadmintoolkit.command.ExecCommand):
                    continue

                method = command.get_function()

                if method not in plugin_method_map:
                    plugin_method_map[method] = {'longhelp': command.get_help(), 'shorthelp': command.get_shorthelp(), 'labels': {}}

                if label not in plugin_method_map[method]['labels']:
                    plugin_method_map[method]['labels'][label] = []

                plugin_method_map[method]['labels'][label].append(modename)
                plugin_method_map[method]['labels'][label].sort()

        plugin_command_doc = []

        plugin_method_map_keys = plugin_method_map.keys()
        for method in plugin_method_map_keys:
            method_dict = plugin_method_map[method]

            label_keys = method_dict['labels'].keys()
            label_keys.sort()

            plugin_command_doc.append('| ')

            for label in label_keys:
                plugin_command_doc.append('| *%s* (%s)' % (label, ', '.join(method_dict['labels'][label])))

                if label is label_keys[-1]:
                    plugin_command_doc.append('| ')

            plugin_command_doc.append('')

            plugin_command_doc.append(sysadmintoolkit.utils.indent_text(method_dict['longhelp'], indent=2))

            if method is not plugin_method_map_keys[-1]:
                plugin_command_doc.append('')
                plugin_command_doc.append('----')
                plugin_command_doc.append('')

        if len(plugin_command_doc):
            plugin_doc_header.append('Plugin Commands')
            plugin_doc_header.append('---------------')

        plugin_doc_footer = []
        plugin_doc_footer.append(plugin.get_doc_footer())
        plugin_doc_footer.append('')

        plugin_doc = '\n'.join(plugin_doc_header + plugin_command_doc + plugin_doc_footer)

        self.logger.debug('Using merged docstrings for plugin %s:\n%s' % (pluginname, plugin_doc))

        return plugin_doc

    # Dynamic keywords

    def get_plugins(self, user_input_obj):
        plugins = self.plugin_set.get_plugins()

        map_plugins = {}

        for plugin in plugins:
            map_plugins[plugin] = 'Use %s plugin' % plugin

        return map_plugins

    def get_modes(self, user_input_obj):
        current_mode = user_input_obj.get_mode()

        map_modes = {}

        for next_mode in self.modes_graph[current_mode]['next_modes']:
            map_modes[next_mode] = self.modes_graph[current_mode]['desc']

        return map_modes

    # Sysadmin-toolkit commands

    def debug(self, user_input_obj):
        '''
        Display registered commands in the current Command Prompt
        '''
        if len(self.cmdstack):
            cmdprompt = self.cmdstack[-1]
        else:
            self.logger.error('This command must be executed interactively, please execute in the CLI')
            return

        width = sysadmintoolkit.utils.get_terminal_size()[1]

        window_ratio = 0.55
        max_block_size = 150
        first_block_len = int(min(width, max_block_size) * window_ratio) - 1
        second_block_len = int(min(width, max_block_size) * (1 - window_ratio)) - 1

        print
        print '  Loaded plugins: %s' % ', '.join(self.plugin_set.get_plugins().keys())
        print
        print '  Command tree (%s)' % cmdprompt.mode
        print '  ' + ('=' * len('Command tree (%s)' % cmdprompt.mode))
        print
        print '    Definitions: non executable, %s, %s' % (sysadmintoolkit.utils.get_underline_text('executable'), \
                                                           sysadmintoolkit.utils.get_reversed_text('<dynamic>'))
        print
        print '  %s%s' % ('Keyword'.ljust(first_block_len), 'Help (plugin)')
        print '-' * (first_block_len + second_block_len)

        labeldict = cmdprompt.command_tree.get_sub_keywords_labels()
        labelkeys = labeldict.keys()
        labelkeys.sort()

        for label in labelkeys:
            # Build the label part of the debug
            labelstr = ''
            for keyword in cmdprompt.split_label(label)[:-1]:
                if cmdprompt.is_dynamic_keyword(keyword):
                    keyword = sysadmintoolkit.utils.get_reversed_text(keyword)

                labelstr += sysadmintoolkit.utils.get_grey_text(keyword) + ' '

            lastkeyword = cmdprompt.split_label(label)[-1]

            if cmdprompt.is_dynamic_keyword(lastkeyword):
                lastkeyword = sysadmintoolkit.utils.get_reversed_text(lastkeyword)

            if len(labeldict[label]['executable_commands']) > 0:
                lastkeyword = sysadmintoolkit.utils.get_underline_text(lastkeyword)

            labelstr += lastkeyword

            label_text_block = {'str': '%s' % labelstr, 'window_ratio': window_ratio, 'maxwidth': max_block_size * window_ratio, 'wrap': False}
            blank_text_block = {'str': ' ', 'window_ratio': window_ratio, 'maxwidth': max_block_size * window_ratio, 'wrap': True}

            # Build the label help blocks
            label_help_blocks = []

            label_help_plugins = labeldict[label]['help'].keys()
            label_help_plugins.sort()

            for plugin in label_help_plugins:
                label_help_blocks += [{'str':'%s (%s)' % (labeldict[label]['help'][plugin].get_shorthelp(), plugin), \
                                       'window_ratio': 1 - window_ratio, \
                                       'maxwidth': max_block_size * (1 - window_ratio), \
                                       'wrap': True
                                       }]

            # Build the command help blocks
            command_help_blocks = []

            command_help_plugins = labeldict[label]['executable_commands'].keys()
            command_help_plugins.sort()

            for plugin in command_help_plugins:
                command_help_blocks += [{'str': '%s (%s)' % (labeldict[label]['executable_commands'][plugin].get_shorthelp(), plugin), \
                                         'window_ratio': 1 - window_ratio, \
                                         'maxwidth': max_block_size * (1 - window_ratio), \
                                         'wrap': True
                                         }]

            # Print all the gathered information
            output_so_far = False

            for right_block in label_help_blocks + command_help_blocks:
                if not output_so_far:
                    left_block = label_text_block
                    output_so_far = True
                else:
                    left_block = blank_text_block

                sysadmintoolkit.utils.print_text_blocks([left_block, right_block])

                print

        print

        print '  Available modes'
        print '  ==============='
        print

        for mode in self.modes_graph:
            print '    %s %s' % (('%s:' % mode).ljust(20), self.modes_graph[mode]['desc'])
            print

            if len(self.modes_graph[mode]['next_modes']):
                print '    %s Next modes: %s' % (' ' * len(('%s:' % mode).ljust(20)), ', '.join(self.modes_graph[mode]['next_modes']))
            else:
                print '    %s Next modes: None' % (' ' * len(('%s:' % mode).ljust(20)))

            print

        print

    def show_plugin_help(self, user_input_obj):
        '''
        Display man page for this plugin

        '''
        pluginname = user_input_obj.get_entered_command().split()[1]
        plugin = self.plugin_set.get_plugins()[pluginname]

        self.logger.debug('Showing help for plugin %s' % pluginname)

        plugin_doc = self.get_plugin_documentation(plugin)

        plugin_doc_manpage_header = sysadmintoolkit.utils.trim_docstring("""
        %s=======
        %s Plugin
        %s=======
        """ % ('=' * len(pluginname), pluginname, '=' * len(pluginname)))

        self.logger.debug('Adding header for manpage:\n%s' % plugin_doc_manpage_header)

        doc_file = tempfile.NamedTemporaryFile()

        # Man page formatting
        manpage_format = docutils.core.publish_string('%s' % (plugin_doc), writer_name='manpage').splitlines()

        manpage_format[0] = '.TH %s COMMANDS "%s" "SYSADMIN-TOOLKIT PLUGIN" ""' % (plugin.get_name().upper(), plugin.get_version())

        if '.SH NAME' in manpage_format[1]:
            manpage_format[1] = ''

        if manpage_format[2].endswith('\\- '):
            manpage_format[2] = manpage_format[2][:-3]

        manpage_format = '\n'.join(manpage_format)

        doc_file.writelines('%s\n' % manpage_format)
        doc_file.flush()

        self.logger.debug('Using manpage for plugin %s:\n%s' % (pluginname, manpage_format))

        sysadmintoolkit.utils.execute_interactive_cmd('man %s' % doc_file.name, self.logger)

    def exit_last_commandprompt_level(self, user_input_obj):
        '''
        Exit the current command prompt

        '''
        return sysadmintoolkit.cmdprompt.EXIT_THIS_CMDPROMPT

    def exit_all_commandprompt_levels(self, user_input_obj):
        '''
        Exit the program

        '''
        return sysadmintoolkit.cmdprompt.EXIT_ALL_CMDPROMPT

    def change_mode(self, user_input_obj):
        '''
        Change the current command prompt mode

        '''
        newmode = user_input_obj.get_entered_command().split()[-1]
        old_commandprompt = user_input_obj.get_cmdprompt()

        self.logger.debug('Changing mode from %s to %s' % (user_input_obj.get_mode(), newmode))

        newmode_commandprompt = sysadmintoolkit.cmdprompt.CmdPrompt(old_commandprompt.logger, mode=newmode, \
                                                         shell_allowed=old_commandprompt.shell_allowed, \
                                                         is_interactive=old_commandprompt.is_interactive)

        def SIGWINCH_handler(signum=None, frame=None):
            (height, width) = sysadmintoolkit.utils.get_terminal_size()
            newmode_commandprompt.update_window_size(width, height)

        old_sigwinch_handler = signal.signal(signal.SIGWINCH, SIGWINCH_handler)
        SIGWINCH_handler()

        for plugin in old_commandprompt.get_plugins():
            newmode_commandprompt.add_plugin(plugin)

        self.logger.debug('Entering cmdloop for command prompt in mode %s' % newmode)

        newmode_commandprompt.cmdloop()

        return_code = newmode_commandprompt.last_return_code

        if return_code is sysadmintoolkit.cmdprompt.EXIT_THIS_CMDPROMPT:
            self.logger.debug('Command prompt in mode %s exited' % newmode)
            return_code = 0
        elif return_code is sysadmintoolkit.cmdprompt.EXIT_ALL_CMDPROMPT:
            self.logger.debug('Command prompt in mode %s asked for a program exit' % newmode)
        else:
            self.logger.debug('Command prompt in mode %s exited with code %s' % (newmode, return_code))

        signal.signal(signal.SIGWINCH, old_sigwinch_handler)

        return return_code
<commit_msg>* Added clear cache broadcast when leaving a mode<commit_after>__version__ = '0.1.0a'

import sysadmintoolkit
import signal
import sys
import readline
import docutils.core
import tempfile
import collections

global plugin_instance

plugin_instance = None
modes_graph = collections.OrderedDict()
modes_graph['root'] = {'next_modes': ['config', 'operator'], 'desc': 'Display system properties, most commands require superuser privilege'}
modes_graph['config'] = {'next_modes': [], 'desc': 'Change system configuration'}
modes_graph['operator'] = {'next_modes': [], 'desc': 'View basic system properties'}

def get_plugin(logger, config):
    '''
    '''
    global plugin_instance, modes_graph

    if plugin_instance is None:
        plugin_instance = CommandPrompt(logger, config, modes_graph)

    return plugin_instance

class CommandPrompt(sysadmintoolkit.plugin.Plugin):
    '''
    Description
    -----------

    Provides basic commands to the sysadmin-toolkit CLI.

    Configuration
    -------------

    *plugin-dir*
      Directory where plugins reside. Plugins must be with the .py extension.

      Default: /etc/sysadmin-toolkit/plugin.d/

    *script-dir*
      Directory where scripts reside. The commandprompt plugin doesn't do anything with this value.

      Other plugins would use it for indirection if needed.

      Default: /etc/sysadmin-toolkit/scripts.d/

    '''
    def __init__(self, logger, config, modes_graph):
        super(CommandPrompt, self).__init__('commandprompt', logger, config, version=__version__)

        self.modes_graph = modes_graph

        self.add_command(sysadmintoolkit.command.LabelHelp('debug', self, 'Debug plugins'))
        self.add_command(sysadmintoolkit.command.ExecCommand('debug commandprompt', self, self.debug))

        # use_help = sysadmintoolkit.command.LabelHelp('use', self, 'Restrict the command to a specific plugin')
        # use_help._Label__is_reserved = True
        # self.add_command(use_help)

        # use_cmd = sysadmintoolkit.command.ExecCommand('use <plugin> <*>', self, self.cmd_input_with_scope)
        # use_cmd._Label__is_reserved = True
        # self.add_command(use_cmd)

        switchmode_help = sysadmintoolkit.command.LabelHelp('switchmode', self, 'Change the command prompt mode')
        switchmode_help._Label__is_reserved = True
        self.add_command(switchmode_help)

        switchmode_cmd = sysadmintoolkit.command.ExecCommand('switchmode <mode>', self, self.change_mode)
        switchmode_cmd._Label__is_reserved = True
        self.add_command(switchmode_cmd)

        help_help = sysadmintoolkit.command.LabelHelp('help', self, 'Displays plugin help page')
        help_help._Label__is_reserved = True
        self.add_command(help_help)

        help_cmd = sysadmintoolkit.command.ExecCommand('help <plugin>', self, self.show_plugin_help)
        help_cmd._Label__is_reserved = True
        self.add_command(help_cmd)

        exit_cmd = sysadmintoolkit.command.ExecCommand('exit', self, self.exit_last_commandprompt_level)
        exit_cmd._Label__is_reserved = True
        self.add_command(exit_cmd)

        quit_cmd = sysadmintoolkit.command.ExecCommand('quit', self, self.exit_all_commandprompt_levels)
        quit_cmd._Label__is_reserved = True
        self.add_command(quit_cmd)

        # set_cmd = sysadmintoolkit.command.ExecCommand('set', self, self.change_global_config)
        # set_cmd._Label__is_reserved = True
        # self.add_command(set_cmd)

        self.add_dynamic_keyword_fn('<plugin>', self.get_plugins)
        self.add_dynamic_keyword_fn('<mode>', self.get_modes)

        def sigint_handler(signum=None, frame=None):
            global plugin_instance
            plugin_instance.logger.info('Received signal %s' % signum)
            print '\n>> Use quit or exit to leave the program'
            sys.stdout.write('%s%s' % (plugin_instance.cmdstack[-1].prompt, readline.get_line_buffer()))
            sys.stdout.flush()

        signal.signal(signal.SIGINT, sigint_handler)

    def get_plugin_documentation(self, plugin):
        pluginname = plugin.get_name()

        plugin_doc_header = []

        plugin_doc_header.append(sysadmintoolkit.utils.trim_docstring(plugin.get_doc_header()))
        plugin_doc_header.append('')

        plugin_method_map = collections.OrderedDict()

        for mode in plugin.label_map:
            modename = mode

            if mode is '':
                modename = 'default'

            for label in plugin.label_map[mode]:
                command = plugin.label_map[mode][label]
                if not isinstance(command, sysadmintoolkit.command.ExecCommand):
                    continue

                method = command.get_function()

                if method not in plugin_method_map:
                    plugin_method_map[method] = {'longhelp': command.get_help(), 'shorthelp': command.get_shorthelp(), 'labels': {}}

                if label not in plugin_method_map[method]['labels']:
                    plugin_method_map[method]['labels'][label] = []

                plugin_method_map[method]['labels'][label].append(modename)
                plugin_method_map[method]['labels'][label].sort()

        plugin_command_doc = []

        plugin_method_map_keys = plugin_method_map.keys()
        for method in plugin_method_map_keys:
            method_dict = plugin_method_map[method]

            label_keys = method_dict['labels'].keys()
            label_keys.sort()

            plugin_command_doc.append('| ')

            for label in label_keys:
                plugin_command_doc.append('| *%s* (%s)' % (label, ', '.join(method_dict['labels'][label])))

                if label is label_keys[-1]:
                    plugin_command_doc.append('| ')

            plugin_command_doc.append('')

            plugin_command_doc.append(sysadmintoolkit.utils.indent_text(method_dict['longhelp'], indent=2))

            if method is not plugin_method_map_keys[-1]:
                plugin_command_doc.append('')
                plugin_command_doc.append('----')
                plugin_command_doc.append('')

        if len(plugin_command_doc):
            plugin_doc_header.append('Plugin Commands')
            plugin_doc_header.append('---------------')

        plugin_doc_footer = []
        plugin_doc_footer.append(plugin.get_doc_footer())
        plugin_doc_footer.append('')

        plugin_doc = '\n'.join(plugin_doc_header + plugin_command_doc + plugin_doc_footer)

        self.logger.debug('Using merged docstrings for plugin %s:\n%s' % (pluginname, plugin_doc))

        return plugin_doc

    # Dynamic keywords

    def get_plugins(self, user_input_obj):
        plugins = self.plugin_set.get_plugins()

        map_plugins = {}

        for plugin in plugins:
            map_plugins[plugin] = 'Use %s plugin' % plugin

        return map_plugins

    def get_modes(self, user_input_obj):
        current_mode = user_input_obj.get_mode()

        map_modes = {}

        for next_mode in self.modes_graph[current_mode]['next_modes']:
            map_modes[next_mode] = self.modes_graph[current_mode]['desc']

        return map_modes

    # Sysadmin-toolkit commands

    def debug(self, user_input_obj):
        '''
        Display registered commands in the current Command Prompt
        '''
        if len(self.cmdstack):
            cmdprompt = self.cmdstack[-1]
        else:
            self.logger.error('This command must be executed interactively, please execute in the CLI')
            return

        width = sysadmintoolkit.utils.get_terminal_size()[1]

        window_ratio = 0.55
        max_block_size = 150
        first_block_len = int(min(width, max_block_size) * window_ratio) - 1
        second_block_len = int(min(width, max_block_size) * (1 - window_ratio)) - 1

        print
        print '  Loaded plugins: %s' % ', '.join(self.plugin_set.get_plugins().keys())
        print
        print '  Command tree (%s)' % cmdprompt.mode
        print '  ' + ('=' * len('Command tree (%s)' % cmdprompt.mode))
        print
        print '    Definitions: non executable, %s, %s' % (sysadmintoolkit.utils.get_underline_text('executable'), \
                                                           sysadmintoolkit.utils.get_reversed_text('<dynamic>'))
        print
        print '  %s%s' % ('Keyword'.ljust(first_block_len), 'Help (plugin)')
        print '-' * (first_block_len + second_block_len)

        labeldict = cmdprompt.command_tree.get_sub_keywords_labels()
        labelkeys = labeldict.keys()
        labelkeys.sort()

        for label in labelkeys:
            # Build the label part of the debug
            labelstr = ''
            for keyword in cmdprompt.split_label(label)[:-1]:
                if cmdprompt.is_dynamic_keyword(keyword):
                    keyword = sysadmintoolkit.utils.get_reversed_text(keyword)

                labelstr += sysadmintoolkit.utils.get_grey_text(keyword) + ' '

            lastkeyword = cmdprompt.split_label(label)[-1]

            if cmdprompt.is_dynamic_keyword(lastkeyword):
                lastkeyword = sysadmintoolkit.utils.get_reversed_text(lastkeyword)

            if len(labeldict[label]['executable_commands']) > 0:
                lastkeyword = sysadmintoolkit.utils.get_underline_text(lastkeyword)

            labelstr += lastkeyword

            label_text_block = {'str': '%s' % labelstr, 'window_ratio': window_ratio, 'maxwidth': max_block_size * window_ratio, 'wrap': False}
            blank_text_block = {'str': ' ', 'window_ratio': window_ratio, 'maxwidth': max_block_size * window_ratio, 'wrap': True}

            # Build the label help blocks
            label_help_blocks = []

            label_help_plugins = labeldict[label]['help'].keys()
            label_help_plugins.sort()

            for plugin in label_help_plugins:
                label_help_blocks += [{'str':'%s (%s)' % (labeldict[label]['help'][plugin].get_shorthelp(), plugin), \
                                       'window_ratio': 1 - window_ratio, \
                                       'maxwidth': max_block_size * (1 - window_ratio), \
                                       'wrap': True
                                       }]

            # Build the command help blocks
            command_help_blocks = []

            command_help_plugins = labeldict[label]['executable_commands'].keys()
            command_help_plugins.sort()

            for plugin in command_help_plugins:
                command_help_blocks += [{'str': '%s (%s)' % (labeldict[label]['executable_commands'][plugin].get_shorthelp(), plugin), \
                                         'window_ratio': 1 - window_ratio, \
                                         'maxwidth': max_block_size * (1 - window_ratio), \
                                         'wrap': True
                                         }]

            # Print all the gathered information
            output_so_far = False

            for right_block in label_help_blocks + command_help_blocks:
                if not output_so_far:
                    left_block = label_text_block
                    output_so_far = True
                else:
                    left_block = blank_text_block

                sysadmintoolkit.utils.print_text_blocks([left_block, right_block])

                print

        print

        print '  Available modes'
        print '  ==============='
        print

        for mode in self.modes_graph:
            print '    %s %s' % (('%s:' % mode).ljust(20), self.modes_graph[mode]['desc'])
            print

            if len(self.modes_graph[mode]['next_modes']):
                print '    %s Next modes: %s' % (' ' * len(('%s:' % mode).ljust(20)), ', '.join(self.modes_graph[mode]['next_modes']))
            else:
                print '    %s Next modes: None' % (' ' * len(('%s:' % mode).ljust(20)))

            print

        print

    def show_plugin_help(self, user_input_obj):
        '''
        Display man page for this plugin

        '''
        pluginname = user_input_obj.get_entered_command().split()[1]
        plugin = self.plugin_set.get_plugins()[pluginname]

        self.logger.debug('Showing help for plugin %s' % pluginname)

        plugin_doc = self.get_plugin_documentation(plugin)

        plugin_doc_manpage_header = sysadmintoolkit.utils.trim_docstring("""
        %s=======
        %s Plugin
        %s=======
        """ % ('=' * len(pluginname), pluginname, '=' * len(pluginname)))

        self.logger.debug('Adding header for manpage:\n%s' % plugin_doc_manpage_header)

        doc_file = tempfile.NamedTemporaryFile()

        # Man page formatting
        manpage_format = docutils.core.publish_string('%s' % (plugin_doc), writer_name='manpage').splitlines()

        manpage_format[0] = '.TH %s COMMANDS "%s" "SYSADMIN-TOOLKIT PLUGIN" ""' % (plugin.get_name().upper(), plugin.get_version())

        if '.SH NAME' in manpage_format[1]:
            manpage_format[1] = ''

        if manpage_format[2].endswith('\\- '):
            manpage_format[2] = manpage_format[2][:-3]

        manpage_format = '\n'.join(manpage_format)

        doc_file.writelines('%s\n' % manpage_format)
        doc_file.flush()

        self.logger.debug('Using manpage for plugin %s:\n%s' % (pluginname, manpage_format))

        sysadmintoolkit.utils.execute_interactive_cmd('man %s' % doc_file.name, self.logger)

    def exit_last_commandprompt_level(self, user_input_obj):
        '''
        Exit the current command prompt

        '''
        return sysadmintoolkit.cmdprompt.EXIT_THIS_CMDPROMPT

    def exit_all_commandprompt_levels(self, user_input_obj):
        '''
        Exit the program

        '''
        return sysadmintoolkit.cmdprompt.EXIT_ALL_CMDPROMPT

    def change_mode(self, user_input_obj):
        '''
        Change the command prompt mode

        '''
        newmode = user_input_obj.get_entered_command().split()[-1]
        old_commandprompt = user_input_obj.get_cmdprompt()

        self.logger.debug('Changing mode from %s to %s' % (user_input_obj.get_mode(), newmode))

        newmode_commandprompt = sysadmintoolkit.cmdprompt.CmdPrompt(old_commandprompt.logger, mode=newmode, \
                                                         shell_allowed=old_commandprompt.shell_allowed, \
                                                         is_interactive=old_commandprompt.is_interactive)

        def SIGWINCH_handler(signum=None, frame=None):
            (height, width) = sysadmintoolkit.utils.get_terminal_size()
            newmode_commandprompt.update_window_size(width, height)

        old_sigwinch_handler = signal.signal(signal.SIGWINCH, SIGWINCH_handler)
        SIGWINCH_handler()

        for plugin in old_commandprompt.get_plugins():
            newmode_commandprompt.add_plugin(plugin)

        self.logger.debug('Entering cmdloop for command prompt in mode %s' % newmode)

        newmode_commandprompt.cmdloop()

        return_code = newmode_commandprompt.last_return_code

        if return_code is sysadmintoolkit.cmdprompt.EXIT_THIS_CMDPROMPT:
            self.logger.debug('Command prompt in mode %s exited' % newmode)
            return_code = 0
        elif return_code is sysadmintoolkit.cmdprompt.EXIT_ALL_CMDPROMPT:
            self.logger.debug('Command prompt in mode %s asked for a program exit' % newmode)
        else:
            self.logger.debug('Command prompt in mode %s exited with code %s' % (newmode, return_code))

        signal.signal(signal.SIGWINCH, old_sigwinch_handler)

        for plugin in self.plugin_set.get_plugins():
            self.plugin_set.get_plugins()[plugin].clear_cache()

        return return_code
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import concurrent.futures
from concurrent.futures import ThreadPoolExecutor
import requests
import whatthepatch
from shipit_uplift.coverage import coverage_service, get_coverage_build, coverage_supported, get_github_commit

def generate(changeset):
    '''
    This function generates a report containing the coverage information of the diff
    introduced by a changeset.
    '''
    desc, build_changeset, overall = get_coverage_build(changeset)
    if any(text in desc for text in ['r=merge', 'a=merge']):
        raise Exception('Retrieving coverage for merge commits is not supported.')

    r = requests.get('https://hg.mozilla.org/mozilla-central/raw-rev/%s' % changeset)
    patch = r.text

    diffs = []

    def parse_diff(diff):
        # Get old and new path, for files that have been renamed.
        new_path = diff.header.new_path[2:] if diff.header.new_path.startswith('b/') else diff.header.new_path

        # If the diff doesn't contain any changes, we skip it.
        if diff.changes is None:
            return None

        # If the file is not a source file, we skip it (as we already know
        # we have no coverage information for it).
        if not coverage_supported(new_path):
            return None

        # Retrieve coverage of added lines.
        coverage = coverage_service.get_file_coverage(build_changeset, new_path)

        # If we don't have coverage for this file, we skip it.
        if coverage is None:
            return None

        # Use hg annotate to report lines in their correct positions and to avoid
        # reporting lines that have been modified by a successive patch in the same push.
        r = requests.get('https://hg.mozilla.org/mozilla-central/json-annotate/%s/%s' % (build_changeset, new_path))
        annotate = r.json()['annotate']

        changes = []
        for new_line, data in enumerate(annotate):
            # Skip lines that were not added by this changeset or were overwritten by
            # another changeset.
            if data['node'] != changeset:
                continue

            if new_line not in coverage or coverage[new_line] is None:
                # We have no coverage information for this line (e.g. a definition, like
                # a variable in a header file).
                covered = '?'
            elif coverage[new_line] > 0:
                covered = 'Y'
            else:
                covered = 'N'

            changes.append({
                'coverage': covered,
                'line': new_line,
            })

        return {
          'name': new_path,
          'changes': changes,
        }

    def parse_diff_task(diff):
        return lambda: parse_diff(diff)

    with ThreadPoolExecutor(max_workers=4) as executor:
        futures = []

        for diff in whatthepatch.parse_patch(patch):
            futures.append(executor.submit(parse_diff_task(diff)))

        for future in concurrent.futures.as_completed(futures):
            res = future.result()
            if res is not None:
                diffs.append(res)

    return {
        'build_changeset': build_changeset,
        'git_build_changeset': get_github_commit(build_changeset),
        'overall_cur': overall['cur'],
        'overall_prev': overall['prev'],
        'diffs': diffs,
    }
<commit_msg>shipit_uplift: Return 'original' line number (that is, the line number when the line was introduced) and fix off-by-one error (enumerate starts from 0, line numbers start from 1) (#720)<commit_after># -*- coding: utf-8 -*-
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import concurrent.futures
from concurrent.futures import ThreadPoolExecutor
import requests
import whatthepatch
from shipit_uplift.coverage import coverage_service, get_coverage_build, coverage_supported, get_github_commit

def generate(changeset):
    '''
    This function generates a report containing the coverage information of the diff
    introduced by a changeset.
    '''
    desc, build_changeset, overall = get_coverage_build(changeset)
    if any(text in desc for text in ['r=merge', 'a=merge']):
        raise Exception('Retrieving coverage for merge commits is not supported.')

    r = requests.get('https://hg.mozilla.org/mozilla-central/raw-rev/%s' % changeset)
    patch = r.text

    diffs = []

    def parse_diff(diff):
        # Get old and new path, for files that have been renamed.
        new_path = diff.header.new_path[2:] if diff.header.new_path.startswith('b/') else diff.header.new_path

        # If the diff doesn't contain any changes, we skip it.
        if diff.changes is None:
            return None

        # If the file is not a source file, we skip it (as we already know
        # we have no coverage information for it).
        if not coverage_supported(new_path):
            return None

        # Retrieve coverage of added lines.
        coverage = coverage_service.get_file_coverage(build_changeset, new_path)

        # If we don't have coverage for this file, we skip it.
        if coverage is None:
            return None

        # Use hg annotate to report lines in their correct positions and to avoid
        # reporting lines that have been modified by a successive patch in the same push.
        r = requests.get('https://hg.mozilla.org/mozilla-central/json-annotate/%s/%s' % (build_changeset, new_path))
        annotate = r.json()['annotate']

        changes = []
        for data in annotate:
            # Skip lines that were not added by this changeset or were overwritten by
            # another changeset.
            if data['node'] != changeset:
                continue

            new_line = data['lineno']

            if new_line not in coverage or coverage[new_line] is None:
                # We have no coverage information for this line (e.g. a definition, like
                # a variable in a header file).
                covered = '?'
            elif coverage[new_line] > 0:
                covered = 'Y'
            else:
                covered = 'N'

            changes.append({
                'coverage': covered,
                'line': data['targetline'],
            })

        return {
          'name': new_path,
          'changes': changes,
        }

    def parse_diff_task(diff):
        return lambda: parse_diff(diff)

    with ThreadPoolExecutor(max_workers=4) as executor:
        futures = []

        for diff in whatthepatch.parse_patch(patch):
            futures.append(executor.submit(parse_diff_task(diff)))

        for future in concurrent.futures.as_completed(futures):
            res = future.result()
            if res is not None:
                diffs.append(res)

    return {
        'build_changeset': build_changeset,
        'git_build_changeset': get_github_commit(build_changeset),
        'overall_cur': overall['cur'],
        'overall_prev': overall['prev'],
        'diffs': diffs,
    }
<|endoftext|>


<commit_before># Copyright 2017 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"TypeScript compilation"

# pylint: disable=unused-argument
# pylint: disable=missing-docstring
load(":common/compilation.bzl", "COMMON_ATTRIBUTES", "compile_ts", "ts_providers_dict_to_struct")
load(":common/tsconfig.bzl", "create_tsconfig")
load(":ts_config.bzl", "TsConfigInfo")

def _compile_action(ctx, inputs, outputs, tsconfig_file, node_opts, description = "prodmode"):
    externs_files = []
    action_outputs = []
    for output in outputs:
        if output.basename.endswith(".externs.js"):
            externs_files.append(output)
        elif output.basename.endswith(".es5.MF"):
            ctx.file_action(output, content = "")
        else:
            action_outputs.append(output)

    # TODO(plf): For now we mock creation of files other than {name}.js.
    for externs_file in externs_files:
        ctx.file_action(output = externs_file, content = "")

    # A ts_library that has only .d.ts inputs will have no outputs,
    # therefore there are no actions to execute
    if not action_outputs:
        return struct()

    action_inputs = inputs + [
        f
        for f in ctx.files.node_modules + ctx.files._tsc_wrapped_deps
        if f.path.endswith(".js") or f.path.endswith(".ts") or f.path.endswith(".json")
    ]
    if ctx.file.tsconfig:
        action_inputs += [ctx.file.tsconfig]
        if TsConfigInfo in ctx.attr.tsconfig:
            action_inputs += ctx.attr.tsconfig[TsConfigInfo].deps

    # Pass actual options for the node binary in the special "--node_options" argument.
    arguments = ["--node_options=%s" % opt for opt in node_opts]

    # One at-sign makes this a params-file, enabling the worker strategy.
    # Two at-signs escapes the argument so it's passed through to tsc_wrapped
    # rather than the contents getting expanded.
    if ctx.attr.supports_workers:
        arguments.append("@@" + tsconfig_file.path)
        mnemonic = "TypeScriptCompile"
    else:
        arguments.append("-p")
        arguments.append(tsconfig_file.path)
        mnemonic = "tsc"

    ctx.action(
        progress_message = "Compiling TypeScript (%s) %s" % (description, ctx.label),
        mnemonic = mnemonic,
        inputs = action_inputs,
        outputs = action_outputs,
        arguments = arguments,
        executable = ctx.executable.compiler,
        execution_requirements = {
            "supports-workers": str(int(ctx.attr.supports_workers)),
        },
    )

    # Enable the replay_params in case an aspect needs to re-build this library.
    return struct(
        label = ctx.label,
        tsconfig = tsconfig_file,
        inputs = action_inputs,
        outputs = action_outputs,
        compiler = ctx.executable.compiler,
    )

def _devmode_compile_action(ctx, inputs, outputs, tsconfig_file, node_opts):
    _compile_action(
        ctx,
        inputs,
        outputs,
        tsconfig_file,
        node_opts,
        description = "devmode",
    )

def tsc_wrapped_tsconfig(
        ctx,
        files,
        srcs,
        devmode_manifest = None,
        jsx_factory = None,
        **kwargs):
    """Produce a tsconfig.json that sets options required under Bazel.
    """

    # The location of tsconfig.json is interpreted as the root of the project
    # when it is passed to the TS compiler with the `-p` option:
    #   https://www.typescriptlang.org/docs/handbook/tsconfig-json.html.
    # Our tsconfig.json is in bazel-foo/bazel-out/local-fastbuild/bin/{package_path}
    # because it's generated in the execution phase. However, our source files are in
    # bazel-foo/ and therefore we need to strip some parent directories for each
    # f.path.

    config = create_tsconfig(
        ctx,
        files,
        srcs,
        devmode_manifest = devmode_manifest,
        **kwargs
    )
    config["bazelOptions"]["nodeModulesPrefix"] = "/".join([p for p in [
        ctx.attr.node_modules.label.workspace_root,
        ctx.attr.node_modules.label.package,
        "node_modules",
    ] if p])

    # If the user gives a tsconfig attribute, the generated file should extend
    # from the user's tsconfig.
    # See https://github.com/Microsoft/TypeScript/issues/9876
    # We subtract the ".json" from the end before handing to TypeScript because
    # this gives extra error-checking.
    if ctx.file.tsconfig:
        workspace_path = config["compilerOptions"]["rootDir"]
        config["extends"] = "/".join([workspace_path, ctx.file.tsconfig.path[:-len(".json")]])

    if jsx_factory:
        config["compilerOptions"]["jsxFactory"] = jsx_factory

    return config

# ************ #
# ts_library   #
# ************ #

def _ts_library_impl(ctx):
    """Implementation of ts_library.

    Args:
      ctx: the context.

    Returns:
      the struct returned by the call to compile_ts.
    """
    ts_providers = compile_ts(
        ctx,
        is_library = True,
        compile_action = _compile_action,
        devmode_compile_action = _devmode_compile_action,
        tsc_wrapped_tsconfig = tsc_wrapped_tsconfig,
    )
    return ts_providers_dict_to_struct(ts_providers)

ts_library = rule(
    _ts_library_impl,
    attrs = dict(COMMON_ATTRIBUTES, **{
        "srcs": attr.label_list(
            doc = "The TypeScript source files to compile.",
            allow_files = [".ts", ".tsx"],
            mandatory = True,
        ),

        # TODO(alexeagle): reconcile with google3: ts_library rules should
        # be portable across internal/external, so we need this attribute
        # internally as well.
        "tsconfig": attr.label(
            doc = """A tsconfig.json file containing settings for TypeScript compilation.
            Note that some properties in the tsconfig are governed by Bazel and will be
            overridden, such as `target` and `module`.

            The default value is set to `//:tsconfig.json` by a macro. This means you must
            either:

            - Have your `tsconfig.json` file in the workspace root directory
            - Use an alias in the root BUILD.bazel file to point to the location of tsconfig:
              `alias(name="tsconfig.json", actual="//path/to:tsconfig-something.json")`
            - Give an explicit `tsconfig` attribute to all `ts_library` targets
            """,
            allow_files = True,
            single_file = True,
        ),
        "compiler": attr.label(
            doc = """Intended for internal use only.
            Sets a different TypeScript compiler binary to use for this library.
            For example, we use the vanilla TypeScript tsc.js for bootstrapping,
            and Angular compilations can replace this with `ngc`.""",
            default = Label("//internal:tsc_wrapped_bin"),
            single_file = False,
            allow_files = True,
            executable = True,
            cfg = "host",
        ),
        "supports_workers": attr.bool(
            doc = """Intended for internal use only.
            Allows you to disable the Bazel Worker strategy for this library.
            Typically used together with the "compiler" setting when using a
            non-worker aware compiler binary.""",
            default = True,
        ),
        "tsickle_typed": attr.bool(default = True),
        "internal_testing_type_check_dependencies": attr.bool(default = False, doc = "Testing only, whether to type check inputs that aren't srcs."),
        "_tsc_wrapped_deps": attr.label(default = Label("@build_bazel_rules_typescript_tsc_wrapped_deps//:node_modules")),
        # @// is special syntax for the "main" repository
        # The default assumes the user specified a target "node_modules" in their
        # root BUILD file.
        "node_modules": attr.label(default = Label("@//:node_modules")),
    }),
    outputs = {
        "tsconfig": "%{name}_tsconfig.json",
    },
)
"""
`ts_library` type-checks and compiles a set of TypeScript sources to JavaScript.

It produces declarations files (`.d.ts`) which are used for compiling downstream
TypeScript targets and JavaScript for the browser and Closure compiler.
"""

def ts_library_macro(tsconfig = None, **kwargs):
    """Wraps `ts_library` to set the default for the `tsconfig` attribute.

    This must be a macro so that the string is converted to a label in the context of the
    workspace that declares the `ts_library` target, rather than the workspace that defines
    `ts_library`, or the workspace where the build is taking place.

    This macro is re-exported as `ts_library` in the public API.

    Args:
      tsconfig: the label pointing to a tsconfig.json file
      **kwargs: remaining args to pass to the ts_library rule
    """
    if not tsconfig:
        tsconfig = "//:tsconfig.json"

    ts_library(tsconfig = tsconfig, **kwargs)
<commit_msg>_compile_action should use the default shell environment when calling `ctx.action()`<commit_after># Copyright 2017 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"TypeScript compilation"

# pylint: disable=unused-argument
# pylint: disable=missing-docstring
load(":common/compilation.bzl", "COMMON_ATTRIBUTES", "compile_ts", "ts_providers_dict_to_struct")
load(":common/tsconfig.bzl", "create_tsconfig")
load(":ts_config.bzl", "TsConfigInfo")

def _compile_action(ctx, inputs, outputs, tsconfig_file, node_opts, description = "prodmode"):
    externs_files = []
    action_outputs = []
    for output in outputs:
        if output.basename.endswith(".externs.js"):
            externs_files.append(output)
        elif output.basename.endswith(".es5.MF"):
            ctx.file_action(output, content = "")
        else:
            action_outputs.append(output)

    # TODO(plf): For now we mock creation of files other than {name}.js.
    for externs_file in externs_files:
        ctx.file_action(output = externs_file, content = "")

    # A ts_library that has only .d.ts inputs will have no outputs,
    # therefore there are no actions to execute
    if not action_outputs:
        return struct()

    action_inputs = inputs + [
        f
        for f in ctx.files.node_modules + ctx.files._tsc_wrapped_deps
        if f.path.endswith(".js") or f.path.endswith(".ts") or f.path.endswith(".json")
    ]
    if ctx.file.tsconfig:
        action_inputs += [ctx.file.tsconfig]
        if TsConfigInfo in ctx.attr.tsconfig:
            action_inputs += ctx.attr.tsconfig[TsConfigInfo].deps

    # Pass actual options for the node binary in the special "--node_options" argument.
    arguments = ["--node_options=%s" % opt for opt in node_opts]

    # One at-sign makes this a params-file, enabling the worker strategy.
    # Two at-signs escapes the argument so it's passed through to tsc_wrapped
    # rather than the contents getting expanded.
    if ctx.attr.supports_workers:
        arguments.append("@@" + tsconfig_file.path)
        mnemonic = "TypeScriptCompile"
    else:
        arguments.append("-p")
        arguments.append(tsconfig_file.path)
        mnemonic = "tsc"

    ctx.action(
        progress_message = "Compiling TypeScript (%s) %s" % (description, ctx.label),
        mnemonic = mnemonic,
        inputs = action_inputs,
        outputs = action_outputs,
        # Use the built-in shell environment
        # Allow for users who set a custom shell that can locate standard binaries like tr and uname
        # See https://github.com/NixOS/nixpkgs/issues/43955#issuecomment-407546331
        use_default_shell_env = True,
        arguments = arguments,
        executable = ctx.executable.compiler,
        execution_requirements = {
            "supports-workers": str(int(ctx.attr.supports_workers)),
        },
    )

    # Enable the replay_params in case an aspect needs to re-build this library.
    return struct(
        label = ctx.label,
        tsconfig = tsconfig_file,
        inputs = action_inputs,
        outputs = action_outputs,
        compiler = ctx.executable.compiler,
    )

def _devmode_compile_action(ctx, inputs, outputs, tsconfig_file, node_opts):
    _compile_action(
        ctx,
        inputs,
        outputs,
        tsconfig_file,
        node_opts,
        description = "devmode",
    )

def tsc_wrapped_tsconfig(
        ctx,
        files,
        srcs,
        devmode_manifest = None,
        jsx_factory = None,
        **kwargs):
    """Produce a tsconfig.json that sets options required under Bazel.
    """

    # The location of tsconfig.json is interpreted as the root of the project
    # when it is passed to the TS compiler with the `-p` option:
    #   https://www.typescriptlang.org/docs/handbook/tsconfig-json.html.
    # Our tsconfig.json is in bazel-foo/bazel-out/local-fastbuild/bin/{package_path}
    # because it's generated in the execution phase. However, our source files are in
    # bazel-foo/ and therefore we need to strip some parent directories for each
    # f.path.

    config = create_tsconfig(
        ctx,
        files,
        srcs,
        devmode_manifest = devmode_manifest,
        **kwargs
    )
    config["bazelOptions"]["nodeModulesPrefix"] = "/".join([p for p in [
        ctx.attr.node_modules.label.workspace_root,
        ctx.attr.node_modules.label.package,
        "node_modules",
    ] if p])

    # If the user gives a tsconfig attribute, the generated file should extend
    # from the user's tsconfig.
    # See https://github.com/Microsoft/TypeScript/issues/9876
    # We subtract the ".json" from the end before handing to TypeScript because
    # this gives extra error-checking.
    if ctx.file.tsconfig:
        workspace_path = config["compilerOptions"]["rootDir"]
        config["extends"] = "/".join([workspace_path, ctx.file.tsconfig.path[:-len(".json")]])

    if jsx_factory:
        config["compilerOptions"]["jsxFactory"] = jsx_factory

    return config

# ************ #
# ts_library   #
# ************ #

def _ts_library_impl(ctx):
    """Implementation of ts_library.

    Args:
      ctx: the context.

    Returns:
      the struct returned by the call to compile_ts.
    """
    ts_providers = compile_ts(
        ctx,
        is_library = True,
        compile_action = _compile_action,
        devmode_compile_action = _devmode_compile_action,
        tsc_wrapped_tsconfig = tsc_wrapped_tsconfig,
    )
    return ts_providers_dict_to_struct(ts_providers)

ts_library = rule(
    _ts_library_impl,
    attrs = dict(COMMON_ATTRIBUTES, **{
        "srcs": attr.label_list(
            doc = "The TypeScript source files to compile.",
            allow_files = [".ts", ".tsx"],
            mandatory = True,
        ),

        # TODO(alexeagle): reconcile with google3: ts_library rules should
        # be portable across internal/external, so we need this attribute
        # internally as well.
        "tsconfig": attr.label(
            doc = """A tsconfig.json file containing settings for TypeScript compilation.
            Note that some properties in the tsconfig are governed by Bazel and will be
            overridden, such as `target` and `module`.

            The default value is set to `//:tsconfig.json` by a macro. This means you must
            either:

            - Have your `tsconfig.json` file in the workspace root directory
            - Use an alias in the root BUILD.bazel file to point to the location of tsconfig:
              `alias(name="tsconfig.json", actual="//path/to:tsconfig-something.json")`
            - Give an explicit `tsconfig` attribute to all `ts_library` targets
            """,
            allow_files = True,
            single_file = True,
        ),
        "compiler": attr.label(
            doc = """Intended for internal use only.
            Sets a different TypeScript compiler binary to use for this library.
            For example, we use the vanilla TypeScript tsc.js for bootstrapping,
            and Angular compilations can replace this with `ngc`.""",
            default = Label("//internal:tsc_wrapped_bin"),
            single_file = False,
            allow_files = True,
            executable = True,
            cfg = "host",
        ),
        "supports_workers": attr.bool(
            doc = """Intended for internal use only.
            Allows you to disable the Bazel Worker strategy for this library.
            Typically used together with the "compiler" setting when using a
            non-worker aware compiler binary.""",
            default = True,
        ),
        "tsickle_typed": attr.bool(default = True),
        "internal_testing_type_check_dependencies": attr.bool(default = False, doc = "Testing only, whether to type check inputs that aren't srcs."),
        "_tsc_wrapped_deps": attr.label(default = Label("@build_bazel_rules_typescript_tsc_wrapped_deps//:node_modules")),
        # @// is special syntax for the "main" repository
        # The default assumes the user specified a target "node_modules" in their
        # root BUILD file.
        "node_modules": attr.label(default = Label("@//:node_modules")),
    }),
    outputs = {
        "tsconfig": "%{name}_tsconfig.json",
    },
)
"""
`ts_library` type-checks and compiles a set of TypeScript sources to JavaScript.

It produces declarations files (`.d.ts`) which are used for compiling downstream
TypeScript targets and JavaScript for the browser and Closure compiler.
"""

def ts_library_macro(tsconfig = None, **kwargs):
    """Wraps `ts_library` to set the default for the `tsconfig` attribute.

    This must be a macro so that the string is converted to a label in the context of the
    workspace that declares the `ts_library` target, rather than the workspace that defines
    `ts_library`, or the workspace where the build is taking place.

    This macro is re-exported as `ts_library` in the public API.

    Args:
      tsconfig: the label pointing to a tsconfig.json file
      **kwargs: remaining args to pass to the ts_library rule
    """
    if not tsconfig:
        tsconfig = "//:tsconfig.json"

    ts_library(tsconfig = tsconfig, **kwargs)
<|endoftext|>


<commit_before># Copyright 2017 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""TypeScript rules.
"""
# pylint: disable=unused-argument
# pylint: disable=missing-docstring
load(":common/compilation.bzl", "COMMON_ATTRIBUTES", "compile_ts", "ts_providers_dict_to_struct")
load(":executables.bzl", "get_tsc")
load(":common/tsconfig.bzl", "create_tsconfig")
load(":ts_config.bzl", "TsConfig")

def _compile_action(ctx, inputs, outputs, config_file_path):
  externs_files = []
  non_externs_files = []
  for output in outputs:
    if output.basename.endswith(".externs.js"):
      externs_files.append(output)
    elif output.basename.endswith(".es5.MF"):
      ctx.file_action(output, content="")
    else:
      non_externs_files.append(output)

  # TODO(plf): For now we mock creation of files other than {name}.js.
  for externs_file in externs_files:
    ctx.file_action(output=externs_file, content="")

  action_inputs = inputs + [f for f in ctx.files.node_modules
                            if f.path.endswith(".ts") or f.path.endswith(".json")]
  if ctx.file.tsconfig:
    action_inputs += [ctx.file.tsconfig]
    if TsConfig in ctx.attr.tsconfig:
      action_inputs += ctx.attr.tsconfig[TsConfig].deps

  # One at-sign makes this a params-file, enabling the worker strategy.
  # Two at-signs escapes the argument so it's passed through to tsc_wrapped
  # rather than the contents getting expanded.
  if ctx.attr.supports_workers:
    arguments = ["@@" + config_file_path]
    mnemonic = "TypeScriptCompile"
  else:
    arguments = ["-p", config_file_path]
    mnemonic = "tsc"

  ctx.action(
      progress_message = "Compiling TypeScript (devmode) %s" % ctx.label,
      mnemonic = mnemonic,
      inputs = action_inputs,
      outputs = non_externs_files,
      arguments = arguments,
      executable = ctx.executable.compiler,
      execution_requirements = {
          "supports-workers": str(int(ctx.attr.supports_workers)),
      },
  )

def _devmode_compile_action(ctx, inputs, outputs, config_file_path):
  _compile_action(ctx, inputs, outputs, config_file_path)

def tsc_wrapped_tsconfig(ctx,
                         files,
                         srcs,
                         devmode_manifest=None,
                         jsx_factory=None,
                         **kwargs):

  # The location of tsconfig.json is interpreted as the root of the project
  # when it is passed to the TS compiler with the `-p` option:
  #   https://www.typescriptlang.org/docs/handbook/tsconfig-json.html.
  # Our tsconfig.json is in bazel-foo/bazel-out/local-fastbuild/bin/{package_path}
  # because it's generated in the execution phase. However, our source files are in
  # bazel-foo/ and therefore we need to strip some parent directories for each
  # f.path.

  config = create_tsconfig(ctx, files, srcs,
                           devmode_manifest=devmode_manifest,
                           **kwargs)
  config["bazelOptions"]["nodeModulesPrefix"] = "node_modules"
  if config["compilerOptions"]["target"] == "es6":
    config["compilerOptions"]["module"] = "es2015"
  else:
    # The "typescript.es5_sources" provider is expected to work
    # in both nodejs and in browsers.
    # NOTE: tsc-wrapped will always name the enclosed AMD modules
    config["compilerOptions"]["module"] = "umd"

  # If the user gives a tsconfig attribute, the generated file should extend
  # from the user's tsconfig.
  # See https://github.com/Microsoft/TypeScript/issues/9876
  # We subtract the ".json" from the end before handing to TypeScript because
  # this gives extra error-checking.
  if ctx.file.tsconfig:
    workspace_path = config["compilerOptions"]["rootDir"]
    config["extends"] = "/".join([workspace_path, ctx.file.tsconfig.path[:-len(".json")]])

  if jsx_factory:
    config["compilerOptions"]["jsxFactory"] = jsx_factory

  return config

# ************ #
# ts_library   #
# ************ #

def _ts_library_impl(ctx):
  """Implementation of ts_library.

  Args:
    ctx: the context.
  Returns:
    the struct returned by the call to compile_ts.
  """
  ts_providers = compile_ts(ctx, is_library=True,
                            compile_action=_compile_action,
                            devmode_compile_action=_devmode_compile_action,
                            tsc_wrapped_tsconfig=tsc_wrapped_tsconfig)
  return ts_providers_dict_to_struct(ts_providers)

ts_library = rule(
    _ts_library_impl,
    attrs = dict(COMMON_ATTRIBUTES, **{
        "srcs":
            attr.label_list(
                allow_files=FileType([
                    ".ts",
                    ".tsx",
                ]),
                mandatory=True,),

        # TODO(alexeagle): reconcile with google3: ts_library rules should
        # be portable across internal/external, so we need this attribute
        # internally as well.
        "tsconfig":
            attr.label(allow_files = True, single_file = True),
        "compiler":
            attr.label(
                default=get_tsc(),
                single_file=False,
                allow_files=True,
                executable=True,
                cfg="host"),
        "supports_workers": attr.bool(default = True),
        # @// is special syntax for the "main" repository
        # The default assumes the user specified a target "node_modules" in their
        # root BUILD file.
        "node_modules": attr.label(default = Label("@//:node_modules")),
    }),
    outputs = {
        "tsconfig": "%{name}_tsconfig.json"
    }
)
<commit_msg>Fix strict deps when node_modules is in non-standard place.<commit_after># Copyright 2017 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""TypeScript rules.
"""
# pylint: disable=unused-argument
# pylint: disable=missing-docstring
load(":common/compilation.bzl", "COMMON_ATTRIBUTES", "compile_ts", "ts_providers_dict_to_struct")
load(":executables.bzl", "get_tsc")
load(":common/tsconfig.bzl", "create_tsconfig")
load(":ts_config.bzl", "TsConfig")

def _compile_action(ctx, inputs, outputs, config_file_path):
  externs_files = []
  non_externs_files = []
  for output in outputs:
    if output.basename.endswith(".externs.js"):
      externs_files.append(output)
    elif output.basename.endswith(".es5.MF"):
      ctx.file_action(output, content="")
    else:
      non_externs_files.append(output)

  # TODO(plf): For now we mock creation of files other than {name}.js.
  for externs_file in externs_files:
    ctx.file_action(output=externs_file, content="")

  action_inputs = inputs + [f for f in ctx.files.node_modules
                            if f.path.endswith(".ts") or f.path.endswith(".json")]
  if ctx.file.tsconfig:
    action_inputs += [ctx.file.tsconfig]
    if TsConfig in ctx.attr.tsconfig:
      action_inputs += ctx.attr.tsconfig[TsConfig].deps

  # One at-sign makes this a params-file, enabling the worker strategy.
  # Two at-signs escapes the argument so it's passed through to tsc_wrapped
  # rather than the contents getting expanded.
  if ctx.attr.supports_workers:
    arguments = ["@@" + config_file_path]
    mnemonic = "TypeScriptCompile"
  else:
    arguments = ["-p", config_file_path]
    mnemonic = "tsc"

  ctx.action(
      progress_message = "Compiling TypeScript (devmode) %s" % ctx.label,
      mnemonic = mnemonic,
      inputs = action_inputs,
      outputs = non_externs_files,
      arguments = arguments,
      executable = ctx.executable.compiler,
      execution_requirements = {
          "supports-workers": str(int(ctx.attr.supports_workers)),
      },
  )

def _devmode_compile_action(ctx, inputs, outputs, config_file_path):
  _compile_action(ctx, inputs, outputs, config_file_path)

def tsc_wrapped_tsconfig(ctx,
                         files,
                         srcs,
                         devmode_manifest=None,
                         jsx_factory=None,
                         **kwargs):

  # The location of tsconfig.json is interpreted as the root of the project
  # when it is passed to the TS compiler with the `-p` option:
  #   https://www.typescriptlang.org/docs/handbook/tsconfig-json.html.
  # Our tsconfig.json is in bazel-foo/bazel-out/local-fastbuild/bin/{package_path}
  # because it's generated in the execution phase. However, our source files are in
  # bazel-foo/ and therefore we need to strip some parent directories for each
  # f.path.

  config = create_tsconfig(ctx, files, srcs,
                           devmode_manifest=devmode_manifest,
                           **kwargs)
  config["bazelOptions"]["nodeModulesPrefix"] = "/".join([p for p in [
    ctx.attr.node_modules.label.workspace_root,
    ctx.attr.node_modules.label.package,
    "node_modules"
  ] if p])

  if config["compilerOptions"]["target"] == "es6":
    config["compilerOptions"]["module"] = "es2015"
  else:
    # The "typescript.es5_sources" provider is expected to work
    # in both nodejs and in browsers.
    # NOTE: tsc-wrapped will always name the enclosed AMD modules
    config["compilerOptions"]["module"] = "umd"

  # If the user gives a tsconfig attribute, the generated file should extend
  # from the user's tsconfig.
  # See https://github.com/Microsoft/TypeScript/issues/9876
  # We subtract the ".json" from the end before handing to TypeScript because
  # this gives extra error-checking.
  if ctx.file.tsconfig:
    workspace_path = config["compilerOptions"]["rootDir"]
    config["extends"] = "/".join([workspace_path, ctx.file.tsconfig.path[:-len(".json")]])

  if jsx_factory:
    config["compilerOptions"]["jsxFactory"] = jsx_factory

  return config

# ************ #
# ts_library   #
# ************ #

def _ts_library_impl(ctx):
  """Implementation of ts_library.

  Args:
    ctx: the context.
  Returns:
    the struct returned by the call to compile_ts.
  """
  ts_providers = compile_ts(ctx, is_library=True,
                            compile_action=_compile_action,
                            devmode_compile_action=_devmode_compile_action,
                            tsc_wrapped_tsconfig=tsc_wrapped_tsconfig)
  return ts_providers_dict_to_struct(ts_providers)

ts_library = rule(
    _ts_library_impl,
    attrs = dict(COMMON_ATTRIBUTES, **{
        "srcs":
            attr.label_list(
                allow_files=FileType([
                    ".ts",
                    ".tsx",
                ]),
                mandatory=True,),

        # TODO(alexeagle): reconcile with google3: ts_library rules should
        # be portable across internal/external, so we need this attribute
        # internally as well.
        "tsconfig":
            attr.label(allow_files = True, single_file = True),
        "compiler":
            attr.label(
                default=get_tsc(),
                single_file=False,
                allow_files=True,
                executable=True,
                cfg="host"),
        "supports_workers": attr.bool(default = True),
        # @// is special syntax for the "main" repository
        # The default assumes the user specified a target "node_modules" in their
        # root BUILD file.
        "node_modules": attr.label(default = Label("@//:node_modules")),
    }),
    outputs = {
        "tsconfig": "%{name}_tsconfig.json"
    }
)
<|endoftext|>


<commit_before># coding=utf-8
# Copyright 2019 The Tensor2Tensor Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Data generators for translation data-sets."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from tensor2tensor.data_generators import problem
from tensor2tensor.data_generators import text_problems
from tensor2tensor.data_generators import translate
from tensor2tensor.data_generators import wiki_lm
from tensor2tensor.utils import registry

_ENDE_TRAIN_DATASETS = [
    [
        "http://data.statmt.org/wmt18/translation-task/training-parallel-nc-v13.tgz",  # pylint: disable=line-too-long
        ("training-parallel-nc-v13/news-commentary-v13.de-en.en",
         "training-parallel-nc-v13/news-commentary-v13.de-en.de")
    ],
    [
        "http://www.statmt.org/wmt13/training-parallel-commoncrawl.tgz",
        ("commoncrawl.de-en.en", "commoncrawl.de-en.de")
    ],
    [
        "http://www.statmt.org/wmt13/training-parallel-europarl-v7.tgz",
        ("training/europarl-v7.de-en.en", "training/europarl-v7.de-en.de")
    ],
]
_ENDE_EVAL_DATASETS = [
    [
        "http://data.statmt.org/wmt17/translation-task/dev.tgz",
        ("dev/newstest2013.en", "dev/newstest2013.de")
    ],
]
_ENDE_PARACRAWL_DATASETS = [
    [
        "https://s3.amazonaws.com/web-language-models/paracrawl/release3/en-de.bicleaner07.tmx.gz",  # pylint: disable=line-too-long
        ("tmx", "en-de.bicleaner07.tmx.gz")
    ]
]

@registry.register_problem
class TranslateEndeWmt8k(translate.TranslateProblem):
  """Problem spec for WMT En-De translation."""

  @property
  def approx_vocab_size(self):
    return 2**13  # 8192

  @property
  def additional_training_datasets(self):
    """Allow subclasses to add training datasets."""
    return []

  def source_data_files(self, dataset_split):
    train = dataset_split == problem.DatasetSplit.TRAIN
    train_datasets = _ENDE_TRAIN_DATASETS + self.additional_training_datasets
    return train_datasets if train else _ENDE_EVAL_DATASETS

@registry.register_problem
class TranslateEndeWmt32k(TranslateEndeWmt8k):

  @property
  def approx_vocab_size(self):
    return 2**15  # 32768

@registry.register_problem
class TranslateEndeWmtClean32k(TranslateEndeWmt32k):

  @property
  def use_vocab_from_other_problem(self):
    return TranslateEndeWmt32k()

  @property
  def datatypes_to_clean(self):
    return ["txt"]

@registry.register_problem
class TranslateEndeParacrawl32k(translate.TranslateProblem):
  """Problem spec for Paracrawl en-de translation."""

  @property
  def use_vocab_from_other_problem(self):
    return TranslateEndeWmt32k()

  @property
  def additional_training_datasets(self):
    """Allow subclasses to add training datasets."""
    return []

  def source_data_files(self, dataset_split):
    train = dataset_split == problem.DatasetSplit.TRAIN
    train_datasets = (
        _ENDE_PARACRAWL_DATASETS + self.additional_training_datasets)
    return train_datasets if train else _ENDE_EVAL_DATASETS

@registry.register_problem
class TranslateEndeParacrawlClean32k(TranslateEndeParacrawl32k):
  """Paracrawl en-de Bicleaner corpus, with additional cleaning."""

  @property
  def datatypes_to_clean(self):
    return ["tmx"]

@registry.register_problem
class TranslateEndeWmtParacrawlBicleaner32k(TranslateEndeWmt32k):
  """WMT en-de corpus with extra data from Paracrawl, cleaned with Bicleaner."""

  @property
  def use_vocab_from_other_problem(self):
    return TranslateEndeWmt32k()

  @property
  def additional_training_datasets(self):
    return _ENDE_PARACRAWL_DATASETS

@registry.register_problem
class TranslateEndeWmtCleanParacrawl32k(TranslateEndeWmtParacrawlBicleaner32k):

  @property
  def datatypes_to_clean(self):
    return ["txt"]

@registry.register_problem
class TranslateEndeWmtParacrawlClean32k(TranslateEndeWmtParacrawlBicleaner32k):

  @property
  def datatypes_to_clean(self):
    return ["tmx"]

@registry.register_problem
class TranslateEndeWmtParacrawlAllClean32k(TranslateEndeWmtParacrawlClean32k):

  @property
  def datatypes_to_clean(self):
    return ["txt", "tmx"]

@registry.register_problem
class TranslateEndeWmt32kPacked(TranslateEndeWmt32k):

  @property
  def packed_length(self):
    return 256

  @property
  def use_vocab_from_other_problem(self):
    return TranslateEndeWmt32k()

@registry.register_problem
class TranslateEndeWmt8kPacked(TranslateEndeWmt8k):

  @property
  def packed_length(self):
    return 256

  @property
  def use_vocab_from_other_problem(self):
    return TranslateEndeWmt8k()

@registry.register_problem
class TranslateEndeWmtCharacters(TranslateEndeWmt8k):
  """Problem spec for WMT En-De translation."""

  @property
  def vocab_type(self):
    return text_problems.VocabType.CHARACTER

@registry.register_problem
class TranslateEndeWmtMulti64k(TranslateEndeWmt8k):
  """Translation with muli-lingual vocabulary."""

  @property
  def use_vocab_from_other_problem(self):
    return wiki_lm.LanguagemodelDeEnFrRoWiki64k()

@registry.register_problem
class TranslateEndeWmtMulti64kPacked1k(TranslateEndeWmtMulti64k):
  """Translation with muli-lingual vocabulary."""

  @property
  def packed_length(self):
    return 1024

  @property
  def num_training_examples(self):
    return 173800

  @property
  def inputs_prefix(self):
    return "translate English German "

  @property
  def targets_prefix(self):
    return "translate German English "
<commit_msg>Update to Paracrawl release4, and simplify/regularize some problem names.<commit_after># coding=utf-8
# Copyright 2019 The Tensor2Tensor Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Data generators for translation data-sets."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from tensor2tensor.data_generators import problem
from tensor2tensor.data_generators import text_problems
from tensor2tensor.data_generators import translate
from tensor2tensor.data_generators import wiki_lm
from tensor2tensor.utils import registry

_ENDE_TRAIN_DATASETS = [
    [
        "http://data.statmt.org/wmt18/translation-task/training-parallel-nc-v13.tgz",  # pylint: disable=line-too-long
        ("training-parallel-nc-v13/news-commentary-v13.de-en.en",
         "training-parallel-nc-v13/news-commentary-v13.de-en.de")
    ],
    [
        "http://www.statmt.org/wmt13/training-parallel-commoncrawl.tgz",
        ("commoncrawl.de-en.en", "commoncrawl.de-en.de")
    ],
    [
        "http://www.statmt.org/wmt13/training-parallel-europarl-v7.tgz",
        ("training/europarl-v7.de-en.en", "training/europarl-v7.de-en.de")
    ],
]
_ENDE_EVAL_DATASETS = [
    [
        "http://data.statmt.org/wmt17/translation-task/dev.tgz",
        ("dev/newstest2013.en", "dev/newstest2013.de")
    ],
]
_ENDE_PARACRAWL_DATASETS = [
    [
        "https://s3.amazonaws.com/web-language-models/paracrawl/release4/en-de.bicleaner07.tmx.gz",  # pylint: disable=line-too-long
        ("tmx", "en-de.bicleaner07.tmx.gz")
    ]
]

@registry.register_problem
class TranslateEndeWmt8k(translate.TranslateProblem):
  """Problem spec for WMT En-De translation."""

  @property
  def approx_vocab_size(self):
    return 2**13  # 8192

  @property
  def additional_training_datasets(self):
    """Allow subclasses to add training datasets."""
    return []

  def source_data_files(self, dataset_split):
    train = dataset_split == problem.DatasetSplit.TRAIN
    train_datasets = _ENDE_TRAIN_DATASETS + self.additional_training_datasets
    return train_datasets if train else _ENDE_EVAL_DATASETS

@registry.register_problem
class TranslateEndeWmt32k(TranslateEndeWmt8k):
  """En-de translation trained on WMT corpus."""

  @property
  def approx_vocab_size(self):
    return 2**15  # 32768

@registry.register_problem
class TranslateEndeWmtClean32k(TranslateEndeWmt32k):
  """En-de translation trained on WMT with further cleaning."""

  @property
  def use_vocab_from_other_problem(self):
    return TranslateEndeWmt32k()

  @property
  def datatypes_to_clean(self):
    return ["txt"]

@registry.register_problem
class TranslateEndePc32k(translate.TranslateProblem):
  """En-de translation trained on Paracrawl (bicleaner corpus)."""

  @property
  def use_vocab_from_other_problem(self):
    return TranslateEndeWmt32k()

  @property
  def additional_training_datasets(self):
    """Allow subclasses to add training datasets."""
    return []

  def source_data_files(self, dataset_split):
    train = dataset_split == problem.DatasetSplit.TRAIN
    train_datasets = (
        _ENDE_PARACRAWL_DATASETS + self.additional_training_datasets)
    return train_datasets if train else _ENDE_EVAL_DATASETS

@registry.register_problem
class TranslateEndePcClean32k(TranslateEndePc32k):
  """En-de translation trained on Paracrawl with further cleaning."""

  @property
  def datatypes_to_clean(self):
    return ["tmx"]

@registry.register_problem
class TranslateEndeWmtPc32k(TranslateEndeWmt32k):
  """En-de translation trained on WMT plus Paracrawl."""

  @property
  def use_vocab_from_other_problem(self):
    return TranslateEndeWmt32k()

  @property
  def additional_training_datasets(self):
    return _ENDE_PARACRAWL_DATASETS

@registry.register_problem
class TranslateEndeWmtCleanPc32k(TranslateEndeWmtPc32k):
  """En-de translation trained on cleaned WMT plus Paracrawl."""

  @property
  def datatypes_to_clean(self):
    return ["txt"]

@registry.register_problem
class TranslateEndeWmtPcClean32k(TranslateEndeWmtPc32k):
  """En-de translation trained on WMT plus cleaned Paracrawl."""

  @property
  def datatypes_to_clean(self):
    return ["tmx"]

@registry.register_problem
class TranslateEndeWmtCleanPcClean32k(TranslateEndeWmtPcClean32k):
  """En-de translation trained on cleaned WMT plus cleaned Paracrawl."""

  @property
  def datatypes_to_clean(self):
    return ["txt", "tmx"]

@registry.register_problem
class TranslateEndeWmt32kPacked(TranslateEndeWmt32k):

  @property
  def packed_length(self):
    return 256

  @property
  def use_vocab_from_other_problem(self):
    return TranslateEndeWmt32k()

@registry.register_problem
class TranslateEndeWmt8kPacked(TranslateEndeWmt8k):

  @property
  def packed_length(self):
    return 256

  @property
  def use_vocab_from_other_problem(self):
    return TranslateEndeWmt8k()

@registry.register_problem
class TranslateEndeWmtCharacters(TranslateEndeWmt8k):
  """Problem spec for WMT En-De translation."""

  @property
  def vocab_type(self):
    return text_problems.VocabType.CHARACTER

@registry.register_problem
class TranslateEndeWmtMulti64k(TranslateEndeWmt8k):
  """Translation with muli-lingual vocabulary."""

  @property
  def use_vocab_from_other_problem(self):
    return wiki_lm.LanguagemodelDeEnFrRoWiki64k()

@registry.register_problem
class TranslateEndeWmtMulti64kPacked1k(TranslateEndeWmtMulti64k):
  """Translation with muli-lingual vocabulary."""

  @property
  def packed_length(self):
    return 1024

  @property
  def num_training_examples(self):
    return 173800

  @property
  def inputs_prefix(self):
    return "translate English German "

  @property
  def targets_prefix(self):
    return "translate German English "
<|endoftext|>