<commit_before>from ..libs.view_helpers import *
from ..libs.text_helpers import *
from ..libs import log
from .event_hub import EventHub

class TimeoutScheduler:
    """
    If there are multiple timeouts set for the same function, only call the
    function in the last timeout in chronological order.
    """
    def __init__(self, handler):
        self.timeout_count = 0 # number of pending timeouts
        self.handler = handler

    def reset_timeout(self, mstime):
        self.timeout_count += 1
        sublime.set_timeout(self._check_timeout_count, mstime)

    def _check_timeout_count(self):
        self.timeout_count -= 1
        if self.timeout_count == 0:
            self.handler()

class IdleListener:
    def __init__(self):
        self.just_changed_focus = False
        self.modified = False  # Flag to check if buffer has just been modified
        self.event_handler_added = False
        self.occurrences_event_handler_added = False

        self.on_idle_timeout_scheduler = TimeoutScheduler(self.on_idle)
        self.on_selection_idle_timeout_scheduler = TimeoutScheduler(self.on_selection_idle)

    # LISTENERS

    def on_activated_with_info(self, view, info):
        # Set modified and selection idle timers, so we can read diagnostics
        # and update status line
        self.on_idle_timeout_scheduler.reset_timeout(IDLE_TIME_LENGTH)
        self.on_selection_idle_timeout_scheduler.reset_timeout(IDLE_TIME_LENGTH)
        self.just_changed_focus = True

    def post_on_modified(self, view):
        if not is_special_view(view):
            self.modified = True
            self.on_idle_timeout_scheduler.reset_timeout(100)

    def on_selection_modified_with_info(self, view, info):
        if self.modified:
            self.on_selection_idle_timeout_scheduler.reset_timeout(1250)
        else:
            self.on_selection_idle_timeout_scheduler.reset_timeout(100)
        self.modified = False

    # HELPER METHODS

    def on_idle(self):
        """
        If file hasn't been modified for a period of time, send a request for
        errors.
        """
        log.debug("on_idle")
        view = active_view()
        info = get_info(view)
        if info:
            self.request_errors(view, info, 500)

    def on_selection_idle(self):
        """
        If selection is idle (cursor is not moving around), update the status
        line (error message or quick info, if any) and update document highlights.
        """
        log.debug("on_selection_idle")
        view = active_view()
        info = get_info(view)
        if info:
            self.update_status(view, info)
            if not IS_ST2 and view.settings().get('typescript_highlight_occurrences', 'true'):
                self.request_document_highlights(view, info)

    def request_errors(self, view, info, error_delay):
        """
        Ask the server for diagnostic information on all opened ts files in
        most-recently-used order
        """
        if not self.event_handler_added:
            cli.service.add_event_handler("syntaxDiag", lambda ev: self.show_errors(ev["body"], syntactic=True))
            cli.service.add_event_handler("semanticDiag", lambda ev: self.show_errors(ev["body"], syntactic=False))
            self.event_handler_added = True

        # Todo: limit this request to ts files currently visible in views
        if info and (self.just_changed_focus or info.change_count_when_last_err_req_sent < change_count(view)):
            self.just_changed_focus = False
            info.change_count_when_last_err_req_sent = change_count(view)
            window = sublime.active_window()
            group_number = window.num_groups()
            files = []
            for i in range(group_number):
                group_active_view = window.active_view_in_group(i)
                info = get_info(group_active_view)
                if info:
                    files.append(group_active_view.file_name())
                    check_update_view(group_active_view)
            if len(files) > 0:
                cli.service.request_get_err(error_delay, files)

    def show_errors(self, diagno_event_body, syntactic):
        """
        Error messages arrived from the server; show them in view
        """
        log.debug("show_errors")
        filename = diagno_event_body["file"]
        if os.name == 'nt' and filename:
            filename = filename.replace('/', '\\')

        info = get_info_with_filename(filename)
        if not info:
            return

        view = info.view
        if not info.change_count_when_last_err_req_sent == change_count(view):
            log.debug("The error info is outdated")
            self.on_idle_timeout_scheduler.reset_timeout(200)
            return

        region_key = 'syntacticDiag' if syntactic else 'semanticDiag'
        view.erase_regions(region_key)

        client_info = cli.get_or_add_file(filename)
        client_info.errors[region_key] = []
        error_regions = []

        diagnos = diagno_event_body["diagnostics"]
        if diagnos:
            for diagno in diagnos:
                start_line, start_offset = extract_line_offset(diagno["start"])
                start = view.text_point(start_line, start_offset)

                end_line, end_offset = extract_line_offset(diagno["end"])
                end = view.text_point(end_line, end_offset)

                if end <= view.size():
                    # Creates a <sublime.Region> from <start> to <end> to
                    # highlight the error. If the region coincides with the
                    # EOF character, use the region of the last visible
                    # character instead so user can still see the highlight.
                    if start == view.size() and start == end:
                        region = last_visible_character_region(view)
                    else:
                        region = sublime.Region(start, end)

                    client_info.errors[region_key].append((region, diagno["text"]))
                    error_regions.append(region)

        # Update status bar with error information
        info.has_errors = cli.has_errors(filename)
        self.update_status(view, info)

        # Highlight error regions in view
        if IS_ST2:
            view.add_regions(region_key, error_regions, "invalid", "",
                             sublime.DRAW_OUTLINED)
        else:
            settings = sublime.load_settings("TypeScript.sublime-settings")
            view.add_regions(region_key,
                             error_regions,
                             settings.get("error_color", "invalid.illegal"),
                             settings.get("error_icon", ""),
                             sublime.DRAW_OUTLINED
                             if settings.get("error_outlined") else
                             sublime.DRAW_NO_FILL +
                             sublime.DRAW_NO_OUTLINE +
                             sublime.DRAW_SQUIGGLY_UNDERLINE)

    def update_status(self, view, info):
        """Update the status line with error info and quick info if no error info"""
        # Error infos
        view.erase_status("typescript_error")

        # Quick info
        error_status = view.get_status('typescript_error')
        if error_status and len(error_status) > 0:
            view.erase_status("typescript_info")
        else:
            view.run_command('typescript_quick_info')

    def request_document_highlights(self, view, info):
        if is_typescript(view):
            location = get_location_from_view(view)
            cli.service.async_document_highlights(view.file_name(), location, self.highlight_occurrences)

    def highlight_occurrences(self, response):
        view = active_view()
        if not view.file_name():
            return

        region_key = 'occurrences'
        view.erase_regions(region_key)

        if not response['success']:
            return

        occurrence_regions = []

        for file_highlight in response['body']:
            if file_highlight['file'] != view.file_name().replace('\\', '/'):
                continue

            for occurrence in file_highlight['highlightSpans']:
                start_line, start_offset = extract_line_offset(occurrence['start'])
                start_point = view.text_point(start_line, start_offset)

                end_line, end_offset = extract_line_offset(occurrence['end'])
                end_point = view.text_point(end_line, end_offset)

                occurrence_regions.append(sublime.Region(start_point, end_point))

        view.add_regions(region_key, occurrence_regions, 'comment', '',
                         sublime.DRAW_NO_FILL |
                         sublime.DRAW_NO_OUTLINE |
                         sublime.DRAW_SOLID_UNDERLINE)

listener = IdleListener()
EventHub.subscribe("on_activated_with_info", listener.on_activated_with_info)
EventHub.subscribe("post_on_modified", listener.post_on_modified)
EventHub.subscribe("on_selection_modified_with_info", listener.on_selection_modified_with_info)
<commit_msg>Correctly use default error color in the presence of an empty st… (#735)<commit_after>from ..libs.view_helpers import *
from ..libs.text_helpers import *
from ..libs import log
from .event_hub import EventHub

class TimeoutScheduler:
    """
    If there are multiple timeouts set for the same function, only call the
    function in the last timeout in chronological order.
    """
    def __init__(self, handler):
        self.timeout_count = 0 # number of pending timeouts
        self.handler = handler

    def reset_timeout(self, mstime):
        self.timeout_count += 1
        sublime.set_timeout(self._check_timeout_count, mstime)

    def _check_timeout_count(self):
        self.timeout_count -= 1
        if self.timeout_count == 0:
            self.handler()

class IdleListener:
    def __init__(self):
        self.just_changed_focus = False
        self.modified = False  # Flag to check if buffer has just been modified
        self.event_handler_added = False
        self.occurrences_event_handler_added = False

        self.on_idle_timeout_scheduler = TimeoutScheduler(self.on_idle)
        self.on_selection_idle_timeout_scheduler = TimeoutScheduler(self.on_selection_idle)

    # LISTENERS

    def on_activated_with_info(self, view, info):
        # Set modified and selection idle timers, so we can read diagnostics
        # and update status line
        self.on_idle_timeout_scheduler.reset_timeout(IDLE_TIME_LENGTH)
        self.on_selection_idle_timeout_scheduler.reset_timeout(IDLE_TIME_LENGTH)
        self.just_changed_focus = True

    def post_on_modified(self, view):
        if not is_special_view(view):
            self.modified = True
            self.on_idle_timeout_scheduler.reset_timeout(100)

    def on_selection_modified_with_info(self, view, info):
        if self.modified:
            self.on_selection_idle_timeout_scheduler.reset_timeout(1250)
        else:
            self.on_selection_idle_timeout_scheduler.reset_timeout(100)
        self.modified = False

    # HELPER METHODS

    def on_idle(self):
        """
        If file hasn't been modified for a period of time, send a request for
        errors.
        """
        log.debug("on_idle")
        view = active_view()
        info = get_info(view)
        if info:
            self.request_errors(view, info, 500)

    def on_selection_idle(self):
        """
        If selection is idle (cursor is not moving around), update the status
        line (error message or quick info, if any) and update document highlights.
        """
        log.debug("on_selection_idle")
        view = active_view()
        info = get_info(view)
        if info:
            self.update_status(view, info)
            if not IS_ST2 and view.settings().get('typescript_highlight_occurrences', 'true'):
                self.request_document_highlights(view, info)

    def request_errors(self, view, info, error_delay):
        """
        Ask the server for diagnostic information on all opened ts files in
        most-recently-used order
        """
        if not self.event_handler_added:
            cli.service.add_event_handler("syntaxDiag", lambda ev: self.show_errors(ev["body"], syntactic=True))
            cli.service.add_event_handler("semanticDiag", lambda ev: self.show_errors(ev["body"], syntactic=False))
            self.event_handler_added = True

        # Todo: limit this request to ts files currently visible in views
        if info and (self.just_changed_focus or info.change_count_when_last_err_req_sent < change_count(view)):
            self.just_changed_focus = False
            info.change_count_when_last_err_req_sent = change_count(view)
            window = sublime.active_window()
            group_number = window.num_groups()
            files = []
            for i in range(group_number):
                group_active_view = window.active_view_in_group(i)
                info = get_info(group_active_view)
                if info:
                    files.append(group_active_view.file_name())
                    check_update_view(group_active_view)
            if len(files) > 0:
                cli.service.request_get_err(error_delay, files)

    def show_errors(self, diagno_event_body, syntactic):
        """
        Error messages arrived from the server; show them in view
        """
        log.debug("show_errors")
        filename = diagno_event_body["file"]
        if os.name == 'nt' and filename:
            filename = filename.replace('/', '\\')

        info = get_info_with_filename(filename)
        if not info:
            return

        view = info.view
        if not info.change_count_when_last_err_req_sent == change_count(view):
            log.debug("The error info is outdated")
            self.on_idle_timeout_scheduler.reset_timeout(200)
            return

        region_key = 'syntacticDiag' if syntactic else 'semanticDiag'
        view.erase_regions(region_key)

        client_info = cli.get_or_add_file(filename)
        client_info.errors[region_key] = []
        error_regions = []

        diagnos = diagno_event_body["diagnostics"]
        if diagnos:
            for diagno in diagnos:
                start_line, start_offset = extract_line_offset(diagno["start"])
                start = view.text_point(start_line, start_offset)

                end_line, end_offset = extract_line_offset(diagno["end"])
                end = view.text_point(end_line, end_offset)

                if end <= view.size():
                    # Creates a <sublime.Region> from <start> to <end> to
                    # highlight the error. If the region coincides with the
                    # EOF character, use the region of the last visible
                    # character instead so user can still see the highlight.
                    if start == view.size() and start == end:
                        region = last_visible_character_region(view)
                    else:
                        region = sublime.Region(start, end)

                    client_info.errors[region_key].append((region, diagno["text"]))
                    error_regions.append(region)

        # Update status bar with error information
        info.has_errors = cli.has_errors(filename)
        self.update_status(view, info)

        # Highlight error regions in view
        if IS_ST2:
            view.add_regions(region_key, error_regions, "invalid", "",
                             sublime.DRAW_OUTLINED)
        else:
            settings = sublime.load_settings("TypeScript.sublime-settings")
            error_color = settings.get("error_color") or "invalid.illegal"
            error_icon = settings.get("error_icon") or ""
            error_draw_style = \
                sublime.DRAW_OUTLINED if settings.get("error_outlined") \
                else sublime.DRAW_NO_FILL | sublime.DRAW_NO_OUTLINE | sublime.DRAW_SQUIGGLY_UNDERLINE
            view.add_regions(region_key,
                             error_regions,
                             error_color,
                             error_icon,
                             error_draw_style)

    def update_status(self, view, info):
        """Update the status line with error info and quick info if no error info"""
        # Error infos
        view.erase_status("typescript_error")

        # Quick info
        error_status = view.get_status('typescript_error')
        if error_status and len(error_status) > 0:
            view.erase_status("typescript_info")
        else:
            view.run_command('typescript_quick_info')

    def request_document_highlights(self, view, info):
        if is_typescript(view):
            location = get_location_from_view(view)
            cli.service.async_document_highlights(view.file_name(), location, self.highlight_occurrences)

    def highlight_occurrences(self, response):
        view = active_view()
        if not view.file_name():
            return

        region_key = 'occurrences'
        view.erase_regions(region_key)

        if not response['success']:
            return

        occurrence_regions = []

        for file_highlight in response['body']:
            if file_highlight['file'] != view.file_name().replace('\\', '/'):
                continue

            for occurrence in file_highlight['highlightSpans']:
                start_line, start_offset = extract_line_offset(occurrence['start'])
                start_point = view.text_point(start_line, start_offset)

                end_line, end_offset = extract_line_offset(occurrence['end'])
                end_point = view.text_point(end_line, end_offset)

                occurrence_regions.append(sublime.Region(start_point, end_point))

        view.add_regions(region_key, occurrence_regions, 'comment', '',
                         sublime.DRAW_NO_FILL |
                         sublime.DRAW_NO_OUTLINE |
                         sublime.DRAW_SOLID_UNDERLINE)

listener = IdleListener()
EventHub.subscribe("on_activated_with_info", listener.on_activated_with_info)
EventHub.subscribe("post_on_modified", listener.post_on_modified)
EventHub.subscribe("on_selection_modified_with_info", listener.on_selection_modified_with_info)
<|endoftext|>


<commit_before>from experience_tables import build_tables

EXP_TABLES = build_tables()
GROUPS = ['fluctuating', 'slow', 'medium_slow',
          'medium_fast', 'fast', 'erratic']

class Experience():
    """Experience.

    The implementation is based around experience tables created in the
    experience tables module. The Experience class will be inherited by
    the Statistics class which will ultimately be inherited by the Pokemon
    class.

    Parameters
    ----------
    group : string, required
            Specifies the experience group the pokemon belongs to.
            It must be either 'fluctuating', 'slow', 'medium_slow', 'medium_fast',
            'fast', or 'erratic', or a callable.

    current_exp : integer, optional (default=0)
            The current experience of the pokemon.

    """

    def __init__(self, group, current_exp=0):

        value_check(group=group, experience=current_exp)
        self.exp_group = group.lower()
        self.current_exp = current_exp
        self.get_current_level() # Calculates and sets current_level

    def get_current_level(self):
        """
        Loops through the keys of the corresponding experience table and
        increases the estimated level by 1 until it overshoots, then sets
        self.current_level to the previous estimate.

        Parameters
        ----------
        None

        Returns
        -------
        self : object
            Returns self.

        Notes
        -----
        The experience table currently goes to level 101 despite there not
        actually being a level 101. It does this so the function has a level
        to overshoot too if the pokemon is at level 100. This should probably
        be a little cleaner.

        """
        estimate = 1
        for key in EXP_TABLES[self.exp_group]:
            if key <= self.current_exp:
                estimate += 1
            else:
                self.current_level = estimate - 1
        return self

    def to_next_level(self):
        """
        Loops through the keys in the corresponding experience table. When a
        key is found that is higher than the current experience, returns the
        difference between those values.

        Parameters
        ----------
        None

        Returns
        -------
        experience : integer
            Returns the experience required to reach the next level.

        """
        for key in EXP_TABLES[self.exp_group]:
            if key > self.current_exp:
                return key - self.current_exp

    def exp_needed_to_reach(self, desired_level):
        """
        Loops through the corresponding experience table for the desired level
        and returns the difference between current_exp and experience required
        for that level.

        Parameters
        ----------
        desired_level : integer
            The level you want to know your distance to

        Returns
        -------
        experience : integer
            Returns the amount of experience needed to reach the desired_level

        """
        for key, value in EXP_TABLES[self.exp_group].items():
            if value == desired_level:
                return key - self.current_exp

    def set_experience_group(self, new_group):
        """
        Verifies the new group as valid then sets the new group.

        Parameters
        ----------
        new_group : string
            The new group you want to assign

        Returns
        -------
        self : object
            Returns self.
        """
        value_check(group=new_group)
        self.exp_group = new_group
        return self
    #
    # def current_experience(self, new_value):
    #     # Change the current experience to this value
    #
    # def experience_multipliers(self, **kwargs):
    #     # Change multiplier values
    #
    # def defeat_opponent(self, **kwargs):
    #     # Return experience gained from defeating opponent pokemon
    #
    # def gain_experience(self, new_current_exp):
    #     # Set current exp to new value

def value_check(group=None, experience=0):
    """
    Makes sure values related to this module are okay.

    Parameters
    ----------
    group : string (default=None)
        The experience group to be checked

    experience : integer (default=0)
        The experience value to be checked

    Returns
    -------
    ValueError : Error
        Only if the value being checked is no bueno

    """
    if group.lower() not in GROUPS:
        raise ValueError("group should be one of {}, {} was given".format(
            GROUPS, group))

    if experience < 0 or isinstance(experience, int) is False:
        raise ValueError("current_exp must be a positive whole number,"
                         " {} was given".format(experience))

if __name__ == '__main__':
    a = Experience(group = 'SLOW', current_exp = 50000)
    print(a.current_level)
    print(a.exp_needed_to_reach(100))
    a.set_experience_group('fast')
<commit_msg>Make exp_needed_to_reach more generalized<commit_after>from experience_tables import build_tables

EXP_TABLES = build_tables()
GROUPS = ['fluctuating', 'slow', 'medium_slow',
          'medium_fast', 'fast', 'erratic']

class Experience():
    """The Experience Class.

    The implementation is based around experience tables created in the
    experience tables module.

    Parameters
    ----------
    group : string, required
            Specifies the experience group the pokemon belongs to.
            It must be either 'fluctuating', 'slow', 'medium_slow', 'medium_fast',
            'fast', or 'erratic', or a callable.

    current_exp : integer, optional (default=0)
            The current experience of the pokemon.

    """

    def __init__(self, group, current_exp=0):

        value_check(group=group, experience=current_exp)
        self.exp_group = group.lower()
        self.current_exp = current_exp
        self.get_current_level() # Calculates and sets current_level

    def get_current_level(self):
        """
        Loops through the keys of the corresponding experience table and
        increases the estimated level by 1 until it overshoots, then sets
        self.current_level to the previous estimate.

        Parameters
        ----------
        None

        Returns
        -------
        self : object
            Returns self.

        Notes
        -----
        The experience table currently goes to level 101 despite there not
        actually being a level 101. It does this so the function has a level
        to overshoot too if the pokemon is at level 100. This should probably
        be a little cleaner.

        """
        estimate = 1
        for key in EXP_TABLES[self.exp_group]:
            if key <= self.current_exp:
                estimate += 1
            else:
                self.current_level = estimate - 1
        return self

    def to_next_level(self):
        """
        Loops through the keys in the corresponding experience table. When a
        key is found that is higher than the current experience, returns the
        difference between those values.

        Parameters
        ----------
        None

        Returns
        -------
        experience : integer
            Returns the experience required to reach the next level.

        """
        for key in EXP_TABLES[self.exp_group]:
            if key > self.current_exp:
                return key - self.current_exp

    def exp_needed_to_reach(self, to_level, from_exp=None):
        """
        Loops through the corresponding experience table for the desired level
        and returns the difference between starting_exp and experience required
        for the desired level.

        Parameters
        ----------
        desired_level : integer
            The level you want to know your distance to

        Returns
        -------
        experience : integer
            Returns the amount of experience needed to reach the desired_level

        """
        if from_exp is None:
            from_exp = self.current_exp
        for key, value in EXP_TABLES[self.exp_group].items():
            if value == to_level:
                return key - from_exp

    def set_experience_group(self, new_group):
        """
        Verifies the new group as valid then sets the new group.

        Parameters
        ----------
        new_group : string
            The new group you want to assign

        Returns
        -------
        self : object
            Returns self.
        """
        value_check(group=new_group)
        self.exp_group = new_group
        return self
    #
    # def current_experience(self, new_value):
    #     # Change the current experience to this value
    #
    # def experience_multipliers(self, **kwargs):
    #     # Change multiplier values
    #
    # def defeat_opponent(self, **kwargs):
    #     # Return experience gained from defeating opponent pokemon
    #
    # def gain_experience(self, new_current_exp):
    #     # Set current exp to new value

def value_check(group=None, experience=0):
    """
    Makes sure values related to this module are okay.

    Parameters
    ----------
    group : string (default=None)
        The experience group to be checked

    experience : integer (default=0)
        The experience value to be checked

    Returns
    -------
    ValueError : Error
        Only if the value being checked is no bueno

    """
    if group.lower() not in GROUPS:
        raise ValueError("group should be one of {}, {} was given".format(
            GROUPS, group))

    if experience < 0 or isinstance(experience, int) is False:
        raise ValueError("current_exp must be a positive whole number,"
                         " {} was given".format(experience))

if __name__ == '__main__':
    a = Experience(group = 'SLOW', current_exp = 50000)
    print(a.current_level)
    print(a.exp_needed_to_reach(100))
    a.set_experience_group('fast')
<|endoftext|>


<commit_before>import logging
import platform
import re
import posixpath

logger = logging.getLogger(__name__)

class ParseError(Exception):
    pass

class HeaderHandler(object):

    def __init__(self, include_paths):
        self.include_paths = list(include_paths)
        self.resolved = {}

    def _open(self, header_path):
        try:
            f = open(header_path)
        except IOError:
            return None
        else:
            return f

    def add_include_paths(self, include_paths):
        self.include_paths.extend(include_paths)

    def _resolve(self, anchor_file):
        if anchor_file is not None:
            yield posixpath.dirname(anchor_file)
        for include_path in self.include_paths:
            yield include_path

    def open_header(self, include_header, skip_file, anchor_file):
        header_path = self.resolved.get(include_header)
        if header_path is not None:
            if skip_file(header_path):
                return SKIP_FILE
            else:
                return self._open(header_path)
        for include_path in self._resolve(anchor_file):
            header_path = posixpath.join(include_path, include_header)
            f = self._open(posixpath.normpath(header_path))
            if f:
                self.resolved[include_header] = f.name
                break
        return f

def calculate_windows_constants(bitness=None):
    if bitness is None:
        bitness, _ = platform.architecture()
    constants = {
        "WIN32": "WIN32", "_WIN64": "_WIN64"}
    if bitness == "64bit":
        constants["WIN64"] = "WIN64"
        constants["_WIN64"] = "_WIN64"
    elif not bitness == "32bit":
        raise Exception("Unsupported bitness %s" % bitness)
    return constants

def calculate_linux_constants(bitness=None):
    if bitness is None:
        bitness, _ = platform.architecture()
    constants = {
        "__linux__": "__linux__"
    }
    if bitness == "32bit":
        constants["__i386__"] = "__i386__"
    elif bitness == "64bit":
        constants["__x86_64__"] = "__x86_64"
    else:
        raise Exception("Unsupported bitness %s" % bitness)
    return constants

def calculate_platform_constants():
    system = platform.system()
    if system == "Windows":
        constants = calculate_windows_constants()
    elif system == "Linux":
        constants = calculate_linux_constants()
    else:
        raise ParseError("Unsupported platform %s" % platform)
    constants["__SIZE_TYPE__"] = "size_t"
    return constants

PLATFORM_CONSTANTS = calculate_platform_constants()
DEFAULT_LINE_ENDING = "\n"
PRAGMA_ONCE = "pragma_once"
IFDEF = "ifdef"
IFNDEF = "ifndef"
ELSE = "else"
SKIP_FILE = object()
TOKEN = re.compile((r"<\w+(?:/\w+)*(?:\.\w+)?>|\".+\"|'\w'|/\*|"
                    r"\*/|//|\b\w+\b|\s+|\W"))
DOUBLE_QUOTE = '"'
SINGLE_QUOTE = "'"
CHAR = re.compile(r"^'\w'$")
CHUNK_MARK = object()
RSTRIP = object()

def _tokenize(s):
    for match in TOKEN.finditer(s):
        yield match.group(0)

class TokenExpander(object):
    def __init__(self, defines):
        self.defines = defines

    def expand_tokens(self, tokens, seen=()):
        for line_num, token in tokens:
            if token not in self.defines or token in seen:
                yield line_num, token
            else:
                new_seen = [token]
                new_seen.extend(seen)
                if len(new_seen) > 20:
                    raise Exception("Stopping with stack %s" % new_seen)
                tokens = [(line_num, token) for token in self.defines[token]]
                for token in self.expand_tokens(tokens, new_seen):
                    yield token

class Tokenizer(object):
    def __init__(self, f_obj, line_ending):
        self.source = enumerate(f_obj)
        self.line_ending = line_ending

    def __iter__(self):
        comment = None
        for line_no, line in self.source:
            line = line.rstrip("\r\n")
            tokens = _tokenize(line)
            token = None
            for token in tokens:
                if token == "*/" and comment == "/*":
                    comment = None
                elif comment:
                    continue
                elif token in ("//", "/*"):
                    comment = token
                    yield line_no, RSTRIP
                else:
                    yield line_no, token
            if comment == "//":
                if token != "\\":
                    comment = None
            if not comment:
                yield line_no, self.line_ending
                if token != "\\":
                    yield line_no, CHUNK_MARK

    def read_chunks(self):
        tokens = iter(self)
        chunk = []
        whitespace_only = True
        for line_no, token in tokens:
            if token is RSTRIP:
                chunk = chunk[:-1]
                continue
            elif token is CHUNK_MARK:
                if chunk:
                    yield chunk
                chunk = []
                whitespace_only = True
                continue
            chunk.append((line_no, token))
            if token == "#":
                if whitespace_only:
                    chunk = [(line_no, "#")]
            elif token.strip():
                whitespace_only = False

class Preprocessor(object):

    def __init__(self, line_ending=DEFAULT_LINE_ENDING, include_paths=(),
                 header_handler=None, platform_constants=PLATFORM_CONSTANTS,
                 ignore_headers=()):
        self.defines = {}
        self.ignore_headers = ignore_headers
        self.include_once = {}
        self.defines.update(platform_constants)
        self.constraints = []
        self.ignore = False
        self.line_ending = line_ending
        self.last_constraint = None
        self.header_stack = []
        self.token_expander = TokenExpander(self.defines)
        if header_handler is None:
            self.headers = HeaderHandler(include_paths)
        else:
            self.headers = header_handler
            self.headers.add_include_paths(include_paths)

    def process_define(self, **kwargs):
        if self.ignore:
            return
        chunk = kwargs["chunk"]
        for i, tokenized in enumerate(chunk):
            define_name = tokenized[1]
            if define_name.strip():
                rest = [e for i, e in chunk[i+2:-1]]
                break
        self.defines[define_name] = rest

    def process_endif(self, **kwargs):
        line_num = kwargs["line_num"]
        if not self.constraints:
            raise ParseError("Unexpected #endif on line %s" % line_num)
        (constraint_type, constraint, ignore,
         original_line_num) = self.constraints.pop()
        if ignore:
            self.ignore = False
        self.last_constraint = constraint, constraint_type, original_line_num

    def process_else(self, **kwargs):
        line_num = kwargs["line_num"]
        if not self.constraints:
            raise ParseError("Unexpected #else on line %s" % line_num)
        _, constraint, ignore, _ = self.constraints.pop()
        if self.ignore and ignore:
            ignore = False
            self.ignore = False
        elif not self.ignore and not ignore:
            ignore = True
            self.ignore = True
        self.constraints.append((ELSE, constraint, ignore, line_num))

    def process_ifdef(self, **kwargs):
        chunk = kwargs["chunk"]
        line_num = kwargs["line_num"]
        for _, condition in chunk:
            if condition.strip():
                break
        if not self.ignore and condition not in self.defines:
            self.ignore = True
            self.constraints.append((IFDEF, condition, True, line_num))
        else:
            self.constraints.append((IFDEF, condition, False, line_num))

    def process_pragma(self, **kwargs):
        chunk = kwargs["chunk"]
        line_num = kwargs["line_num"]
        pragma = None
        for _, pragma_name in chunk:
            if pragma_name.strip():
                method_name = "process_pragma_%s" % pragma_name
                pragma = getattr(self, method_name, None)
                break
        if pragma is None:
            raise Exception("Unsupported pragma %s on line %s" % (pragma_name,
                                                                  line_num))
        else:
            pragma(chunk=chunk, line_num=line_num)

    def process_pragma_once(self, **_):
        self.include_once[self.current_name()] = PRAGMA_ONCE

    def current_name(self):
        return self.header_stack[-1].name

    def process_ifndef(self, **kwargs):
        chunk = kwargs["chunk"]
        line_num = kwargs["line_num"]
        for _, condition in chunk:
            if condition.strip():
                break
        if not self.ignore and condition in self.defines:
            self.ignore = True
            self.constraints.append((IFNDEF, condition, True, line_num))
        else:
            self.constraints.append((IFNDEF, condition, False, line_num))

    def process_undef(self, **kwargs):
        chunk = kwargs["chunk"]
        for _, undefine in chunk:
            if undefine.strip():
                break
        try:
            del self.defines[undefine]
        except KeyError:
            pass

    def process_source_chunks(self, chunk):
        if not self.ignore:
            for chunk in self.token_expander.expand_tokens(chunk):
                yield chunk

    def skip_file(self, name):
        item = self.include_once.get(name)
        if item is PRAGMA_ONCE:
            return True
        elif item is None:
            return False
        else:
            constraint, constraint_type = item
            if constraint_type == IFDEF:
                return constraint not in self.defines
            elif constraint_type == IFNDEF:
                return constraint in self.defines
            else:
                raise Exception("Bug, constraint type %s" % constraint_type)

    def _read_header(self, header, error, anchor_file=None):
        if header not in self.ignore_headers:
            f = self.headers.open_header(header, self.skip_file, anchor_file)
            if f is None:
                raise error
            elif f is not SKIP_FILE:
                with f:
                    for line in self.preprocess(f):
                        yield None, line

    def process_include(self, **kwargs):
        chunk = kwargs["chunk"]
        line_num = kwargs["line_num"]
        for _, item in chunk:
            if item.strip():
                break
        s = "Line %s includes a file %s that can't be found" % (line_num,
                                                                item)
        error = ParseError(s)
        if item.startswith("<") and item.endswith(">"):
            header = item.strip("<>")
            return self._read_header(header, error)
        elif item.startswith('"') and item.endswith('"'):
            header = item.strip('"')
            return self._read_header(header, error, self.current_name())
        else:
            fmt = "Invalid include on line %s, got %r for include name"
            raise ParseError(fmt % (line_num, item))

    def check_fullfile_guard(self):
        if self.last_constraint is None:
            return
        constraint, constraint_type, begin = self.last_constraint
        if begin != 0:
            return
        self.include_once[self.current_name()] = constraint, constraint_type

    def preprocess(self, f_object, depth=0):
        self.header_stack.append(f_object)
        tokenizer = Tokenizer(f_object, self.line_ending)
        for chunk in tokenizer.read_chunks():
            self.last_constraint = None
            if chunk[0][1] == "#":
                line_num = chunk[0][0]
                macro_name = chunk[1][1]
                macro_chunk = chunk[2:]
                macro = getattr(self, "process_%s" % macro_name, None)
                if macro is None:
                    fmt = "Line number %s contains unsupported macro %s"
                    raise ParseError(fmt % (line_num, macro_name))
                ret = macro(line_num=line_num, chunk=macro_chunk)
                if ret is not None:
                    for _, token in ret:
                        yield token
            else:
                for _, token in self.process_source_chunks(chunk):
                    yield token
        self.check_fullfile_guard()
        self.header_stack.pop()
        if not self.header_stack and self.constraints:
            constraint_type, name, _, line_num = self.constraints[-1]
            if constraint_type is IFDEF:
                fmt = "#ifdef %s from line %s left open"
            elif constraint_type is IFNDEF:
                fmt = "#ifndef %s from line %s left open"
            else:
                fmt = "#else from line %s left open"
            raise ParseError(fmt % (name, line_num))

def preprocess(f_object, line_ending="\n", include_paths=(),
               header_handler=None, platform_constants=PLATFORM_CONSTANTS,
               ignore_headers=()):
    r"""
    This preprocessor yields lines with \n at the end
    """
    preprocessor = Preprocessor(line_ending, include_paths, header_handler,
                                platform_constants, ignore_headers)
    return preprocessor.preprocess(f_object)
<commit_msg>Introduce a Token class There were tuples being passed throughout the preprocessor with implied structure. Create a token class whose instances will now be the tokens. Slight changes in the structure of defines as a result<commit_after>import logging
import platform
import re
import posixpath

logger = logging.getLogger(__name__)

class ParseError(Exception):
    pass

class HeaderHandler(object):

    def __init__(self, include_paths):
        self.include_paths = list(include_paths)
        self.resolved = {}

    def _open(self, header_path):
        try:
            f = open(header_path)
        except IOError:
            return None
        else:
            return f

    def add_include_paths(self, include_paths):
        self.include_paths.extend(include_paths)

    def _resolve(self, anchor_file):
        if anchor_file is not None:
            yield posixpath.dirname(anchor_file)
        for include_path in self.include_paths:
            yield include_path

    def open_header(self, include_header, skip_file, anchor_file):
        header_path = self.resolved.get(include_header)
        if header_path is not None:
            if skip_file(header_path):
                return SKIP_FILE
            else:
                return self._open(header_path)
        for include_path in self._resolve(anchor_file):
            header_path = posixpath.join(include_path, include_header)
            f = self._open(posixpath.normpath(header_path))
            if f:
                self.resolved[include_header] = f.name
                break
        return f

def calculate_windows_constants(bitness=None):
    if bitness is None:
        bitness, _ = platform.architecture()
    constants = {
        "WIN32": "WIN32", "_WIN64": "_WIN64"}
    if bitness == "64bit":
        constants["WIN64"] = "WIN64"
        constants["_WIN64"] = "_WIN64"
    elif not bitness == "32bit":
        raise Exception("Unsupported bitness %s" % bitness)
    return constants

def calculate_linux_constants(bitness=None):
    if bitness is None:
        bitness, _ = platform.architecture()
    constants = {
        "__linux__": "__linux__"
    }
    if bitness == "32bit":
        constants["__i386__"] = "__i386__"
    elif bitness == "64bit":
        constants["__x86_64__"] = "__x86_64"
    else:
        raise Exception("Unsupported bitness %s" % bitness)
    return constants

def calculate_platform_constants():
    system = platform.system()
    if system == "Windows":
        constants = calculate_windows_constants()
    elif system == "Linux":
        constants = calculate_linux_constants()
    else:
        raise ParseError("Unsupported platform %s" % platform)
    constants["__SIZE_TYPE__"] = "size_t"
    return constants

PLATFORM_CONSTANTS = calculate_platform_constants()
DEFAULT_LINE_ENDING = "\n"
PRAGMA_ONCE = "pragma_once"
IFDEF = "ifdef"
IFNDEF = "ifndef"
ELSE = "else"
SKIP_FILE = object()
TOKEN = re.compile((r"<\w+(?:/\w+)*(?:\.\w+)?>|\".+\"|'\w'|/\*|"
                    r"\*/|//|\b\w+\b|\s+|\W"))
DOUBLE_QUOTE = '"'
SINGLE_QUOTE = "'"
CHAR = re.compile(r"^'\w'$")
CHUNK_MARK = object()
RSTRIP = object()

def _tokenize(s):
    for match in TOKEN.finditer(s):
        yield match.group(0)

class Token(object):
    def __init__(self, line_no, value, whitespace):
        self.line_no = line_no
        self.value = value
        self.whitespace = whitespace

    @classmethod
    def from_string(cls, line_no, value):
        return cls(line_no, value, not value.strip())

    @classmethod
    def from_constant(cls, line_no, value):
        return cls(line_no, value, False)

    def __repr__(self):
        return "Line {}, value {}".format(self.line_no, self.value)

class TokenExpander(object):
    def __init__(self, defines):
        self.defines = defines

    def expand_tokens(self, tokens, seen=()):
        for token in tokens:
            if token.value not in self.defines or token.value in seen:
                yield token.value
            else:
                new_seen = [token.value]
                new_seen.extend(seen)
                if len(new_seen) > 20:
                    raise Exception("Stopping with stack %s" % new_seen)
                tokens = self.defines[token.value]
                for token in self.expand_tokens(tokens, new_seen):
                    yield token

class Tokenizer(object):
    def __init__(self, f_obj, line_ending):
        self.source = enumerate(f_obj)
        self.line_ending = line_ending

    def __iter__(self):
        comment = None
        for line_no, line in self.source:
            line = line.rstrip("\r\n")
            tokens = _tokenize(line)
            token = None
            for token in tokens:
                if token == "*/" and comment == "/*":
                    comment = None
                elif comment:
                    continue
                elif token in ("//", "/*"):
                    comment = token
                    yield Token.from_constant(line_no, RSTRIP)
                else:
                    yield Token.from_string(line_no, token)
            if comment == "//":
                if token != "\\":
                    comment = None
            if not comment:
                yield Token.from_string(line_no, self.line_ending)
                if token != "\\":
                    yield Token.from_constant(line_no, CHUNK_MARK)

    def read_chunks(self):
        tokens = iter(self)
        chunk = []
        whitespace_only = True
        for token in tokens:
            if token.value is RSTRIP:
                chunk = chunk[:-1]
                continue
            elif token.value is CHUNK_MARK:
                if chunk:
                    yield chunk
                chunk = []
                whitespace_only = True
                continue
            chunk.append(token)
            if token.value == "#":
                if whitespace_only:
                    chunk = [Token.from_string(token.line_no, "#")]
            elif not token.whitespace:
                whitespace_only = False

class Preprocessor(object):

    def __init__(self, line_ending=DEFAULT_LINE_ENDING, include_paths=(),
                 header_handler=None, platform_constants=PLATFORM_CONSTANTS,
                 ignore_headers=()):
        self.ignore_headers = ignore_headers
        self.include_once = {}
        self.defines = {key: [Token.from_string(None, value)]
                        for key, value in platform_constants.items()}
        self.constraints = []
        self.ignore = False
        self.line_ending = line_ending
        self.last_constraint = None
        self.header_stack = []
        self.token_expander = TokenExpander(self.defines)
        if header_handler is None:
            self.headers = HeaderHandler(include_paths)
        else:
            self.headers = header_handler
            self.headers.add_include_paths(include_paths)

    def process_define(self, **kwargs):
        if self.ignore:
            return
        chunk = kwargs["chunk"]
        for i, tokenized in enumerate(chunk):
            if not tokenized.whitespace:
                define_name = tokenized.value
                break
        self.defines[define_name] = chunk[i+2:-1]

    def process_endif(self, **kwargs):
        line_no = kwargs["line_no"]
        if not self.constraints:
            raise ParseError("Unexpected #endif on line %s" % line_no)
        (constraint_type, constraint, ignore,
         original_line_no) = self.constraints.pop()
        if ignore:
            self.ignore = False
        self.last_constraint = constraint, constraint_type, original_line_no

    def process_else(self, **kwargs):
        line_no = kwargs["line_no"]
        if not self.constraints:
            raise ParseError("Unexpected #else on line %s" % line_no)
        _, constraint, ignore, _ = self.constraints.pop()
        if self.ignore and ignore:
            ignore = False
            self.ignore = False
        elif not self.ignore and not ignore:
            ignore = True
            self.ignore = True
        self.constraints.append((ELSE, constraint, ignore, line_no))

    def process_ifdef(self, **kwargs):
        chunk = kwargs["chunk"]
        line_no = kwargs["line_no"]
        for token in chunk:
            if not token.whitespace:
                condition = token.value
                break
        if not self.ignore and condition not in self.defines:
            self.ignore = True
            self.constraints.append((IFDEF, condition, True, line_no))
        else:
            self.constraints.append((IFDEF, condition, False, line_no))

    def process_pragma(self, **kwargs):
        chunk = kwargs["chunk"]
        line_no = kwargs["line_no"]
        pragma = None
        for token in chunk:
            if not token.whitespace:
                method_name = "process_pragma_%s" % token.value
                pragma = getattr(self, method_name, None)
                break
        if pragma is None:
            raise Exception("Unsupported pragma %s on line %s" % (pragma_name,
                                                                  line_no))
        else:
            pragma(chunk=chunk, line_no=line_no)

    def process_pragma_once(self, **_):
        self.include_once[self.current_name()] = PRAGMA_ONCE

    def current_name(self):
        return self.header_stack[-1].name

    def process_ifndef(self, **kwargs):
        chunk = kwargs["chunk"]
        line_no = kwargs["line_no"]
        for token in chunk:
            if not token.whitespace:
                condition = token.value
                break
        if not self.ignore and condition in self.defines:
            self.ignore = True
            self.constraints.append((IFNDEF, condition, True, line_no))
        else:
            self.constraints.append((IFNDEF, condition, False, line_no))

    def process_undef(self, **kwargs):
        chunk = kwargs["chunk"]
        for token in chunk:
            if not token.whitespace:
                undefine = token.value
                break
        try:
            del self.defines[undefine]
        except KeyError:
            pass

    def process_source_chunks(self, chunk):
        if not self.ignore:
            for chunk in self.token_expander.expand_tokens(chunk):
                yield chunk

    def skip_file(self, name):
        item = self.include_once.get(name)
        if item is PRAGMA_ONCE:
            return True
        elif item is None:
            return False
        else:
            constraint, constraint_type = item
            if constraint_type == IFDEF:
                return constraint not in self.defines
            elif constraint_type == IFNDEF:
                return constraint in self.defines
            else:
                raise Exception("Bug, constraint type %s" % constraint_type)

    def _read_header(self, header, error, anchor_file=None):
        if header not in self.ignore_headers:
            f = self.headers.open_header(header, self.skip_file, anchor_file)
            if f is None:
                raise error
            elif f is not SKIP_FILE:
                with f:
                    for line in self.preprocess(f):
                        yield line

    def process_include(self, **kwargs):
        chunk = kwargs["chunk"]
        line_no = kwargs["line_no"]
        for token in chunk:
            if not token.whitespace:
                item = token.value
                break
        s = "Line %s includes a file %s that can't be found" % (line_no,
                                                                item)
        error = ParseError(s)
        if item.startswith("<") and item.endswith(">"):
            header = item.strip("<>")
            return self._read_header(header, error)
        elif item.startswith('"') and item.endswith('"'):
            header = item.strip('"')
            return self._read_header(header, error, self.current_name())
        else:
            fmt = "Invalid include on line %s, got %r for include name"
            raise ParseError(fmt % (line_no, item))

    def check_fullfile_guard(self):
        if self.last_constraint is None:
            return
        constraint, constraint_type, begin = self.last_constraint
        if begin != 0:
            return
        self.include_once[self.current_name()] = constraint, constraint_type

    def preprocess(self, f_object, depth=0):
        self.header_stack.append(f_object)
        tokenizer = Tokenizer(f_object, self.line_ending)
        for chunk in tokenizer.read_chunks():
            self.last_constraint = None
            if chunk[0].value == "#":
                line_no = chunk[0].line_no
                macro_name = chunk[1].value
                macro_chunk = chunk[2:]
                macro = getattr(self, "process_%s" % macro_name, None)
                if macro is None:
                    fmt = "Line number %s contains unsupported macro %s"
                    raise ParseError(fmt % (line_no, macro_name))
                ret = macro(line_no=line_no, chunk=macro_chunk)
                if ret is not None:
                    for token in ret:
                        yield token
            else:
                for token in self.process_source_chunks(chunk):
                    yield token
        self.check_fullfile_guard()
        self.header_stack.pop()
        if not self.header_stack and self.constraints:
            constraint_type, name, _, line_no = self.constraints[-1]
            if constraint_type is IFDEF:
                fmt = "#ifdef %s from line %s left open"
            elif constraint_type is IFNDEF:
                fmt = "#ifndef %s from line %s left open"
            else:
                fmt = "#else from line %s left open"
            raise ParseError(fmt % (name, line_no))

def preprocess(f_object, line_ending="\n", include_paths=(),
               header_handler=None, platform_constants=PLATFORM_CONSTANTS,
               ignore_headers=()):
    r"""
    This preprocessor yields lines with \n at the end
    """
    preprocessor = Preprocessor(line_ending, include_paths, header_handler,
                                platform_constants, ignore_headers)
    return preprocessor.preprocess(f_object)
<|endoftext|>


<commit_before>import spacy
from spacy.tokenizer import Tokenizer
from spacy.util import (
    compile_infix_regex,
    compile_prefix_regex,
    compile_suffix_regex,
)

from initialize import spacy_nlp
from interfaces.SentenceOperation import SentenceOperation
from interfaces.SentencePairOperation import SentencePairOperation
from tasks.TaskTypes import TaskType

"""
Auxiliary Negation Removal.
    Remove auxiliary negations generating a sentence with oposite meaning.
"""

def reinitialize_spacy():
    nlp = spacy_nlp if spacy_nlp else spacy.load("en_core_web_sm")
    rules = nlp.Defaults.tokenizer_exceptions
    infix_re = compile_infix_regex(nlp.Defaults.infixes)
    prefix_re = compile_prefix_regex(nlp.Defaults.prefixes)
    suffix_re = compile_suffix_regex(nlp.Defaults.suffixes)

    nlp.tokenizer = Tokenizer(
        nlp.vocab,
        rules=rules,
        prefix_search=prefix_re.search,
        suffix_search=suffix_re.search,
        infix_finditer=infix_re.finditer,
    )

    return nlp

def auxiliary_negation_removal(sentence, nlp):

    # Tokenize Sentence
    doc = nlp(sentence)

    # Initialize Variables
    changed = False
    new_sentence = []
    supported_auxiliaries = [
        "am",
        "are",
        "can",
        "could",
        "had",
        "has",
        "have",
        "is",
        "may",
        "might",
        "must",
        "shall",
        "should",
        "was",
        "were",
        "will",
        "would",
    ]

    # Evaluate Tokens
    for token in doc:
        # Add Token to Output Sentence
        new_sentence.append(token)

        # Initialize Variables
        token_lowercased_lemma = token.lemma_.lower()

        # Process Negations
        if token_lowercased_lemma == "not" or token_lowercased_lemma == "n't":
            # Get not position
            not_index = token.i

            # Process Auxiliaries
            if not_index > 0:

                # Get Previous Token
                previous_index = not_index - 1
                previous_token = doc[previous_index]
                previous_surface = previous_token.text
                previous_lowercase_surface = previous_surface.lower()

                # Remove Negation
                if previous_lowercase_surface in supported_auxiliaries:
                    new_sentence = new_sentence[:-1]
                    changed = True

                elif previous_lowercase_surface in ["do"]:
                    new_sentence = new_sentence[:-2]
                    changed = True

                # Handle Spacing
                if (
                    token_lowercased_lemma == "n't"
                    and changed
                    and new_sentence
                ):
                    new_sentence[-1] = nlp(
                        new_sentence[-1].text + token.whitespace_
                    )[0]

    # Rebuild Sentence
    new_sentence = [t.text + t.whitespace_ for t in new_sentence]
    new_sentence = "".join(new_sentence)

    return (new_sentence, changed)

class SentenceAuxiliaryNegationRemoval(SentenceOperation):
    tasks = [
        TaskType.TEXT_CLASSIFICATION,
        TaskType.TEXT_TO_TEXT_GENERATION,
        TaskType.QUALITY_ESTIMATION,
    ]
    languages = ["en"]
    keywords = [
        "morphological",
        "lexical",
        "rule-based",
        "tokenizer-required",
        "meaning-alteration",
        "high-precision",
        "low-coverage",
        "low-generations",
        "causal-reasoning",
    ]

    def __init__(self, seed=0, max_outputs=1):
        super().__init__(seed, max_outputs=max_outputs)
        self.nlp = reinitialize_spacy()

    def generate(self, sentence: str):

        # Initialize Variables
        output_sentence = sentence

        # Process sentence
        new_sentence, changed = auxiliary_negation_removal(sentence, self.nlp)

        if changed:
            output_sentence = new_sentence

        return [output_sentence]

class PairAuxiliaryNegationRemoval(SentencePairOperation):
    tasks = [TaskType.PARAPHRASE_DETECTION]
    languages = ["en"]
    keywords = [
        "morphological",
        "lexical",
        "rule-based",
        "tokenizer-required",
        "meaning-alteration",
        "high-precision",
        "low-coverage",
        "low-generations",
        "causal-reasoning",
    ]

    def __init__(self, seed=0, max_outputs=3, pos_label="1", neg_label="0"):
        super().__init__(seed, max_outputs=max_outputs)
        self.nlp = reinitialize_spacy()
        self.pos_label = pos_label
        self.neg_label = neg_label

    def generate(self, sentence1: str, sentence2: str, target: str):

        # Initialize Variables
        output_sentences = []
        changed_sentences = {}

        # Only process equivalent pairs
        if target == self.pos_label:

            for n, sentence in enumerate([sentence1, sentence2]):
                # Process sentence
                new_sentence, changed = auxiliary_negation_removal(
                    sentence, self.nlp
                )
                if changed:
                    changed_sentences[n] = new_sentence

        if 0 in changed_sentences.keys():
            output_sentences.append(
                (changed_sentences[0], sentence2, self.neg_label)
            )

        if 1 in changed_sentences.keys():
            output_sentences.append(
                (sentence1, changed_sentences[1], self.neg_label)
            )

        if 0 in changed_sentences.keys() and 1 in changed_sentences.keys():
            output_sentences.append(
                (changed_sentences[0], changed_sentences[1], self.pos_label)
            )

        if not output_sentences:
            output_sentences = [(sentence1, sentence2, target)]

        return output_sentences

"""
# Sample code to demonstrate usage. Can also assist in adding test cases.
# You don't need to keep this code in your transformation.
if __name__ == '__main__':
    import json
    from TestRunner import convert_to_snake_case
    tf = SentenceAuxiliaryNegationRemoval()
    test_cases = []
    for sentence["Andrew has not returned the French book to the library.",
                 "Sentences with gapping, such as Paul likes coffee and Mary tea, do not have an overt predicate.",
                 "Alice in Wonderland isn't a 1997 American live-action/animated dark fantasy adventure film.",
                 "Ujjal Dev Dosanjh was not the 1st Premier of British Columbia from 1871 to 1872.",
                 "The fighters would not give up."]:
        test_cases.append({
            "class": tf.name(),
            "inputs": {"sentence": sentence},
            "outputs": [{"sentence": o[0]} for o in tf.generate(sentence)]}
        )
    json_file = {"type": convert_to_snake_case(tf.name()), "test_cases": test_cases}
    print(json.dumps(json_file))
    tf = PairAuxiliaryNegationRemoval(max_outputs=3)
    test_cases = []
    for sentence1, sentence2, target in zip(["Andrew has not returned the French book to the library.",
                                     "Sentences with gapping, such as Paul likes coffee and Mary tea, do not have an overt predicate.",
                                     "Alice in Wonderland isn't a 1997 American live-action/animated dark fantasy adventure film.",
                                     "Ujjal Dev Dosanjh was not the 1st Premier of British Columbia from 1871 to 1872.",
                                     "The fighters would not give up."],
                                    ["He hasn't brought back the library's books.",
                                     "Gapping sentences, such as Paul likes coffee and Mary tea, lack an overt predicate.",
                                     "Alice in Wonderland is not an American animated, dark fantasy adventure film from 1997.",
                                     "U.D. Dosanjh wasn't the 1st Premier of British Columbia for a year from 1871.",
                                     "The warriors wouldn't leave the battlefield."],
                                     ["1",
                                      "1",
                                      "1",
                                      "1",
                                      "1"
                                     ]
                                    ):
        test_cases.append({
            "class": tf.name(),
            "inputs": {"sentence1": sentence1, "sentence2": sentence2, "target": target},
            "outputs": [{"sentence1": o[0], "sentence2": o[1], "target": o[2]} for o in tf.generate(sentence1, sentence2, target)]}
        )
    json_file = {"type": convert_to_snake_case(tf.name()), "test_cases": test_cases}
    print(json.dumps(json_file))
"""
<commit_msg>Moved spacy reinitialization to generate() function<commit_after>import spacy
from spacy.tokenizer import Tokenizer
from spacy.util import (
    compile_infix_regex,
    compile_prefix_regex,
    compile_suffix_regex,
)

from initialize import spacy_nlp
from interfaces.SentenceOperation import SentenceOperation
from interfaces.SentencePairOperation import SentencePairOperation
from tasks.TaskTypes import TaskType

"""
Auxiliary Negation Removal.
    Remove auxiliary negations generating a sentence with oposite meaning.
"""

def reinitialize_spacy():
    nlp = spacy_nlp if spacy_nlp else spacy.load("en_core_web_sm")
    rules = nlp.Defaults.tokenizer_exceptions
    infix_re = compile_infix_regex(nlp.Defaults.infixes)
    prefix_re = compile_prefix_regex(nlp.Defaults.prefixes)
    suffix_re = compile_suffix_regex(nlp.Defaults.suffixes)

    nlp.tokenizer = Tokenizer(
        nlp.vocab,
        rules=rules,
        prefix_search=prefix_re.search,
        suffix_search=suffix_re.search,
        infix_finditer=infix_re.finditer,
    )

    return nlp

def auxiliary_negation_removal(sentence, nlp):

    # Tokenize Sentence
    doc = nlp(sentence)

    # Initialize Variables
    changed = False
    new_sentence = []
    supported_auxiliaries = [
        "am",
        "are",
        "can",
        "could",
        "had",
        "has",
        "have",
        "is",
        "may",
        "might",
        "must",
        "shall",
        "should",
        "was",
        "were",
        "will",
        "would",
    ]

    # Evaluate Tokens
    for token in doc:
        # Add Token to Output Sentence
        new_sentence.append(token)

        # Initialize Variables
        token_lowercased_lemma = token.lemma_.lower()

        # Process Negations
        if token_lowercased_lemma == "not" or token_lowercased_lemma == "n't":
            # Get not position
            not_index = token.i

            # Process Auxiliaries
            if not_index > 0:

                # Get Previous Token
                previous_index = not_index - 1
                previous_token = doc[previous_index]
                previous_surface = previous_token.text
                previous_lowercase_surface = previous_surface.lower()

                # Remove Negation
                if previous_lowercase_surface in supported_auxiliaries:
                    new_sentence = new_sentence[:-1]
                    changed = True

                elif previous_lowercase_surface in ["do"]:
                    new_sentence = new_sentence[:-2]
                    changed = True

                # Handle Spacing
                if (
                    token_lowercased_lemma == "n't"
                    and changed
                    and new_sentence
                ):
                    new_sentence[-1] = nlp(
                        new_sentence[-1].text + token.whitespace_
                    )[0]

    # Rebuild Sentence
    new_sentence = [t.text + t.whitespace_ for t in new_sentence]
    new_sentence = "".join(new_sentence)

    return (new_sentence, changed)

class SentenceAuxiliaryNegationRemoval(SentenceOperation):
    tasks = [
        TaskType.TEXT_CLASSIFICATION,
        TaskType.TEXT_TO_TEXT_GENERATION,
        TaskType.QUALITY_ESTIMATION,
    ]
    languages = ["en"]
    keywords = [
        "morphological",
        "lexical",
        "rule-based",
        "tokenizer-required",
        "meaning-alteration",
        "high-precision",
        "low-coverage",
        "low-generations",
        "causal-reasoning",
    ]

    def __init__(self, seed=0, max_outputs=1):
        super().__init__(seed, max_outputs=max_outputs)
        self.nlp = spacy_nlp if spacy_nlp else spacy.load("en_core_web_sm")

    def generate(self, sentence: str):

        # Reinitialize spacy
        self.nlp = reinitialize_spacy()

        # Initialize Variables
        output_sentence = sentence

        # Process sentence
        new_sentence, changed = auxiliary_negation_removal(sentence, self.nlp)

        if changed:
            output_sentence = new_sentence

        return [output_sentence]

class PairAuxiliaryNegationRemoval(SentencePairOperation):
    tasks = [TaskType.PARAPHRASE_DETECTION]
    languages = ["en"]
    keywords = [
        "morphological",
        "lexical",
        "rule-based",
        "tokenizer-required",
        "meaning-alteration",
        "high-precision",
        "low-coverage",
        "low-generations",
        "causal-reasoning",
    ]

    def __init__(self, seed=0, max_outputs=3, pos_label="1", neg_label="0"):
        super().__init__(seed, max_outputs=max_outputs)
        self.nlp = spacy_nlp if spacy_nlp else spacy.load("en_core_web_sm")
        self.pos_label = pos_label
        self.neg_label = neg_label

    def generate(self, sentence1: str, sentence2: str, target: str):

        # Reinitialize spacy
        self.nlp = reinitialize_spacy()

        # Initialize Variables
        output_sentences = []
        changed_sentences = {}

        # Only process equivalent pairs
        if target == self.pos_label:

            for n, sentence in enumerate([sentence1, sentence2]):
                # Process sentence
                new_sentence, changed = auxiliary_negation_removal(
                    sentence, self.nlp
                )
                if changed:
                    changed_sentences[n] = new_sentence

        if 0 in changed_sentences.keys():
            output_sentences.append(
                (changed_sentences[0], sentence2, self.neg_label)
            )

        if 1 in changed_sentences.keys():
            output_sentences.append(
                (sentence1, changed_sentences[1], self.neg_label)
            )

        if 0 in changed_sentences.keys() and 1 in changed_sentences.keys():
            output_sentences.append(
                (changed_sentences[0], changed_sentences[1], self.pos_label)
            )

        if not output_sentences:
            output_sentences = [(sentence1, sentence2, target)]

        return output_sentences

"""
# Sample code to demonstrate usage. Can also assist in adding test cases.
# You don't need to keep this code in your transformation.
if __name__ == '__main__':
    import json
    from TestRunner import convert_to_snake_case
    tf = SentenceAuxiliaryNegationRemoval()
    test_cases = []
    for sentence["Andrew has not returned the French book to the library.",
                 "Sentences with gapping, such as Paul likes coffee and Mary tea, do not have an overt predicate.",
                 "Alice in Wonderland isn't a 1997 American live-action/animated dark fantasy adventure film.",
                 "Ujjal Dev Dosanjh was not the 1st Premier of British Columbia from 1871 to 1872.",
                 "The fighters would not give up."]:
        test_cases.append({
            "class": tf.name(),
            "inputs": {"sentence": sentence},
            "outputs": [{"sentence": o[0]} for o in tf.generate(sentence)]}
        )
    json_file = {"type": convert_to_snake_case(tf.name()), "test_cases": test_cases}
    print(json.dumps(json_file))
    tf = PairAuxiliaryNegationRemoval(max_outputs=3)
    test_cases = []
    for sentence1, sentence2, target in zip(["Andrew has not returned the French book to the library.",
                                     "Sentences with gapping, such as Paul likes coffee and Mary tea, do not have an overt predicate.",
                                     "Alice in Wonderland isn't a 1997 American live-action/animated dark fantasy adventure film.",
                                     "Ujjal Dev Dosanjh was not the 1st Premier of British Columbia from 1871 to 1872.",
                                     "The fighters would not give up."],
                                    ["He hasn't brought back the library's books.",
                                     "Gapping sentences, such as Paul likes coffee and Mary tea, lack an overt predicate.",
                                     "Alice in Wonderland is not an American animated, dark fantasy adventure film from 1997.",
                                     "U.D. Dosanjh wasn't the 1st Premier of British Columbia for a year from 1871.",
                                     "The warriors wouldn't leave the battlefield."],
                                     ["1",
                                      "1",
                                      "1",
                                      "1",
                                      "1"
                                     ]
                                    ):
        test_cases.append({
            "class": tf.name(),
            "inputs": {"sentence1": sentence1, "sentence2": sentence2, "target": target},
            "outputs": [{"sentence1": o[0], "sentence2": o[1], "target": o[2]} for o in tf.generate(sentence1, sentence2, target)]}
        )
    json_file = {"type": convert_to_snake_case(tf.name()), "test_cases": test_cases}
    print(json.dumps(json_file))
"""
<|endoftext|>