<commit_before>"""
file: ti_encode.py
language: python3
author: Nate Levesque <public@thenaterhood.com>
description: Compiles plaintext to TI-Basic files and saves the file.

TODO:
    complete overhaul of how the file is read so that comments and
    text stop being a problem.

    Syntax checking.  Dream big, right?  Few basics would be nice, shouln't
    be too hard

    validation to make sure the file put in is actually a text file.  Doesn't
    break things if it isn't but it looks nasty

"""
import binascii
from copy import deepcopy
import dictionaries
from tiFile import tiFile
import common
from metadata import createMeta

def init():
    print("TI BASIC file encoder.  Compiles text to TIBASIC.\n")
    print("Nate Levesque <public@thenaterhood.com>.")
    print("Visit www.thenaterhood.com/projects for more software\n")

def readFile(filename):
    """
    Reads a file into an array as chunks separated by spaces.

    Arguments:
        filename (string): the name of the file to open
    Returns:
        fileContents (list): byte values from the file

    """
    fileContents = []

    # Opens the file and reads it one line at a time into an array
    for line in open(filename, "r"):
        fileContents.append(line)
    return fileContents

def translate(dictionary, content, escapeCharExists, escapeChar):
    """
    Takes a dictionary and a list of items (such as bytes in a file)
    and checks each byte to see if it's a key in the dictionary.  If it
    is, it replaces the item with the key's value.  Supports having
    preceding escape characters.

    Arguments:
        dictionary (dict): a dictionary mapping byte values to values
        content (list): a list containing byte values
        escapeCharExists (boolean): whether or not to look for an escape
            character when replacing values
        escapeChar (string): an escape character to look for. Has no
            effect if escapeCharExists is valse

    Returns:
        translation (list): a copy of the content list with every
            recognized value replaced
    """

    # Make a copy of the input so the original isn't modified
    #translation = deepcopy(content)
    translation = []

    # Iterate through each index of the copy of the content
    for item in content:
        # If there is no escape character set, check if the item
        # exists in the dictionary and replace it with its value if it does
        if ( not escapeCharExists) and (item in dictionary):
            translation.append(dictionary[item])

        # If an escape character is set and the character at the current
        # iteration was found in the dictionary, switch out the item for
        # its compiled byte value and insert the escape character into
        # the preceding index.
        if (escapeCharExists and item in dictionary):
            translation.append(dictionary[item])
            translation.insert(-1, escapeChar)

        elif (item not in dictionary):
            translation.append(item)

    return translation

def parseASCII(fileContents):
    """
    Parses the TI83 hex codes for standard ascii characters and replaces
    them with the ascii equivalent.

    Arguments:
        fileContents (list): a list containing byte values read from
            a file
    Returns:
        a list containing the file with ascii characters in the place
        of byte values that were interpreted and converted
    """
    # make a deepcopy of the list so the original isn't modified
    ascii_parsed = deepcopy(fileContents)

    # Grab the dictionary with values for ascii uppercase and numbers
    # and reverse the key/value pairs since the dictionary is written
    # for decompiling
    ascii_dict = dictionaries.standardASCII(True)

    # Call translate with the uppercase dictionary, the file contents
    # and no escape code set.
    ascii_parsed = translate(ascii_dict, ascii_parsed, False, '')

    # Grab the dictionary with values for the lowercase letters
    # and reverse the key/value pairs since the dictionary is written
    # for decompiling
    ascii_lower_dict = dictionaries.lowercaseASCII(True)

    # Call translate with the lowercase dictionary, the file contents,
    # and the escape character b'\xbb' set
    ascii_parsed = translate(ascii_lower_dict, ascii_parsed, True, b'\xbb')

    # Grab the dictionary mapping symbols to their plaintext values and
    # reverse the key/values since the dictionary is written for decompiling.
    ascii_symbol_dict = dictionaries.symbolsASCII(True)

    # Call translate with the ascii symbol dictionary, the file contents,
    # and no escape character set
    ascii_parsed = translate(ascii_symbol_dict, ascii_parsed, False, '')

    # Convert spaces
    whitespace_dict = dictionaries.whitespace(True)
    ascii_parsed = translate(whitespace_dict, ascii_parsed, False, '')

    return ascii_parsed + [b'?']

def parseWhitespace(fileContents):
    """
    Parses the TI83 whitespace codes

    Arguments:
        fileContents (list): a list of the byte values read from the file
    Returns:
        a list of byte values with the whitespace codes replaced with
        their ascii equivalents
    """

    parsedFile = []
    """
    try:
        for i in range(0, len(fileContents)):
            if (fileContents[i][0] == ":"):
                parsedFile.append(b'?')
                parsedFile.append(fileContents[i][1:])

            if (fileContents[i][0] != ":"):
                parsedFile.append(fileContents[i])
    except:
        pass
    """
    # Commented, because doing this doesn't actually hold any benefit
    # with how the file is currently dealt with

    whitespace_dict = dictionaries.whitespace(True)
    return translate(whitespace_dict, fileContents, False, '')
    #return parsedFile

def parseText(line):
    """
    Parses a line of TI-Basic code from plaintext -> tiBasic.
    Observes text and comments, so such things are now safe to compile.
    Runs through and parses comments and text, followed by newlines,
    followed by TI-Basic tokens.

    Arguments:
        line (str): a line of plaintext tibasic code
    Returns:
        parsedLine (list): a list of bytes from the parsed line
    """
    unParsed = deepcopy(line)[1:]

    if (len(unParsed) == 0):
        return ''

    for i in range(0, len(unParsed)):
        if (unParsed[i] == '"'):
            j = i+1
            while j < len(unParsed) and unParsed[j] != '"':
                j+=1

            parsed = [ unParsed[:i] ] + parseWhitespace( parseASCII(unParsed[i:j]) ) + [ unParsed[j:] ]
            return parsed

    return [line, b'?']

def parseFunction(fileContents):
    """
    Converts the TI83 byte values for TI-BASIC functions to plaintext

    Arguments:
        fileContents (list): a list of byte values read from the file
    Returns:
        a list with the byte values for functions replaced with their
        plaintext equivalents
    """

    # Grab the dictionary mapping function hex codes to their plaintext
    # values and reverse the key/values since the dictionary is written
    # for decompiling.
    function_dict = dictionaries.tibasicFunctions(True)

    parsedFunction = []
    for item in fileContents:
        item = item.partition('(')
        item = item[0] + item[1]
        if ( item in function_dict ):
            parsedFunction = parsedFunction + [ function_dict[item] ]

    return parsedFunction
    """

    # calls the translate function with the function dictionary,
    # contents of the file, and no escape character set
    parsed = translate(function_dict, fileContents, False, '')
    # Because of issues with spaces, now stripping spaces from the
    # code and trying again.
    stripped_dict = dict()
    for key in function_dict:
        stripped_dict[key.strip()] = function_dict[key]

    parsed = translate(stripped_dict, parsed, False, '')

    return parsed
    """

def splitBytes(contents):
    """
    Splits a list of items up into byte-sized chunks.  They'd probably
    taste nice on salad.

    Arguments:
        contents (list): a list of strings
    Returns:
        splitBytes (list): a list of bytes
    """
    splitBytes = []

    for item in contents:
        if (isinstance(item, str)):
            for byte in item:
                splitBytes.append(byte)
        if (not isinstance(item, str)):
            splitBytes.append(item)

    return splitBytes

def main():
    """
    Calls the functions in order to decode the .8Xp file.  Order
    DOES matter here for each parsing function or garbage results

    Arguments:
        none
    Returns:
        none
    """
    init()

    # Request a filename from the user
    filename = common.getFName()
    tiData = tiFile()

    # Read the file
    fileContents = readFile(filename)

    # Create an array to store the compiled data
    code = [ b'?' ] * len(fileContents)

    # Split the lines up over spaces to isolate functions
    splitLines = []
    for line in fileContents:
        splitLines.append( line[1:].split() )

    parsedLines = []
    # Parse any code comments and empty newlines
    i = 0
    while i < len(splitLines):
        try:
            if ( splitLines[i][0][0] == '"' ):
                code[i] = parseASCII( ' '.join( splitLines[i] ) )
                parsedLines.append(i)
        except:
            parsedLines.append(i)

        i+=1

    i = 0
    while i < len(splitLines):
        if ( i not in parsedLines ):
            code[i] = parseFunction( splitLines[i] ) + [b'?']
            parsedLines.append(i)
        i+=1

    tiData.prgmdata = []
    for item in code:
        try:
            tiData.prgmdata = tiData.prgmdata + item
        except:
            tiData.prgmdata = tiData.prgmdata + [item]
    print(tiData.prgmdata)

    # Parses any code comments
    #parsed = []
    #for line in fileContents:
    #    if ( isinstance(line, list) or isinstance(line, str) ):
    #        parsed = parsed + parseText(line)
    #    else:
    #        parsed = parsed + [line]

    #parsed = []

    #for line in fileContents:
    #    if ( isinstance(line, list) or isinstance(line, str) ):
    #        parsed = parsed + [parseFunction(line)]
    #    else:
    #        parsed = parsed + [line]

    # Splits the remainder into pieces and parses it like before.
    #split = []
    #for i in range(0,len(parsed)):
    #    if (isinstance(parsed[i], str)):
    #        split = split + parsed[i].strip(':').split()
    #    else:
    #        split.append(parsed[i])
    #parsed = split

    # Parse the file.  Again, order matters here
    #tiData.prgmdata = parseWhitespace(parsed)
    #tiData.prgmdata = parseFunction(parsed)
    #tiData.prgmdata = splitBytes(parsed)
    #tiData.prgmdata = parseASCII(tiData.prgmdata)
    #tiData.prgmdata = parseWhitespace(tiData.prgmdata)

    # Break the name of the program off the filename
    name = (filename.split('.')[0])

    # Add the metadata and a footer of null bytes to the tidata object
    tiData.metadata = createMeta(tiData.prgmdata, name)
    tiData.footer = [b'\x00', b'\x00']

    # Write the tile to disk
    tiData.write(name)

    print("Written to file: ")
    print(tiData)
    print("Saved to " +name +'.8Xp')

# Call the main method
main()
<commit_msg>now successfully parses inside parenthasis<commit_after>"""
file: ti_encode.py
language: python3
author: Nate Levesque <public@thenaterhood.com>
description: Compiles plaintext to TI-Basic files and saves the file.

TODO:
    complete overhaul of how the file is read so that comments and
    text stop being a problem.

    Syntax checking.  Dream big, right?  Few basics would be nice, shouln't
    be too hard

    validation to make sure the file put in is actually a text file.  Doesn't
    break things if it isn't but it looks nasty

"""
import binascii
from copy import deepcopy
import dictionaries
from tiFile import tiFile
import common
from metadata import createMeta

def init():
    print("TI BASIC file encoder.  Compiles text to TIBASIC.\n")
    print("Nate Levesque <public@thenaterhood.com>.")
    print("Visit www.thenaterhood.com/projects for more software\n")

def readFile(filename):
    """
    Reads a file into an array as chunks separated by spaces.

    Arguments:
        filename (string): the name of the file to open
    Returns:
        fileContents (list): byte values from the file

    """
    fileContents = []

    # Opens the file and reads it one line at a time into an array
    for line in open(filename, "r"):
        fileContents.append(line)
    return fileContents

def translate(dictionary, content, escapeCharExists, escapeChar):
    """
    Takes a dictionary and a list of items (such as bytes in a file)
    and checks each byte to see if it's a key in the dictionary.  If it
    is, it replaces the item with the key's value.  Supports having
    preceding escape characters.

    Arguments:
        dictionary (dict): a dictionary mapping byte values to values
        content (list): a list containing byte values
        escapeCharExists (boolean): whether or not to look for an escape
            character when replacing values
        escapeChar (string): an escape character to look for. Has no
            effect if escapeCharExists is valse

    Returns:
        translation (list): a copy of the content list with every
            recognized value replaced
    """

    # Make a copy of the input so the original isn't modified
    #translation = deepcopy(content)
    translation = []

    # Iterate through each index of the copy of the content
    for item in content:
        # If there is no escape character set, check if the item
        # exists in the dictionary and replace it with its value if it does
        if ( not escapeCharExists) and (item in dictionary):
            translation.append(dictionary[item])

        # If an escape character is set and the character at the current
        # iteration was found in the dictionary, switch out the item for
        # its compiled byte value and insert the escape character into
        # the preceding index.
        if (escapeCharExists and item in dictionary):
            translation.append(dictionary[item])
            translation.insert(-1, escapeChar)

        elif (item not in dictionary):
            translation.append(item)

    return translation

def parseASCII(fileContents):
    """
    Parses the TI83 hex codes for standard ascii characters and replaces
    them with the ascii equivalent.

    Arguments:
        fileContents (list): a list containing byte values read from
            a file
    Returns:
        a list containing the file with ascii characters in the place
        of byte values that were interpreted and converted
    """

    if ( len(fileContents) == 0):
        return [ fileContents ]

    # make a deepcopy of the list so the original isn't modified
    ascii_parsed = deepcopy(fileContents)

    # Grab the dictionary with values for ascii uppercase and numbers
    # and reverse the key/value pairs since the dictionary is written
    # for decompiling
    ascii_dict = dictionaries.standardASCII(True)

    # Call translate with the uppercase dictionary, the file contents
    # and no escape code set.
    ascii_parsed = translate(ascii_dict, ascii_parsed, False, '')

    # Grab the dictionary with values for the lowercase letters
    # and reverse the key/value pairs since the dictionary is written
    # for decompiling
    ascii_lower_dict = dictionaries.lowercaseASCII(True)

    # Call translate with the lowercase dictionary, the file contents,
    # and the escape character b'\xbb' set
    ascii_parsed = translate(ascii_lower_dict, ascii_parsed, True, b'\xbb')

    # Grab the dictionary mapping symbols to their plaintext values and
    # reverse the key/values since the dictionary is written for decompiling.
    ascii_symbol_dict = dictionaries.symbolsASCII(True)

    # Call translate with the ascii symbol dictionary, the file contents,
    # and no escape character set
    ascii_parsed = translate(ascii_symbol_dict, ascii_parsed, False, '')

    # Convert spaces
    whitespace_dict = dictionaries.whitespace(True)
    ascii_parsed = translate(whitespace_dict, ascii_parsed, False, '')

    if fileContents[0] == '"':
        return ascii_parsed + [ b'?' ]
    else:
        return ascii_parsed

def parseWhitespace(fileContents):
    """
    Parses the TI83 whitespace codes

    Arguments:
        fileContents (list): a list of the byte values read from the file
    Returns:
        a list of byte values with the whitespace codes replaced with
        their ascii equivalents
    """

    parsedFile = []
    """
    try:
        for i in range(0, len(fileContents)):
            if (fileContents[i][0] == ":"):
                parsedFile.append(b'?')
                parsedFile.append(fileContents[i][1:])

            if (fileContents[i][0] != ":"):
                parsedFile.append(fileContents[i])
    except:
        pass
    """
    # Commented, because doing this doesn't actually hold any benefit
    # with how the file is currently dealt with

    whitespace_dict = dictionaries.whitespace(True)
    return translate(whitespace_dict, fileContents, False, '')
    #return parsedFile

def parseText(line):
    """
    Parses a line of TI-Basic code from plaintext -> tiBasic.
    Observes text and comments, so such things are now safe to compile.
    Runs through and parses comments and text, followed by newlines,
    followed by TI-Basic tokens.

    Arguments:
        line (str): a line of plaintext tibasic code
    Returns:
        parsedLine (list): a list of bytes from the parsed line
    """
    unParsed = deepcopy(line)[1:]

    if (len(unParsed) == 0):
        return ''

    for i in range(0, len(unParsed)):
        if (unParsed[i] == '"'):
            j = i+1
            while j < len(unParsed) and unParsed[j] != '"':
                j+=1

            parsed = [ unParsed[:i] ] + parseWhitespace( parseASCII(unParsed[i:j]) ) + [ unParsed[j:] ]
            return parsed

    return [line, b'?']

def parseFunction(fileContents):
    """
    Converts the TI83 byte values for TI-BASIC functions to plaintext

    Arguments:
        fileContents (list): a list of byte values read from the file
    Returns:
        a list with the byte values for functions replaced with their
        plaintext equivalents
    """

    # Grab the dictionary mapping function hex codes to their plaintext
    # values and reverse the key/values since the dictionary is written
    # for decompiling.
    function_dict = dictionaries.tibasicFunctions(True)

    parsedFunction = []
    for item in fileContents:
        if ( '(' in item ):
            parts = item.partition('(')
            item = parts[0] + parts[1]
            arg = parts[2]
            if ( item in function_dict ):
                parsedFunction = parsedFunction + [ function_dict[item] ] + parseASCII(arg)
        else:
            parts = item.partition(' ')
            item = parts[0]
            arg = parts[2]

            if ( item in function_dict ):
                parsedFunction = parsedFunction + [ function_dict[item] ] + parseASCII(arg)

    return parsedFunction
    """

    # calls the translate function with the function dictionary,
    # contents of the file, and no escape character set
    parsed = translate(function_dict, fileContents, False, '')
    # Because of issues with spaces, now stripping spaces from the
    # code and trying again.
    stripped_dict = dict()
    for key in function_dict:
        stripped_dict[key.strip()] = function_dict[key]

    parsed = translate(stripped_dict, parsed, False, '')

    return parsed
    """

def splitBytes(contents):
    """
    Splits a list of items up into byte-sized chunks.  They'd probably
    taste nice on salad.

    Arguments:
        contents (list): a list of strings
    Returns:
        splitBytes (list): a list of bytes
    """
    splitBytes = []

    for item in contents:
        if (isinstance(item, str)):
            for byte in item:
                splitBytes.append(byte)
        if (not isinstance(item, str)):
            splitBytes.append(item)

    return splitBytes

def main():
    """
    Calls the functions in order to decode the .8Xp file.  Order
    DOES matter here for each parsing function or garbage results

    Arguments:
        none
    Returns:
        none
    """
    init()

    # Request a filename from the user
    filename = common.getFName()
    tiData = tiFile()

    # Read the file
    fileContents = readFile(filename)

    # Create an array to store the compiled data
    code = [ None ] * len(fileContents)

    # Split the lines up over spaces to isolate functions
    splitLines = []
    for line in fileContents:
        splitLines.append( line[1:].split() )

    parsedLines = []
    # Parse any code comments and empty newlines
    i = 0
    while i < len(splitLines):
        try:
            if ( splitLines[i][0][0] == '"' ):
                code[i] = parseASCII( ' '.join( splitLines[i] ) )
                parsedLines.append(i)
        except:
            code[i] = b'?'
            parsedLines.append(i)

        i+=1

    i = 0
    while i < len(splitLines):
        if ( i not in parsedLines ):
            code[i] = parseFunction( splitLines[i] ) + [b'?']
            parsedLines.append(i)
        i+=1

    tiData.prgmdata = []
    for item in code:
        try:
            tiData.prgmdata = tiData.prgmdata + item
        except:
            tiData.prgmdata = tiData.prgmdata + [item]
    print(tiData.prgmdata)

    # Parses any code comments
    #parsed = []
    #for line in fileContents:
    #    if ( isinstance(line, list) or isinstance(line, str) ):
    #        parsed = parsed + parseText(line)
    #    else:
    #        parsed = parsed + [line]

    #parsed = []

    #for line in fileContents:
    #    if ( isinstance(line, list) or isinstance(line, str) ):
    #        parsed = parsed + [parseFunction(line)]
    #    else:
    #        parsed = parsed + [line]

    # Splits the remainder into pieces and parses it like before.
    #split = []
    #for i in range(0,len(parsed)):
    #    if (isinstance(parsed[i], str)):
    #        split = split + parsed[i].strip(':').split()
    #    else:
    #        split.append(parsed[i])
    #parsed = split

    # Parse the file.  Again, order matters here
    #tiData.prgmdata = parseWhitespace(parsed)
    #tiData.prgmdata = parseFunction(parsed)
    #tiData.prgmdata = splitBytes(parsed)
    #tiData.prgmdata = parseASCII(tiData.prgmdata)
    #tiData.prgmdata = parseWhitespace(tiData.prgmdata)

    # Break the name of the program off the filename
    name = (filename.split('.')[0])

    # Add the metadata and a footer of null bytes to the tidata object
    tiData.metadata = createMeta(tiData.prgmdata, name)
    tiData.footer = [b'\x00', b'\x00']

    # Write the tile to disk
    tiData.write(name)

    print("Written to file: ")
    print(tiData)
    print("Saved to " +name +'.8Xp')

# Call the main method
main()
<|endoftext|>


<commit_before>"""IO with fif files containing events
"""

# Authors: Alexandre Gramfort <gramfort@nmr.mgh.harvard.edu>
#          Matti Hamalainen <msh@nmr.mgh.harvard.edu>
#
# License: BSD (3-clause)

import numpy as np
from os.path import splitext

import logging
logger = logging.getLogger('mne')

from .fiff.constants import FIFF
from .fiff.tree import dir_tree_find
from .fiff.tag import read_tag
from .fiff.open import fiff_open
from .fiff.write import write_int, start_block, start_file, end_block, end_file
from .fiff.pick import pick_channels
from . import verbose

def pick_events(events, include=None, exclude=None):
    """Select some events

    Parameters
    ----------
    include : int | list | None
        A event id to include or a list of them.
        If None all events are included.
    exclude : int | list | None
        A event id to exclude or a list of them.
        If None no event is excluded. If include is not None
        the exclude parameter is ignored.

    Returns
    -------
    events : array, shape (n_events, 3)
        The list of events
    """
    if include is not None:
        if not isinstance(include, list):
            include = [include]
        mask = np.zeros(len(events), dtype=np.bool)
        for e in include:
            mask = np.logical_or(mask, events[:, 2] == e)
        events = events[mask]
    elif exclude is not None:
        if not isinstance(exclude, list):
            exclude = [exclude]
        mask = np.ones(len(events), dtype=np.bool)
        for e in exclude:
            mask = np.logical_and(mask, events[:, 2] != e)
        events = events[mask]
    else:
        events = np.copy(events)

    if len(events) == 0:
        raise RuntimeError("No events found")

    return events

def define_target_events(events, reference_id, target_id, sfreq, tmin, tmax,
            new_id=None, fill_na=None):
    """ Define new events by co-occurrence of existing events

    This function can be used to evaluate events depending on the
    temporal lag to another event. For example, this can be used to
    analyze evoked responses which were followed by a button press wihtin
    a defined time window.

    Parameters
    ----------

    events : ndarray
        Array as returned by mne.find_events.
    reference_id : int
        The reference event. The event defining the epoch of interest.
    target_id : int
        The target event. The event co-occurring in within a certain time
        window around the reference event.
    sfreq : float
        The sampling frequency of the data.
    tmin : float
        The lower limit in seconds from the target event.
    tmax : float
        The upper limit border in seconds from the target event.
    new_id : int
        new_id for the new event
    fill_na : int | None
        Fill event to be inserted if target is not available within the time
        window specified. If None, the 'null' events will be dropped.

    Returns
    -------
    new_events : ndarray
        The new defined events
    lag : ndarray
        time lag between reference and target in milliseconds.
    """

    if new_id is None:
        new_id = reference_id

    tsample = 1e3 / sfreq
    imin = int(tmin * sfreq)
    imax = int(tmax * sfreq)

    new_events = []
    lag = []
    for event in events.copy().astype('f8'):
        if event[2] == reference_id:
            lower = event[0] + imin
            upper = event[0] + imax
            res = events[(events[:, 0] > lower) &
                    (events[:, 0] < upper) & (events[:, 2] == target_id)]
            if res.any():
                lag += [event[0] - res[0][0]]
                event[2] = new_id
                new_events += [event]
            elif fill_na is not None:
                event[2] = fill_na
                new_events += [event]
                lag += [fill_na]

    new_events = np.array(new_events)

    lag = np.abs(lag, dtype='f8')
    if lag.any():
        lag[lag != fill_na] *= tsample
    else:
        lag = np.array([])

    return new_events if new_events.any() else np.array([]), lag

def _read_events_fif(fid, tree):
    #   Find the desired block
    events = dir_tree_find(tree, FIFF.FIFFB_MNE_EVENTS)

    if len(events) == 0:
        fid.close()
        raise ValueError('Could not find event data')

    events = events[0]

    for d in events['directory']:
        kind = d.kind
        pos = d.pos
        if kind == FIFF.FIFF_MNE_EVENT_LIST:
            tag = read_tag(fid, pos)
            event_list = tag.data
            break
    else:
        raise ValueError('Could not find any events')

    mappings = dir_tree_find(tree, FIFF.FIFFB_MNE_EVENTS)
    mappings = mappings[0]

    for d in mappings['directory']:
        kind = d.kind
        pos = d.pos
        if kind == FIFF.FIFF_DESCRIPTION:
            tag = read_tag(fid, pos)
            mappings = tag.data
            break
    else:
        mappings = None

    if mappings is not None:
        m_ = (m.split(':') for m in mappings.split(';'))
        mappings = dict((k, int(v)) for k, v in m_)
    event_list = event_list.reshape(len(event_list) / 3, 3)
    return event_list, mappings

def read_events(filename, include=None, exclude=None):
    """Reads events from fif or text file

    Parameters
    ----------
    filename : string
        Name of the input file.
        If the extension is .fif, events are read assuming
        the file is in FIF format, otherwise (e.g., .eve,
        .lst, .txt) events are read as coming from text.
        Note that new format event files do not contain
        the "time" column (used to be the second column).
    include : int | list | None
        A event id to include or a list of them.
        If None all events are included.
    exclude : int | list | None
        A event id to exclude or a list of them.
        If None no event is excluded. If include is not None
        the exclude parameter is ignored.

    Returns
    -------
    events: array, shape (n_events, 3)
        The list of events

    Notes
    -----
    This function will discard the offset line (i.e., first line with zero
    event number) if it is present in a text file.
    """
    ext = splitext(filename)[1].lower()
    if ext == '.fif' or ext == '.gz':
        fid, tree, _ = fiff_open(filename)
        event_list, _ = _read_events_fif(fid, tree)
        fid.close()
    else:
        #  Have to read this in as float64 then convert because old style
        #  eve/lst files had a second float column that will raise errors
        lines = np.loadtxt(filename, dtype=np.float64).astype(np.uint32)
        if len(lines) == 0:
            raise ValueError('No text lines found')

        if lines.ndim == 1:  # Special case for only one event
            lines = lines[np.newaxis, :]

        if len(lines[0]) == 4:  # Old format eve/lst
            goods = [0, 2, 3]   # Omit "time" variable
        elif len(lines[0]) == 3:
            goods = [0, 1, 2]
        else:
            raise ValueError('Unknown number of columns in event text file')

        event_list = lines[:, goods]
        if event_list.shape[0] > 0 and event_list[0, 2] == 0:
            event_list = event_list[1:]

    event_list = pick_events(event_list, include, exclude)
    return event_list

def write_events(filename, event_list):
    """Write events to file

    Parameters
    ----------
    filename : string
        Name of the output file.
        If the extension is .fif, events are written in
        binary FIF format, otherwise (e.g., .eve, .lst,
        .txt) events are written as plain text.
        Note that new format event files do not contain
        the "time" column (used to be the second column).

    event_list : array, shape (n_events, 3)
        The list of events
    """
    ext = splitext(filename)[1].lower()
    if ext == '.fif' or ext == '.gz':
        #   Start writing...
        fid = start_file(filename)

        start_block(fid, FIFF.FIFFB_MNE_EVENTS)
        write_int(fid, FIFF.FIFF_MNE_EVENT_LIST, event_list.T)
        end_block(fid, FIFF.FIFFB_MNE_EVENTS)

        end_file(fid)
    else:
        f = open(filename, 'w')
        [f.write('%6d %6d %3d\n' % tuple(e)) for e in event_list]
        f.close()

@verbose
def find_events(raw, stim_channel='STI 014', verbose=None):
    """Find events from raw file

    Parameters
    ----------
    raw : Raw object
        The raw data.
    stim_channel : string or list of string
        Name of the stim channel or all the stim channels
        affected by the trigger.
    verbose : bool, str, int, or None
        If not None, override default verbose level (see mne.verbose).

    Returns
    -------
    events : array
        The array of event onsets in time samples.
    """
    if not isinstance(stim_channel, list):
        stim_channel = [stim_channel]

    pick = pick_channels(raw.info['ch_names'], include=stim_channel)
    if len(pick) == 0:
        raise ValueError('No stim channel found to extract event triggers.')
    data, times = raw[pick, :]
    if np.any(data < 0):
        logger.warn('Trigger channel contains negative values. '
                    'Taking absolute value.')
        data = np.abs(data)  # make sure trig channel is positive
    data = data.astype(np.int)
    idx = np.where(np.all(np.diff(data, axis=1) > 0, axis=0))[0]
    events_id = data[0, idx + 1].astype(np.int)
    idx += raw.first_samp + 1
    events = np.c_[idx, np.zeros_like(idx), events_id]
    logger.info("%s events found" % len(events))
    logger.info("Events id: %s" % np.unique(events[:, 2]))
    return events

def merge_events(events, ids, new_id):
    """Merge a set of events

    Parameters
    ----------
    events : array
        Events.
    ids : array of int
        The ids of events to merge.
    new_id : int
        The new id.

    Returns
    -------
    new_events: array
        The new events
    """
    events = events.copy()
    events_numbers = events[:, 2]
    for i in ids:
        events_numbers[events_numbers == i] = new_id
    return events

def make_fixed_length_events(raw, id, start=0, stop=None, duration=1.):
    """Make a set of events separated by a fixed duration

    Parameters
    ----------
    raw : instance of Raw
        A raw object to use the data from.
    duration: float
        The duration to separate events by.
    id : int
        The id to use.

    Returns
    -------
    new_events: array
        The new events
    """
    start = raw.time_as_index(start)
    start = start[0] + raw.first_samp
    if stop is not None:
        stop = raw.time_as_index(stop)
        stop = min([stop[0] + raw.first_samp, raw.last_samp + 1])
    else:
        stop = raw.last_samp + 1
    if not isinstance(id, int):
        raise ValueError('id must be an integer')
    # Make sure we don't go out the end of the file:
    stop -= np.ceil(raw.info['sfreq'] * duration)
    ts = np.arange(start, stop, raw.info['sfreq'] * duration).astype(int)
    n_events = len(ts)
    events = np.c_[ts, np.zeros(n_events, dtype=int),
                   id * np.ones(n_events, dtype=int)]
    return events

def concatenate_events(events, first_samps, last_samps):
    """Concatenate event lists in a manner compatible with
    concatenate_raws

    This is useful, for example, if you processed and/or changed
    events in raw files separately before combining them using
    concatenate_raws.

    Parameters
    ----------
    events : list of arrays
        List of event arrays, typically each extracted from a
        corresponding raw file that is being concatenated.

    first_samps : list or array of int
        First sample numbers of the raw files concatenated.

    last_samps : list or array of int
        Last sample numbers of the raw files concatenated.

    Returns
    -------
    events : array
        The concatenated events.
    """
    if not isinstance(events, list):
        raise ValueError('events must be a list of arrays')
    if not (len(events) == len(last_samps) and
            len(events) == len(first_samps)):
        raise ValueError('events, first_samps, and last_samps must all have '
                         'the same lengths')
    first_samps = np.array(first_samps)
    last_samps = np.array(last_samps)
    n_samps = np.cumsum(last_samps - first_samps + 1)
    events_out = events[0]
    for e, f, n in zip(events[1:], first_samps[1:], n_samps[:-1]):
        # remove any skip since it doesn't exist in concatenated files
        e2 = e.copy()
        e2[:, 0] -= f
        # add offset due to previous files, plus original file offset
        e2[:, 0] += n + first_samps[0]
        events_out = np.concatenate((events_out, e2), axis=0)

    return events_out
<commit_msg>Consider negative differences in events channel as events<commit_after>"""IO with fif files containing events
"""

# Authors: Alexandre Gramfort <gramfort@nmr.mgh.harvard.edu>
#          Matti Hamalainen <msh@nmr.mgh.harvard.edu>
#
# License: BSD (3-clause)

import numpy as np
from os.path import splitext

import logging
logger = logging.getLogger('mne')

from .fiff.constants import FIFF
from .fiff.tree import dir_tree_find
from .fiff.tag import read_tag
from .fiff.open import fiff_open
from .fiff.write import write_int, start_block, start_file, end_block, end_file
from .fiff.pick import pick_channels
from . import verbose

def pick_events(events, include=None, exclude=None):
    """Select some events

    Parameters
    ----------
    include : int | list | None
        A event id to include or a list of them.
        If None all events are included.
    exclude : int | list | None
        A event id to exclude or a list of them.
        If None no event is excluded. If include is not None
        the exclude parameter is ignored.

    Returns
    -------
    events : array, shape (n_events, 3)
        The list of events
    """
    if include is not None:
        if not isinstance(include, list):
            include = [include]
        mask = np.zeros(len(events), dtype=np.bool)
        for e in include:
            mask = np.logical_or(mask, events[:, 2] == e)
        events = events[mask]
    elif exclude is not None:
        if not isinstance(exclude, list):
            exclude = [exclude]
        mask = np.ones(len(events), dtype=np.bool)
        for e in exclude:
            mask = np.logical_and(mask, events[:, 2] != e)
        events = events[mask]
    else:
        events = np.copy(events)

    if len(events) == 0:
        raise RuntimeError("No events found")

    return events

def define_target_events(events, reference_id, target_id, sfreq, tmin, tmax,
            new_id=None, fill_na=None):
    """ Define new events by co-occurrence of existing events

    This function can be used to evaluate events depending on the
    temporal lag to another event. For example, this can be used to
    analyze evoked responses which were followed by a button press wihtin
    a defined time window.

    Parameters
    ----------

    events : ndarray
        Array as returned by mne.find_events.
    reference_id : int
        The reference event. The event defining the epoch of interest.
    target_id : int
        The target event. The event co-occurring in within a certain time
        window around the reference event.
    sfreq : float
        The sampling frequency of the data.
    tmin : float
        The lower limit in seconds from the target event.
    tmax : float
        The upper limit border in seconds from the target event.
    new_id : int
        new_id for the new event
    fill_na : int | None
        Fill event to be inserted if target is not available within the time
        window specified. If None, the 'null' events will be dropped.

    Returns
    -------
    new_events : ndarray
        The new defined events
    lag : ndarray
        time lag between reference and target in milliseconds.
    """

    if new_id is None:
        new_id = reference_id

    tsample = 1e3 / sfreq
    imin = int(tmin * sfreq)
    imax = int(tmax * sfreq)

    new_events = []
    lag = []
    for event in events.copy().astype('f8'):
        if event[2] == reference_id:
            lower = event[0] + imin
            upper = event[0] + imax
            res = events[(events[:, 0] > lower) &
                    (events[:, 0] < upper) & (events[:, 2] == target_id)]
            if res.any():
                lag += [event[0] - res[0][0]]
                event[2] = new_id
                new_events += [event]
            elif fill_na is not None:
                event[2] = fill_na
                new_events += [event]
                lag += [fill_na]

    new_events = np.array(new_events)

    lag = np.abs(lag, dtype='f8')
    if lag.any():
        lag[lag != fill_na] *= tsample
    else:
        lag = np.array([])

    return new_events if new_events.any() else np.array([]), lag

def _read_events_fif(fid, tree):
    #   Find the desired block
    events = dir_tree_find(tree, FIFF.FIFFB_MNE_EVENTS)

    if len(events) == 0:
        fid.close()
        raise ValueError('Could not find event data')

    events = events[0]

    for d in events['directory']:
        kind = d.kind
        pos = d.pos
        if kind == FIFF.FIFF_MNE_EVENT_LIST:
            tag = read_tag(fid, pos)
            event_list = tag.data
            break
    else:
        raise ValueError('Could not find any events')

    mappings = dir_tree_find(tree, FIFF.FIFFB_MNE_EVENTS)
    mappings = mappings[0]

    for d in mappings['directory']:
        kind = d.kind
        pos = d.pos
        if kind == FIFF.FIFF_DESCRIPTION:
            tag = read_tag(fid, pos)
            mappings = tag.data
            break
    else:
        mappings = None

    if mappings is not None:
        m_ = (m.split(':') for m in mappings.split(';'))
        mappings = dict((k, int(v)) for k, v in m_)
    event_list = event_list.reshape(len(event_list) / 3, 3)
    return event_list, mappings

def read_events(filename, include=None, exclude=None):
    """Reads events from fif or text file

    Parameters
    ----------
    filename : string
        Name of the input file.
        If the extension is .fif, events are read assuming
        the file is in FIF format, otherwise (e.g., .eve,
        .lst, .txt) events are read as coming from text.
        Note that new format event files do not contain
        the "time" column (used to be the second column).
    include : int | list | None
        A event id to include or a list of them.
        If None all events are included.
    exclude : int | list | None
        A event id to exclude or a list of them.
        If None no event is excluded. If include is not None
        the exclude parameter is ignored.

    Returns
    -------
    events: array, shape (n_events, 3)
        The list of events

    Notes
    -----
    This function will discard the offset line (i.e., first line with zero
    event number) if it is present in a text file.
    """
    ext = splitext(filename)[1].lower()
    if ext == '.fif' or ext == '.gz':
        fid, tree, _ = fiff_open(filename)
        event_list, _ = _read_events_fif(fid, tree)
        fid.close()
    else:
        #  Have to read this in as float64 then convert because old style
        #  eve/lst files had a second float column that will raise errors
        lines = np.loadtxt(filename, dtype=np.float64).astype(np.uint32)
        if len(lines) == 0:
            raise ValueError('No text lines found')

        if lines.ndim == 1:  # Special case for only one event
            lines = lines[np.newaxis, :]

        if len(lines[0]) == 4:  # Old format eve/lst
            goods = [0, 2, 3]   # Omit "time" variable
        elif len(lines[0]) == 3:
            goods = [0, 1, 2]
        else:
            raise ValueError('Unknown number of columns in event text file')

        event_list = lines[:, goods]
        if event_list.shape[0] > 0 and event_list[0, 2] == 0:
            event_list = event_list[1:]

    event_list = pick_events(event_list, include, exclude)
    return event_list

def write_events(filename, event_list):
    """Write events to file

    Parameters
    ----------
    filename : string
        Name of the output file.
        If the extension is .fif, events are written in
        binary FIF format, otherwise (e.g., .eve, .lst,
        .txt) events are written as plain text.
        Note that new format event files do not contain
        the "time" column (used to be the second column).

    event_list : array, shape (n_events, 3)
        The list of events
    """
    ext = splitext(filename)[1].lower()
    if ext == '.fif' or ext == '.gz':
        #   Start writing...
        fid = start_file(filename)

        start_block(fid, FIFF.FIFFB_MNE_EVENTS)
        write_int(fid, FIFF.FIFF_MNE_EVENT_LIST, event_list.T)
        end_block(fid, FIFF.FIFFB_MNE_EVENTS)

        end_file(fid)
    else:
        f = open(filename, 'w')
        [f.write('%6d %6d %3d\n' % tuple(e)) for e in event_list]
        f.close()

@verbose
def find_events(raw, stim_channel='STI 014', verbose=None):
    """Find events from raw file

    Parameters
    ----------
    raw : Raw object
        The raw data.
    stim_channel : string or list of string
        Name of the stim channel or all the stim channels
        affected by the trigger.
    verbose : bool, str, int, or None
        If not None, override default verbose level (see mne.verbose).

    Returns
    -------
    events : array
        The array of event onsets in time samples.
    """
    if not isinstance(stim_channel, list):
        stim_channel = [stim_channel]

    pick = pick_channels(raw.info['ch_names'], include=stim_channel)
    if len(pick) == 0:
        raise ValueError('No stim channel found to extract event triggers.')
    data, times = raw[pick, :]
    if np.any(data < 0):
        logger.warn('Trigger channel contains negative values. '
                    'Taking absolute value.')
        data = np.abs(data)  # make sure trig channel is positive
    data = data.astype(np.int)
    idx = np.where(np.all(np.logical_and(np.diff(data, axis=1) != 0, data[:, 1:] > 0), axis=0))[0]
    events_id = data[0, idx + 1].astype(np.int)
    idx += raw.first_samp + 1
    events = np.c_[idx, np.zeros_like(idx), events_id]
    logger.info("%s events found" % len(events))
    logger.info("Events id: %s" % np.unique(events[:, 2]))
    return events

def merge_events(events, ids, new_id):
    """Merge a set of events

    Parameters
    ----------
    events : array
        Events.
    ids : array of int
        The ids of events to merge.
    new_id : int
        The new id.

    Returns
    -------
    new_events: array
        The new events
    """
    events = events.copy()
    events_numbers = events[:, 2]
    for i in ids:
        events_numbers[events_numbers == i] = new_id
    return events

def make_fixed_length_events(raw, id, start=0, stop=None, duration=1.):
    """Make a set of events separated by a fixed duration

    Parameters
    ----------
    raw : instance of Raw
        A raw object to use the data from.
    duration: float
        The duration to separate events by.
    id : int
        The id to use.

    Returns
    -------
    new_events: array
        The new events
    """
    start = raw.time_as_index(start)
    start = start[0] + raw.first_samp
    if stop is not None:
        stop = raw.time_as_index(stop)
        stop = min([stop[0] + raw.first_samp, raw.last_samp + 1])
    else:
        stop = raw.last_samp + 1
    if not isinstance(id, int):
        raise ValueError('id must be an integer')
    # Make sure we don't go out the end of the file:
    stop -= np.ceil(raw.info['sfreq'] * duration)
    ts = np.arange(start, stop, raw.info['sfreq'] * duration).astype(int)
    n_events = len(ts)
    events = np.c_[ts, np.zeros(n_events, dtype=int),
                   id * np.ones(n_events, dtype=int)]
    return events

def concatenate_events(events, first_samps, last_samps):
    """Concatenate event lists in a manner compatible with
    concatenate_raws

    This is useful, for example, if you processed and/or changed
    events in raw files separately before combining them using
    concatenate_raws.

    Parameters
    ----------
    events : list of arrays
        List of event arrays, typically each extracted from a
        corresponding raw file that is being concatenated.

    first_samps : list or array of int
        First sample numbers of the raw files concatenated.

    last_samps : list or array of int
        Last sample numbers of the raw files concatenated.

    Returns
    -------
    events : array
        The concatenated events.
    """
    if not isinstance(events, list):
        raise ValueError('events must be a list of arrays')
    if not (len(events) == len(last_samps) and
            len(events) == len(first_samps)):
        raise ValueError('events, first_samps, and last_samps must all have '
                         'the same lengths')
    first_samps = np.array(first_samps)
    last_samps = np.array(last_samps)
    n_samps = np.cumsum(last_samps - first_samps + 1)
    events_out = events[0]
    for e, f, n in zip(events[1:], first_samps[1:], n_samps[:-1]):
        # remove any skip since it doesn't exist in concatenated files
        e2 = e.copy()
        e2[:, 0] -= f
        # add offset due to previous files, plus original file offset
        e2[:, 0] += n + first_samps[0]
        events_out = np.concatenate((events_out, e2), axis=0)

    return events_out
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Licensed to Cloudera, Inc. under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  Cloudera, Inc. licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Decorators and methods related to access log.
This assumes a single-threaded server.
"""

import logging
import re
import threading
import time

import desktop.conf

ACCESS_LOG = logging.getLogger('access')

def access_log_level(lvl):
  """Decorator to set the access log level of a view function."""
  if lvl not in (logging.DEBUG, logging.WARN, logging.ERROR, logging.CRITICAL, logging.FATAL):
    raise ValueError('%s is not a valid logging level' % (lvl,))

  def deco_view(func):
    func.access_log_level = lvl
    return func
  return deco_view

#
# Keep most recent per user per app per view access info
#
# This is a dictionary (indexed by user)
# of dictionary (indexed by app)
# of dictionary (indexed by path)
# of list (of AccessInfo) sorted by time most recent first
#
recent_access_map = { }
_recent_access_map_lk = threading.Lock()
_per_user_lk = { }      # Indexed by username

# Store a map of usernames and a dictionary of
# their IP addresses and last access times
last_access_map = { }

# Max number of records per user per view to keep
_USER_ACCESS_HISTORY_SIZE = desktop.conf.USER_ACCESS_HISTORY_SIZE.get()

class AccessInfo(dict):
  """
  Represents details on a user access.

  In addition to the attributes specified in __init__, it may contain
  ``msg`` -- A message associated with the access
  ``app`` -- The top level package name of the view function, which
             need NOT be a valid Desktop application name
  """
  def __init__(self, request):
    self['username'] = request.user.username or '-anon-'
    if request.META.has_key('HTTP_X_FORWARDED_FOR'):
      self['remote_ip'] = request.META.get('HTTP_X_FORWARDED_FOR', '-')
    else:
      self['remote_ip'] = request.META.get('REMOTE_ADDR', '-')
    self['method'] = request.method
    self['path'] = request.path
    self['proto'] = request.META.get('SERVER_PROTOCOL', '-')
    self['agent'] = request.META.get('HTTP_USER_AGENT', '-')
    self['time'] = time.time()
    self['duration'] = None

  def log(self, level, msg=None, start_time=None):
    self['duration'] = ' returned in %dms' % ((time.time() - start_time) * 1000) if start_time is not None else ''

    if msg is not None:
      self['msg'] = msg
      ACCESS_LOG.log(level, '%(remote_ip)s %(username)s - "%(method)s %(path)s %(proto)s"%(duration)s -- %(msg)s' % self)
    else:
      ACCESS_LOG.log(level, '%(remote_ip)s %(username)s - "%(method)s %(path)s %(proto)s"%(duration)s' % self)

  def add_to_access_history(self, app):
    """Record this user access to the recent access map"""
    self['app'] = app
    user = self['username']
    path = self['path']
    try:
      app_dict = recent_access_map[user]
    except KeyError:
      # Hold the global lock when modifying recent_access_map
      _recent_access_map_lk.acquire()
      try:
        app_dict = { }
        _per_user_lk[user] = threading.Lock()
        recent_access_map[user] = app_dict
      finally:
        _recent_access_map_lk.release()

    # Hold the per user lock when modifying adding the access record.
    # We could further break down the locking granularity but that seems silly.
    user_lk = _per_user_lk[user]
    user_lk.acquire()
    try:
      try:
        path_dict = app_dict[app]
      except KeyError:
        path_dict = { }
        app_dict[app] = path_dict

      try:
        view_access_list = path_dict[path]
      except KeyError:
        view_access_list = [ ]
        path_dict[path] = view_access_list

      # Most recent first
      view_access_list.insert(0, self)
      if len(view_access_list) > _USER_ACCESS_HISTORY_SIZE:
        view_access_list.pop()

      # Update the IP address and last access time of the user
      last_access_map[user] = {'ip': self['remote_ip'], 'time': self['time']}
    finally:
      user_lk.release()

_MODULE_RE = re.compile('[^.]*')

def log_page_hit(request, view_func, level=None, start_time=None):
  """Log the request to the access log"""
  if level is None:
    level = logging.INFO
  ai = AccessInfo(request)
  ai.log(level, start_time=start_time)
  # Find the app
  app_re_match = _MODULE_RE.match(view_func.__module__)
  app = app_re_match and app_re_match.group(0) or '-'
  ai.add_to_access_history(app)

def access_log(request, msg=None, level=None):
  """
  access_log(request, msg=None, level=None) -> None

  Write to the access log. This could be a page hit, or general auditing information.
  """
  if level is None:
    level = logging.INFO
  ai = AccessInfo(request)
  ai.log(level, msg)

def access_warn(request, msg=None):
  """Write to access log with a WARN log level"""
  ai = AccessInfo(request)
  ai.log(logging.WARN, msg)
<commit_msg>HUE-7447 [core] Stop logging user pages counts<commit_after>#!/usr/bin/env python
# Licensed to Cloudera, Inc. under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  Cloudera, Inc. licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Decorators and methods related to access log.
This assumes a single-threaded server.
"""

import logging
import re
import threading
import time

import desktop.conf

ACCESS_LOG = logging.getLogger('access')

def access_log_level(lvl):
  """Decorator to set the access log level of a view function."""
  if lvl not in (logging.DEBUG, logging.WARN, logging.ERROR, logging.CRITICAL, logging.FATAL):
    raise ValueError('%s is not a valid logging level' % (lvl,))

  def deco_view(func):
    func.access_log_level = lvl
    return func
  return deco_view

#
# Keep most recent per user per app per view access info
#
# This is a dictionary (indexed by user)
# of dictionary (indexed by app)
# of dictionary (indexed by path)
# of list (of AccessInfo) sorted by time most recent first
#
recent_access_map = { }
_recent_access_map_lk = threading.Lock()
_per_user_lk = { }      # Indexed by username

# Store a map of usernames and a dictionary of
# their IP addresses and last access times
last_access_map = { }

# Max number of records per user per view to keep
_USER_ACCESS_HISTORY_SIZE = desktop.conf.USER_ACCESS_HISTORY_SIZE.get()

class AccessInfo(dict):
  """
  Represents details on a user access.

  In addition to the attributes specified in __init__, it may contain
  ``msg`` -- A message associated with the access
  ``app`` -- The top level package name of the view function, which
             need NOT be a valid Desktop application name
  """
  def __init__(self, request):
    self['username'] = request.user.username or '-anon-'
    if request.META.has_key('HTTP_X_FORWARDED_FOR'):
      self['remote_ip'] = request.META.get('HTTP_X_FORWARDED_FOR', '-')
    else:
      self['remote_ip'] = request.META.get('REMOTE_ADDR', '-')
    self['method'] = request.method
    self['path'] = request.path
    self['proto'] = request.META.get('SERVER_PROTOCOL', '-')
    self['agent'] = request.META.get('HTTP_USER_AGENT', '-')
    self['time'] = time.time()
    self['duration'] = None

  def log(self, level, msg=None, start_time=None):
    self['duration'] = ' returned in %dms' % ((time.time() - start_time) * 1000) if start_time is not None else ''

    if msg is not None:
      self['msg'] = msg
      ACCESS_LOG.log(level, '%(remote_ip)s %(username)s - "%(method)s %(path)s %(proto)s"%(duration)s -- %(msg)s' % self)
    else:
      ACCESS_LOG.log(level, '%(remote_ip)s %(username)s - "%(method)s %(path)s %(proto)s"%(duration)s' % self)

  def add_to_access_history(self, app):
    """Record this user access to the recent access map"""
    self['app'] = app
    user = self['username']
    path = self['path']
    try:
      app_dict = recent_access_map[user]
    except KeyError:
      # Hold the global lock when modifying recent_access_map
      _recent_access_map_lk.acquire()
      try:
        app_dict = { }
        _per_user_lk[user] = threading.Lock()
        recent_access_map[user] = app_dict
      finally:
        _recent_access_map_lk.release()

    # Hold the per user lock when modifying adding the access record.
    # We could further break down the locking granularity but that seems silly.
    user_lk = _per_user_lk[user]
    user_lk.acquire()
    try:
      try:
        path_dict = app_dict[app]
      except KeyError:
        path_dict = { }
        app_dict[app] = path_dict

      try:
        view_access_list = path_dict[path]
      except KeyError:
        view_access_list = [ ]
        path_dict[path] = view_access_list

      # Most recent first
      view_access_list.insert(0, self)
      if len(view_access_list) > _USER_ACCESS_HISTORY_SIZE:
        view_access_list.pop()

      # Update the IP address and last access time of the user
      last_access_map[user] = {'ip': self['remote_ip'], 'time': self['time']}
    finally:
      user_lk.release()

_MODULE_RE = re.compile('[^.]*')

def log_page_hit(request, view_func, level=None, start_time=None):
  """Log the request to the access log"""
  if level is None:
    level = logging.INFO
  ai = AccessInfo(request)
  ai.log(level, start_time=start_time)

  # Disabled for now as not used
  # Find the app
#   app_re_match = _MODULE_RE.match(view_func.__module__)
#   app = app_re_match and app_re_match.group(0) or '-'
#   ai.add_to_access_history(app)

def access_log(request, msg=None, level=None):
  """
  access_log(request, msg=None, level=None) -> None

  Write to the access log. This could be a page hit, or general auditing information.
  """
  if level is None:
    level = logging.INFO
  ai = AccessInfo(request)
  ai.log(level, msg)

def access_warn(request, msg=None):
  """Write to access log with a WARN log level"""
  ai = AccessInfo(request)
  ai.log(logging.WARN, msg)
<|endoftext|>


<commit_before>from pybuilder.core import use_plugin, init, Author

use_plugin("python.core")
use_plugin("python.unittest")
use_plugin("python.integrationtest")
use_plugin("python.install_dependencies")
use_plugin("python.flake8")
use_plugin("python.coverage")
use_plugin("python.distutils")
use_plugin('copy_resources')
use_plugin('filter_resources')

name = "yum-repos"
summary = "yum-repos: simple yum repositories with minimal rest api"
url = "https://github.com/arnehilmann/yum-repos"
version = "0.7.7"

authors = [Author('Arne Hilmann', 'arne.hilmann@gmail.com')]

description = """yum-repos
- serve yum repositories as simple folders
- ... via web server
- offer rest api for
   - create/remove/link of repositories
   - upload/stage/remove of rpms
"""

default_task = ["clean", "analyze", "publish"]

@init
def set_properties(project):
    project.build_depends_on('requests')

    project.depends_on("flask")

    project.set_property('copy_resources_target', '$dir_dist')
    project.get_property('copy_resources_glob').extend(['setup.cfg'])
    project.get_property('filter_resources_glob').extend(['**/setup.cfg'])
<commit_msg>bump version<commit_after>from pybuilder.core import use_plugin, init, Author

use_plugin("python.core")
use_plugin("python.unittest")
use_plugin("python.integrationtest")
use_plugin("python.install_dependencies")
use_plugin("python.flake8")
use_plugin("python.coverage")
use_plugin("python.distutils")
use_plugin('copy_resources')
use_plugin('filter_resources')

name = "yum-repos"
summary = "yum-repos: simple yum repositories with minimal rest api"
url = "https://github.com/arnehilmann/yum-repos"
version = "0.7.8"

authors = [Author('Arne Hilmann', 'arne.hilmann@gmail.com')]

description = """yum-repos
- serve yum repositories as simple folders
- ... via web server
- offer rest api for
   - create/remove/link of repositories
   - upload/stage/remove of rpms
"""

default_task = ["clean", "analyze", "publish"]

@init
def set_properties(project):
    project.build_depends_on('requests')

    project.depends_on("flask")

    project.set_property('copy_resources_target', '$dir_dist')
    project.get_property('copy_resources_glob').extend(['setup.cfg'])
    project.get_property('filter_resources_glob').extend(['**/setup.cfg'])
<|endoftext|>


<commit_before>from flask import current_app
from .text import slugify

def expose(url='/', methods=('GET',)):
    """
        Use this decorator to expose views in your view classes.

        `url`
            Relative URL for the view
        `methods`
            Allowed HTTP methods. By default only GET is allowed.
    """
    def wrap(f):
        if not hasattr(f, '_urls'):
            f._urls = []
        f._urls.append((url, methods))
        return f

    return wrap

def get_content_url(content):
    category = content.get('category')
    title = content.get('title')
    slug = content.get('slug')
    ext = current_app.config.get('CONTENT_EXTENSION', 'html')
    if category:
        url = f'/{slugify(category)}/{slug or slugify(title)}.{ext}'
    else:
        url = f'/{slug or slugify(title)}.{ext}'
    return url
<commit_msg>removed unused code form utils.routing<commit_after>

def expose(url='/', methods=('GET',)):
    """
        Use this decorator to expose views in your view classes.

        `url`
            Relative URL for the view
        `methods`
            Allowed HTTP methods. By default only GET is allowed.
    """
    def wrap(f):
        if not hasattr(f, '_urls'):
            f._urls = []
        f._urls.append((url, methods))
        return f

    return wrap
<|endoftext|>


<commit_before>from __future__ import division # confidence high

pkg = [ 'pytools', 'pytools.tests' ]

setupargs = {

    'version' :         '3.0',
    'description' :     "General Use Python Tools",
    'author' :          "Warren Hack, Christopher Hanley",
    'author_email' :    "help@stsci.edu",
    'license' :         "http://www.stsci.edu/resources/software_hardware/pyraf/LICENSE",
    'platforms' :       ["Linux","Solaris","Mac OS X","Win"],
    'scripts' :         [ 'lib/fitsdiff','lib/convertwaiveredfits','lib/updatenpol'] ,
    'package_dir' :     { 'pytools' : 'lib', 'pytools.tests' : 'lib/tests' },
    'data_files' :      [ ( 'pytools/tests', [ 'lib/tests/*.fits' ] ) ],
    }

<commit_msg>Updated 'defsetup' in pytools to remove 'updatenpol' installation, since it is now installed as part of the acstools package. WJH<commit_after>from __future__ import division # confidence high

pkg = [ 'pytools', 'pytools.tests' ]

setupargs = {

    'version' :         '3.0',
    'description' :     "General Use Python Tools",
    'author' :          "Warren Hack, Christopher Hanley",
    'author_email' :    "help@stsci.edu",
    'license' :         "http://www.stsci.edu/resources/software_hardware/pyraf/LICENSE",
    'platforms' :       ["Linux","Solaris","Mac OS X","Win"],
    'scripts' :         [ 'lib/fitsdiff','lib/convertwaiveredfits'] ,
    'package_dir' :     { 'pytools' : 'lib', 'pytools.tests' : 'lib/tests' },
    'data_files' :      [ ( 'pytools/tests', [ 'lib/tests/*.fits' ] ) ],
    }

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import dns.resolver

from django.contrib.auth.models import User
from post_office import mail

def send_templated_email(template_name, user_list):
    for u in user_list:
        mail.send(
            recipients=[u.email],
            sender='iva@srdu.ru',
            template=template_name,
            context={'user': u},
            priority=3
        )

def validate_email_domain(email):
    domain = email.split('@')[1]
    dns.resolver.query(domain, 'MX')
<commit_msg>Fixed empty mail validation<commit_after># -*- coding: utf-8 -*-
import dns.resolver

from django.contrib.auth.models import User
from post_office import mail

def send_templated_email(template_name, user_list):
    for u in user_list:
        mail.send(
            recipients=[u.email],
            sender='iva@srdu.ru',
            template=template_name,
            context={'user': u},
            priority=3
        )

def validate_email_domain(email):
    if email != '':
        domain = email.split('@')[1]
        dns.resolver.query(domain, 'MX')
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
# Copyright 2010 Santhosh Thottingal <santhosh.thottingal@gmail.com>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
import cgitb
import os
import cgi
import sys
from common import *

class Webfonts(SilpaModule):
    def __init__(self):
        self.template = os.path.join(os.path.dirname(__file__), 'index.html')
        self.font=None
        self.available_fonts=['Meera','Rachana', 'Suruma', 'AnjaliOldLipi', 'Kalyani','RaghuMalayalam','Lohit Malayalam','Dyuthi','Mallige-n','Kedage-n','lohit_kn']

    def set_request(self,request):
        self.request=request
        self.font = self.request.get('font')

    def is_self_serve(self) :
        if self.font:
            return True
        else:
            return False

    def get_mimetype(self):
        return "text/css"

    def serve(self,font=None):
        """
        Provide the css for the given font. CSS will differ for IE and Other browsers
        """
        http_ref =self.request.get('HTTP_REFERER')
        http_ref=http_ref.replace("Webfonts","")
        if self.font not in self.available_fonts:
            return "Error!, Font not available"
        user_agent= self.request.get('HTTP_USER_AGENT')
        if user_agent.find("MSIE")<0:
            css = "@font-face {font-family: '$$FONTFAMILY$$';font-style: normal;font-weight: normal;src: local('$$FONTFAMILY$$'), url('$$FONTURL$$') format('truetype');}"
            css=css.replace('$$FONTURL$$', http_ref +'/modules/webfonts/font/' + self.font + '.ttf')
        else:
            css = "@font-face {font-family: '$$FONTFAMILY$$';font-style: normal;font-weight: normal;src:local('$$FONTFAMILY$$'), url('$$FONTURL$$');}"
            css=css.replace('$$FONTURL$$', http_ref +'/modules/webfonts/font/' + self.font + '.eot')

        css=css.replace('$$FONTFAMILY$$',self.font)
        return css

    @ServiceMethod
    def get_fonts_list(self, language=None):
        """
        return a list of available fonts names for the given Language
        If the language is not given, return all the available fonts
        """
        return self.available_fonts

    def get_module_name(self):
        return "Webfonts"

    def get_info(self):
        return  "Indic Webfonts"

def getInstance():
    return Webfonts()

<commit_msg>http_ref wont work. changing to http_host<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
# Copyright 2010 Santhosh Thottingal <santhosh.thottingal@gmail.com>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
import cgitb
import os
import cgi
import sys
from common import *
from utils import *

class Webfonts(SilpaModule):
    def __init__(self):
        self.template = os.path.join(os.path.dirname(__file__), 'index.html')
        self.font=None
        self.available_fonts=['Meera','Rachana', 'Suruma', 'AnjaliOldLipi', 'Kalyani','RaghuMalayalam','Lohit Malayalam','Dyuthi','Mallige-n','Kedage-n','lohit_kn']

    def set_request(self,request):
        self.request=request
        self.font = self.request.get('font')

    def is_self_serve(self) :
        if self.font:
            return True
        else:
            return False

    def get_mimetype(self):
        return "text/css"

    def serve(self,font=None):
        """
        Provide the css for the given font. CSS will differ for IE and Other browsers
        """
        http_host =self.request.get('HTTP_HOST')
        request_uri =self.request.get('REQUEST_URI')
        if request_uri!=None:
            http_host+="/silpa"
        if self.font not in self.available_fonts:
            return "Error!, Font not available"
        user_agent= self.request.get('HTTP_USER_AGENT')
        if user_agent.find("MSIE")<0:
            css = "@font-face {font-family: '$$FONTFAMILY$$';font-style: normal;font-weight: normal;src: local('$$FONTFAMILY$$'), url('$$FONTURL$$') format('truetype');}"
            css=css.replace('$$FONTURL$$', "http://"+http_host +'/modules/webfonts/font/' + self.font + '.ttf')
        else:
            css = "@font-face {font-family: '$$FONTFAMILY$$';font-style: normal;font-weight: normal;src:local('$$FONTFAMILY$$'), url('$$FONTURL$$');}"
            css=css.replace('$$FONTURL$$', "http://"+http_host +'/modules/webfonts/font/' + self.font + '.eot')

        css=css.replace('$$FONTFAMILY$$',self.font)
        return css

    @ServiceMethod
    def get_fonts_list(self, language=None):
        """
        return a list of available fonts names for the given Language
        If the language is not given, return all the available fonts
        """
        return self.available_fonts

    def get_module_name(self):
        return "Webfonts"

    def get_info(self):
        return  "Indic Webfonts"

def getInstance():
    return Webfonts()

<|endoftext|>


<commit_before>#!/usr/bin/env python3
# SPDX-License-Identifier: BSD-3-Clause
# Copyright(c) 2020 Intel Corporation

"""
Configure an entire Intel DSA instance, using idxd kernel driver, for DPDK use
"""

import sys
import argparse
import os
import os.path

class SysfsDir:
    "Used to read/write paths in a sysfs directory"
    def __init__(self, path):
        self.path = path

    def read_int(self, filename):
        "Return a value from sysfs file"
        with open(os.path.join(self.path, filename)) as f:
            return int(f.readline())

    def write_values(self, values):
        "write dictionary, where key is filename and value is value to write"
        for filename, contents in values.items():
            with open(os.path.join(self.path, filename), "w") as f:
                f.write(str(contents))

def reset_device(dsa_id):
    "Reset the DSA device and all its queues"
    drv_dir = SysfsDir("/sys/bus/dsa/drivers/dsa")
    drv_dir.write_values({"unbind": f"dsa{dsa_id}"})

def get_pci_dir(pci):
    "Search for the sysfs directory of the PCI device"
    base_dir = '/sys/bus/pci/devices/'
    for path, dirs, files in os.walk(base_dir):
        for dir in dirs:
            if pci in dir:
                return os.path.join(base_dir, dir)
    sys.exit(f"Could not find sysfs directory for device {pci}")

def get_dsa_id(pci):
    "Get the DSA instance ID using the PCI address of the device"
    pci_dir = get_pci_dir(pci)
    for path, dirs, files in os.walk(pci_dir):
        for dir in dirs:
            if dir.startswith('dsa') and 'wq' not in dir:
                return int(dir[3:])
    sys.exit(f"Could not get device ID for device {pci}")

def configure_dsa(dsa_id, queues, prefix):
    "Configure the DSA instance with appropriate number of queues"
    dsa_dir = SysfsDir(f"/sys/bus/dsa/devices/dsa{dsa_id}")
    drv_dir = SysfsDir("/sys/bus/dsa/drivers/dsa")

    max_groups = dsa_dir.read_int("max_groups")
    max_engines = dsa_dir.read_int("max_engines")
    max_queues = dsa_dir.read_int("max_work_queues")
    max_tokens = dsa_dir.read_int("max_tokens")

    nb_queues = min(queues, max_queues)
    if queues > nb_queues:
        print(f"Setting number of queues to max supported value: {max_queues}")

    # we want one engine per group, and no more engines than queues
    nb_groups = min(max_engines, max_groups, nb_queues)
    for grp in range(nb_groups):
        dsa_dir.write_values({f"engine{dsa_id}.{grp}/group_id": grp})

    # configure each queue
    for q in range(nb_queues):
        wq_dir = SysfsDir(os.path.join(dsa_dir.path, f"wq{dsa_id}.{q}"))
        wq_dir.write_values({"group_id": q % nb_groups,
                             "type": "user",
                             "mode": "dedicated",
                             "name": f"{prefix}_wq{dsa_id}.{q}",
                             "priority": 1,
                             "size": int(max_tokens / nb_queues)})

    # enable device and then queues
    drv_dir.write_values({"bind": f"dsa{dsa_id}"})
    for q in range(nb_queues):
        drv_dir.write_values({"bind": f"wq{dsa_id}.{q}"})

def main(args):
    "Main function, does arg parsing and calls config function"
    arg_p = argparse.ArgumentParser(
        description="Configure whole DSA device instance for DPDK use")
    arg_p.add_argument('dsa_id',
                       help="Specify DSA instance either via DSA instance number or PCI address")
    arg_p.add_argument('-q', metavar='queues', type=int, default=255,
                       help="Number of queues to set up")
    arg_p.add_argument('--name-prefix', metavar='prefix', dest='prefix',
                       default="dpdk",
                       help="Prefix for workqueue name to mark for DPDK use [default: 'dpdk']")
    arg_p.add_argument('--reset', action='store_true',
                       help="Reset DSA device and its queues")
    parsed_args = arg_p.parse_args(args[1:])

    dsa_id = parsed_args.dsa_id
    dsa_id = get_dsa_id(dsa_id) if ':' in dsa_id else dsa_id
    if parsed_args.reset:
        reset_device(dsa_id)
    else:
        configure_dsa(dsa_id, parsed_args.q, parsed_args.prefix)

if __name__ == "__main__":
    main(sys.argv)
<commit_msg>raw/ioat: fix config script queue size calculation<commit_after>#!/usr/bin/env python3
# SPDX-License-Identifier: BSD-3-Clause
# Copyright(c) 2020 Intel Corporation

"""
Configure an entire Intel DSA instance, using idxd kernel driver, for DPDK use
"""

import sys
import argparse
import os
import os.path

class SysfsDir:
    "Used to read/write paths in a sysfs directory"
    def __init__(self, path):
        self.path = path

    def read_int(self, filename):
        "Return a value from sysfs file"
        with open(os.path.join(self.path, filename)) as f:
            return int(f.readline())

    def write_values(self, values):
        "write dictionary, where key is filename and value is value to write"
        for filename, contents in values.items():
            with open(os.path.join(self.path, filename), "w") as f:
                f.write(str(contents))

def reset_device(dsa_id):
    "Reset the DSA device and all its queues"
    drv_dir = SysfsDir("/sys/bus/dsa/drivers/dsa")
    drv_dir.write_values({"unbind": f"dsa{dsa_id}"})

def get_pci_dir(pci):
    "Search for the sysfs directory of the PCI device"
    base_dir = '/sys/bus/pci/devices/'
    for path, dirs, files in os.walk(base_dir):
        for dir in dirs:
            if pci in dir:
                return os.path.join(base_dir, dir)
    sys.exit(f"Could not find sysfs directory for device {pci}")

def get_dsa_id(pci):
    "Get the DSA instance ID using the PCI address of the device"
    pci_dir = get_pci_dir(pci)
    for path, dirs, files in os.walk(pci_dir):
        for dir in dirs:
            if dir.startswith('dsa') and 'wq' not in dir:
                return int(dir[3:])
    sys.exit(f"Could not get device ID for device {pci}")

def configure_dsa(dsa_id, queues, prefix):
    "Configure the DSA instance with appropriate number of queues"
    dsa_dir = SysfsDir(f"/sys/bus/dsa/devices/dsa{dsa_id}")
    drv_dir = SysfsDir("/sys/bus/dsa/drivers/dsa")

    max_groups = dsa_dir.read_int("max_groups")
    max_engines = dsa_dir.read_int("max_engines")
    max_queues = dsa_dir.read_int("max_work_queues")
    max_work_queues_size = dsa_dir.read_int("max_work_queues_size")

    nb_queues = min(queues, max_queues)
    if queues > nb_queues:
        print(f"Setting number of queues to max supported value: {max_queues}")

    # we want one engine per group, and no more engines than queues
    nb_groups = min(max_engines, max_groups, nb_queues)
    for grp in range(nb_groups):
        dsa_dir.write_values({f"engine{dsa_id}.{grp}/group_id": grp})

    # configure each queue
    for q in range(nb_queues):
        wq_dir = SysfsDir(os.path.join(dsa_dir.path, f"wq{dsa_id}.{q}"))
        wq_dir.write_values({"group_id": q % nb_groups,
                             "type": "user",
                             "mode": "dedicated",
                             "name": f"{prefix}_wq{dsa_id}.{q}",
                             "priority": 1,
                             "size": int(max_work_queues_size / nb_queues)})

    # enable device and then queues
    drv_dir.write_values({"bind": f"dsa{dsa_id}"})
    for q in range(nb_queues):
        drv_dir.write_values({"bind": f"wq{dsa_id}.{q}"})

def main(args):
    "Main function, does arg parsing and calls config function"
    arg_p = argparse.ArgumentParser(
        description="Configure whole DSA device instance for DPDK use")
    arg_p.add_argument('dsa_id',
                       help="Specify DSA instance either via DSA instance number or PCI address")
    arg_p.add_argument('-q', metavar='queues', type=int, default=255,
                       help="Number of queues to set up")
    arg_p.add_argument('--name-prefix', metavar='prefix', dest='prefix',
                       default="dpdk",
                       help="Prefix for workqueue name to mark for DPDK use [default: 'dpdk']")
    arg_p.add_argument('--reset', action='store_true',
                       help="Reset DSA device and its queues")
    parsed_args = arg_p.parse_args(args[1:])

    dsa_id = parsed_args.dsa_id
    dsa_id = get_dsa_id(dsa_id) if ':' in dsa_id else dsa_id
    if parsed_args.reset:
        reset_device(dsa_id)
    else:
        configure_dsa(dsa_id, parsed_args.q, parsed_args.prefix)

if __name__ == "__main__":
    main(sys.argv)
<|endoftext|>


<commit_before>import re
import time

import pywikibot

from service.ws_re.scanner.tasks.base_task import ReScannerTask
from tools import save_if_changed
from tools.bots.pi import WikiLogger

REGEX_TRANSLATION = str.maketrans({"(": r"\(",
                                   ")": r"\)",
                                   "?": r"\?"})

class CHRETask(ReScannerTask):
    error_category = "RE:Links führen auf Redirects"

    def __init__(self, wiki: pywikibot.Site, logger: WikiLogger, debug: bool = True):
        ReScannerTask.__init__(self, wiki, logger, debug)
        self.filter_regex = re.compile(
            r"(Benutzer|"
            r"Paulys Realencyclopädie der classischen Altertumswissenschaft/Register/|"
            r"Wikisource:"
            r").*"
        )

    def task(self) -> bool:
        self.iterate_throw_redirects(repair=True)
        self.iterate_throw_redirects(report=True)
        return True

    def iterate_throw_redirects(self, repair: bool = False, report: bool = False):
        for redirect in self.re_page.get_redirects():
            filtered_list = self.filter_link_list(self.get_backlinks(redirect))
            if filtered_list:
                if repair:
                    self.rename_redirects_to_target(filtered_list, redirect)
                    time.sleep(2)
                if report:
                    self.re_page.add_error_category(self.error_category)
                    return
        self.re_page.remove_error_category(self.error_category)

    def rename_redirects_to_target(self, filtered_list: list[str], redirect: pywikibot.Page):
        for entry in filtered_list:
            page = pywikibot.Page(self.wiki, entry)
            new_text = self.replace_redirect_links(page.text, redirect.title()[3:], self.re_page.lemma_without_prefix)
            save_if_changed(page, new_text, f"Link korrigiert von {redirect} zu {self.re_page.page}")

    @staticmethod
    def get_backlinks(redirect_page: pywikibot.Page) -> list[str]:
        return [page.title() for page in redirect_page.backlinks()]

    def filter_link_list(self, link_list: list[str]) -> list[str]:
        return [link for link in link_list if self.filter_regex.search(link) is None]

    @staticmethod
    def replace_redirect_links(text: str, redirect: str, target: str):
        redirect_re = redirect.translate(REGEX_TRANSLATION)
        # [[RE:Redirect]] -> [[RE:Target]], [[RE:Redirect|Something]] -> [[RE:Target|Something]]
        temp_text = re.sub(rf"\[\[RE:{redirect_re}(\||\]\])", rf"[[RE:{target}\g<1>", text)
        # {{RE siehe|Redirect}} -> {{RE siehe|Target|Redirect}}
        temp_text = re.sub(rf"{{RE siehe\|{redirect_re}}}", f"{{RE siehe|{target}|{redirect}}}", temp_text)
        # {{RE siehe|Redirect|Something}} -> {{RE siehe|Target|Something}}
        temp_text = re.sub(rf"{{RE siehe\|{redirect_re}\|(.+?)}}", rf"{{RE siehe|{target}|\g<1>}}", temp_text)
        # VORGÄNGER=Redirect -> VORGÄNGER=Target
        temp_text = re.sub(rf"(VORGÄNGER|NACHFOLGER)={redirect_re}\n", rf"\g<1>={target}\n", temp_text)
        return temp_text
<commit_msg>fix quality<commit_after>import re
import time

import pywikibot

from service.ws_re.scanner.tasks.base_task import ReScannerTask
from tools import save_if_changed
from tools.bots.pi import WikiLogger

REGEX_TRANSLATION = str.maketrans({"(": r"\(",
                                   ")": r"\)",
                                   "?": r"\?"})

class CHRETask(ReScannerTask):
    error_category = "RE:Links führen auf Redirects"

    def __init__(self, wiki: pywikibot.Site, logger: WikiLogger, debug: bool = True):
        ReScannerTask.__init__(self, wiki, logger, debug)
        self.filter_regex = re.compile(
            r"(Benutzer|"
            r"Paulys Realencyclopädie der classischen Altertumswissenschaft/Register/|"
            r"Wikisource:"
            r").*"
        )

    def task(self) -> bool:
        self.iterate_throw_redirects(repair=True)
        self.iterate_throw_redirects(report=True)
        return True

    def iterate_throw_redirects(self, repair: bool = False, report: bool = False):
        for redirect in self.re_page.get_redirects():
            filtered_list = self.filter_link_list(self.get_backlinks(redirect))
            if filtered_list:
                if repair:
                    self.rename_redirects_to_target(filtered_list, redirect)
                    time.sleep(2)
                if report:
                    self.re_page.add_error_category(self.error_category)
                    return
        self.re_page.remove_error_category(self.error_category)

    def rename_redirects_to_target(self, filtered_list: list[str], redirect: pywikibot.Page):
        for entry in filtered_list:
            page = pywikibot.Page(self.wiki, entry)
            new_text = self.replace_redirect_links(page.text, redirect.title()[3:], self.re_page.lemma_without_prefix)
            save_if_changed(page, new_text, f"Link korrigiert von {redirect} zu {self.re_page.page}")

    @staticmethod
    def get_backlinks(redirect_page: pywikibot.Page) -> list[str]:
        return [page.title() for page in redirect_page.backlinks()]

    def filter_link_list(self, link_list: list[str]) -> list[str]:
        return [link for link in link_list if self.filter_regex.search(link) is None]

    @staticmethod
    def replace_redirect_links(text: str, redirect: str, target: str):
        redirect_re = redirect.translate(REGEX_TRANSLATION)
        # [[RE:Redirect]] -> [[RE:Target]], [[RE:Redirect|Something]] -> [[RE:Target|Something]]
        temp_text = re.sub(rf"\[\[RE:{redirect_re}(\||\]\])", rf"[[RE:{target}\g<1>", text)
        # {{RE siehe|Redirect}} -> {{RE siehe|Target|Redirect}}
        temp_text = re.sub(rf"{{RE siehe\|{redirect_re}}}", f"{{RE siehe|{target}|{redirect}}}", temp_text)
        # {{RE siehe|Redirect|Something}} -> {{RE siehe|Target|Something}}
        temp_text = re.sub(rf"{{RE siehe\|{redirect_re}\|(.+?)}}", rf"{{RE siehe|{target}|\g<1>}}", temp_text)
        # VORGÄNGER=Redirect -> VORGÄNGER=Target
        temp_text = re.sub(rf"(VORGÄNGER|NACHFOLGER)={redirect_re}\n", rf"\g<1>={target}\n", temp_text)
        return temp_text
<|endoftext|>