<commit_before>"""backup_verifier.py

MIT license
Copyright (c) 2016 by Doug Mahugh

Tool for analyzing a set of directory listings from backup drives and
identifying discrepancies: the extra, missing, or modified files on each drive.
"""
import csv
import datetime
from pathlib import Path
import subprocess
import sys
import time

def backup_compare(backup, master, report_file):
    """Compare a backup to a master copy, return the differences.

    backup = name of the data file for a backup copy; either a .csv with four columns
             (folder/filename/timestamp/size) or a .dir capture of a Windows
             directory listing
    master = dictionary of the master, keys = filename, values = timestamp+size
    report_file = file handle to open report file for detailed output

    Returns (nmissing, ndiffer, nextra)
    """
    nmissing = 0
    ndiffer = 0
    nextra = 0
    # /// just pass a Path object?
    backup_path = Path(backup)  # create Path object from passed string
    display("analyzing " + backup_path.stem + " ...", report_file, "cn")
    display(">>> " + backup_path.stem.upper() + " <<<", report_file, "f")
    backup_dict = {}  # dictionary of this backup, populated in loop below

    if backup_path.suffix.lower() == ".csv":
        datafile = backup
    else:
        # /// does open() take a Path() instead of string? Should we do that?
        datafile = backup_path.stem + ".csv"
        display("parsing " + backup + " ...", report_file, "cn")
        convert_to_csv(infile=backup, outfile=datafile)

    # scan through this backup and compare each file to master dictionary ...
    for row in csv.reader(open(datafile), delimiter=",", quotechar='"'):
        fullpath = row[0].lower() + "\\" + row[1].lower()
        ts_size = row[2] + row[3]
        backup_dict[fullpath] = ts_size
        if fullpath in master:
            if files_differ(master[fullpath], ts_size):
                display("modified: " + fullpath, report_file, "f")
                ndiffer += 1
        else:
            display("extra: " + fullpath, report_file, "f")
            nextra += 1

    # scan through master_dict to identify any files missing from the backup ...
    for fullpath in master:
        if fullpath not in backup_dict:
            display("missing: " + fullpath, report_file, "f")
            nmissing += 1

    return (nmissing, ndiffer, nextra)

def convert_to_csv(infile=None, outfile=None):
    """Parse a text file containing a Windows directory listing and write the
    data to a CSV file.

    infile = a file captured with a command such as DIR *.* /S >filename.dir
    outfile = name of a CSV file to be written
    """
    if not infile or not outfile:
        return  # nothing to do

    # path = a string indicating the root path of the backup structure, so that
    # this can be removed from all data in the output file. This root path is
    # extracted from the first directory found in the input file below.
    path = None

    # open file, write header row to output file
    fhandle = open(outfile, "w", newline="")
    csvwriter = csv.writer(fhandle, dialect="excel")
    csvwriter.writerow(["folder", "filename", "timestamp", "bytes"])

    lines_written = 0
    current_folder = None
    for line in open(infile, "r"):
        linetext = line.strip()

        folder, filename, timestamp, filesize = parseline(linetext)
        if folder:
            # reset the current folder value, to  process files in this folder
            if not path:
                # path not set yet, so set it now. Note that the approach used
                # here is specific to our setup: a master server with a complete
                # backup stored under c:\backup-master, and a set of USB drives
                # with backups stored in the root of each one.
                if folder.startswith(r"c:\backup-master"):
                    path = r"c:\backup-master"
                else:
                    path = folder[0:2]  # first 2 characters; e.g., 'd:'
            if folder.startswith(path):
                # remove the backup set root path from beginning of folder name
                current_folder = folder[len(path) :]
            else:
                current_folder = folder
            continue

        if not filename:
            continue  # not a filename, so nothing to do

        if excluded_folder(current_folder):
            continue

        # remove \backup-master from start of path ...
        if current_folder.startswith(r"\backup-master"):
            current_folder = current_folder[14:]

        csvwriter.writerow([current_folder, filename, str(timestamp), str(filesize)])

        lines_written += 1
        if lines_written % 10000 == 0:
            display(
                "{0} lines parsed from {1} and written to ".format(
                    lines_written, infile
                )
                + outfile,
                None,
                "cn",
            )

    fhandle.close()
    display("{} lines written to ".format(lines_written) + outfile, None, "cn")

def diff_report(datafiles=None):
    """Generate a difference analysis for a list of data files.

    First entry in the list is the "master" copy for analysis purposes.

    Each file can be either a .csv file or a .dir file containing a Windows
    directory listing, in which case the same-named .csv file is created on the
    fly.

    A full-detail log file is created as a CSV file, and the filename is returned.
    Also prints a brief summary to the console.
    """
    if not datafiles:
        # if no CSV files were specified, use our current defaults
        datafiles = ["master.csv", "drive1.csv", "drive2.csv", "drive3.csv"]

    # open output report file
    report_filename = "backups-" + time.strftime("%Y-%m-%d-%H%M%S") + ".rpt"
    report_file = open(report_filename, "w")
    display("  MASTER: " + datafiles[0], report_file, "f")
    display("  copies: " + str(datafiles[1:]), report_file, "f")
    display("-" * 80, report_file, "f")

    master_path = Path(datafiles[0])
    if master_path.suffix.lower() == '.csv':
        master_data = str(master_path)
    else:
        display("parsing " + datafiles[0] + " ...", None, "cn")
        master_data = str(master_path.with_suffix('.csv'))
        convert_to_csv(str(master_path), master_data)

    # master_dict = a dictionary created from the master backup (first file)
    #    key = folder + r'\' + filename
    #    value = timestamp + filesize
    display("creating dictionary from MASTER COPY ...", report_file, "cn")
    master_dict = {}
    for row in csv.reader(open(master_data, newline=""), delimiter=",", quotechar='"'):
        master_dict[row[0].lower() + "\\" + row[1].lower()] = row[2] + str(row[3])

    masterfilesumm = str(master_path.with_suffix('')) + " -- MASTER COPY ({:,} files)".format(len(master_dict))
    display(masterfilesumm, report_file)
    display("-" * 80, report_file, "f")

    # compare each backup against the master
    for nbackup, filename in enumerate(datafiles):
        if nbackup == 0:
            continue  # skip the master
        nmissing, ndiffer, nextra = backup_compare(filename, master_dict, report_file)
        summary = summary_msg(filename, datafiles[0], nmissing, ndiffer, nextra)
        display(summary, report_file)
        display("-" * 80, report_file, "f")

    report_file.close()

    return report_filename

def display(message, report_file, flags="cf"):
    """Display a message and/or write it to report file

    message = the text message
    report_file = file handle of open report file
    flags = contains 'c' for console output
            contains 'n' to suppress newline on console output
            contains 'f' for file output
            default is 'cf'
    """
    flags = flags.lower() if flags else "cf"
    if len(message) < 80:
        message = message.ljust(80)
    if "c" in flags:
        if "n" in flags:
            print("\r" + message, end="")
        else:
            print("\r" + message)
    if "f" in flags:
        report_file.write(message + "\n")

def excluded_folder(folder=None):
    """Determine whether a folder is to be excluded from the output CSV file.

    folder = a folder name
    Returns True if its contents should be excluded, false if not.
    """
    if not folder:
        return True

    if folder.endswith("__pycache__"):
        return True
    if folder.endswith("\\.git"):
        return True
    if "\\.git\\" in folder:
        return True
    if "\\$RECYCLE.BIN\\" in folder:
        return True

    return False

def files_differ(file1, file2):
    """Determine whether two files (which are expected to be identical copies
    of the same file on different drives) have differences in ther timestamps
    or sizes.

    Inputs: file1 and file2 are strings that encode a file's timestamp and size
    as captured in DIR listings, in this format (where NNNNNNNNN is the file
    size): "YYYY-MM-DD HH:MM:SSNNNNNNNNN"

    Output: returns True if the files differ, False if they have same timestamp/size.

    NOTE: we now only check the minutes and seconds of the timestamp as well as
    the file size, because we've found that if each file's DIR listing was done
    during a Daylight Savings Time setting, they'll have different hours, which
    can roll over into different days, months, or years, even though the files
    are in fact identical.

    NOTE #2, which overrides the above note: fuck it, we now only verify the
    size and entirely ignore the timestamp. It seems that Windows at some point
    changed how it rounds off seconds in a DIR listing so that you can have a
    meaningless 1-minute difference in the timestamp depending on which version
    of Windows did the DIR listing - for example, 04:53:59 used to print in a
    DIR listing as 04:53 but now it prints as 04:54, and honestly WHO HAS TIME
    FOR THIS SHIT?
    """
    return file1[19:] != file2[19:]

def parseline(linetext=None):
    """Parse a line from a directory listing, and return a tuple containing the
    folder name (if it's a Directory line), filename, timestamp, and filesize.
    """
    novalues = (None, None, None, None)
    if not linetext:
        return novalues

    # some lines are just noise, so don't parse them
    if (
        " <DIR> " in linetext
        or "File(s)" in linetext
        or linetext.endswith("bytes free")
        or linetext.startswith("Volume ")
        or linetext.startswith("Total Files")
    ):
        return novalues

    if linetext.startswith("Directory of "):
        return (linetext[13:], None, None, None)

    filename = linetext[39:]
    timestamp = ts_to_datetime(linetext)
    filesize = int(linetext[20:38].strip().replace(",", ""))
    return (None, filename, timestamp, filesize)

def summary_msg(backup, master, nmissing, ndiffer, nextra):
    """Create a 1-liner summary message for a backup comparison.

    backup = filename of the backup copy
    master = filename of the master copy that the backup was compared with
    nmissing = # of files in master that are missing from backup
    ndiffer = # of files that have different timestamp or size from master
    nextra = # of files in backup that are not in master

    Returns a string that describes/summarizes the results of the comparison.
    """
    backup_path = Path(backup)

    clauses = []
    if nmissing > 0:
        clauses.append(
            "{0} missing file".format(nmissing) + ("s" if nmissing > 1 else "")
        )
    if ndiffer > 0:
        clauses.append("{0} different timestamp/size".format(ndiffer))
    if nextra > 0:
        clauses.append("{0} extra file".format(nextra) + ("s" if nextra > 1 else ""))

    if nmissing == 0 and ndiffer == 0 and nextra == 0:
        return (
            str(backup_path.with_suffix('')) + " -- clean backup, all files match " + master
        )

    return str(backup_path.with_suffix('')) + " -- " + ", ".join(clauses)

def ts_to_datetime(linetext):
    """Convert a directory listing timestamp (as displayed by Windows DIR, for
    example "12/08/2016  02:33 PM") to a datetime value.

    NOTE: these values don't include the seconds portion of the timestamp.
    This is not a problem for the original use of this function, but could be
    an issue if this code is used elsewhere.
    """
    windows_timestamp = linetext[:20]
    return datetime.datetime.strptime(windows_timestamp, "%m/%d/%Y %I:%M %p")

def test_backup_verifier():
    """Run tests
    """

    test_cases = [
        ("testdata\\folder1", "folder1.dir"),
        ("testdata\\folder2", "folder2.dir"),
        ("testdata\\folder3", "folder3.dir"),
    ]

    # capture DIR listings to CSV files
    for folder, filename in test_cases:
        with open(filename, "w") as fhandle:
            fhandle.write(subprocess.getoutput(f"dir {folder}"))

    # generate diff report
    output = diff_report([test_case[1] for test_case in test_cases])

    # compare generated output to expected output
    with open("testdata\\expected_output.txt") as fhandle:
        expected = fhandle.read()
    with open(output) as fhandle:
        actual = fhandle.read()
    print("TESTS PASSED" if actual == expected else "TESTS FAILED")

if __name__ == "__main__":

    # switch console output to utf8 encoding, so that we don't crash on
    # display of filenames with non-ASCII characters. (The need for this should go
    # away with new CMD prompt updates coming in Windows 10.)
    sys.stdout = open(sys.stdout.fileno(), mode="w", encoding="utf8", buffering=1)

    #test_backup_verifier()

    # generate a diference report for a set of .CSV or .DIR files passed as
    # command line arguments
    # diff_report(sys.argv[1:])

    # LIVE USAGE - for verifying our backup drives
    diff_report(["drive5-2019-11-23.csv", "archive/drive1-2019-06-08.csv", "archive/drive2-2019-06-08.csv", "archive/drive3-2019-06-08.csv", "archive/drive4-2019-06-08.csv"])
    #diff_report(["drive1.dir", "drive2.dir", "drive3.dir", "drive4.dir"])
    # diff_report(['server.csv', 'drive2.csv', 'drive3.csv', 'drive4.csv'])

    # this enables command-line usage for converting a .dir to .csv
    # if len(sys.argv) == 3:
    #    INPUT_FILE = sys.argv[1]
    #    OUTPUT_FILE = sys.argv[2]
    #    print('Input file: ' + INPUT_FILE)
    #    print('Output file: ' + OUTPUT_FILE)
    #    convert_to_csv(infile=INPUT_FILE, outfile=OUTPUT_FILE)
    # else:
    #    print('Wrong number of arguments. Syntax = '+ \
    #        'python backup_verifier.py infile.dir outfile.csv')
<commit_msg>exclude \System Volume Information<commit_after>"""backup_verifier.py

MIT license
Copyright (c) 2016 by Doug Mahugh

Tool for analyzing a set of directory listings from backup drives and
identifying discrepancies: the extra, missing, or modified files on each drive.
"""
import csv
import datetime
from pathlib import Path
import subprocess
import sys
import time

def backup_compare(backup, master, report_file):
    """Compare a backup to a master copy, return the differences.

    backup = name of the data file for a backup copy; either a .csv with four columns
             (folder/filename/timestamp/size) or a .dir capture of a Windows
             directory listing
    master = dictionary of the master, keys = filename, values = timestamp+size
    report_file = file handle to open report file for detailed output

    Returns (nmissing, ndiffer, nextra)
    """
    nmissing = 0
    ndiffer = 0
    nextra = 0
    # /// just pass a Path object?
    backup_path = Path(backup)  # create Path object from passed string
    display("analyzing " + backup_path.stem + " ...", report_file, "cn")
    display(">>> " + backup_path.stem.upper() + " <<<", report_file, "f")
    backup_dict = {}  # dictionary of this backup, populated in loop below

    if backup_path.suffix.lower() == ".csv":
        datafile = backup
    else:
        # /// does open() take a Path() instead of string? Should we do that?
        datafile = backup_path.stem + ".csv"
        display("parsing " + backup + " ...", report_file, "cn")
        convert_to_csv(infile=backup, outfile=datafile)

    # scan through this backup and compare each file to master dictionary ...
    for row in csv.reader(open(datafile), delimiter=",", quotechar='"'):
        fullpath = row[0].lower() + "\\" + row[1].lower()
        ts_size = row[2] + row[3]
        backup_dict[fullpath] = ts_size
        if fullpath in master:
            if files_differ(master[fullpath], ts_size):
                display("modified: " + fullpath, report_file, "f")
                ndiffer += 1
        else:
            display("extra: " + fullpath, report_file, "f")
            nextra += 1

    # scan through master_dict to identify any files missing from the backup ...
    for fullpath in master:
        if fullpath not in backup_dict:
            display("missing: " + fullpath, report_file, "f")
            nmissing += 1

    return (nmissing, ndiffer, nextra)

def convert_to_csv(infile=None, outfile=None):
    """Parse a text file containing a Windows directory listing and write the
    data to a CSV file.

    infile = a file captured with a command such as DIR *.* /S >filename.dir
    outfile = name of a CSV file to be written
    """
    if not infile or not outfile:
        return  # nothing to do

    # path = a string indicating the root path of the backup structure, so that
    # this can be removed from all data in the output file. This root path is
    # extracted from the first directory found in the input file below.
    path = None

    # open file, write header row to output file
    fhandle = open(outfile, "w", newline="")
    csvwriter = csv.writer(fhandle, dialect="excel")
    csvwriter.writerow(["folder", "filename", "timestamp", "bytes"])

    lines_written = 0
    current_folder = None
    for line in open(infile, "r"):
        linetext = line.strip()

        folder, filename, timestamp, filesize = parseline(linetext)
        if folder:
            # reset the current folder value, to  process files in this folder
            if not path:
                # path not set yet, so set it now. Note that the approach used
                # here is specific to our setup: a master server with a complete
                # backup stored under c:\backup-master, and a set of USB drives
                # with backups stored in the root of each one.
                if folder.startswith(r"c:\backup-master"):
                    path = r"c:\backup-master"
                else:
                    path = folder[0:2]  # first 2 characters; e.g., 'd:'
            if folder.startswith(path):
                # remove the backup set root path from beginning of folder name
                current_folder = folder[len(path) :]
            else:
                current_folder = folder
            continue

        if not filename:
            continue  # not a filename, so nothing to do

        if excluded_folder(current_folder):
            continue

        # remove \backup-master from start of path ...
        if current_folder.startswith(r"\backup-master"):
            current_folder = current_folder[14:]

        csvwriter.writerow([current_folder, filename, str(timestamp), str(filesize)])

        lines_written += 1
        if lines_written % 10000 == 0:
            display(
                "{0} lines parsed from {1} and written to ".format(
                    lines_written, infile
                )
                + outfile,
                None,
                "cn",
            )

    fhandle.close()
    display("{} lines written to ".format(lines_written) + outfile, None, "cn")

def diff_report(datafiles=None):
    """Generate a difference analysis for a list of data files.

    First entry in the list is the "master" copy for analysis purposes.

    Each file can be either a .csv file or a .dir file containing a Windows
    directory listing, in which case the same-named .csv file is created on the
    fly.

    A full-detail log file is created as a CSV file, and the filename is returned.
    Also prints a brief summary to the console.
    """
    if not datafiles:
        # if no CSV files were specified, use our current defaults
        datafiles = ["master.csv", "drive1.csv", "drive2.csv", "drive3.csv"]

    # open output report file
    report_filename = "backups-" + time.strftime("%Y-%m-%d-%H%M%S") + ".rpt"
    report_file = open(report_filename, "w")
    display("  MASTER: " + datafiles[0], report_file, "f")
    display("  copies: " + str(datafiles[1:]), report_file, "f")
    display("-" * 80, report_file, "f")

    master_path = Path(datafiles[0])
    if master_path.suffix.lower() == '.csv':
        master_data = str(master_path)
    else:
        display("parsing " + datafiles[0] + " ...", None, "cn")
        master_data = str(master_path.with_suffix('.csv'))
        convert_to_csv(str(master_path), master_data)

    # master_dict = a dictionary created from the master backup (first file)
    #    key = folder + r'\' + filename
    #    value = timestamp + filesize
    display("creating dictionary from MASTER COPY ...", report_file, "cn")
    master_dict = {}
    for row in csv.reader(open(master_data, newline=""), delimiter=",", quotechar='"'):
        master_dict[row[0].lower() + "\\" + row[1].lower()] = row[2] + str(row[3])

    masterfilesumm = str(master_path.with_suffix('')) + " -- MASTER COPY ({:,} files)".format(len(master_dict))
    display(masterfilesumm, report_file)
    display("-" * 80, report_file, "f")

    # compare each backup against the master
    for nbackup, filename in enumerate(datafiles):
        if nbackup == 0:
            continue  # skip the master
        nmissing, ndiffer, nextra = backup_compare(filename, master_dict, report_file)
        summary = summary_msg(filename, datafiles[0], nmissing, ndiffer, nextra)
        display(summary, report_file)
        display("-" * 80, report_file, "f")

    report_file.close()

    return report_filename

def display(message, report_file, flags="cf"):
    """Display a message and/or write it to report file

    message = the text message
    report_file = file handle of open report file
    flags = contains 'c' for console output
            contains 'n' to suppress newline on console output
            contains 'f' for file output
            default is 'cf'
    """
    flags = flags.lower() if flags else "cf"
    if len(message) < 80:
        message = message.ljust(80)
    if "c" in flags:
        if "n" in flags:
            print("\r" + message, end="")
        else:
            print("\r" + message)
    if "f" in flags:
        report_file.write(message + "\n")

def excluded_folder(folder=None):
    """Determine whether a folder is to be excluded from the output CSV file.

    folder = a folder name
    Returns True if its contents should be excluded, false if not.
    """
    if not folder:
        return True

    if folder.endswith("__pycache__"):
        return True
    if folder.endswith("\\.git"):
        return True
    if "\\.git\\" in folder:
        return True
    if "\\$RECYCLE.BIN\\" in folder:
        return True
    if folder.lower().startswith("\\system volume information"):
        # This folder is always present on USB drives, but NTFS hides it and
        # other file systems may show it, which can make it appear to be a
        # "difference" between two drives with identical content.
        return True

    return False

def files_differ(file1, file2):
    """Determine whether two files (which are expected to be identical copies
    of the same file on different drives) have differences in ther timestamps
    or sizes.

    Inputs: file1 and file2 are strings that encode a file's timestamp and size
    as captured in DIR listings, in this format (where NNNNNNNNN is the file
    size): "YYYY-MM-DD HH:MM:SSNNNNNNNNN"

    Output: returns True if the files differ, False if they have same timestamp/size.

    NOTE: we now only check the minutes and seconds of the timestamp as well as
    the file size, because we've found that if each file's DIR listing was done
    during a different Daylight Savings Time setting, they'll have different
    hours, which can roll over into different days, months, or years, even
    though the files are in fact identical.

    NOTE #2, which overrides the above note: fuck it, we now only verify the
    size and entirely ignore the timestamp. It seems that Windows at some point
    changed how it rounds off seconds in a DIR listing so that you can have a
    meaningless 1-minute difference in the timestamp depending on which version
    of Windows did the DIR listing - for example, 04:53:59 used to print in a
    DIR listing as 04:53 but now it prints as 04:54, and honestly WHO HAS TIME
    FOR THIS SHIT?
    """
    return file1[19:] != file2[19:]

def parseline(linetext=None):
    """Parse a line from a directory listing, and return a tuple containing the
    folder name (if it's a Directory line), filename, timestamp, and filesize.
    """
    novalues = (None, None, None, None)
    if not linetext:
        return novalues

    # some lines are just noise, so don't parse them
    if (
        " <DIR> " in linetext
        or "File(s)" in linetext
        or linetext.endswith("bytes free")
        or linetext.startswith("Volume ")
        or linetext.startswith("Total Files")
    ):
        return novalues

    if linetext.startswith("Directory of "):
        return (linetext[13:], None, None, None)

    filename = linetext[39:]
    timestamp = ts_to_datetime(linetext)
    filesize = int(linetext[20:38].strip().replace(",", ""))
    return (None, filename, timestamp, filesize)

def summary_msg(backup, master, nmissing, ndiffer, nextra):
    """Create a 1-liner summary message for a backup comparison.

    backup = filename of the backup copy
    master = filename of the master copy that the backup was compared with
    nmissing = # of files in master that are missing from backup
    ndiffer = # of files that have different timestamp or size from master
    nextra = # of files in backup that are not in master

    Returns a string that describes/summarizes the results of the comparison.
    """
    backup_path = Path(backup)

    clauses = []
    if nmissing > 0:
        clauses.append(
            "{0} missing file".format(nmissing) + ("s" if nmissing > 1 else "")
        )
    if ndiffer > 0:
        clauses.append("{0} different timestamp/size".format(ndiffer))
    if nextra > 0:
        clauses.append("{0} extra file".format(nextra) + ("s" if nextra > 1 else ""))

    if nmissing == 0 and ndiffer == 0 and nextra == 0:
        return (
            str(backup_path.with_suffix('')) + " -- clean backup, all files match " + master
        )

    return str(backup_path.with_suffix('')) + " -- " + ", ".join(clauses)

def ts_to_datetime(linetext):
    """Convert a directory listing timestamp (as displayed by Windows DIR, for
    example "12/08/2016  02:33 PM") to a datetime value.

    NOTE: these values don't include the seconds portion of the timestamp.
    This is not a problem for the original use of this function, but could be
    an issue if this code is used elsewhere.
    """
    windows_timestamp = linetext[:20]
    return datetime.datetime.strptime(windows_timestamp, "%m/%d/%Y %I:%M %p")

def test_backup_verifier():
    """Run tests
    """

    test_cases = [
        ("testdata\\folder1", "folder1.dir"),
        ("testdata\\folder2", "folder2.dir"),
        ("testdata\\folder3", "folder3.dir"),
    ]

    # capture DIR listings to CSV files
    for folder, filename in test_cases:
        with open(filename, "w") as fhandle:
            fhandle.write(subprocess.getoutput(f"dir {folder}"))

    # generate diff report
    output = diff_report([test_case[1] for test_case in test_cases])

    # compare generated output to expected output
    with open("testdata\\expected_output.txt") as fhandle:
        expected = fhandle.read()
    with open(output) as fhandle:
        actual = fhandle.read()
    print("TESTS PASSED" if actual == expected else "TESTS FAILED")

if __name__ == "__main__":

    # switch console output to utf8 encoding, so that we don't crash on
    # display of filenames with non-ASCII characters. (The need for this should go
    # away with new CMD prompt updates coming in Windows 10.)
    sys.stdout = open(sys.stdout.fileno(), mode="w", encoding="utf8", buffering=1)

    #test_backup_verifier()

    # generate a diference report for a set of .CSV or .DIR files passed as
    # command line arguments
    # diff_report(sys.argv[1:])

    # LIVE USAGE - for verifying our backup drives
    diff_report(["drive5-2019-11-24.dir", "archive/drive1-2019-06-08.csv", "archive/drive2-2019-06-08.csv", "archive/drive3-2019-06-08.csv", "archive/drive4-2019-06-08.csv"])
    #diff_report(["drive1.dir", "drive2.dir", "drive3.dir", "drive4.dir"])
    # diff_report(['server.csv', 'drive2.csv', 'drive3.csv', 'drive4.csv'])

    # this enables command-line usage for converting a .dir to .csv
    # if len(sys.argv) == 3:
    #    INPUT_FILE = sys.argv[1]
    #    OUTPUT_FILE = sys.argv[2]
    #    print('Input file: ' + INPUT_FILE)
    #    print('Output file: ' + OUTPUT_FILE)
    #    convert_to_csv(infile=INPUT_FILE, outfile=OUTPUT_FILE)
    # else:
    #    print('Wrong number of arguments. Syntax = '+ \
    #        'python backup_verifier.py infile.dir outfile.csv')
<|endoftext|>


<commit_before>"""data.py
   author: Gerry LaMontagne
   date: 04/23/2013

    Contains data structures to insulate the application code from the details
    of the various data structures used by the XMLRPC methods.
"""
from xmlrpclib import DateTime
import time

################################################################################
"""DataError

    Class for data exceptions
"""
class DataError(Exception):

    def __init__(self, errmsg):
        self._message = "DataError Exception:  %s" % errmsg

    def __str__(self):
        return self.message

################################################################################
"""post

    An kind of container class for post structures.  It serves to provide a
    consistent interface between the application and the actual data.

    Can be called with no initialization parms, in which case it returns an
    empty object, otherwise it contains the data structure which will be
    accessible through instance attributes.

    Objects that are initialized as non-native types are essentially read-only.
    The native type objects can hae their attibutes modified.
"""
class Post(object):

    _supported_types = ('native', 'wp', 'metaweblog')
    _wp_map = { 'id'         : 'post_id',
                'title'      : 'post_title',
                'dategmt'    : 'post_date_gmt',
                'status'     : 'post_status',
                'posttype'   : 'post_type',
                'author'     : 'post_author',
                'excerpt'    : 'post_excerpt',
                'comments'   : 'comment_status',
                'ping'       : 'ping_status',
                'content'    : 'post_content', }

    _metaweblog_map = { 'id'         : 'postid',
                        'title'      : 'title',
                        'dategmt'    : 'date_created_gmt',
                        'status'     : 'post_status',
                        'posttype'   : 'post_type',
                        'author'     : 'wp_author_id',
                        'excerpt'    : 'mt_excerpt',
                        'comments'   : 'mt_allow_comments',
                        'ping'       : 'mt_allow_pings',
                        'content'    : 'description',
                        'publish'    : 'publish',
                        'categories' : 'categories',
                        'tags'       : 'mt_keywords', }

    def __init__(self, data = None, type = 'native'):

        if data != None:
            if type not in self._supported_types:
                raise DataError("Post object type '%s' not supported." % type)
        self._data = data
        self._type = type

    ############################################################################
    """_getter

        Returns a closure of a function designed to return a public attribute
        value.  The closure is made based on the attribute name, which will be
        returned from either a properly interpreted internal dict called
        ``_data`` or will be returned from an internl version of the attribute
        which has an identical name prepended with an ``_`` character.

        It is meant to be used to on attributes decorated with the ``property``
        decorator so the attributes don't look like methods to the caller.

        ``attr``:: name of the attribute to close the ``_get`` function over.
                   This is the public name of the attribute.

        ``returns``: closure of ``_get`` function.
    """
    @staticmethod
    def _getter(attr):
        def _get(self):
            def native():
                return getattr(self, "_%s" % attr)
            def wp():
                return self._data[self._wp_map[attr]]
            def metaweblog():
                return self._data[self._metaweblog_map[attr]]

            return { 'native'     : native,
                     'wp'         : wp,
                     'metaweblog' : metaweblog, }[self._type]()

        return _get

    ############################################################################
    """_setter

        Creates a closure of a function designed to set an internal attribute
        value that maps to a public value.  **Only** internal attribute setting
        is supported.  If the object is not of type ``native`` then a
        ``ValueError`` exception is raised when trying to set a value.

        ``attr``:: name of attribute to set.  Note that an internal version of
                   this name will be created.  The actual name of the attribute
                   is a method disguised as an attribute by the property
                   decorator.

        ``returns``:: closure of ``_set`` function over the attribute name
    """
    @staticmethod
    def _setter(attr):
        def _set(self, value):
            if self._type != 'native':
                raise ValueError
            setattr(self, "_%s" % attr, value)
        return _set

    ################################################################################
    """_convertTime

        method that attempts to convert a date time string to a datetime object
        in UTC time.  Defaults to assuming string is a local time representation.
    """
    def _convertTime(self, timestr):
        # List of formats to attempt to match up.
        time_fmts = [
                      "%Y%m%dT%H:%M",        #YYYYMMDDThh:mm
                      "%Y%m%dT%I:%M%p",      #YYYYMMDDThh:mmAM/PM
                      "%Y%m%dT%H:%M:%S",     #YYYYMMDDThh:mm:ss
                      "%Y%m%dT%I:%M:%S%p",   #YYYYMMDDThh:mm:ssAM/PM
                      "%b %d, %Y %H:%M",     #Month Day, Year hour:min
                      "%b %d, %Y %I:%M%p",   #Month Day, Year hour:min AM/PM
                      "%m/%d/%Y %H:%M",      #MM/DD/YYYY hh:mm
                      "%m/%d/%Y %I:%M%p",    #MM/DD/YYYY hh:mmAM/PM
                      "%H:%M %m/%d/%Y",      #hh:mm MM/DD/YYYY
                      "%I:%M%p %m/%d/%Y",    #hh:mmAM/PM MM/DD/YYYY
                    ]

        # the timestamp is provided as "local time" so we need to convert it to
        # UTC time- do this by converting timestamp to seconds from epoch, then
        # to UTC time.  Finally, pass it to xmlrpclib for formatting
        for tf in time_fmts:
            try:
                timeStruct = time.strptime(timestr, tf)
                utctime = time.gmtime(time.mktime(timeStruct))
                posttime = time.strftime("%Y%m%dT%H:%M:%SZ", utctime)

                # the following merely makes the string into a xmlrpc datetime
                # object
                return DateTime(posttime)

            except ValueError:
                continue
        else:
            # the time format could not be parsed properly
            raise DataError("Unable to parse timestring: %s" % timestamp)

    ############################################################################
    """ Beginning of public API... """

    ############################################################################
    """metaweblogStruct

        Returns a metaweblog post structure suitable for use with the metaWeblog
        XMLRPC methods.  It is implemented as a property.  It raises a DataError
        if the object is not of type ``native``- we aren't trying to convert
        between different post structure types.
    """
    @property
    def metaweblogStruct(self):
        if self._type != 'native':
            raise DataError("Can't build metaweblog post structure, post object not 'native'.")
        post = { v: getattr(self, k) for k,v in self._metaweblog_map.iteritems()
                                     if hasattr(self,k) }
        return post

    ############################################################################
    """wpStruct

        Returns a Wordpress post structure suitable for use with the Wordpress
        XMLRPC methods.  It is implemented as a property.  A ``DataError`` is
        raised if the object is not of type ``native``.
    """
    @property
    def wpStruct(self):
        if self._type != 'native':
            raise DataError("Can't build Wordpress post structure, post object not 'native'.")
        post = { v: getattr(self, k) for k,v in self._wp_map.iteritems()
                                     if hasattr(self,k) }
        post['terms_names'] = { 'category' : self._categories }
        if len(self._tags) != 0:
            post['terms_names']['post_tag'] = self._tags
        return post

    ############################################################################
    """id
    """
    @property
    def id(self):
        return self._getter('id')(self)

    @id.setter
    def id(self, value):
        self._setter('id')(self, value)

    ############################################################################
    """title
    """
    @property
    def title(self):
        return self._getter('title')(self)

    @title.setter
    def title(self, value):
        self._setter('title')(self, value)

    ############################################################################
    """dategmt
    """
    @property
    def dategmt(self):
        return self._getter('dategmt')(self)

    @dategmt.setter
    def dategmt(self, value):
        self._setter('dategmt')(self, self._convertTime(value))

    ############################################################################
    """status
    """
    @property
    def status(self):
        return self._getter('status')(self)

    @status.setter
    def status(self, value):
        self._setter('status')(self, value)

    ############################################################################
    """posttype
    """
    @property
    def posttype(self):
        return self._getter('posttype')(self)

    @posttype.setter
    def posttype(self, value):
        self._setter('posttype')(self, value)

    ############################################################################
    """author
    """
    @property
    def author(self):
        return self._getter('author')(self)

    @author.setter
    def author(self, value):
        self._setter('author')(self, value)

    ############################################################################
    """excerpt
    """
    @property
    def excerpt(self):
        return self._getter('excerpt')(self)

    @excerpt.setter
    def excerpt(self, value):
        self._setter('excerpt')(self, value)

    ############################################################################
    """comments
    """
    @property
    def comments(self):
        return self._getter('comments')(self)

    @comments.setter
    def comments(self, value):
        self._setter('comments')(self, value)

    ############################################################################
    """ping
    """
    @property
    def ping(self):
        return self._getter('ping')(self)

    @ping.setter
    def ping(self, value):
        self._setter('ping')(self, value)

    ############################################################################
    """publish
    """
    @property
    def publish(self):
        def native():
            return self._publish
        def wp():
            if self._data['post_status'] == 'publish':
                return 1
            else:
                return 0
        def metaweblog():
            return self._data['publish']

        return { 'native'     : native,
                 'wp'         : wp,
                 'metaweblog' : metaweblog, }[self._type]()

    @publish.setter
    def publish(self, value):
        def native(v):
            self._publish = v
        def wp(v):
            if v == 1:
                self._data['post_status'] = 'publish'
            else:
                self._data['post_status'] = 'draft'
        def metaweblog(v):
            self._data['publish'] = v

        { 'native'     : native,
          'wp'         : wp,
          'metaweblog' : metaweblog, }[self._type](value)

    ############################################################################
    """content
    """
    @property
    def content(self):
        def native():
            return self._content

        def wp():
            return self._data['post_content']

        def metaweblog():
            if self._data['mt_text_more']:
                if self._data['wp_more_text']:
                    more = "more %s" % self._data['wp_more_text']
                else:
                    more = "more"
                return "%s<!--%s-->%s" % (self._data['description'],
                                          more,
                                          self._data['mt_text_more'])
            else:
                return self._data['description']

        return { 'native'     : native,
                 'wp'         : wp,
                 'metaweblog' : metaweblog, }[self._type]()

    @content.setter
    def content(self, value):
        self._setter('content')(self, value)

    ############################################################################
    """categories
    """
    @property
    def categories(self):
        def native():
            return self._categories

        def wp():
            cats = []
            for term in self._data['terms']:
                if term['taxonomy'] == 'category':
                    cats.append(term['name'])
            return cats

        def metaweblog():
            return self._data['categories']

        return { 'native'     : native,
                 'wp'         : wp,
                 'metaweblog' : metaweblog, }[self._type]()

    @categories.setter
    def categories(self, value):
        self._setter('categories')(self, value)

    ############################################################################
    """tags
    """
    @property
    def tags(self):
        def native():
            return self._tags

        def wp():
            tags = ''
            for term in self._data['terms']:
                if term['taxonomy'] == 'post_tag':
                    if tags != '':
                        tags += ', '
                    tags += term['name']
            return tags

        def metaweblog():
            return self._data['mt_keywords']

        return { 'native'     : native,
                 'wp'         : wp,
                 'metaweblog' : metaweblog, }[self._type]()

    @tags.setter
    def tags(self, value):
        self._setter('tags')(self, value)
<commit_msg>Made convertTime a staticmethod<commit_after>"""data.py
   author: Gerry LaMontagne
   date: 04/23/2013

    Contains data structures to insulate the application code from the details
    of the various data structures used by the XMLRPC methods.
"""
from xmlrpclib import DateTime
import time

################################################################################
"""DataError

    Class for data exceptions
"""
class DataError(Exception):

    def __init__(self, errmsg):
        self._message = "DataError Exception:  %s" % errmsg

    def __str__(self):
        return self.message

################################################################################
"""Post

    An kind of container class for post structures.  It serves to provide a
    consistent interface between the application and the actual data.

    Can be called with no initialization parms, in which case it returns an
    empty object, otherwise it contains the data structure which will be
    accessible through instance attributes.

    Objects that are initialized as non-native types are essentially read-only.
    The native type objects can hae their attibutes modified.
"""
class Post(object):

    _supported_types = ('native', 'wp', 'metaweblog')
    _wp_map = { 'id'         : 'post_id',
                'title'      : 'post_title',
                'dategmt'    : 'post_date_gmt',
                'status'     : 'post_status',
                'posttype'   : 'post_type',
                'author'     : 'post_author',
                'excerpt'    : 'post_excerpt',
                'comments'   : 'comment_status',
                'ping'       : 'ping_status',
                'content'    : 'post_content', }

    _metaweblog_map = { 'id'         : 'postid',
                        'title'      : 'title',
                        'dategmt'    : 'date_created_gmt',
                        'status'     : 'post_status',
                        'posttype'   : 'post_type',
                        'author'     : 'wp_author_id',
                        'excerpt'    : 'mt_excerpt',
                        'comments'   : 'mt_allow_comments',
                        'ping'       : 'mt_allow_pings',
                        'content'    : 'description',
                        'publish'    : 'publish',
                        'categories' : 'categories',
                        'tags'       : 'mt_keywords', }

    def __init__(self, data = None, type = 'native'):

        if data != None:
            if type not in self._supported_types:
                raise DataError("Post object type '%s' not supported." % type)
        self._data = data
        self._type = type

    ############################################################################
    """_getter

        Returns a closure of a function designed to return a public attribute
        value.  The closure is made based on the attribute name, which will be
        returned from either a properly interpreted internal dict called
        ``_data`` or will be returned from an internl version of the attribute
        which has an identical name prepended with an ``_`` character.

        It is meant to be used to on attributes decorated with the ``property``
        decorator so the attributes don't look like methods to the caller.

        ``attr``:: name of the attribute to close the ``_get`` function over.
                   This is the public name of the attribute.

        ``returns``: closure of ``_get`` function.
    """
    @staticmethod
    def _getter(attr):
        def _get(self):
            def native():
                return getattr(self, "_%s" % attr)
            def wp():
                return self._data[self._wp_map[attr]]
            def metaweblog():
                return self._data[self._metaweblog_map[attr]]

            return { 'native'     : native,
                     'wp'         : wp,
                     'metaweblog' : metaweblog, }[self._type]()

        return _get

    ############################################################################
    """_setter

        Creates a closure of a function designed to set an internal attribute
        value that maps to a public value.  **Only** internal attribute setting
        is supported.  If the object is not of type ``native`` then a
        ``ValueError`` exception is raised when trying to set a value.

        ``attr``:: name of attribute to set.  Note that an internal version of
                   this name will be created.  The actual name of the attribute
                   is a method disguised as an attribute by the property
                   decorator.

        ``returns``:: closure of ``_set`` function over the attribute name
    """
    @staticmethod
    def _setter(attr):
        def _set(self, value):
            if self._type != 'native':
                raise ValueError
            setattr(self, "_%s" % attr, value)
        return _set

    ################################################################################
    """_convertTime

        method that attempts to convert a date time string to a datetime object
        in UTC time.  Defaults to assuming string is a local time representation.

        It's a staticmethod because it doesn't affect a `Post` instance, ie it
        has no side-effects.  This means it can also be called directly from the
        class.

       ``timestr``:: a string object that specifies a time like
                     "8:00AM 12/31/2002"

       ``returns``:: an XMLRPC DateTime object for the time string.
                     Raises ``DataError`` if unable to convert ``timestr``
    """
    @staticmethod
    def _convertTime(timestr):
        # List of formats to attempt to match up.
        time_fmts = [
                      "%Y%m%dT%H:%M",        #YYYYMMDDThh:mm
                      "%Y%m%dT%I:%M%p",      #YYYYMMDDThh:mmAM/PM
                      "%Y%m%dT%H:%M:%S",     #YYYYMMDDThh:mm:ss
                      "%Y%m%dT%I:%M:%S%p",   #YYYYMMDDThh:mm:ssAM/PM
                      "%b %d, %Y %H:%M",     #Month Day, Year hour:min
                      "%b %d, %Y %I:%M%p",   #Month Day, Year hour:min AM/PM
                      "%m/%d/%Y %H:%M",      #MM/DD/YYYY hh:mm
                      "%m/%d/%Y %I:%M%p",    #MM/DD/YYYY hh:mmAM/PM
                      "%H:%M %m/%d/%Y",      #hh:mm MM/DD/YYYY
                      "%I:%M%p %m/%d/%Y",    #hh:mmAM/PM MM/DD/YYYY
                    ]

        # the timestamp is provided as "local time" so we need to convert it to
        # UTC time- do this by converting timestamp to seconds from epoch, then
        # to UTC time.  Finally, pass it to xmlrpclib for formatting
        for tf in time_fmts:
            try:
                timeStruct = time.strptime(timestr, tf)
                utctime = time.gmtime(time.mktime(timeStruct))
                posttime = time.strftime("%Y%m%dT%H:%M:%SZ", utctime)

                # the following merely makes the string into a xmlrpc datetime
                # object
                return DateTime(posttime)

            except ValueError:
                continue
        else:
            # the time format could not be parsed properly
            raise DataError("Unable to parse timestring: %s" % timestamp)

    ############################################################################
    """ Beginning of public API... """

    ############################################################################
    """metaweblogStruct

        Returns a metaweblog post structure suitable for use with the metaWeblog
        XMLRPC methods.  It is implemented as a property.  It raises a DataError
        if the object is not of type ``native``- we aren't trying to convert
        between different post structure types.
    """
    @property
    def metaweblogStruct(self):
        if self._type != 'native':
            raise DataError("Can't build metaweblog post structure, post object not 'native'.")
        post = { v: getattr(self, k) for k,v in self._metaweblog_map.iteritems()
                                     if hasattr(self,k) }
        return post

    ############################################################################
    """wpStruct

        Returns a Wordpress post structure suitable for use with the Wordpress
        XMLRPC methods.  It is implemented as a property.  A ``DataError`` is
        raised if the object is not of type ``native``.
    """
    @property
    def wpStruct(self):
        if self._type != 'native':
            raise DataError("Can't build Wordpress post structure, post object not 'native'.")
        post = { v: getattr(self, k) for k,v in self._wp_map.iteritems()
                                     if hasattr(self,k) }
        post['terms_names'] = { 'category' : self._categories }
        if len(self._tags) != 0:
            post['terms_names']['post_tag'] = self._tags
        return post

    ############################################################################
    """id
    """
    @property
    def id(self):
        return self._getter('id')(self)

    @id.setter
    def id(self, value):
        self._setter('id')(self, value)

    ############################################################################
    """title
    """
    @property
    def title(self):
        return self._getter('title')(self)

    @title.setter
    def title(self, value):
        self._setter('title')(self, value)

    ############################################################################
    """dategmt
    """
    @property
    def dategmt(self):
        return self._getter('dategmt')(self)

    @dategmt.setter
    def dategmt(self, value):
        self._setter('dategmt')(self, self._convertTime(value))

    ############################################################################
    """status
    """
    @property
    def status(self):
        return self._getter('status')(self)

    @status.setter
    def status(self, value):
        self._setter('status')(self, value)

    ############################################################################
    """posttype
    """
    @property
    def posttype(self):
        return self._getter('posttype')(self)

    @posttype.setter
    def posttype(self, value):
        self._setter('posttype')(self, value)

    ############################################################################
    """author
    """
    @property
    def author(self):
        return self._getter('author')(self)

    @author.setter
    def author(self, value):
        self._setter('author')(self, value)

    ############################################################################
    """excerpt
    """
    @property
    def excerpt(self):
        return self._getter('excerpt')(self)

    @excerpt.setter
    def excerpt(self, value):
        self._setter('excerpt')(self, value)

    ############################################################################
    """comments
    """
    @property
    def comments(self):
        return self._getter('comments')(self)

    @comments.setter
    def comments(self, value):
        self._setter('comments')(self, value)

    ############################################################################
    """ping
    """
    @property
    def ping(self):
        return self._getter('ping')(self)

    @ping.setter
    def ping(self, value):
        self._setter('ping')(self, value)

    ############################################################################
    """publish
    """
    @property
    def publish(self):
        def native():
            return self._publish
        def wp():
            if self._data['post_status'] == 'publish':
                return 1
            else:
                return 0
        def metaweblog():
            return self._data['publish']

        return { 'native'     : native,
                 'wp'         : wp,
                 'metaweblog' : metaweblog, }[self._type]()

    @publish.setter
    def publish(self, value):
        def native(v):
            self._publish = v
        def wp(v):
            if v == 1:
                self._data['post_status'] = 'publish'
            else:
                self._data['post_status'] = 'draft'
        def metaweblog(v):
            self._data['publish'] = v

        { 'native'     : native,
          'wp'         : wp,
          'metaweblog' : metaweblog, }[self._type](value)

    ############################################################################
    """content
    """
    @property
    def content(self):
        def native():
            return self._content

        def wp():
            return self._data['post_content']

        def metaweblog():
            if self._data['mt_text_more']:
                if self._data['wp_more_text']:
                    more = "more %s" % self._data['wp_more_text']
                else:
                    more = "more"
                return "%s<!--%s-->%s" % (self._data['description'],
                                          more,
                                          self._data['mt_text_more'])
            else:
                return self._data['description']

        return { 'native'     : native,
                 'wp'         : wp,
                 'metaweblog' : metaweblog, }[self._type]()

    @content.setter
    def content(self, value):
        self._setter('content')(self, value)

    ############################################################################
    """categories
    """
    @property
    def categories(self):
        def native():
            return self._categories

        def wp():
            cats = []
            for term in self._data['terms']:
                if term['taxonomy'] == 'category':
                    cats.append(term['name'])
            return cats

        def metaweblog():
            return self._data['categories']

        return { 'native'     : native,
                 'wp'         : wp,
                 'metaweblog' : metaweblog, }[self._type]()

    @categories.setter
    def categories(self, value):
        self._setter('categories')(self, value)

    ############################################################################
    """tags
    """
    @property
    def tags(self):
        def native():
            return self._tags

        def wp():
            tags = ''
            for term in self._data['terms']:
                if term['taxonomy'] == 'post_tag':
                    if tags != '':
                        tags += ', '
                    tags += term['name']
            return tags

        def metaweblog():
            return self._data['mt_keywords']

        return { 'native'     : native,
                 'wp'         : wp,
                 'metaweblog' : metaweblog, }[self._type]()

    @tags.setter
    def tags(self, value):
        self._setter('tags')(self, value)
<|endoftext|>


<commit_before>from __future__ import unicode_literals

import django
import sys
from unittest.case import skipIf

from django.test.testcases import TransactionTestCase
from mock import patch, Mock

from messaging.backends.amqp import AMQPMessagingBackend
from messaging.backends.factory import add_message_backend

if not sys.platform.startswith('win'):
    import scheduler

@skipIf(sys.platform.startswith('win'), 'The scheduler uses the mesos native api.'
        '  Our development environment does not have this available on windows')
class TestScheduler(TransactionTestCase):
    """Tests core functionality of the scheduler."""

    def setUp(self):
        django.setup()

        add_message_backend(AMQPMessagingBackend)

        # mock out threading.start

    @patch('scheduler.scale_scheduler.initialize_system')
    @patch('scheduler.scale_scheduler.threading.Thread.start')
    def _get_mocked_scheduler_driver_master(self, _m1, _m2):
        """gets a registered scheduler with some stuff mocked out for testing"""
        return self._get_registered_scheduler_driver_master()

    def _get_registered_scheduler_driver_master(self):
        driver = Mock()
        framework_id = Mock()
        framework_id.value = 'framework_id'
        master_info = Mock()
        master_info.hostname = 'localhost'
        master_info.port = 1234

        my_scheduler = scheduler.scale_scheduler.ScaleScheduler()
        my_scheduler.registered(driver, framework_id, master_info)
        return my_scheduler, driver, master_info

    @patch('scheduler.scale_scheduler.initialize_system')
    @patch('scheduler.scale_scheduler.threading.Thread.start')
    def testRegistration(self, mock_thread_start, mock_initializer):
        my_scheduler, driver, master_info = self._get_registered_scheduler_driver_master()
        self.assertTrue(mock_initializer.called, 'initializer should be called on registration')
        self.assertEqual(
            mock_thread_start.call_count, 7,
            '7 threads should be started (7 != %d)' % mock_thread_start.call_count
        )

    @patch('scheduler.scale_scheduler.ScaleScheduler._reconcile_running_jobs')
    def test_reregistration_triggers_reconciliation(self, mock_reconcile_running_jobs):
        my_scheduler, driver, master_info = self._get_mocked_scheduler_driver_master()
        reconcile_calls_before = mock_reconcile_running_jobs.call_count
        my_scheduler.reregistered(driver, master_info)
        reconcile_calls_after = mock_reconcile_running_jobs.call_count
        self.assertEqual(reconcile_calls_after - reconcile_calls_before, 1,
                         're-registering the scheduler should trigger a call to reconcile running jobs')
<commit_msg>Fixed unit test<commit_after>from __future__ import unicode_literals

import django
import sys
from unittest.case import skipIf

from django.test.testcases import TransactionTestCase
from mock import patch, Mock

from messaging.backends.amqp import AMQPMessagingBackend
from messaging.backends.factory import add_message_backend

if not sys.platform.startswith('win'):
    import scheduler

@skipIf(sys.platform.startswith('win'), 'The scheduler uses the mesos native api.'
        '  Our development environment does not have this available on windows')
class TestScheduler(TransactionTestCase):
    """Tests core functionality of the scheduler."""

    def setUp(self):
        django.setup()

        add_message_backend(AMQPMessagingBackend)

        # mock out threading.start

    @patch('scheduler.scale_scheduler.initialize_system')
    @patch('scheduler.scale_scheduler.threading.Thread.start')
    def _get_mocked_scheduler_driver_master(self, _m1, _m2):
        """gets a registered scheduler with some stuff mocked out for testing"""
        return self._get_registered_scheduler_driver_master()

    def _get_registered_scheduler_driver_master(self):
        driver = Mock()
        framework_id = Mock()
        framework_id.value = 'framework_id'
        master_info = Mock()
        master_info.hostname = 'localhost'
        master_info.port = 1234

        my_scheduler = scheduler.scale_scheduler.ScaleScheduler()
        my_scheduler.initialize()
        my_scheduler.registered(driver, framework_id, master_info)
        return my_scheduler, driver, master_info

    @patch('scheduler.scale_scheduler.initialize_system')
    @patch('scheduler.scale_scheduler.threading.Thread.start')
    def testRegistration(self, mock_thread_start, mock_initializer):
        my_scheduler, driver, master_info = self._get_registered_scheduler_driver_master()
        self.assertTrue(mock_initializer.called, 'initializer should be called on registration')
        self.assertEqual(
            mock_thread_start.call_count, 7,
            '7 threads should be started (7 != %d)' % mock_thread_start.call_count
        )

    @patch('scheduler.scale_scheduler.ScaleScheduler._reconcile_running_jobs')
    def test_reregistration_triggers_reconciliation(self, mock_reconcile_running_jobs):
        my_scheduler, driver, master_info = self._get_mocked_scheduler_driver_master()
        reconcile_calls_before = mock_reconcile_running_jobs.call_count
        my_scheduler.reregistered(driver, master_info)
        reconcile_calls_after = mock_reconcile_running_jobs.call_count
        self.assertEqual(reconcile_calls_after - reconcile_calls_before, 1,
                         're-registering the scheduler should trigger a call to reconcile running jobs')
<|endoftext|>


<commit_before>
from future import standard_library

standard_library.install_aliases()

from urllib.parse import urlencode

from builtins import object
import json
import logging
import socket
from collections import namedtuple
from importlib import import_module

import html2text

import bleach
import pygeoip
from django.conf import settings
from django.contrib.auth.management import create_permissions
from django.contrib.auth.models import Permission, Group
from django.core.exceptions import ObjectDoesNotExist
from django.core.signing import TimestampSigner
from django.core.urlresolvers import reverse
from django.db import connection
from django_fsm import TransitionNotAllowed
from django_tools.middlewares import ThreadLocal

from bluebottle.clients import properties

to_text = html2text.HTML2Text()
to_text.ignore_tables = True
to_text.ignore_images = True

TAGS = ['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'strong', 'em', 'b', 'i', 'ul', 'li', 'ol', 'a',
        'br', 'pre', 'blockquote', 'img', 'hr', 'span', 'em', 'u']
ATTRIBUTES = {'a': ['target', 'href'], 'img': ['src', 'alt', 'width', 'height', 'align']}

def clean_html(content):
    return bleach.clean(content, tags=TAGS, attributes=ATTRIBUTES)

def get_languages():
    return properties.LANGUAGES

class StatusDefinition(object):
    """
    Various status definitions for FSM's
    """
    NEW = 'new'
    IN_PROGRESS = 'in_progress'
    PENDING = 'pending'
    NEEDS_APPROVAL = 'needs_approval'
    CREATED = 'created'
    LOCKED = 'locked'
    PLEDGED = 'pledged'
    APPROVED = 'approved'
    SUCCESS = 'success'
    STARTED = 'started'
    SCHEDULED = 'scheduled'
    RE_SCHEDULED = 're_scheduled'
    CANCELLED = 'cancelled'
    AUTHORIZED = 'authorized'
    SETTLED = 'settled'
    CONFIRMED = 'confirmed'
    CHARGED_BACK = 'charged_back'
    REFUND_REQUESTED = 'refund_requested'
    REFUNDED = 'refunded'
    PAID = 'paid'
    FAILED = 'failed'
    RETRY = 'retry'
    PARTIAL = 'partial'
    UNKNOWN = 'unknown'

class FSMTransition(object):
    """
    Class mixin to add transition_to method for Django FSM
    """

    def transition_to(self, new_status, save=True):
        # If the new_status is the same as then current then return early
        if self.status == new_status:
            return

        # Lookup the available next transition - from Django FSM
        available_transitions = self.get_available_status_transitions()

        logging.debug("{0} (pk={1}) state changing: '{2}' to '{3}'".format(
            self.__class__.__name__, self.pk, self.status, new_status))

        # Check that the new_status is in the available transitions -
        # created with Django FSM decorator
        for transition in available_transitions:
            if transition.name == new_status:
                transition_method = transition.method

        # Call state transition method
        try:
            instance_method = getattr(self, transition_method.__name__)
            instance_method()
        except UnboundLocalError:
            tenant = connection.tenant.schema_name
            raise TransitionNotAllowed(
                "Can't switch from state '{0}' to state '{1}' for {2} {3} {4}".format(
                    self.status,
                    new_status,
                    self.__class__.__name__,
                    self.id,
                    tenant
                ))

        if save:
            self.save()

    def refresh_from_db(self):
        """Refreshes this instance from db"""
        new_self = self.__class__.objects.get(pk=self.pk)
        self.__dict__.update(new_self.__dict__)

def get_client_ip(request=None):
    """
    A utility method that returns the client IP for the given request.
    """
    if not request:
        request = ThreadLocal.get_current_request()

    try:
        x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR')
    except AttributeError:
        x_forwarded_for = None

    if x_forwarded_for:
        ipa = x_forwarded_for.split(',')[-1]
    else:
        try:
            ipa = request.META.get('REMOTE_ADDR')
        except AttributeError:
            ipa = None

    return ipa.strip() if ipa else ipa

def set_author_editor_ip(request, obj):
    """
    A utility method to set the author, editor and IP address on an
    object based on information in a request.
    """
    if not hasattr(obj, 'author'):
        obj.author = request.user
    else:
        obj.editor = request.user
    obj.ip_address = get_client_ip(request)

def clean_for_hashtag(text):
    """
    Strip non alphanumeric charachters.

    Sometimes, text bits are made up of two parts, sepated by a slash. Split
    those into two tags. Otherwise, join the parts separated by a space.
    """
    tags = []
    bits = text.split('/')
    for bit in bits:
        # keep the alphanumeric bits and capitalize the first letter
        _bits = [_bit.title() for _bit in bit.split() if _bit.isalnum()]
        tag = "".join(_bits)
        tags.append(tag)

    return " #".join(tags)

class GetClassError(Exception):
    """ Custom exception for an GetClass """
    pass

def get_class(cls):
    # Get the class from dotted string
    try:
        # try to call handler
        parts = cls.split('.')
        module_path, class_name = '.'.join(parts[:-1]), parts[-1]
        module = import_module(module_path)
        return getattr(module, class_name)

    except (ImportError, AttributeError, ValueError) as err:
        error_message = "Could not import '%s'. %s: %s." % (cls, err.__class__.__name__, err)
        raise GetClassError(error_message)

def get_current_host(include_scheme=True):
    """
    Get the current hostname with protocol
    E.g. http://localhost:8000 or https://bluebottle.org
    """
    request = ThreadLocal.get_current_request()
    if request:
        host = request.get_host()
    else:
        host = connection.tenant.domain_url
    if include_scheme:
        if request and request.is_secure():
            scheme = 'https'
        else:
            scheme = 'http'
        return '{0}://{1}'.format(scheme, host)
    else:
        return host

def get_current_language():
    """
    Get the current language from request
    """
    request = ThreadLocal.get_current_request()
    if request:
        return request.LANGUAGE_CODE
    else:
        return properties.LANGUAGE_CODE

class InvalidIpError(Exception):
    """ Custom exception for an invalid IP address """

    def __init__(self, value):
        self.value = value

    def __str__(self):
        return repr(self.value)

def get_country_by_ip(ip_address=None):
    """
    Returns the country associated with the IP address. Uses pygeoip
    library which is based on
    """
    if not ip_address:
        return None

    try:
        socket.inet_aton(ip_address)
    except socket.error:
        raise InvalidIpError("Invalid IP address")

    gip = pygeoip.GeoIP(settings.PROJECT_ROOT + '/GeoIP.dat')
    return gip.country_name_by_addr(ip_address)

def get_country_code_by_ip(ip_address=None):
    """
    Returns the country associated with the IP address. Uses pygeoip
    library which is based on
    the popular Maxmind's GeoIP C API
    """
    if not ip_address:
        return None

    try:
        socket.inet_aton(ip_address)
    except socket.error:
        raise InvalidIpError("Invalid IP address")

    gip = pygeoip.GeoIP(settings.PROJECT_ROOT + '/GeoIP.dat')
    return gip.country_code_by_name(ip_address)

def update_group_permissions(label, group_perms, apps):
    for app_config in apps.get_app_configs():
        app_config.models_module = True
        create_permissions(app_config, apps=apps, verbosity=0)
        app_config.models_module = None

    for group_name, permissions in list(group_perms.items()):
        group, _ = Group.objects.get_or_create(name=group_name)
        for perm_codename in permissions['perms']:
            try:
                permissions = Permission.objects.filter(codename=perm_codename)
                permissions = permissions.filter(content_type__app_label=label)
                permission = permissions.last()
                if not permission:
                    raise Exception(
                        'Could not add permission: {}: Not found'.format(perm_codename)
                    )
                else:
                    group.permissions.add(permission)
            except Permission.DoesNotExist as err:
                logging.debug(err)
                raise Exception(
                    'Could not add permission: {}: {}'.format(perm_codename, err)
                )
        group.save()

class PreviousStatusMixin(object):
    """
    Store the status of the instance on init to be accessed as _original_status
    """

    def __init__(self, *args, **kwargs):
        super(PreviousStatusMixin, self).__init__(*args, **kwargs)

        try:
            self._original_status = self.status
        except ObjectDoesNotExist:
            self._original_status = None

signer = TimestampSigner()

def reverse_signed(name, args):
    url = reverse(name, args=args)
    signature = signer.sign(url)
    return '{}?{}'.format(url, urlencode({'signature': signature}))

def get_language_from_request(request):
    return request.META.get('HTTP_X_APPLICATION_LANGUAGE', None)

def _json_object_hook(d):
    return namedtuple('X', list(d.keys()))(*list(d.values()))

def json2obj(data):
    return json.loads(data, object_hook=_json_object_hook)
<commit_msg>Do not line wrap txt emails. Do disable emphasis in txt emails<commit_after>
from future import standard_library

standard_library.install_aliases()

from urllib.parse import urlencode

from builtins import object
import json
import logging
import socket
from collections import namedtuple
from importlib import import_module

import html2text

import bleach
import pygeoip
from django.conf import settings
from django.contrib.auth.management import create_permissions
from django.contrib.auth.models import Permission, Group
from django.core.exceptions import ObjectDoesNotExist
from django.core.signing import TimestampSigner
from django.core.urlresolvers import reverse
from django.db import connection
from django_fsm import TransitionNotAllowed
from django_tools.middlewares import ThreadLocal

from bluebottle.clients import properties

to_text = html2text.HTML2Text()
to_text.ignore_tables = True
to_text.ignore_images = True
to_text.body_width = 0
to_text.ignore_emphasis = True

TAGS = ['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'strong', 'em', 'b', 'i', 'ul', 'li', 'ol', 'a',
        'br', 'pre', 'blockquote', 'img', 'hr', 'span', 'em', 'u']
ATTRIBUTES = {'a': ['target', 'href'], 'img': ['src', 'alt', 'width', 'height', 'align']}

def clean_html(content):
    return bleach.clean(content, tags=TAGS, attributes=ATTRIBUTES)

def get_languages():
    return properties.LANGUAGES

class StatusDefinition(object):
    """
    Various status definitions for FSM's
    """
    NEW = 'new'
    IN_PROGRESS = 'in_progress'
    PENDING = 'pending'
    NEEDS_APPROVAL = 'needs_approval'
    CREATED = 'created'
    LOCKED = 'locked'
    PLEDGED = 'pledged'
    APPROVED = 'approved'
    SUCCESS = 'success'
    STARTED = 'started'
    SCHEDULED = 'scheduled'
    RE_SCHEDULED = 're_scheduled'
    CANCELLED = 'cancelled'
    AUTHORIZED = 'authorized'
    SETTLED = 'settled'
    CONFIRMED = 'confirmed'
    CHARGED_BACK = 'charged_back'
    REFUND_REQUESTED = 'refund_requested'
    REFUNDED = 'refunded'
    PAID = 'paid'
    FAILED = 'failed'
    RETRY = 'retry'
    PARTIAL = 'partial'
    UNKNOWN = 'unknown'

class FSMTransition(object):
    """
    Class mixin to add transition_to method for Django FSM
    """

    def transition_to(self, new_status, save=True):
        # If the new_status is the same as then current then return early
        if self.status == new_status:
            return

        # Lookup the available next transition - from Django FSM
        available_transitions = self.get_available_status_transitions()

        logging.debug("{0} (pk={1}) state changing: '{2}' to '{3}'".format(
            self.__class__.__name__, self.pk, self.status, new_status))

        # Check that the new_status is in the available transitions -
        # created with Django FSM decorator
        for transition in available_transitions:
            if transition.name == new_status:
                transition_method = transition.method

        # Call state transition method
        try:
            instance_method = getattr(self, transition_method.__name__)
            instance_method()
        except UnboundLocalError:
            tenant = connection.tenant.schema_name
            raise TransitionNotAllowed(
                "Can't switch from state '{0}' to state '{1}' for {2} {3} {4}".format(
                    self.status,
                    new_status,
                    self.__class__.__name__,
                    self.id,
                    tenant
                ))

        if save:
            self.save()

    def refresh_from_db(self):
        """Refreshes this instance from db"""
        new_self = self.__class__.objects.get(pk=self.pk)
        self.__dict__.update(new_self.__dict__)

def get_client_ip(request=None):
    """
    A utility method that returns the client IP for the given request.
    """
    if not request:
        request = ThreadLocal.get_current_request()

    try:
        x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR')
    except AttributeError:
        x_forwarded_for = None

    if x_forwarded_for:
        ipa = x_forwarded_for.split(',')[-1]
    else:
        try:
            ipa = request.META.get('REMOTE_ADDR')
        except AttributeError:
            ipa = None

    return ipa.strip() if ipa else ipa

def set_author_editor_ip(request, obj):
    """
    A utility method to set the author, editor and IP address on an
    object based on information in a request.
    """
    if not hasattr(obj, 'author'):
        obj.author = request.user
    else:
        obj.editor = request.user
    obj.ip_address = get_client_ip(request)

def clean_for_hashtag(text):
    """
    Strip non alphanumeric charachters.

    Sometimes, text bits are made up of two parts, sepated by a slash. Split
    those into two tags. Otherwise, join the parts separated by a space.
    """
    tags = []
    bits = text.split('/')
    for bit in bits:
        # keep the alphanumeric bits and capitalize the first letter
        _bits = [_bit.title() for _bit in bit.split() if _bit.isalnum()]
        tag = "".join(_bits)
        tags.append(tag)

    return " #".join(tags)

class GetClassError(Exception):
    """ Custom exception for an GetClass """
    pass

def get_class(cls):
    # Get the class from dotted string
    try:
        # try to call handler
        parts = cls.split('.')
        module_path, class_name = '.'.join(parts[:-1]), parts[-1]
        module = import_module(module_path)
        return getattr(module, class_name)

    except (ImportError, AttributeError, ValueError) as err:
        error_message = "Could not import '%s'. %s: %s." % (cls, err.__class__.__name__, err)
        raise GetClassError(error_message)

def get_current_host(include_scheme=True):
    """
    Get the current hostname with protocol
    E.g. http://localhost:8000 or https://bluebottle.org
    """
    request = ThreadLocal.get_current_request()
    if request:
        host = request.get_host()
    else:
        host = connection.tenant.domain_url
    if include_scheme:
        if request and request.is_secure():
            scheme = 'https'
        else:
            scheme = 'http'
        return '{0}://{1}'.format(scheme, host)
    else:
        return host

def get_current_language():
    """
    Get the current language from request
    """
    request = ThreadLocal.get_current_request()
    if request:
        return request.LANGUAGE_CODE
    else:
        return properties.LANGUAGE_CODE

class InvalidIpError(Exception):
    """ Custom exception for an invalid IP address """

    def __init__(self, value):
        self.value = value

    def __str__(self):
        return repr(self.value)

def get_country_by_ip(ip_address=None):
    """
    Returns the country associated with the IP address. Uses pygeoip
    library which is based on
    """
    if not ip_address:
        return None

    try:
        socket.inet_aton(ip_address)
    except socket.error:
        raise InvalidIpError("Invalid IP address")

    gip = pygeoip.GeoIP(settings.PROJECT_ROOT + '/GeoIP.dat')
    return gip.country_name_by_addr(ip_address)

def get_country_code_by_ip(ip_address=None):
    """
    Returns the country associated with the IP address. Uses pygeoip
    library which is based on
    the popular Maxmind's GeoIP C API
    """
    if not ip_address:
        return None

    try:
        socket.inet_aton(ip_address)
    except socket.error:
        raise InvalidIpError("Invalid IP address")

    gip = pygeoip.GeoIP(settings.PROJECT_ROOT + '/GeoIP.dat')
    return gip.country_code_by_name(ip_address)

def update_group_permissions(label, group_perms, apps):
    for app_config in apps.get_app_configs():
        app_config.models_module = True
        create_permissions(app_config, apps=apps, verbosity=0)
        app_config.models_module = None

    for group_name, permissions in list(group_perms.items()):
        group, _ = Group.objects.get_or_create(name=group_name)
        for perm_codename in permissions['perms']:
            try:
                permissions = Permission.objects.filter(codename=perm_codename)
                permissions = permissions.filter(content_type__app_label=label)
                permission = permissions.last()
                if not permission:
                    raise Exception(
                        'Could not add permission: {}: Not found'.format(perm_codename)
                    )
                else:
                    group.permissions.add(permission)
            except Permission.DoesNotExist as err:
                logging.debug(err)
                raise Exception(
                    'Could not add permission: {}: {}'.format(perm_codename, err)
                )
        group.save()

class PreviousStatusMixin(object):
    """
    Store the status of the instance on init to be accessed as _original_status
    """

    def __init__(self, *args, **kwargs):
        super(PreviousStatusMixin, self).__init__(*args, **kwargs)

        try:
            self._original_status = self.status
        except ObjectDoesNotExist:
            self._original_status = None

signer = TimestampSigner()

def reverse_signed(name, args):
    url = reverse(name, args=args)
    signature = signer.sign(url)
    return '{}?{}'.format(url, urlencode({'signature': signature}))

def get_language_from_request(request):
    return request.META.get('HTTP_X_APPLICATION_LANGUAGE', None)

def _json_object_hook(d):
    return namedtuple('X', list(d.keys()))(*list(d.values()))

def json2obj(data):
    return json.loads(data, object_hook=_json_object_hook)
<|endoftext|>


<commit_before>from datetime import datetime

from bottle import abort, default_app, response, route, run
from rss2producer import RSS2Feed
import dataset

from settings import TABLE, PAGES

# copy pasted from scraper
LIMIT = 50

slug_to_name = {slug: name for slug, pk, name in PAGES}
slug_to_id = {slug: pk for slug, pk, name in PAGES}

@route('/')
def index():
    out = []
    for slug, pk, name in PAGES:
        out.append('
{}
'.format(slug, name))
    return u''.join(out)

@route('/<slug>/')
def feed_detail(slug):
    if slug != 'all' and slug not in slug_to_name:
        abort(404, 'No results for that Board or Commission')
    db = dataset.connect()  # uses DATABASE_URL
    # table = db.load_table(TABLE)  # don't allow dataset to create the table
    if slug == 'all':
        feed_info = dict(
            title='Boards and Commissions',
            link='http://www.austintexas.gov/cityclerk/boards_commissions/',
            description='Feed of Boards and Commissions activity',
        )
    else:
        feed_info = dict(
            title=slug_to_name[slug],
            link=('http://www.austintexas.gov/cityclerk/boards_commissions/'
                'meetings/{}_1.htm'.format(slug_to_id[slug])),
            description='Feed of {} activity'.format(slug_to_name[slug]),
        )
    feed = RSS2Feed(**feed_info)
    filter_kwargs = {}
    if slug != 'all':
        filter_kwargs['bandc'] = slug
    # results = table.find(_limit=LIMIT, order_by='-date', **filter_kwargs)
    # HACK for 'order_by' not working
    sql = 'SELECT * from {} ORDER BY date DESC LIMIT {}'.format(
        TABLE,
        LIMIT,
    )
    results = db.query(sql)
    import re
    for row in results:
        title = row['title'] or row['type']
        if slug == 'all':
            title = '[{}] {}'.format(row['bandc'], title)
        text = row['text'].strip().encode('ascii', 'ignore')[:600].strip()
        text = re.sub(r'\s+', ' ', text)[:500]
        if text:
            print text
        feed.append_item(
            title=title,
            link=row['url'],
            description=text if text and not text.startswith('no text') else row['type'],
            # convert date into datetime
            pub_date=datetime.combine(row['date'], datetime.min.time()),
        )
    response.content_type = 'application/rss+xml'
    return feed.get_xml()

app = default_app()

if __name__ == '__main__':
    run(host='localhost', port=8000, reloader=True)
<commit_msg>more copy tweaks to feed page<commit_after>from datetime import datetime

from bottle import abort, default_app, response, route, run
from rss2producer import RSS2Feed
import dataset

from settings import TABLE, PAGES

# copy pasted from scraper
LIMIT = 50

slug_to_name = {slug: name for slug, pk, name in PAGES}
slug_to_id = {slug: pk for slug, pk, name in PAGES}

@route('/')
def index():
    out = []
    out.append('
{}
'.format('all', 'All'))
    for slug, pk, name in PAGES:
        out.append('
{}
'.format(slug, name))
    return u''.join(out)

@route('/<slug>/')
def feed_detail(slug):
    if slug != 'all' and slug not in slug_to_name:
        abort(404, 'No results for that Board or Commission')
    db = dataset.connect()  # uses DATABASE_URL
    # table = db.load_table(TABLE)  # don't allow dataset to create the table
    if slug == 'all':
        feed_info = dict(
            title='Boards and Commissions',
            link='http://www.austintexas.gov/cityclerk/boards_commissions/',
            description='Feed of Boards and Commissions activity',
        )
    else:
        feed_info = dict(
            title=slug_to_name[slug],
            link=('http://www.austintexas.gov/cityclerk/boards_commissions/'
                'meetings/{}_1.htm'.format(slug_to_id[slug])),
            description='Feed of {} activity'.format(slug_to_name[slug]),
        )
    feed = RSS2Feed(**feed_info)
    filter_kwargs = {}
    if slug != 'all':
        filter_kwargs['bandc'] = slug
    # results = table.find(_limit=LIMIT, order_by='-date', **filter_kwargs)
    # HACK for 'order_by' not working
    sql = 'SELECT * from {} ORDER BY date DESC LIMIT {}'.format(
        TABLE,
        LIMIT,
    )
    results = db.query(sql)
    import re
    for row in results:
        title = row['title'] or row['type']
        if slug == 'all':
            title = '[{}] {}'.format(row['bandc'], title)
        text = row['text'].strip().encode('ascii', 'ignore')[:600].strip()
        text = re.sub(r'\s+', ' ', text)[:500]
        if text.startswith('no text'):
            text = '{}: {}'.format(row['type'], text)
        feed.append_item(
            title=title,
            link=row['url'],
            description=text if text else row['type'],
            # convert date into datetime
            pub_date=datetime.combine(row['date'], datetime.min.time()),
        )
    response.content_type = 'application/rss+xml'
    return feed.get_xml()

app = default_app()

if __name__ == '__main__':
    run(host='localhost', port=8000, reloader=True)
<|endoftext|>


<commit_before>import sys
from collections import OrderedDict
import StringIO
import blinker

class DataNode(object):
    '''A branch/leaf in a data tree.
    '''
    def __init__(self, parent=None):
        self.parent = parent
        self.children_loaded = False
        self._children = []
        self._data_object = None

    node_type = "Unknown"

    child_added = blinker.Signal("child_added")

    child_removed = blinker.Signal("child_removed")

    @property
    def icon(self):
        return None

    @property
    def data_object(self):
        return None

    def has_object(self):
        return (self._data_object is not None) or hasattr(self, "create_data_object")

    @property
    def data_object(self):
        if self._data_object is None:
            if hasattr(self, "create_data_object"):
                self._data_object = self.create_data_object()
        return self._data_object

    # TODO data_object setter

    @property
    def properties(self):
        return OrderedDict()

    @property
    def children(self):
        return []

    @property
    def title(self):
        return unicode(self)

    @property
    def full_title(self):
        if self.parent:
            return self.parent.full_title + "/" + self.title
        else:
            return self.title

    @property
    def descendants(self):
        """Recursive iterator of all descendants."""
        for child in self.children:
            yield child
            for descendant in child.descendants:
                yield descendant

    #TODO: enumerate descendants with indices

    def subtree(self):
        return None

    def has_subtree(self):
        return self.subtree() is not None

    @property
    def children(self):
        """Lazy access to children."""
        # TODO: Add option to disable caching
        if not self.children_loaded:
            self.load_children()
            self.children_loaded = True
        return self._children

    def add_child(self, child, send_signal=True):
        child.parent = self
        self._children.append(child)
        if send_signal:
            self.child_added.send(self, child=child)

    def remove_child(self, child, send_signal=True):
        self._children.remove(child)
        if send_signal:
            self.child_removed.send(self, child=child)

    def load_children(self):
        pass

    def reload_children(self):
        '''Forces children reloading.'''
        self._children = []
        self.children_loaded = False

    def dump(self, stream=sys.stdout, indent=u"  ", subtree=False, in_depth=0, children_only=False, data_object_info=False):
        if not children_only:
            stream.write(in_depth * indent)
            stream.write(self.title)
            # stream.write(str(self.has_object()))
            if data_object_info and self.has_object():
                # stream.write("!")
                stream.write(" (" + " x ".join(str(i) for i in self.data_object.shape) + ")")
        if self.has_subtree() and subtree:
            stream.write(":")
            self.subtree().dump(stream, indent, subtree, in_depth, children_only=True, data_object_info=data_object_info)
        else:
            stream.write("\n")
        for child in self.children:
            child.dump(stream, indent, subtree, in_depth+1, data_object_info=data_object_info)

class DataTree(DataNode):
    '''A node that can be top-level in the tree view.
    '''

    @property
    def menu_title(self):
        '''Title of the menu displayed in main menu bar.

        Override if not equal to title.
        '''
        return self.title

    @property
    def menu_actions(self):
        '''Qt actions that should be put into the menu in main menu bar.'''
        return []<commit_msg>Bit more __doc__<commit_after>import sys
from collections import OrderedDict
import StringIO
import blinker

class DataNode(object):
    '''A branch/leaf in a data tree.
    '''

    def __init__(self, parent=None):
        self.parent = parent
        self.children_loaded = False
        self._children = []
        self._data_object = None

    node_type = "Unknown"

    # Signals
    child_added = blinker.Signal("child_added")
    child_removed = blinker.Signal("child_removed")

    @property
    def icon(self):
        return None

    @property
    def data_object(self):
        return None

    def has_object(self):
        return (self._data_object is not None) or hasattr(self, "create_data_object")

    @property
    def data_object(self):
        if self._data_object is None:
            if hasattr(self, "create_data_object"):
                self._data_object = self.create_data_object()
        return self._data_object

    # TODO: data_object setter
    # TODO: signal data_object changed

    @property
    def properties(self):
        return OrderedDict()

    @property
    def children(self):
        return []

    @property
    def title(self):
        return unicode(self)

    @property
    def full_title(self):
        if self.parent:
            return self.parent.full_title + "/" + self.title
        else:
            return self.title

    @property
    def descendants(self):
        """Recursive iterator of all descendants."""
        for child in self.children:
            yield child
            for descendant in child.descendants:
                yield descendant

    #TODO: enumerate descendants with indices

    def subtree(self):
        return None

    def has_subtree(self):
        '''Whether the node can serve as a root of another tree.'''
        return self.subtree() is not None

    @property
    def children(self):
        '''Lazy access to children.'''
        # TODO: Add option to disable caching
        if not self.children_loaded:
            self.load_children()
            self.children_loaded = True
        return self._children

    def add_child(self, child, send_signal=True):
        child.parent = self
        self._children.append(child)
        if send_signal:
            self.child_added.send(self, child=child)

    def remove_child(self, child, send_signal=True):
        self._children.remove(child)
        if send_signal:
            self.child_removed.send(self, child=child)

    def load_children(self):
        pass

    def reload_children(self):
        '''Forces children reloading.'''
        self._children = []
        self.children_loaded = False

    def dump(self, stream=sys.stdout, indent=u"  ", subtree=False, in_depth=0, children_only=False, data_object_info=False):
        if not children_only:
            stream.write(in_depth * indent)
            stream.write(self.title)
            # stream.write(str(self.has_object()))
            if data_object_info and self.has_object():
                # stream.write("!")
                stream.write(" (" + " x ".join(str(i) for i in self.data_object.shape) + ")")
        if self.has_subtree() and subtree:
            stream.write(":")
            self.subtree().dump(stream, indent, subtree, in_depth, children_only=True, data_object_info=data_object_info)
        else:
            stream.write("\n")
        for child in self.children:
            child.dump(stream, indent, subtree, in_depth+1, data_object_info=data_object_info)

class DataTree(DataNode):
    '''A node that can be top-level in the tree view.
    '''

    @property
    def menu_title(self):
        '''Title of the menu displayed in main menu bar.

        Override if not equal to title.
        '''
        return self.title

    @property
    def menu_actions(self):
        '''Qt actions that should be put into the menu in main menu bar.'''
        return []<|endoftext|>