<commit_before>#!/usr/bin/env python
# encoding: utf-8

#------------------------------------------------------------------------------
# decrypto
# Copyright 2014 Christopher Simpkins
# MIT license
#------------------------------------------------------------------------------

# Application start
def main():
    import os
    import sys
    import getpass
    from Naked.commandline import Command
    from Naked.toolshed.shell import execute, muterun
    from Naked.toolshed.system import dir_exists, directory, filename, file_exists, list_all_files, make_path, stdout, stderr

    #------------------------------------------------------------------------------------------
    # [ Instantiate command line object ]
    #   used for all subsequent conditional logic in the CLI application
    #------------------------------------------------------------------------------------------
    c = Command(sys.argv[0], sys.argv[1:])
    #------------------------------------------------------------------------------------------
    # [ VALIDATION LOGIC ] - early validation of appropriate command syntax
    # Test that user entered at least one argument to the executable, print usage if not
    #------------------------------------------------------------------------------------------
    if not c.command_suite_validates():
        from crypto.settings import usage as crypto_usage
        print(crypto_usage)
        sys.exit(1)
    #------------------------------------------------------------------------------------------
    # [ HELP, VERSION, USAGE LOGIC ]
    # Naked framework provides default help, usage, and version commands for all applications
    #   --> settings for user messages are assigned in the lib/crypto/settings.py file
    #------------------------------------------------------------------------------------------
    if c.help():      # User requested crypto help information
        from crypto.settings import help as crypto_help
        print(crypto_help)
        sys.exit(0)
    elif c.usage():   # User requested crypto usage information
        from crypto.settings import usage as crypto_usage
        print(crypto_usage)
        sys.exit(0)
    elif c.version(): # User requested crypto version information
        from crypto.settings import app_name, major_version, minor_version, patch_version
        version_display_string = app_name + ' ' + major_version + '.' + minor_version + '.' + patch_version
        print(version_display_string)
        sys.exit(0)
    #------------------------------------------------------------------------------------------
    # [ APPLICATION LOGIC ]
    #
    #------------------------------------------------------------------------------------------
    elif c.argc > 1:
    # code for multi-file processing and commands that include options
        use_standard_output = False # print to stdout flag
        use_file_overwrite = False # overwrite existing files

        # set user option flags
        if c.option('--stdout') or c.option('-s'):
            use_standard_output = True
        if c.option('--overwrite') or c.option('-o'):
            use_file_overwrite = True

        directory_list = [] # directory paths included in the user entered paths from the command line
        file_list = [] # file paths included in the user entered paths from the command line (and inside directories entered)

        for argument in c.argv:
            if file_exists(argument): # user included a file, add it to the file_list for decryption
                if argument.endswith('.crypt'):
                    file_list.append(argument) # add .crypt files to the list of files for decryption
                elif argument.endswith('.gpg'):
                    file_list.append(argument)
                elif argument.endswith('.asc'):
                    file_list.append(argument)
                elif argument.endswith('.pgp'):
                    file_list.append(argument)
                else:
                    # cannot identify as an encrypted file, give it a shot anyways but warn user
                    file_list.append(argument)
                    stdout("Could not confirm that '" + argument + "' is encrypted based upon the file type.  Attempting decryption.  Keep your fingers crossed...")
            elif dir_exists(argument): # user included a directory, add it to the directory_list
                directory_list.append(argument)
            else:
                if argument[0] == "-":
                    pass # if it is an option, do nothing
                else:
                    stderr("'" + argument + "' does not appear to be an existing file or directory.  Aborting decryption attempt for this request.", 0)

        # unroll the contained directory files into the file_list IF they are encrypted file types
        if len(directory_list) > 0:
            for directory in directory_list:
                directory_file_list = list_all_files(directory)
                for contained_file in directory_file_list:
                    if contained_file.endswith('.crypt'):
                        file_list.append(make_path(directory, contained_file)) # include the file with a filepath 'directory path/contained_file path'
                    elif contained_file.endswith('.gpg'):
                        file_list.append(make_path(directory, contained_file))
                    elif contained_file.endswith('asc'):
                        file_list.append(make_path(directory, contained_file))
                    elif contained_file.endswith('.pgp'):
                        file_list.append(make_path(directory, contained_file))

        # confirm that there are files for decryption, if not abort
        if len(file_list) == 0:
            stderr("Could not identify files for decryption")
            sys.exit(1)

        # get passphrase used to symmetrically decrypt the file
        passphrase = getpass.getpass("Please enter your passphrase: ")
        if len(passphrase) == 0: # confirm that user entered a passphrase
                stderr("You did not enter a passphrase. Please repeat your command and try again.")
                sys.exit(1)
        passphrase_confirm = getpass.getpass("Please enter your passphrase again: ")

        if passphrase == passphrase_confirm:
            # begin decryption of each requested file.  the directory path was already added to the file path above
            for encrypted_file in file_list:
                # create the decrypted file name
                decrypted_filename = ""
                if encrypted_file.endswith('.crypt'):
                    decrypted_filename = encrypted_file[0:-6]
                elif encrypted_file.endswith('.gpg') or encrypted_file.endswith('.asc') or encrypted_file.endswith('.pgp'):
                    decrypted_filename = encrypted_file[0:-4]
                else:
                    decrypted_filename = encrypted_file + '.decrypt' # if it was a file without a known encrypted file type, add the .decrypt suffix

                # determine whether file overwrite will take place with the decrypted file
                skip_file = False # flag that indicates this file should not be encrypted
                created_tmp_files = False
                if not use_standard_output: # if not writing a file, no need to check for overwrite
                    if file_exists(decrypted_filename):
                        if use_file_overwrite: # rename the existing file to temp file which will be erased or replaced (on decryption failures) below
                            tmp_filename = decrypted_filename + '.tmp'
                            os.rename(decrypted_filename, tmp_filename)
                            created_tmp_files = True
                        else:
                            stdout("The file path '" + decrypted_filename + "' already exists.  This file was not decrypted.")
                            skip_file = True

                # begin decryption
                if not skip_file:
                    if use_standard_output: # using --quiet flag to suppress stdout messages from gpg, just want the file data in stdout stream
                        system_command = "gpg --batch --quiet --passphrase '" + passphrase + "' -d " + encrypted_file
                        successful_execution = execute(system_command) # use naked execute function to directly push to stdout, rather than return stdout

                        if not successful_execution:
                            stderr("Unable to decrypt file '" + encrypted_file + "'", 0)
                            if created_tmp_files: # restore the moved tmp file to original if decrypt failed
                                tmp_filename = decrypted_filename + '.tmp'
                                if file_exists(tmp_filename):
                                    os.rename(tmp_filename, decrypted_filename)
                        else: # decryption successful but we are in stdout flag so do not include any other output from decrypto
                            pass
                    else:
                        system_command = "gpg --batch -o " + decrypted_filename + " --passphrase '" + passphrase + "' -d " + encrypted_file
                        response = muterun(system_command)

                        if response.exitcode == 0:
                            stdout("'" + encrypted_file + "' decrypted to '" + decrypted_filename + "'")
                        else: # failed decryption
                            if created_tmp_files:# restore the moved tmp file to original if decrypt failed
                                tmp_filename = decrypted_filename + '.tmp'
                                if file_exists(tmp_filename):
                                    os.rename(tmp_filename, decrypted_filename)
                            # report the error
                            stderr(response.stderr)
                            stderr("Decryption failed for " + encrypted_file)

                # cleanup: remove the tmp file
                if created_tmp_files:
                    tmp_filename = decrypted_filename + '.tmp'
                    if file_exists(tmp_filename):
                        os.remove(tmp_filename)

            # overwrite the entered passphrases after file decryption is complete for all files
            passphrase = ""
            passphrase_confirm = ""

        else: # passphrases did not match
            passphrase = ""
            passphrase_confirm = ""
            stderr("The passphrases did not match.  Please enter your command again.")
            sys.exit(1)

    elif c.argc == 1:
    # simple single file or directory processing with default settings
        path = c.arg0
        if file_exists(path): ## SINGLE FILE
            check_existing_file = False # check for a file with the name of new decrypted filename in the directory

            if path.endswith('.crypt'):
                new_filename = path[0:-6] # remove the .crypt suffix
                check_existing_file = True
            elif path.endswith('.gpg') or path.endswith('.pgp') or path.endswith('.asc'):
                new_filename = path[0:-4]
                check_existing_file = True
            else:
                new_filename = path + ".decrypt" #if there is not a standard file type, then add a .decrypt suffix to the decrypted file name
                stdout("Could not confirm that the requested file is encrypted based upon the file type.  Attempting decryption.  Keep your fingers crossed...")

            # confirm that the decrypted path does not already exist, if so abort with warning message to user
            if check_existing_file == True:
                if file_exists(new_filename):
                    stderr("Your file will be decrypted to '" + new_filename + "' and this file path already exists.  Please move the file or use the --overwrite option with your command if you intend to replace the current file.")
                    sys.exit(1)

            # get passphrase used to symmetrically decrypt the file
            passphrase = getpass.getpass("Please enter your passphrase: ")
            if len(passphrase) == 0: # confirm that user entered a passphrase
                stderr("You did not enter a passphrase. Please repeat your command and try again.")
                sys.exit(1)
            passphrase_confirm = getpass.getpass("Please enter your passphrase again: ")

            # confirm that the passphrases match
            if passphrase == passphrase_confirm:
                system_command = "gpg --batch -o " + new_filename + " --passphrase '" + passphrase + "' -d " + path
                response = muterun(system_command)
                # overwrite user entered passphrases
                passphrase = ""
                passphrase_confirm = ""

                if response.exitcode == 0:
                    stdout("Decryption complete")
                    sys.exit(0)
                else:
                    stderr(response.stderr, 0)
                    stderr("Decryption failed")
                    sys.exit(1)
            else:
                stderr("The passphrases did not match.  Please enter your command again.")
                sys.exit(1)
        elif dir_exists(path):  ## SINGLE DIRECTORY
            dirty_directory_file_list = list_all_files(path)
            directory_file_list = [x for x in dirty_directory_file_list if (x.endswith('.crypt') or x.endswith('.gpg') or x.endswith('.pgp') or x.endswith('.asc'))]

            # if there are no encrypted files found, warn and abort
            if len(directory_file_list) == 0:
                stderr("There are no encrypted files in the directory")
                sys.exit(1)

            #prompt for the passphrase
            passphrase = getpass.getpass("Please enter your passphrase: ")
            if len(passphrase) == 0: # confirm that user entered a passphrase
                stderr("You did not enter a passphrase. Please repeat your command and try again.")
                sys.exit(1)
            passphrase_confirm = getpass.getpass("Please enter your passphrase again: ")

            if passphrase == passphrase_confirm:
                # decrypt all of the encypted files in the directory
                for filepath in directory_file_list:
                    absolute_filepath = make_path(path, filepath) # combine the directory path and file name into absolute path

                    # remove file suffix from the decrypted file path that writes to disk
                    if absolute_filepath.endswith('.crypt'):
                        decrypted_filepath = absolute_filepath[0:-6] # remove the .crypt suffix
                    elif absolute_filepath.endswith('.gpg') or absolute_filepath.endswith('.pgp') or absolute_filepath.endswith('.asc'):
                        decrypted_filepath = absolute_filepath[0:-4]

                    # confirm that the file does not already exist
                    if file_exists(decrypted_filepath):
                        stdout("The file path '" + decrypted_filepath + "' already exists.  This file was not decrypted.")
                    else:
                        system_command = "gpg --batch -o " + decrypted_filepath + " --passphrase '" + passphrase + "' -d " + absolute_filepath
                        response = muterun(system_command)

                        if response.exitcode == 0:
                            stdout("'" + absolute_filepath + "' decrypted to '" + decrypted_filepath + "'")
                        else:
                            stderr(response.stderr)
                            stderr("Decryption failed for " + absolute_filepath)
                # overwrite user entered passphrases
                passphrase = ""
                passphrase_confirm = ""
            else:
                # overwrite user entered passphrases
                passphrase = ""
                passphrase_confirm = ""
                stderr("The passphrases did not match.  Please enter your command again.")
                sys.exit(1)
        else:
            # error message, not a file or directory.  user entry error
            stderr("The path that you entered does not appear to be an existing file or directory.  Please try again.")
            sys.exit(1)

    #------------------------------------------------------------------------------------------
    # [ DEFAULT MESSAGE FOR MATCH FAILURE ]
    #  Message to provide to the user when all above conditional logic fails to meet a true condition
    #------------------------------------------------------------------------------------------
    else:
        print("Could not complete your request.  Please try again.")
        sys.exit(1)

if __name__ == '__main__':
    main()
<commit_msg>decryption pauses added<commit_after>#!/usr/bin/env python
# encoding: utf-8

#------------------------------------------------------------------------------
# decrypto
# Copyright 2014 Christopher Simpkins
# MIT license
#------------------------------------------------------------------------------

# Application start
def main():
    import os
    import sys
    from time import sleep
    import getpass
    from Naked.commandline import Command
    from Naked.toolshed.shell import execute, muterun
    from Naked.toolshed.system import dir_exists, directory, filename, file_exists, list_all_files, make_path, stdout, stderr

    #------------------------------------------------------------------------------------------
    # [ Instantiate command line object ]
    #   used for all subsequent conditional logic in the CLI application
    #------------------------------------------------------------------------------------------
    c = Command(sys.argv[0], sys.argv[1:])
    #------------------------------------------------------------------------------------------
    # [ VALIDATION LOGIC ] - early validation of appropriate command syntax
    # Test that user entered at least one argument to the executable, print usage if not
    #------------------------------------------------------------------------------------------
    if not c.command_suite_validates():
        from crypto.settings import usage as crypto_usage
        print(crypto_usage)
        sys.exit(1)
    #------------------------------------------------------------------------------------------
    # [ HELP, VERSION, USAGE LOGIC ]
    # Naked framework provides default help, usage, and version commands for all applications
    #   --> settings for user messages are assigned in the lib/crypto/settings.py file
    #------------------------------------------------------------------------------------------
    if c.help():      # User requested crypto help information
        from crypto.settings import help as crypto_help
        print(crypto_help)
        sys.exit(0)
    elif c.usage():   # User requested crypto usage information
        from crypto.settings import usage as crypto_usage
        print(crypto_usage)
        sys.exit(0)
    elif c.version(): # User requested crypto version information
        from crypto.settings import app_name, major_version, minor_version, patch_version
        version_display_string = app_name + ' ' + major_version + '.' + minor_version + '.' + patch_version
        print(version_display_string)
        sys.exit(0)
    #------------------------------------------------------------------------------------------
    # [ APPLICATION LOGIC ]
    #
    #------------------------------------------------------------------------------------------
    elif c.argc > 1:
    # code for multi-file processing and commands that include options
        use_standard_output = False # print to stdout flag
        use_file_overwrite = False # overwrite existing files

        # set user option flags
        if c.option('--stdout') or c.option('-s'):
            use_standard_output = True
        if c.option('--overwrite') or c.option('-o'):
            use_file_overwrite = True

        directory_list = [] # directory paths included in the user entered paths from the command line
        file_list = [] # file paths included in the user entered paths from the command line (and inside directories entered)

        for argument in c.argv:
            if file_exists(argument): # user included a file, add it to the file_list for decryption
                if argument.endswith('.crypt'):
                    file_list.append(argument) # add .crypt files to the list of files for decryption
                elif argument.endswith('.gpg'):
                    file_list.append(argument)
                elif argument.endswith('.asc'):
                    file_list.append(argument)
                elif argument.endswith('.pgp'):
                    file_list.append(argument)
                else:
                    # cannot identify as an encrypted file, give it a shot anyways but warn user
                    file_list.append(argument)
                    stdout("Could not confirm that '" + argument + "' is encrypted based upon the file type.  Attempting decryption.  Keep your fingers crossed...")
            elif dir_exists(argument): # user included a directory, add it to the directory_list
                directory_list.append(argument)
            else:
                if argument[0] == "-":
                    pass # if it is an option, do nothing
                else:
                    stderr("'" + argument + "' does not appear to be an existing file or directory.  Aborting decryption attempt for this request.", 0)

        # unroll the contained directory files into the file_list IF they are encrypted file types
        if len(directory_list) > 0:
            for directory in directory_list:
                directory_file_list = list_all_files(directory)
                for contained_file in directory_file_list:
                    if contained_file.endswith('.crypt'):
                        file_list.append(make_path(directory, contained_file)) # include the file with a filepath 'directory path/contained_file path'
                    elif contained_file.endswith('.gpg'):
                        file_list.append(make_path(directory, contained_file))
                    elif contained_file.endswith('asc'):
                        file_list.append(make_path(directory, contained_file))
                    elif contained_file.endswith('.pgp'):
                        file_list.append(make_path(directory, contained_file))

        # confirm that there are files for decryption, if not abort
        if len(file_list) == 0:
            stderr("Could not identify files for decryption")
            sys.exit(1)

        # get passphrase used to symmetrically decrypt the file
        passphrase = getpass.getpass("Please enter your passphrase: ")
        if len(passphrase) == 0: # confirm that user entered a passphrase
                stderr("You did not enter a passphrase. Please repeat your command and try again.")
                sys.exit(1)
        passphrase_confirm = getpass.getpass("Please enter your passphrase again: ")

        if passphrase == passphrase_confirm:
            # begin decryption of each requested file.  the directory path was already added to the file path above
            for encrypted_file in file_list:
                # create the decrypted file name
                decrypted_filename = ""
                if encrypted_file.endswith('.crypt'):
                    decrypted_filename = encrypted_file[0:-6]
                elif encrypted_file.endswith('.gpg') or encrypted_file.endswith('.asc') or encrypted_file.endswith('.pgp'):
                    decrypted_filename = encrypted_file[0:-4]
                else:
                    decrypted_filename = encrypted_file + '.decrypt' # if it was a file without a known encrypted file type, add the .decrypt suffix

                # determine whether file overwrite will take place with the decrypted file
                skip_file = False # flag that indicates this file should not be encrypted
                created_tmp_files = False
                if not use_standard_output: # if not writing a file, no need to check for overwrite
                    if file_exists(decrypted_filename):
                        if use_file_overwrite: # rename the existing file to temp file which will be erased or replaced (on decryption failures) below
                            tmp_filename = decrypted_filename + '.tmp'
                            os.rename(decrypted_filename, tmp_filename)
                            created_tmp_files = True
                        else:
                            stdout("The file path '" + decrypted_filename + "' already exists.  This file was not decrypted.")
                            skip_file = True

                # begin decryption
                if not skip_file:
                    if use_standard_output: # using --quiet flag to suppress stdout messages from gpg, just want the file data in stdout stream
                        system_command = "gpg --batch --quiet --passphrase '" + passphrase + "' -d " + encrypted_file
                        successful_execution = execute(system_command) # use naked execute function to directly push to stdout, rather than return stdout

                        if not successful_execution:
                            stderr("Unable to decrypt file '" + encrypted_file + "'", 0)
                            if created_tmp_files: # restore the moved tmp file to original if decrypt failed
                                tmp_filename = decrypted_filename + '.tmp'
                                if file_exists(tmp_filename):
                                    os.rename(tmp_filename, decrypted_filename)
                        else: # decryption successful but we are in stdout flag so do not include any other output from decrypto
                            pass
                    else:
                        system_command = "gpg --batch -o " + decrypted_filename + " --passphrase '" + passphrase + "' -d " + encrypted_file
                        response = muterun(system_command)

                        if response.exitcode == 0:
                            stdout("'" + encrypted_file + "' decrypted to '" + decrypted_filename + "'")
                        else: # failed decryption
                            if created_tmp_files:# restore the moved tmp file to original if decrypt failed
                                tmp_filename = decrypted_filename + '.tmp'
                                if file_exists(tmp_filename):
                                    os.rename(tmp_filename, decrypted_filename)
                            # report the error
                            stderr(response.stderr)
                            stderr("Decryption failed for " + encrypted_file)

                # cleanup: remove the tmp file
                if created_tmp_files:
                    tmp_filename = decrypted_filename + '.tmp'
                    if file_exists(tmp_filename):
                        os.remove(tmp_filename)

            # overwrite the entered passphrases after file decryption is complete for all files
            passphrase = ""
            passphrase_confirm = ""

            # add a short pause to hinder brute force pexpect style password attacks with decrypto
            sleep(0.2)  # 200ms pause

        else: # passphrases did not match
            passphrase = ""
            passphrase_confirm = ""
            stderr("The passphrases did not match.  Please enter your command again.")
            sys.exit(1)

    elif c.argc == 1:
    # simple single file or directory processing with default settings
        path = c.arg0
        if file_exists(path): ## SINGLE FILE
            check_existing_file = False # check for a file with the name of new decrypted filename in the directory

            if path.endswith('.crypt'):
                new_filename = path[0:-6] # remove the .crypt suffix
                check_existing_file = True
            elif path.endswith('.gpg') or path.endswith('.pgp') or path.endswith('.asc'):
                new_filename = path[0:-4]
                check_existing_file = True
            else:
                new_filename = path + ".decrypt" #if there is not a standard file type, then add a .decrypt suffix to the decrypted file name
                stdout("Could not confirm that the requested file is encrypted based upon the file type.  Attempting decryption.  Keep your fingers crossed...")

            # confirm that the decrypted path does not already exist, if so abort with warning message to user
            if check_existing_file == True:
                if file_exists(new_filename):
                    stderr("Your file will be decrypted to '" + new_filename + "' and this file path already exists.  Please move the file or use the --overwrite option with your command if you intend to replace the current file.")
                    sys.exit(1)

            # get passphrase used to symmetrically decrypt the file
            passphrase = getpass.getpass("Please enter your passphrase: ")
            if len(passphrase) == 0: # confirm that user entered a passphrase
                stderr("You did not enter a passphrase. Please repeat your command and try again.")
                sys.exit(1)
            passphrase_confirm = getpass.getpass("Please enter your passphrase again: ")

            # confirm that the passphrases match
            if passphrase == passphrase_confirm:
                system_command = "gpg --batch -o " + new_filename + " --passphrase '" + passphrase + "' -d " + path
                response = muterun(system_command)
                # overwrite user entered passphrases
                passphrase = ""
                passphrase_confirm = ""

                if response.exitcode == 0:
                    stdout("Decryption complete")
                    sys.exit(0)
                else:
                    stderr(response.stderr)
                    stderr("Decryption failed")
                    # add a short pause to hinder brute force pexpect style password attacks with decrypto
                    sleep(0.2)  # 200ms pause
                    sys.exit(1)
            else:
                stderr("The passphrases did not match.  Please enter your command again.")
                sys.exit(1)
        elif dir_exists(path):  ## SINGLE DIRECTORY
            dirty_directory_file_list = list_all_files(path)
            directory_file_list = [x for x in dirty_directory_file_list if (x.endswith('.crypt') or x.endswith('.gpg') or x.endswith('.pgp') or x.endswith('.asc'))]

            # if there are no encrypted files found, warn and abort
            if len(directory_file_list) == 0:
                stderr("There are no encrypted files in the directory")
                sys.exit(1)

            #prompt for the passphrase
            passphrase = getpass.getpass("Please enter your passphrase: ")
            if len(passphrase) == 0: # confirm that user entered a passphrase
                stderr("You did not enter a passphrase. Please repeat your command and try again.")
                sys.exit(1)
            passphrase_confirm = getpass.getpass("Please enter your passphrase again: ")

            if passphrase == passphrase_confirm:
                # decrypt all of the encypted files in the directory
                for filepath in directory_file_list:
                    absolute_filepath = make_path(path, filepath) # combine the directory path and file name into absolute path

                    # remove file suffix from the decrypted file path that writes to disk
                    if absolute_filepath.endswith('.crypt'):
                        decrypted_filepath = absolute_filepath[0:-6] # remove the .crypt suffix
                    elif absolute_filepath.endswith('.gpg') or absolute_filepath.endswith('.pgp') or absolute_filepath.endswith('.asc'):
                        decrypted_filepath = absolute_filepath[0:-4]

                    # confirm that the file does not already exist
                    if file_exists(decrypted_filepath):
                        stdout("The file path '" + decrypted_filepath + "' already exists.  This file was not decrypted.")
                    else:
                        system_command = "gpg --batch -o " + decrypted_filepath + " --passphrase '" + passphrase + "' -d " + absolute_filepath
                        response = muterun(system_command)

                        if response.exitcode == 0:
                            stdout("'" + absolute_filepath + "' decrypted to '" + decrypted_filepath + "'")
                        else:
                            stderr(response.stderr)
                            stderr("Decryption failed for " + absolute_filepath)
                # overwrite user entered passphrases
                passphrase = ""
                passphrase_confirm = ""

                # add a short pause to hinder brute force pexpect style password attacks with decrypto
                sleep(0.2)  # 200ms pause
            else:
                # overwrite user entered passphrases
                passphrase = ""
                passphrase_confirm = ""
                stderr("The passphrases did not match.  Please enter your command again.")
                sys.exit(1)
        else:
            # error message, not a file or directory.  user entry error
            stderr("The path that you entered does not appear to be an existing file or directory.  Please try again.")
            sys.exit(1)

    #------------------------------------------------------------------------------------------
    # [ DEFAULT MESSAGE FOR MATCH FAILURE ]
    #  Message to provide to the user when all above conditional logic fails to meet a true condition
    #------------------------------------------------------------------------------------------
    else:
        print("Could not complete your request.  Please try again.")
        sys.exit(1)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import os
import socket
hostname = socket.gethostname()

build_repos = (
    'buildapi',
    'buildbot-configs',
    'buildbotcustom',
    'mozharness',
    'partner-repacks',
    'puppet',
    'talos',
    'tools',
)

conversion_repos = []
remote_targets = {}

for repo in build_repos:
    conversion_repos.append({
        "repo": "https://hg.mozilla.org/build/%s" % repo,
        "repo_name": "build-%s" % repo,
        "conversion_dir": "build-%s" % repo,
        "targets": [{
            "target_dest": "build-%s-github" % repo,
            "force_push": True
        }],
        "vcs": "hg",
        "mapper": {
            "url": "https://api.pub.build.mozilla.org/mapper",
            "project": "build-%s" % repo,
        },
        "branch_config": {
            "branches": {
                "default": "master",
            },
            "branch_regexes": [
                "^.*$"
            ]
        },
# Bug 1036819 - build/* repos currently not able to push tags to github
# temporarily disable tags in conversion.
# When bug 1020613 is resolved, this tag_config below can be enabled again.
#       "tag_config": {
#           "tag_regexes": [
#               "^.*$"
#           ]
#       },
    })
    remote_targets["build-%s-github" % repo] = {
        "repo": "git@github.com:mozilla/build-%s.git" % repo,
        "ssh_key": "~/.ssh/releng-github-id_rsa",
        "vcs": "git",
    }

# version-control-tools is here because adding it to this job was easier
# than defining its own config file and job.
conversion_repos.append({
    "repo": "https://hg.mozilla.org/hgcustom/version-control-tools",
    "repo_name": "version-control-tools",
    "conversion_dir": "version-control-tools",
    "targets": [{
        "target_dest": "version-control-tools-github",
        "force_push": True,
    }],
    "vcs": "hg",
    "mapper": {
        "url": "https://api.pub.build.mozilla.org/mapper",
        "project": "version-control-tools",
    },
    "branch_config": {
        "branches": {
            "default": "master",
        },
    },
})

remote_targets["version-control-tools-github"] = {
    "repo": "git@github.com:mozilla/version-control-tools.git",
    "ssh_key": "~/.ssh/releng-github-id_rsa",
    "vcs": "git",
}

# nss is here because adding it to this job was easier
# than defining its own config file and job.
conversion_repos.append({
    "repo": "https://hg.mozilla.org/projects/nss",
    "repo_name": "test-nss",
    "conversion_dir": "nss-dev",
    "targets": [{
        "target_dest": "nss-github",
        "force_push": True,
    }],
    "vcs": "hg",
    "mapper": {
        "url": "https://api.pub.build.mozilla.org/mapper",
        "project": "nss",
    },
    "branch_config": {
        "branches": {
            "default": "master",
        },
    },
})

remote_targets["nss-github"] = {
    "repo": "git@github.com:mozilla/test-nss.git",
    "ssh_key": "~/.ssh/releng-github-id_rsa",
    "vcs": "git",
}

config = {
    "log_name": "build-repos",
    "log_max_rotate": 99,
    "job_name": "build-repos",
    "env": {
        "PATH": "%(PATH)s:/usr/libexec/git-core",
    },
    "conversion_repos": conversion_repos,
    "remote_targets": remote_targets,
    "virtualenv_modules": [
        "dulwich==0.9.0",
        "ordereddict==1.1",
        "hg-git==0.4.0-moz2",
        "mapper==0.1",
        "mercurial==3.7.3",
        "mozfile==0.9",
        "mozinfo==0.5",
        "mozprocess==0.11",
        "requests==2.8.1",
    ],
    "find_links": [
        "http://pypi.pub.build.mozilla.org/pub"
    ],
    "pip_index": False,

    "default_notify_from": "developer-services+%s@mozilla.org" % hostname,
    "notify_config": [{
        "to": "releng-ops-trial@mozilla.com",
        "failure_only": False,
        "skip_empty_messages": True,
    }],

    # Disallow sharing, since we want pristine .hg and .git directories.
    "vcs_share_base": None,
    "hg_share_base": None,

    # any hg command line options
    "hg_options": (
        "--config",
        "web.cacerts=/etc/pki/tls/certs/ca-bundle.crt"
    ),

    "default_actions": [
        'list-repos',
        'create-virtualenv',
        'update-stage-mirror',
        'update-work-mirror',
        'publish-to-mapper',
        'push',
        'combine-mapfiles',
        'notify',
    ],
}
<commit_msg>bug 1285392 - switch to production repo on github<commit_after>import os
import socket
hostname = socket.gethostname()

build_repos = (
    'buildapi',
    'buildbot-configs',
    'buildbotcustom',
    'mozharness',
    'partner-repacks',
    'puppet',
    'talos',
    'tools',
)

conversion_repos = []
remote_targets = {}

for repo in build_repos:
    conversion_repos.append({
        "repo": "https://hg.mozilla.org/build/%s" % repo,
        "repo_name": "build-%s" % repo,
        "conversion_dir": "build-%s" % repo,
        "targets": [{
            "target_dest": "build-%s-github" % repo,
            "force_push": True
        }],
        "vcs": "hg",
        "mapper": {
            "url": "https://api.pub.build.mozilla.org/mapper",
            "project": "build-%s" % repo,
        },
        "branch_config": {
            "branches": {
                "default": "master",
            },
            "branch_regexes": [
                "^.*$"
            ]
        },
# Bug 1036819 - build/* repos currently not able to push tags to github
# temporarily disable tags in conversion.
# When bug 1020613 is resolved, this tag_config below can be enabled again.
#       "tag_config": {
#           "tag_regexes": [
#               "^.*$"
#           ]
#       },
    })
    remote_targets["build-%s-github" % repo] = {
        "repo": "git@github.com:mozilla/build-%s.git" % repo,
        "ssh_key": "~/.ssh/releng-github-id_rsa",
        "vcs": "git",
    }

# version-control-tools is here because adding it to this job was easier
# than defining its own config file and job.
conversion_repos.append({
    "repo": "https://hg.mozilla.org/hgcustom/version-control-tools",
    "repo_name": "version-control-tools",
    "conversion_dir": "version-control-tools",
    "targets": [{
        "target_dest": "version-control-tools-github",
        "force_push": True,
    }],
    "vcs": "hg",
    "mapper": {
        "url": "https://api.pub.build.mozilla.org/mapper",
        "project": "version-control-tools",
    },
    "branch_config": {
        "branches": {
            "default": "master",
        },
    },
})

remote_targets["version-control-tools-github"] = {
    "repo": "git@github.com:mozilla/version-control-tools.git",
    "ssh_key": "~/.ssh/releng-github-id_rsa",
    "vcs": "git",
}

# nss is here because adding it to this job was easier
# than defining its own config file and job.
conversion_repos.append({
    "repo": "https://hg.mozilla.org/projects/nss",
    "repo_name": "nss-dev",
    "conversion_dir": "nss-dev",
    "targets": [{
        "target_dest": "nss-github",
        "force_push": True,
    }],
    "vcs": "hg",
    "mapper": {
        "url": "https://api.pub.build.mozilla.org/mapper",
        "project": "nss",
    },
    "branch_config": {
        "branches": {
            "default": "master",
        },
    },
})

remote_targets["nss-github"] = {
    "repo": "git@github.com:nss-dev/nss.git",
    "ssh_key": "~/.ssh/releng-github-id_rsa",
    "vcs": "git",
}

config = {
    "log_name": "build-repos",
    "log_max_rotate": 99,
    "job_name": "build-repos",
    "env": {
        "PATH": "%(PATH)s:/usr/libexec/git-core",
    },
    "conversion_repos": conversion_repos,
    "remote_targets": remote_targets,
    "virtualenv_modules": [
        "dulwich==0.9.0",
        "ordereddict==1.1",
        "hg-git==0.4.0-moz2",
        "mapper==0.1",
        "mercurial==3.7.3",
        "mozfile==0.9",
        "mozinfo==0.5",
        "mozprocess==0.11",
        "requests==2.8.1",
    ],
    "find_links": [
        "http://pypi.pub.build.mozilla.org/pub"
    ],
    "pip_index": False,

    "default_notify_from": "developer-services+%s@mozilla.org" % hostname,
    "notify_config": [{
        "to": "releng-ops-trial@mozilla.com",
        "failure_only": False,
        "skip_empty_messages": True,
    }],

    # Disallow sharing, since we want pristine .hg and .git directories.
    "vcs_share_base": None,
    "hg_share_base": None,

    # any hg command line options
    "hg_options": (
        "--config",
        "web.cacerts=/etc/pki/tls/certs/ca-bundle.crt"
    ),

    "default_actions": [
        'list-repos',
        'create-virtualenv',
        'update-stage-mirror',
        'update-work-mirror',
        'publish-to-mapper',
        'push',
        'combine-mapfiles',
        'notify',
    ],
}
<|endoftext|>


<commit_before>"""
Slider
======

.. image:: images/slider.jpg

The :class:`Slider` widget looks like a scrollbar. It supports horizontal and
vertical orientations, min/max values and a default value.

To create a slider from -100 to 100 starting from 25::

    from kivy.uix.slider import Slider
    s = Slider(min=-100, max=100, value=25)

To create a vertical slider::

    from kivy.uix.slider import Slider
    s = Slider(orientation='vertical')

"""
__all__ = ('Slider', )

from kivy.uix.widget import Widget
from kivy.properties import (NumericProperty, AliasProperty, OptionProperty,
                             ReferenceListProperty, BoundedNumericProperty)
from kivy.metrics import sp

class Slider(Widget):
    """Class for creating a Slider widget.

    Check module documentation for more details.
    """

    value = NumericProperty(0.)
    '''Current value used for the slider.

    :attr:`value` is a :class:`~kivy.properties.NumericProperty` and defaults
    to 0.'''

    min = NumericProperty(0.)
    '''Minimum value allowed for :attr:`value`.

    :attr:`min` is a :class:`~kivy.properties.NumericProperty` and defaults to
    0.'''

    max = NumericProperty(100.)
    '''Maximum value allowed for :attr:`value`.

    :attr:`max` is a :class:`~kivy.properties.NumericProperty` and defaults to
    100.'''

    padding = NumericProperty(sp(16))
    '''Padding of the slider. The padding is used for graphical representation
    and interaction. It prevents the cursor from going out of the bounds of the
    slider bounding box.

    By default, padding is sp(16). The range of the slider is reduced from
    padding \*2 on the screen. It allows drawing the default cursor of sp(32)
    width without having the cursor go out of the widget.

    :attr:`padding` is a :class:`~kivy.properties.NumericProperty` and defaults
    to sp(16).'''

    orientation = OptionProperty('horizontal', options=(
        'vertical', 'horizontal'))
    '''Orientation of the slider.

    :attr:`orientation` is an :class:`~kivy.properties.OptionProperty` and
    defaults to 'horizontal'. Can take a value of 'vertical' or 'horizontal'.
    '''

    range = ReferenceListProperty(min, max)
    '''Range of the slider in the format (minimum value, maximum value)::

        >>> slider = Slider(min=10, max=80)
        >>> slider.range
        [10, 80]
        >>> slider.range = (20, 100)
        >>> slider.min
        20
        >>> slider.max
        100

    :attr:`range` is a :class:`~kivy.properties.ReferenceListProperty` of
    (:attr:`min`, :attr:`max`) properties.
    '''

    step = BoundedNumericProperty(0, min=0)
    '''Step size of the slider.

    .. versionadded:: 1.4.0

    Determines the size of each interval or step the slider takes between
    min and max. If the value range can't be evenly divisible by step the
    last step will be capped by slider.max

    :attr:`step` is a :class:`~kivy.properties.NumericProperty` and defaults
    to 1.'''

    # The following two methods constrain the slider's value
    # to range(min,max). Otherwise it may happen that self.value < self.min
    # at init.

    def on_min(self, *largs):
        self.value = min(self.max, max(self.min, self.value))

    def on_max(self, *largs):
        self.value = min(self.max, max(self.min, self.value))

    def get_norm_value(self):
        vmin = self.min
        d = self.max - vmin
        if d == 0:
            return 0
        return (self.value - vmin) / float(d)

    def set_norm_value(self, value):
        vmin = self.min
        vmax = self.max
        step = self.step
        val = min(value * (vmax - vmin) + vmin, vmax)
        if step == 0:
            self.value = val
        else:
            self.value = min(round((val - vmin) / step) * step + vmin,
                             vmax)
    value_normalized = AliasProperty(get_norm_value, set_norm_value,
                                     bind=('value', 'min', 'max', 'step'))
    '''Normalized value inside the :attr:`range` (min/max) to 0-1 range::

        >>> slider = Slider(value=50, min=0, max=100)
        >>> slider.value
        50
        >>> slider.value_normalized
        0.5
        >>> slider.value = 0
        >>> slider.value_normalized
        0
        >>> slider.value = 100
        >>> slider.value_normalized
        1

    You can also use it for setting the real value without knowing the minimum
    and maximum::

        >>> slider = Slider(min=0, max=200)
        >>> slider.value_normalized = .5
        >>> slider.value
        100
        >>> slider.value_normalized = 1.
        >>> slider.value
        200

    :attr:`value_normalized` is an :class:`~kivy.properties.AliasProperty`.
    '''

    def get_value_pos(self):
        padding = self.padding
        x = self.x
        y = self.y
        nval = self.value_normalized
        if self.orientation == 'horizontal':
            return (x + padding + nval * (self.width - 2 * padding), y)
        else:
            return (x, y + padding + nval * (self.height - 2 * padding))

    def set_value_pos(self, pos):
        padding = self.padding
        x = min(self.right - padding, max(pos[0], self.x + padding))
        y = min(self.top - padding, max(pos[1], self.y + padding))
        if self.orientation == 'horizontal':
            if self.width == 0:
                self.value_normalized = 0
            else:
                self.value_normalized = (x - self.x - padding
                                         ) / float(self.width - 2 * padding)
        else:
            if self.height == 0:
                self.value_normalized = 0
            else:
                self.value_normalized = (y - self.y - padding
                                         ) / float(self.height - 2 * padding)
    value_pos = AliasProperty(get_value_pos, set_value_pos,
                              bind=('x', 'y', 'width', 'height', 'min',
                                    'max', 'value_normalized', 'orientation'))
    '''Position of the internal cursor, based on the normalized value.

    :attr:`value_pos` is an :class:`~kivy.properties.AliasProperty`.
    '''

    def on_touch_down(self, touch):
        if self.disabled or not self.collide_point(*touch.pos):
            return
        if touch.is_mouse_scrolling:
            if 'down' in touch.button or 'left' in touch.button:
                if self.step:
                    self.value = min(self.max, self.value + self.step)
                else:
                    self.value = min(
                        self.max,
                        self.value + (self.max - self.min) / 20)
            if 'up' in touch.button or 'right' in touch.button:
                if self.step:
                    self.value = max(self.min, self.value - self.step)
                else:
                    self.value = max(
                        self.min,
                        self.value - (self.max - self.min) / 20)
        else:
            touch.grab(self)
            self.value_pos = touch.pos
        return True

    def on_touch_move(self, touch):
        if touch.grab_current == self:
            self.value_pos = touch.pos
            return True

    def on_touch_up(self, touch):
        if touch.grab_current == self:
            self.value_pos = touch.pos
            return True

if __name__ == '__main__':
    from kivy.app import App

    class SliderApp(App):
        def build(self):
            return Slider(padding=25)

    SliderApp().run()
<commit_msg>change use of dimension conversion to pattern used elsewhere, e.g. kivy.uix.label.<commit_after>"""
Slider
======

.. image:: images/slider.jpg

The :class:`Slider` widget looks like a scrollbar. It supports horizontal and
vertical orientations, min/max values and a default value.

To create a slider from -100 to 100 starting from 25::

    from kivy.uix.slider import Slider
    s = Slider(min=-100, max=100, value=25)

To create a vertical slider::

    from kivy.uix.slider import Slider
    s = Slider(orientation='vertical')

"""
__all__ = ('Slider', )

from kivy.uix.widget import Widget
from kivy.properties import (NumericProperty, AliasProperty, OptionProperty,
                             ReferenceListProperty, BoundedNumericProperty)

class Slider(Widget):
    """Class for creating a Slider widget.

    Check module documentation for more details.
    """

    value = NumericProperty(0.)
    '''Current value used for the slider.

    :attr:`value` is a :class:`~kivy.properties.NumericProperty` and defaults
    to 0.'''

    min = NumericProperty(0.)
    '''Minimum value allowed for :attr:`value`.

    :attr:`min` is a :class:`~kivy.properties.NumericProperty` and defaults to
    0.'''

    max = NumericProperty(100.)
    '''Maximum value allowed for :attr:`value`.

    :attr:`max` is a :class:`~kivy.properties.NumericProperty` and defaults to
    100.'''

    padding = NumericProperty('16sp')
    '''Padding of the slider. The padding is used for graphical representation
    and interaction. It prevents the cursor from going out of the bounds of the
    slider bounding box.

    By default, padding is 16sp. The range of the slider is reduced from
    padding \*2 on the screen. It allows drawing the default cursor of 32sp
    width without having the cursor go out of the widget.

    :attr:`padding` is a :class:`~kivy.properties.NumericProperty` and defaults
    to 16sp.'''

    orientation = OptionProperty('horizontal', options=(
        'vertical', 'horizontal'))
    '''Orientation of the slider.

    :attr:`orientation` is an :class:`~kivy.properties.OptionProperty` and
    defaults to 'horizontal'. Can take a value of 'vertical' or 'horizontal'.
    '''

    range = ReferenceListProperty(min, max)
    '''Range of the slider in the format (minimum value, maximum value)::

        >>> slider = Slider(min=10, max=80)
        >>> slider.range
        [10, 80]
        >>> slider.range = (20, 100)
        >>> slider.min
        20
        >>> slider.max
        100

    :attr:`range` is a :class:`~kivy.properties.ReferenceListProperty` of
    (:attr:`min`, :attr:`max`) properties.
    '''

    step = BoundedNumericProperty(0, min=0)
    '''Step size of the slider.

    .. versionadded:: 1.4.0

    Determines the size of each interval or step the slider takes between
    min and max. If the value range can't be evenly divisible by step the
    last step will be capped by slider.max

    :attr:`step` is a :class:`~kivy.properties.NumericProperty` and defaults
    to 1.'''

    # The following two methods constrain the slider's value
    # to range(min,max). Otherwise it may happen that self.value < self.min
    # at init.

    def on_min(self, *largs):
        self.value = min(self.max, max(self.min, self.value))

    def on_max(self, *largs):
        self.value = min(self.max, max(self.min, self.value))

    def get_norm_value(self):
        vmin = self.min
        d = self.max - vmin
        if d == 0:
            return 0
        return (self.value - vmin) / float(d)

    def set_norm_value(self, value):
        vmin = self.min
        vmax = self.max
        step = self.step
        val = min(value * (vmax - vmin) + vmin, vmax)
        if step == 0:
            self.value = val
        else:
            self.value = min(round((val - vmin) / step) * step + vmin,
                             vmax)
    value_normalized = AliasProperty(get_norm_value, set_norm_value,
                                     bind=('value', 'min', 'max', 'step'))
    '''Normalized value inside the :attr:`range` (min/max) to 0-1 range::

        >>> slider = Slider(value=50, min=0, max=100)
        >>> slider.value
        50
        >>> slider.value_normalized
        0.5
        >>> slider.value = 0
        >>> slider.value_normalized
        0
        >>> slider.value = 100
        >>> slider.value_normalized
        1

    You can also use it for setting the real value without knowing the minimum
    and maximum::

        >>> slider = Slider(min=0, max=200)
        >>> slider.value_normalized = .5
        >>> slider.value
        100
        >>> slider.value_normalized = 1.
        >>> slider.value
        200

    :attr:`value_normalized` is an :class:`~kivy.properties.AliasProperty`.
    '''

    def get_value_pos(self):
        padding = self.padding
        x = self.x
        y = self.y
        nval = self.value_normalized
        if self.orientation == 'horizontal':
            return (x + padding + nval * (self.width - 2 * padding), y)
        else:
            return (x, y + padding + nval * (self.height - 2 * padding))

    def set_value_pos(self, pos):
        padding = self.padding
        x = min(self.right - padding, max(pos[0], self.x + padding))
        y = min(self.top - padding, max(pos[1], self.y + padding))
        if self.orientation == 'horizontal':
            if self.width == 0:
                self.value_normalized = 0
            else:
                self.value_normalized = (x - self.x - padding
                                         ) / float(self.width - 2 * padding)
        else:
            if self.height == 0:
                self.value_normalized = 0
            else:
                self.value_normalized = (y - self.y - padding
                                         ) / float(self.height - 2 * padding)
    value_pos = AliasProperty(get_value_pos, set_value_pos,
                              bind=('x', 'y', 'width', 'height', 'min',
                                    'max', 'value_normalized', 'orientation'))
    '''Position of the internal cursor, based on the normalized value.

    :attr:`value_pos` is an :class:`~kivy.properties.AliasProperty`.
    '''

    def on_touch_down(self, touch):
        if self.disabled or not self.collide_point(*touch.pos):
            return
        if touch.is_mouse_scrolling:
            if 'down' in touch.button or 'left' in touch.button:
                if self.step:
                    self.value = min(self.max, self.value + self.step)
                else:
                    self.value = min(
                        self.max,
                        self.value + (self.max - self.min) / 20)
            if 'up' in touch.button or 'right' in touch.button:
                if self.step:
                    self.value = max(self.min, self.value - self.step)
                else:
                    self.value = max(
                        self.min,
                        self.value - (self.max - self.min) / 20)
        else:
            touch.grab(self)
            self.value_pos = touch.pos
        return True

    def on_touch_move(self, touch):
        if touch.grab_current == self:
            self.value_pos = touch.pos
            return True

    def on_touch_up(self, touch):
        if touch.grab_current == self:
            self.value_pos = touch.pos
            return True

if __name__ == '__main__':
    from kivy.app import App

    class SliderApp(App):
        def build(self):
            return Slider(padding=25)

    SliderApp().run()
<|endoftext|>


<commit_before>import os
import socket
hostname = socket.gethostname()

build_repos = (
    'buildapi',
    'buildbot-configs',
    'buildbotcustom',
    'mozharness',
    'partner-repacks',
    'puppet',
    'talos',
    'tools',
)

conversion_repos = []
remote_targets = {}

for repo in build_repos:
    conversion_repos.append({
        "repo": "https://hg.mozilla.org/build/%s" % repo,
        "repo_name": "build-%s" % repo,
        "conversion_dir": "build-%s" % repo,
        "targets": [{
            "target_dest": "build-%s-github" % repo,
            "force_push": True
        }],
        "vcs": "hg",
        "mapper": {
            "url": "https://api.pub.build.mozilla.org/mapper",
            "project": "build-%s" % repo,
        },
        "branch_config": {
            "branches": {
                "default": "master",
            },
            "branch_regexes": [
                "^.*$"
            ]
        },
# Bug 1036819 - build/* repos currently not able to push tags to github
# temporarily disable tags in conversion.
# When bug 1020613 is resolved, this tag_config below can be enabled again.
#       "tag_config": {
#           "tag_regexes": [
#               "^.*$"
#           ]
#       },
    })
    remote_targets["build-%s-github" % repo] = {
        "repo": "git@github.com:mozilla/build-%s.git" % repo,
        "ssh_key": "~/.ssh/releng-github-id_rsa",
        "vcs": "git",
    }

# version-control-tools is here because adding it to this job was easier
# than defining its own config file and job.
conversion_repos.append({
    "repo": "https://hg.mozilla.org/hgcustom/version-control-tools",
    "repo_name": "version-control-tools",
    "conversion_dir": "version-control-tools",
    "targets": [{
        "target_dest": "version-control-tools-github",
        "force_push": True,
    }],
    "vcs": "hg",
    "mapper": {
        "url": "https://api.pub.build.mozilla.org/mapper",
        "project": "version-control-tools",
    },
    "branch_config": {
        "branches": {
            "default": "master",
        },
    },
})

remote_targets["version-control-tools-github"] = {
    "repo": "git@github.com:mozilla/version-control-tools.git",
    "ssh_key": "~/.ssh/releng-github-id_rsa",
    "vcs": "git",
}

# nss is here because adding it to this job was easier
# than defining its own config file and job.
conversion_repos.append({
    "repo": "https://hg.mozilla.org/projects/nss",
    "repo_name": "nss-dev",
    "conversion_dir": "nss-dev",
    "targets": [{
        "target_dest": "nss-github",
        "force_push": True,
    }],
    "vcs": "hg",
    "mapper": {
        "url": "https://api.pub.build.mozilla.org/mapper",
        "project": "nss",
    },
    "branch_config": {
        "branches": {
            "default": "master",
        },
    },
})

remote_targets["nss-github"] = {
    "repo": "git@github.com:nss-dev/nss.git",
    "ssh_key": "~/.ssh/releng-github-id_rsa", "vcs": "git",
}

# comm-central is here because adding it to this job was easier
# than defining its own config file and job.
conversion_repos.append({
    "repo": "https://hg.mozilla.org/comm-central",
    "repo_name": "comm-central",
    "conversion_dir": "comm-central",
    "targets": [{
        "target_dest": "comm-central-github",
        "force_push": True,
    }],
    "vcs": "hg",
    "mapper": {
        "url": "https://api.pub.build.mozilla.org/mapper",
        "project": "comm-central",
    },
    "branch_config": {
        "branches": {
            "default": "master",
        },
    },
})

remote_targets["comm-central-github"] = {
    "repo": "git@github.com:mozilla/releases-comm-central.git",
    "ssh_key": "~/.ssh/releng-github-id_rsa",
    "vcs": "git",
}

config = {
    "log_name": "build-repos",
    "log_max_rotate": 99,
    "job_name": "build-repos",
    "env": {
        "PATH": "%(PATH)s:/usr/libexec/git-core",
    },
    "conversion_repos": conversion_repos,
    "remote_targets": remote_targets,
    "virtualenv_modules": [
        "dulwich==0.9.0",
        "ordereddict==1.1",
        "hg-git==0.4.0-moz2",
        "mapper==0.1",
        "mercurial==3.7.3",
        "mozfile==0.9",
        "mozinfo==0.5",
        "mozprocess==0.11",
        "requests==2.8.1",
    ],
    "find_links": [
        "http://pypi.pub.build.mozilla.org/pub"
    ],
    "pip_index": False,

    "default_notify_from": "developer-services+%s@mozilla.org" % hostname,
    "notify_config": [{
        "to": "releng-ops-trial@mozilla.com",
        "failure_only": False,
        "skip_empty_messages": True,
    }],

    # Disallow sharing, since we want pristine .hg and .git directories.
    "vcs_share_base": None,
    "hg_share_base": None,

    # any hg command line options
    "hg_options": (
        "--config",
        "web.cacerts=/etc/pki/tls/certs/ca-bundle.crt"
    ),

    "default_actions": [
        'list-repos',
        'create-virtualenv',
        'update-stage-mirror',
        'update-work-mirror',
        'publish-to-mapper',
        'push',
        'combine-mapfiles',
        'notify',
    ],
}
<commit_msg>bug 1307588 - push tags for nss-dev; r=dhouse<commit_after>import os
import socket
hostname = socket.gethostname()

build_repos = (
    'buildapi',
    'buildbot-configs',
    'buildbotcustom',
    'mozharness',
    'partner-repacks',
    'puppet',
    'talos',
    'tools',
)

conversion_repos = []
remote_targets = {}

for repo in build_repos:
    conversion_repos.append({
        "repo": "https://hg.mozilla.org/build/%s" % repo,
        "repo_name": "build-%s" % repo,
        "conversion_dir": "build-%s" % repo,
        "targets": [{
            "target_dest": "build-%s-github" % repo,
            "force_push": True
        }],
        "vcs": "hg",
        "mapper": {
            "url": "https://api.pub.build.mozilla.org/mapper",
            "project": "build-%s" % repo,
        },
        "branch_config": {
            "branches": {
                "default": "master",
            },
            "branch_regexes": [
                "^.*$"
            ]
        },
# Bug 1036819 - build/* repos currently not able to push tags to github
# temporarily disable tags in conversion.
# When bug 1020613 is resolved, this tag_config below can be enabled again.
#       "tag_config": {
#           "tag_regexes": [
#               "^.*$"
#           ]
#       },
    })
    remote_targets["build-%s-github" % repo] = {
        "repo": "git@github.com:mozilla/build-%s.git" % repo,
        "ssh_key": "~/.ssh/releng-github-id_rsa",
        "vcs": "git",
    }

# version-control-tools is here because adding it to this job was easier
# than defining its own config file and job.
conversion_repos.append({
    "repo": "https://hg.mozilla.org/hgcustom/version-control-tools",
    "repo_name": "version-control-tools",
    "conversion_dir": "version-control-tools",
    "targets": [{
        "target_dest": "version-control-tools-github",
        "force_push": True,
    }],
    "vcs": "hg",
    "mapper": {
        "url": "https://api.pub.build.mozilla.org/mapper",
        "project": "version-control-tools",
    },
    "branch_config": {
        "branches": {
            "default": "master",
        },
    },
})

remote_targets["version-control-tools-github"] = {
    "repo": "git@github.com:mozilla/version-control-tools.git",
    "ssh_key": "~/.ssh/releng-github-id_rsa",
    "vcs": "git",
}

# nss is here because adding it to this job was easier
# than defining its own config file and job.
conversion_repos.append({
    "repo": "https://hg.mozilla.org/projects/nss",
    "repo_name": "nss-dev",
    "conversion_dir": "nss-dev",
    "targets": [{
        "target_dest": "nss-github",
        "force_push": True,
        # push all tags for parity with legacy vcs-sync behavior
        "tag_config": {
            "tag_regexes": [
                "^.*$"
            ]
        },
    }],
    "vcs": "hg",
    "mapper": {
        "url": "https://api.pub.build.mozilla.org/mapper",
        "project": "nss",
    },
    "branch_config": {
        "branches": {
            "default": "master",
        },
    },
})

remote_targets["nss-github"] = {
    "repo": "git@github.com:nss-dev/nss.git",
    "ssh_key": "~/.ssh/releng-github-id_rsa", "vcs": "git",
}

# comm-central is here because adding it to this job was easier
# than defining its own config file and job.
conversion_repos.append({
    "repo": "https://hg.mozilla.org/comm-central",
    "repo_name": "comm-central",
    "conversion_dir": "comm-central",
    "targets": [{
        "target_dest": "comm-central-github",
        "force_push": True,
    }],
    "vcs": "hg",
    "mapper": {
        "url": "https://api.pub.build.mozilla.org/mapper",
        "project": "comm-central",
    },
    "branch_config": {
        "branches": {
            "default": "master",
        },
    },
})

remote_targets["comm-central-github"] = {
    "repo": "git@github.com:mozilla/releases-comm-central.git",
    "ssh_key": "~/.ssh/releng-github-id_rsa",
    "vcs": "git",
}

config = {
    "log_name": "build-repos",
    "log_max_rotate": 99,
    "job_name": "build-repos",
    "env": {
        "PATH": "%(PATH)s:/usr/libexec/git-core",
    },
    "conversion_repos": conversion_repos,
    "remote_targets": remote_targets,
    "virtualenv_modules": [
        "dulwich==0.9.0",
        "ordereddict==1.1",
        "hg-git==0.4.0-moz2",
        "mapper==0.1",
        "mercurial==3.7.3",
        "mozfile==0.9",
        "mozinfo==0.5",
        "mozprocess==0.11",
        "requests==2.8.1",
    ],
    "find_links": [
        "http://pypi.pub.build.mozilla.org/pub"
    ],
    "pip_index": False,

    "default_notify_from": "developer-services+%s@mozilla.org" % hostname,
    "notify_config": [{
        "to": "releng-ops-trial@mozilla.com",
        "failure_only": False,
        "skip_empty_messages": True,
    }],

    # Disallow sharing, since we want pristine .hg and .git directories.
    "vcs_share_base": None,
    "hg_share_base": None,

    # any hg command line options
    "hg_options": (
        "--config",
        "web.cacerts=/etc/pki/tls/certs/ca-bundle.crt"
    ),

    "default_actions": [
        'list-repos',
        'create-virtualenv',
        'update-stage-mirror',
        'update-work-mirror',
        'publish-to-mapper',
        'push',
        'combine-mapfiles',
        'notify',
    ],
}
<|endoftext|>


<commit_before>#!/usr/bin/python3

"""
AUTHOR: Matthew May - mcmay.web@gmail.com
"""

# Imports
import json
#import logging
import maxminddb
#import re
import redis
import io

from const import META, PORTMAP

from argparse import ArgumentParser, RawDescriptionHelpFormatter
from os import getuid
from sys import exit
from textwrap import dedent
from time import gmtime, localtime, sleep, strftime

# start the Redis server if it isn't started already.
# $ redis-server
# default port is 6379
# make sure system can use a lot of memory and overcommit memory

redis_ip = '127.0.0.1'
redis_instance = None

# required input paths
syslog_path = '/var/log/syslog'
#syslog_path = '/var/log/reverse-proxy.log'
db_path = '../DataServerDB/GeoLite2-City.mmdb'

# file to log data
#log_file_out = '/var/log/map_data_server.out'

# ip for headquarters
hq_ip = '8.8.8.8'

# stats
#server_start_time = strftime("%Y-%m-%d %H:%M:%S", gmtime())
event_count = 0
continents_tracked = {}
countries_tracked = {}
country_to_code = {}
ip_to_code = {}
ips_tracked = {}
unknowns = {}

# @IDEA
#---------------------------------------------------------
# Use a class to nicely wrap everything:
# Could attempt to do an access here
# now without worrying about key errors,
# or just keep the filled data structure
#
#class Instance(dict):
#
#    defaults = {
#                'city': {'names':{'en':None}},
#                'continent': {'names':{'en':None}},
#                'continent': {'code':None},
#                'country': {'names':{'en':None}},
#                'country': {'iso_code':None},
#                'location': {'latitude':None},
#                'location': {'longitude':None},
#                'location': {'metro_code':None},
#                'postal': {'code':None}
#                }
#
#    def __init__(self, seed):
#        self(seed)
#        backfill()
#
#    def backfill(self):
#        for default in self.defaults:
#            if default not in self:
#                self[default] = defaults[default]
#---------------------------------------------------------

# Create clean dictionary using unclean db dictionary contents
def clean_db(unclean):
    selected = {}
    for tag in META:
        head = None
        if tag['tag'] in unclean:
            head = unclean[tag['tag']]
            for node in tag['path']:
                if node in head:
                    head = head[node]
                else:
                    head = None
                    break
            selected[tag['lookup']] = head

    return selected

def connect_redis(redis_ip):
    r = redis.StrictRedis(host=redis_ip, port=6379, db=0)
    return r

def get_msg_type():
    # @TODO
    # Add support for more message types later
    return "Traffic"

# Check to see if packet is using an interesting TCP/UDP protocol based on source or destination port
def get_tcp_udp_proto(src_port, dst_port):
    src_port = int(src_port)
    dst_port = int(dst_port)

    if src_port in PORTMAP:
        return PORTMAP[src_port]
    if dst_port in PORTMAP:
        return PORTMAP[dst_port]

    return "OTHER"

def find_hq_lat_long(hq_ip):
    hq_ip_db_unclean = parse_maxminddb(db_path, hq_ip)
    if hq_ip_db_unclean:
        hq_ip_db_clean = clean_db(hq_ip_db_unclean)
        dst_lat = hq_ip_db_clean['latitude']
        dst_long = hq_ip_db_clean['longitude']
        hq_dict = {
                'dst_lat': dst_lat,
                'dst_long': dst_long
                }
        return hq_dict
    else:
        print('Please provide a valid IP address for headquarters')
        exit()

def parse_maxminddb(db_path, ip):
    try:
        reader = maxminddb.open_database(db_path)
        response = reader.get(ip)
        reader.close()
        return response
    except FileNotFoundError:
        print('DB not found')
        print('SHUTTING DOWN')
        exit()
    except ValueError:
        return False

# @TODO
# Refactor/improve parsing
# This function depends heavily on which appliances are generating logs
# For now it is only here for testing

def parse_syslog(line):
    line = line.split()
    data = line[-1]
    data = data.split(',')

    if len(data) != 6:
    #if len(data) != 4:
        print('NOT A VALID LOG')
        return False
    else:
        src_ip = data[0]
        dst_ip = data[1]
        src_port = data[2]
        dst_port = data[3]
        #test
        type_attack = data[4]
        cve_attack = data[5]
        data_dict = {
                    'src_ip': src_ip,
                    'dst_ip': dst_ip,
                    'src_port': src_port,
                    'dst_port': dst_port,
                    #test
                    'type_attack': type_attack,
                    'cve_attack': cve_attack
                    }
        return data_dict

def shutdown_and_report_stats():
    print('\nSHUTTING DOWN')
    # report stats tracked
    print('\nREPORTING STATS...')
    print('\nEvent Count: {}'.format(event_count)) # report event count
    print('\nContinent Stats...') # report continents stats
    for key in continents_tracked:
        print('{}: {}'.format(key, continents_tracked[key]))
    print('\nCountry Stats...') # report country stats
    for country in countries_tracked:
        print('{}: {}'.format(country, countries_tracked[country]))
    print('\nCountries to iso_codes...')
    for key in country_to_code:
        print('{}: {}'.format(key, country_to_code[key]))
    print('\nIP Stats...') # report IP stats
    for ip in ips_tracked:
        print('{}: {}'.format(ip, ips_tracked[ip]))
    print('\nIPs to iso_codes...')
    for key in ip_to_code:
        print('{}: {}'.format(key, ip_to_code[key]))
    print('\nUnknowns...')
    for key in unknowns:
        print('{}: {}'.format(key, unknowns[key]))
    # log stats tracked
    exit()

def menu():
    # instantiate parser
    parser = ArgumentParser(
            prog='DataServer.py',
            usage='%(progs)s [OPTIONS]',
            formatter_class=RawDescriptionHelpFormatter,
            description=dedent('''\
                    --------------------------------------------------------------
                    Data server for attack map application.
                    --------------------------------------------------------------'''))

    # @TODO --> Add support for command line args?
    # define command line arguments
    # parser.add_argument('-db', '--database', dest='db_path', required=True, type=str, help='path to maxmind database')
    # parser.add_argument('-m', '--readme', dest='readme', help='print readme')
    # parser.add_argument('-o', '--output', dest='output', help='file to write logs to')
    # parser.add_argument('-r', '--random', action='store_true', dest='randomize', help='generate random IPs/protocols for demo')
    # parser.add_argument('-rs', '--redis-server-ip', dest='redis_ip', type=str, help='redis server ip address')
    # parser.add_argument('-sp', '--syslog-path', dest='syslog_path', type=str, help='path to syslog file')
    parser.add_argument('-v', '--verbose', action='store_true', dest='verbose', help='run server in verbose mode')

    # parse arguments/options
    args = parser.parse_args()
    return args

def merge_dicts(*args):
    super_dict = {}
    for arg in args:
        super_dict.update(arg)
    return super_dict

def track_flags(super_dict, tracking_dict, key1, key2):
    if key1 in super_dict:
        if key2 in super_dict:
            if key1 in tracking_dict:
                return None
            else:
                tracking_dict[super_dict[key1]] = super_dict[key2]
        else:
            return None
    else:
        return None

def track_stats(super_dict, tracking_dict, key):
    if key in super_dict:
        node = super_dict[key]
        if node in tracking_dict:
            tracking_dict[node] += 1
        else:
            tracking_dict[node] = 1
    else:
        if key in unknowns:
            unknowns[key] += 1
        else:
            unknowns[key] = 1

def main():
    if getuid() != 0:
        print('Please run this script as root')
        print('SHUTTING DOWN')
        exit()

    global db_path, log_file_out, redis_ip, redis_instance, syslog_path, hq_ip
    global continents_tracked, countries_tracked, ips_tracked, postal_codes_tracked, event_count, unknown, ip_to_code, country_to_code

    args = menu()

    # connect to Redis
    redis_instance = connect_redis(redis_ip)

    # find HQ lat/long
    hq_dict = find_hq_lat_long(hq_ip)

    # follow/parse/format/publish syslog data
    with io.open(syslog_path, "r", encoding='ISO-8859-1') as syslog_file:
        while True:
            where = syslog_file.tell()
            line = syslog_file.readline()
            if not line:
                sleep(.1)
                syslog_file.seek(where)
            else:
                syslog_data_dict = parse_syslog(line)
                if syslog_data_dict:
                    ip_db_unclean = parse_maxminddb(db_path, syslog_data_dict['src_ip'])
                    if ip_db_unclean:
                        event_count += 1
                        ip_db_clean = clean_db(ip_db_unclean)

                        msg_type = {'msg_type': get_msg_type()}
                        msg_type2 = {'msg_type2': syslog_data_dict['type_attack']}
                        msg_type3 = {'msg_type3': syslog_data_dict['cve_attack']}

                        proto = {'protocol': get_tcp_udp_proto(
                                                            syslog_data_dict['src_port'],
                                                            syslog_data_dict['dst_port']
                                                            )}
                        super_dict = merge_dicts(
                                                hq_dict,
                                                ip_db_clean,
                                                msg_type,
                                                msg_type2,
                                                msg_type3,
                                                proto,
                                                syslog_data_dict
                                                )

                        # Track Stats
                        track_stats(super_dict, continents_tracked, 'continent')
                        track_stats(super_dict, countries_tracked, 'country')
                        track_stats(super_dict, ips_tracked, 'src_ip')
                        event_time = strftime("%d-%m-%Y %H:%M:%S", localtime()) # local time
                        #event_time = strftime("%Y-%m-%d %H:%M:%S", gmtime()) # UTC time
                        track_flags(super_dict, country_to_code, 'country', 'iso_code')
                        track_flags(super_dict, ip_to_code, 'src_ip', 'iso_code')

                        # Append stats to super_dict
                        super_dict['event_count'] = event_count
                        super_dict['continents_tracked'] = continents_tracked
                        super_dict['countries_tracked'] = countries_tracked
                        super_dict['ips_tracked'] = ips_tracked
                        super_dict['unknowns'] = unknowns
                        super_dict['event_time'] = event_time
                        super_dict['country_to_code'] = country_to_code
                        super_dict['ip_to_code'] = ip_to_code

                        json_data = json.dumps(super_dict)
                        redis_instance.publish('attack-map-production', json_data)

                        if args.verbose:
                            print(ip_db_unclean)
                            print('------------------------')
                            print(json_data)
                            print('Event Count: {}'.format(event_count))
                            print('------------------------')

                        print('Event Count: {}'.format(event_count))
                        print('------------------------')

                    else:
                        continue

if __name__ == '__main__':
    try:
        main()
    except KeyboardInterrupt:
        shutdown_and_report_stats()
<commit_msg>file no longer starts reading from beginning<commit_after>#!/usr/bin/python3

"""
AUTHOR: Matthew May - mcmay.web@gmail.com
"""

# Imports
import json
#import logging
import maxminddb
#import re
import redis
import io

from const import META, PORTMAP

from argparse import ArgumentParser, RawDescriptionHelpFormatter
from os import getuid
from sys import exit
#from textwrap import dedent
from time import gmtime, localtime, sleep, strftime

# start the Redis server if it isn't started already.
# $ redis-server
# default port is 6379
# make sure system can use a lot of memory and overcommit memory

redis_ip = '127.0.0.1'
redis_instance = None

# required input paths
syslog_path = '/var/log/syslog'
#syslog_path = '/var/log/reverse-proxy.log'
db_path = '../DataServerDB/GeoLite2-City.mmdb'

# file to log data
#log_file_out = '/var/log/map_data_server.out'

# ip for headquarters
hq_ip = '8.8.8.8'

# stats
server_start_time = strftime("%d-%m-%Y %H:%M:%S", localtime()) # local time
event_count = 0
continents_tracked = {}
countries_tracked = {}
country_to_code = {}
ip_to_code = {}
ips_tracked = {}
unknowns = {}

# @IDEA
#---------------------------------------------------------
# Use a class to nicely wrap everything:
# Could attempt to do an access here
# now without worrying about key errors,
# or just keep the filled data structure
#
#class Instance(dict):
#
#    defaults = {
#                'city': {'names':{'en':None}},
#                'continent': {'names':{'en':None}},
#                'continent': {'code':None},
#                'country': {'names':{'en':None}},
#                'country': {'iso_code':None},
#                'location': {'latitude':None},
#                'location': {'longitude':None},
#                'location': {'metro_code':None},
#                'postal': {'code':None}
#                }
#
#    def __init__(self, seed):
#        self(seed)
#        backfill()
#
#    def backfill(self):
#        for default in self.defaults:
#            if default not in self:
#                self[default] = defaults[default]
#---------------------------------------------------------

# Create clean dictionary using unclean db dictionary contents
def clean_db(unclean):
    selected = {}
    for tag in META:
        head = None
        if tag['tag'] in unclean:
            head = unclean[tag['tag']]
            for node in tag['path']:
                if node in head:
                    head = head[node]
                else:
                    head = None
                    break
            selected[tag['lookup']] = head

    return selected

def connect_redis(redis_ip):
    r = redis.StrictRedis(host=redis_ip, port=6379, db=0)
    return r

def get_msg_type():
    # @TODO
    # Add support for more message types later
    return "Traffic"

# Check to see if packet is using an interesting TCP/UDP protocol based on source or destination port
def get_tcp_udp_proto(src_port, dst_port):
    src_port = int(src_port)
    dst_port = int(dst_port)

    if src_port in PORTMAP:
        return PORTMAP[src_port]
    if dst_port in PORTMAP:
        return PORTMAP[dst_port]

    return "OTHER"

def find_hq_lat_long(hq_ip):
    hq_ip_db_unclean = parse_maxminddb(db_path, hq_ip)
    if hq_ip_db_unclean:
        hq_ip_db_clean = clean_db(hq_ip_db_unclean)
        dst_lat = hq_ip_db_clean['latitude']
        dst_long = hq_ip_db_clean['longitude']
        hq_dict = {
                'dst_lat': dst_lat,
                'dst_long': dst_long
                }
        return hq_dict
    else:
        print('Please provide a valid IP address for headquarters')
        exit()

def parse_maxminddb(db_path, ip):
    try:
        reader = maxminddb.open_database(db_path)
        response = reader.get(ip)
        reader.close()
        return response
    except FileNotFoundError:
        print('DB not found')
        print('SHUTTING DOWN')
        exit()
    except ValueError:
        return False

# @TODO
# Refactor/improve parsing
# This function depends heavily on which appliances are generating logs
# For now it is only here for testing

def parse_syslog(line):
    line = line.split()
    data = line[-1]
    data = data.split(',')

    if len(data) != 6:
        print('NOT A VALID LOG')
        return False
    else:
        src_ip = data[0]
        dst_ip = data[1]
        src_port = data[2]
        dst_port = data[3]
        type_attack = data[4]
        cve_attack = data[5]
        data_dict = {
                    'src_ip': src_ip,
                    'dst_ip': dst_ip,
                    'src_port': src_port,
                    'dst_port': dst_port,
                    'type_attack': type_attack,
                    'cve_attack': cve_attack
                    }
        return data_dict

def shutdown_and_report_stats():
    print('\nSHUTTING DOWN')
    # Report stats tracked
    print('\nREPORTING STATS...')
    print('\nEvent Count: {}'.format(event_count)) # report event count
    print('\nContinent Stats...') # report continents stats
    for key in continents_tracked:
        print('{}: {}'.format(key, continents_tracked[key]))
    print('\nCountry Stats...') # report country stats
    for country in countries_tracked:
        print('{}: {}'.format(country, countries_tracked[country]))
    print('\nCountries to iso_codes...')
    for key in country_to_code:
        print('{}: {}'.format(key, country_to_code[key]))
    print('\nIP Stats...') # report IP stats
    for ip in ips_tracked:
        print('{}: {}'.format(ip, ips_tracked[ip]))
    print('\nIPs to iso_codes...')
    for key in ip_to_code:
        print('{}: {}'.format(key, ip_to_code[key]))
    print('\nUnknowns...')
    for key in unknowns:
        print('{}: {}'.format(key, unknowns[key]))
    exit()

def menu():
    # Instantiate parser
    #parser = ArgumentParser(
    #        prog='DataServer.py',
    #        usage='%(progs)s [OPTIONS]',
    #        formatter_class=RawDescriptionHelpFormatter,
    #        description=dedent('''\
    #                --------------------------------------------------------------
    #                Data server for attack map application.
    #                --------------------------------------------------------------'''))

    # @TODO --> Add support for command line args?
    #define command line arguments
    #parser.add_argument('-db', '--database', dest='db_path', required=True, type=str, help='path to maxmind database')
    #parser.add_argument('-m', '--readme', dest='readme', help='print readme')
    #parser.add_argument('-o', '--output', dest='output', help='file to write logs to')
    #parser.add_argument('-r', '--random', action='store_true', dest='randomize', help='generate random IPs/protocols for demo')
    #parser.add_argument('-rs', '--redis-server-ip', dest='redis_ip', type=str, help='redis server ip address')
    #parser.add_argument('-sp', '--syslog-path', dest='syslog_path', type=str, help='path to syslog file')
    #parser.add_argument('-v', '--verbose', action='store_true', dest='verbose', help='run server in verbose mode')

    # Parse arguments/options
    args = parser.parse_args()
    return args

def merge_dicts(*args):
    super_dict = {}
    for arg in args:
        super_dict.update(arg)
    return super_dict

def track_flags(super_dict, tracking_dict, key1, key2):
    if key1 in super_dict:
        if key2 in super_dict:
            if key1 in tracking_dict:
                return None
            else:
                tracking_dict[super_dict[key1]] = super_dict[key2]
        else:
            return None
    else:
        return None

def track_stats(super_dict, tracking_dict, key):
    if key in super_dict:
        node = super_dict[key]
        if node in tracking_dict:
            tracking_dict[node] += 1
        else:
            tracking_dict[node] = 1
    else:
        if key in unknowns:
            unknowns[key] += 1
        else:
            unknowns[key] = 1

def main():
    if getuid() != 0:
        print('Please run this script as root')
        print('SHUTTING DOWN')
        exit()

    global db_path, log_file_out, redis_ip, redis_instance, syslog_path, hq_ip
    global continents_tracked, countries_tracked, ips_tracked, postal_codes_tracked, event_count, unknown, ip_to_code, country_to_code

    args = menu()

    # Connect to Redis
    redis_instance = connect_redis(redis_ip)

    # Find HQ lat/long
    hq_dict = find_hq_lat_long(hq_ip)

    # Follow/parse/format/publish syslog data
    with io.open(syslog_path, "r", encoding='ISO-8859-1') as syslog_file:
        syslog_file.readlines()
        while True:
            where = syslog_file.tell()
            line = syslog_file.readline()
            if not line:
                sleep(.1)
                syslog_file.seek(where)
            else:
                syslog_data_dict = parse_syslog(line)
                if syslog_data_dict:
                    ip_db_unclean = parse_maxminddb(db_path, syslog_data_dict['src_ip'])
                    if ip_db_unclean:
                        event_count += 1
                        ip_db_clean = clean_db(ip_db_unclean)

                        msg_type = {'msg_type': get_msg_type()}
                        msg_type2 = {'msg_type2': syslog_data_dict['type_attack']}
                        msg_type3 = {'msg_type3': syslog_data_dict['cve_attack']}

                        proto = {'protocol': get_tcp_udp_proto(
                                                            syslog_data_dict['src_port'],
                                                            syslog_data_dict['dst_port']
                                                            )}
                        super_dict = merge_dicts(
                                                hq_dict,
                                                ip_db_clean,
                                                msg_type,
                                                msg_type2,
                                                msg_type3,
                                                proto,
                                                syslog_data_dict
                                                )

                        # Track Stats
                        track_stats(super_dict, continents_tracked, 'continent')
                        track_stats(super_dict, countries_tracked, 'country')
                        track_stats(super_dict, ips_tracked, 'src_ip')
                        event_time = strftime("%d-%m-%Y %H:%M:%S", localtime()) # local time
                        #event_time = strftime("%Y-%m-%d %H:%M:%S", gmtime()) # UTC time
                        track_flags(super_dict, country_to_code, 'country', 'iso_code')
                        track_flags(super_dict, ip_to_code, 'src_ip', 'iso_code')

                        # Append stats to super_dict
                        super_dict['event_count'] = event_count
                        super_dict['continents_tracked'] = continents_tracked
                        super_dict['countries_tracked'] = countries_tracked
                        super_dict['ips_tracked'] = ips_tracked
                        super_dict['unknowns'] = unknowns
                        super_dict['event_time'] = event_time
                        super_dict['country_to_code'] = country_to_code
                        super_dict['ip_to_code'] = ip_to_code

                        json_data = json.dumps(super_dict)
                        redis_instance.publish('attack-map-production', json_data)

                        #if args.verbose:
                        #    print(ip_db_unclean)
                        #    print('------------------------')
                        #    print(json_data)
                        #    print('Event Count: {}'.format(event_count))
                        #    print('------------------------')

                        print('Event Count: {}'.format(event_count))
                        print('------------------------')

                    else:
                        continue

if __name__ == '__main__':
    try:
        main()
    except KeyboardInterrupt:
        shutdown_and_report_stats()
<|endoftext|>