<commit_before>#!/usr/bin/env python
#
#/* ***** BEGIN LICENSE BLOCK	****
# Version: MPL 1.1/GPL 2.0/LGPL 2.1
#
# The contents of this file are subject to the Mozilla Public License Version
# 1.1 (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
# http://www.mozilla.org/MPL/
#
# Software distributed under the License is distributed on an "AS IS" basis,
# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
# for the specific language governing rights and limitations under the
# License.
#
# The Original Code is startjsfunfuzz.
#
# The Initial Developer of the Original Code is
# Gary Kwong.
# Portions created by the Initial Developer are Copyright (C) 2008-2010
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
#
# Alternatively, the contents of this file may be used under the terms of
# either the GNU General Public License Version 2 or later (the "GPL"), or
# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
# in which case the provisions of the GPL or the LGPL are applicable instead
# of those above. If you wish to allow use of your version of this file only
# under the terms of either the GPL or the LGPL, and not to allow others to
# use your version of this file under the terms of the MPL, indicate your
# decision by deleting the provisions above and replace them with the notice
# and other provisions required by the GPL or the LGPL. If you do not delete
# the provisions above, a recipient may use your version of this file under
# the terms of any one of the MPL, the GPL or the LGPL.

'''
This file contains functions for startjsfunfuzz.py.
'''

import os, platform, shutil, subprocess

verbose = True  # Turn this to True to enable verbose output for debugging.

def exceptionBadCompileType():
    raise Exception('Unknown compileType')
def exceptionBadBranchType():
    raise Exception('Unknown branchType')
def exceptionBadOs():
    raise Exception("Unknown OS - Platform is unsupported.")

def verboseDump(input):
    '''
    This function appends the word 'DEBUG' to any verbose output.
    '''
    if verbose:
        print 'DEBUG -', input

def osCheck():
    '''
    This function checks for supported operating systems.
    It returns macVer in the case of 10.5.x or 10.6.x.
    '''
    if os.name == 'posix':
        if os.uname()[0] == 'Darwin':
            macVer, _, _ = platform.mac_ver()
            macVer = float('.'.join(macVer.split('.')[:2]))
            if ('10.5' or '10.6' in str(macVer)):
                return str(macVer)
            else:
                exceptionBadOs()  # Only 10.5.x and 10.6.x is supported.
        elif os.uname()[0] == 'Linux':
            pass
    elif os.name == 'nt':
        if float(sys.version[:3]) < 2.6:
            raise Exception('A minimum Python version of 2.6 is required.')
    else:
        print '\nOnly Windows XP/Vista/7, Linux or Mac OS X 10.6.x are supported.\n'
        raise Exception('Unknown OS - Platform is unsupported.')

def error(branchSupp):
    '''
    This function prints the corresponding CLI requirements that should be input.
    '''
    print '\n==========\n| Error! |\n=========='
    print 'General usage: python startjsfunfuzz.py [32|64] [dbg|opt]',
    print '%s [patch <directory to patch>] [patch <directory to patch>]' % branchSupp,
    print '[valgrind]\n'
    print
    print 'Requirements: Python 2.6.x, Mozilla build prerequisites and repositories at "/" (WinXP) or "~/" (other platforms).'
    print
    print 'Windows platforms only compile in 32-bit.'
    print 'Valgrind only works for Linux platforms.\n'

def captureStdout(input):
    '''
    This function captures standard output into a python string.
    '''
    p = subprocess.Popen([input], stdin=subprocess.PIPE,stdout=subprocess.PIPE, shell=True)
    (stdout, stderr) = p.communicate()
    return stdout

def hgHashAddToFuzzPath(fuzzPath):
    '''
    This function finds the mercurial revision and appends it to the directory name.
    It also prompts if the user wants to continue, should the repository not be on tip.
    '''
    print
    verboseDump('About to start running `hg identify` commands...')
    tipOrNot = captureStdout('hg identify')[:-1]
    hgIdentifynMinus1 = captureStdout('hg identify -n')[:-1]
    # -5 is to remove the " tip\n" portion of `hg identify` output if on tip.
    hgIdentifyMinus5 = tipOrNot[:-4]
    onTip = True
    if tipOrNot.endswith('tip'):
        fuzzPath2 = fuzzPath[:-1] + '-' + hgIdentifynMinus1
        fuzzPath = fuzzPath2 + '-' + hgIdentifyMinus5
    else:
        print '`hg identify` shows the repository is on this changeset -', hgIdentifynMinus1 + ':' + tipOrNot
        notOnTipApproval = str(raw_input('Not on tip! Are you sure you want to continue? (y/n): '))
        if notOnTipApproval == ('y' or 'yes'):
            onTip = False
            fuzzPath = fuzzPath[:-1] + '-' + hgIdentifynMinus1 + '-' + tipOrNot
        else:
            switchToTipApproval = str(raw_input('Do you want to switch to the default tip? (y/n): '))
            if switchToTipApproval == ('y' or 'yes'):
                subprocess.call(['hg', 'up', 'default'])
                fuzzPath2 = fuzzPath[:-1] + '-' + hgIdentifynMinus1
                fuzzPath = fuzzPath2 + '-' + hgIdentifyMinus5
            else:
                raise Exception('Not on tip.')
    fuzzPath += '/'
    print
    verboseDump('Finished running `hg identify` commands.')
    return fuzzPath, onTip

def cpJsTreeOrPymakeDir(repo, jsOrBuild):
    '''
    This function copies the js tree or the pymake build directory.
    '''
    repo += 'js/src/' if jsOrBuild == 'js' else 'build/'
    if not platform.platform() == 'Windows-XP-5.1.2600-SP3':
        repo = os.path.expanduser(repo)
    try:
        jsOrBuildText = 'js tree' if jsOrBuild == 'js' else 'pymake build dir'
        verboseDump('Copying the ' + jsOrBuildText + ', which is located at ' + repo)
        shutil.copytree(repo, "compilePath", ignore=shutil.ignore_patterns('tests', 'trace-test', 'xpconnect')) \
            if jsOrBuild == 'js' else shutil.copytree(repo, "build")
        verboseDump('Finished copying the ' + jsOrBuildText)
    except OSError:
        raise Exception("The', jsOrBuildText, 'directory located at '" + repo + "' doesn't exist!")

def autoconfRun():
    '''
    Sniff platform and run different autoconf types:
    '''
    if os.name == 'posix':
        if os.uname()[0] == 'Darwin':
            subprocess.call(['autoconf213'])
        elif os.uname()[0] == 'Linux':
            subprocess.call(['autoconf2.13'])
    elif os.name == 'nt':
        subprocess.call(['sh', 'autoconf-2.13'])

def cfgJsBin(archNum, compileType, branchType, traceJit, methodJit,
                      valgrindSupport, threadsafe, macver):
    '''
    This function configures a js binary depending on the parameters.
    '''
    cfgCmd = 'sh ../configure'
    if (archNum == '32') and (os.name == 'posix'):
        if os.uname()[0] == "Darwin":
            if macver == '10.6':
                cfgCmd = 'CC="gcc-4.2 -arch i386" CXX="g++-4.2 -arch i386" ' + \
                             'HOST_CC="gcc-4.2" HOST_CXX="g++-4.2" ' + \
                             'RANLIB=ranlib AR=ar AS=$CC LD=ld' + \
                             'STRIP="strip -x -S" CROSS_COMPILE=1' + \
                             'sh ../configure --target=i386-apple-darwin8.0.0'
        elif (os.uname()[0] == "Linux") and (os.uname()[4] != 'armv7l'):
            # Apt-get `ia32-libs gcc-multilib g++-multilib` first, if on 64-bit Linux.
            cfgCmd = 'CC="gcc -m32" CXX="g++ -m32" AR=ar sh ../configure --target=i686-pc-linux'
        elif os.uname()[4] == 'armv7l':
            cfgCmd = 'CC=/opt/cs2007q3/bin/gcc CXX=/opt/cs2007q3/bin/g++ ' + \
                         'sh ../configure'
    if (archNum == '64') and (macver == '10.5'):
        cfgCmd = 'CC="gcc -m64" CXX="g++ -m64" AR=ar ' + \
                     'sh ../configure --target=x86_64-apple-darwin10.0.0'

    if compileType == 'dbg':
        cfgCmd += ' --disable-tests --disable-optimize --enable-debug'
    elif compileType == 'opt':
        cfgCmd += ' --disable-tests --enable-optimize --disable-debug'

    # Trace JIT is on by default.
    if not traceJit:
        cfgCmd += ' --disable-tracejit'
    # Method JIT is off by default.
    if methodJit:
        cfgCmd += ' --enable-methodjit'
    if valgrindSupport:
        cfgCmd += ' --enable-valgrind'
    if threadsafe:
        cfgCmd += ' --enable-threadsafe --with-system-nspr'

    verboseDump('This is the configure command:')
    verboseDump('%s\n' % cfgCmd)

    subprocess.call([cfgCmd], shell=True)

def compileCopy(archNum, compileType, branchType, usePymake):
    '''
    This function compiles and copies a binary.
    '''
    # Run make using 2 cores, not sure if pymake allows parallel compilation yet.
    subprocess.call(['python', '-O', '../../build/pymake/make.py', '-j2', '-s']) if usePymake \
        else subprocess.call(['make', '-j2', '-s'])
    # Sniff platform and rename executable accordingly:
    if os.name == 'posix':
        shellName = 'js-' + compileType + '-' + archNum + '-' + branchType + '-' + os.uname()[0].lower()
        shutil.copy2('js', '../../' + shellName)
    elif os.name == 'nt':
        shellName = 'js-' + compileType + '-' + archNum + '-' + branchType + '-' + os.name.lower()
        shutil.copy2('js.exe', '../../' + shellName + '.exe')
    return shellName

def test32or64bit(jsShellName, archNum):
    '''
    This function tests if a binary is 32-bit or 64-bit.
    '''
    test32or64bitCmd = 'file ' + jsShellName
    test32or64bitStr = captureStdout(test32or64bitCmd)[:-1]
    if archNum == '32':
        if verbose:
            if '386' in test32or64bitStr:
                print 'test32or64bitStr is:', test32or64bitStr
                verboseDump('Compiled binary is 32-bit.')
        if '386' not in test32or64bitStr:
            raise Exception('Compiled binary is not 32-bit.')
    elif archNum == '64':
        if verbose:
            if '64-bit' in test32or64bitStr:
                print 'test32or64bitStr is:', test32or64bitStr
                verboseDump('Compiled binary is 64-bit.')
        if '64-bit' not in test32or64bitStr:
            raise Exception('Compiled binary is not 64-bit.')

def testDbgOrOpt(jsShellName, compileType):
    '''
    This function tests if a binary is a debug or optimized shell.
    '''
    # Create a testfile with the gczeal() function.
    compileTypeTestName = 'compileTypeTest.js'
    compileTypeTestFile = open(compileTypeTestName, 'w')
    compileTypeTestFile.writelines('gczeal()')
    compileTypeTestFile.close()
    # Test that compileTypeTestFile is indeed created.
    if not os.path.isfile(compileTypeTestName):
        raise Exception(compileTypeTestName, 'does not exist.')

    if os.name == 'posix':
        testFileErrNum = subprocess.call(['./' + jsShellName, compileTypeTestName])
    elif os.name == 'nt':
        testFileErrNum = subprocess.call([jsShellName, compileTypeTestName], shell=True)
    os.remove(compileTypeTestName)  # Remove testfile after grabbing the error code.

    verboseDump('The error code for debug shells should be 0.')
    verboseDump('The error code for opt shells should be 3.')
    verboseDump('The actual error code for ' + jsShellName + ' now, is: ' + str(testFileErrNum))

    # The error code for debug shells when passing in the gczeal() function should be 0.
    if compileType == 'dbg' and testFileErrNum != 0:
        print 'ERROR: A debug shell when tested with the gczeal() should return "0" as the error code.'
        print 'compileType is: ' + compileType
        print 'testFileErrNum is: ' + str(testFileErrNum)
        print
        raise Exception('The compiled binary is not a debug shell.')
    # The error code for debug shells when passing in the gczeal() function
    # should be 3, because they don't have the function compiled in.
    elif compileType == 'opt' and testFileErrNum != 3:
        print 'ERROR: An optimized shell when tested with the gczeal() should return "3" as the error code.'
        print 'compileType is: ' + compileType
        print 'testFileErrNum is: ' + str(testFileErrNum)
        print
        raise Exception('The compiled binary is not an optimized shell.')

if __name__ == '__main__':
    pass
<commit_msg>Fix up 32-bit binary detection logic.<commit_after>#!/usr/bin/env python
#
#/* ***** BEGIN LICENSE BLOCK	****
# Version: MPL 1.1/GPL 2.0/LGPL 2.1
#
# The contents of this file are subject to the Mozilla Public License Version
# 1.1 (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
# http://www.mozilla.org/MPL/
#
# Software distributed under the License is distributed on an "AS IS" basis,
# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
# for the specific language governing rights and limitations under the
# License.
#
# The Original Code is startjsfunfuzz.
#
# The Initial Developer of the Original Code is
# Gary Kwong.
# Portions created by the Initial Developer are Copyright (C) 2008-2010
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
#
# Alternatively, the contents of this file may be used under the terms of
# either the GNU General Public License Version 2 or later (the "GPL"), or
# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
# in which case the provisions of the GPL or the LGPL are applicable instead
# of those above. If you wish to allow use of your version of this file only
# under the terms of either the GPL or the LGPL, and not to allow others to
# use your version of this file under the terms of the MPL, indicate your
# decision by deleting the provisions above and replace them with the notice
# and other provisions required by the GPL or the LGPL. If you do not delete
# the provisions above, a recipient may use your version of this file under
# the terms of any one of the MPL, the GPL or the LGPL.

'''
This file contains functions for startjsfunfuzz.py.
'''

import os, platform, shutil, subprocess

verbose = True  # Turn this to True to enable verbose output for debugging.

def exceptionBadCompileType():
    raise Exception('Unknown compileType')
def exceptionBadBranchType():
    raise Exception('Unknown branchType')
def exceptionBadOs():
    raise Exception("Unknown OS - Platform is unsupported.")

def verboseDump(input):
    '''
    This function appends the word 'DEBUG' to any verbose output.
    '''
    if verbose:
        print 'DEBUG -', input

def osCheck():
    '''
    This function checks for supported operating systems.
    It returns macVer in the case of 10.5.x or 10.6.x.
    '''
    if os.name == 'posix':
        if os.uname()[0] == 'Darwin':
            macVer, _, _ = platform.mac_ver()
            macVer = float('.'.join(macVer.split('.')[:2]))
            if ('10.5' or '10.6' in str(macVer)):
                return str(macVer)
            else:
                exceptionBadOs()  # Only 10.5.x and 10.6.x is supported.
        elif os.uname()[0] == 'Linux':
            pass
    elif os.name == 'nt':
        if float(sys.version[:3]) < 2.6:
            raise Exception('A minimum Python version of 2.6 is required.')
    else:
        print '\nOnly Windows XP/Vista/7, Linux or Mac OS X 10.6.x are supported.\n'
        raise Exception('Unknown OS - Platform is unsupported.')

def error(branchSupp):
    '''
    This function prints the corresponding CLI requirements that should be input.
    '''
    print '\n==========\n| Error! |\n=========='
    print 'General usage: python startjsfunfuzz.py [32|64] [dbg|opt]',
    print '%s [patch <directory to patch>] [patch <directory to patch>]' % branchSupp,
    print '[valgrind]\n'
    print
    print 'Requirements: Python 2.6.x, Mozilla build prerequisites and repositories at "/" (WinXP) or "~/" (other platforms).'
    print
    print 'Windows platforms only compile in 32-bit.'
    print 'Valgrind only works for Linux platforms.\n'

def captureStdout(input):
    '''
    This function captures standard output into a python string.
    '''
    p = subprocess.Popen([input], stdin=subprocess.PIPE,stdout=subprocess.PIPE, shell=True)
    (stdout, stderr) = p.communicate()
    return stdout

def hgHashAddToFuzzPath(fuzzPath):
    '''
    This function finds the mercurial revision and appends it to the directory name.
    It also prompts if the user wants to continue, should the repository not be on tip.
    '''
    print
    verboseDump('About to start running `hg identify` commands...')
    tipOrNot = captureStdout('hg identify')[:-1]
    hgIdentifynMinus1 = captureStdout('hg identify -n')[:-1]
    # -5 is to remove the " tip\n" portion of `hg identify` output if on tip.
    hgIdentifyMinus5 = tipOrNot[:-4]
    onTip = True
    if tipOrNot.endswith('tip'):
        fuzzPath2 = fuzzPath[:-1] + '-' + hgIdentifynMinus1
        fuzzPath = fuzzPath2 + '-' + hgIdentifyMinus5
    else:
        print '`hg identify` shows the repository is on this changeset -', hgIdentifynMinus1 + ':' + tipOrNot
        notOnTipApproval = str(raw_input('Not on tip! Are you sure you want to continue? (y/n): '))
        if notOnTipApproval == ('y' or 'yes'):
            onTip = False
            fuzzPath = fuzzPath[:-1] + '-' + hgIdentifynMinus1 + '-' + tipOrNot
        else:
            switchToTipApproval = str(raw_input('Do you want to switch to the default tip? (y/n): '))
            if switchToTipApproval == ('y' or 'yes'):
                subprocess.call(['hg', 'up', 'default'])
                fuzzPath2 = fuzzPath[:-1] + '-' + hgIdentifynMinus1
                fuzzPath = fuzzPath2 + '-' + hgIdentifyMinus5
            else:
                raise Exception('Not on tip.')
    fuzzPath += '/'
    print
    verboseDump('Finished running `hg identify` commands.')
    return fuzzPath, onTip

def cpJsTreeOrPymakeDir(repo, jsOrBuild):
    '''
    This function copies the js tree or the pymake build directory.
    '''
    repo += 'js/src/' if jsOrBuild == 'js' else 'build/'
    if not platform.platform() == 'Windows-XP-5.1.2600-SP3':
        repo = os.path.expanduser(repo)
    try:
        jsOrBuildText = 'js tree' if jsOrBuild == 'js' else 'pymake build dir'
        verboseDump('Copying the ' + jsOrBuildText + ', which is located at ' + repo)
        shutil.copytree(repo, "compilePath", ignore=shutil.ignore_patterns('tests', 'trace-test', 'xpconnect')) \
            if jsOrBuild == 'js' else shutil.copytree(repo, "build")
        verboseDump('Finished copying the ' + jsOrBuildText)
    except OSError:
        raise Exception("The', jsOrBuildText, 'directory located at '" + repo + "' doesn't exist!")

def autoconfRun():
    '''
    Sniff platform and run different autoconf types:
    '''
    if os.name == 'posix':
        if os.uname()[0] == 'Darwin':
            subprocess.call(['autoconf213'])
        elif os.uname()[0] == 'Linux':
            subprocess.call(['autoconf2.13'])
    elif os.name == 'nt':
        subprocess.call(['sh', 'autoconf-2.13'])

def cfgJsBin(archNum, compileType, branchType, traceJit, methodJit,
                      valgrindSupport, threadsafe, macver):
    '''
    This function configures a js binary depending on the parameters.
    '''
    cfgCmd = 'sh ../configure'
    if (archNum == '32') and (os.name == 'posix'):
        if os.uname()[0] == "Darwin":
            if macver == '10.6':
                cfgCmd = 'CC="gcc-4.2 -arch i386" CXX="g++-4.2 -arch i386" ' + \
                             'HOST_CC="gcc-4.2" HOST_CXX="g++-4.2" ' + \
                             'RANLIB=ranlib AR=ar AS=$CC LD=ld' + \
                             'STRIP="strip -x -S" CROSS_COMPILE=1' + \
                             'sh ../configure --target=i386-apple-darwin8.0.0'
        elif (os.uname()[0] == "Linux") and (os.uname()[4] != 'armv7l'):
            # Apt-get `ia32-libs gcc-multilib g++-multilib` first, if on 64-bit Linux.
            cfgCmd = 'CC="gcc -m32" CXX="g++ -m32" AR=ar sh ../configure --target=i686-pc-linux'
        elif os.uname()[4] == 'armv7l':
            cfgCmd = 'CC=/opt/cs2007q3/bin/gcc CXX=/opt/cs2007q3/bin/g++ ' + \
                         'sh ../configure'
    if (archNum == '64') and (macver == '10.5'):
        cfgCmd = 'CC="gcc -m64" CXX="g++ -m64" AR=ar ' + \
                     'sh ../configure --target=x86_64-apple-darwin10.0.0'

    if compileType == 'dbg':
        cfgCmd += ' --disable-tests --disable-optimize --enable-debug'
    elif compileType == 'opt':
        cfgCmd += ' --disable-tests --enable-optimize --disable-debug'

    # Trace JIT is on by default.
    if not traceJit:
        cfgCmd += ' --disable-tracejit'
    # Method JIT is off by default.
    if methodJit:
        cfgCmd += ' --enable-methodjit'
    if valgrindSupport:
        cfgCmd += ' --enable-valgrind'
    if threadsafe:
        cfgCmd += ' --enable-threadsafe --with-system-nspr'

    verboseDump('This is the configure command:')
    verboseDump('%s\n' % cfgCmd)

    subprocess.call([cfgCmd], shell=True)

def compileCopy(archNum, compileType, branchType, usePymake):
    '''
    This function compiles and copies a binary.
    '''
    # Run make using 2 cores, not sure if pymake allows parallel compilation yet.
    subprocess.call(['python', '-O', '../../build/pymake/make.py', '-j2', '-s']) if usePymake \
        else subprocess.call(['make', '-j2', '-s'])
    # Sniff platform and rename executable accordingly:
    if os.name == 'posix':
        shellName = 'js-' + compileType + '-' + archNum + '-' + branchType + '-' + os.uname()[0].lower()
        shutil.copy2('js', '../../' + shellName)
    elif os.name == 'nt':
        shellName = 'js-' + compileType + '-' + archNum + '-' + branchType + '-' + os.name.lower()
        shutil.copy2('js.exe', '../../' + shellName + '.exe')
    return shellName

def test32or64bit(jsShellName, archNum):
    '''
    This function tests if a binary is 32-bit or 64-bit.
    '''
    test32or64bitCmd = 'file ' + jsShellName
    test32or64bitStr = captureStdout(test32or64bitCmd)[:-1]
    if archNum == '32':
        if verbose:
            if ('386' in test32or64bitStr) or ('32-bit' in test32or64bitStr):
                print 'test32or64bitStr is:', test32or64bitStr
                verboseDump('Compiled binary is 32-bit.')
        if ('386' not in test32or64bitStr) and ('32-bit' not in test32or64bitStr):
            raise Exception('Compiled binary is not 32-bit.')
    elif archNum == '64':
        if verbose:
            if '64-bit' in test32or64bitStr:
                print 'test32or64bitStr is:', test32or64bitStr
                verboseDump('Compiled binary is 64-bit.')
        if '64-bit' not in test32or64bitStr:
            raise Exception('Compiled binary is not 64-bit.')

def testDbgOrOpt(jsShellName, compileType):
    '''
    This function tests if a binary is a debug or optimized shell.
    '''
    # Create a testfile with the gczeal() function.
    compileTypeTestName = 'compileTypeTest.js'
    compileTypeTestFile = open(compileTypeTestName, 'w')
    compileTypeTestFile.writelines('gczeal()')
    compileTypeTestFile.close()
    # Test that compileTypeTestFile is indeed created.
    if not os.path.isfile(compileTypeTestName):
        raise Exception(compileTypeTestName, 'does not exist.')

    if os.name == 'posix':
        testFileErrNum = subprocess.call(['./' + jsShellName, compileTypeTestName])
    elif os.name == 'nt':
        testFileErrNum = subprocess.call([jsShellName, compileTypeTestName], shell=True)
    os.remove(compileTypeTestName)  # Remove testfile after grabbing the error code.

    verboseDump('The error code for debug shells should be 0.')
    verboseDump('The error code for opt shells should be 3.')
    verboseDump('The actual error code for ' + jsShellName + ' now, is: ' + str(testFileErrNum))

    # The error code for debug shells when passing in the gczeal() function should be 0.
    if compileType == 'dbg' and testFileErrNum != 0:
        print 'ERROR: A debug shell when tested with the gczeal() should return "0" as the error code.'
        print 'compileType is: ' + compileType
        print 'testFileErrNum is: ' + str(testFileErrNum)
        print
        raise Exception('The compiled binary is not a debug shell.')
    # The error code for debug shells when passing in the gczeal() function
    # should be 3, because they don't have the function compiled in.
    elif compileType == 'opt' and testFileErrNum != 3:
        print 'ERROR: An optimized shell when tested with the gczeal() should return "3" as the error code.'
        print 'compileType is: ' + compileType
        print 'testFileErrNum is: ' + str(testFileErrNum)
        print
        raise Exception('The compiled binary is not an optimized shell.')

if __name__ == '__main__':
    pass
<|endoftext|>


<commit_before>import pandas as pd

def get_top_topic_words(topics, opinions, t, top=10):
    """Return dataframe containing top topics and opinions.

    Parameters
        t : str - index of topic number
        top : int - the number of words to store in the dataframe

    Returns Pandas DataFrame
        The DataFrame contains top topic words, weights of topic words and for
        each perspective opinion words and weigths of opinion words.
    """
    t = str(t)
    topic = topics[t].copy()
    topic.sort(ascending=False)
    topic = topic[0:top]
    df_t = pd.DataFrame(topic)
    df_t.reset_index(level=0, inplace=True)
    df_t.columns = ['topic', 'weights_topic']
    dfs = [df_t]

    for p, o in opinions.iteritems():
        opinion = o[t].copy()
        opinion.sort(ascending=False)
        opinion = opinion[0:top]
        df_o = pd.DataFrame(opinion)
        df_o.reset_index(level=0, inplace=True)
        df_o.columns = ['{}'.format(p),
                        'weights_{}'.format(p)]
        dfs.append(df_o)
    return pd.concat(dfs, axis=1)

def topic_str(df, single_line=False, weights=False):
    opinion_labels = [l for l in df.columns if not l.startswith('weights')]

    if not single_line:
        if not weights:
            return str(df[opinion_labels])
        else:
            return str(df)
    else:
        lines = []
        if not weights:
            for l in opinion_labels:
                lines.append('{}:\t'.format(l)+' '.join(df[l]))
        else:
            for l in opinion_labels:
                zipped = zip(df[l], df['weigths_{}'.format(l)])
                line = ['{}*{:.4f}'.format(wo, we) for wo, we in zipped]
                lines.append(' '.join(['{}:\t'.format(l)]+line))
        return '\n'.join(lines)
<commit_msg>Fix bug caused by typo<commit_after>import pandas as pd

def get_top_topic_words(topics, opinions, t, top=10):
    """Return dataframe containing top topics and opinions.

    Parameters
        t : str - index of topic number
        top : int - the number of words to store in the dataframe

    Returns Pandas DataFrame
        The DataFrame contains top topic words, weights of topic words and for
        each perspective opinion words and weigths of opinion words.
    """
    t = str(t)
    topic = topics[t].copy()
    topic.sort(ascending=False)
    topic = topic[0:top]
    df_t = pd.DataFrame(topic)
    df_t.reset_index(level=0, inplace=True)
    df_t.columns = ['topic', 'weights_topic']
    dfs = [df_t]

    for p, o in opinions.iteritems():
        opinion = o[t].copy()
        opinion.sort(ascending=False)
        opinion = opinion[0:top]
        df_o = pd.DataFrame(opinion)
        df_o.reset_index(level=0, inplace=True)
        df_o.columns = ['{}'.format(p),
                        'weights_{}'.format(p)]
        dfs.append(df_o)
    return pd.concat(dfs, axis=1)

def topic_str(df, single_line=False, weights=False):
    opinion_labels = [l for l in df.columns if not l.startswith('weights')]

    if not single_line:
        if not weights:
            return str(df[opinion_labels])
        else:
            return str(df)
    else:
        lines = []
        if not weights:
            for l in opinion_labels:
                lines.append('{}:\t'.format(l)+' '.join(df[l]))
        else:
            for l in opinion_labels:
                zipped = zip(df[l], df['weights_{}'.format(l)])
                line = ['{}*{:.4f}'.format(wo, we) for wo, we in zipped]
                lines.append(' '.join(['{}:\t'.format(l)]+line))
        return '\n'.join(lines)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
# Copyright (c) 2016 Mozilla Corporation

# Imports auth0.com logs into MozDef

import hjson
import sys
import os
import requests
import traceback

import mozdef_client as mozdef

from mozdef_util.utilities.dot_dict import DotDict
from mozdef_util.utilities.logger import logger

def fatal(msg):
    print(msg)
    sys.exit(1)

# This is from https://auth0.com/docs/api/management/v2#!/Logs/get_logs
# auth0 calls these events with an acronym and description
# The logs have the acronym, but not the description
# but do include a 'description' field that is additional detailed words
# about what happened.
# See also: https://auth0.com/docs/logs#log-data-event-listing
# levels
#     0 = Debug
#     1 = Info
#     2 = Warning
#     3 = Error
#     4 = Critical
log_types = DotDict(
    {
        "s": {"event": "Success Login", "level": 1},
        "slo": {"event": "Success Logout", "level": 1},
        "flo": {"event": "Failed Logout", "level": 3},
        "seacft": {"event": "Success Exchange (Authorization Code for Access Token)", "level": 1},
        "feacft": {"event": "Failed Exchange (Authorization Code for Access Token)", "level": 3},
        "f": {"event": "Failed Login", "level": 3},
        "w": {"event": "Warnings During Login", "level": 2},
        "du": {"event": "Deleted User", "level": 1},
        "fu": {"event": "Failed Login (invalid email/username)", "level": 3},
        "fp": {"event": "Failed Login (wrong password)", "level": 3},
        "fc": {"event": "Failed by Connector", "level": 3},
        "fco": {"event": "Failed by CORS", "level": 3},
        "con": {"event": "Connector Online", "level": 1},
        "coff": {"event": "Connector Offline", "level": 3},
        "fcpro": {"event": "Failed Connector Provisioning", "level": 4},
        "ss": {"event": "Success Signup", "level": 1},
        "fs": {"event": "Failed Signup", "level": 3},
        "cs": {"event": "Code Sent", "level": 0},
        "cls": {"event": "Code/Link Sent", "level": 0},
        "sv": {"event": "Success Verification Email", "level": 0},
        "fv": {"event": "Failed Verification Email", "level": 0},
        "scp": {"event": "Success Change Password", "level": 1},
        "fcp": {"event": "Failed Change Password", "level": 3},
        "sce": {"event": "Success Change Email", "level": 1},
        "fce": {"event": "Failed Change Email", "level": 3},
        "scu": {"event": "Success Change Username", "level": 1},
        "fcu": {"event": "Failed Change Username", "level": 3},
        "scpn": {"event": "Success Change Phone Number", "level": 1},
        "fcpn": {"event": "Failed Change Phone Number", "level": 3},
        "svr": {"event": "Success Verification Email Request", "level": 0},
        "fvr": {"event": "Failed Verification Email Request", "level": 3},
        "scpr": {"event": "Success Change Password Request", "level": 0},
        "fcpr": {"event": "Failed Change Password Request", "level": 3},
        "fn": {"event": "Failed Sending Notification", "level": 3},
        "sapi": {"event": "API Operation", "level": 1},
        "fapi": {"event": "Failed API Operation", "level": 3},
        "limit_wc": {"event": "Blocked Account", "level": 4},
        "limit_ui": {"event": "Too Many Calls to /userinfo", "level": 4},
        "api_limit": {"event": "Rate Limit On API", "level": 4},
        "sdu": {"event": "Successful User Deletion", "level": 1},
        "fdu": {"event": "Failed User Deletion", "level": 3},
        "sd": {"event": "Success Delegation", "level": 3},
        "fd": {"event": "Failed Delegation", "level": 3},
        "seccft": {"event": "Success Exchange (Client Credentials for Access Token)", "level": 1},
        "feccft": {"event": "Failed Exchange (Client Credentials for Access Token)", "level": 1},
        "fsa": {"event": "Failed Silent Auth", "level": 3},
        "ssa": {"event": "Success Silent Auth", "level": 1},
        "fepft": {"event": "Failed Exchange (Password for Access Token)", "level": 3},
        "limit_mu": {"event": "Blocked IP Address", "level": 3},
        "sepft": {"event": "Success Exchange (Password for Access Token)", "level": 1},
        "fcoa": {"event": "Failed Cross Origin Authentication", "level": 3},
        "depnote": {"event": "Deprecation Note", "level": 1},
    }
)

def process_msg(mozmsg, msg):
    """Normalization function for auth0 msg.
    @mozmsg: MozDefEvent (mozdef message as DotDict)
    @msg: DotDict (json with auth0 raw message data).

    All the try-except loops handle cases where the auth0 msg may or may not contain expected fields.
    The msg structure is not garanteed.
    See also https://auth0.com/docs/api/management/v2#!/Logs/get_logs
    """
    details = DotDict({})

    # key words used to set category and success/failure markers
    authentication_words = ["Login", "Logout", "Auth"]
    authorization_words = ["Authorization", "Access", "Delegation"]
    success_words = ["Success"]
    failed_words = ["Failed"]

    # default category (might be modified below to be more specific)
    mozmsg.set_category("iam")
    mozmsg.source = "auth0"
    # fields that should always exist
    mozmsg.timestamp = msg.date
    details["messageid"] = msg._id
    details["sourceipaddress"] = msg.ip

    try:
        details["userid"] = msg.user_id
    except KeyError:
        pass

    try:
        details["username"] = msg.user_name
    except KeyError:
        pass

    try:
        # the details.request/response exist for api calls
        # but not for logins and other events
        # check and prefer them if present.
        details["username"] = msg.details.request.auth.user.name
        if type(msg.details.response.body) is not list:
            details["action"] = msg.details.response.body.name
    except KeyError:
        pass

    try:
        details["useragent"] = msg.user_agent
    except KeyError:
        pass

    try:
        # auth0 calls these events with an acronym and name
        details["eventname"] = log_types[msg.type].event
        # determine the event category
        if any(authword in details["eventname"] for authword in authentication_words):
            mozmsg.set_category("authentication")
        if any(authword in details["eventname"] for authword in authorization_words):
            mozmsg.set_category("authorization")
        # determine success/failure
        if any(failword in details["eventname"] for failword in failed_words):
            details.success = False
        if any(successword in details["eventname"] for successword in success_words):
            details.success = True
    except KeyError:
        # New message type, check https://manage-dev.mozilla.auth0.com/docs/api/management/v2#!/Logs/get_logs for ex.
        logger.error("New auth0 message type, please add support: {}".format(msg.type))
        details["eventname"] = msg.type

    # determine severity level
    if log_types[msg.type].level == 3:
        mozmsg.set_severity(mozdef.MozDefEvent.SEVERITY_ERROR)
    elif log_types[msg.type].level > 3:
        mozmsg.set_severity(mozdef.MozDefEvent.SEVERITY_CRITICAL)

    # default description
    details["description"] = ""
    try:
        if "description" in msg and msg.description is not None:
            # use the detailed description of the operation sent from auth0
            # Update a rule, add a site, update a site, etc
            details["description"] = msg.description
    except KeyError:
        details["description"] = ""

    # set the summary
    if "auth" in mozmsg._category:
        # make summary be action/username (success login user@place.com)
        # include UNKNOWN as username value in summary
        # if no details.username field exists
        tmp_username = "UNKNOWN"
        if 'username' in details:
            tmp_username = details.username
        mozmsg.summary = "{event} {username}".format(event=details.eventname, username=tmp_username)
    else:
        # default summary as action and description (if it exists)
        mozmsg.summary = "{event} {desc}".format(event=details.eventname, desc=details.description)

    try:
        details["clientname"] = msg.client_name
    except KeyError:
        pass

    try:
        details["connection"] = msg.connection
    except KeyError:
        pass

    try:
        details["clientid"] = msg.client_id
    except KeyError:
        pass

    # Differenciate auto login (session cookie check validated) from logged in and had password verified

    try:
        for i in msg.details.prompt:
            # Session cookie check
            if i.get("name") == "authenticate":
                details["authtype"] = "Login succeeded due to a valid session cookie being supplied"
            elif i.get("name") == "lock-password-authenticate":
                details["authtype"] = "Login succeeded due to a valid plaintext password being supplied"
    except KeyError:
        pass

    mozmsg.details = details
    mozmsg.details["raw"] = str(msg)

    return mozmsg

def load_state(fpath):
    """Load last msg id we've read from auth0 (log index).
    @fpath string (path to state file)
    """
    state = 0
    try:
        with open(fpath) as fd:
            state = fd.read().split("\n")[0]
    except IOError:
        pass
    return state

def save_state(fpath, state):
    """Saves last msg id we've read from auth0 (log index).
    @fpath string (path to state file)
    @state int (state value)
    """
    with open(fpath, mode="w") as fd:
        fd.write(str(state) + "\n")

def byteify(input):
    """Convert input to ascii"""
    if isinstance(input, dict):
        return {byteify(key): byteify(value) for key, value in input.items()}
    elif isinstance(input, list):
        return [byteify(element) for element in input]
    elif not isinstance(input, str):
        return str(input)
    else:
        return input

def fetch_auth0_logs(config, headers, fromid):
    lastid = fromid

    r = requests.get(
        "{url}?take={reqnr}&sort=date:1&per_page={reqnr}&from={fromid}&include_totals=true".format(
            url=config.auth0.url, reqnr=config.auth0.reqnr, fromid=fromid
        ),
        headers=headers,
    )

    # If we fail here, auth0 is not responding to us the way we expected it
    if not r.ok:
        raise Exception(r.url, r.reason, r.status_code, r.json())
    ret = r.json()

    # Sometimes API give us the requested totals.. sometimes not.
    # To be clear; totals are now only returned when using `page=..` and not using `from=..` parameters
    # The issue is that when using `page`, auth0 internally splices the log by page, which is extremely slow and the
    # call takes 10-20s to return each time.
    # When using `from` auth0 queries the index for that location which is fast, so we use `from`
    # this means we can't properly page results, so we have to "try to fetch" until no more logs are returned
    # Finally note that when using `from` the `sort` ordering is not guaranteed to work according to the API docs
    if type(ret) is dict and "logs" in ret:
        have_totals = True
        all_msgs = ret["logs"]
    else:
        have_totals = False
        all_msgs = ret

    # Process all new auth0 log msgs, normalize and send them to mozdef
    for msg in all_msgs:
        mozmsg = mozdef.MozDefEvent(config.mozdef.url)
        if config.DEBUG == "True":
            mozmsg.set_send_to_syslog(True, only_syslog=True)
        mozmsg.hostname = config.auth0.url
        mozmsg.tags = ["auth0"]
        msg = byteify(msg)
        msg = DotDict(msg)
        lastid = msg._id

        # Fill in mozdef msg fields from the auth0 msg
        try:
            mozmsg = process_msg(mozmsg, msg)
        except KeyError as e:
            # if this happens the msg was malformed in some way
            mozmsg.details["error"] = "true"
            mozmsg.details["errormsg"] = '"' + str(e) + '"'
            mozmsg.summary = "Failed to parse auth0 message"
            if config.DEBUG == "True":
                traceback.print_exc()
        mozmsg.send()

    if have_totals:
        return (int(ret["total"]), int(ret["start"]), int(ret["length"]), lastid)
    else:
        return (-1, -1, -1, lastid)

def main():
    # Configuration loading
    config_location = os.path.dirname(sys.argv[0]) + "/" + "auth02mozdef.json"
    with open(config_location) as fd:
        config = DotDict(hjson.load(fd))

    if config is None:
        logger.error("No configuration file 'auth02mozdef.json' found.")
        sys.exit(1)

    headers = {"Authorization": "Bearer {}".format(config.auth0.token), "Accept": "application/json"}

    fromid = load_state(config.state_file)
    # Auth0 will interpret a 0 state as an error on our hosted instance, but will accept an empty parameter "as if it was 0"
    if fromid == 0 or fromid == "0":
        fromid = ""
    totals = 1
    start = 0
    length = 0

    # Fetch until we've gotten all messages
    while totals > start + length:
        (totals, start, length, lastid) = fetch_auth0_logs(config, headers, fromid)

        if totals == -1:
            if fromid == lastid:
                # We got everything, we're done!
                break
        fromid = lastid

    save_state(config.state_file, lastid)

if __name__ == "__main__":
    main()
<commit_msg>Modify auth0 cron script to ensure always save raw msg<commit_after>#!/usr/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
# Copyright (c) 2016 Mozilla Corporation

# Imports auth0.com logs into MozDef

import hjson
import sys
import os
import requests
import traceback

import mozdef_client as mozdef

from mozdef_util.utilities.dot_dict import DotDict
from mozdef_util.utilities.logger import logger

def fatal(msg):
    print(msg)
    sys.exit(1)

# This is from https://auth0.com/docs/api/management/v2#!/Logs/get_logs
# auth0 calls these events with an acronym and description
# The logs have the acronym, but not the description
# but do include a 'description' field that is additional detailed words
# about what happened.
# See also: https://auth0.com/docs/logs#log-data-event-listing
# levels
#     0 = Debug
#     1 = Info
#     2 = Warning
#     3 = Error
#     4 = Critical
log_types = DotDict(
    {
        "s": {"event": "Success Login", "level": 1},
        "slo": {"event": "Success Logout", "level": 1},
        "flo": {"event": "Failed Logout", "level": 3},
        "seacft": {"event": "Success Exchange (Authorization Code for Access Token)", "level": 1},
        "feacft": {"event": "Failed Exchange (Authorization Code for Access Token)", "level": 3},
        "f": {"event": "Failed Login", "level": 3},
        "w": {"event": "Warnings During Login", "level": 2},
        "du": {"event": "Deleted User", "level": 1},
        "fu": {"event": "Failed Login (invalid email/username)", "level": 3},
        "fp": {"event": "Failed Login (wrong password)", "level": 3},
        "fc": {"event": "Failed by Connector", "level": 3},
        "fco": {"event": "Failed by CORS", "level": 3},
        "con": {"event": "Connector Online", "level": 1},
        "coff": {"event": "Connector Offline", "level": 3},
        "fcpro": {"event": "Failed Connector Provisioning", "level": 4},
        "ss": {"event": "Success Signup", "level": 1},
        "fs": {"event": "Failed Signup", "level": 3},
        "cs": {"event": "Code Sent", "level": 0},
        "cls": {"event": "Code/Link Sent", "level": 0},
        "sv": {"event": "Success Verification Email", "level": 0},
        "fv": {"event": "Failed Verification Email", "level": 0},
        "scp": {"event": "Success Change Password", "level": 1},
        "fcp": {"event": "Failed Change Password", "level": 3},
        "sce": {"event": "Success Change Email", "level": 1},
        "fce": {"event": "Failed Change Email", "level": 3},
        "scu": {"event": "Success Change Username", "level": 1},
        "fcu": {"event": "Failed Change Username", "level": 3},
        "scpn": {"event": "Success Change Phone Number", "level": 1},
        "fcpn": {"event": "Failed Change Phone Number", "level": 3},
        "svr": {"event": "Success Verification Email Request", "level": 0},
        "fvr": {"event": "Failed Verification Email Request", "level": 3},
        "scpr": {"event": "Success Change Password Request", "level": 0},
        "fcpr": {"event": "Failed Change Password Request", "level": 3},
        "fn": {"event": "Failed Sending Notification", "level": 3},
        "sapi": {"event": "API Operation", "level": 1},
        "fapi": {"event": "Failed API Operation", "level": 3},
        "limit_wc": {"event": "Blocked Account", "level": 4},
        "limit_ui": {"event": "Too Many Calls to /userinfo", "level": 4},
        "api_limit": {"event": "Rate Limit On API", "level": 4},
        "sdu": {"event": "Successful User Deletion", "level": 1},
        "fdu": {"event": "Failed User Deletion", "level": 3},
        "sd": {"event": "Success Delegation", "level": 3},
        "fd": {"event": "Failed Delegation", "level": 3},
        "seccft": {"event": "Success Exchange (Client Credentials for Access Token)", "level": 1},
        "feccft": {"event": "Failed Exchange (Client Credentials for Access Token)", "level": 1},
        "fsa": {"event": "Failed Silent Auth", "level": 3},
        "ssa": {"event": "Success Silent Auth", "level": 1},
        "fepft": {"event": "Failed Exchange (Password for Access Token)", "level": 3},
        "limit_mu": {"event": "Blocked IP Address", "level": 3},
        "sepft": {"event": "Success Exchange (Password for Access Token)", "level": 1},
        "fcoa": {"event": "Failed Cross Origin Authentication", "level": 3},
        "depnote": {"event": "Deprecation Note", "level": 1},
    }
)

def process_msg(mozmsg, msg):
    """Normalization function for auth0 msg.
    @mozmsg: MozDefEvent (mozdef message as DotDict)
    @msg: DotDict (json with auth0 raw message data).

    All the try-except loops handle cases where the auth0 msg may or may not contain expected fields.
    The msg structure is not garanteed.
    See also https://auth0.com/docs/api/management/v2#!/Logs/get_logs
    """
    details = DotDict({})

    # key words used to set category and success/failure markers
    authentication_words = ["Login", "Logout", "Auth"]
    authorization_words = ["Authorization", "Access", "Delegation"]
    success_words = ["Success"]
    failed_words = ["Failed"]

    # default category (might be modified below to be more specific)
    mozmsg.set_category("iam")
    mozmsg.source = "auth0"
    # fields that should always exist
    mozmsg.timestamp = msg.date
    details["messageid"] = msg._id
    details["sourceipaddress"] = msg.ip

    try:
        details["userid"] = msg.user_id
    except KeyError:
        pass

    try:
        details["username"] = msg.user_name
    except KeyError:
        pass

    try:
        # the details.request/response exist for api calls
        # but not for logins and other events
        # check and prefer them if present.
        details["username"] = msg.details.request.auth.user.name
        if type(msg.details.response.body) is not list:
            details["action"] = msg.details.response.body.name
    except KeyError:
        pass

    try:
        details["useragent"] = msg.user_agent
    except KeyError:
        pass

    try:
        # auth0 calls these events with an acronym and name
        details["eventname"] = log_types[msg.type].event
        # determine the event category
        if any(authword in details["eventname"] for authword in authentication_words):
            mozmsg.set_category("authentication")
        if any(authword in details["eventname"] for authword in authorization_words):
            mozmsg.set_category("authorization")
        # determine success/failure
        if any(failword in details["eventname"] for failword in failed_words):
            details.success = False
        if any(successword in details["eventname"] for successword in success_words):
            details.success = True
    except KeyError:
        # New message type, check https://manage-dev.mozilla.auth0.com/docs/api/management/v2#!/Logs/get_logs for ex.
        logger.error("New auth0 message type, please add support: {}".format(msg.type))
        details["eventname"] = msg.type

    # determine severity level
    if log_types[msg.type].level == 3:
        mozmsg.set_severity(mozdef.MozDefEvent.SEVERITY_ERROR)
    elif log_types[msg.type].level > 3:
        mozmsg.set_severity(mozdef.MozDefEvent.SEVERITY_CRITICAL)

    # default description
    details["description"] = ""
    try:
        if "description" in msg and msg.description is not None:
            # use the detailed description of the operation sent from auth0
            # Update a rule, add a site, update a site, etc
            details["description"] = msg.description
    except KeyError:
        details["description"] = ""

    # set the summary
    if "auth" in mozmsg._category:
        # make summary be action/username (success login user@place.com)
        # include UNKNOWN as username value in summary
        # if no details.username field exists
        tmp_username = "UNKNOWN"
        if 'username' in details:
            tmp_username = details.username
        mozmsg.summary = "{event} {username}".format(event=details.eventname, username=tmp_username)
    else:
        # default summary as action and description (if it exists)
        mozmsg.summary = "{event} {desc}".format(event=details.eventname, desc=details.description)

    try:
        details["clientname"] = msg.client_name
    except KeyError:
        pass

    try:
        details["connection"] = msg.connection
    except KeyError:
        pass

    try:
        details["clientid"] = msg.client_id
    except KeyError:
        pass

    # Differenciate auto login (session cookie check validated) from logged in and had password verified

    try:
        for i in msg.details.prompt:
            # Session cookie check
            if i.get("name") == "authenticate":
                details["authtype"] = "Login succeeded due to a valid session cookie being supplied"
            elif i.get("name") == "lock-password-authenticate":
                details["authtype"] = "Login succeeded due to a valid plaintext password being supplied"
    except KeyError:
        pass

    mozmsg.details = details

    return mozmsg

def load_state(fpath):
    """Load last msg id we've read from auth0 (log index).
    @fpath string (path to state file)
    """
    state = 0
    try:
        with open(fpath) as fd:
            state = fd.read().split("\n")[0]
    except IOError:
        pass
    return state

def save_state(fpath, state):
    """Saves last msg id we've read from auth0 (log index).
    @fpath string (path to state file)
    @state int (state value)
    """
    with open(fpath, mode="w") as fd:
        fd.write(str(state) + "\n")

def byteify(input):
    """Convert input to ascii"""
    if isinstance(input, dict):
        return {byteify(key): byteify(value) for key, value in input.items()}
    elif isinstance(input, list):
        return [byteify(element) for element in input]
    elif not isinstance(input, str):
        return str(input)
    else:
        return input

def fetch_auth0_logs(config, headers, fromid):
    lastid = fromid

    r = requests.get(
        "{url}?take={reqnr}&sort=date:1&per_page={reqnr}&from={fromid}&include_totals=true".format(
            url=config.auth0.url, reqnr=config.auth0.reqnr, fromid=fromid
        ),
        headers=headers,
    )

    # If we fail here, auth0 is not responding to us the way we expected it
    if not r.ok:
        raise Exception(r.url, r.reason, r.status_code, r.json())
    ret = r.json()

    # Sometimes API give us the requested totals.. sometimes not.
    # To be clear; totals are now only returned when using `page=..` and not using `from=..` parameters
    # The issue is that when using `page`, auth0 internally splices the log by page, which is extremely slow and the
    # call takes 10-20s to return each time.
    # When using `from` auth0 queries the index for that location which is fast, so we use `from`
    # this means we can't properly page results, so we have to "try to fetch" until no more logs are returned
    # Finally note that when using `from` the `sort` ordering is not guaranteed to work according to the API docs
    if type(ret) is dict and "logs" in ret:
        have_totals = True
        all_msgs = ret["logs"]
    else:
        have_totals = False
        all_msgs = ret

    # Process all new auth0 log msgs, normalize and send them to mozdef
    for msg in all_msgs:
        mozmsg = mozdef.MozDefEvent(config.mozdef.url)
        if config.DEBUG == "True":
            mozmsg.set_send_to_syslog(True, only_syslog=True)
        mozmsg.hostname = config.auth0.url
        mozmsg.tags = ["auth0"]
        msg = byteify(msg)
        msg = DotDict(msg)
        lastid = msg._id

        # Fill in mozdef msg fields from the auth0 msg
        try:
            mozmsg = process_msg(mozmsg, msg)
        except KeyError as e:
            # if this happens the msg was malformed in some way
            mozmsg.details["error"] = "true"
            mozmsg.details["errormsg"] = '"' + str(e) + '"'
            mozmsg.summary = "Failed to parse auth0 message"
            traceback.print_exc()

        # Save raw initial message in final message
        # in case we ran into parsing errors
        if "details" not in mozmsg:
            mozmsg.details = {}
        mozmsg.details["raw"] = str(msg)

        mozmsg.send()

    if have_totals:
        return (int(ret["total"]), int(ret["start"]), int(ret["length"]), lastid)
    else:
        return (-1, -1, -1, lastid)

def main():
    # Configuration loading
    config_location = os.path.dirname(sys.argv[0]) + "/" + "auth02mozdef.json"
    with open(config_location) as fd:
        config = DotDict(hjson.load(fd))

    if config is None:
        logger.error("No configuration file 'auth02mozdef.json' found.")
        sys.exit(1)

    headers = {"Authorization": "Bearer {}".format(config.auth0.token), "Accept": "application/json"}

    fromid = load_state(config.state_file)
    # Auth0 will interpret a 0 state as an error on our hosted instance, but will accept an empty parameter "as if it was 0"
    if fromid == 0 or fromid == "0":
        fromid = ""
    totals = 1
    start = 0
    length = 0

    # Fetch until we've gotten all messages
    while totals > start + length:
        (totals, start, length, lastid) = fetch_auth0_logs(config, headers, fromid)

        if totals == -1:
            if fromid == lastid:
                # We got everything, we're done!
                break
        fromid = lastid

    save_state(config.state_file, lastid)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import os
import sys
import transaction
import datetime

from sqlalchemy import engine_from_config
from sqlalchemy.exc import IntegrityError

from pyramid.paster import (
    get_appsettings,
    setup_logging,
    )

from ..models import *

def usage(argv):
    cmd = os.path.basename(argv[0])
    print('usage: %s <config_uri>\n'
          '(example: "%s development.ini")' % (cmd, cmd))
    sys.exit(1)

def main(argv=sys.argv):
    if len(argv) != 2:
        usage(argv)
    config_uri = argv[1]
    setup_logging(config_uri)
    settings = get_appsettings(config_uri)
    engine = engine_from_config(settings, 'sqlalchemy.')
    DBSession.configure(bind=engine)
    Base.metadata.create_all(engine)
    try:
        with transaction.manager:
            user = User()
            user.username = "xistence"
            user.realname = "Bert JW Regeer"
            user.credentials = "test"

            DBSession.add(user)
            DBSession.flush()

            user = user.id

            tag1 = insert_new_tag("c++")
            tag2 = insert_new_tag("c")
            tag3 = insert_new_tag("database")
            tag4 = insert_new_tag("testing")
            tag5 = insert_new_tag("not used")

            insert_new_rev_entry("one", "Post number 1", "one", user, [tag1, tag2], published=True)
            insert_new_rev_entry("two", "Post number 2", "two", user, [tag2, tag3], published=True)
            insert_new_rev_entry("three", "Post number 3", "three", user, [tag4], published=False)
            insert_new_rev_entry("four", "Post number 4", "four", user, [tag1], published=True)
            insert_new_rev_entry("five", "Post number 5", "five", user, [], published=True)
    except IntegrityError:
        pass

def insert_new_rev_entry(title, entry, slug, user, tags, published=False):
    revision = Revision()
    revision.revision = 0
    revision.user_id = user
    revision.title = title
    revision.entry = entry

    DBSession.add(revision)
    DBSession.flush()

    entry = Entry()
    entry.current_rev = revision.id
    entry.slug = slug
    if published:
        entry.pubdate = datetime.datetime.now()

    DBSession.add(entry)
    DBSession.flush()

    author = EntryAuthors()
    author.entry_id = entry.id
    author.user_id = user
    author.revision_id = revision.id

    DBSession.add(entry)

    for tag in tags:
        tagforrev = RevisionTags()

        tagforrev.revision_id = revision.id
        tagforrev.tag_id = tag
        DBSession.add(tagforrev)

    DBSession.flush()

def insert_new_tag(tagname):
    tag = Tag()
    tag.tag = tagname
    tag.description = "Unknown"

    DBSession.add(tag)
    DBSession.flush()

    return tag.id
<commit_msg>Make initialise DB much better<commit_after>import os
import sys
import transaction
import datetime

from sqlalchemy import engine_from_config
from sqlalchemy.exc import IntegrityError

from pyramid.paster import (
    get_appsettings,
    setup_logging,
    )

from ..models import *

defaults = {
            'users': [
                (u'xistence', 'Bert JW Regeer', 'test')
                ],
            'tags': [
                (u'c++', "The C++ programming language"),
                (u'c', "The C progrmaming langauge"),
                (u'database', "Databases store stuff"),
                (u'testing', "Not sure..."),
                (u'not used', "Don't look at me that way"),
                ],
            'domains': [
                (u'test.alexandra.network.lan', "xistence"),
                (u'whatever.alexandra.network.lan', "xistence"),
                ],
            'entries': [
                # Title, entry, slug, user, tags, published
                (u'one', u'Post number 1', 'one', 1, ["c++", "c"], u'test.alexandra.network.lan', True),
                (u'two', u'Post number 2', 'two', 1, ["database", "testing"], u'test.alexandra.network.lan', True),
                (u'three', u'Post number 3', 'three', 1, ["c++", "database"], u'whatever.alexandra.network.lan', True),
                (u'four', u'Post number 4', 'four', 1, ["testing", "c"], u'whatever.alexandra.network.lan', True),
                (u'five', u'Post number 5', 'five', 1, ["c++", "c", "database"], u'test.alexandra.network.lan', True),
                (u'No tags', u'This post has no tags', 'no-tags', 1, [], u'whatever.alexandra.network.lan', True),
                ],
        }

def usage(argv):
    cmd = os.path.basename(argv[0])
    print('usage: %s <config_uri>\n'
          '(example: "%s development.ini")' % (cmd, cmd))
    sys.exit(1)

def main(argv=sys.argv):
    if len(argv) != 2:
        usage(argv)
    config_uri = argv[1]
    setup_logging(config_uri)
    settings = get_appsettings(config_uri)
    engine = engine_from_config(settings, 'sqlalchemy.')
    DBSession.configure(bind=engine)
    Base.metadata.create_all(engine)

    with transaction.manager:
        for (u, r, c) in defaults['users']:
            sp = transaction.savepoint()
            try:
                user = User(username=u, realname=r, credentials=c)
                DBSession.add(user)
                DBSession.flush()
            except IntegrityError:
                sp.rollback()
                print 'Username "{}" already exists.'.format(u)

        for (t, d) in defaults['tags']:
            sp = transaction.savepoint()
            try:
                tag = Tag(tag = t, description = d)
                DBSession.add(tag)
                DBSession.flush()
            except IntegrityError:
                sp.rollback()
                print 'Tag "{}" already exists.'.format(t)

        for (d, o) in defaults['domains']:
            sp = transaction.savepoint()
            try:
                domain = Domain(domain = d, owner = DBSession.query(User).filter(User.username == o).first().id)
                DBSession.add(domain)
                DBSession.flush()
            except IntegrityError:
                print e
                sp.rollback()
                print 'Domain "{}" already exists.'.format(d)

        for (t, e, s, u, ta, d, p) in defaults['entries']:
            sp = transaction.savepoint()
            try:
                insert_new_rev_entry(t, e, s, u, ta, d, published=p)
            except IntegrityError:
                sp.rollback()
                print 'Entry "{}" already exists.'.format(t)

def insert_new_rev_entry(title, entry, slug, user, tags, domain, published=False):
    revision = Revision()
    revision.revision = 0
    revision.user_id = user
    revision.title = title
    revision.entry = entry

    for tag in tags:
        revision.tags.append(DBSession.query(Tag).filter(Tag.tag == tag).first())

    DBSession.add(revision)
    DBSession.flush()

    entry = Entry(current_rev = revision.id, slug = slug, domain_id = DBSession.query(Domain).filter(Domain.domain == domain).first().id)
    if published:
        entry.pubdate = datetime.datetime.now()

    DBSession.add(entry)
    DBSession.flush()

    author = EntryAuthors()
    author.entry_id = entry.id
    author.user_id = user
    author.revision_id = revision.id

    DBSession.add(entry)
    DBSession.flush()
<|endoftext|>


<commit_before>#!/usr/bin/env python

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Copyright (c) 2014 Mozilla Corporation
#
# Contributors:
# Jeff Bryner jbryner@mozilla.com

import json
import logging
import math
import random
import requests
import sys
from configlib import getConfig, OptionParser
from datetime import datetime, date, timedelta

import os
sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), '../lib'))
from utilities.toUTC import toUTC
from elasticsearch_client import ElasticsearchClient

logger = logging.getLogger(sys.argv[0])

def loggerTimeStamp(self, record, datefmt=None):
    return toUTC(datetime.now()).isoformat()

def initLogger():
    logger.level = logging.INFO
    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    formatter.formatTime = loggerTimeStamp
    sh = logging.StreamHandler(sys.stderr)
    sh.setFormatter(formatter)
    logger.addHandler(sh)

def isNumber(s):
    'check if a token is numeric, return bool'
    try:
        float(s)  # for int, long and float
    except ValueError:
        try:
            complex(s)  # for complex
        except ValueError:
            return False
    return True

def digits(n):
    '''return the number of digits in a number'''
    if n > 0:
        digits = int(math.log10(n))+1
    elif n == 0:
        digits = 1
    else:
        digits = int(math.log10(-n))+2
    return digits

def isJVMMemoryHigh():
    url = "http://{0}/_nodes/stats?pretty=true".format(random.choice(options.esservers))
    r = requests.get(url)
    logger.debug(r)
    if r.status_code == 200:
        nodestats=r.json()
        #logger.debug(json.dumps(nodestats, indent=4, sort_keys=True))

        for node in nodestats['nodes']:
            loadaverage=nodestats['nodes'][node]['os']['load_average']
            cpuusage=nodestats['nodes'][node]['os']['cpu_percent']
            nodename=nodestats['nodes'][node]['name']
            jvmused=nodestats['nodes'][node]['jvm']['mem']['heap_used_percent']
            logger.debug('{0}: cpu {1}%  jvm {2}% load average: {3}'.format(nodename,cpuusage,jvmused,loadaverage))
            if jvmused> options.jvmlimit:
                logger.info('{0}: cpu {1}%  jvm {2}% load average: {3} recommending cache clear'.format(nodename,cpuusage,jvmused,loadaverage))
                return True
        return False
    else:
        logger.error(r)
        return False

def clearESCache():
    es = ElasticsearchClient((list('{0}'.format(s) for s in options.esservers)))
    indices = es.get_indices()
    # assums index names  like events-YYYYMMDD etc.
    # used to avoid operating on current indexes
    dtNow = datetime.utcnow()
    indexSuffix = date.strftime(dtNow, '%Y%m%d')
    previousSuffix = date.strftime(dtNow - timedelta(days=1), '%Y%m%d')
    for targetindex in sorted(indices):
        if indexSuffix not in targetindex and previousSuffix not in targetindex:
            url = 'http://{0}/{1}/_stats'.format(random.choice(options.esservers), targetindex)
            r = requests.get(url)
            if r.status_code == 200:
                indexstats = json.loads(r.text)
                if indexstats['_all']['total']['search']['query_current'] == 0:
                    fielddata = indexstats['_all']['total']['fielddata']['memory_size_in_bytes']
                    if fielddata > 0:
                        logger.info('target: {0}: field data {1}'.format(targetindex, indexstats['_all']['total']['fielddata']['memory_size_in_bytes']))
                        clearurl = 'http://{0}/{1}/_cache/clear'.format(random.choice(options.esservers), targetindex)
                        clearRequest = requests.post(clearurl)
                        logger.info(clearRequest.text)
                        if options.conservative:
                            return
                else:
                    logger.debug('{0}: <ignoring due to current search > field data {1}'.format(targetindex, indexstats['_all']['total']['fielddata']['memory_size_in_bytes']))
            else:
                logger.error('{0} returned {1}'.format(url, r.status_code))

def main():
    if isJVMMemoryHigh():
        logger.info('initiating cache clearing')
        clearESCache()

def initConfig():
    # elastic search options.
    options.esservers = list(getConfig('esservers', 'http://localhost:9200', options.configfile).split(','))

    # memory watermark, set to 90 (percent) by default
    options.jvmlimit = getConfig('jvmlimit', 90, options.configfile)

    # be conservative? if set only clears cache for the first index found with no searches and cached field data
    # if false, will continue to clear for any index not matching the date suffix.
    options.conservative = getConfig('conservative', True, options.configfile)

if __name__ == '__main__':
    # configure ourselves
    parser = OptionParser()
    parser.add_option("-c", dest='configfile', default=sys.argv[0].replace('.py', '.conf'), help="configuration file to use")
    (options, args) = parser.parse_args()
    initConfig()
    initLogger()
    logger.debug('starting')

    main()
<commit_msg>Update cron script to remove http prefix<commit_after>#!/usr/bin/env python

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Copyright (c) 2014 Mozilla Corporation
#
# Contributors:
# Jeff Bryner jbryner@mozilla.com

import json
import logging
import math
import random
import requests
import sys
from configlib import getConfig, OptionParser
from datetime import datetime, date, timedelta

import os
sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), '../lib'))
from utilities.toUTC import toUTC
from elasticsearch_client import ElasticsearchClient

logger = logging.getLogger(sys.argv[0])

def loggerTimeStamp(self, record, datefmt=None):
    return toUTC(datetime.now()).isoformat()

def initLogger():
    logger.level = logging.INFO
    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    formatter.formatTime = loggerTimeStamp
    sh = logging.StreamHandler(sys.stderr)
    sh.setFormatter(formatter)
    logger.addHandler(sh)

def isNumber(s):
    'check if a token is numeric, return bool'
    try:
        float(s)  # for int, long and float
    except ValueError:
        try:
            complex(s)  # for complex
        except ValueError:
            return False
    return True

def digits(n):
    '''return the number of digits in a number'''
    if n > 0:
        digits = int(math.log10(n))+1
    elif n == 0:
        digits = 1
    else:
        digits = int(math.log10(-n))+2
    return digits

def isJVMMemoryHigh():
    url = "http://{0}/_nodes/stats?pretty=true".format(random.choice(options.esservers))
    r = requests.get(url)
    logger.debug(r)
    if r.status_code == 200:
        nodestats=r.json()
        #logger.debug(json.dumps(nodestats, indent=4, sort_keys=True))

        for node in nodestats['nodes']:
            loadaverage=nodestats['nodes'][node]['os']['load_average']
            cpuusage=nodestats['nodes'][node]['os']['cpu_percent']
            nodename=nodestats['nodes'][node]['name']
            jvmused=nodestats['nodes'][node]['jvm']['mem']['heap_used_percent']
            logger.debug('{0}: cpu {1}%  jvm {2}% load average: {3}'.format(nodename,cpuusage,jvmused,loadaverage))
            if jvmused> options.jvmlimit:
                logger.info('{0}: cpu {1}%  jvm {2}% load average: {3} recommending cache clear'.format(nodename,cpuusage,jvmused,loadaverage))
                return True
        return False
    else:
        logger.error(r)
        return False

def clearESCache():
    es = ElasticsearchClient((list('{0}'.format(s) for s in options.esservers)))
    indices = es.get_indices()
    # assums index names  like events-YYYYMMDD etc.
    # used to avoid operating on current indexes
    dtNow = datetime.utcnow()
    indexSuffix = date.strftime(dtNow, '%Y%m%d')
    previousSuffix = date.strftime(dtNow - timedelta(days=1), '%Y%m%d')
    for targetindex in sorted(indices):
        if indexSuffix not in targetindex and previousSuffix not in targetindex:
            url = '{0}/{1}/_stats'.format(random.choice(options.esservers), targetindex)
            r = requests.get(url)
            if r.status_code == 200:
                indexstats = json.loads(r.text)
                if indexstats['_all']['total']['search']['query_current'] == 0:
                    fielddata = indexstats['_all']['total']['fielddata']['memory_size_in_bytes']
                    if fielddata > 0:
                        logger.info('target: {0}: field data {1}'.format(targetindex, indexstats['_all']['total']['fielddata']['memory_size_in_bytes']))
                        clearurl = 'http://{0}/{1}/_cache/clear'.format(random.choice(options.esservers), targetindex)
                        clearRequest = requests.post(clearurl)
                        logger.info(clearRequest.text)
                        if options.conservative:
                            return
                else:
                    logger.debug('{0}: <ignoring due to current search > field data {1}'.format(targetindex, indexstats['_all']['total']['fielddata']['memory_size_in_bytes']))
            else:
                logger.error('{0} returned {1}'.format(url, r.status_code))

def main():
    if isJVMMemoryHigh():
        logger.info('initiating cache clearing')
        clearESCache()

def initConfig():
    # elastic search options.
    options.esservers = list(getConfig('esservers', 'http://localhost:9200', options.configfile).split(','))

    # memory watermark, set to 90 (percent) by default
    options.jvmlimit = getConfig('jvmlimit', 90, options.configfile)

    # be conservative? if set only clears cache for the first index found with no searches and cached field data
    # if false, will continue to clear for any index not matching the date suffix.
    options.conservative = getConfig('conservative', True, options.configfile)

if __name__ == '__main__':
    # configure ourselves
    parser = OptionParser()
    parser.add_option("-c", dest='configfile', default=sys.argv[0].replace('.py', '.conf'), help="configuration file to use")
    (options, args) = parser.parse_args()
    initConfig()
    initLogger()
    logger.debug('starting')

    main()
<|endoftext|>


<commit_before>"""Create cron schedule from exported data from `crontab -l` ."""
from crontab import CronTab
from datetime import timedelta
import os
import re
import sys
import warnings

# Ignore cases when reading crontab file.
IGNORE_CASES = ['MAILTO', '#']

def stop_future_warning(func):
    """Stop printing warning."""
    import functools
    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        with warnings.catch_warnings():
            warnings.simplefilter('ignore', FutureWarning)
            return func(*args, **kwargs)
    return wrapper

def __create_file_object_by_server(directory):
    """Read cronfile and create file object."""
    server_list = {}
    for server_file in os.listdir(directory):
        file_path = os.path.join(directory, server_file)
        with open(file_path, 'r') as f:
            lines = f.readlines()
            name, __ext = os.path.splitext(server_file)
            server_list[name] = lines
    return server_list

@stop_future_warning
def main(directory, start, end):
    """Main process."""
    targets = {}

    for sv, lines in __create_file_object_by_server(directory).items():
        for li in lines:
            li = li.rstrip()
            if not is_cron_script(li):
                continue

            nz_crnon = normalize_cron_script(li)
            cron_time = nz_crnon[0]
            cron_script = nz_crnon[1]

            # TODO: has to handle error occuring missed cron script.
            c = CronTab(cron_time)

            current_time = start
            while current_time <= end:
                interval_second = c.next(current_time)
                current_time = current_time + \
                    timedelta(seconds=interval_second)
                if current_time > end:
                    break
                cron_combo = (sv, cron_script)
                if current_time not in targets:
                    targets[current_time] = [cron_combo]
                else:
                    targets[current_time].append(cron_combo)
    return targets

def is_cron_script(cron_script):
    for ic in IGNORE_CASES:
        if ic in cron_script or cron_script == '':
            return False
    return True

def normalize_cron_script(cron_script):
    result = re.sub(r'  +', '\t', cron_script)
    result_list = result.split('\t')
    return (' '.join(result_list[0:5]), result_list[-1])

def export(targets):
    header = 'date, hour, miniute, second, scrip, server\n'
    with open('result.csv', 'w') as f:
        f.write(header)
        for t, scripts in sorted(targets.items()):
            for s in scripts:
                result = '{0},{1},{2}\n'.format(
                        t.strftime('%Y-%m-%d,%H,%M,%S'),
                        s[1],
                        s[0])
                f.write(result)

def execute_from_console():
    return parse_command()

def parse_command():
    import argparse
    from datetime import datetime
    p = argparse.ArgumentParser(description="Download sites.")
    p.add_argument('-s', '--start', type=str,
                   default=datetime.now().strftime('%Y%m%d000000'),
                   help='Cron start datetime.(YYYYmmddHHMMSS)',
                   dest='start_datetime' )
    p.add_argument('-e', '--end', type=str,
                   help='Cron end datetime.(YYYYmmddHHMMSS)',
                   default=datetime.now().strftime('%Y%m%d235959'),
                   dest='end_datetime')
    p.add_argument('-d', '--directory', type=str,
                   help='Directory name excluding cron files.',
                   default='crontab',
                   dest='directory')
    args = p.parse_args()
    start = datetime.strptime(args.start_datetime, '%Y%m%d%H%M%S')
    end = datetime.strptime(args.end_datetime, '%Y%m%d%H%M%S')
    directory = args.directory
    if not has_directory(directory):
        sys.exit(1)

    t = main(directory, start, end)
    export(t)

def has_directory(directory):
    if os.path.isdir(directory):
        return True
    print("""
You must make `crontab` directory to load crontab files.
If you don't want this name, you can chose your prefere name with argument of
`-d`. Please see help command.\n""")
    return False

if __name__ == '__main__':
    parse_command()
<commit_msg>Change to call iterable.<commit_after>"""Create cron schedule from exported data from `crontab -l` ."""
from crontab import CronTab
from datetime import timedelta
import os
import re
import sys
import warnings

# Ignore cases when reading crontab file.
IGNORE_CASES = ['MAILTO', '#']

def stop_future_warning(func):
    """Stop printing warning."""
    import functools
    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        with warnings.catch_warnings():
            warnings.simplefilter('ignore', FutureWarning)
            return func(*args, **kwargs)
    return wrapper

def export(targets):
    header = 'date, hour, miniute, second, scrip, server\n'
    with open('result.csv', 'w') as f:
        f.write(header)
        for t, scripts in sorted(targets.items()):
            for s in scripts:
                result = '{0},{1},{2}\n'.format(
                        t.strftime('%Y-%m-%d,%H,%M,%S'),
                        s[1],
                        s[0])
                f.write(result)

def execute_from_console():
    return parse_command()

def parse_command():
    import argparse
    from datetime import datetime
    p = argparse.ArgumentParser(description="Download sites.")
    p.add_argument('-s', '--start', type=str,
                   default=datetime.now().strftime('%Y%m%d000000'),
                   help='Cron start datetime.(YYYYmmddHHMMSS)',
                   dest='start_datetime' )
    p.add_argument('-e', '--end', type=str,
                   help='Cron end datetime.(YYYYmmddHHMMSS)',
                   default=datetime.now().strftime('%Y%m%d235959'),
                   dest='end_datetime')
    p.add_argument('-d', '--directory', type=str,
                   help='Directory name excluding cron files.',
                   default='crontab',
                   dest='directory')
    args = p.parse_args()
    start = datetime.strptime(args.start_datetime, '%Y%m%d%H%M%S')
    end = datetime.strptime(args.end_datetime, '%Y%m%d%H%M%S')
    directory = args.directory
    if not has_directory(directory):
        sys.exit(1)

    c = Cron(directory, start, end)
    for cc in c:
        print cc
    # export(t.crons)

def has_directory(directory):
    if os.path.isdir(directory):
        return True
    print("""
You must make `crontab` directory to load crontab files.
If you don't want this name, you can chose your prefere name with argument of
`-d`. Please see help command.\n""")
    return False

class Cron(object):

    def __init__(self, directory, start, end):
        self.crons = self.__quote(directory, start, end)

    @stop_future_warning
    def __quote(self, directory, start, end):
        targets = {}
        for sv, lines in self.create_file_object_by_server(directory).items():
            for li in lines:
                li = li.rstrip()
                if not self.is_cron_script(li):
                    continue

                nz_crnon = self.normalize_cron_script(li)
                cron_time = nz_crnon[0]
                cron_script = nz_crnon[1]

                # TODO: has to handle error occuring missed cron script.
                c = CronTab(cron_time)

                current_time = start
                while current_time <= end:
                    interval_second = c.next(current_time)
                    current_time = current_time + \
                        timedelta(seconds=interval_second)
                    if current_time > end:
                        break
                    cron_combo = (sv, cron_script)
                    if current_time not in targets:
                        targets[current_time] = [cron_combo]
                    else:
                        targets[current_time].append(cron_combo)
        return targets

    def __iter__(self):
        for t, s in sorted(self.crons.items()):
            yield (t, s)

    def create_file_object_by_server(self, directory):
        """Read cronfile and create file object."""
        server_list = {}
        for server_file in os.listdir(directory):
            file_path = os.path.join(directory, server_file)
            with open(file_path, 'r') as f:
                lines = f.readlines()
                name, __ext = os.path.splitext(server_file)
                server_list[name] = lines
        return server_list

    def is_cron_script(self, cron_script):
        for ic in IGNORE_CASES:
            if ic in cron_script or cron_script == '':
                return False
        return True

    def normalize_cron_script(self, cron_script):
        result = re.sub(r'  +', '\t', cron_script)
        result_list = result.split('\t')
        return (' '.join(result_list[0:5]), result_list[-1])

if __name__ == '__main__':
    parse_command()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Predicts article quality and priority with respect to a WikiProject.
Copyright (C) 2015 James Hare
Licensed under MIT License: http://mitlicense.org
"""

import json
import operator
import pywikibot
import re
import requests
from math import log  # https://www.youtube.com/watch?v=RTrAVpK9blw
from project_index import WikiProjectTools

def getpageviews(article):
    '''
    Queries stats.grok.se for the number of page views in the last 30 days
    Takes string *article* as input, returns view count.
    Does NOT take the logarithm of the view count.
    '''

    grok = requests.get("http://stats.grok.se/json/en/latest30/{0}".format(article))
    result = grok.json()
    counter = 0
    for dailyvalue in result['daily_views'].values():
        counter += dailyvalue

    return counter

def getlinkcount(wptools, package):
    '''
    Gets a list of inbound links for a list of articles
    Takes list *package* as input, returns list of tuples (article, linkcount)
    Input MUST be a list. If there is just one article, enter it as such: [article]
    '''

    if len(package) > 1:
        query_builder = 'select pl_title, count(*) from pagelinks where pl_namespace = 0 and pl_title in {0} group by pl_title;'.format(tuple(package))
    else:
        query_builder = 'select pl_title, count(*) from pagelinks where pl_namespace = 0 and pl_title in {0} group by pl_title;'.format(package[0])

    output = []
    for row in wptools.query('wiki', query_builder, None):
        output.append((row[0].decode('utf-8'), log(row[1])))

    return output

class QualityPredictor:
    def qualitypredictor(self, pagetitle):
        print("Argh! Not ready yet!")
        # chat it up with ORES

class PriorityPredictor:
    def __init__(self, wikiproject, unknownpriority):
        self.wptools = WikiProjectTools()
        self.score = []  # Sorted list of tuples; allows for ranking
        self.score_unranked = {}  # Unsorted dictionary "article: value"; allows for easily looking up scores later

        # We need all the articles for a WikiProject, since the system works by comparing stats for an article to the others.
        self.articles = []   # List of strings (article titles)
        pageviews = []  # List of tuples (article title, log of view count)
        linkcount = []  # List of tuples (article title, log of link count)
        for row in self.wptools.query('index', 'select pi_page from projectindex where pi_project = "Wikipedia:{0}";'.format(wikiproject), None):
            if row[0].startswith("Talk:"):  #
                article = row[0][5:] # Stripping out "Talk:"
                self.articles.append(article)

                # Page view count
                # Unfortunately, there is no way to batch this.
                pageviews.append((article, log(getpageviews(article))))

        # Inbound link count
        # This *is* batched, thus broken out of the loop
        packages = []
        for i in range(0, len(self.articles), 10000):
            packages.append(self.articles[i:i+10000])

        for package in packages:
                toappend = getlinkcount(wptools, package)
                for item in toappend:
                    linkcount.append(item)

        # Sorting...
        pageviews = sorted(pageviews, key=operator.itemgetter(1), reverse=True)
        linkcount = sorted(linkcount, key=operator.itemgetter(1), reverse=True)

        # Computing relative pageviews and linkcount
        # "Relative" means "as a ratio to the highest rank".
        # The most viewed article has a relative pageview score of 1.00. Goes lower from there.

        pageviews_relative = {}
        linkcount_relative = {}

        self.mostviews = pageviews[0][1]
        self.mostlinks = linkcount[0][1]

        for pair in pageviews:
            article = pair[0]
            count = pair[1]
            pageviews_relative[article] = count / self.mostviews

        for pair in linkcount:
            article = pair[0]
            count = pair[1]
            linkcount_relative[article] = count / self.mostlinks

        for article in self.articles:
            weightedscore = (pageviews_relative[article] * 0.75) + (linkcount_relative[article] * 0.25)
            self.score.append((article, weightscored))
            self.score_unranked[article] = weightedscore

        self.score = sorted(self.score, key=operator.itemgetter(1), reverse=True)

        # Calculating minimum scores
        # The idea is that there is a minimum score for something to be top, high, or mid-priority
        # The script is fed the category name for the unknown-importance/unknown-priority category
        # Based on this, derive category names for top/high/mid/low, add all the counts together...
        # ...then calculate ratio for top/high/mid as a ratio of the total...
        # ...multiply that ratio by the count of self.score, convert to an integer
        # ...and then threshold = self.score[that integer][1]
        # This gives us a general sense of what proportion of articles should be considered top/high/mid/low
        # Far from perfect but it's a start.

        toppriority = unknownpriority.replace("Unknown-", "Top-")
        highpriority = unknownpriority.replace("Unknown-", "High-")
        midpriority = unknownpriority.replace("Unknown-", "Mid-")
        lowpriority = unknownpriority.replace("Unknown-", "Low-")  # Easy enough...

        toppriority_count = wptools.query('wiki', 'select count(*) from categorylinks where cl_type = "page" and cl_to = {0}'.format(toppriority), None)[0][0]
        highpriority_count = wptools.query('wiki', 'select count(*) from categorylinks where cl_type = "page" and cl_to = {0}'.format(highpriority), None)[0][0]
        midpriority_count = wptools.query('wiki', 'select count(*) from categorylinks where cl_type = "page" and cl_to = {0}'.format(midpriority), None)[0][0]
        lowpriority_count = wptools.query('wiki', 'select count(*) from categorylinks where cl_type = "page" and cl_to = {0}'.format(lowpriority), None)[0][0]

        total_assessed = toppriority_count + highpriority_count + midpriority_count + lowpriority_count

        top_index = int((toppriority_count / total_assessed) * len(self.articles)) - 1
        high_index = int((highpriority_count / total_assessed) * len(self.articles)) - 1
        mid_index = int((midpriority_count / total_assessed) * len(self.articles)) - 1

        self.threshold_top = self.score[top_index][1]
        self.threshold_high = self.score[high_index][1]
        self.threshold_mid = self.score[mid_index][1]

    def prioritypredictor(self, pagetitle):
        # Pull pagescore if already defined
        # Otherwise, compute it "de novo"
        if pagetitle in self.articles:
            pagescore = self.score_unranked[pagetitle]
        else:
            pageviews = getpageviews(pagetitle) / self.mostviews
            linkcount = getlinkcount([pagetitle])[0][0] / self.mostlinks
            pagescore = (pageviews * 0.75) + (linkcount * 0.25)

        if pagescore >= self.threshold_top:
            return "Top"

        if pagescore >= self.threshold_high:
            return "High"

        if pagescore >= self.threshold_mid:
            return "Mid"

        # If none of these...
        return "Low"<commit_msg>Bug fixes, adding print statements<commit_after># -*- coding: utf-8 -*-
"""
Predicts article quality and priority with respect to a WikiProject.
Copyright (C) 2015 James Hare
Licensed under MIT License: http://mitlicense.org
"""

import json
import operator
import pywikibot
import re
import requests
from math import log  # https://www.youtube.com/watch?v=RTrAVpK9blw
from project_index import WikiProjectTools

def getpageviews(article):
    '''
    Queries stats.grok.se for the number of page views in the last 30 days
    Takes string *article* as input, returns view count.
    Does NOT take the logarithm of the view count.
    '''

    grok = requests.get("http://stats.grok.se/json/en/latest30/{0}".format(article))
    result = grok.json()
    counter = 0
    for dailyvalue in result['daily_views'].values():
        counter += dailyvalue

    return counter

def getlinkcount(wptools, package):
    '''
    Gets a list of inbound links for a list of articles
    Takes list *package* as input, returns list of tuples (article, log of linkcount)
    Input MUST be a list. If there is just one article, enter it as such: [article]
    '''

    if len(package) > 1:
        query_builder = 'select pl_title, count(*) from pagelinks where pl_namespace = 0 and pl_title in {0} group by pl_title;'.format(tuple(package))
    else:
        query_builder = 'select pl_title, count(*) from pagelinks where pl_namespace = 0 and pl_title in {0} group by pl_title;'.format(package[0])

    output = []
    for row in wptools.query('wiki', query_builder, None):
        output.append((row[0].decode('utf-8'), log(row[1])))

    return output

class QualityPredictor:
    def qualitypredictor(self, pagetitle):
        print("Argh! Not ready yet!")
        # chat it up with ORES

class PriorityPredictor:
    def __init__(self, wikiproject, unknownpriority):
        print("Initializing the Priority Predictor for: " + wikiproject)
        self.wptools = WikiProjectTools()
        self.score = []  # Sorted list of tuples; allows for ranking
        self.score_unranked = {}  # Unsorted dictionary "article: value"; allows for easily looking up scores later

        # We need all the articles for a WikiProject, since the system works by comparing stats for an article to the others.
        print("Getting list of articles in the WikiProject...")
        self.articles = []   # List of strings (article titles)
        pageviews = []  # List of tuples (article title, log of view count)
        linkcount = []  # List of tuples (article title, log of link count)
        for row in self.wptools.query('index', 'select pi_page from projectindex where pi_project = "Wikipedia:{0}";'.format(wikiproject), None):
            if row[0].startswith("Talk:"):  #
                article = row[0][5:] # Stripping out "Talk:"
                self.articles.append(article)

                # Page view count
                # Unfortunately, there is no way to batch this.
                print("Getting pageviews for: " + article)
                pageviews.append((article, log(getpageviews(article))))

        # Inbound link count
        # This *is* batched, thus broken out of the loop
        print("Getting inbound link count...")
        packages = []
        for i in range(0, len(self.articles), 10000):
            packages.append(self.articles[i:i+10000])

        for package in packages:
                toappend = getlinkcount(wptools, package)
                for item in toappend:
                    linkcount.append(item)

        # Sorting...
        pageviews = sorted(pageviews, key=operator.itemgetter(1), reverse=True)
        linkcount = sorted(linkcount, key=operator.itemgetter(1), reverse=True)

        # Computing relative pageviews and linkcount
        # "Relative" means "as a ratio to the highest rank".
        # The most viewed article has a relative pageview score of 1.00. Goes lower from there.

        print("Computing relative pageviews and linkcount...")
        pageviews_relative = {}
        linkcount_relative = {}

        self.mostviews = pageviews[0][1]
        self.mostlinks = linkcount[0][1]

        for pair in pageviews:
            article = pair[0]
            count = pair[1]
            pageviews_relative[article] = count / self.mostviews

        for pair in linkcount:
            article = pair[0]
            count = pair[1]
            linkcount_relative[article] = count / self.mostlinks

        for article in self.articles:
            weightedscore = (pageviews_relative[article] * 0.75) + (linkcount_relative[article] * 0.25)
            self.score.append((article, weightscored))
            self.score_unranked[article] = weightedscore

        self.score = sorted(self.score, key=operator.itemgetter(1), reverse=True)

        # Calculating minimum scores
        # The idea is that there is a minimum score for something to be top, high, or mid-priority
        # The script is fed the category name for the unknown-importance/unknown-priority category
        # Based on this, derive category names for top/high/mid/low, add all the counts together...
        # ...then calculate ratio for top/high/mid as a ratio of the total...
        # ...multiply that ratio by the count of self.score, convert to an integer
        # ...and then threshold = self.score[that integer][1]
        # This gives us a general sense of what proportion of articles should be considered top/high/mid/low
        # Far from perfect but it's a start.

        print("Calculating priority thresholds...")
        toppriority = unknownpriority.replace("Unknown-", "Top-")
        highpriority = unknownpriority.replace("Unknown-", "High-")
        midpriority = unknownpriority.replace("Unknown-", "Mid-")
        lowpriority = unknownpriority.replace("Unknown-", "Low-")  # Easy enough...

        toppriority_count = wptools.query('wiki', 'select count(*) from categorylinks where cl_type = "page" and cl_to = {0}'.format(toppriority), None)[0][0]
        highpriority_count = wptools.query('wiki', 'select count(*) from categorylinks where cl_type = "page" and cl_to = {0}'.format(highpriority), None)[0][0]
        midpriority_count = wptools.query('wiki', 'select count(*) from categorylinks where cl_type = "page" and cl_to = {0}'.format(midpriority), None)[0][0]
        lowpriority_count = wptools.query('wiki', 'select count(*) from categorylinks where cl_type = "page" and cl_to = {0}'.format(lowpriority), None)[0][0]

        total_assessed = toppriority_count + highpriority_count + midpriority_count + lowpriority_count

        top_index = int((toppriority_count / total_assessed) * len(self.articles) - 1)
        high_index = int((highpriority_count / total_assessed) * len(self.articles) -1)
        mid_index = int((midpriority_count / total_assessed) * len(self.articles) -1)

        self.threshold_top = self.score[top_index][1]
        self.threshold_high = self.score[high_index][1]
        self.threshold_mid = self.score[mid_index][1]

    def prioritypredictor(self, pagetitle):
        # Pull pagescore if already defined
        # Otherwise, compute it "de novo"
        if pagetitle in self.articles:
            pagescore = self.score_unranked[pagetitle]
        else:
            pageviews = log(getpageviews(pagetitle)) / self.mostviews
            linkcount = getlinkcount([pagetitle])[0][1] / self.mostlinks
            pagescore = (pageviews * 0.75) + (linkcount * 0.25)

        if pagescore >= self.threshold_top:
            return "Top"

        if pagescore >= self.threshold_high:
            return "High"

        if pagescore >= self.threshold_mid:
            return "Mid"

        # If none of these...
        return "Low"<|endoftext|>