<commit_before>#!/usr/bin/env python

import os
import platform
import re
import shutil
import stat
from HTMLParser import HTMLParser

from optparse import OptionParser
from subprocesses import captureStdout, normExpUserPath, shellify, vdump

# Use curl/wget rather than urllib because urllib can't check certs.
useCurl = False

def readFromURL(url):
    '''
    Reads in a URL and returns its contents as a list.
    '''
    inpCmdList = ['curl', '--silent', url] if useCurl else ['wget', '-q', '-O', '-', url]
    out, retVal = captureStdout(inpCmdList, combineStderr=True, ignoreExitCode=True)
    if not useCurl and retVal == 5:
        print 'Unable to read from URL. If you installed wget using MacPorts, you should put ' + \
              '"CA_CERTIFICATE=/opt/local/share/curl/curl-ca-bundle.crt" (without the quotes) ' + \
              'in ~/.wgetrc'
        raise Exception('Unable to read from URL. Please check your ~/.wgetrc file.')
    elif retVal != 0:
        print 'inpCmdList is: ' + shellify(inpCmdList)
        raise Exception('The following exit code was returned: ' + str(retVal))
    else:
        return out

def downloadURL(url, dest):
    '''
    Reads in a URL and downloads it to a destination.
    '''
    inpCmdList = ['curl', '--output', dest, url] if useCurl else ['wget', '-O', dest, url]
    out, retVal = captureStdout(inpCmdList, combineStderr=True, ignoreExitCode=True)
    if retVal != 0:
        print out
        raise Exception('Return code is not 0, but is: ' + str(retVal))
    return dest

def parseOptions():
    usage = 'Usage: %prog [options]'
    parser = OptionParser(usage)
    parser.disable_interspersed_args()

    parser.set_defaults(
        compileType = 'dbg',
        downloadFolder = os.getcwdu(),
        repoName = 'mozilla-central',
        enableJsShell = False,
    )

    parser.add_option('-c', '--compiletype', dest='compileType',
                      help='Sets the compile type to be downloaded. Must be "dbg" or "opt".' + \
                           'Defaults to "%default".')
    parser.add_option('-a', '--architecture',
                      dest='arch',
                      type='choice',
                      choices=['32', '64'],
                      help='Test architecture. Only accepts "32" or "64"')
    parser.add_option('-w', '--downloadfolder', dest='downloadFolder',
                      help='Sets the folder to download builds in. Defaults to the current ' + \
                           'working directory, which is "%default".')
    parser.add_option('-r', '--repoName', dest='repoName',
                      help='Sets the repository to be fuzzed. Defaults to "%default".')
    parser.add_option('-d', '--remotedir', dest='remoteDir',
                      help='Sets the remote directory from which the files are to be obtained ' + \
                           'from. The default is to grab the latest from mozilla-central.')
    parser.add_option('-s', '--enable-jsshell', dest='enableJsShell', action='store_true',
                      help='Sets the compile type to be fuzzed. Defaults to "%default".')

    options, args = parser.parse_args()
    assert options.compileType in ['dbg', 'opt']
    assert len(args) == 0
    return options

class MyHTMLParser(HTMLParser):
    def getHrefLinks(self, html):
        '''
        Returns href links as a list.
        '''
        self.hrefLinksList = []
        self.feed(html)
        return self.hrefLinksList
    def handle_starttag(self, tag, attrs):
        aTagFound = False
        if tag == 'a':
            aTagFound = True
        for attr in attrs:
            if not aTagFound:
                break
            if aTagFound and attr[0] == 'href' and attr[1][:1] not in ('?', '/'):
                self.hrefLinksList.append(attr[1])

def httpDirList(directory):
    '''
    Reads in an input directory and returns the href links in the directory as a list.
    '''
    print "Looking in " + directory + "..."
    page = readFromURL(directory)
    vdump('Finished reading from: ' + directory)

    parser = MyHTMLParser()
    fileList = parser.getHrefLinks(page)
    return fileList

def unzip(fn, dest):
    '''
    Extracts .zip files to their destination.
    '''
    captureStdout(['unzip', fn, '-d', dest])

def untarbz2(fn, dest):
    '''
    Extracts .tar.bz2 files to their destination.
    '''
    if not os.path.exists(dest):
        os.mkdir(dest)
    captureStdout(['tar', '-C', dest, '-xjf', os.path.abspath(fn)])

def undmg(fn, dest, mountpoint):
    '''
    Extracts .dmg files to their destination via a mount point.
    '''
    if os.path.exists(mountpoint):
        # If the mount point already exists, detach it first.
        captureStdout(['hdiutil', 'detach', mountpoint, '-force'])
    captureStdout(['hdiutil', 'attach', '-quiet', '-mountpoint', mountpoint, fn])
    try:
        apps = [x for x in os.listdir(mountpoint) if x.endswith('app')]
        assert len(apps) == 1
        shutil.copytree(mountpoint + '/' + apps[0], dest + '/' + apps[0])
    finally:
        captureStdout(['hdiutil', 'detach', mountpoint])

def downloadBuild(httpDir, targetDir, jsShell=False, wantSymbols=True, wantTests=True):
    '''
    Downloads the build specified, along with symbols and tests. Returns True when all are obtained.
    '''
    wantSymbols = wantSymbols and not jsShell  # Bug 715365, js shell currently lacks native symbols
    wantTests = wantTests and not jsShell
    gotApp = False
    gotTests = False
    gotTxtFile = False
    gotSyms = False
    # Create build folder and a download subfolder.
    buildDir = os.path.abspath(normExpUserPath(os.path.join(targetDir, 'build')))
    if os.path.exists(buildDir):
        print "Deleting old build..."
        shutil.rmtree(buildDir)
    os.mkdir(buildDir)
    downloadFolder = os.path.join(buildDir, 'download')
    os.mkdir(downloadFolder)

    with open(os.path.join(downloadFolder, "source-url.txt"), "w") as f:
        f.writelines([httpDir])

    # Hack #1 for making os.path.join(reftestScriptDir, automation.DEFAULT_APP) work is to:
    # Call this directory "dist".
    appDir = os.path.join(buildDir, 'dist') + os.sep
    testsDir = os.path.join(buildDir, 'tests') + os.sep
    symbolsDir = os.path.join(buildDir, 'symbols') + os.sep
    fileHttpRawList = httpDirList(httpDir)
    # We only want files, those with file extensions, not folders.
    fileHttpList = [httpDir + x for x in fileHttpRawList if '.' in x and 'mozilla.org' not in x]

    for remotefn in fileHttpList:
        localfn = os.path.join(downloadFolder, remotefn.split('/')[-1])
        if remotefn.endswith('.tests.zip') and wantTests:
            print 'Downloading tests...',
            dlAction = downloadURL(remotefn, localfn)
            print 'extracting...',
            unzip(dlAction, testsDir)
            print 'completed!'
            gotTests = True
        if remotefn.split('/')[-1].endswith('.txt'):
            print 'Downloading text file...',
            downloadURL(remotefn, localfn)
            print 'completed!'
            gotTxtFile = True
        if jsShell:
            if remotefn.split('/')[-1].startswith('jsshell-'):
                print 'Downloading js shell...',
                dlAction = downloadURL(remotefn, localfn)
                print 'extracting...',
                unzip(dlAction, appDir)
                print 'completed!'
                gotApp = True  # Bug 715365 - note that js shell currently lacks native symbols
        else:
            if remotefn.endswith('.linux-i686.tar.bz2') or \
                remotefn.endswith('.linux-x86_64.tar.bz2'):
                print 'Downloading application...',
                dlAction = downloadURL(remotefn, localfn)
                print 'extracting...',
                untarbz2(dlAction, appDir)
                print 'completed!'

                # Hack #2 to make os.path.join(reftestScriptDir, automation.DEFAULT_APP) work.
                os.rename(os.path.join(appDir, 'firefox'), os.path.join(appDir, 'bin'))
                stackwalk = os.path.join(buildDir, 'minidump_stackwalk')
                stackwalkUrl = \
            'https://hg.mozilla.org/build/tools/raw-file/default/breakpad/linux/minidump_stackwalk' \
            if remotefn.endswith('.linux-i686.tar.bz2') else \
            'https://hg.mozilla.org/build/tools/raw-file/default/breakpad/linux64/minidump_stackwalk'
                downloadURL(stackwalkUrl, stackwalk)
                os.chmod(stackwalk, stat.S_IRWXU)
                gotApp = True
            if remotefn.endswith('.win32.zip'):
                print 'Downloading application...',
                dlAction = downloadURL(remotefn, localfn)
                print 'extracting...',
                unzip(dlAction, appDir)
                print 'completed!'

                # Hack #2 for making os.path.join(reftestScriptDir, automation.DEFAULT_APP) work.
                os.rename(os.path.join(appDir, 'firefox'), os.path.join(appDir, 'bin'))
                for filename in ['minidump_stackwalk.exe', 'cyggcc_s-1.dll',
                                 'cygstdc++-6.dll', 'cygwin1.dll']:
                    remoteURL = \
            'https://hg.mozilla.org/build/tools/raw-file/default/breakpad/win32/%s' % filename
                    localfile = os.path.join(buildDir, filename)
                    downloadURL(remoteURL, localfile)
                gotApp = True
            if remotefn.endswith('.mac.dmg') or remotefn.endswith('.mac64.dmg'):
                print 'Downloading application...',
                dlAction = downloadURL(remotefn, localfn)
                print 'extracting...',
                undmg(dlAction, appDir, os.path.join(buildDir, 'MOUNTEDDMG'))
                print 'completed!'

                stackwalk = os.path.join(buildDir, 'minidump_stackwalk')
                stackwalkUrl = \
            'https://hg.mozilla.org/build/tools/raw-file/default/breakpad/osx/minidump_stackwalk'
                downloadURL(stackwalkUrl, stackwalk)
                os.chmod(stackwalk, stat.S_IRWXU)
                gotApp = True
            if remotefn.endswith('.crashreporter-symbols.zip') and wantSymbols:
                print 'Downloading crash reporter symbols...',
                dlAction = downloadURL(remotefn, localfn)
                print 'extracting...',
                unzip(dlAction, symbolsDir)
                print 'completed!'
                gotSyms = True
    return gotApp and gotTxtFile and (gotTests or not wantTests) and (gotSyms or not wantSymbols)

def isNumericSubDir(n):
    '''
    Returns True if input is a numeric directory, False if not. e.g. 1234/ returns True
    '''
    return re.match(r'^\d+$', n.split('/')[0])

def downloadLatestBuild(buildType, workingDir, getJsShell=False):
    '''
    Downloads the latest build based on machine type, e.g. mozilla-central-macosx-debug.
    '''
    buildsHttpDir = 'https://ftp.mozilla.org/pub/mozilla.org/firefox/tinderbox-builds/' + \
                    buildType + '/'
    buildsRawList = httpDirList(buildsHttpDir)
    buildsSubDirList = [x for x in buildsRawList if isNumericSubDir(x)]
    builds = [c[:-1] for c in buildsSubDirList]
    for b in reversed(builds):  # Try downloading the latest build first.
        fullb = buildsHttpDir + b + '/'
        if downloadBuild(fullb, workingDir, jsShell=getJsShell):
            return fullb
    raise Exception('No builds in ' + buildsHttpDir + '!')

def mozPlatformDetails():
    '''
    Determines the platform of the system and returns the RelEng-specific build type.
    '''
    s = platform.system()
    if s == "Darwin":
        return ("macosx", "macosx64", platform.architecture()[0] == "64bit")
    elif s == "Linux":
        return ("linux", "linux64", platform.machine() == "x86_64")
    elif s == 'Windows':
        return ("win32", "win64", False)
    else:
        raise Exception("Unknown platform.system(): " + s)

def mozPlatform(arch):
    '''
    Returns the native build type of the current machine.
    '''
    (name32, name64, native64) = mozPlatformDetails()
    if arch == "64":
        return name64
    elif arch == "32":
        return name32
    elif arch == None:
        # FIXME: Eventually, we should set 64-bit as native for Win64. We should also aim to test
        # both 32-bit and 64-bit Firefox builds on any platform that supports both. Let us make
        # sure Python detects 32-bit Windows vs 64-bit Windows correctly before changing this.
        return (name64 if native64 else name32)
    else:
        raise Exception("The arch passed to mozPlatform must be '64', '32', or None")

def defaultBuildType(options):
    '''Returns the default build type as per RelEng, e.g. mozilla-central-macosx-debug.'''
    return options.repoName + '-' + mozPlatform(options.arch) + \
        ('-debug' if options.compileType == 'dbg' else '')

def main():
    options = parseOptions()
    if options.remoteDir is not None:
        print downloadBuild(options.remoteDir, options.downloadFolder, jsShell=options.enableJsShell)
    else:
        downloadLatestBuild(defaultBuildType(options), options.downloadFolder,
                            getJsShell=options.enableJsShell)

if __name__ == "__main__":
    main()
<commit_msg>Hack around https://bugzilla.mozilla.org/show_bug.cgi?id=888748 by moving crashinject.exe to where automation.py won't see it.<commit_after>#!/usr/bin/env python

import os
import platform
import re
import shutil
import stat
from HTMLParser import HTMLParser

from optparse import OptionParser
from subprocesses import captureStdout, normExpUserPath, shellify, vdump

# Use curl/wget rather than urllib because urllib can't check certs.
useCurl = False

def readFromURL(url):
    '''
    Reads in a URL and returns its contents as a list.
    '''
    inpCmdList = ['curl', '--silent', url] if useCurl else ['wget', '-q', '-O', '-', url]
    out, retVal = captureStdout(inpCmdList, combineStderr=True, ignoreExitCode=True)
    if not useCurl and retVal == 5:
        print 'Unable to read from URL. If you installed wget using MacPorts, you should put ' + \
              '"CA_CERTIFICATE=/opt/local/share/curl/curl-ca-bundle.crt" (without the quotes) ' + \
              'in ~/.wgetrc'
        raise Exception('Unable to read from URL. Please check your ~/.wgetrc file.')
    elif retVal != 0:
        print 'inpCmdList is: ' + shellify(inpCmdList)
        raise Exception('The following exit code was returned: ' + str(retVal))
    else:
        return out

def downloadURL(url, dest):
    '''
    Reads in a URL and downloads it to a destination.
    '''
    inpCmdList = ['curl', '--output', dest, url] if useCurl else ['wget', '-O', dest, url]
    out, retVal = captureStdout(inpCmdList, combineStderr=True, ignoreExitCode=True)
    if retVal != 0:
        print out
        raise Exception('Return code is not 0, but is: ' + str(retVal))
    return dest

def parseOptions():
    usage = 'Usage: %prog [options]'
    parser = OptionParser(usage)
    parser.disable_interspersed_args()

    parser.set_defaults(
        compileType = 'dbg',
        downloadFolder = os.getcwdu(),
        repoName = 'mozilla-central',
        enableJsShell = False,
    )

    parser.add_option('-c', '--compiletype', dest='compileType',
                      help='Sets the compile type to be downloaded. Must be "dbg" or "opt".' + \
                           'Defaults to "%default".')
    parser.add_option('-a', '--architecture',
                      dest='arch',
                      type='choice',
                      choices=['32', '64'],
                      help='Test architecture. Only accepts "32" or "64"')
    parser.add_option('-w', '--downloadfolder', dest='downloadFolder',
                      help='Sets the folder to download builds in. Defaults to the current ' + \
                           'working directory, which is "%default".')
    parser.add_option('-r', '--repoName', dest='repoName',
                      help='Sets the repository to be fuzzed. Defaults to "%default".')
    parser.add_option('-d', '--remotedir', dest='remoteDir',
                      help='Sets the remote directory from which the files are to be obtained ' + \
                           'from. The default is to grab the latest from mozilla-central.')
    parser.add_option('-s', '--enable-jsshell', dest='enableJsShell', action='store_true',
                      help='Sets the compile type to be fuzzed. Defaults to "%default".')

    options, args = parser.parse_args()
    assert options.compileType in ['dbg', 'opt']
    assert len(args) == 0
    return options

class MyHTMLParser(HTMLParser):
    def getHrefLinks(self, html):
        '''
        Returns href links as a list.
        '''
        self.hrefLinksList = []
        self.feed(html)
        return self.hrefLinksList
    def handle_starttag(self, tag, attrs):
        aTagFound = False
        if tag == 'a':
            aTagFound = True
        for attr in attrs:
            if not aTagFound:
                break
            if aTagFound and attr[0] == 'href' and attr[1][:1] not in ('?', '/'):
                self.hrefLinksList.append(attr[1])

def httpDirList(directory):
    '''
    Reads in an input directory and returns the href links in the directory as a list.
    '''
    print "Looking in " + directory + "..."
    page = readFromURL(directory)
    vdump('Finished reading from: ' + directory)

    parser = MyHTMLParser()
    fileList = parser.getHrefLinks(page)
    return fileList

def unzip(fn, dest):
    '''
    Extracts .zip files to their destination.
    '''
    captureStdout(['unzip', fn, '-d', dest])

def untarbz2(fn, dest):
    '''
    Extracts .tar.bz2 files to their destination.
    '''
    if not os.path.exists(dest):
        os.mkdir(dest)
    captureStdout(['tar', '-C', dest, '-xjf', os.path.abspath(fn)])

def undmg(fn, dest, mountpoint):
    '''
    Extracts .dmg files to their destination via a mount point.
    '''
    if os.path.exists(mountpoint):
        # If the mount point already exists, detach it first.
        captureStdout(['hdiutil', 'detach', mountpoint, '-force'])
    captureStdout(['hdiutil', 'attach', '-quiet', '-mountpoint', mountpoint, fn])
    try:
        apps = [x for x in os.listdir(mountpoint) if x.endswith('app')]
        assert len(apps) == 1
        shutil.copytree(mountpoint + '/' + apps[0], dest + '/' + apps[0])
    finally:
        captureStdout(['hdiutil', 'detach', mountpoint])

def downloadBuild(httpDir, targetDir, jsShell=False, wantSymbols=True, wantTests=True):
    '''
    Downloads the build specified, along with symbols and tests. Returns True when all are obtained.
    '''
    wantSymbols = wantSymbols and not jsShell  # Bug 715365, js shell currently lacks native symbols
    wantTests = wantTests and not jsShell
    gotApp = False
    gotTests = False
    gotTxtFile = False
    gotSyms = False
    # Create build folder and a download subfolder.
    buildDir = os.path.abspath(normExpUserPath(os.path.join(targetDir, 'build')))
    if os.path.exists(buildDir):
        print "Deleting old build..."
        shutil.rmtree(buildDir)
    os.mkdir(buildDir)
    downloadFolder = os.path.join(buildDir, 'download')
    os.mkdir(downloadFolder)

    with open(os.path.join(downloadFolder, "source-url.txt"), "w") as f:
        f.writelines([httpDir])

    # Hack #1 for making os.path.join(reftestScriptDir, automation.DEFAULT_APP) work is to:
    # Call this directory "dist".
    appDir = os.path.join(buildDir, 'dist') + os.sep
    testsDir = os.path.join(buildDir, 'tests') + os.sep
    symbolsDir = os.path.join(buildDir, 'symbols') + os.sep
    fileHttpRawList = httpDirList(httpDir)
    # We only want files, those with file extensions, not folders.
    fileHttpList = [httpDir + x for x in fileHttpRawList if '.' in x and 'mozilla.org' not in x]

    for remotefn in fileHttpList:
        localfn = os.path.join(downloadFolder, remotefn.split('/')[-1])
        if remotefn.endswith('.tests.zip') and wantTests:
            print 'Downloading tests...',
            dlAction = downloadURL(remotefn, localfn)
            print 'extracting...',
            unzip(dlAction, testsDir)
            moveCrashInjector(testsDir)
            print 'completed!'
            gotTests = True
        if remotefn.split('/')[-1].endswith('.txt'):
            print 'Downloading text file...',
            downloadURL(remotefn, localfn)
            print 'completed!'
            gotTxtFile = True
        if jsShell:
            if remotefn.split('/')[-1].startswith('jsshell-'):
                print 'Downloading js shell...',
                dlAction = downloadURL(remotefn, localfn)
                print 'extracting...',
                unzip(dlAction, appDir)
                print 'completed!'
                gotApp = True  # Bug 715365 - note that js shell currently lacks native symbols
        else:
            if remotefn.endswith('.linux-i686.tar.bz2') or \
                remotefn.endswith('.linux-x86_64.tar.bz2'):
                print 'Downloading application...',
                dlAction = downloadURL(remotefn, localfn)
                print 'extracting...',
                untarbz2(dlAction, appDir)
                print 'completed!'

                # Hack #2 to make os.path.join(reftestScriptDir, automation.DEFAULT_APP) work.
                os.rename(os.path.join(appDir, 'firefox'), os.path.join(appDir, 'bin'))
                stackwalk = os.path.join(buildDir, 'minidump_stackwalk')
                stackwalkUrl = \
            'https://hg.mozilla.org/build/tools/raw-file/default/breakpad/linux/minidump_stackwalk' \
            if remotefn.endswith('.linux-i686.tar.bz2') else \
            'https://hg.mozilla.org/build/tools/raw-file/default/breakpad/linux64/minidump_stackwalk'
                downloadURL(stackwalkUrl, stackwalk)
                os.chmod(stackwalk, stat.S_IRWXU)
                gotApp = True
            if remotefn.endswith('.win32.zip'):
                print 'Downloading application...',
                dlAction = downloadURL(remotefn, localfn)
                print 'extracting...',
                unzip(dlAction, appDir)
                print 'completed!'

                # Hack #2 for making os.path.join(reftestScriptDir, automation.DEFAULT_APP) work.
                os.rename(os.path.join(appDir, 'firefox'), os.path.join(appDir, 'bin'))
                for filename in ['minidump_stackwalk.exe', 'cyggcc_s-1.dll',
                                 'cygstdc++-6.dll', 'cygwin1.dll']:
                    remoteURL = \
            'https://hg.mozilla.org/build/tools/raw-file/default/breakpad/win32/%s' % filename
                    localfile = os.path.join(buildDir, filename)
                    downloadURL(remoteURL, localfile)
                gotApp = True
            if remotefn.endswith('.mac.dmg') or remotefn.endswith('.mac64.dmg'):
                print 'Downloading application...',
                dlAction = downloadURL(remotefn, localfn)
                print 'extracting...',
                undmg(dlAction, appDir, os.path.join(buildDir, 'MOUNTEDDMG'))
                print 'completed!'

                stackwalk = os.path.join(buildDir, 'minidump_stackwalk')
                stackwalkUrl = \
            'https://hg.mozilla.org/build/tools/raw-file/default/breakpad/osx/minidump_stackwalk'
                downloadURL(stackwalkUrl, stackwalk)
                os.chmod(stackwalk, stat.S_IRWXU)
                gotApp = True
            if remotefn.endswith('.crashreporter-symbols.zip') and wantSymbols:
                print 'Downloading crash reporter symbols...',
                dlAction = downloadURL(remotefn, localfn)
                print 'extracting...',
                unzip(dlAction, symbolsDir)
                print 'completed!'
                gotSyms = True
    return gotApp and gotTxtFile and (gotTests or not wantTests) and (gotSyms or not wantSymbols)

def moveCrashInjector(tests):
    # Hackaround for crashinject.exe not being a reliable way to kill firefox.exe (see bug 888748)
    testsBin = os.path.join(tests, "bin")
    crashinject = os.path.join(testsBin, "crashinject.exe")
    if os.path.exists(crashinject):
        os.rename(crashinject, os.path.join(testsBin, "crashinject-disabled.exe"))

def isNumericSubDir(n):
    '''
    Returns True if input is a numeric directory, False if not. e.g. 1234/ returns True
    '''
    return re.match(r'^\d+$', n.split('/')[0])

def downloadLatestBuild(buildType, workingDir, getJsShell=False):
    '''
    Downloads the latest build based on machine type, e.g. mozilla-central-macosx-debug.
    '''
    buildsHttpDir = 'https://ftp.mozilla.org/pub/mozilla.org/firefox/tinderbox-builds/' + \
                    buildType + '/'
    buildsRawList = httpDirList(buildsHttpDir)
    buildsSubDirList = [x for x in buildsRawList if isNumericSubDir(x)]
    builds = [c[:-1] for c in buildsSubDirList]
    for b in reversed(builds):  # Try downloading the latest build first.
        fullb = buildsHttpDir + b + '/'
        if downloadBuild(fullb, workingDir, jsShell=getJsShell):
            return fullb
    raise Exception('No builds in ' + buildsHttpDir + '!')

def mozPlatformDetails():
    '''
    Determines the platform of the system and returns the RelEng-specific build type.
    '''
    s = platform.system()
    if s == "Darwin":
        return ("macosx", "macosx64", platform.architecture()[0] == "64bit")
    elif s == "Linux":
        return ("linux", "linux64", platform.machine() == "x86_64")
    elif s == 'Windows':
        return ("win32", "win64", False)
    else:
        raise Exception("Unknown platform.system(): " + s)

def mozPlatform(arch):
    '''
    Returns the native build type of the current machine.
    '''
    (name32, name64, native64) = mozPlatformDetails()
    if arch == "64":
        return name64
    elif arch == "32":
        return name32
    elif arch == None:
        # FIXME: Eventually, we should set 64-bit as native for Win64. We should also aim to test
        # both 32-bit and 64-bit Firefox builds on any platform that supports both. Let us make
        # sure Python detects 32-bit Windows vs 64-bit Windows correctly before changing this.
        return (name64 if native64 else name32)
    else:
        raise Exception("The arch passed to mozPlatform must be '64', '32', or None")

def defaultBuildType(options):
    '''Returns the default build type as per RelEng, e.g. mozilla-central-macosx-debug.'''
    return options.repoName + '-' + mozPlatform(options.arch) + \
        ('-debug' if options.compileType == 'dbg' else '')

def main():
    options = parseOptions()
    if options.remoteDir is not None:
        print downloadBuild(options.remoteDir, options.downloadFolder, jsShell=options.enableJsShell)
    else:
        downloadLatestBuild(defaultBuildType(options), options.downloadFolder,
                            getJsShell=options.enableJsShell)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import sys
from threading import Thread
from findspam import FindSpam
import datahandling
from globalvars import GlobalVars
from datetime import datetime
import parsing
import metasmoke
import deletionwatcher
import excepthook
import regex
import json
import time

def should_whitelist_prevent_alert(user_url, reasons):
    is_whitelisted = datahandling.is_whitelisted_user(parsing.get_user_from_url(user_url))
    if not is_whitelisted:
        return False
    reasons_comparison = [r for r in set(reasons) if "username" not in r]
    return len(reasons_comparison) == 0

def should_reasons_prevent_tavern_posting(reasons):
    reasons_comparison = [r for r in set(reasons) if r not in GlobalVars.non_tavern_reasons]
    return len(reasons_comparison) == 0

def check_if_spam(title, body, user_name, user_url, post_site, post_id, is_answer, body_is_summary, owner_rep, post_score):
    if not body:
        body = ""
    test, why = FindSpam.test_post(title, body, user_name, post_site, is_answer, body_is_summary, owner_rep, post_score)
    if datahandling.is_blacklisted_user(parsing.get_user_from_url(user_url)):
        test.append("blacklisted user")
        blacklisted_user_data = datahandling.get_blacklisted_user_data(parsing.get_user_from_url(user_url))
        if len(blacklisted_user_data) > 1:
            message_url = 'http:' + blacklisted_user_data[1]
            blacklisted_post_url = blacklisted_user_data[2]
            if blacklisted_post_url:
                rel_url = blacklisted_post_url.replace("http:", "", 1)
                why += u"\nBlacklisted user - blacklisted for {} (https://m.erwaysoftware.com/posts/by-url?url={}) by {}".format(blacklisted_post_url, rel_url, message_url)
            else:
                why += u"\n" + u"Blacklisted user - blacklisted by {}".format(message_url)
    if 0 < len(test):
        if datahandling.has_already_been_posted(post_site, post_id, title) or datahandling.is_false_positive((post_id, post_site)) \
                or should_whitelist_prevent_alert(user_url, test) \
                or datahandling.is_ignored_post((post_id, post_site)) \
                or datahandling.is_auto_ignored_post((post_id, post_site)):
            return False, None, ""  # Don't repost. Reddit will hate you.
        return True, test, why
    return False, None, ""

def check_if_spam_json(json_data):
    text_data = json.loads(json_data)["data"]
    if text_data == "hb":
        return False, None, ""
    try:
        data = json.loads(text_data)
    except ValueError:
        GlobalVars.charcoal_hq.send_message(u"Encountered ValueError parsing the following:\n{0}".format(json_data), False)
        return False, None, ""
    if "ownerUrl" not in data:
        # owner's account doesn't exist anymore, no need to post it in chat:
        # http://chat.stackexchange.com/transcript/message/18380776#18380776
        return False, None, ""
    title = data["titleEncodedFancy"]
    title = parsing.unescape_title(title)
    body = data["bodySummary"]
    poster = data["ownerDisplayName"]
    url = data["url"]
    post_id = str(data["id"])
    print time.strftime("%Y-%m-%d %H:%M:%S"), title.encode("ascii", errors="replace")
    site = data["siteBaseHostAddress"]
    site = site.encode("ascii", errors="replace")
    sys.stdout.flush()
    is_spam, reason, why = check_if_spam(title=title,
                                         body=body,
                                         user_name=poster,
                                         user_url=url,
                                         post_site=site,
                                         post_id=post_id,
                                         is_answer=False,
                                         body_is_summary=True,
                                         owner_rep=1,
                                         post_score=0)
    return is_spam, reason, why

def handle_spam(title, body, poster, site, post_url, poster_url, post_id, reasons, is_answer, why="", owner_rep=None, post_score=None, up_vote_count=None, down_vote_count=None, question_id=None):
    post_url = parsing.to_protocol_relative(parsing.url_to_shortlink(post_url))
    poster_url = parsing.to_protocol_relative(parsing.user_url_to_shortlink(poster_url))
    reason = ", ".join(reasons)
    reason = reason[:1].upper() + reason[1:]  # reason is capitalised, unlike the entries of reasons list
    datahandling.append_to_latest_questions(site, post_id, title if not is_answer else "")
    if len(reasons) == 1 and ("all-caps title" in reasons or
                              "repeating characters in title" in reasons or
                              "repeating characters in body" in reasons or
                              "repeating characters in answer" in reasons or
                              "repeating words in title" in reasons or
                              "repeating words in body" in reasons or
                              "repeating words in answer" in reasons):
        datahandling.add_auto_ignored_post((post_id, site, datetime.now()))
    if why is not None and why != "":
        datahandling.add_why(site, post_id, why)
    if is_answer and question_id is not None:
        datahandling.add_post_site_id_link((post_id, site, "answer"), question_id)
    try:
        title = parsing.escape_special_chars_in_title(title)
        sanitized_title = regex.sub('(https?://|\n)', '', title)

        prefix = u"[ [SmokeDetector](//git.io/vgx7b) ]"
        if GlobalVars.metasmoke_key:
            prefix_ms = u"[ [SmokeDetector](//git.io/vgx7b) | [MS](//m.erwaysoftware.com/posts/by-url?url=" + post_url + ") ]"
        else:
            prefix_ms = prefix

        if not poster.strip():
            s = u" {}: [{}]({}) by a deleted user on `{}`".format(reason, sanitized_title.strip(), post_url, site)
            username = ""
            user_link = ""
        else:
            s = u" {}: [{}]({}) by [{}]({}) on `{}`" .format(reason, sanitized_title.strip(), post_url + "?smokeypost=yes", poster.strip(), poster_url, site)
            username = poster.strip()
            user_link = poster_url

        t_metasmoke = Thread(target=metasmoke.Metasmoke.send_stats_on_post,
                             args=(title, post_url, reason.split(", "), body, username, user_link, why, owner_rep, post_score, up_vote_count, down_vote_count))
        t_metasmoke.start()

        print GlobalVars.parser.unescape(s).encode('ascii', errors='replace')
        if time.time() >= GlobalVars.blockedTime["all"]:
            datahandling.append_to_latest_questions(site, post_id, title)
            if reason not in GlobalVars.experimental_reasons:
                if time.time() >= GlobalVars.blockedTime[GlobalVars.charcoal_room_id]:
                    chq_pings = datahandling.get_user_names_on_notification_list("stackexchange.com", GlobalVars.charcoal_room_id, site, GlobalVars.wrap)
                    chq_msg = prefix + s
                    chq_msg_pings = prefix + datahandling.append_pings(s, chq_pings)
                    chq_msg_pings_ms = prefix_ms + datahandling.append_pings(s, chq_pings)
                    msg_to_send = chq_msg_pings_ms if len(chq_msg_pings_ms) <= 500 else chq_msg_pings if len(chq_msg_pings) <= 500 else chq_msg[0:500]
                    GlobalVars.charcoal_hq.send_message(msg_to_send)
                if not should_reasons_prevent_tavern_posting(reasons) and site not in GlobalVars.non_tavern_sites and time.time() >= GlobalVars.blockedTime[GlobalVars.meta_tavern_room_id]:
                    tavern_pings = datahandling.get_user_names_on_notification_list("meta.stackexchange.com", GlobalVars.meta_tavern_room_id, site, GlobalVars.wrapm)
                    tavern_msg = prefix + s
                    tavern_msg_pings = prefix + datahandling.append_pings(s, tavern_pings)
                    tavern_msg_pings_ms = prefix_ms + datahandling.append_pings(s, tavern_pings)
                    msg_to_send = tavern_msg_pings_ms if len(tavern_msg_pings_ms) <= 500 else tavern_msg_pings if len(tavern_msg_pings) <= 500 else tavern_msg[0:500]
                    t_check_websocket = Thread(target=deletionwatcher.DeletionWatcher.post_message_if_not_deleted, args=((post_id, site, "answer" if is_answer else "question"), post_url, msg_to_send, GlobalVars.tavern_on_the_meta))
                    t_check_websocket.daemon = True
                    t_check_websocket.start()
                if site == "stackoverflow.com" and reason not in GlobalVars.non_socvr_reasons and time.time() >= GlobalVars.blockedTime[GlobalVars.socvr_room_id]:
                    socvr_pings = datahandling.get_user_names_on_notification_list("stackoverflow.com", GlobalVars.socvr_room_id, site, GlobalVars.wrapso)
                    socvr_msg = prefix + s
                    socvr_msg_pings = prefix + datahandling.append_pings(s, socvr_pings)
                    socvr_msg_pings_ms = prefix_ms + datahandling.append_pings(s, socvr_pings)
                    msg_to_send = socvr_msg_pings_ms if len(socvr_msg_pings_ms) <= 500 else socvr_msg_pings if len(socvr_msg_pings) <= 500 else socvr_msg[0:500]
                    GlobalVars.socvr.send_message(msg_to_send)

            for specialroom in GlobalVars.specialrooms:
                sites = specialroom["sites"]
                if site in sites and reason not in specialroom["unwantedReasons"]:
                    room = specialroom["room"]
                    if room.id not in GlobalVars.blockedTime or time.time() >= GlobalVars.blockedTime[room.id]:
                        room_site = room._client.host
                        room_id = int(room.id)
                        room_pings = datahandling.get_user_names_on_notification_list(room_site, room_id, site, room._client)
                        room_msg = prefix + s
                        room_msg_pings = prefix + datahandling.append_pings(s, room_pings)
                        room_msg_pings_ms = prefix_ms + datahandling.append_pings(s, room_pings)
                        msg_to_send = room_msg_pings_ms if len(room_msg_pings_ms) <= 500 else room_msg_pings if len(room_msg_pings) <= 500 else room_msg[0:500]
                        specialroom["room"].send_message(msg_to_send)
    except:
        exc_type, exc_obj, exc_tb = sys.exc_info()
        excepthook.uncaught_exception(exc_type, exc_obj, exc_tb)

def handle_user_with_all_spam(user, why):
    user_id = user[0]
    site = user[1]
    tab = "activity" if site == "stackexchange.com" else "topactivity"
    s = "[ [SmokeDetector](//git.io/vgx7b) ] All of this user's posts are spam: [user {} on {}](//{}/users/{}?tab={})" \
        .format(user_id, site, site, user_id, tab)
    print GlobalVars.parser.unescape(s).encode('ascii', errors='replace')
    datahandling.add_why_allspam(user, why)
    if time.time() >= GlobalVars.blockedTime[GlobalVars.charcoal_room_id]:
        GlobalVars.charcoal_hq.send_message(s)
    for specialroom in GlobalVars.specialrooms:
        room = specialroom["room"]
        if site in specialroom["sites"] and (room.id not in GlobalVars.blockedTime or time.time() >= GlobalVars.blockedTime[room.id]):
            room.send_message(s)
<commit_msg>Change 'yes' to 'true'<commit_after>import sys
from threading import Thread
from findspam import FindSpam
import datahandling
from globalvars import GlobalVars
from datetime import datetime
import parsing
import metasmoke
import deletionwatcher
import excepthook
import regex
import json
import time

def should_whitelist_prevent_alert(user_url, reasons):
    is_whitelisted = datahandling.is_whitelisted_user(parsing.get_user_from_url(user_url))
    if not is_whitelisted:
        return False
    reasons_comparison = [r for r in set(reasons) if "username" not in r]
    return len(reasons_comparison) == 0

def should_reasons_prevent_tavern_posting(reasons):
    reasons_comparison = [r for r in set(reasons) if r not in GlobalVars.non_tavern_reasons]
    return len(reasons_comparison) == 0

def check_if_spam(title, body, user_name, user_url, post_site, post_id, is_answer, body_is_summary, owner_rep, post_score):
    if not body:
        body = ""
    test, why = FindSpam.test_post(title, body, user_name, post_site, is_answer, body_is_summary, owner_rep, post_score)
    if datahandling.is_blacklisted_user(parsing.get_user_from_url(user_url)):
        test.append("blacklisted user")
        blacklisted_user_data = datahandling.get_blacklisted_user_data(parsing.get_user_from_url(user_url))
        if len(blacklisted_user_data) > 1:
            message_url = 'http:' + blacklisted_user_data[1]
            blacklisted_post_url = blacklisted_user_data[2]
            if blacklisted_post_url:
                rel_url = blacklisted_post_url.replace("http:", "", 1)
                why += u"\nBlacklisted user - blacklisted for {} (https://m.erwaysoftware.com/posts/by-url?url={}) by {}".format(blacklisted_post_url, rel_url, message_url)
            else:
                why += u"\n" + u"Blacklisted user - blacklisted by {}".format(message_url)
    if 0 < len(test):
        if datahandling.has_already_been_posted(post_site, post_id, title) or datahandling.is_false_positive((post_id, post_site)) \
                or should_whitelist_prevent_alert(user_url, test) \
                or datahandling.is_ignored_post((post_id, post_site)) \
                or datahandling.is_auto_ignored_post((post_id, post_site)):
            return False, None, ""  # Don't repost. Reddit will hate you.
        return True, test, why
    return False, None, ""

def check_if_spam_json(json_data):
    text_data = json.loads(json_data)["data"]
    if text_data == "hb":
        return False, None, ""
    try:
        data = json.loads(text_data)
    except ValueError:
        GlobalVars.charcoal_hq.send_message(u"Encountered ValueError parsing the following:\n{0}".format(json_data), False)
        return False, None, ""
    if "ownerUrl" not in data:
        # owner's account doesn't exist anymore, no need to post it in chat:
        # http://chat.stackexchange.com/transcript/message/18380776#18380776
        return False, None, ""
    title = data["titleEncodedFancy"]
    title = parsing.unescape_title(title)
    body = data["bodySummary"]
    poster = data["ownerDisplayName"]
    url = data["url"]
    post_id = str(data["id"])
    print time.strftime("%Y-%m-%d %H:%M:%S"), title.encode("ascii", errors="replace")
    site = data["siteBaseHostAddress"]
    site = site.encode("ascii", errors="replace")
    sys.stdout.flush()
    is_spam, reason, why = check_if_spam(title=title,
                                         body=body,
                                         user_name=poster,
                                         user_url=url,
                                         post_site=site,
                                         post_id=post_id,
                                         is_answer=False,
                                         body_is_summary=True,
                                         owner_rep=1,
                                         post_score=0)
    return is_spam, reason, why

def handle_spam(title, body, poster, site, post_url, poster_url, post_id, reasons, is_answer, why="", owner_rep=None, post_score=None, up_vote_count=None, down_vote_count=None, question_id=None):
    post_url = parsing.to_protocol_relative(parsing.url_to_shortlink(post_url))
    poster_url = parsing.to_protocol_relative(parsing.user_url_to_shortlink(poster_url))
    reason = ", ".join(reasons)
    reason = reason[:1].upper() + reason[1:]  # reason is capitalised, unlike the entries of reasons list
    datahandling.append_to_latest_questions(site, post_id, title if not is_answer else "")
    if len(reasons) == 1 and ("all-caps title" in reasons or
                              "repeating characters in title" in reasons or
                              "repeating characters in body" in reasons or
                              "repeating characters in answer" in reasons or
                              "repeating words in title" in reasons or
                              "repeating words in body" in reasons or
                              "repeating words in answer" in reasons):
        datahandling.add_auto_ignored_post((post_id, site, datetime.now()))
    if why is not None and why != "":
        datahandling.add_why(site, post_id, why)
    if is_answer and question_id is not None:
        datahandling.add_post_site_id_link((post_id, site, "answer"), question_id)
    try:
        title = parsing.escape_special_chars_in_title(title)
        sanitized_title = regex.sub('(https?://|\n)', '', title)

        prefix = u"[ [SmokeDetector](//git.io/vgx7b) ]"
        if GlobalVars.metasmoke_key:
            prefix_ms = u"[ [SmokeDetector](//git.io/vgx7b) | [MS](//m.erwaysoftware.com/posts/by-url?url=" + post_url + ") ]"
        else:
            prefix_ms = prefix

        if not poster.strip():
            s = u" {}: [{}]({}) by a deleted user on `{}`".format(reason, sanitized_title.strip(), post_url, site)
            username = ""
            user_link = ""
        else:
            s = u" {}: [{}]({}) by [{}]({}) on `{}`" .format(reason, sanitized_title.strip(), post_url + "?smokeypost=true", poster.strip(), poster_url, site)
            username = poster.strip()
            user_link = poster_url

        t_metasmoke = Thread(target=metasmoke.Metasmoke.send_stats_on_post,
                             args=(title, post_url, reason.split(", "), body, username, user_link, why, owner_rep, post_score, up_vote_count, down_vote_count))
        t_metasmoke.start()

        print GlobalVars.parser.unescape(s).encode('ascii', errors='replace')
        if time.time() >= GlobalVars.blockedTime["all"]:
            datahandling.append_to_latest_questions(site, post_id, title)
            if reason not in GlobalVars.experimental_reasons:
                if time.time() >= GlobalVars.blockedTime[GlobalVars.charcoal_room_id]:
                    chq_pings = datahandling.get_user_names_on_notification_list("stackexchange.com", GlobalVars.charcoal_room_id, site, GlobalVars.wrap)
                    chq_msg = prefix + s
                    chq_msg_pings = prefix + datahandling.append_pings(s, chq_pings)
                    chq_msg_pings_ms = prefix_ms + datahandling.append_pings(s, chq_pings)
                    msg_to_send = chq_msg_pings_ms if len(chq_msg_pings_ms) <= 500 else chq_msg_pings if len(chq_msg_pings) <= 500 else chq_msg[0:500]
                    GlobalVars.charcoal_hq.send_message(msg_to_send)
                if not should_reasons_prevent_tavern_posting(reasons) and site not in GlobalVars.non_tavern_sites and time.time() >= GlobalVars.blockedTime[GlobalVars.meta_tavern_room_id]:
                    tavern_pings = datahandling.get_user_names_on_notification_list("meta.stackexchange.com", GlobalVars.meta_tavern_room_id, site, GlobalVars.wrapm)
                    tavern_msg = prefix + s
                    tavern_msg_pings = prefix + datahandling.append_pings(s, tavern_pings)
                    tavern_msg_pings_ms = prefix_ms + datahandling.append_pings(s, tavern_pings)
                    msg_to_send = tavern_msg_pings_ms if len(tavern_msg_pings_ms) <= 500 else tavern_msg_pings if len(tavern_msg_pings) <= 500 else tavern_msg[0:500]
                    t_check_websocket = Thread(target=deletionwatcher.DeletionWatcher.post_message_if_not_deleted, args=((post_id, site, "answer" if is_answer else "question"), post_url, msg_to_send, GlobalVars.tavern_on_the_meta))
                    t_check_websocket.daemon = True
                    t_check_websocket.start()
                if site == "stackoverflow.com" and reason not in GlobalVars.non_socvr_reasons and time.time() >= GlobalVars.blockedTime[GlobalVars.socvr_room_id]:
                    socvr_pings = datahandling.get_user_names_on_notification_list("stackoverflow.com", GlobalVars.socvr_room_id, site, GlobalVars.wrapso)
                    socvr_msg = prefix + s
                    socvr_msg_pings = prefix + datahandling.append_pings(s, socvr_pings)
                    socvr_msg_pings_ms = prefix_ms + datahandling.append_pings(s, socvr_pings)
                    msg_to_send = socvr_msg_pings_ms if len(socvr_msg_pings_ms) <= 500 else socvr_msg_pings if len(socvr_msg_pings) <= 500 else socvr_msg[0:500]
                    GlobalVars.socvr.send_message(msg_to_send)

            for specialroom in GlobalVars.specialrooms:
                sites = specialroom["sites"]
                if site in sites and reason not in specialroom["unwantedReasons"]:
                    room = specialroom["room"]
                    if room.id not in GlobalVars.blockedTime or time.time() >= GlobalVars.blockedTime[room.id]:
                        room_site = room._client.host
                        room_id = int(room.id)
                        room_pings = datahandling.get_user_names_on_notification_list(room_site, room_id, site, room._client)
                        room_msg = prefix + s
                        room_msg_pings = prefix + datahandling.append_pings(s, room_pings)
                        room_msg_pings_ms = prefix_ms + datahandling.append_pings(s, room_pings)
                        msg_to_send = room_msg_pings_ms if len(room_msg_pings_ms) <= 500 else room_msg_pings if len(room_msg_pings) <= 500 else room_msg[0:500]
                        specialroom["room"].send_message(msg_to_send)
    except:
        exc_type, exc_obj, exc_tb = sys.exc_info()
        excepthook.uncaught_exception(exc_type, exc_obj, exc_tb)

def handle_user_with_all_spam(user, why):
    user_id = user[0]
    site = user[1]
    tab = "activity" if site == "stackexchange.com" else "topactivity"
    s = "[ [SmokeDetector](//git.io/vgx7b) ] All of this user's posts are spam: [user {} on {}](//{}/users/{}?tab={})" \
        .format(user_id, site, site, user_id, tab)
    print GlobalVars.parser.unescape(s).encode('ascii', errors='replace')
    datahandling.add_why_allspam(user, why)
    if time.time() >= GlobalVars.blockedTime[GlobalVars.charcoal_room_id]:
        GlobalVars.charcoal_hq.send_message(s)
    for specialroom in GlobalVars.specialrooms:
        room = specialroom["room"]
        if site in specialroom["sites"] and (room.id not in GlobalVars.blockedTime or time.time() >= GlobalVars.blockedTime[room.id]):
            room.send_message(s)
<|endoftext|>


<commit_before># Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
GoGrid driver
"""
from libcloud.providers import Provider
from libcloud.types import NodeState, InvalidCredsException
from libcloud.base import Node, ConnectionUserAndKey, Response, NodeDriver
from libcloud.base import NodeSize, NodeImage, NodeLocation
import time
import hashlib

# JSON is included in the standard library starting with Python 2.6.  For 2.5
# and 2.4, there's a simplejson egg at: http://pypi.python.org/pypi/simplejson
try: import json
except: import simplejson as json

HOST = 'api.gogrid.com'
PORTS_BY_SECURITY = { True: 443, False: 80 }
API_VERSION = '1.3'

STATE = {
    "Starting": NodeState.PENDING,
    "On": NodeState.RUNNING,
    "Off": NodeState.PENDING,
    "Restarting": NodeState.REBOOTING,
    "Saving": NodeState.PENDING,
    "Restoring": NodeState.PENDING,
}

GOGRID_INSTANCE_TYPES = {'512MB': {'id': '512MB',
                       'name': '512MB',
                       'ram': 512,
                       'disk': 30,
                       'bandwidth': None,
                       'price':0.095},
        '1GB': {'id': '1GB',
                       'name': '1GB',
                       'ram': 1024,
                       'disk': 60,
                       'bandwidth': None,
                       'price':0.19},
        '2GB': {'id': '2GB',
                       'name': '2GB',
                       'ram': 2048,
                       'disk': 120,
                       'bandwidth': None,
                       'price':0.38},
        '4GB': {'id': '4GB',
                       'name': '4GB',
                       'ram': 4096,
                       'disk': 240,
                       'bandwidth': None,
                       'price':0.76},
        '8GB': {'id': '8GB',
                       'name': '8GB',
                       'ram': 8192,
                       'disk': 480,
                       'bandwidth': None,
                       'price':1.52}}

class GoGridResponse(Response):
    def success(self):
        if self.status == 403:
            raise InvalidCredsException()
        if not self.body:
            return None
        return json.loads(self.body)['status'] == 'success'

    def parse_body(self):
        if not self.body:
            return None
        return json.loads(self.body)

    def parse_error(self):
        if not self.object:
            return None
        return self.object['message']

class GoGridConnection(ConnectionUserAndKey):
    """
    Connection class for the GoGrid driver
    """

    host = HOST
    responseCls = GoGridResponse

    def add_default_params(self, params):
        params["api_key"] = self.user_id
        params["v"] = API_VERSION
        params["format"] = 'json'
        params["sig"] = self.get_signature(self.user_id, self.key)

        return params

    def get_signature(self, key, secret):
        """ create sig from md5 of key + secret + time """
        m = hashlib.md5(key+secret+str(int(time.time())))
        return m.hexdigest()

class GoGridNode(Node):
    # Generating uuid based on public ip to get around missing id on
    # create_node in gogrid api
    #
    # Used public ip since it is not mutable and specified at create time,
    # so uuid of node should not change after add is completed
    def get_uuid(self):
        return hashlib.sha1(
            "%s:%d" % (self.public_ip,self.driver.type)
        ).hexdigest()

class GoGridNodeDriver(NodeDriver):
    """
    GoGrid node driver
    """

    connectionCls = GoGridConnection
    type = Provider.GOGRID
    name = 'GoGrid API'

    _instance_types = GOGRID_INSTANCE_TYPES

    def _get_state(self, element):
        try:
            return STATE[element['state']['name']]
        except:
            pass
        return NodeState.UNKNOWN

    def _get_ip(self, element):
        return element.get('ip').get('ip')

    def _get_id(self, element):
        return element.get('id')

    def _to_node(self, element):
        state = self._get_state(element)
        ip = self._get_ip(element)
        id = self._get_id(element)
        n = GoGridNode(id=id,
                 name=element['name'],
                 state=state,
                 public_ip=[ip],
                 private_ip=[],
                 extra={'ram':element.get('ram').get('name')},
                 driver=self.connection.driver)
        return n

    def _to_image(self, element):
        n = NodeImage(id=element['id'],
                      name=element['friendlyName'],
                      driver=self.connection.driver)
        return n

    def _to_images(self, object):
        return [ self._to_image(el)
                 for el in object['list'] ]

    def list_images(self, location=None):
        images = self._to_images(
                    self.connection.request('/api/grid/image/list').object)
        return images

    def list_nodes(self):
        res = self._server_list()
        return [ self._to_node(el)
                 for el
                 in res['list'] ]

    def reboot_node(self, node):
        id = node.id
        power = 'restart'
        res = self._server_power(id, power)
        if not res.success():
            raise Exception(res.parse_error())
        return True

    def destroy_node(self, node):
        id = node.id
        res = self._server_delete(id)
        if not res.success():
            raise Exception(res.parse_error())
        return True

    def _server_list(self):
        return self.connection.request('/api/grid/server/list').object

    def _server_power(self, id, power):
        # power in ['start', 'stop', 'restart']
        params = {'id': id, 'power': power}
        return self.connection.request("/api/grid/server/power", params)

    def _server_delete(self, id):
        params = {'id': id}
        return self.connection.request("/api/grid/server/delete", params)

    def _get_first_ip(self):
        params = {'ip.state': 'Unassigned', 'ip.type': 'public'}
        object = self.connection.request("/api/grid/ip/list", params).object
        return object['list'][0]['ip']

    def list_sizes(self, location=None):
        return [ NodeSize(driver=self.connection.driver, **i)
                    for i in self._instance_types.values() ]

    def list_locations(self):
        return [NodeLocation(0, "GoGrid Los Angeles", 'US', self)]

    def ex_create_node_nowait(self, **kwargs):
        """Don't block until GoGrid allocates id for a node
        but return right away with id == None.

        The existance of this method is explained by the fact
        that GoGrid assigns id to a node only few minutes after
        creation."""
        name = kwargs['name']
        image = kwargs['image']
        size = kwargs['size']
        first_ip = self._get_first_ip()
        params = {'name': name,
                  'image': image.id,
                  'description': kwargs.get('ex_description', ''),
                  'server.ram': size.id,
                  'ip': first_ip}

        object = self.connection.request('/api/grid/server/add',
                                         params=params).object
        node = self._to_node(object['list'][0])

        return node

    def create_node(self, **kwargs):
        """Create a new GoGird node

        See L{NodeDriver.create_node} for more keyword args.

        @keyword    ex_description: Description of a Node
        @type       ex_description: C{string}
        """
        node = self.ex_create_node_nowait(**kwargs)

        timeout = 60 * 20
        waittime = 0
        interval = 2 * 60

        while node.id is None and waittime < timeout:
            nodes = self.list_nodes()

            for i in nodes:
                if i.public_ip[0] == node.public_ip[0] and i.id is not None:
                    node.id = i.id
                    return node

            waittime += interval
            time.sleep(interval)

        return node
<commit_msg>Raise an exception if after timeout id wasn't allocated.<commit_after># Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
GoGrid driver
"""
from libcloud.providers import Provider
from libcloud.types import NodeState, InvalidCredsException
from libcloud.base import Node, ConnectionUserAndKey, Response, NodeDriver
from libcloud.base import NodeSize, NodeImage, NodeLocation
import time
import hashlib

# JSON is included in the standard library starting with Python 2.6.  For 2.5
# and 2.4, there's a simplejson egg at: http://pypi.python.org/pypi/simplejson
try: import json
except: import simplejson as json

HOST = 'api.gogrid.com'
PORTS_BY_SECURITY = { True: 443, False: 80 }
API_VERSION = '1.3'

STATE = {
    "Starting": NodeState.PENDING,
    "On": NodeState.RUNNING,
    "Off": NodeState.PENDING,
    "Restarting": NodeState.REBOOTING,
    "Saving": NodeState.PENDING,
    "Restoring": NodeState.PENDING,
}

GOGRID_INSTANCE_TYPES = {'512MB': {'id': '512MB',
                       'name': '512MB',
                       'ram': 512,
                       'disk': 30,
                       'bandwidth': None,
                       'price':0.095},
        '1GB': {'id': '1GB',
                       'name': '1GB',
                       'ram': 1024,
                       'disk': 60,
                       'bandwidth': None,
                       'price':0.19},
        '2GB': {'id': '2GB',
                       'name': '2GB',
                       'ram': 2048,
                       'disk': 120,
                       'bandwidth': None,
                       'price':0.38},
        '4GB': {'id': '4GB',
                       'name': '4GB',
                       'ram': 4096,
                       'disk': 240,
                       'bandwidth': None,
                       'price':0.76},
        '8GB': {'id': '8GB',
                       'name': '8GB',
                       'ram': 8192,
                       'disk': 480,
                       'bandwidth': None,
                       'price':1.52}}

class GoGridResponse(Response):
    def success(self):
        if self.status == 403:
            raise InvalidCredsException()
        if not self.body:
            return None
        return json.loads(self.body)['status'] == 'success'

    def parse_body(self):
        if not self.body:
            return None
        return json.loads(self.body)

    def parse_error(self):
        if not self.object:
            return None
        return self.object['message']

class GoGridConnection(ConnectionUserAndKey):
    """
    Connection class for the GoGrid driver
    """

    host = HOST
    responseCls = GoGridResponse

    def add_default_params(self, params):
        params["api_key"] = self.user_id
        params["v"] = API_VERSION
        params["format"] = 'json'
        params["sig"] = self.get_signature(self.user_id, self.key)

        return params

    def get_signature(self, key, secret):
        """ create sig from md5 of key + secret + time """
        m = hashlib.md5(key+secret+str(int(time.time())))
        return m.hexdigest()

class GoGridNode(Node):
    # Generating uuid based on public ip to get around missing id on
    # create_node in gogrid api
    #
    # Used public ip since it is not mutable and specified at create time,
    # so uuid of node should not change after add is completed
    def get_uuid(self):
        return hashlib.sha1(
            "%s:%d" % (self.public_ip,self.driver.type)
        ).hexdigest()

class GoGridNodeDriver(NodeDriver):
    """
    GoGrid node driver
    """

    connectionCls = GoGridConnection
    type = Provider.GOGRID
    name = 'GoGrid API'

    _instance_types = GOGRID_INSTANCE_TYPES

    def _get_state(self, element):
        try:
            return STATE[element['state']['name']]
        except:
            pass
        return NodeState.UNKNOWN

    def _get_ip(self, element):
        return element.get('ip').get('ip')

    def _get_id(self, element):
        return element.get('id')

    def _to_node(self, element):
        state = self._get_state(element)
        ip = self._get_ip(element)
        id = self._get_id(element)
        n = GoGridNode(id=id,
                 name=element['name'],
                 state=state,
                 public_ip=[ip],
                 private_ip=[],
                 extra={'ram':element.get('ram').get('name')},
                 driver=self.connection.driver)
        return n

    def _to_image(self, element):
        n = NodeImage(id=element['id'],
                      name=element['friendlyName'],
                      driver=self.connection.driver)
        return n

    def _to_images(self, object):
        return [ self._to_image(el)
                 for el in object['list'] ]

    def list_images(self, location=None):
        images = self._to_images(
                    self.connection.request('/api/grid/image/list').object)
        return images

    def list_nodes(self):
        res = self._server_list()
        return [ self._to_node(el)
                 for el
                 in res['list'] ]

    def reboot_node(self, node):
        id = node.id
        power = 'restart'
        res = self._server_power(id, power)
        if not res.success():
            raise Exception(res.parse_error())
        return True

    def destroy_node(self, node):
        id = node.id
        res = self._server_delete(id)
        if not res.success():
            raise Exception(res.parse_error())
        return True

    def _server_list(self):
        return self.connection.request('/api/grid/server/list').object

    def _server_power(self, id, power):
        # power in ['start', 'stop', 'restart']
        params = {'id': id, 'power': power}
        return self.connection.request("/api/grid/server/power", params)

    def _server_delete(self, id):
        params = {'id': id}
        return self.connection.request("/api/grid/server/delete", params)

    def _get_first_ip(self):
        params = {'ip.state': 'Unassigned', 'ip.type': 'public'}
        object = self.connection.request("/api/grid/ip/list", params).object
        return object['list'][0]['ip']

    def list_sizes(self, location=None):
        return [ NodeSize(driver=self.connection.driver, **i)
                    for i in self._instance_types.values() ]

    def list_locations(self):
        return [NodeLocation(0, "GoGrid Los Angeles", 'US', self)]

    def ex_create_node_nowait(self, **kwargs):
        """Don't block until GoGrid allocates id for a node
        but return right away with id == None.

        The existance of this method is explained by the fact
        that GoGrid assigns id to a node only few minutes after
        creation."""
        name = kwargs['name']
        image = kwargs['image']
        size = kwargs['size']
        first_ip = self._get_first_ip()
        params = {'name': name,
                  'image': image.id,
                  'description': kwargs.get('ex_description', ''),
                  'server.ram': size.id,
                  'ip': first_ip}

        object = self.connection.request('/api/grid/server/add',
                                         params=params).object
        node = self._to_node(object['list'][0])

        return node

    def create_node(self, **kwargs):
        """Create a new GoGird node

        See L{NodeDriver.create_node} for more keyword args.

        @keyword    ex_description: Description of a Node
        @type       ex_description: C{string}
        """
        node = self.ex_create_node_nowait(**kwargs)

        timeout = 60 * 20
        waittime = 0
        interval = 2 * 60

        while node.id is None and waittime < timeout:
            nodes = self.list_nodes()

            for i in nodes:
                if i.public_ip[0] == node.public_ip[0] and i.id is not None:
                    node.id = i.id
                    return node

            waittime += interval
            time.sleep(interval)

        if id is None:
            raise Exception("Wasn't able to wait for id allocation for the node %s" % str(node))

        return node
<|endoftext|>


<commit_before># coding: utf-8
from django.conf import settings
from django.contrib.auth.models import AnonymousUser
from django.core.exceptions import FieldError
from django.db.models import Case, Count, IntegerField, Value, When
from django.db.models.query import QuerySet
from rest_framework import filters
from rest_framework.request import Request

from kpi.constants import (
    ASSET_SEARCH_DEFAULT_FIELD_LOOKUPS,
    ASSET_STATUS_SHARED,
    ASSET_STATUS_DISCOVERABLE,
    ASSET_STATUS_PRIVATE,
    ASSET_STATUS_PUBLIC,
    PERM_DISCOVER_ASSET,
    PERM_VIEW_ASSET,
)
from kpi.exceptions import SearchQueryTooShortException
from kpi.models.asset import UserAssetSubscription
from kpi.utils.query_parser import parse, ParseError
from .models import Asset, ObjectPermission
from .models.object_permission import (
    get_objects_for_user,
    get_anonymous_user,
    get_perm_ids_from_code_names
)

class AssetOwnerFilterBackend(filters.BaseFilterBackend):
    """
    For use with nested models of Asset.
    Restricts access to items that are owned by the current user
    """

    def filter_queryset(self, request, queryset, view):
        fields = {"asset__owner": request.user}
        return queryset.filter(**fields)

class AssetOrderingFilter(filters.OrderingFilter):

    def filter_queryset(self, request, queryset, view):
        query_params = request.query_params
        collections_first = query_params.get('collections_first',
                                             'false').lower() == 'true'
        ordering = self.get_ordering(request, queryset, view)

        if collections_first:
            # If `collections_first` is `True`, we want the collections to be
            # displayed at the beginning. We add a temporary field to set a
            # priority to 1 for collections and all other asset types to 0.
            # The results are sorted by this priority first and then by what
            # comes next, either:
            # - `ordering` from querystring
            # - model default option
            queryset = queryset.annotate(
                ordering_priority=Case(
                    When(asset_type='collection', then=Value(1)),
                    default=Value(0),
                    output_field=IntegerField(),
                ),
            )
            if ordering is None:
                # Make a copy, we don't want to alter Asset._meta.ordering
                ordering = Asset._meta.ordering.copy()
            ordering.insert(0, '-ordering_priority')

        if ordering:
            if 'subscribers_count' in ordering or \
                    '-subscribers_count' in ordering:
                queryset = queryset.annotate(subscribers_count=
                                             Count('userassetsubscription__user'))
            return queryset.order_by(*ordering)

        return queryset

class KpiObjectPermissionsFilter:

    def filter_queryset(self, request, queryset, view):

        user = request.user
        if user.is_superuser and view.action != 'list':
            # For a list, we won't deluge the superuser with everyone else's
            # stuff. This isn't a list, though, so return it all
            return queryset

        queryset = self._get_queryset_for_collection_statuses(request, queryset)
        if self._return_queryset:
            return queryset.distinct()

        # Getting the children of discoverable public collections
        queryset = self._get_queryset_for_discoverable_child_assets(
            request, queryset
        )
        if self._return_queryset:
            return queryset.distinct()

        owned_and_explicit_shared = self._get_owned_and_explicitly_shared(user)

        if view.action != 'list':
            # Not a list, so discoverability doesn't matter
            assets = owned_and_explicit_shared.union(self._get_publics())
            return queryset.filter(pk__in=assets)

        subscribed = self._get_subscribed(user)

        # As other places in the code, coerce `asset_ids` as a list to force
        # the query to be processed right now. Otherwise, because queryset is
        # a lazy query, Django creates (left) joins on tables when queryset is
        # interpreted and it is way slower than running this extra query.
        asset_ids = list(
            (
                owned_and_explicit_shared
                    .union(subscribed)
                    # Since user would be subscribed to a collection and not
                    # the assets themselves, we append children of subscribed
                    # collections to the queryset in order for `?q=parent__uid`
                    # queries to return the collection's children
                    .union(queryset.filter(parent__in=subscribed).values("pk")
                )
            ).values_list("id", flat=True)
        )
        return queryset.filter(pk__in=asset_ids)

    def _get_discoverable(self, queryset):
        # We were asked not to consider subscriptions; return all
        # discoverable objects
        return get_objects_for_user(
            get_anonymous_user(), PERM_DISCOVER_ASSET,
            queryset
        )

    def _get_queryset_for_collection_statuses(self, request, queryset):
        """
        Narrow down the queryset based on `status` parameter.
        It is useful when fetching Assets of type `collection`

        If `status` is not detected in `q`, it returns the queryset as is.
        Otherwise, there are 4 scenarios:
        - `status` == 'private': collections owned by user
        - `status` == 'shared': collections user can view
        - `status` == 'public': collections user has subscribed to (ONLY)
        - `status` == 'public-discoverable': all public collections

        Args:
            request
            queryset
        Returns:
            QuerySet
        """

        # Governs whether returned queryset should be processed immediately
        # and should stop other filtering on `queryset` in parent method of
        # `_get_queryset_for_collection_statuses()`
        self._return_queryset = False
        user = request.user
        STATUS_PARAMETER = 'status'

        try:
            status = request.query_params[STATUS_PARAMETER].strip()
        except KeyError:
            return queryset

        if status == ASSET_STATUS_PRIVATE:
            self._return_queryset = True
            return queryset.filter(owner=request.user)

        elif status == ASSET_STATUS_SHARED:
            self._return_queryset = True
            return get_objects_for_user(user, PERM_VIEW_ASSET, queryset)

        elif status == ASSET_STATUS_PUBLIC:
            self._return_queryset = True
            return queryset.filter(pk__in=self._get_subscribed(user))

        elif status == ASSET_STATUS_DISCOVERABLE:
            self._return_queryset = True
            discoverable = self._get_discoverable(queryset)
            # We were asked not to consider subscriptions; return all
            # discoverable objects
            return discoverable

        return queryset

    def _get_queryset_for_discoverable_child_assets(
        self, request: Request, queryset: QuerySet
    ) -> QuerySet:
        """
        Returns a queryset containing the children of publically discoverable
        assets based on the discoverability of the child's parent. The parent
        uid is passed in the request query string.

        args:
            request (Request)
            queryset (QuerySet)

        returns:
            QuerySet
        """

        self._return_queryset = False
        PARENT_UID_PARAMETER = 'parent__uid'

        if 'q' not in request.query_params:
            return queryset

        request_query = request.query_params['q']
        parent_uid = (
            request_query.split(PARENT_UID_PARAMETER)[1].strip(':')
            if PARENT_UID_PARAMETER in request_query
            else None
        )

        if parent_uid is None:
            return queryset

        try:
            parent_obj = queryset.get(uid=parent_uid)
        except Exception as e:
            return queryset

        if not isinstance(parent_obj, Asset):
            return queryset

        if parent_obj.has_perm(get_anonymous_user(), PERM_DISCOVER_ASSET):
            self._return_queryset = True
            return queryset.filter(pk__in=self._get_publics())

        return queryset

    @staticmethod
    def _get_owned_and_explicitly_shared(user):
        view_asset_perm_id = get_perm_ids_from_code_names(PERM_VIEW_ASSET)
        if user.is_anonymous:
            # Avoid giving anonymous users special treatment when viewing
            # public objects
            perms = ObjectPermission.objects.none()
        else:
            perms = ObjectPermission.objects.filter(
                deny=False,
                user=user,
                permission_id=view_asset_perm_id)

        return perms.values('asset')

    @staticmethod
    def _get_publics():
        view_asset_perm_id = get_perm_ids_from_code_names(PERM_VIEW_ASSET)
        return ObjectPermission.objects.filter(
            deny=False,
            user=get_anonymous_user(),
            permission_id=view_asset_perm_id).values('asset')

    @classmethod
    def _get_subscribed(cls, user):
        # Of the public objects, determine to which the user has subscribed
        if user.is_anonymous:
            user = get_anonymous_user()

        return UserAssetSubscription.objects.filter(asset__in=cls._get_publics(),
                                                    user=user).values('asset')

class RelatedAssetPermissionsFilter(KpiObjectPermissionsFilter):
    """
    Uses KpiObjectPermissionsFilter to determine which assets the user
    may access, and then filters the provided queryset to include only objects
    related to those assets. The queryset's model must be related to `Asset`
    via a field named `asset`.
    """

    def filter_queryset(self, request, queryset, view):
        available_assets = super().filter_queryset(
            request=request,
            queryset=Asset.objects.all(),
            view=view
        )
        return queryset.filter(asset__in=available_assets)

class SearchFilter(filters.BaseFilterBackend):
    """
    If the request includes a `q` parameter specifying a Boolean search string
    with a Whoosh-like syntax, filter the queryset accordingly using the ORM.
    If no `q` is present, return the queryset untouched. If `q` is not
    parseable, references a field that does not exist, or specifies an invalid
    value for a field (e.g. text for an integer field), return an empty
    queryset to make the problem obvious.
    """

    def filter_queryset(self, request, queryset, view):
        try:
            q = request.query_params['q']
        except KeyError:
            return queryset

        try:
            q_obj = parse(
                q, default_field_lookups=ASSET_SEARCH_DEFAULT_FIELD_LOOKUPS
            )
        except ParseError:
            return queryset.model.objects.none()
        except SearchQueryTooShortException as e:
            # raising an exception if the default search query without a
            # specified field is less than a set length of characters -
            # currently 3
            raise e

        try:
            # If no search field is specified, the search term is compared
            # to several default fields and therefore may return a copies
            # of the same match, therefore the `distinct()` method is required
            return queryset.filter(q_obj).distinct()
        except (FieldError, ValueError):
            return queryset.model.objects.none()

class KpiAssignedObjectPermissionsFilter(filters.BaseFilterBackend):
    def filter_queryset(self, request, queryset, view):
        # TODO: omit objects for which the user has only a deny permission
        user = request.user
        if isinstance(request.user, AnonymousUser):
            user = get_anonymous_user()
        if user.is_superuser:
            # Superuser sees all
            return queryset
        if user.pk == settings.ANONYMOUS_USER_ID:
            # Hide permissions from anonymous users
            return queryset.none()
        """
        A regular user sees permissions for objects to which they have access.
        For example, if Alana has view access to an object owned by Richard,
        she should see all permissions for that object, including those
        assigned to other users.
        """
        result = ObjectPermission.objects.filter(
            asset__permissions__user=user
        ).distinct()
        return result
<commit_msg>Use regex to access `parent__uid`<commit_after># coding: utf-8
import re

from django.conf import settings
from django.contrib.auth.models import AnonymousUser
from django.core.exceptions import FieldError
from django.db.models import Case, Count, IntegerField, Value, When
from django.db.models.query import QuerySet
from rest_framework import filters
from rest_framework.request import Request

from kpi.constants import (
    ASSET_SEARCH_DEFAULT_FIELD_LOOKUPS,
    ASSET_STATUS_SHARED,
    ASSET_STATUS_DISCOVERABLE,
    ASSET_STATUS_PRIVATE,
    ASSET_STATUS_PUBLIC,
    PERM_DISCOVER_ASSET,
    PERM_VIEW_ASSET,
)
from kpi.exceptions import SearchQueryTooShortException
from kpi.models.asset import UserAssetSubscription
from kpi.utils.query_parser import parse, ParseError
from .models import Asset, ObjectPermission
from .models.object_permission import (
    get_objects_for_user,
    get_anonymous_user,
    get_perm_ids_from_code_names
)

class AssetOwnerFilterBackend(filters.BaseFilterBackend):
    """
    For use with nested models of Asset.
    Restricts access to items that are owned by the current user
    """

    def filter_queryset(self, request, queryset, view):
        fields = {"asset__owner": request.user}
        return queryset.filter(**fields)

class AssetOrderingFilter(filters.OrderingFilter):

    def filter_queryset(self, request, queryset, view):
        query_params = request.query_params
        collections_first = query_params.get('collections_first',
                                             'false').lower() == 'true'
        ordering = self.get_ordering(request, queryset, view)

        if collections_first:
            # If `collections_first` is `True`, we want the collections to be
            # displayed at the beginning. We add a temporary field to set a
            # priority to 1 for collections and all other asset types to 0.
            # The results are sorted by this priority first and then by what
            # comes next, either:
            # - `ordering` from querystring
            # - model default option
            queryset = queryset.annotate(
                ordering_priority=Case(
                    When(asset_type='collection', then=Value(1)),
                    default=Value(0),
                    output_field=IntegerField(),
                ),
            )
            if ordering is None:
                # Make a copy, we don't want to alter Asset._meta.ordering
                ordering = Asset._meta.ordering.copy()
            ordering.insert(0, '-ordering_priority')

        if ordering:
            if 'subscribers_count' in ordering or \
                    '-subscribers_count' in ordering:
                queryset = queryset.annotate(subscribers_count=
                                             Count('userassetsubscription__user'))
            return queryset.order_by(*ordering)

        return queryset

class KpiObjectPermissionsFilter:

    def filter_queryset(self, request, queryset, view):

        user = request.user
        if user.is_superuser and view.action != 'list':
            # For a list, we won't deluge the superuser with everyone else's
            # stuff. This isn't a list, though, so return it all
            return queryset

        queryset = self._get_queryset_for_collection_statuses(request, queryset)
        if self._return_queryset:
            return queryset.distinct()

        # Getting the children of discoverable public collections
        queryset = self._get_queryset_for_discoverable_child_assets(
            request, queryset
        )
        if self._return_queryset:
            return queryset.distinct()

        owned_and_explicit_shared = self._get_owned_and_explicitly_shared(user)

        if view.action != 'list':
            # Not a list, so discoverability doesn't matter
            assets = owned_and_explicit_shared.union(self._get_publics())
            return queryset.filter(pk__in=assets)

        subscribed = self._get_subscribed(user)

        # As other places in the code, coerce `asset_ids` as a list to force
        # the query to be processed right now. Otherwise, because queryset is
        # a lazy query, Django creates (left) joins on tables when queryset is
        # interpreted and it is way slower than running this extra query.
        asset_ids = list(
            (
                owned_and_explicit_shared
                    .union(subscribed)
                    # Since user would be subscribed to a collection and not
                    # the assets themselves, we append children of subscribed
                    # collections to the queryset in order for `?q=parent__uid`
                    # queries to return the collection's children
                    .union(queryset.filter(parent__in=subscribed).values("pk")
                )
            ).values_list("id", flat=True)
        )
        return queryset.filter(pk__in=asset_ids)

    def _get_discoverable(self, queryset):
        # We were asked not to consider subscriptions; return all
        # discoverable objects
        return get_objects_for_user(
            get_anonymous_user(), PERM_DISCOVER_ASSET,
            queryset
        )

    def _get_queryset_for_collection_statuses(self, request, queryset):
        """
        Narrow down the queryset based on `status` parameter.
        It is useful when fetching Assets of type `collection`

        If `status` is not detected in `q`, it returns the queryset as is.
        Otherwise, there are 4 scenarios:
        - `status` == 'private': collections owned by user
        - `status` == 'shared': collections user can view
        - `status` == 'public': collections user has subscribed to (ONLY)
        - `status` == 'public-discoverable': all public collections

        Args:
            request
            queryset
        Returns:
            QuerySet
        """

        # Governs whether returned queryset should be processed immediately
        # and should stop other filtering on `queryset` in parent method of
        # `_get_queryset_for_collection_statuses()`
        self._return_queryset = False
        user = request.user
        STATUS_PARAMETER = 'status'

        try:
            status = request.query_params[STATUS_PARAMETER].strip()
        except KeyError:
            return queryset

        if status == ASSET_STATUS_PRIVATE:
            self._return_queryset = True
            return queryset.filter(owner=request.user)

        elif status == ASSET_STATUS_SHARED:
            self._return_queryset = True
            return get_objects_for_user(user, PERM_VIEW_ASSET, queryset)

        elif status == ASSET_STATUS_PUBLIC:
            self._return_queryset = True
            return queryset.filter(pk__in=self._get_subscribed(user))

        elif status == ASSET_STATUS_DISCOVERABLE:
            self._return_queryset = True
            discoverable = self._get_discoverable(queryset)
            # We were asked not to consider subscriptions; return all
            # discoverable objects
            return discoverable

        return queryset

    def _get_queryset_for_discoverable_child_assets(
        self, request: Request, queryset: QuerySet
    ) -> QuerySet:
        """
        Returns a queryset containing the children of publically discoverable
        assets based on the discoverability of the child's parent. The parent
        uid is passed in the request query string.

        args:
            request (Request)
            queryset (QuerySet)

        returns:
            QuerySet
        """

        self._return_queryset = False
        PARENT_UID_PARAMETER = 'parent__uid'

        if 'q' not in request.query_params:
            return queryset

        request_query = request.query_params['q']
        parent_uid = re.search(
            f'{PARENT_UID_PARAMETER}:([a-zA-Z0-9_.-]*)', request_query
        )

        if parent_uid is None:
            return queryset

        try:
            parent_obj = queryset.get(uid=parent_uid.group(1))
        except Exception as e:
            return queryset

        if not isinstance(parent_obj, Asset):
            return queryset

        if parent_obj.has_perm(get_anonymous_user(), PERM_DISCOVER_ASSET):
            self._return_queryset = True
            return queryset.filter(pk__in=self._get_publics())

        return queryset

    @staticmethod
    def _get_owned_and_explicitly_shared(user):
        view_asset_perm_id = get_perm_ids_from_code_names(PERM_VIEW_ASSET)
        if user.is_anonymous:
            # Avoid giving anonymous users special treatment when viewing
            # public objects
            perms = ObjectPermission.objects.none()
        else:
            perms = ObjectPermission.objects.filter(
                deny=False,
                user=user,
                permission_id=view_asset_perm_id)

        return perms.values('asset')

    @staticmethod
    def _get_publics():
        view_asset_perm_id = get_perm_ids_from_code_names(PERM_VIEW_ASSET)
        return ObjectPermission.objects.filter(
            deny=False,
            user=get_anonymous_user(),
            permission_id=view_asset_perm_id).values('asset')

    @classmethod
    def _get_subscribed(cls, user):
        # Of the public objects, determine to which the user has subscribed
        if user.is_anonymous:
            user = get_anonymous_user()

        return UserAssetSubscription.objects.filter(asset__in=cls._get_publics(),
                                                    user=user).values('asset')

class RelatedAssetPermissionsFilter(KpiObjectPermissionsFilter):
    """
    Uses KpiObjectPermissionsFilter to determine which assets the user
    may access, and then filters the provided queryset to include only objects
    related to those assets. The queryset's model must be related to `Asset`
    via a field named `asset`.
    """

    def filter_queryset(self, request, queryset, view):
        available_assets = super().filter_queryset(
            request=request,
            queryset=Asset.objects.all(),
            view=view
        )
        return queryset.filter(asset__in=available_assets)

class SearchFilter(filters.BaseFilterBackend):
    """
    If the request includes a `q` parameter specifying a Boolean search string
    with a Whoosh-like syntax, filter the queryset accordingly using the ORM.
    If no `q` is present, return the queryset untouched. If `q` is not
    parseable, references a field that does not exist, or specifies an invalid
    value for a field (e.g. text for an integer field), return an empty
    queryset to make the problem obvious.
    """

    def filter_queryset(self, request, queryset, view):
        try:
            q = request.query_params['q']
        except KeyError:
            return queryset

        try:
            q_obj = parse(
                q, default_field_lookups=ASSET_SEARCH_DEFAULT_FIELD_LOOKUPS
            )
        except ParseError:
            return queryset.model.objects.none()
        except SearchQueryTooShortException as e:
            # raising an exception if the default search query without a
            # specified field is less than a set length of characters -
            # currently 3
            raise e

        try:
            # If no search field is specified, the search term is compared
            # to several default fields and therefore may return a copies
            # of the same match, therefore the `distinct()` method is required
            return queryset.filter(q_obj).distinct()
        except (FieldError, ValueError):
            return queryset.model.objects.none()

class KpiAssignedObjectPermissionsFilter(filters.BaseFilterBackend):
    def filter_queryset(self, request, queryset, view):
        # TODO: omit objects for which the user has only a deny permission
        user = request.user
        if isinstance(request.user, AnonymousUser):
            user = get_anonymous_user()
        if user.is_superuser:
            # Superuser sees all
            return queryset
        if user.pk == settings.ANONYMOUS_USER_ID:
            # Hide permissions from anonymous users
            return queryset.none()
        """
        A regular user sees permissions for objects to which they have access.
        For example, if Alana has view access to an object owned by Richard,
        she should see all permissions for that object, including those
        assigned to other users.
        """
        result = ObjectPermission.objects.filter(
            asset__permissions__user=user
        ).distinct()
        return result
<|endoftext|>