<commit_before>import unittest, time, sys, random
sys.path.extend(['.','..','py'])
import h2o, h2o_cmd, h2o_hosts
import h2o_browse as h2b

# ord('a') gives 97. Use that when you pass it as url param to h2o
# str(unichr(97)) gives 'a'

print "Data rows in header_from_file are ignored unless that file was part of the parse pattern"
print "Tab in header is not auto-detected separator. comma and space are"

print "Maybe a rows count problem if using tabs? temporary only tabs"
print "passes if only , is used"
print "just do one file with tab"
paramsDict = {
    # don't worry about these variants in this test (just parse normal)
    # 'parser_type': [None, 'AUTO', 'XLS', 'XLSX', 'SVMLight'],
    # I suppose, libsvm could have strangeness!..but there is no header with libsvm?
    'parser_type': [None, 'AUTO'],
    # gets used for separator=
    # 'separator': [",", " ", "\t"],
    # 'separator': ["\t", "|", ",", " "],
    'separator': ["\t", ",", " "],
    # 'separator': ["\t"],
    'header': [None, 0,1],
    # we can point to the 'wrong' file!
    # assume this is always used, otherwise we sum data rows without knowing if we'll use the header file?
    # always point to the header file..again, if we switch it around, the counts are off
    'header_from_file': ['syn_header'],
    # 'header_from_file': [None],
}

# ability to selectively comment the first line (which may be data or header)
# Does not write an extra line as the comment first in that case (header can be picked from the comment line)
# H2O always just strips the comment from the first line? and continues parsing?

# Don't write headerString if None (for non-header files)
# Don't write data if rowCount is None
def write_syn_dataset(csvPathname, rowCount, headerString, rList, commentFirst=False, sepChar=','):
    commentDone = False
    dsf = open(csvPathname, "w+")

    # this should never add to the actual count of data rows
    # UPDATE: comments are always ignored. So a commented header, commented data, is ignored.
    headerRowsDone = 0
    if headerString is not None:
        if commentFirst and not commentDone:
            h = "# " + headerString
            commentDone = True
        # FIX: for now, always put the header in even if we're told to!
        h = headerString
        headerRowsDone += 1
        dsf.write(h + "\n")

    if rowCount is not None:
        for i in range(rowCount):
            # two choices on the input. Make output choices random
            r = rList[random.randint(0,1)] + sepChar + str(random.randint(0,7))
            if commentFirst and not commentDone:
                # if commented, then write it twice, so the count of rows done is what we expect
                # the commented row will be ignored
                dsf.write("# " + r + "\n")
                commentDone = True
            dsf.write(r + "\n")
        dsf.close()
        return (headerRowsDone, rowCount) # rows done
    else:
        return (headerRowsDone, 0) # rows done

def rand_rowData(colCount, sepChar):
    rowData = [random.randint(0,7) for i in range(colCount)]
    rowData1= sepChar.join(map(str,rowData))
    rowData = [random.randint(0,7) for i in range(colCount)]
    rowData2= sepChar.join(map(str,rowData))
    # RF will complain if all inputs are the same
    r = [rowData1, rowData2]
    return r

class Basic(unittest.TestCase):
    def tearDown(self):
        h2o.check_sandbox_for_errors()

    @classmethod
    def setUpClass(cls):
        global SEED, localhost
        SEED = h2o.setup_random_seed()
        # SEED = h2o.setup_random_seed(8968685305521902318)

        localhost = h2o.decide_if_localhost()
        if (localhost):
            h2o.build_cloud(2,java_heap_MB=1300,use_flatfile=True)
        else:
            h2o_hosts.build_cloud_with_hosts()

    @classmethod
    def tearDownClass(cls):
        h2o.tear_down_cloud()

    def test_parse_multi_header_rand(self):
        h2b.browseTheCloud()
        SYNDATASETS_DIR = h2o.make_syn_dir()
        csvFilename = "syn_ints.csv"
        csvPathname = SYNDATASETS_DIR + '/' + csvFilename

        headerList = [
            # consistent with A, so we can decide when A should have been used (see below)
            ['aA','aB','aC','aD','aE','aF','aG','aH','aI','output'],
            # FIX! don't mess with the header size to data size now (stack trace
            # ['A','B2','C2','D2','E2','F2','G2','H2','I2','output2'],
            # ['A','B','C','D','E','F','G','H','I'],
            # ['A','B','C','D','E','F','G','H','I','output','junk'],
        ]

        # cols must be 9 to match the header above, otherwise a different bug is hit
        # extra output is added, so it's 10 total
        tryList = [
            # FIX! one fails count for now
            # (1, 5, 9, 'cA', 60, 0),
            (1, 5, 9, 'cA', 60, 0),

            # try with col mismatch on header.
            # FIX! causes exception? don't test for now
            # (7, 300, 10, 'cA', 60, 0),
            # (7, 300, 10, 'cB', 60, 1),
            # (7, 300, 10, 'cC', 60, 2),
            # (7, 300, 10, 'cD', 60, 3),

            # (7, 300, 8, 'cA', 60, 0),
            # (7, 300, 8, 'cB', 60, 1),
            # (7, 300, 8, 'cC', 60, 2),
            # (7, 300, 8, 'cD', 60, 3),
            ]

        # so many random combos..rather than walk tryList, just do random for some amount of time
        for trial in range(20):
            (fileNum, rowCount, colCount, key2, timeoutSecs, dataRowsWithHeader) = random.choice(tryList)
            print fileNum, rowCount, colCount, key2, timeoutSecs, dataRowsWithHeader
            # FIX! should we add a header to them randomly???
            print "Wait while", fileNum, "synthetic files are created in", SYNDATASETS_DIR
            rowxcol = str(rowCount) + 'x' + str(colCount)
            totalCols = colCount + 1 # 1 extra for output
            totalDataRows = 0
            totalHeaderRows = 0

            HEADER_HAS_HDR_ROW = random.randint(0,1)
            DATA_HAS_HDR_ROW = random.randint(0,1)
            PARSE_PATTERN_INCLUDES_HEADER = random.randint(0,1)
            ## DATA_FIRST_IS_COMMENT = random.randint(0,1)
            ## HEADER_FIRST_IS_COMMENT = random.randint(0,1)
            print "TEMPORARY: don't put any comments in"
            DATA_FIRST_IS_COMMENT = 0
            HEADER_FIRST_IS_COMMENT = 0
            # none is not legal
            SEP_CHAR_GEN = random.choice(paramsDict['separator'])

            print '\nHEADER_HAS_HDR_ROW:', HEADER_HAS_HDR_ROW
            print 'DATA_HAS_HDR_ROW:', DATA_HAS_HDR_ROW
            print 'PARSE_PATTERN_INCLUDES_HEADER', PARSE_PATTERN_INCLUDES_HEADER
            print 'DATA_FIRST_IS_COMMENT:', DATA_FIRST_IS_COMMENT
            print 'HEADER_FIRST_IS_COMMENT:', HEADER_FIRST_IS_COMMENT
            print 'SEP_CHAR_GEN:', SEP_CHAR_GEN

            # they need to both use the same separator (h2o rule)
            headerForHeader = SEP_CHAR_GEN.join(random.choice(headerList))
            headerForData = SEP_CHAR_GEN.join(random.choice(headerList))

            # random selection of parse param choices
            kwargs = {}
            for k,v in paramsDict.items():
                aChoice = random.choice(v)
                # can tell h2o something different compared to what we actually used!
                if k == 'separator':
                    if aChoice:
                        sepChar = aChoice
                        sepCharInt = ord(aChoice) # make it an integer for h2o
                    else:
                        sepChar = ',' # default char for None, need it for header/data file creation
                        sepCharInt = None
                    aChoice = sepCharInt

                kwargs[k] = aChoice

            # FOR NOW: ..override the rand choice if it exists, so we can parse and expect 'A' to be found
            # match what was gen'ed if choice is not None
            if kwargs['separator']:
                if SEP_CHAR_GEN==" "  or SEP_CHAR_GEN==",": # parse doesn't auto-detect tab. will autodetect space and comma
                    del kwargs['separator']
                else:
                    kwargs['separator'] = ord(SEP_CHAR_GEN)

            # create data files
            for fileN in range(fileNum):
                csvFilename = 'syn_data_' + str(fileN) + "_" + str(SEED) + "_" + str(trial) + "_" + rowxcol + '.csv'
                csvPathname = SYNDATASETS_DIR + '/' + csvFilename
                rList = rand_rowData(colCount, sepChar=SEP_CHAR_GEN)
                (headerRowsDone, dataRowsDone) = write_syn_dataset(csvPathname, rowCount,
                    headerString=(headerForData if DATA_HAS_HDR_ROW else None), rList=rList,
                    commentFirst=DATA_FIRST_IS_COMMENT, sepChar=SEP_CHAR_GEN)
                totalDataRows += dataRowsDone
                totalHeaderRows += headerRowsDone

            # create the header file
            hdrFilename = 'syn_header_' + str(SEED) + "_" + str(trial) + "_" + rowxcol + '.csv'
            hdrPathname = SYNDATASETS_DIR + '/' + hdrFilename
            # dataRowsWithHeader = 0 # temp hack
            (headerRowsDone, dataRowsDone) = write_syn_dataset(hdrPathname, dataRowsWithHeader,
                headerString=(headerForHeader if HEADER_HAS_HDR_ROW else None), rList=rList,
                commentFirst=HEADER_FIRST_IS_COMMENT, sepChar=SEP_CHAR_GEN)
            if PARSE_PATTERN_INCLUDES_HEADER: # only include header file data rows if the parse pattern includes it
                totalDataRows += dataRowsDone
            totalHeaderRows += headerRowsDone

            # make sure all key names are unique, when we re-put and re-parse (h2o caching issues)
            key = "syn_" + str(trial)
            key2 = "syn_" + str(trial) + ".hex"

            # DON"T get redirected to S3! (EC2 hack in config, remember!)
            # use it at the node level directly (because we gen'ed the files.
            # I suppose we could force the redirect state bits in h2o.nodes[0] to False, instead?:w
            xs = h2o.nodes[0].import_files(SYNDATASETS_DIR)['keys']
            headerKey = [x for x in xs if hdrFilename in x][0]
            dataKey = [x for x in xs if csvFilename not in x][0]

            # use regex. the only files in the dir will be the ones we just created with  *fileN* match
            print "Header Key =", headerKey

            # put the right name in
            if kwargs['header_from_file'] == 'syn_header':
                kwargs['header_from_file'] = headerKey
            # use one of the data files?
            elif kwargs['header_from_file'] == 'syn_data':
                kwargs['header_from_file'] = dataKey

            print "If header_from_file= is used, we are currently required to force header=1 for h2o"
            if kwargs['header_from_file']:
                kwargs['header'] =  1
            # if we have a header in a data file, tell h2o (for now)
            elif DATA_HAS_HDR_ROW:
                kwargs['header'] =  1
            else:
                kwargs['header'] =  0

            # may have error if h2o doesn't get anything!
            start = time.time()
            if PARSE_PATTERN_INCLUDES_HEADER:
                pattern = '*syn_*'+str(trial)+"_"+rowxcol+'*'
            else:
                pattern = '*syn_data_*'+str(trial)+"_"+rowxcol+'*'
            parseKey = h2o.nodes[0].parse(pattern, key2=key2, timeoutSecs=timeoutSecs, **kwargs)

            print "parseKey['destination_key']: " + parseKey['destination_key']
            print 'parse time:', parseKey['response']['time']

            inspect = h2o_cmd.runInspect(None, parseKey['destination_key'])
            h2o_cmd.infoFromInspect(inspect, csvPathname)
            print "\n" + csvPathname, \
                "    num_rows:", "{:,}".format(inspect['num_rows']), \
                "    num_cols:", "{:,}".format(inspect['num_cols'])

            # more reporting: (we can error here if extra col in header, causes all NA for missing col of data)
            h2o_cmd.columnInfoFromInspect(parseKey['destination_key'], exceptionOnMissingValues=False)

            # should match # of cols in header or ??
            self.assertEqual(inspect['num_cols'], totalCols, \
                "parse created result with the wrong number of cols %s %s" % (inspect['num_cols'], totalCols))

            # do we end up parsing one data rows as a header because of mismatch in gen/param
            h2oLosesOneData = (headerRowsDone==0) and (kwargs['header']==1) and not DATA_HAS_HDR_ROW
            print "h2oLosesOneData:", h2oLosesOneData
            if h2oLosesOneData:
                totalDataRows -= 1

            self.assertEqual(inspect['num_rows'], totalDataRows,
                "parse created result with the wrong number of rows (header rows don't count) h2o: %s gen'ed: %s" % \
                (inspect['num_rows'], totalDataRows))

            # put in an ignore param, that will fail unless headers were parsed correctly
            # doesn't matter if the header got a comment, should see it
            h2oShouldSeeHeader = (HEADER_HAS_HDR_ROW and (kwargs['header_from_file']==1)) or DATA_HAS_HDR_ROW
            if h2oShouldSeeHeader:
                kwargs = {'sample': 75, 'depth': 25, 'ntree': 1, 'ignore': 'A'}
            else:
                kwargs = {'sample': 75, 'depth': 25, 'ntree': 1}

            start = time.time()
            elapsed = time.time() - start
            print "%d pct. of timeout" % ((elapsed/timeoutSecs) * 100)
            print "trial #", trial, "totalDataRows:", totalDataRows, "parse end on ", csvFilename, \
                'took', time.time() - start, 'seconds'

            h2o.check_sandbox_for_errors()

if __name__ == '__main__':
    h2o.unit_main()
<commit_msg>passes with tomas' latest<commit_after>import unittest, time, sys, random
sys.path.extend(['.','..','py'])
import h2o, h2o_cmd, h2o_hosts
import h2o_browse as h2b

# ord('a') gives 97. Use that when you pass it as url param to h2o
# str(unichr(97)) gives 'a'

print "Data rows in header_from_file are ignored unless that file was part of the parse pattern"
print "Tab in header is not auto-detected separator. comma and space are"

print "Maybe a rows count problem if using tabs? temporary only tabs"
print "passes if only , is used"
print "just do one file with tab"
paramsDict = {
    # don't worry about these variants in this test (just parse normal)
    # 'parser_type': [None, 'AUTO', 'XLS', 'XLSX', 'SVMLight'],
    # I suppose, libsvm could have strangeness!..but there is no header with libsvm?
    'parser_type': [None, 'AUTO'],
    # gets used for separator=
    # 'separator': [",", " ", "\t"],
    # 'separator': ["\t", "|", ",", " "],
    'separator': ["\t", ",", " "],
    # 'separator': ["\t"],
    'header': [None, 0,1],
    # we can point to the 'wrong' file!
    # assume this is always used, otherwise we sum data rows without knowing if we'll use the header file?
    # always point to the header file..again, if we switch it around, the counts are off
    'header_from_file': ['syn_header'],
    # 'header_from_file': [None],
}

# ability to selectively comment the first line (which may be data or header)
# Does not write an extra line as the comment first in that case (header can be picked from the comment line)
# H2O always just strips the comment from the first line? and continues parsing?

# Don't write headerString if None (for non-header files)
# Don't write data if rowCount is None
def write_syn_dataset(csvPathname, rowCount, headerString, rList, commentFirst=False, sepChar=','):
    commentDone = False
    dsf = open(csvPathname, "w+")

    # this should never add to the actual count of data rows
    # UPDATE: comments are always ignored. So a commented header, commented data, is ignored.
    headerRowsDone = 0
    if headerString is not None:
        if commentFirst and not commentDone:
            h = "# " + headerString
            commentDone = True
        # FIX: for now, always put the header in even if we're told to!
        h = headerString
        headerRowsDone += 1
        dsf.write(h + "\n")

    if rowCount is not None:
        for i in range(rowCount):
            # two choices on the input. Make output choices random
            r = rList[random.randint(0,1)] + sepChar + str(random.randint(0,7))
            if commentFirst and not commentDone:
                # if commented, then write it twice, so the count of rows done is what we expect
                # the commented row will be ignored
                dsf.write("# " + r + "\n")
                commentDone = True
            dsf.write(r + "\n")
        dsf.close()
        return (headerRowsDone, rowCount) # rows done
    else:
        return (headerRowsDone, 0) # rows done

def rand_rowData(colCount, sepChar):
    rowData = [random.randint(0,7) for i in range(colCount)]
    rowData1= sepChar.join(map(str,rowData))
    rowData = [random.randint(0,7) for i in range(colCount)]
    rowData2= sepChar.join(map(str,rowData))
    # RF will complain if all inputs are the same
    r = [rowData1, rowData2]
    return r

class Basic(unittest.TestCase):
    def tearDown(self):
        h2o.check_sandbox_for_errors()

    @classmethod
    def setUpClass(cls):
        global SEED, localhost
        SEED = h2o.setup_random_seed()
        # SEED = h2o.setup_random_seed(8968685305521902318)

        localhost = h2o.decide_if_localhost()
        if (localhost):
            h2o.build_cloud(2,java_heap_MB=1300,use_flatfile=True)
        else:
            h2o_hosts.build_cloud_with_hosts()

    @classmethod
    def tearDownClass(cls):
        h2o.tear_down_cloud()

    def test_parse_multi_header_rand(self):
        ### h2b.browseTheCloud()
        SYNDATASETS_DIR = h2o.make_syn_dir()
        csvFilename = "syn_ints.csv"
        csvPathname = SYNDATASETS_DIR + '/' + csvFilename

        headerChoices = ['aA','aB','aC','aD','aE','aF','aG','aH','aI', 'tomas']

        # cols must be 9 to match the header above, otherwise a different bug is hit
        # extra output is added, so it's 10 total
        tryList = [
            # FIX! one fails count for now
            # (1, 5, 9, 'cA', 60, 0),
            (1, 5, 9, 'cA', 60, 0),
            (1, 5, 25, 'cA', 60, 0),

            # try with col mismatch on header.
            # FIX! causes exception? don't test for now
            # (7, 300, 10, 'cA', 60, 0),
            # (7, 300, 10, 'cB', 60, 1),
            # (7, 300, 10, 'cC', 60, 2),
            # (7, 300, 10, 'cD', 60, 3),

            # (7, 300, 8, 'cA', 60, 0),
            # (7, 300, 8, 'cB', 60, 1),
            # (7, 300, 8, 'cC', 60, 2),
            # (7, 300, 8, 'cD', 60, 3),
            ]

        # so many random combos..rather than walk tryList, just do random for some amount of time
        for trial in range(50):
            (fileNum, rowCount, colCount, key2, timeoutSecs, dataRowsWithHeader) = random.choice(tryList)
            print fileNum, rowCount, colCount, key2, timeoutSecs, dataRowsWithHeader
            # FIX! should we add a header to them randomly???
            print "Wait while", fileNum, "synthetic files are created in", SYNDATASETS_DIR
            rowxcol = str(rowCount) + 'x' + str(colCount)
            totalCols = colCount + 1 # 1 extra for output
            totalDataRows = 0
            totalHeaderRows = 0

            HEADER_HAS_HDR_ROW = random.randint(0,1)
            DATA_HAS_HDR_ROW = random.randint(0,1)
            PARSE_PATTERN_INCLUDES_HEADER = random.randint(0,1)
            ## DATA_FIRST_IS_COMMENT = random.randint(0,1)
            ## HEADER_FIRST_IS_COMMENT = random.randint(0,1)
            print "TEMPORARY: don't put any comments in"
            DATA_FIRST_IS_COMMENT = 0
            HEADER_FIRST_IS_COMMENT = 0
            # none is not legal
            SEP_CHAR_GEN = random.choice(paramsDict['separator'])

            print '\nHEADER_HAS_HDR_ROW:', HEADER_HAS_HDR_ROW
            print 'DATA_HAS_HDR_ROW:', DATA_HAS_HDR_ROW
            print 'PARSE_PATTERN_INCLUDES_HEADER', PARSE_PATTERN_INCLUDES_HEADER
            print 'DATA_FIRST_IS_COMMENT:', DATA_FIRST_IS_COMMENT
            print 'HEADER_FIRST_IS_COMMENT:', HEADER_FIRST_IS_COMMENT
            print 'SEP_CHAR_GEN:', SEP_CHAR_GEN

            # they need to both use the same separator (h2o rule)
            hh = [random.choice(headerChoices) for h in range(colCount)] + ["output"]
            print hh
            headerForHeader = SEP_CHAR_GEN.join(hh)
            # make these different
            hh = [random.choice(headerChoices) for h in range(colCount)] + ["output"]
            headerForData   = SEP_CHAR_GEN.join(hh)

            # random selection of parse param choices
            kwargs = {}
            for k,v in paramsDict.items():
                aChoice = random.choice(v)
                # can tell h2o something different compared to what we actually used!
                if k == 'separator':
                    if aChoice:
                        sepChar = aChoice
                        sepCharInt = ord(aChoice) # make it an integer for h2o
                    else:
                        sepChar = ',' # default char for None, need it for header/data file creation
                        sepCharInt = None
                    aChoice = sepCharInt

                kwargs[k] = aChoice

            # FOR NOW: ..override the rand choice if it exists, so we can parse and expect 'A' to be found
            # match what was gen'ed if choice is not None
            if kwargs['separator']:
                if SEP_CHAR_GEN==" "  or SEP_CHAR_GEN==",": # parse doesn't auto-detect tab. will autodetect space and comma
                    del kwargs['separator']
                else:
                    kwargs['separator'] = ord(SEP_CHAR_GEN)

            # create data files
            for fileN in range(fileNum):
                csvFilename = 'syn_data_' + str(fileN) + "_" + str(SEED) + "_" + str(trial) + "_" + rowxcol + '.csv'
                csvPathname = SYNDATASETS_DIR + '/' + csvFilename
                rList = rand_rowData(colCount, sepChar=SEP_CHAR_GEN)
                (headerRowsDone, dataRowsDone) = write_syn_dataset(csvPathname, rowCount,
                    headerString=(headerForData if DATA_HAS_HDR_ROW else None), rList=rList,
                    commentFirst=DATA_FIRST_IS_COMMENT, sepChar=SEP_CHAR_GEN)
                totalDataRows += dataRowsDone
                totalHeaderRows += headerRowsDone

            # create the header file
            hdrFilename = 'syn_header_' + str(SEED) + "_" + str(trial) + "_" + rowxcol + '.csv'
            hdrPathname = SYNDATASETS_DIR + '/' + hdrFilename
            # dataRowsWithHeader = 0 # temp hack
            (headerRowsDone, dataRowsDone) = write_syn_dataset(hdrPathname, dataRowsWithHeader,
                headerString=(headerForHeader if HEADER_HAS_HDR_ROW else None), rList=rList,
                commentFirst=HEADER_FIRST_IS_COMMENT, sepChar=SEP_CHAR_GEN)
            if PARSE_PATTERN_INCLUDES_HEADER: # only include header file data rows if the parse pattern includes it
                totalDataRows += dataRowsDone
            totalHeaderRows += headerRowsDone

            # make sure all key names are unique, when we re-put and re-parse (h2o caching issues)
            key = "syn_dst" + str(trial)
            key2 = "syn_dst" + str(trial) + ".hex"

            # DON"T get redirected to S3! (EC2 hack in config, remember!)
            # use it at the node level directly (because we gen'ed the files.
            # I suppose we could force the redirect state bits in h2o.nodes[0] to False, instead?:w
            xs = h2o.nodes[0].import_files(SYNDATASETS_DIR)['keys']
            headerKey = [x for x in xs if hdrFilename in x][0]
            dataKey = [x for x in xs if csvFilename not in x][0]

            # use regex. the only files in the dir will be the ones we just created with  *fileN* match
            print "Header Key =", headerKey

            # put the right name in
            if kwargs['header_from_file'] == 'syn_header':
                kwargs['header_from_file'] = headerKey
            # use one of the data files?
            elif kwargs['header_from_file'] == 'syn_data':
                kwargs['header_from_file'] = dataKey

            # if there's no header in the header file, turn off the header_from_file
            if not HEADER_HAS_HDR_ROW:
                kwargs['header_from_file'] = None

            print "If header_from_file= is used, we are currently required to force header=1 for h2o"
            if kwargs['header_from_file']:
                kwargs['header'] =  1
            # if we have a header in a data file, tell h2o (for now)
            elif DATA_HAS_HDR_ROW:
                kwargs['header'] =  1
            else:
                kwargs['header'] =  0

            # may have error if h2o doesn't get anything!
            start = time.time()
            if PARSE_PATTERN_INCLUDES_HEADER and HEADER_HAS_HDR_ROW:
                pattern = '*syn_*'+str(trial)+"_"+rowxcol+'*'
            else:
                pattern = '*syn_data_*'+str(trial)+"_"+rowxcol+'*'
            parseKey = h2o.nodes[0].parse(pattern, key2=key2, timeoutSecs=timeoutSecs, **kwargs)

            print "parseKey['destination_key']: " + parseKey['destination_key']
            print 'parse time:', parseKey['response']['time']

            inspect = h2o_cmd.runInspect(None, parseKey['destination_key'])
            h2o_cmd.infoFromInspect(inspect, csvPathname)
            print "\n" + csvPathname, \
                "    num_rows:", "{:,}".format(inspect['num_rows']), \
                "    num_cols:", "{:,}".format(inspect['num_cols'])

            # more reporting: (we can error here if extra col in header, causes all NA for missing col of data)
            h2o_cmd.columnInfoFromInspect(parseKey['destination_key'], exceptionOnMissingValues=False)

            # should match # of cols in header or ??
            self.assertEqual(inspect['num_cols'], totalCols, \
                "parse created result with the wrong number of cols %s %s" % (inspect['num_cols'], totalCols))

            # do we end up parsing one data rows as a header because of mismatch in gen/param
            h2oLosesOneData = (headerRowsDone==0) and (kwargs['header']==1) and not DATA_HAS_HDR_ROW
            # header in data file gets treated as data
            h2oGainsOneData = (headerRowsDone!=0) and (kwargs['header']==1) and \
                DATA_HAS_HDR_ROW and (kwargs['header_from_file'] is not None)
            print "h2oLosesOneData:", h2oLosesOneData
            print "h2oGainsOneData:", h2oGainsOneData
            ### print (headerRowsDone!=0), (kwargs['header']==1), DATA_HAS_HDR_ROW, (kwargs['header_from_file'] is not None)
            if h2oLosesOneData:
                totalDataRows -= 1
            if h2oGainsOneData:
                totalDataRows += 1

            self.assertEqual(inspect['num_rows'], totalDataRows,
                "parse created result with the wrong number of rows (header rows don't count) h2o: %s gen'ed: %s" % \
                (inspect['num_rows'], totalDataRows))

            # put in an ignore param, that will fail unless headers were parsed correctly
            # doesn't matter if the header got a comment, should see it
            h2oShouldSeeHeader = (HEADER_HAS_HDR_ROW and (kwargs['header_from_file'] is not None)) or DATA_HAS_HDR_ROW
            if h2oShouldSeeHeader:
                kwargs = {'sample': 75, 'depth': 25, 'ntree': 1, 'ignore': 'A'}
            else:
                kwargs = {'sample': 75, 'depth': 25, 'ntree': 1}

            start = time.time()
            elapsed = time.time() - start
            print "%d pct. of timeout" % ((elapsed/timeoutSecs) * 100)
            print "trial #", trial, "totalDataRows:", totalDataRows, "parse end on ", csvFilename, \
                'took', time.time() - start, 'seconds'

            h2o.check_sandbox_for_errors()

if __name__ == '__main__':
    h2o.unit_main()
<|endoftext|>


<commit_before># Copyright 2013-2018 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class RArgparse(RPackage):
    """A command line parser to be used with Rscript to write "#!"
       shebang scripts that gracefully accept positional and optional
       arguments and automatically generate usage."""

    homepage = "https://github.com/trevorld/argparse"
    url      = "https://cran.r-project.org/src/contrib/argparse_1.1.1.tar.gz"
    list_url = "https://cran.r-project.org/src/contrib/Archive/argparse"

    version('1.1.1', sha256='441449f0816411a868fd1b15cf4b2bc45931bbd4b67d6592dbe48875905cf93b')

    depends_on('r-proto@1:', type=('build', 'run'))
    depends_on('r-findpython', type=('build', 'run'))
    depends_on('r-getopt@1.19', type=('build', 'run'))
    depends_on('r-jsonlite', type=('build', 'run'))
<commit_msg>r-argparse: remove unnecessary r-getopt version (#9638)<commit_after># Copyright 2013-2018 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class RArgparse(RPackage):
    """A command line parser to be used with Rscript to write "#!"
       shebang scripts that gracefully accept positional and optional
       arguments and automatically generate usage."""

    homepage = "https://github.com/trevorld/argparse"
    url      = "https://cran.r-project.org/src/contrib/argparse_1.1.1.tar.gz"
    list_url = "https://cran.r-project.org/src/contrib/Archive/argparse"

    version('1.1.1', sha256='441449f0816411a868fd1b15cf4b2bc45931bbd4b67d6592dbe48875905cf93b')

    depends_on('r-proto@1:', type=('build', 'run'))
    depends_on('r-findpython', type=('build', 'run'))
    depends_on('r-getopt', type=('build', 'run'))
    depends_on('r-jsonlite', type=('build', 'run'))
<|endoftext|>


<commit_before>import os
import signal
import subprocess
import sys
import time

import pytest

import dcos.util as util
from dcos.util import create_schema

from dcoscli.test.common import (assert_command, assert_lines, delete_zk_node,
                                 delete_zk_nodes, exec_command, ssh_output)
from dcoscli.test.marathon import remove_app
from dcoscli.test.package import (package, package_install,
                                  setup_universe_server,
                                  teardown_universe_server)
from dcoscli.test.service import (get_services, service_shutdown,
                                  wait_for_service)
from ..fixtures.service import framework_fixture

def setup_module(module):
    setup_universe_server()

def teardown_module(module):
    delete_zk_nodes()
    teardown_universe_server()

def test_help():
    with open('dcoscli/data/help/service.txt') as content:
        assert_command(['dcos', 'service', '--help'],
                       stdout=content.read().encode('utf-8'))

def test_info():
    stdout = b"Manage DC/OS services\n"
    assert_command(['dcos', 'service', '--info'], stdout=stdout)

def test_service():
    services = get_services()

    schema = _get_schema(framework_fixture())
    for srv in services:
        assert not util.validate_json(srv, schema)

def test_service_table():
    assert_lines(['dcos', 'service'], 3)

def test_service_inactive_and_completed():
    package_install('kafka', True, ['--app'])
    wait_for_service('kafka')

    # get kafka's framework ID
    kafka_id = None
    for service in get_services():
        if service['name'] == 'kafka':
            kafka_id = service['id']
            break

    assert kafka_id is not None

    # assert kafka is listed
    services = get_services()
    assert any(
        service['name'] == 'kafka' for service in services)

    # kill scheduler and test for inactive...
    exec_command(['dcos', 'marathon', 'app', 'stop', '/kafka', '--force'])

    time.sleep(5)
    # assert kafka is not listed
    assert not any(
        service['name'] == 'kafka' for service in get_services())

    # assert kafka is inactive
    inactive = get_services(args=['--inactive'])
    assert any(service['name'] == 'kafka' for service in inactive)

    # teardown framework and test for completed...
    service_shutdown(kafka_id)
    delete_zk_node('kafka-mesos')

    # assert kafka is not listed
    assert not any(
        service['name'] == 'kafka' for service in get_services())

    # assert kafka is completed
    services = get_services(args=['--completed'])
    assert len(services) >= 3
    assert any(service['id'] == kafka_id for service in services)

    exec_command(['dcos', 'package', 'uninstall', 'kafka', '--yes'])

def test_log():
    with package('hello-world', deploy=True):
        returncode, stdout, stderr = exec_command(
            ['dcos', 'service', 'log', 'hello-world'])

        assert returncode == 0
        assert len(stdout.decode('utf-8').split('\n')) > 1
        assert stderr == b''

        returncode, stdout, stderr = exec_command(
            ['dcos', 'service', 'log', 'hello-world', 'stderr'])

        assert returncode == 0
        assert len(stdout.decode('utf-8').split('\n')) > 1
        assert stderr == b''

def test_log_marathon_file():
    assert_command(['dcos', 'service', 'log', 'marathon', 'stderr'],
                   stderr=(b'The <file> argument is invalid for marathon. ' +
                           b'The systemd journal is always used for the ' +
                           b'marathon log.\n'),
                   returncode=1)

@pytest.mark.skipif(sys.platform == 'win32',
                    reason='No pseduo terminal on windows')
def test_log_marathon_config():
    stdout, stderr, _ = ssh_output(
        'dcos service log marathon ' +
        '--ssh-config-file=tests/data/node/ssh_config')

    assert stdout == b''
    assert b'ignoring bad proto spec' in stderr

@pytest.mark.skipif(True,
                    reason=(
                        "Now that we test against an AWS cluster, this test "
                        "is blocked on DCOS-3104requires python3.3"))
def test_log_marathon():
    stdout, stderr, _ = ssh_output(
        'dcos service log marathon ' +
        '--ssh-config-file=tests/data/service/ssh_config')

    assert len(stdout.decode('utf-8').split('\n')) > 10

    assert b"Running `" in stderr
    num_lines = len(stderr.decode('utf-8').split('\n'))
    assert ((num_lines == 2) or
            (num_lines == 3 and b'Warning: Permanently added' in stderr))

def test_log_config():
    assert_command(
        ['dcos', 'service', 'log', 'chronos', '--ssh-config-file=/path'],
        stderr=(b'The `--ssh-config-file` argument is invalid for '
                b'non-marathon services. SSH is not used.\n'),
        returncode=1)

@pytest.mark.skipif(os.environ.get('DCOS_ENABLE_LOG_TEST') != 1,
                    reason='disable python buffering')
def test_log_follow():
    package_install('chronos', deploy=True)

    args = ['dcos', 'service', 'log', 'chronos', '--follow']

    if sys.platform == 'win32':
        proc = subprocess.Popen(
            args,
            creationflags=subprocess.CREATE_NEW_PROCESS_GROUP,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE)
    else:
        # disable stdout/stderr buffering:
        # https://docs.python.org/3/using/cmdline.html#cmdoption-u
        my_env = os.environ.copy()
        my_env['PYTHONUNBUFFERED'] = 'x'

        # os.setsid is only available for Unix:
        # https://docs.python.org/2/library/os.html#os.setsid
        proc = subprocess.Popen(
            args,
            preexec_fn=os.setsid,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            env=my_env)

    time.sleep(10)

    proc.poll()
    assert proc.returncode is None

    if sys.platform == 'win32':
        os.kill(proc.pid, signal.CTRL_BREAK_EVENT)
    else:
        # using Unix-only commands os.killpg + os.getgid
        # https://docs.python.org/2/library/os.html#os.killpg
        # https://docs.python.org/2/library/os.html#os.getpgid
        os.killpg(os.getpgid(proc.pid), 15)

    stdout = proc.stdout.read()
    stderr = proc.stderr.read()

    print('STDOUT: {}'.format(stdout))
    print('STDERR: {}'.format(stderr))
    assert len(stdout.decode('utf-8').split('\n')) > 3

    assert_lines(['dcos', 'service', 'log', 'chronos', '--lines=4'], 4)

    exec_command(['dcos', 'package', 'uninstall', 'chronos', '--yes'])

@pytest.mark.skipif(True, reason='Broken Marathon but we need to release')
def test_log_multiple_apps():
    package_install('marathon',
                    True,
                    ['--options=tests/data/service/marathon-user.json'])
    package_install('marathon',
                    True,
                    ['--options=tests/data/service/marathon-user2.json',
                     '--app-id=marathon-user2'])
    wait_for_service('marathon-user', number_of_services=2)

    try:
        stderr = (b'Multiple marathon apps found for service name ' +
                  b'[marathon-user]: [/marathon-user], [/marathon-user2]\n')
        assert_command(['dcos', 'service', 'log', 'marathon-user'],
                       returncode=1,
                       stderr=stderr)
    finally:
        # We can't use `dcos package uninstall`. The services have the same
        # name. Manually remove the dcos services.
        remove_app('marathon-user')
        remove_app('marathon-user2')
        for service in get_services():
            if service['name'] == 'marathon-user':
                service_shutdown(service['id'])

        delete_zk_node('universe')

def test_log_no_apps():
    assert_command(['dcos', 'service', 'log', 'bogus'],
                   stderr=b'No marathon apps found for service name [bogus]\n',
                   returncode=1)

def _get_schema(service):
    schema = create_schema(service.dict(), True)
    schema['required'].remove('reregistered_time')
    schema['required'].remove('pid')
    schema['required'].remove('executors')
    schema['properties']['offered_resources']['required'].remove('ports')
    schema['properties']['resources']['required'].remove('ports')
    schema['properties']['used_resources']['required'].remove('ports')

    return schema
<commit_msg>Use a variable in a test<commit_after>import os
import signal
import subprocess
import sys
import time

import pytest

import dcos.util as util
from dcos.util import create_schema

from dcoscli.test.common import (assert_command, assert_lines, delete_zk_node,
                                 delete_zk_nodes, exec_command, ssh_output)
from dcoscli.test.marathon import remove_app
from dcoscli.test.package import (package, package_install,
                                  setup_universe_server,
                                  teardown_universe_server)
from dcoscli.test.service import (get_services, service_shutdown,
                                  wait_for_service)
from ..fixtures.service import framework_fixture

def setup_module(module):
    setup_universe_server()

def teardown_module(module):
    delete_zk_nodes()
    teardown_universe_server()

def test_help():
    with open('dcoscli/data/help/service.txt') as content:
        assert_command(['dcos', 'service', '--help'],
                       stdout=content.read().encode('utf-8'))

def test_info():
    stdout = b"Manage DC/OS services\n"
    assert_command(['dcos', 'service', '--info'], stdout=stdout)

def test_service():
    services = get_services()

    schema = _get_schema(framework_fixture())
    for srv in services:
        assert not util.validate_json(srv, schema)

def test_service_table():
    assert_lines(['dcos', 'service'], 3)

def test_service_inactive_and_completed():
    package_install('kafka', True, ['--app'])
    wait_for_service('kafka')

    # get kafka's framework ID
    kafka_id = None
    for service in get_services():
        if service['name'] == 'kafka':
            kafka_id = service['id']
            break

    assert kafka_id is not None

    # assert kafka is listed
    services = get_services()
    assert any(
        service['name'] == 'kafka' for service in services)

    # kill scheduler and test for inactive...
    exec_command(['dcos', 'marathon', 'app', 'stop', '/kafka', '--force'])

    time.sleep(5)
    # assert kafka is not listed
    assert not any(
        service['name'] == 'kafka' for service in get_services())

    # assert kafka is inactive
    inactive = get_services(args=['--inactive'])
    assert any(service['name'] == 'kafka' for service in inactive)

    # teardown framework and test for completed...
    service_shutdown(kafka_id)
    delete_zk_node('kafka-mesos')

    # assert kafka is not listed
    assert not any(
        service['name'] == 'kafka' for service in get_services())

    # assert kafka is completed
    services = get_services(args=['--completed'])
    assert len(services) >= 3
    assert any(service['id'] == kafka_id for service in services)

    exec_command(['dcos', 'package', 'uninstall', 'kafka', '--yes'])

def test_log():
    package_name = 'hello-world'

    with package(package_name, deploy=True):
        returncode, stdout, stderr = exec_command(
            ['dcos', 'service', 'log', package_name])

        assert returncode == 0
        assert len(stdout.decode('utf-8').split('\n')) > 1
        assert stderr == b''

        returncode, stdout, stderr = exec_command(
            ['dcos', 'service', 'log', package_name, 'stderr'])

        assert returncode == 0
        assert len(stdout.decode('utf-8').split('\n')) > 1
        assert stderr == b''

def test_log_marathon_file():
    assert_command(['dcos', 'service', 'log', 'marathon', 'stderr'],
                   stderr=(b'The <file> argument is invalid for marathon. ' +
                           b'The systemd journal is always used for the ' +
                           b'marathon log.\n'),
                   returncode=1)

@pytest.mark.skipif(sys.platform == 'win32',
                    reason='No pseduo terminal on windows')
def test_log_marathon_config():
    stdout, stderr, _ = ssh_output(
        'dcos service log marathon ' +
        '--ssh-config-file=tests/data/node/ssh_config')

    assert stdout == b''
    assert b'ignoring bad proto spec' in stderr

@pytest.mark.skipif(True,
                    reason=(
                        "Now that we test against an AWS cluster, this test "
                        "is blocked on DCOS-3104requires python3.3"))
def test_log_marathon():
    stdout, stderr, _ = ssh_output(
        'dcos service log marathon ' +
        '--ssh-config-file=tests/data/service/ssh_config')

    assert len(stdout.decode('utf-8').split('\n')) > 10

    assert b"Running `" in stderr
    num_lines = len(stderr.decode('utf-8').split('\n'))
    assert ((num_lines == 2) or
            (num_lines == 3 and b'Warning: Permanently added' in stderr))

def test_log_config():
    assert_command(
        ['dcos', 'service', 'log', 'chronos', '--ssh-config-file=/path'],
        stderr=(b'The `--ssh-config-file` argument is invalid for '
                b'non-marathon services. SSH is not used.\n'),
        returncode=1)

@pytest.mark.skipif(os.environ.get('DCOS_ENABLE_LOG_TEST') != 1,
                    reason='disable python buffering')
def test_log_follow():
    package_install('chronos', deploy=True)

    args = ['dcos', 'service', 'log', 'chronos', '--follow']

    if sys.platform == 'win32':
        proc = subprocess.Popen(
            args,
            creationflags=subprocess.CREATE_NEW_PROCESS_GROUP,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE)
    else:
        # disable stdout/stderr buffering:
        # https://docs.python.org/3/using/cmdline.html#cmdoption-u
        my_env = os.environ.copy()
        my_env['PYTHONUNBUFFERED'] = 'x'

        # os.setsid is only available for Unix:
        # https://docs.python.org/2/library/os.html#os.setsid
        proc = subprocess.Popen(
            args,
            preexec_fn=os.setsid,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            env=my_env)

    time.sleep(10)

    proc.poll()
    assert proc.returncode is None

    if sys.platform == 'win32':
        os.kill(proc.pid, signal.CTRL_BREAK_EVENT)
    else:
        # using Unix-only commands os.killpg + os.getgid
        # https://docs.python.org/2/library/os.html#os.killpg
        # https://docs.python.org/2/library/os.html#os.getpgid
        os.killpg(os.getpgid(proc.pid), 15)

    stdout = proc.stdout.read()
    stderr = proc.stderr.read()

    print('STDOUT: {}'.format(stdout))
    print('STDERR: {}'.format(stderr))
    assert len(stdout.decode('utf-8').split('\n')) > 3

    assert_lines(['dcos', 'service', 'log', 'chronos', '--lines=4'], 4)

    exec_command(['dcos', 'package', 'uninstall', 'chronos', '--yes'])

@pytest.mark.skipif(True, reason='Broken Marathon but we need to release')
def test_log_multiple_apps():
    package_install('marathon',
                    True,
                    ['--options=tests/data/service/marathon-user.json'])
    package_install('marathon',
                    True,
                    ['--options=tests/data/service/marathon-user2.json',
                     '--app-id=marathon-user2'])
    wait_for_service('marathon-user', number_of_services=2)

    try:
        stderr = (b'Multiple marathon apps found for service name ' +
                  b'[marathon-user]: [/marathon-user], [/marathon-user2]\n')
        assert_command(['dcos', 'service', 'log', 'marathon-user'],
                       returncode=1,
                       stderr=stderr)
    finally:
        # We can't use `dcos package uninstall`. The services have the same
        # name. Manually remove the dcos services.
        remove_app('marathon-user')
        remove_app('marathon-user2')
        for service in get_services():
            if service['name'] == 'marathon-user':
                service_shutdown(service['id'])

        delete_zk_node('universe')

def test_log_no_apps():
    assert_command(['dcos', 'service', 'log', 'bogus'],
                   stderr=b'No marathon apps found for service name [bogus]\n',
                   returncode=1)

def _get_schema(service):
    schema = create_schema(service.dict(), True)
    schema['required'].remove('reregistered_time')
    schema['required'].remove('pid')
    schema['required'].remove('executors')
    schema['properties']['offered_resources']['required'].remove('ports')
    schema['properties']['resources']['required'].remove('ports')
    schema['properties']['used_resources']['required'].remove('ports')

    return schema
<|endoftext|>


<commit_before>"""Export a STEP file
"""

from onshape_client.client import Client
from onshape_client.onshape_url import OnshapeElement
from onshape_client.models.bt_translate_format_params import BTTranslateFormatParams
import time
import json
import os.path

client = Client()

# We're saving these files to the home directory ("~")
client.configuration.temp_folder_path = os.path.expanduser("~")

# Turn the URL into an "OnshapeElement"
url = "https://cad.onshape.com/documents/d52538842d07c098d986361f/w/2d3de2dce7022ea4bdd19149/e/5da80a9e428fc1110d0baaf2"
cube = OnshapeElement(url, client=client)

# Create the params. Note there are far more params that can be specified.
params = BTTranslateFormatParams(element_id=cube.eid, format_name="STEP", store_in_document=False, configuration='')

# Call the client
response = client.part_studios_api.translate_format5(cube.did, cube.wvm, cube.wvmid, cube.eid, bt_translate_format_params=params)

translation_id = response.id
state = response.request_state

# Long poll every 2 seconds until the translation is marked as done - alternatively you could submit a webhook
parsed_response = None

# Seconds between polls
polling_interval = 2

time_to_translate=0
while state == 'ACTIVE':
    time.sleep(polling_interval)
    time_to_translate += polling_interval
    response = client.translation_api.get_translation(translation_id, _preload_content=False)
    parsed_response = json.loads(response.data.decode("UTF-8"))
    state = parsed_response["requestState"]
    print("Still actively translating. Time thus far = {}".format(time_to_translate))

# Make the actual download when the translation is done, otherwise report the error
if state == "DONE":
    client.documents_api.download_external_data(cube.did, parsed_response["resultExternalDataIds"][0])
    print("File saved to {}. Took {} seconds to complete the translation.".format(client.configuration.temp_folder_path, time_to_translate))
else:
    print("An error ocurred on the server! Here is the response: \n" + parsed_response)
"""
File saved to <temp_folder_path>
"""<commit_msg>Show requesting a STEP file with a configuration<commit_after>"""Export a STEP file
"""

from onshape_client.client import Client
from onshape_client.onshape_url import OnshapeElement
from onshape_client.models.bt_translate_format_params import BTTranslateFormatParams
import time
import json
import os.path

client = Client()

# We're saving these files to the home directory ("~")
client.configuration.temp_folder_path = os.path.expanduser("~")

# Turn the URL into an "OnshapeElement"
url = "https://cad.onshape.com/documents/cca81d10f239db0db9481e6f/v/aaa25d18038e06f0b2964e2d/e/69c9eedda86512966b20bc90"
cube = OnshapeElement(url, client=client)

# Create the params. Note there are far more params that can be specified.
params = BTTranslateFormatParams(element_id=cube.eid, format_name="STEP", store_in_document=False, configuration='List_UKkGODiz574chc=chamfered;show_cube=true;size=0.254+meter')

# Call the client
response = client.part_studios_api.translate_format5(cube.did, cube.wvm, cube.wvmid, cube.eid, bt_translate_format_params=params)

translation_id = response.id
state = response.request_state

# Long poll every 2 seconds until the translation is marked as done - alternatively you could submit a webhook
parsed_response = None

# Seconds between polls
polling_interval = 2

time_to_translate=0
while state == 'ACTIVE':
    time.sleep(polling_interval)
    time_to_translate += polling_interval
    response = client.translation_api.get_translation(translation_id, _preload_content=False)
    parsed_response = json.loads(response.data.decode("UTF-8"))
    state = parsed_response["requestState"]
    print("Still actively translating. Time thus far = {}".format(time_to_translate))

# Make the actual download when the translation is done, otherwise report the error
if state == "DONE":
    client.documents_api.download_external_data(cube.did, parsed_response["resultExternalDataIds"][0])
    print("File saved to {}. Took {} seconds to complete the translation.".format(client.configuration.temp_folder_path, time_to_translate))
else:
    print("An error ocurred on the server! Here is the response: \n" + parsed_response)
"""
File saved to <temp_folder_path>
"""<|endoftext|>


<commit_before>from django.test import TestCase
from casexml.apps.case.mock import CaseBlock
from casexml.apps.case.models import CommCareCase
from casexml.apps.case.util import post_case_blocks
from casexml.apps.case.xml import V2
from casexml.apps.phone.xml import date_to_xml_string
from corehq.apps.domain.shortcuts import create_domain
from corehq.apps.users.models import CommCareUser
from corehq.apps.users.util import format_username
from corehq.apps.cloudcare.api import get_filtered_cases, CaseAPIResult, CASE_STATUS_OPEN, CASE_STATUS_ALL,\
    CASE_STATUS_CLOSED
import uuid

TEST_DOMAIN = "test-domain"

class CaseAPITest(TestCase):
    """
    Tests some of the Case API functions
    """
    domain = TEST_DOMAIN
    case_types = ['t1', 't2']
    user_ids = ['TEST_API1', 'TEST_API2']

    def setUp(self):
        create_domain(self.domain)
        self._clearData()
        password = "****"

        def create_user(username):
            return CommCareUser.create(self.domain,
                                       format_username(username, self.domain),
                                       password)

        self.users = [create_user(id) for id in self.user_ids]

        def create_case_set(user, child_user):
            # for each user we need one open and one closed case of
            # two different types.
            for type in self.case_types:
                c1 = _create_case(user, type, close=False) # open
                _create_case(user, type, close=True)       # closed
                # child
                _create_case(child_user,
                             _child_case_type(type), close=False,
                             index={'parent': ('parent-case', c1._id)})

        for i, user in enumerate(self.users):
            create_case_set(user, self.users[(i + 1) % len(self.users)])

        self.test_type = self.case_types[0]
        self.test_user_id = self.users[0]._id

    def tearDown(self):
        for user in self.users:
            django_user = user.get_django_user()
            django_user.delete()
            user.delete()
        self._clearData()

    def _clearData(self):
        for case_type in self.case_types:
            for subtype in [case_type, _child_case_type(case_type)]:
                for c in CommCareCase.get_all_cases(self.domain, case_type=subtype, include_docs=True):
                    c.delete()

    def assertListMatches(self, list, function):
        for item in list:
            self.assertTrue(function(item))

    @property
    def expectedOpenByType(self):
        return len(self.user_ids)

    @property
    def expectedClosedByType(self):
        return len(self.user_ids)

    @property
    def expectedByType(self):
        return self.expectedOpenByType + self.expectedClosedByType

    @property
    def expectedOpenByUser(self):
        return len(self.case_types) * 2 # one per type and child type

    @property
    def expectedClosedByUser(self):
        return len(self.case_types)

    @property
    def expectedByUser(self):
        return self.expectedOpenByUser + self.expectedClosedByUser

    @property
    def expectedOpenByUserWithFootprint(self):
        # each user gets 1 additional child case belonging to their cases
        # for each case type
        return self.expectedOpenByUser + len(self.case_types)

    @property
    def expectedClosedByUserWithFootprint(self):
        # each user gets 1 additional child case belonging to their cases
        return self.expectedClosedByUser

    @property
    def expectedOpen(self):
        return self.expectedOpenByUser * len(self.user_ids)

    @property
    def expectedClosed(self):
        return self.expectedClosedByUser * len(self.user_ids)

    @property
    def expectedAll(self):
        return len(self.user_ids) * len(self.case_types) * 3

    def testGetAllOpen(self):
        list = get_filtered_cases(self.domain, status=CASE_STATUS_OPEN)
        self.assertEqual(self.expectedOpen, len(list))
        self.assertListMatches(list, lambda c: not c['closed'])

    def testGetAllWithClosed(self):
        list = get_filtered_cases(self.domain, status=CASE_STATUS_ALL)
        self.assertEqual(self.expectedOpen + self.expectedClosed, len(list))

    def testGetAllOpenWithType(self):
        list = get_filtered_cases(self.domain, status=CASE_STATUS_OPEN, case_type=self.test_type)
        self.assertEqual(self.expectedOpenByType, len(list))
        self.assertListMatches(list, lambda c: not c['closed'] and c['properties']['case_type'] == self.test_type)

    def testGetAllWithClosedAndType(self):
        list = get_filtered_cases(self.domain, status=CASE_STATUS_ALL, case_type=self.test_type)
        self.assertEqual(self.expectedByType, len(list))
        self.assertListMatches(list, lambda c: c['properties']['case_type'] == self.test_type)

    def testGetOwnedOpen(self):
        list = get_filtered_cases(self.domain, user_id=self.test_user_id, status=CASE_STATUS_OPEN, footprint=False)
        self.assertEqual(self.expectedOpenByUser, len(list))
        self.assertListMatches(list, lambda c: not c['closed'] and c['user_id'] == self.test_user_id)

    def testGetOwnedClosed(self):
        list = get_filtered_cases(self.domain, user_id=self.test_user_id, status=CASE_STATUS_CLOSED, footprint=False)
        self.assertEqual(self.expectedClosedByUser, len(list))
        self.assertListMatches(list, lambda c: c['closed'] and c['user_id'] == self.test_user_id)

    def testGetOwnedBoth(self):
        list = get_filtered_cases(self.domain, user_id=self.test_user_id, status=CASE_STATUS_ALL, footprint=False)
        self.assertEqual(self.expectedByUser, len(list))
        self.assertListMatches(list, lambda c: c['user_id'] == self.test_user_id)

    def testGetOwnedOpenWithFootprint(self):
        list = get_filtered_cases(self.domain, user_id=self.test_user_id, status=CASE_STATUS_OPEN, footprint=True)
        self.assertEqual(self.expectedOpenByUserWithFootprint, len(list))
        self.assertListMatches(list, lambda c: not c['closed'] or c['user_id'] != self.test_user_id)

    def testGetOwnedClosedWithFootprint(self):
        list = get_filtered_cases(self.domain, user_id=self.test_user_id, status=CASE_STATUS_CLOSED, footprint=True)
        self.assertEqual(self.expectedClosedByUserWithFootprint, len(list))
        self.assertListMatches(list, lambda c: c['closed'] or c['user_id'] != self.test_user_id)

    def testGetOwnedBothWithFootprint(self):
        list = get_filtered_cases(self.domain, user_id=self.test_user_id, status=CASE_STATUS_ALL, footprint=True)
        self.assertEqual(self.expectedOpenByUserWithFootprint + self.expectedClosedByUserWithFootprint, len(list))
        # I don't think we can say anything super useful about this base set

    def testGetAllStripHistory(self):
        list = get_filtered_cases(self.domain, status=CASE_STATUS_ALL, footprint=True, include_children=True,
                                  strip_history=True)
        self.assertEqual(self.expectedAll, len(list))
        self.assertListMatches(list, lambda c: len(c._couch_doc.actions) == 0)
        self.assertListMatches(list, lambda c: len(c._couch_doc.xform_ids) == 0)

    def testGetAllIdsOnly(self):
        list = get_filtered_cases(self.domain, status=CASE_STATUS_ALL, footprint=True, include_children=True,
                                  ids_only=True)
        self.assertEqual(self.expectedAll, len(list))
        self.assertListMatches(list, lambda c: isinstance(c._couch_doc, dict))
        self.assertListMatches(list, lambda c: isinstance(c.to_json(), basestring))

    def testGetAllIdsOnlyStripHistory(self):
        list = get_filtered_cases(self.domain, status=CASE_STATUS_ALL, footprint=True, include_children=True,
                                  ids_only=True, strip_history=True)
        self.assertEqual(self.expectedAll, len(list))
        self.assertListMatches(list, lambda c: isinstance(c._couch_doc, dict))
        self.assertListMatches(list, lambda c: 'actions' not in c._couch_doc)
        self.assertListMatches(list, lambda c: 'xform_ids' not in c._couch_doc)
        self.assertListMatches(list, lambda c: isinstance(c.to_json(), basestring))

    def testFiltersOnAll(self):
        list = get_filtered_cases(self.domain, status=CASE_STATUS_ALL,
                                  filters={"properties/case_name": _type_to_name(self.test_type)})
        self.assertEqual(self.expectedByType, len(list))
        self.assertListMatches(list, lambda c: c['properties']['case_name'] == _type_to_name(self.test_type))

    def testFiltersOnOwned(self):
        list = get_filtered_cases(self.domain, user_id=self.test_user_id, status=CASE_STATUS_ALL,
                                  filters={"properties/case_name": _type_to_name(self.test_type)})
        self.assertEqual(2, len(list))
        self.assertListMatches(list, lambda c: c['properties']['case_name'] == _type_to_name(self.test_type))

    def testFiltersWithoutFootprint(self):
        name = _type_to_name(_child_case_type(self.test_type))
        list = get_filtered_cases(self.domain, user_id=self.test_user_id, status=CASE_STATUS_ALL,
                                  footprint=False,
                                  filters={"properties/case_name": name})
        self.assertEqual(1, len(list))
        self.assertListMatches(list, lambda c: c['properties']['case_name'] == name)

    def testFiltersWithFootprint(self):
        name = _type_to_name(_child_case_type(self.test_type))
        list = get_filtered_cases(self.domain, user_id=self.test_user_id, status=CASE_STATUS_ALL,
                                  footprint=True,
                                  filters={"properties/case_name": name})
        # when filtering with footprint, the filters get intentionally ignored
        # so just ensure the whole footprint including open and closed is available
        self.assertEqual(self.expectedOpenByUserWithFootprint + self.expectedClosedByUserWithFootprint, len(list))

    def testCaseAPIResultJSON(self):
        case = _create_case(self.users[0], None, close=False)
        res_sanitized = CaseAPIResult(id=case._id, couch_doc=case, sanitize=True)
        res_unsanitized = CaseAPIResult(id=case._id, couch_doc=case, sanitize=False)

        json = res_sanitized.case_json
        self.assertEqual(json['properties']['case_type'], '')

        json = res_unsanitized.case_json
        self.assertEqual(json['properties']['case_type'], None)

def _child_case_type(type):
    return "%s-child" % type

def _type_to_name(type):
    return "%s-name" % type

def _create_case(user, type, close=False, **extras):
    case_id = uuid.uuid4().hex
    blocks = [CaseBlock(
        create=True,
        case_id=case_id,
        case_name=_type_to_name(type),
        case_type=type,
        user_id=user.user_id,
        owner_id=user.user_id,
        version=V2,
        **extras
    ).as_xml(format_datetime=date_to_xml_string)]
    if close:
        blocks.append(CaseBlock(
            create=False,
            case_id=case_id,
            close=True,
            version=V2,
        ).as_xml(format_datetime=date_to_xml_string))
    post_case_blocks(blocks, {'domain': TEST_DOMAIN})
    case = CommCareCase.get(case_id)
    assert case.closed == close
    return case
<commit_msg>Fix tests<commit_after>from django.test import TestCase
from casexml.apps.case.mock import CaseBlock
from casexml.apps.case.models import CommCareCase
from casexml.apps.case.util import post_case_blocks
from casexml.apps.case.xml import V2
from casexml.apps.phone.xml import date_to_xml_string
from corehq.apps.domain.shortcuts import create_domain
from corehq.apps.users.models import CommCareUser
from corehq.apps.users.util import format_username
from corehq.apps.cloudcare.api import get_filtered_cases, CaseAPIResult, CASE_STATUS_OPEN, CASE_STATUS_ALL,\
    CASE_STATUS_CLOSED
import uuid

TEST_DOMAIN = "test-domain"

class CaseAPITest(TestCase):
    """
    Tests some of the Case API functions
    """
    domain = TEST_DOMAIN
    case_types = ['t1', 't2']
    user_ids = ['TEST_API1', 'TEST_API2']

    def setUp(self):
        create_domain(self.domain)
        self._clearData()
        password = "****"

        def create_user(username):
            return CommCareUser.create(self.domain,
                                       format_username(username, self.domain),
                                       password)

        self.users = [create_user(id) for id in self.user_ids]

        def create_case_set(user, child_user):
            # for each user we need one open and one closed case of
            # two different types.
            for type in self.case_types:
                c1 = _create_case(user, type, close=False) # open
                _create_case(user, type, close=True)       # closed
                # child
                _create_case(child_user,
                             _child_case_type(type), close=False,
                             index={'parent': ('parent-case', c1._id)})

        for i, user in enumerate(self.users):
            create_case_set(user, self.users[(i + 1) % len(self.users)])

        self.test_type = self.case_types[0]
        self.test_user_id = self.users[0]._id

    def tearDown(self):
        for user in self.users:
            django_user = user.get_django_user()
            django_user.delete()
            user.delete()
        self._clearData()

    def _clearData(self):
        for case_type in self.case_types:
            for subtype in [case_type, _child_case_type(case_type)]:
                for c in CommCareCase.get_all_cases(self.domain, case_type=subtype, include_docs=True):
                    c.delete()

    def assertListMatches(self, list, function):
        for item in list:
            self.assertTrue(function(item))

    @property
    def expectedOpenByType(self):
        return len(self.user_ids)

    @property
    def expectedClosedByType(self):
        return len(self.user_ids)

    @property
    def expectedByType(self):
        return self.expectedOpenByType + self.expectedClosedByType

    @property
    def expectedOpenByUser(self):
        return len(self.case_types) * 2 # one per type and child type

    @property
    def expectedClosedByUser(self):
        return len(self.case_types)

    @property
    def expectedByUser(self):
        return self.expectedOpenByUser + self.expectedClosedByUser

    @property
    def expectedOpenByUserWithFootprint(self):
        # each user gets 1 additional child case belonging to their cases
        # for each case type
        return self.expectedOpenByUser + len(self.case_types)

    @property
    def expectedClosedByUserWithFootprint(self):
        # each user gets 1 additional child case belonging to their cases
        return self.expectedClosedByUser

    @property
    def expectedOpen(self):
        return self.expectedOpenByUser * len(self.user_ids)

    @property
    def expectedClosed(self):
        return self.expectedClosedByUser * len(self.user_ids)

    @property
    def expectedAll(self):
        return len(self.user_ids) * len(self.case_types) * 3

    def testGetAllOpen(self):
        list = get_filtered_cases(self.domain, status=CASE_STATUS_OPEN)
        self.assertEqual(self.expectedOpen, len(list))
        self.assertListMatches(list, lambda c: not c['closed'])

    def testGetAllWithClosed(self):
        list = get_filtered_cases(self.domain, status=CASE_STATUS_ALL)
        self.assertEqual(self.expectedOpen + self.expectedClosed, len(list))

    def testGetAllOpenWithType(self):
        list = get_filtered_cases(self.domain, status=CASE_STATUS_OPEN, case_type=self.test_type)
        self.assertEqual(self.expectedOpenByType, len(list))
        self.assertListMatches(list, lambda c: not c['closed'] and c['properties']['case_type'] == self.test_type)

    def testGetAllWithClosedAndType(self):
        list = get_filtered_cases(self.domain, status=CASE_STATUS_ALL, case_type=self.test_type)
        self.assertEqual(self.expectedByType, len(list))
        self.assertListMatches(list, lambda c: c['properties']['case_type'] == self.test_type)

    def testGetOwnedOpen(self):
        list = get_filtered_cases(self.domain, user_id=self.test_user_id, status=CASE_STATUS_OPEN, footprint=False)
        self.assertEqual(self.expectedOpenByUser, len(list))
        self.assertListMatches(list, lambda c: not c['closed'] and c['user_id'] == self.test_user_id)

    def testGetOwnedClosed(self):
        list = get_filtered_cases(self.domain, user_id=self.test_user_id, status=CASE_STATUS_CLOSED, footprint=False)
        self.assertEqual(self.expectedClosedByUser, len(list))
        self.assertListMatches(list, lambda c: c['closed'] and c['user_id'] == self.test_user_id)

    def testGetOwnedBoth(self):
        list = get_filtered_cases(self.domain, user_id=self.test_user_id, status=CASE_STATUS_ALL, footprint=False)
        self.assertEqual(self.expectedByUser, len(list))
        self.assertListMatches(list, lambda c: c['user_id'] == self.test_user_id)

    def testGetOwnedOpenWithFootprint(self):
        list = get_filtered_cases(self.domain, user_id=self.test_user_id, status=CASE_STATUS_OPEN, footprint=True)
        self.assertEqual(self.expectedOpenByUserWithFootprint, len(list))
        self.assertListMatches(list, lambda c: not c['closed'] or c['user_id'] != self.test_user_id)

    def testGetOwnedClosedWithFootprint(self):
        list = get_filtered_cases(self.domain, user_id=self.test_user_id, status=CASE_STATUS_CLOSED, footprint=True)
        self.assertEqual(self.expectedClosedByUserWithFootprint, len(list))
        self.assertListMatches(list, lambda c: c['closed'] or c['user_id'] != self.test_user_id)

    def testGetOwnedBothWithFootprint(self):
        list = get_filtered_cases(self.domain, user_id=self.test_user_id, status=CASE_STATUS_ALL, footprint=True)
        self.assertEqual(self.expectedOpenByUserWithFootprint + self.expectedClosedByUserWithFootprint, len(list))
        # I don't think we can say anything super useful about this base set

    def testGetAllStripHistory(self):
        list = get_filtered_cases(self.domain, status=CASE_STATUS_ALL, footprint=True, include_children=True,
                                  strip_history=True)
        self.assertEqual(self.expectedAll, len(list))
        self.assertListMatches(list, lambda c: len(c._couch_doc.actions) == 0)
        self.assertListMatches(list, lambda c: len(c._couch_doc.xform_ids) == 0)

    def testGetAllIdsOnly(self):
        list = get_filtered_cases(self.domain, status=CASE_STATUS_ALL, footprint=True, include_children=True,
                                  ids_only=True)
        self.assertEqual(self.expectedAll, len(list))
        self.assertListMatches(list, lambda c: isinstance(c._couch_doc, dict))
        self.assertListMatches(list, lambda c: isinstance(c.to_json(), basestring))

    def testGetAllIdsOnlyStripHistory(self):
        list = get_filtered_cases(self.domain, status=CASE_STATUS_ALL, footprint=True, include_children=True,
                                  ids_only=True, strip_history=True)
        self.assertEqual(self.expectedAll, len(list))
        self.assertListMatches(list, lambda c: isinstance(c._couch_doc, dict))
        self.assertListMatches(list, lambda c: 'actions' not in c._couch_doc)
        self.assertListMatches(list, lambda c: 'xform_ids' not in c._couch_doc)
        self.assertListMatches(list, lambda c: isinstance(c.to_json(), basestring))

    def testFiltersOnAll(self):
        list = get_filtered_cases(self.domain, status=CASE_STATUS_ALL,
                                  filters={"properties/case_name": _type_to_name(self.test_type)})
        self.assertEqual(self.expectedByType, len(list))
        self.assertListMatches(list, lambda c: c['properties']['case_name'] == _type_to_name(self.test_type))

    def testFiltersOnOwned(self):
        list = get_filtered_cases(self.domain, user_id=self.test_user_id, status=CASE_STATUS_ALL,
                                  filters={"properties/case_name": _type_to_name(self.test_type)})
        self.assertEqual(2, len(list))
        self.assertListMatches(list, lambda c: c['properties']['case_name'] == _type_to_name(self.test_type))

    def testFiltersWithoutFootprint(self):
        name = _type_to_name(_child_case_type(self.test_type))
        list = get_filtered_cases(self.domain, user_id=self.test_user_id, status=CASE_STATUS_ALL,
                                  footprint=False,
                                  filters={"properties/case_name": name})
        self.assertEqual(1, len(list))
        self.assertListMatches(list, lambda c: c['properties']['case_name'] == name)

    def testFiltersWithFootprint(self):
        name = _type_to_name(_child_case_type(self.test_type))
        list = get_filtered_cases(self.domain, user_id=self.test_user_id, status=CASE_STATUS_ALL,
                                  footprint=True,
                                  filters={"properties/case_name": name})
        # when filtering with footprint, the filters get intentionally ignored
        # so just ensure the whole footprint including open and closed is available
        self.assertEqual(self.expectedOpenByUserWithFootprint + self.expectedClosedByUserWithFootprint, len(list))

    def testCaseAPIResultJSON(self):
        case = _create_case(self.users[0], None, close=False)
        res_sanitized = CaseAPIResult(id=case._id, couch_doc=case, sanitize=True)
        res_unsanitized = CaseAPIResult(id=case._id, couch_doc=case, sanitize=False)

        json = res_sanitized.case_json
        self.assertEqual(json['properties']['case_type'], '')

        json = res_unsanitized.case_json
        self.assertEqual(json['properties']['case_type'], None)
        case.delete()

def _child_case_type(type):
    return "%s-child" % type

def _type_to_name(type):
    return "%s-name" % type

def _create_case(user, type, close=False, **extras):
    case_id = uuid.uuid4().hex
    blocks = [CaseBlock(
        create=True,
        case_id=case_id,
        case_name=_type_to_name(type),
        case_type=type,
        user_id=user.user_id,
        owner_id=user.user_id,
        version=V2,
        **extras
    ).as_xml(format_datetime=date_to_xml_string)]
    if close:
        blocks.append(CaseBlock(
            create=False,
            case_id=case_id,
            close=True,
            version=V2,
        ).as_xml(format_datetime=date_to_xml_string))
    post_case_blocks(blocks, {'domain': TEST_DOMAIN})
    case = CommCareCase.get(case_id)
    assert case.closed == close
    return case
<|endoftext|>


<commit_before>import django_dynamic_fixture as fixture
from django_dynamic_fixture import get
from django.contrib.auth.models import User
from django.test import TestCase

from readthedocs.projects.models import Feature, Project
from readthedocs.projects.constants import PRIVATE, PUBLIC, PROTECTED
from readthedocs.builds.models import Version
from readthedocs.projects.querysets import (
    ChildRelatedProjectQuerySet,
    ParentRelatedProjectQuerySet,
)

class ProjectQuerySetTests(TestCase):

    def setUp(self):
        self.user = get(User)
        self.another_user = get(User)

        self.project = get(
            Project,
            privacy_level=PUBLIC,
            users=[self.user],
            main_language_project=None,
        )
        self.project_private = get(
            Project,
            privacy_level=PRIVATE,
            users=[self.user],
            main_language_project=None,
        )
        self.project_protected = get(
            Project,
            privacy_level=PROTECTED,
            users=[self.user],
            main_language_project=None,
        )

        self.another_project = get(
            Project,
            privacy_level=PUBLIC,
            users=[self.another_user],
            main_language_project=None,
        )
        self.another_project_private = get(
            Project,
            privacy_level=PRIVATE,
            users=[self.another_user],
            main_language_project=None,
        )
        self.another_project_protected = get(
            Project,
            privacy_level=PROTECTED,
            users=[self.another_user],
            main_language_project=None,
        )

        self.shared_project = get(
            Project,
            privacy_level=PUBLIC,
            users=[self.user, self.another_user],
            main_language_project=None,
        )
        self.shared_project_private = get(
            Project,
            privacy_level=PRIVATE,
            users=[self.user, self.another_user],
            main_language_project=None,
        )
        self.shared_project_protected = get(
            Project,
            privacy_level=PROTECTED,
            users=[self.user, self.another_user],
            main_language_project=None,
        )

        self.user_projects = {
            self.project,
            self.project_private,
            self.project_protected,
            self.shared_project,
            self.shared_project_private,
            self.shared_project_protected,
        }

        self.another_user_projects = {
            self.another_project,
            self.another_project_private,
            self.another_project_protected,
            self.shared_project,
            self.shared_project_private,
            self.shared_project_protected,
        }

    def test_subproject_queryset_attributes(self):
        self.assertEqual(ParentRelatedProjectQuerySet.project_field, 'parent')
        self.assertTrue(ParentRelatedProjectQuerySet.use_for_related_fields)
        self.assertEqual(ChildRelatedProjectQuerySet.project_field, 'child')
        self.assertTrue(ChildRelatedProjectQuerySet.use_for_related_fields)

    def test_subproject_queryset_as_manager_gets_correct_class(self):
        mgr = ChildRelatedProjectQuerySet.as_manager()
        self.assertEqual(
            mgr.__class__.__name__,
            'ManagerFromChildRelatedProjectQuerySetBase',
        )
        mgr = ParentRelatedProjectQuerySet.as_manager()
        self.assertEqual(
            mgr.__class__.__name__,
            'ManagerFromParentRelatedProjectQuerySetBase',
        )

    def test_is_active(self):
        project = fixture.get(Project, skip=False)
        self.assertTrue(Project.objects.is_active(project))

        project = fixture.get(Project, skip=True)
        self.assertFalse(Project.objects.is_active(project))

        user = fixture.get(User)
        user.profile.banned = True
        user.profile.save()
        project = fixture.get(Project, skip=False, users=[user])
        self.assertFalse(Project.objects.is_active(project))

    def test_dashboard(self):
        query = Project.objects.dashboard(user=self.user)
        self.assertEqual(query.count(), len(self.user_projects))
        self.assertEqual(set(query), self.user_projects)

        query = Project.objects.dashboard(user=self.another_user)
        self.assertEqual(query.count(), len(self.another_user_projects))
        self.assertEqual(set(query), self.another_user_projects)

    def test_private(self):
        query = Project.objects.private()
        projects = {
            self.project_private,
            self.another_project_private,
            self.shared_project_private,
        }
        self.assertEqual(query.count(), len(projects))
        self.assertEqual(set(query), projects)

    def test_private_user(self):
        query = Project.objects.private(user=self.user)
        projects = (
            self.user_projects |
            {self.another_project_private}
        )
        self.assertEqual(query.count(), len(projects))
        self.assertEqual(set(query), projects)

        query = Project.objects.private(user=self.another_user)
        projects = (
            self.another_user_projects |
            {self.project_private}
        )
        self.assertEqual(query.count(), len(projects))
        self.assertEqual(set(query), projects)

    def test_public(self):
        query = Project.objects.public()
        projects = {
            self.project,
            self.another_project,
            self.shared_project,
        }
        self.assertEqual(query.count(), len(projects))
        self.assertEqual(set(query), projects)

    def test_public_user(self):
        query = Project.objects.public(user=self.user)
        projects = (
            self.user_projects |
            {self.another_project}
        )
        self.assertEqual(query.count(), len(projects))
        self.assertEqual(set(query), projects)

        query = Project.objects.public(user=self.another_user)
        projects = (
            self.another_user_projects |
            {self.project}
        )
        self.assertEqual(query.count(), len(projects))
        self.assertEqual(set(query), projects)

    def test_protected(self):
        query = Project.objects.protected()
        projects = {
            self.project,
            self.project_protected,
            self.another_project,
            self.another_project_protected,
            self.shared_project,
            self.shared_project_protected,
        }
        self.assertEqual(query.count(), len(projects))
        self.assertEqual(set(query), projects)

    def test_protected_user(self):
        query = Project.objects.protected(user=self.user)
        projects = (
            self.user_projects |
            {self.another_project, self.another_project_protected}
        )
        self.assertEqual(query.count(), len(projects))
        self.assertEqual(set(query), projects)

        query = Project.objects.protected(user=self.another_user)
        projects = (
            self.another_user_projects |
            {self.project, self.project_protected}
        )
        self.assertEqual(query.count(), len(projects))
        self.assertEqual(set(query), projects)

class FeatureQuerySetTests(TestCase):

    def test_feature_for_project_is_explicit_applied(self):
        project = fixture.get(Project, main_language_project=None)
        feature = fixture.get(Feature, projects=[project])
        self.assertTrue(project.has_feature(feature.feature_id))

    def test_feature_for_project_is_implicitly_applied(self):
        project = fixture.get(Project, main_language_project=None)
        # Explicit feature
        feature1 = fixture.get(Feature, projects=[project])
        # False implicit feature
        feature2 = fixture.get(
            Feature,
            projects=[],
            add_date=project.pub_date + timedelta(days=1),
            default_true=False,
        )
        # True implicit feature before add date
        feature3 = fixture.get(
            Feature,
            projects=[],
            add_date=project.pub_date + timedelta(days=1),
            default_true=True,
        )
        # True implicit feature after add date
        feature4 = fixture.get(
            Feature,
            projects=[],
            add_date=project.pub_date - timedelta(days=1),
            default_true=True,
        )
        self.assertQuerysetEqual(
            Feature.objects.for_project(project),
            [repr(feature1), repr(feature3)],
            ordered=False,
        )

    def test_feature_multiple_projects(self):
        project1 = fixture.get(Project, main_language_project=None)
        project2 = fixture.get(Project, main_language_project=None)
        feature = fixture.get(Feature, projects=[project1, project2])
        self.assertQuerysetEqual(
            Feature.objects.for_project(project1),
            [repr(feature)],
            ordered=False,
        )
<commit_msg>Missing import<commit_after>from datetime import timedelta

import django_dynamic_fixture as fixture
from django_dynamic_fixture import get
from django.contrib.auth.models import User
from django.test import TestCase

from readthedocs.projects.models import Feature, Project
from readthedocs.projects.constants import PRIVATE, PUBLIC, PROTECTED
from readthedocs.builds.models import Version
from readthedocs.projects.querysets import (
    ChildRelatedProjectQuerySet,
    ParentRelatedProjectQuerySet,
)

class ProjectQuerySetTests(TestCase):

    def setUp(self):
        self.user = get(User)
        self.another_user = get(User)

        self.project = get(
            Project,
            privacy_level=PUBLIC,
            users=[self.user],
            main_language_project=None,
        )
        self.project_private = get(
            Project,
            privacy_level=PRIVATE,
            users=[self.user],
            main_language_project=None,
        )
        self.project_protected = get(
            Project,
            privacy_level=PROTECTED,
            users=[self.user],
            main_language_project=None,
        )

        self.another_project = get(
            Project,
            privacy_level=PUBLIC,
            users=[self.another_user],
            main_language_project=None,
        )
        self.another_project_private = get(
            Project,
            privacy_level=PRIVATE,
            users=[self.another_user],
            main_language_project=None,
        )
        self.another_project_protected = get(
            Project,
            privacy_level=PROTECTED,
            users=[self.another_user],
            main_language_project=None,
        )

        self.shared_project = get(
            Project,
            privacy_level=PUBLIC,
            users=[self.user, self.another_user],
            main_language_project=None,
        )
        self.shared_project_private = get(
            Project,
            privacy_level=PRIVATE,
            users=[self.user, self.another_user],
            main_language_project=None,
        )
        self.shared_project_protected = get(
            Project,
            privacy_level=PROTECTED,
            users=[self.user, self.another_user],
            main_language_project=None,
        )

        self.user_projects = {
            self.project,
            self.project_private,
            self.project_protected,
            self.shared_project,
            self.shared_project_private,
            self.shared_project_protected,
        }

        self.another_user_projects = {
            self.another_project,
            self.another_project_private,
            self.another_project_protected,
            self.shared_project,
            self.shared_project_private,
            self.shared_project_protected,
        }

    def test_subproject_queryset_attributes(self):
        self.assertEqual(ParentRelatedProjectQuerySet.project_field, 'parent')
        self.assertTrue(ParentRelatedProjectQuerySet.use_for_related_fields)
        self.assertEqual(ChildRelatedProjectQuerySet.project_field, 'child')
        self.assertTrue(ChildRelatedProjectQuerySet.use_for_related_fields)

    def test_subproject_queryset_as_manager_gets_correct_class(self):
        mgr = ChildRelatedProjectQuerySet.as_manager()
        self.assertEqual(
            mgr.__class__.__name__,
            'ManagerFromChildRelatedProjectQuerySetBase',
        )
        mgr = ParentRelatedProjectQuerySet.as_manager()
        self.assertEqual(
            mgr.__class__.__name__,
            'ManagerFromParentRelatedProjectQuerySetBase',
        )

    def test_is_active(self):
        project = fixture.get(Project, skip=False)
        self.assertTrue(Project.objects.is_active(project))

        project = fixture.get(Project, skip=True)
        self.assertFalse(Project.objects.is_active(project))

        user = fixture.get(User)
        user.profile.banned = True
        user.profile.save()
        project = fixture.get(Project, skip=False, users=[user])
        self.assertFalse(Project.objects.is_active(project))

    def test_dashboard(self):
        query = Project.objects.dashboard(user=self.user)
        self.assertEqual(query.count(), len(self.user_projects))
        self.assertEqual(set(query), self.user_projects)

        query = Project.objects.dashboard(user=self.another_user)
        self.assertEqual(query.count(), len(self.another_user_projects))
        self.assertEqual(set(query), self.another_user_projects)

    def test_private(self):
        query = Project.objects.private()
        projects = {
            self.project_private,
            self.another_project_private,
            self.shared_project_private,
        }
        self.assertEqual(query.count(), len(projects))
        self.assertEqual(set(query), projects)

    def test_private_user(self):
        query = Project.objects.private(user=self.user)
        projects = (
            self.user_projects |
            {self.another_project_private}
        )
        self.assertEqual(query.count(), len(projects))
        self.assertEqual(set(query), projects)

        query = Project.objects.private(user=self.another_user)
        projects = (
            self.another_user_projects |
            {self.project_private}
        )
        self.assertEqual(query.count(), len(projects))
        self.assertEqual(set(query), projects)

    def test_public(self):
        query = Project.objects.public()
        projects = {
            self.project,
            self.another_project,
            self.shared_project,
        }
        self.assertEqual(query.count(), len(projects))
        self.assertEqual(set(query), projects)

    def test_public_user(self):
        query = Project.objects.public(user=self.user)
        projects = (
            self.user_projects |
            {self.another_project}
        )
        self.assertEqual(query.count(), len(projects))
        self.assertEqual(set(query), projects)

        query = Project.objects.public(user=self.another_user)
        projects = (
            self.another_user_projects |
            {self.project}
        )
        self.assertEqual(query.count(), len(projects))
        self.assertEqual(set(query), projects)

    def test_protected(self):
        query = Project.objects.protected()
        projects = {
            self.project,
            self.project_protected,
            self.another_project,
            self.another_project_protected,
            self.shared_project,
            self.shared_project_protected,
        }
        self.assertEqual(query.count(), len(projects))
        self.assertEqual(set(query), projects)

    def test_protected_user(self):
        query = Project.objects.protected(user=self.user)
        projects = (
            self.user_projects |
            {self.another_project, self.another_project_protected}
        )
        self.assertEqual(query.count(), len(projects))
        self.assertEqual(set(query), projects)

        query = Project.objects.protected(user=self.another_user)
        projects = (
            self.another_user_projects |
            {self.project, self.project_protected}
        )
        self.assertEqual(query.count(), len(projects))
        self.assertEqual(set(query), projects)

class FeatureQuerySetTests(TestCase):

    def test_feature_for_project_is_explicit_applied(self):
        project = fixture.get(Project, main_language_project=None)
        feature = fixture.get(Feature, projects=[project])
        self.assertTrue(project.has_feature(feature.feature_id))

    def test_feature_for_project_is_implicitly_applied(self):
        project = fixture.get(Project, main_language_project=None)
        # Explicit feature
        feature1 = fixture.get(Feature, projects=[project])
        # False implicit feature
        feature2 = fixture.get(
            Feature,
            projects=[],
            add_date=project.pub_date + timedelta(days=1),
            default_true=False,
        )
        # True implicit feature before add date
        feature3 = fixture.get(
            Feature,
            projects=[],
            add_date=project.pub_date + timedelta(days=1),
            default_true=True,
        )
        # True implicit feature after add date
        feature4 = fixture.get(
            Feature,
            projects=[],
            add_date=project.pub_date - timedelta(days=1),
            default_true=True,
        )
        self.assertQuerysetEqual(
            Feature.objects.for_project(project),
            [repr(feature1), repr(feature3)],
            ordered=False,
        )

    def test_feature_multiple_projects(self):
        project1 = fixture.get(Project, main_language_project=None)
        project2 = fixture.get(Project, main_language_project=None)
        feature = fixture.get(Feature, projects=[project1, project2])
        self.assertQuerysetEqual(
            Feature.objects.for_project(project1),
            [repr(feature)],
            ordered=False,
        )
<|endoftext|>


<commit_before>from .adapter import get_indicator_table, IndicatorSqlAdapter, metadata
from .columns import get_expanded_column_config, SqlColumnConfig
from .connection import create_engine
from .util import get_column_name, get_table_name, truncate_value

<commit_msg>[ci skip] whitespace<commit_after>from .adapter import get_indicator_table, IndicatorSqlAdapter, metadata
from .columns import get_expanded_column_config, SqlColumnConfig
from .connection import create_engine
from .util import get_column_name, get_table_name, truncate_value
<|endoftext|>