<commit_before>import numpy as np
import os
import re

from singlecell_constants import DATADIR, MEHTA_ZSCORE_DATAFILE_PATH

# TODO pass metadata to all functions?
# TODO test and optimize read_exptdata_from_files

def read_datafile_simple(datapath, verbose=True, txt=False):
    if txt:
        assert datapath[-4:] == '.txt'
        arr = np.loadtxt(datapath, skiprows=1, usecols=(1,3,4))
    else:
        arr = np.load(datapath)
    if verbose:
        print "Loaded dim %s data at %s" % (arr.shape, datapath)
    return arr

def read_datafile_manual(datapath, verbose=True, save_as_sep_npy=False):
    """
    Datafile form is N+1 x M+1 array with column and row labels (NxM expression matrix)
    Function created for loading mouse cell atlas data formats
    http://bis.zju.edu.cn/MCA/contact.html (DGE matrices)
    """
    assert datapath[-4:] == '.txt'
    with open(datapath) as f:
        count = 0
        # do first pass to get gene count and cell names
        for idx, line in enumerate(f):
            if idx == 0:
                line = line.rstrip()
                line = line.split('\t')
                cell_names = [a.strip('"') for a in line]
            else:
                count += 1
        arr = np.zeros((count, len(cell_names)), dtype=np.int16)  # max size ~ 33k (unsigned twice that)
        gene_names = [0] * count
        print "data size will be (genes x cells):", arr.shape
        # second pass to get gene names and array data
        f.seek(0)
        for idx, line in enumerate(f):
            if idx > 0:
                if idx % 1000 == 0:
                    print idx
                line = line.rstrip()
                line = line.split('\t')
                gene_names[idx-1] = line[0].strip('"')
                arr[idx-1, :] = [int(val) for val in line[1:]]
    if verbose:
        print "Loaded dim %s data at %s" % (arr.shape, datapath)
        print "Max val in arr:", np.max(arr)
        print "Size of arr in memory (bytes)", arr.nbytes

    datadir = os.path.abspath(os.path.join(datapath, os.pardir))
    if save_as_sep_npy:
        np.save(datadir + os.sep + "raw_arr.npy", arr)
        np.save(datadir + os.sep + "raw_genes.npy", np.array(gene_names))
        np.save(datadir + os.sep + "raw_cells.npy", np.array(cell_names))
    else:
        genes = np.array(gene_names)
        cells = np.array(cell_names)
        np.savez_compressed(datadir + os.sep + "arr_genes_cells_raw_compressed.npz", arr=arr, genes=genes, cells=cells)
    return arr, gene_names, cell_names

def load_singlecell_data(zscore_datafile=MEHTA_ZSCORE_DATAFILE_PATH, savenpz='mems_genes_clusters_raw_compressed'):
    """
    Returns list of cell types (size p), list of TFs (size N), and xi array where xi_ij is ith TF value in cell type j
    Note the Mehta SI file has odd formatting (use regex to parse); array text file is read in as single line:
    http://journals.plos.org/ploscompbiol/article/file?id=10.1371/journal.pcbi.1003734.s005&type=supplementary
    """
    gene_labels = []
    with open(zscore_datafile) as f:
        origline = f.readline()
        filelines = origline.split('\r')
        for idx_row, row in enumerate(filelines):
            row_split = re.split(r'\t', row)
            if idx_row == 0:  # celltypes line, first row
                celltype_labels = row_split[1:]
            else:
                gene_labels.append(row_split[0])
    # reloop to get data without excessive append calls
    expression_data = np.zeros((len(gene_labels), len(celltype_labels)))
    with open(zscore_datafile) as f:
        origline = f.readline()
        filelines_dataonly = origline.split('\r')[1:]
        for idx_row, row in enumerate(filelines_dataonly):
            row_split_dataonly = re.split(r'\t', row)[1:]
            expression_data[idx_row,:] = [float(val) for val in row_split_dataonly]
    if savenpz is not None:
        datadir = os.path.abspath(os.path.join(zscore_datafile, os.pardir))
        npzpath = datadir + os.sep + savenpz
        save_npz_of_arr_genes_cells(npzpath, expression_data, gene_labels, celltype_labels)
    return expression_data, gene_labels, celltype_labels

def load_npz_of_arr_genes_cells(npzpath, verbose=True):
    """
    Can also use for memory array, gene labels, and cell cluster names!
    """
    if verbose:
        print "loading npz of arr genes cells at", npzpath, "..."
    loaded = np.load(npzpath)
    arr = loaded['arr']
    genes = loaded['genes']
    cells = loaded['cells']
    if verbose:
        print "loaded arr, genes, cells:", arr.shape, genes.shape, cells.shape
    return arr, genes, cells

def binarize_data(xi):
    return 1.0 * np.where(xi > 0, 1, -1)  # mult by 1.0 to cast as float

def save_npz_of_arr_genes_cells(npzpath, arr, genes, cells):
    np.savez_compressed(npzpath, arr=arr, genes=genes, cells=cells)
    return

def attach_cluster_id_arr_manual(npzpath, clusterpath, save=True, one_indexed=True):
    """
    one_indexed: if true, assume cluster index starts at 1 (as in scMCA)
    """
    arr, genes, cells = load_npz_of_arr_genes_cells(npzpath)
    # generate cell_to_cluster_idx mapping
    cluster_info = {}
    # expected format of csv file is "cell name, cluster idx, tissue origin"
    with open(clusterpath) as f:
        for idx, line in enumerate(f):
            line = line.rstrip()
            line = line.split(',')
            cluster_info[line[0]] = int(line[1])
    # adjust genes and arr contents
    arr = np.insert(arr, 0, 0, axis=0)
    genes = np.insert(genes, 0, 'cluster_id')  # TODO should have global constant for this mock gene label
    if one_indexed:
        for idx in xrange(len(cells)):
            arr[0,idx] = cluster_info[cells[idx]] - 1
    else:
        for idx in xrange(len(cells)):
            arr[0,idx] = cluster_info[cells[idx]]
    # save and return data
    if save:
        print "saving cluster-appended arrays..."
        np.savez_compressed(datadir + os.sep + "arr_genes_cells_withcluster_compressed.npz", arr=arr, genes=genes, cells=cells)
    return arr, genes, cells

def prune_boring_rows(npzpath, save=True):
    """
    Delete rows from array and corresponding genes that are self-duplicates
    NOTE: very similar to reduce_gene_set(xi, gene_labels)
    """

    arr, genes, cells = load_npz_of_arr_genes_cells(npzpath)
    # collect rows to delete
    rows_duplicates = np.all(arr.T == arr.T[0,:], axis=0)
    rows_to_delete = np.array([idx for idx, val in enumerate(rows_duplicates) if val])
    # note pruned rows
    print "number of self-duplicate rows:", len(rows_to_delete)
    print rows_to_delete
    print rows_to_delete[0:10]
    # adjust genes and arr contents
    print "Orig shape arr, genes, cells:", arr.shape, genes.shape, cells.shape
    arr = np.delete(arr, rows_to_delete, axis=0)
    genes = np.delete(genes, rows_to_delete)  # TODO should have global constant for this mock gene label
    print "New shape arr, genes, cells:", arr.shape, genes.shape, cells.shape  # TODO not operating as expected
    # save and return data
    if save:
        print "saving pruned arrays..."
        datadir = os.path.abspath(os.path.join(npzpath, os.pardir))
        base = os.path.basename(npzpath)
        basestr = os.path.splitext(base)[0]
        savestr = basestr + '_pruned.npz'
        np.savez_compressed(datadir + os.sep + savestr, arr=arr, genes=genes, cells=cells)
    return rows_to_delete, arr, genes, cells

def reduce_gene_set(xi, gene_labels):  # TODO: my removal ends with 1339 left but theirs with 1337 why?
    """
    NOTE: very similar to prune_boring_rows(...)
    """
    genes_to_remove = []
    for row_idx, row in enumerate(xi):
        if all(map(lambda x: x == row[0], row)):
            genes_to_remove.append(row_idx)
    reduced_gene_labels = [gene_labels[idx] for idx in xrange(len(xi)) if idx not in genes_to_remove]
    reduced_xi = np.array([row for idx, row in enumerate(xi) if idx not in genes_to_remove])
    return reduced_gene_labels, reduced_xi

def load_cluster_labels(clusterpath, one_indexed=True):
    """
    one_indexed: if true, assume cluster index starts at 1 (as in scMCA)
    """
    cluster_labels = {}
    if one_indexed:
        dec = 1
    else:
        dec = 0
    # expected format of csv file is "cluster number, name"
    with open(clusterpath) as f:
        for idx, line in enumerate(f):
            line = line.rstrip()
            line = line.split(',')
            print line
            cluster_labels[int(line[0])-dec] = line[1]
    return cluster_labels

def parse_exptdata(states_raw, gene_labels, verbose=True):
    """
    Args:
        - states_raw: stores array of state data and cluster labels for each cell state (column)
        - gene_labels: stores row names i.e. gene or PCA labels
    Notes: data format may change with time
        - convention is first row stores cluster index, from 0 to np.max(row 0) == K - 1
        - future convention may be to store unique integer ID for each column corresponding to earlier in pipeline
        - maybe also extract and pass metadata_dict info (i.e. K, N, M, filename information on pipeline)
    Returns:
        - cluster_dict: {cluster_idx: N x M array of raw cell states in the cluster (i.e. not binarized)}
        - metadata: dict, mainly stores N x 1 array of 'gene_labels' for each row
    """
    states_row0 = states_raw[0, :]
    states_truncated = states_raw[1:, :]
    num_genes, num_cells = states_truncated.shape  # aka N, M
    num_clusters = np.max(states_raw[0, :]) + 1
    if verbose:
        print "raw data dimension: %d x %d" % (states_raw.shape)
        print "cleaned data dimension: %d x %d" % (states_truncated.shape)
        print "num_clusters is %d" % num_clusters

    # process gene labels
    assert len(gene_labels) == num_genes or len(gene_labels) == num_genes + 1
    if len(gene_labels) == num_genes + 1:
        assert 'cluster' in gene_labels[0]
        gene_labels = gene_labels[1:]

    # prep cluster_dict
    cluster_dict = {}
    cluster_indices = {k: [] for k in xrange(num_clusters)}
    # TODO optimize this chunk if needed
    for cell_idx in xrange(num_cells):
        cluster_idx = states_row0[cell_idx]
        cluster_indices[cluster_idx].append(cell_idx)

    # build cluster dict
    if verbose:
        print "cluster_indices collected; building cluster arrays..."
    for k in xrange(num_clusters):
        print k
        cluster_dict[k] = states_truncated.take(cluster_indices[k], axis=1)

    # fill metatadata dict
    metadata = {}
    metadata['gene_labels'] = gene_labels
    metadata['num_clusters'] = num_clusters
    metadata['K'] = num_clusters
    metadata['num_genes'] = num_genes
    metadata['N'] = num_genes
    metadata['num_cells'] = num_cells
    metadata['M'] = num_cells
    return cluster_dict, metadata

if __name__ == '__main__':
    # run flags
    datadir = DATADIR + os.sep + "2018_scMCA"
    flag_load_simple = False
    flag_process_manual = False
    flag_load_sep_npy = False
    flag_load_compressed_npz = False
    flag_attach_clusters_resave = False
    flag_prune_boring_rows = False
    flag_process_data_mehta = False

    # simple data load
    if flag_load_simple:
        datapath = datadir + os.sep + "SI_Figure2-batch-removed.txt"
        arr = read_datafile_simple(datapath, verbose=True, txt=True)

    if flag_process_manual:
        datapath = datadir + os.sep + "SI_Figure2-batch-removed.txt"
        arr, genes, cells = read_datafile_manual(datapath, verbose=True)

    if flag_load_sep_npy:
        arrpath = datadir + os.sep + "raw_arr.npy"
        genespath = datadir + os.sep + "raw_genes.npy"
        cellspath = datadir + os.sep + "raw_cells.npy"
        arr = read_datafile_simple(arrpath, verbose=True)
        genes = read_datafile_simple(genespath, verbose=True)
        cells = read_datafile_simple(cellspath, verbose=True)

    if flag_load_compressed_npz:
        compressed_file = datadir + os.sep + "arr_genes_cells_raw_compressed.npz"
        arr, genes, cells = load_npz_of_arr_genes_cells(compressed_file)

    if flag_attach_clusters_resave:
        compressed_file = datadir + os.sep + "arr_genes_cells_raw_compressed.npz"
        clusterpath = datadir + os.sep + "SI_cells_to_clusters.csv"
        arr, genes, cells = attach_cluster_id_arr_manual(compressed_file, clusterpath, save=True)

    if flag_prune_boring_rows:
        compressed_file = datadir + os.sep + "arr_genes_cells_raw_compressed.npz"
        prune_boring_rows(compressed_file)

    if flag_process_data_mehta:
        # part 1: load their zscore textfile, save in standard npz format
        expression_data, genes, celltypes = load_singlecell_data(zscore_datafile=MEHTA_ZSCORE_DATAFILE_PATH,
                                                                 savenpz='mehta_mems_genes_clusters_zscore_compressed.npz')
        # part 2: load npz, binarize, save
        npzpath = DATADIR + os.sep + "2014_mehta" + os.sep + 'mehta_mems_genes_clusters_zscore_compressed.npz'
        expression_data, genes, celltypes = load_npz_of_arr_genes_cells(npzpath, verbose=True)
        xi = binarize_data(expression_data)
        compressed_boolean = datadir + os.sep + "mehta_mems_genes_clusters_boolean_compressed.npz"
        save_npz_of_arr_genes_cells(compressed_boolean, xi, genes, celltypes)
        # part 3: load npz, prune, save
        rows_to_delete, xi, genes, celltypes = prune_boring_rows(compressed_boolean)
<commit_msg>prune rows fn extended<commit_after>import numpy as np
import os
import re

from singlecell_constants import DATADIR, MEHTA_ZSCORE_DATAFILE_PATH

# TODO pass metadata to all functions?
# TODO test and optimize read_exptdata_from_files

def read_datafile_simple(datapath, verbose=True, txt=False):
    if txt:
        assert datapath[-4:] == '.txt'
        arr = np.loadtxt(datapath, skiprows=1, usecols=(1,3,4))
    else:
        arr = np.load(datapath)
    if verbose:
        print "Loaded dim %s data at %s" % (arr.shape, datapath)
    return arr

def read_datafile_manual(datapath, verbose=True, save_as_sep_npy=False):
    """
    Datafile form is N+1 x M+1 array with column and row labels (NxM expression matrix)
    Function created for loading mouse cell atlas data formats
    http://bis.zju.edu.cn/MCA/contact.html (DGE matrices)
    """
    assert datapath[-4:] == '.txt'
    with open(datapath) as f:
        count = 0
        # do first pass to get gene count and cell names
        for idx, line in enumerate(f):
            if idx == 0:
                line = line.rstrip()
                line = line.split('\t')
                cell_names = [a.strip('"') for a in line]
            else:
                count += 1
        arr = np.zeros((count, len(cell_names)), dtype=np.int16)  # max size ~ 33k (unsigned twice that)
        gene_names = [0] * count
        print "data size will be (genes x cells):", arr.shape
        # second pass to get gene names and array data
        f.seek(0)
        for idx, line in enumerate(f):
            if idx > 0:
                if idx % 1000 == 0:
                    print idx
                line = line.rstrip()
                line = line.split('\t')
                gene_names[idx-1] = line[0].strip('"')
                arr[idx-1, :] = [int(val) for val in line[1:]]
    if verbose:
        print "Loaded dim %s data at %s" % (arr.shape, datapath)
        print "Max val in arr:", np.max(arr)
        print "Size of arr in memory (bytes)", arr.nbytes

    datadir = os.path.abspath(os.path.join(datapath, os.pardir))
    if save_as_sep_npy:
        np.save(datadir + os.sep + "raw_arr.npy", arr)
        np.save(datadir + os.sep + "raw_genes.npy", np.array(gene_names))
        np.save(datadir + os.sep + "raw_cells.npy", np.array(cell_names))
    else:
        genes = np.array(gene_names)
        cells = np.array(cell_names)
        np.savez_compressed(datadir + os.sep + "arr_genes_cells_raw_compressed.npz", arr=arr, genes=genes, cells=cells)
    return arr, gene_names, cell_names

def load_singlecell_data(zscore_datafile=MEHTA_ZSCORE_DATAFILE_PATH, savenpz='mems_genes_clusters_raw_compressed'):
    """
    Returns list of cell types (size p), list of TFs (size N), and xi array where xi_ij is ith TF value in cell type j
    Note the Mehta SI file has odd formatting (use regex to parse); array text file is read in as single line:
    http://journals.plos.org/ploscompbiol/article/file?id=10.1371/journal.pcbi.1003734.s005&type=supplementary
    """
    gene_labels = []
    with open(zscore_datafile) as f:
        origline = f.readline()
        filelines = origline.split('\r')
        for idx_row, row in enumerate(filelines):
            row_split = re.split(r'\t', row)
            if idx_row == 0:  # celltypes line, first row
                celltype_labels = row_split[1:]
            else:
                gene_labels.append(row_split[0])
    # reloop to get data without excessive append calls
    expression_data = np.zeros((len(gene_labels), len(celltype_labels)))
    with open(zscore_datafile) as f:
        origline = f.readline()
        filelines_dataonly = origline.split('\r')[1:]
        for idx_row, row in enumerate(filelines_dataonly):
            row_split_dataonly = re.split(r'\t', row)[1:]
            expression_data[idx_row,:] = [float(val) for val in row_split_dataonly]
    if savenpz is not None:
        datadir = os.path.abspath(os.path.join(zscore_datafile, os.pardir))
        npzpath = datadir + os.sep + savenpz
        save_npz_of_arr_genes_cells(npzpath, expression_data, gene_labels, celltype_labels)
    return expression_data, gene_labels, celltype_labels

def load_npz_of_arr_genes_cells(npzpath, verbose=True):
    """
    Can also use for memory array, gene labels, and cell cluster names!
    """
    if verbose:
        print "loading npz of arr genes cells at", npzpath, "..."
    loaded = np.load(npzpath)
    arr = loaded['arr']
    genes = loaded['genes']
    cells = loaded['cells']
    if verbose:
        print "loaded arr, genes, cells:", arr.shape, genes.shape, cells.shape
    return arr, genes, cells

def binarize_data(xi):
    return 1.0 * np.where(xi > 0, 1, -1)  # mult by 1.0 to cast as float

def save_npz_of_arr_genes_cells(npzpath, arr, genes, cells):
    np.savez_compressed(npzpath, arr=arr, genes=genes, cells=cells)
    return

def attach_cluster_id_arr_manual(npzpath, clusterpath, save=True, one_indexed=True):
    """
    one_indexed: if true, assume cluster index starts at 1 (as in scMCA)
    """
    arr, genes, cells = load_npz_of_arr_genes_cells(npzpath)
    # generate cell_to_cluster_idx mapping
    cluster_info = {}
    # expected format of csv file is "cell name, cluster idx, tissue origin"
    with open(clusterpath) as f:
        for idx, line in enumerate(f):
            line = line.rstrip()
            line = line.split(',')
            cluster_info[line[0]] = int(line[1])
    # adjust genes and arr contents
    arr = np.insert(arr, 0, 0, axis=0)
    genes = np.insert(genes, 0, 'cluster_id')  # TODO should have global constant for this mock gene label
    if one_indexed:
        for idx in xrange(len(cells)):
            arr[0,idx] = cluster_info[cells[idx]] - 1
    else:
        for idx in xrange(len(cells)):
            arr[0,idx] = cluster_info[cells[idx]]
    # save and return data
    if save:
        print "saving cluster-appended arrays..."
        np.savez_compressed(datadir + os.sep + "arr_genes_cells_withcluster_compressed.npz", arr=arr, genes=genes, cells=cells)
    return arr, genes, cells

def prune_boring_rows(npzpath, specified_rows=None, save=True):
    """
    Delete rows from array and corresponding genes that are self-duplicates
    NOTE: very similar to reduce_gene_set(xi, gene_labels)
    """
    arr, genes, cells = load_npz_of_arr_genes_cells(npzpath)
    print "CHECK FIRST ROW NOT CLUSTER ROW:", genes[0]
    if specified_rows is None:
        # collect rows to delete
        rows_duplicates = np.all(arr.T == arr.T[0,:], axis=0)
        rows_to_delete = np.array([idx for idx, val in enumerate(rows_duplicates) if val])
        # note pruned rows
        print "number of self-duplicate rows:", len(rows_to_delete)
        print rows_to_delete
        print rows_to_delete[0:10]
    else:
        rows_to_delete = np.array(specified_rows)
    # adjust genes and arr contents
    print "Orig shape arr, genes, cells:", arr.shape, genes.shape, cells.shape
    arr = np.delete(arr, rows_to_delete, axis=0)
    genes = np.delete(genes, rows_to_delete)  # TODO should have global constant for this mock gene label
    print "New shape arr, genes, cells:", arr.shape, genes.shape, cells.shape
    # save and return data
    if save:
        print "saving pruned arrays..."
        datadir = os.path.abspath(os.path.join(npzpath, os.pardir))
        base = os.path.basename(npzpath)
        basestr = os.path.splitext(base)[0]
        savestr = basestr + '_pruned.npz'
        np.savez_compressed(datadir + os.sep + savestr, arr=arr, genes=genes, cells=cells)
    return rows_to_delete, arr, genes, cells

def reduce_gene_set(xi, gene_labels):  # TODO: my removal ends with 1339 left but theirs with 1337 why?
    """
    NOTE: very similar to prune_boring_rows(...)
    """
    genes_to_remove = []
    for row_idx, row in enumerate(xi):
        if all(map(lambda x: x == row[0], row)):
            genes_to_remove.append(row_idx)
    reduced_gene_labels = [gene_labels[idx] for idx in xrange(len(xi)) if idx not in genes_to_remove]
    reduced_xi = np.array([row for idx, row in enumerate(xi) if idx not in genes_to_remove])
    return reduced_gene_labels, reduced_xi

def load_cluster_labels(clusterpath, one_indexed=True):
    """
    one_indexed: if true, assume cluster index starts at 1 (as in scMCA)
    """
    cluster_labels = {}
    if one_indexed:
        dec = 1
    else:
        dec = 0
    # expected format of csv file is "cluster number, name"
    with open(clusterpath) as f:
        for idx, line in enumerate(f):
            line = line.rstrip()
            line = line.split(',')
            print line
            cluster_labels[int(line[0])-dec] = line[1]
    return cluster_labels

def parse_exptdata(states_raw, gene_labels, verbose=True):
    """
    Args:
        - states_raw: stores array of state data and cluster labels for each cell state (column)
        - gene_labels: stores row names i.e. gene or PCA labels
    Notes: data format may change with time
        - convention is first row stores cluster index, from 0 to np.max(row 0) == K - 1
        - future convention may be to store unique integer ID for each column corresponding to earlier in pipeline
        - maybe also extract and pass metadata_dict info (i.e. K, N, M, filename information on pipeline)
    Returns:
        - cluster_dict: {cluster_idx: N x M array of raw cell states in the cluster (i.e. not binarized)}
        - metadata: dict, mainly stores N x 1 array of 'gene_labels' for each row
    """
    states_row0 = states_raw[0, :]
    states_truncated = states_raw[1:, :]
    num_genes, num_cells = states_truncated.shape  # aka N, M
    num_clusters = np.max(states_raw[0, :]) + 1
    if verbose:
        print "raw data dimension: %d x %d" % (states_raw.shape)
        print "cleaned data dimension: %d x %d" % (states_truncated.shape)
        print "num_clusters is %d" % num_clusters

    # process gene labels
    assert len(gene_labels) == num_genes or len(gene_labels) == num_genes + 1
    if len(gene_labels) == num_genes + 1:
        assert 'cluster' in gene_labels[0]
        gene_labels = gene_labels[1:]

    # prep cluster_dict
    cluster_dict = {}
    cluster_indices = {k: [] for k in xrange(num_clusters)}
    # TODO optimize this chunk if needed
    for cell_idx in xrange(num_cells):
        cluster_idx = states_row0[cell_idx]
        cluster_indices[cluster_idx].append(cell_idx)

    # build cluster dict
    if verbose:
        print "cluster_indices collected; building cluster arrays..."
    for k in xrange(num_clusters):
        print k
        cluster_dict[k] = states_truncated.take(cluster_indices[k], axis=1)

    # fill metatadata dict
    metadata = {}
    metadata['gene_labels'] = gene_labels
    metadata['num_clusters'] = num_clusters
    metadata['K'] = num_clusters
    metadata['num_genes'] = num_genes
    metadata['N'] = num_genes
    metadata['num_cells'] = num_cells
    metadata['M'] = num_cells
    return cluster_dict, metadata

if __name__ == '__main__':
    # run flags
    datadir = DATADIR + os.sep + "2018_scMCA"
    flag_load_simple = False
    flag_process_manual = False
    flag_load_sep_npy = False
    flag_load_compressed_npz = False
    flag_attach_clusters_resave = False
    flag_prune_boring_rows = False
    flag_process_data_mehta = False

    # simple data load
    if flag_load_simple:
        datapath = datadir + os.sep + "SI_Figure2-batch-removed.txt"
        arr = read_datafile_simple(datapath, verbose=True, txt=True)

    if flag_process_manual:
        datapath = datadir + os.sep + "SI_Figure2-batch-removed.txt"
        arr, genes, cells = read_datafile_manual(datapath, verbose=True)

    if flag_load_sep_npy:
        arrpath = datadir + os.sep + "raw_arr.npy"
        genespath = datadir + os.sep + "raw_genes.npy"
        cellspath = datadir + os.sep + "raw_cells.npy"
        arr = read_datafile_simple(arrpath, verbose=True)
        genes = read_datafile_simple(genespath, verbose=True)
        cells = read_datafile_simple(cellspath, verbose=True)

    if flag_load_compressed_npz:
        compressed_file = datadir + os.sep + "arr_genes_cells_raw_compressed.npz"
        arr, genes, cells = load_npz_of_arr_genes_cells(compressed_file)

    if flag_attach_clusters_resave:
        compressed_file = datadir + os.sep + "arr_genes_cells_raw_compressed.npz"
        clusterpath = datadir + os.sep + "SI_cells_to_clusters.csv"
        arr, genes, cells = attach_cluster_id_arr_manual(compressed_file, clusterpath, save=True)

    if flag_prune_boring_rows:
        compressed_file = datadir + os.sep + "arr_genes_cells_raw_compressed.npz"
        prune_boring_rows(compressed_file)

    if flag_process_data_mehta:
        # part 1: load their zscore textfile, save in standard npz format
        expression_data, genes, celltypes = load_singlecell_data(zscore_datafile=MEHTA_ZSCORE_DATAFILE_PATH,
                                                                 savenpz='mehta_mems_genes_types_zscore_compressed.npz')
        # part 2: load npz, binarize, save
        npzpath = DATADIR + os.sep + "2014_mehta" + os.sep + 'mehta_mems_genes_types_zscore_compressed.npz'
        expression_data, genes, celltypes = load_npz_of_arr_genes_cells(npzpath, verbose=True)
        xi = binarize_data(expression_data)
        compressed_boolean = datadir + os.sep + "mehta_mems_genes_types_boolean_compressed.npz"
        save_npz_of_arr_genes_cells(compressed_boolean, xi, genes, celltypes)
        # part 3: load npz, prune, save
        rows_to_delete, xi, genes, celltypes = prune_boring_rows(compressed_boolean)
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Copyright 2016 Feei. All Rights Reserved
#
# Author:   Feei <wufeifei@wufeifei.com>
# Homepage: https://github.com/wufeifei/cobra
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See the file 'doc/COPYING' for copying permission
#
import ConfigParser
import os
import sys
import time
import subprocess

from flask import Flask
from flask_migrate import MigrateCommand, Migrate
from flask_script import Manager, Server, Option, Command
from flask_sqlalchemy import SQLAlchemy
from flask_bootstrap import Bootstrap

from utils import log

reload(sys)
sys.setdefaultencoding('utf-8')

template = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'templates')
asset = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'templates/asset')
web = Flask(__name__, template_folder=template, static_folder=asset)

config = ConfigParser.ConfigParser()
config.read('config')
web.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = True
web.config['SQLALCHEMY_DATABASE_URI'] = config.get('database', 'mysql')
web.secret_key = config.get('cobra', 'secret_key')

bootstrap = Bootstrap(web)

db = SQLAlchemy(web)

# just use the migration script's app context when you import the models
# http://stackoverflow.com/questions/33905706/flask-migrate-seemed-to-delete-all-my-database-data
with web.app_context():
    from models import *

migrate = Migrate(web, db)
manager = Manager(web)

class Statistic(Command):
    option_list = (
        Option('--target', '-t', dest='target', help='directory'),
        Option('--tid', '-i', dest='tid', help='scan task id')
    )

    def run(self, target=None, tid=None):
        if target is None:
            log.critical("Please set --target param")
            sys.exit()
        if tid is None:
            log.critical("Please set --tid param")
            sys.exit()

        # Statistic Code
        p = subprocess.Popen(
            ['cloc', target], stdout=subprocess.PIPE)
        (output, err) = p.communicate()
        rs = output.split("\n")
        for r in rs:
            r_e = r.split()
            if len(r_e) > 3 and r_e[0] == 'SUM:':
                t = CobraTaskInfo.query.filter_by(id=tid).first()
                if t is not None:
                    t.code_number = r_e[4]
                    try:
                        db.session.add(t)
                        db.session.commit()
                        log.info("Statistic code number done")
                    except Exception as e:
                        log.error("Statistic code number failed" + str(e.message))

class Scan(Command):
    option_list = (
        Option('--target', '-t', dest='target', help='scan target(directory/git repository/svn url/file path)'),
        Option('--tid', '-i', dest='tid', help='scan task id'),
        Option('--pid', '-p', dest='pid', help='scan project id'),
    )

    def parse_target(self, target=None):
        if target[len(target) - 4:] == '.git':
            return 'git'
        elif os.path.isdir(target) is True:
            return 'directory'
        elif os.path.isfile(target) is True:
            filename, file_extension = os.path.splitext(target)
            if file_extension in ['.tar.gz', '.rar', '.zip']:
                return 'compress'
            else:
                return 'file'
        elif target[0:7] == 'http://' or target[0:8] == 'https://':
            return 'svn'
        else:
            return False

    def run(self, target=None, tid=None, pid=None):
        if target is None:
            log.critical("Please set --target param")
            sys.exit()
        if tid is not None:
            task_id = tid
            # Start Time For Task
            t = CobraTaskInfo.query.filter_by(id=tid).first()
            if t is None:
                log.critical("Task id doesn't exists.")
                sys.exit()
            if t.status not in [0, 1]:
                log.critical("Task Already Scan.")
                sys.exit()
            t.status = 1
            t.time_start = int(time.time())
            t.updated_at = time.strftime('%Y-%m-%d %X', time.localtime())
            try:
                db.session.add(t)
                db.session.commit()
            except Exception as e:
                log.error("Set start time failed" + str(e.message))
        else:
            task_id = None

        target_type = self.parse_target(target)
        if target_type is False:
            log.error("""
                Git Repository: must .git end
                SVN Repository: can http:// or https://
                Directory: must be local directory
                File: must be single file or tar.gz/zip/rar compress file
                """)
        from engine import static
        s = static.Static(target, task_id=task_id, project_id=pid)
        if target_type is 'directory':
            s.analyse()
        elif target_type is 'compress':
            from utils.decompress import Decompress
            # load an compressed file. only tar.gz, rar, zip supported.
            dc = Decompress(target)
            # decompress it. And there will create a directory named "222_test.tar".
            dc.decompress()
            s.analyse()
        elif target_type is 'file':
            s.analyse()
        elif target_type is 'git':
            from pickup.GitTools import Git
            g = Git(target, branch='master')
            g.get_repo()
            if g.clone() is True:
                s.analyse()
            else:
                log.critical("Git clone failed")
        elif target_type is 'svn':
            log.warning("Not Support SVN Repository")

host = config.get('cobra', 'host')
port = config.get('cobra', 'port')
port = int(port)

manager.add_command('db', MigrateCommand)
manager.add_command('start', Server(host=host, port=port))
manager.add_command('scan', Scan())
manager.add_command('statistic', Statistic())

# frontend and api
from app.controller import route
from app.controller import api

# backend
from app.controller.backend import BackendAPIController
from app.controller.backend import DashboardController
from app.controller.backend import IndexController
from app.controller.backend import LanguagesController
from app.controller.backend import ProjectsController
from app.controller.backend import RulesController
from app.controller.backend import SearchController
from app.controller.backend import TasksController
from app.controller.backend import VulsController
from app.controller.backend import WhiteListsController

log.info('Cobra Engine Started')
<commit_msg>替换bootstrap.css等静态文件从CDN到本地<commit_after>#!/usr/bin/env python
#
# Copyright 2016 Feei. All Rights Reserved
#
# Author:   Feei <wufeifei@wufeifei.com>
# Homepage: https://github.com/wufeifei/cobra
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See the file 'doc/COPYING' for copying permission
#
import ConfigParser
import os
import sys
import time
import subprocess

from flask import Flask
from flask_migrate import MigrateCommand, Migrate
from flask_script import Manager, Server, Option, Command
from flask_sqlalchemy import SQLAlchemy
from flask_bootstrap import Bootstrap

from utils import log

reload(sys)
sys.setdefaultencoding('utf-8')

template = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'templates')
asset = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'templates/asset')
web = Flask(__name__, template_folder=template, static_folder=asset)

config = ConfigParser.ConfigParser()
config.read('config')
web.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = True
web.config['SQLALCHEMY_DATABASE_URI'] = config.get('database', 'mysql')
web.secret_key = config.get('cobra', 'secret_key')

bootstrap = Bootstrap(web)

# set bootstrap css and jquery js to local
web.config['BOOTSTRAP_SERVE_LOCAL'] = True

db = SQLAlchemy(web)

# just use the migration script's app context when you import the models
# http://stackoverflow.com/questions/33905706/flask-migrate-seemed-to-delete-all-my-database-data
with web.app_context():
    from models import *

migrate = Migrate(web, db)
manager = Manager(web)

class Statistic(Command):
    option_list = (
        Option('--target', '-t', dest='target', help='directory'),
        Option('--tid', '-i', dest='tid', help='scan task id')
    )

    def run(self, target=None, tid=None):
        if target is None:
            log.critical("Please set --target param")
            sys.exit()
        if tid is None:
            log.critical("Please set --tid param")
            sys.exit()

        # Statistic Code
        p = subprocess.Popen(
            ['cloc', target], stdout=subprocess.PIPE)
        (output, err) = p.communicate()
        rs = output.split("\n")
        for r in rs:
            r_e = r.split()
            if len(r_e) > 3 and r_e[0] == 'SUM:':
                t = CobraTaskInfo.query.filter_by(id=tid).first()
                if t is not None:
                    t.code_number = r_e[4]
                    try:
                        db.session.add(t)
                        db.session.commit()
                        log.info("Statistic code number done")
                    except Exception as e:
                        log.error("Statistic code number failed" + str(e.message))

class Scan(Command):
    option_list = (
        Option('--target', '-t', dest='target', help='scan target(directory/git repository/svn url/file path)'),
        Option('--tid', '-i', dest='tid', help='scan task id'),
        Option('--pid', '-p', dest='pid', help='scan project id'),
    )

    def parse_target(self, target=None):
        if target[len(target) - 4:] == '.git':
            return 'git'
        elif os.path.isdir(target) is True:
            return 'directory'
        elif os.path.isfile(target) is True:
            filename, file_extension = os.path.splitext(target)
            if file_extension in ['.tar.gz', '.rar', '.zip']:
                return 'compress'
            else:
                return 'file'
        elif target[0:7] == 'http://' or target[0:8] == 'https://':
            return 'svn'
        else:
            return False

    def run(self, target=None, tid=None, pid=None):
        if target is None:
            log.critical("Please set --target param")
            sys.exit()
        if tid is not None:
            task_id = tid
            # Start Time For Task
            t = CobraTaskInfo.query.filter_by(id=tid).first()
            if t is None:
                log.critical("Task id doesn't exists.")
                sys.exit()
            if t.status not in [0, 1]:
                log.critical("Task Already Scan.")
                sys.exit()
            t.status = 1
            t.time_start = int(time.time())
            t.updated_at = time.strftime('%Y-%m-%d %X', time.localtime())
            try:
                db.session.add(t)
                db.session.commit()
            except Exception as e:
                log.error("Set start time failed" + str(e.message))
        else:
            task_id = None

        target_type = self.parse_target(target)
        if target_type is False:
            log.error("""
                Git Repository: must .git end
                SVN Repository: can http:// or https://
                Directory: must be local directory
                File: must be single file or tar.gz/zip/rar compress file
                """)
        from engine import static
        s = static.Static(target, task_id=task_id, project_id=pid)
        if target_type is 'directory':
            s.analyse()
        elif target_type is 'compress':
            from utils.decompress import Decompress
            # load an compressed file. only tar.gz, rar, zip supported.
            dc = Decompress(target)
            # decompress it. And there will create a directory named "222_test.tar".
            dc.decompress()
            s.analyse()
        elif target_type is 'file':
            s.analyse()
        elif target_type is 'git':
            from pickup.GitTools import Git
            g = Git(target, branch='master')
            g.get_repo()
            if g.clone() is True:
                s.analyse()
            else:
                log.critical("Git clone failed")
        elif target_type is 'svn':
            log.warning("Not Support SVN Repository")

host = config.get('cobra', 'host')
port = config.get('cobra', 'port')
port = int(port)

manager.add_command('db', MigrateCommand)
manager.add_command('start', Server(host=host, port=port))
manager.add_command('scan', Scan())
manager.add_command('statistic', Statistic())

# frontend and api
from app.controller import route
from app.controller import api

# backend
from app.controller.backend import BackendAPIController
from app.controller.backend import DashboardController
from app.controller.backend import IndexController
from app.controller.backend import LanguagesController
from app.controller.backend import ProjectsController
from app.controller.backend import RulesController
from app.controller.backend import SearchController
from app.controller.backend import TasksController
from app.controller.backend import VulsController
from app.controller.backend import WhiteListsController

log.info('Cobra Engine Started')
<|endoftext|>


<commit_before>import tw2.core as twc
import datetime

from fedoracommunity.widgets.grid import Grid
from fedoracommunity.connectors.api import get_connector

class BugStatsWidget(twc.Widget):
    template = "mako:fedoracommunity.widgets.package.templates.bugs_stats_widget"
    id = twc.Param(default='bugs_widget')
    kwds = twc.Param(default=None)
    product = twc.Param(default='Fedora')
    version = twc.Param(default='rawhide')
    package = twc.Param(default=None)
    num_closed = twc.Param(default='-')
    num_open = twc.Param(default='-')
    num_new = twc.Param(default='-')
    num_new_this_week = twc.Param(default='')
    num_closed_this_week = twc.Param(default='')

    bz_prefix = "https://bugzilla.redhat.com/buglist.cgi"
    status_open_string = "bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED"
    status_closed_string = "bug_status=CLOSED"

    base_query_string = twc.Variable(default='')
    open_query_string = twc.Variable(default='')
    closed_query_string = twc.Variable(default='')

    def prepare(self):
        super(BugStatsWidget, self).prepare()
        def to_query_string(query):
            return "&".join([
                "{key}={value}".format(key=key, value=value)
                for key, value in query.items()
            ])
        self.base_query_string = to_query_string({
            "query_format": "advanced",
            "product": self.product,
            "component": self.package,
        })
        self.open_query_string = to_query_string({
            "chfieldto": "Now",
            "chfield": "[Bug creation]",
            "chfieldfrom": datetime.datetime.now().isoformat().split('T')[0],
        })
        self.closed_query_string = to_query_string({
            "chfieldto": "Now",
            "chfield": "bug_status",
            "chfieldvalue": "CLOSED",
            "chfieldfrom": datetime.datetime.now().isoformat().split('T')[0],
        })

class BugsGrid(Grid):
    resource = 'bugzilla'
    resource_path = 'query_bugs'
    release_table = twc.Param()
    package = twc.Param()
    template = "mako:fedoracommunity.widgets.package.templates.bugs_table_widget"

    def prepare(self):
        releases = []
        self.filters = {'package': self.package}
        pkgdb = get_connector('pkgdb')
        collections = pkgdb.get_collection_table(active_only=True)

        for id, collection in collections.items():
            name = collection['name']
            ver = collection['version']
            label = "%s %s" % (name, ver)
            value = str(ver)
            if ver == 'devel':
                name = 'Rawhide'
                ver = 9999999
                label = 'Rawhide'
                value = 'rawhide'

            if name in ('Fedora', 'Rawhide', 'Fedora EPEL'):
                releases.append({'label': label, 'value': value, 'version': ver})

        def _sort(a,b):
            return cmp(int(b['version']), int(a['version']))

        releases.sort(_sort)
        self.release_table = releases

        super(BugsGrid, self).prepare()

class BugsWidget(twc.Widget):
    bug_stats = BugStatsWidget
    bug_grid = BugsGrid
    kwds = twc.Param()
    package = twc.Param()
    template = "mako:fedoracommunity.widgets.package.templates.bugs"

    def prepare(self):
        super(BugsWidget, self).prepare()
        self.package = self.kwds['package_name']
        self.main_package = self.kwds.get('subpackage_of', '')
        if not self.main_package:
            self.main_package = self.package
<commit_msg>Support for the {package}/bugs/all path.  Fixes #390.<commit_after>import tw2.core as twc
import datetime

from fedoracommunity.widgets.grid import Grid
from fedoracommunity.connectors.api import get_connector

class BugStatsWidget(twc.Widget):
    template = "mako:fedoracommunity.widgets.package.templates.bugs_stats_widget"
    id = twc.Param(default='bugs_widget')
    kwds = twc.Param(default=None)
    product = twc.Param(default='Fedora')
    version = twc.Param(default='rawhide')
    package = twc.Param(default=None)
    num_closed = twc.Param(default='-')
    num_open = twc.Param(default='-')
    num_new = twc.Param(default='-')
    num_new_this_week = twc.Param(default='')
    num_closed_this_week = twc.Param(default='')

    bz_prefix = "https://bugzilla.redhat.com/buglist.cgi"
    status_open_string = "bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED"
    status_closed_string = "bug_status=CLOSED"

    base_query_string = twc.Variable(default='')
    open_query_string = twc.Variable(default='')
    closed_query_string = twc.Variable(default='')

    def prepare(self):
        super(BugStatsWidget, self).prepare()
        def to_query_string(query):
            return "&".join([
                "{key}={value}".format(key=key, value=value)
                for key, value in query.items()
            ])
        self.base_query_string = to_query_string({
            "query_format": "advanced",
            "product": self.product,
            "component": self.package,
        })
        self.open_query_string = to_query_string({
            "chfieldto": "Now",
            "chfield": "[Bug creation]",
            "chfieldfrom": datetime.datetime.now().isoformat().split('T')[0],
        })
        self.closed_query_string = to_query_string({
            "chfieldto": "Now",
            "chfield": "bug_status",
            "chfieldvalue": "CLOSED",
            "chfieldfrom": datetime.datetime.now().isoformat().split('T')[0],
        })

class BugsGrid(Grid):
    resource = 'bugzilla'
    resource_path = 'query_bugs'
    release_table = twc.Param()
    package = twc.Param()
    template = "mako:fedoracommunity.widgets.package.templates.bugs_table_widget"

    def prepare(self):
        releases = []
        self.filters = {'package': self.package}
        pkgdb = get_connector('pkgdb')
        collections = pkgdb.get_collection_table(active_only=True)

        for id, collection in collections.items():
            name = collection['name']
            ver = collection['version']
            label = "%s %s" % (name, ver)
            value = str(ver)
            if ver == 'devel':
                name = 'Rawhide'
                ver = 9999999
                label = 'Rawhide'
                value = 'rawhide'

            if name in ('Fedora', 'Rawhide', 'Fedora EPEL'):
                releases.append({'label': label, 'value': value, 'version': ver})

        def _sort(a,b):
            return cmp(int(b['version']), int(a['version']))

        releases.sort(_sort)
        self.release_table = releases

        super(BugsGrid, self).prepare()

class BugsWidget(twc.Widget):
    bug_stats = BugStatsWidget
    bug_grid = BugsGrid
    kwds = twc.Param()
    package = twc.Param()
    template = "mako:fedoracommunity.widgets.package.templates.bugs"

    def prepare(self):
        super(BugsWidget, self).prepare()
        self.package = self.kwds['package_name']
        self.main_package = self.kwds.get('subpackage_of', '')

        if not self.main_package:
            self.main_package = self.package

        # This is here so you can hit packages/kernel/bugs/all
        if self.args == ['all']:
            self.children[1].rows_per_page = 100000
<|endoftext|>


<commit_before>from sympy import bernoulli, Symbol, symbols, Sum, harmonic, Rational, oo, \
    zoo, pi, I, bell, fibonacci, lucas, euler, catalan, \
    binomial, gamma, sqrt, hyper, log, polygamma, diff, \
    Expr, sympify

from sympy.utilities.pytest import XFAIL, raises

x = Symbol('x')

def test_bernoulli():
    assert bernoulli(0) == 1
    assert bernoulli(1) == Rational(-1, 2)
    assert bernoulli(2) == Rational(1, 6)
    assert bernoulli(3) == 0
    assert bernoulli(4) == Rational(-1, 30)
    assert bernoulli(5) == 0
    assert bernoulli(6) == Rational(1, 42)
    assert bernoulli(7) == 0
    assert bernoulli(8) == Rational(-1, 30)
    assert bernoulli(10) == Rational(5, 66)
    assert bernoulli(1000001) == 0

    assert bernoulli(0, x) == 1
    assert bernoulli(1, x) == x - Rational(1, 2)
    assert bernoulli(2, x) == x**2 - x + Rational(1, 6)
    assert bernoulli(3, x) == x**3 - (3*x**2)/2 + x/2

    # Should be fast; computed with mpmath
    b = bernoulli(1000)
    assert b.p % 10**10 == 7950421099
    assert b.q == 342999030

    b = bernoulli(10**6, evaluate=False).evalf()
    assert str(b) == '-2.23799235765713e+4767529'

def test_fibonacci():
    assert [fibonacci(n) for n in range(-3, 5)] == [2, -1, 1, 0, 1, 1, 2, 3]
    assert fibonacci(100) == 354224848179261915075
    assert [lucas(n) for n in range(-3, 5)] == [-4, 3, -1, 2, 1, 3, 4, 7]
    assert lucas(100) == 792070839848372253127

    assert fibonacci(1, x) == 1
    assert fibonacci(2, x) == x
    assert fibonacci(3, x) == x**2 + 1
    assert fibonacci(4, x) == x**3 + 2*x

def test_bell():
    assert [bell(n) for n in range(8)] == [1, 1, 2, 5, 15, 52, 203, 877]

    assert bell(0, x) == 1
    assert bell(1, x) == x
    assert bell(2, x) == x**2 + x
    assert bell(5, x) == x**5 + 10*x**4 + 25*x**3 + 15*x**2 + x

    X = symbols('x:6')
    # X = (x0, x1, .. x5)
    # at the same time: X[1] = x1, X[2] = x2 for standard readablity.
    # but we must supply zero-based indexed object X[1:] = (x1, .. x5)

    assert bell(6, 2, X[1:]) == 6*X[5]*X[1] + 15*X[4]*X[2] + 10*X[3]**2
    assert bell(
        6, 3, X[1:]) == 15*X[4]*X[1]**2 + 60*X[3]*X[2]*X[1] + 15*X[2]**3

    X = (1, 10, 100, 1000, 10000)
    assert bell(6, 2, X) == (6 + 15 + 10)*10000

    X = (1, 2, 3, 3, 5)
    assert bell(6, 2, X) == 6*5 + 15*3*2 + 10*3**2

    X = (1, 2, 3, 5)
    assert bell(6, 3, X) == 15*5 + 60*3*2 + 15*2**3

def test_harmonic():
    assert harmonic(1, 1) == 1
    assert harmonic(2, 1) == Rational(3, 2)
    assert harmonic(3, 1) == Rational(11, 6)
    assert harmonic(4, 1) == Rational(25, 12)
    assert harmonic(3, 1) == harmonic(3)
    assert harmonic(3, 5) == 1 + Rational(1, 2**5) + Rational(1, 3**5)
    assert harmonic(10, 0) == 10
    assert harmonic(oo, 1) == zoo
    assert harmonic(oo, 2) == (pi**2)/6

def test_euler():
    assert euler(0) == 1
    assert euler(1) == 0
    assert euler(2) == -1
    assert euler(3) == 0
    assert euler(4) == 5
    assert euler(6) == -61
    assert euler(8) == 1385

    assert euler(20, evaluate=False) != 370371188237525

    n = Symbol('n', integer=True)
    assert euler(n) != -1
    assert euler(n).subs(n, 2) == -1

    assert euler(20).evalf() == 370371188237525.0
    assert euler(20, evaluate=False).evalf() == 370371188237525.0

    assert euler(n).rewrite(Sum) == euler(n)
    # XXX: Not sure what the guy who wrote this test was trying to do with the _j and _k stuff
    assert euler(2*n + 1).rewrite(Sum) == 0

@XFAIL
def test_euler_failing():
    # depends on dummy variables being implemented http://code.google.com/p/sympy/issues/detail?id=2566
    assert euler(2*n).rewrite(Sum) == I*Sum(Sum((-1)**_j*2**(-_k)*I**(-_k)*(-2*_j + _k)**(2*n + 1)*binomial(_k, _j)/_k, (_j, 0, _k)), (_k, 1, 2*n + 1))

def test_catalan():
    assert catalan(1) == 1
    assert catalan(2) == 2
    assert catalan(3) == 5
    assert catalan(4) == 14

    assert catalan(x) == catalan(x)
    assert catalan(2*x).rewrite(binomial) == binomial(4*x, 2*x)/(2*x + 1)
    assert catalan(Rational(1, 2)).rewrite(gamma) == 8/(3*pi)
    assert catalan(3*x).rewrite(gamma) == 4**(
        3*x)*gamma(3*x + Rational(1, 2))/(sqrt(pi)*gamma(3*x + 2))
    assert catalan(x).rewrite(hyper) == hyper((-x + 1, -x), (2,), 1)

    assert diff(catalan(x), x) == (polygamma(
        0, x + Rational(1, 2)) - polygamma(0, x + 2) + 2*log(2))*catalan(x)

    c = catalan(0.5).evalf()
    assert str(c) == '0.848826363156775'

def test_nC_nP_nT():
    from sympy.utilities.iterables import (
        multiset_permutations, multiset_combinations, multiset_partitions,
        partitions, subsets, permutations)
    from sympy.functions.combinatorial.numbers import (
        nP, nC, nT, stirling, _data, _gen_poly)
    from sympy.combinatorics.permutations import Permutation
    from sympy.core.numbers import oo
    from random import choice
    from string import lowercase as c

    for i in range(100):
        s = ''.join(choice(c) for i in range(7))
        u = len(s) == len(set(s))
        try:
            tot = 0
            for i in range(8):
                check = nP(s, i)
                tot += check
                assert len(list(multiset_permutations(s, i))) == check
                assert tot == nP(s, -i)
                if u:
                    assert nP(len(s), i) == check
                    assert nP(len(s), -i) == tot
        except AssertionError:
            print s, i, 'failed perm test'
            raise ValueError()

    for i in range(100):
        s = ''.join(choice(c) for i in range(7))
        u = len(s) == len(set(s))
        try:
            tot = 0
            for i in range(8):
                check = nC(s, i)
                tot += check
                assert len(list(multiset_combinations(s, i))) == check
                assert tot == nC(s, -i)
                if u:
                    assert nC(len(s), i) == check
                    assert nC(len(s), -i) == tot
        except AssertionError:
            print s, i, 'failed combo test'
            raise ValueError()

    for i in range(1, 10):
        tot = 0
        for j in range(1, i + 2):
            check = nT(i, j)
            tot += check
            assert sum(1 for p in partitions(i, j, size=True) if p[0] == j) == check
            assert nT(i, -j) == tot

    for i in range(1, 10):
        tot = 0
        for j in range(1, i + 2):
            check = nT(range(i), j)
            tot += check
            assert len(list(multiset_partitions(range(i), j))) == check
            assert nT(range(i), -j) == tot

    for i in range(100):
        s = ''.join(choice(c) for i in range(7))
        u = len(s) == len(set(s))
        try:
            tot = 0
            for i in range(1, 8):
                check = nT(s, i)
                tot += check
                assert len(list(multiset_partitions(s, i))) == check
                assert nT(s, -i) == tot
                if u:
                    assert nT(range(len(s)), i) == check
                    assert nT(range(len(s)), -i) == tot
        except AssertionError:
            print s, i, 'failed partition test'
            raise ValueError()

    # tests for Stirling numbers of the first kind that are not tested in the
    # above
    assert [stirling(9, i, kind=1) for i in range(11)] == [
        0, 40320, 109584, 118124, 67284, 22449, 4536, 546, 36, 1, 0]
    perms = list(permutations(range(4)))
    assert [sum(1 for p in perms if Permutation(p).cycles == i)
            for i in range(5)] == [0, 6, 11, 6, 1] == [
            stirling(4, i, kind=1) for i in range(5)]

    def delta(p):
        if len(p) == 1:
            return oo
        return min(abs(i[0] - i[1]) for i in subsets(p, 2))
    parts = multiset_partitions(range(5), 3)
    d = 2
    assert (sum(1 for p in parts if all(delta(i) >= d for i in p)) ==
            stirling(5, 3, d=d) == 7)

    # other coverage tests
    assert nC('abb', 2) == nC('aab', 2) == 2
    assert nP(3, 3, replacement=True) == nP('aabc', 3, replacement=True) == 27
    assert nP(3, 4) == 0
    assert nP('aabc', 5) == 0
    assert nC(4, 2, replacement=True) == nC('abcdd', 2, replacement=True) == \
        len(list(multiset_combinations('aabbccdd', 2))) == 10
    assert nC('abcdd', -5) == sum(nC('abcdd', i) for i in range(6)) == 24
    assert nC(list('abcdd'), 4) == 4
    assert nT('aaaa') == nT(4) == len(list(partitions(4))) == 5
    assert nT('aaab') == len(list(multiset_partitions('aaab'))) == 7
    assert nC('aabb'*3, 3) == 4  # aaa, bbb, abb, baa
    assert dict(_gen_poly((4,1,1,1))) == {
        0: 1, 1: 4, 2: 7, 3: 8, 4: 8, 5: 7, 6: 4, 7: 1}
    raises(ValueError, lambda: nP('aabc', replacement=True))
    raises(ValueError, lambda: _data({1:'a'}))
<commit_msg>do not use lowercase in Py 3<commit_after>from sympy import bernoulli, Symbol, symbols, Sum, harmonic, Rational, oo, \
    zoo, pi, I, bell, fibonacci, lucas, euler, catalan, \
    binomial, gamma, sqrt, hyper, log, polygamma, diff, \
    Expr, sympify

from sympy.utilities.pytest import XFAIL, raises

x = Symbol('x')

def test_bernoulli():
    assert bernoulli(0) == 1
    assert bernoulli(1) == Rational(-1, 2)
    assert bernoulli(2) == Rational(1, 6)
    assert bernoulli(3) == 0
    assert bernoulli(4) == Rational(-1, 30)
    assert bernoulli(5) == 0
    assert bernoulli(6) == Rational(1, 42)
    assert bernoulli(7) == 0
    assert bernoulli(8) == Rational(-1, 30)
    assert bernoulli(10) == Rational(5, 66)
    assert bernoulli(1000001) == 0

    assert bernoulli(0, x) == 1
    assert bernoulli(1, x) == x - Rational(1, 2)
    assert bernoulli(2, x) == x**2 - x + Rational(1, 6)
    assert bernoulli(3, x) == x**3 - (3*x**2)/2 + x/2

    # Should be fast; computed with mpmath
    b = bernoulli(1000)
    assert b.p % 10**10 == 7950421099
    assert b.q == 342999030

    b = bernoulli(10**6, evaluate=False).evalf()
    assert str(b) == '-2.23799235765713e+4767529'

def test_fibonacci():
    assert [fibonacci(n) for n in range(-3, 5)] == [2, -1, 1, 0, 1, 1, 2, 3]
    assert fibonacci(100) == 354224848179261915075
    assert [lucas(n) for n in range(-3, 5)] == [-4, 3, -1, 2, 1, 3, 4, 7]
    assert lucas(100) == 792070839848372253127

    assert fibonacci(1, x) == 1
    assert fibonacci(2, x) == x
    assert fibonacci(3, x) == x**2 + 1
    assert fibonacci(4, x) == x**3 + 2*x

def test_bell():
    assert [bell(n) for n in range(8)] == [1, 1, 2, 5, 15, 52, 203, 877]

    assert bell(0, x) == 1
    assert bell(1, x) == x
    assert bell(2, x) == x**2 + x
    assert bell(5, x) == x**5 + 10*x**4 + 25*x**3 + 15*x**2 + x

    X = symbols('x:6')
    # X = (x0, x1, .. x5)
    # at the same time: X[1] = x1, X[2] = x2 for standard readablity.
    # but we must supply zero-based indexed object X[1:] = (x1, .. x5)

    assert bell(6, 2, X[1:]) == 6*X[5]*X[1] + 15*X[4]*X[2] + 10*X[3]**2
    assert bell(
        6, 3, X[1:]) == 15*X[4]*X[1]**2 + 60*X[3]*X[2]*X[1] + 15*X[2]**3

    X = (1, 10, 100, 1000, 10000)
    assert bell(6, 2, X) == (6 + 15 + 10)*10000

    X = (1, 2, 3, 3, 5)
    assert bell(6, 2, X) == 6*5 + 15*3*2 + 10*3**2

    X = (1, 2, 3, 5)
    assert bell(6, 3, X) == 15*5 + 60*3*2 + 15*2**3

def test_harmonic():
    assert harmonic(1, 1) == 1
    assert harmonic(2, 1) == Rational(3, 2)
    assert harmonic(3, 1) == Rational(11, 6)
    assert harmonic(4, 1) == Rational(25, 12)
    assert harmonic(3, 1) == harmonic(3)
    assert harmonic(3, 5) == 1 + Rational(1, 2**5) + Rational(1, 3**5)
    assert harmonic(10, 0) == 10
    assert harmonic(oo, 1) == zoo
    assert harmonic(oo, 2) == (pi**2)/6

def test_euler():
    assert euler(0) == 1
    assert euler(1) == 0
    assert euler(2) == -1
    assert euler(3) == 0
    assert euler(4) == 5
    assert euler(6) == -61
    assert euler(8) == 1385

    assert euler(20, evaluate=False) != 370371188237525

    n = Symbol('n', integer=True)
    assert euler(n) != -1
    assert euler(n).subs(n, 2) == -1

    assert euler(20).evalf() == 370371188237525.0
    assert euler(20, evaluate=False).evalf() == 370371188237525.0

    assert euler(n).rewrite(Sum) == euler(n)
    # XXX: Not sure what the guy who wrote this test was trying to do with the _j and _k stuff
    assert euler(2*n + 1).rewrite(Sum) == 0

@XFAIL
def test_euler_failing():
    # depends on dummy variables being implemented http://code.google.com/p/sympy/issues/detail?id=2566
    assert euler(2*n).rewrite(Sum) == I*Sum(Sum((-1)**_j*2**(-_k)*I**(-_k)*(-2*_j + _k)**(2*n + 1)*binomial(_k, _j)/_k, (_j, 0, _k)), (_k, 1, 2*n + 1))

def test_catalan():
    assert catalan(1) == 1
    assert catalan(2) == 2
    assert catalan(3) == 5
    assert catalan(4) == 14

    assert catalan(x) == catalan(x)
    assert catalan(2*x).rewrite(binomial) == binomial(4*x, 2*x)/(2*x + 1)
    assert catalan(Rational(1, 2)).rewrite(gamma) == 8/(3*pi)
    assert catalan(3*x).rewrite(gamma) == 4**(
        3*x)*gamma(3*x + Rational(1, 2))/(sqrt(pi)*gamma(3*x + 2))
    assert catalan(x).rewrite(hyper) == hyper((-x + 1, -x), (2,), 1)

    assert diff(catalan(x), x) == (polygamma(
        0, x + Rational(1, 2)) - polygamma(0, x + 2) + 2*log(2))*catalan(x)

    c = catalan(0.5).evalf()
    assert str(c) == '0.848826363156775'

def test_nC_nP_nT():
    from sympy.utilities.iterables import (
        multiset_permutations, multiset_combinations, multiset_partitions,
        partitions, subsets, permutations)
    from sympy.functions.combinatorial.numbers import (
        nP, nC, nT, stirling, _data, _gen_poly)
    from sympy.combinatorics.permutations import Permutation
    from sympy.core.numbers import oo
    from random import choice

    c = 'abcdefghijklmnopqrstuvwxyz'
    for i in range(100):
        s = ''.join(choice(c) for i in range(7))
        u = len(s) == len(set(s))
        try:
            tot = 0
            for i in range(8):
                check = nP(s, i)
                tot += check
                assert len(list(multiset_permutations(s, i))) == check
                assert tot == nP(s, -i)
                if u:
                    assert nP(len(s), i) == check
                    assert nP(len(s), -i) == tot
        except AssertionError:
            print s, i, 'failed perm test'
            raise ValueError()

    for i in range(100):
        s = ''.join(choice(c) for i in range(7))
        u = len(s) == len(set(s))
        try:
            tot = 0
            for i in range(8):
                check = nC(s, i)
                tot += check
                assert len(list(multiset_combinations(s, i))) == check
                assert tot == nC(s, -i)
                if u:
                    assert nC(len(s), i) == check
                    assert nC(len(s), -i) == tot
        except AssertionError:
            print s, i, 'failed combo test'
            raise ValueError()

    for i in range(1, 10):
        tot = 0
        for j in range(1, i + 2):
            check = nT(i, j)
            tot += check
            assert sum(1 for p in partitions(i, j, size=True) if p[0] == j) == check
            assert nT(i, -j) == tot

    for i in range(1, 10):
        tot = 0
        for j in range(1, i + 2):
            check = nT(range(i), j)
            tot += check
            assert len(list(multiset_partitions(range(i), j))) == check
            assert nT(range(i), -j) == tot

    for i in range(100):
        s = ''.join(choice(c) for i in range(7))
        u = len(s) == len(set(s))
        try:
            tot = 0
            for i in range(1, 8):
                check = nT(s, i)
                tot += check
                assert len(list(multiset_partitions(s, i))) == check
                assert nT(s, -i) == tot
                if u:
                    assert nT(range(len(s)), i) == check
                    assert nT(range(len(s)), -i) == tot
        except AssertionError:
            print s, i, 'failed partition test'
            raise ValueError()

    # tests for Stirling numbers of the first kind that are not tested in the
    # above
    assert [stirling(9, i, kind=1) for i in range(11)] == [
        0, 40320, 109584, 118124, 67284, 22449, 4536, 546, 36, 1, 0]
    perms = list(permutations(range(4)))
    assert [sum(1 for p in perms if Permutation(p).cycles == i)
            for i in range(5)] == [0, 6, 11, 6, 1] == [
            stirling(4, i, kind=1) for i in range(5)]

    def delta(p):
        if len(p) == 1:
            return oo
        return min(abs(i[0] - i[1]) for i in subsets(p, 2))
    parts = multiset_partitions(range(5), 3)
    d = 2
    assert (sum(1 for p in parts if all(delta(i) >= d for i in p)) ==
            stirling(5, 3, d=d) == 7)

    # other coverage tests
    assert nC('abb', 2) == nC('aab', 2) == 2
    assert nP(3, 3, replacement=True) == nP('aabc', 3, replacement=True) == 27
    assert nP(3, 4) == 0
    assert nP('aabc', 5) == 0
    assert nC(4, 2, replacement=True) == nC('abcdd', 2, replacement=True) == \
        len(list(multiset_combinations('aabbccdd', 2))) == 10
    assert nC('abcdd', -5) == sum(nC('abcdd', i) for i in range(6)) == 24
    assert nC(list('abcdd'), 4) == 4
    assert nT('aaaa') == nT(4) == len(list(partitions(4))) == 5
    assert nT('aaab') == len(list(multiset_partitions('aaab'))) == 7
    assert nC('aabb'*3, 3) == 4  # aaa, bbb, abb, baa
    assert dict(_gen_poly((4,1,1,1))) == {
        0: 1, 1: 4, 2: 7, 3: 8, 4: 8, 5: 7, 6: 4, 7: 1}
    raises(ValueError, lambda: nP('aabc', replacement=True))
    raises(ValueError, lambda: _data({1:'a'}))
<|endoftext|>


<commit_before>from datetime import timedelta, datetime

from flask import Flask, request, redirect
from flask.ext.bootstrap import Bootstrap
from flask_login import LoginManager
from flask_wtf.csrf import CsrfProtect
from dmutils import apiclient, init_app, flask_featureflags, formats
from dmutils.user import User
from dmutils.content_loader import ContentLoader

from config import configs

bootstrap = Bootstrap()
csrf = CsrfProtect()
data_api_client = apiclient.DataAPIClient()
feature_flags = flask_featureflags.FeatureFlag()
login_manager = LoginManager()
DISPLAY_DATE_FORMAT = '%d/%m/%Y'
DISPLAY_TIME_FORMAT = '%H:%M:%S'
DISPLAY_DATETIME_FORMAT = '%A, %d %B %Y at %H:%M'

service_content = ContentLoader(
    "app/section_order.yml", "app/content/g6/"
)

def create_app(config_name):

    application = Flask(__name__,
                        static_folder='static/',
                        static_url_path=configs[config_name].STATIC_URL_PATH)

    init_app(
        application,
        configs[config_name],
        bootstrap=bootstrap,
        data_api_client=data_api_client,
        feature_flags=feature_flags,
        login_manager=login_manager
    )
    # Should be incorporated into digitalmarketplace-utils as well
    csrf.init_app(application)

    application.permanent_session_lifetime = timedelta(hours=1)
    from .main import main as main_blueprint
    from .status import status as status_blueprint

    application.register_blueprint(status_blueprint, url_prefix='/admin')
    application.register_blueprint(main_blueprint, url_prefix='/admin')
    login_manager.login_view = 'main.render_login'
    main_blueprint.config = application.config.copy()

    @application.before_request
    def remove_trailing_slash():
        if request.path != '/' and request.path.endswith('/'):
            return redirect(request.path[:-1], code=301)

    @application.template_filter('timeformat')
    def timeformat(value):
        return datetime.strptime(
            value, formats.DATETIME_FORMAT).strftime(DISPLAY_TIME_FORMAT)

    @application.template_filter('dateformat')
    def dateformat(value):
        return datetime.strptime(
            value, formats.DATETIME_FORMAT).strftime(DISPLAY_DATE_FORMAT)

    @application.template_filter('displaydateformat')
    def display_date_format(value):
        return value.strftime(DISPLAY_DATE_FORMAT)

    return application

@login_manager.user_loader
def load_user(user_id):
    return User.load_user(user_id)
<commit_msg>Hotfix: Pass api client in to load_user method<commit_after>from datetime import timedelta, datetime

from flask import Flask, request, redirect
from flask.ext.bootstrap import Bootstrap
from flask_login import LoginManager
from flask_wtf.csrf import CsrfProtect
from dmutils import apiclient, init_app, flask_featureflags, formats
from dmutils.user import User
from dmutils.content_loader import ContentLoader

from config import configs

bootstrap = Bootstrap()
csrf = CsrfProtect()
data_api_client = apiclient.DataAPIClient()
feature_flags = flask_featureflags.FeatureFlag()
login_manager = LoginManager()
DISPLAY_DATE_FORMAT = '%d/%m/%Y'
DISPLAY_TIME_FORMAT = '%H:%M:%S'
DISPLAY_DATETIME_FORMAT = '%A, %d %B %Y at %H:%M'

service_content = ContentLoader(
    "app/section_order.yml", "app/content/g6/"
)

def create_app(config_name):

    application = Flask(__name__,
                        static_folder='static/',
                        static_url_path=configs[config_name].STATIC_URL_PATH)

    init_app(
        application,
        configs[config_name],
        bootstrap=bootstrap,
        data_api_client=data_api_client,
        feature_flags=feature_flags,
        login_manager=login_manager
    )
    # Should be incorporated into digitalmarketplace-utils as well
    csrf.init_app(application)

    application.permanent_session_lifetime = timedelta(hours=1)
    from .main import main as main_blueprint
    from .status import status as status_blueprint

    application.register_blueprint(status_blueprint, url_prefix='/admin')
    application.register_blueprint(main_blueprint, url_prefix='/admin')
    login_manager.login_view = 'main.render_login'
    main_blueprint.config = application.config.copy()

    @application.before_request
    def remove_trailing_slash():
        if request.path != '/' and request.path.endswith('/'):
            return redirect(request.path[:-1], code=301)

    @application.template_filter('timeformat')
    def timeformat(value):
        return datetime.strptime(
            value, formats.DATETIME_FORMAT).strftime(DISPLAY_TIME_FORMAT)

    @application.template_filter('dateformat')
    def dateformat(value):
        return datetime.strptime(
            value, formats.DATETIME_FORMAT).strftime(DISPLAY_DATE_FORMAT)

    @application.template_filter('displaydateformat')
    def display_date_format(value):
        return value.strftime(DISPLAY_DATE_FORMAT)

    return application

@login_manager.user_loader
def load_user(user_id):
    return User.load_user(data_api_client, user_id)
<|endoftext|>


<commit_before>from flask import Flask, render_template, json, jsonify
from app.wikipedia import get_search_articles, get_random_articles

app = Flask(__name__)

@app.route('/')
def render_random_content():
    return render_template('posts.html', articles=get_random_articles(5))

@app.route('/search/<query>')
@app.route('/category/<query>')
def show_posts(query):
    return render_template('posts.html', query=query, articles=get_search_articles(query))
<commit_msg>Remove unneeded requirements<commit_after>from flask import Flask, render_template
from app.wikipedia import get_search_articles, get_random_articles

app = Flask(__name__)

@app.route('/')
def show_random_posts():
    return render_template('articles.html', articles=get_random_articles(5))

@app.route('/search/<query>')
@app.route('/category/<query>')
def show_posts(query):
    return render_template('articles.html', query=query,
                           articles=get_search_articles(query))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    app
    ~~~

    Provides the flask application
"""
from __future__ import (
    absolute_import, division, print_function, with_statement,
    unicode_literals)

import config

from flask import Flask
from flask.ext.sqlalchemy import SQLAlchemy

__version__ = '0.6.2'

__title__ = 'hdxscraper-acled'
__author__ = 'Reuben Cummings'
__description__ = (
    'Armed Conflict Location & Event Data Project (ACLED) Realtime Data'
    ' Collector')

__email__ = 'reubano@gmail.com'
__license__ = 'MIT'
__copyright__ = 'Copyright 2015 Reuben Cummings'

db = SQLAlchemy()

def create_app(mode=None):
    app = Flask(__name__)
    db.init_app(app)

    if mode:
        app.config.from_object(getattr(config, mode))
    else:
        app.config.from_envvar('APP_SETTINGS', silent=True)

    return app
<commit_msg>Bump to version 0.7.0<commit_after># -*- coding: utf-8 -*-
"""
    app
    ~~~

    Provides the flask application
"""
from __future__ import (
    absolute_import, division, print_function, with_statement,
    unicode_literals)

import config

from flask import Flask
from flask.ext.sqlalchemy import SQLAlchemy

__version__ = '0.7.0'

__title__ = 'hdxscraper-acled'
__author__ = 'Reuben Cummings'
__description__ = (
    'Armed Conflict Location & Event Data Project (ACLED) Realtime Data'
    ' Collector')

__email__ = 'reubano@gmail.com'
__license__ = 'MIT'
__copyright__ = 'Copyright 2015 Reuben Cummings'

db = SQLAlchemy()

def create_app(mode=None):
    app = Flask(__name__)
    db.init_app(app)

    if mode:
        app.config.from_object(getattr(config, mode))
    else:
        app.config.from_envvar('APP_SETTINGS', silent=True)

    return app
<|endoftext|>


<commit_before>'''
Copyright (c) 2013, Agora Games, LLC All rights reserved.

https://github.com/agoragames/torus/blob/master/LICENSE.txt
'''

import re
import time

import ujson
from urlparse import *
from gevent.pywsgi import WSGIServer

from .util import parse_time
from .exceptions import *

FUNC_MATCH = re.compile('^(?P<func>[a-zA-Z0-9_]+)\((?P<stat>[^\)]+)\)$')

def extract(dct, transform):
  '''
  Recursively extract the transformed data from a given dictionary.
  '''
  # If we're at the point where we've found the transformed data, return it
  if transform in dct:
    return dct[transform]

  rval = type(dct)()
  for k,v in dct.iteritems():
    rval[k] = extract(v, transform)
  return rval

class Web(WSGIServer):
  '''
  Web server to mine data out of kairos.
  '''

  def __init__(self, **kwargs):
    '''
    Initialize with the given configuration and start the server.
    '''
    self.host = kwargs.get('host', '')
    self.port = kwargs.get('port', 8080)
    self._configuration = kwargs.get('configuration')
    super(Web,self).__init__( (self.host,int(self.port)), self.handle_request, log=None )

  def handle_request(self, env, start_response):
    cmd = env['PATH_INFO'][1:]
    if cmd.endswith('/'):
      cmd = cmd[:-1]
    params = parse_qs( env['QUERY_STRING'] )

    try:
      if cmd == 'series':
        result = self._series(params)

      elif cmd == 'list':
        result = self._list(params)

      elif cmd == 'properties':
        result = self._properties(params)

      else:
        raise HttpNotFound()

      start_response('200 OK', [('content-type','application/json')] )
      return [ ujson.dumps(result, double_precision=4) ]

    except HttpError as e:
      start_response( '%s %s'%(e.code, e.msg),
        [('content-type','application/json')] )
      return []

    except Exception as e:
      import traceback
      traceback.print_exc()
      start_response( '500 Internal Server Error',
        [('content-type','application/json')] )
      return []

    start_response( '404 Not Found', [('content-type','application/json')] )
    return []

  def _list(self, params):
    '''
    Return a list of all stored stat names.
    '''
    # Future versions may add an "extended" view that includes properties.
    schema_name = params.get('schema',[None])[0]
    rval = set()

    if schema_name:
      schema = self._configuration.schema(schema_name)
      if not schema:
        raise HttpNotFound()
      rval.update( schema.list() )
    else:
      for schema in self._configuration.schemas():
        rval.update( schema.list() )
    return sorted(rval)

  def _properties(self, params):
    '''
    Fetch the properties of a stat.
    '''
    rval = {}

    for stat in params['stat']:
      rval.setdefault( stat, {} )
      for schema in self._configuration.schemas(stat):
        rval[stat][schema.name] = schema.properties(stat)

    return rval

  def _series(self, params):
    '''
    Handle the data URL.
    '''
    rval = []

    format = params.setdefault('format',['graphite'])[0]
    condense = False
    fetch = None
    process_row = None
    join_rows = None

    # Force condensed data for graphite return
    if format=='graphite':
      condense = True
    else:
      condense = bool(params.get('condense',[False])[0])

    collapse = bool(params.get('collapse',[False])[0])

    # If start or end times are defined, process them
    start = params.get('start', [''])[0]
    end = params.get('end', [''])[0]
    if start:
      start = parse_time(start)
    if end:
      end = parse_time(end)

    steps = int(params.get('steps',[0])[0])
    schema_name = params.get('schema',[None])[0]
    interval = params.get('interval',[None])[0]

    # First assemble the unique stats and the functions.
    stat_queries = {}
    for stat_spec in params['stat']:
      func_match = FUNC_MATCH.match(stat_spec)
      if func_match:
        func_name = func_match.groupdict()['func']
        stat = func_match.groupdict()['stat']
      else:
        if format=='graphite':
          func_name = 'mean'
        else:
          func_name = None
        stat = stat_spec

      stat = tuple(stat.split(','))
      stat_queries.setdefault( stat, {} )

      # First process as a macro
      if func_name:
        macro = self._configuration.macro(func_name)
        if macro:
          format = macro.get( 'format', format )
          fetch = macro.get( 'fetch' )
          process_row = macro.get( 'process_row' )
          join_rows = macro.get( 'join_rows' )
          condense = macro.get( 'condense', condense )
          collapse = macro.get( 'collapse', collapse )
          start = macro.get( 'start', start )
          end = macro.get( 'end', end )
          steps = macro.get( 'steps', steps )
          func_name = macro.get( 'transform' )
          schema_name = macro.get( 'schema', schema_name )
          interval = macro.get( 'interval', interval )
          if start:
            start = parse_time(start)
          if end:
            end = parse_time(end)

      # If not a macro, or the macro has defined its own transform
      if func_name:
        func = self._configuration.transform(func_name) or func_name
        stat_queries[stat][stat_spec] = (func_name, func)
      else:
        # essentially a "null" transform, we'll get our data back
        stat_queries[stat][stat_spec] = (None, None)

    # For each unique stat, walk trough all the schemas until we find one that
    # matches the stat and has a matching interval if one is specified. If there
    # isn't one specified, then pick the first match and the first interval.
    for stat,specs in stat_queries.iteritems():
      schema = self._configuration.schema(schema_name)

      # If user-requested schema (or none) not found, try to find one.
      if not schema and not schema_name:
        schemas = self._configuration.schemas(stat)
        for schema in schemas:
          if interval in schema.config['intervals'].keys():
            break
        else:
          # No matching interval found, so if there were any schemas and the
          # user didn't define an interval, try to find one.
          if schema and not interval:
            interval = schema.config['intervals'].keys()[0]

      # If user-requested schema found, resolve interval if necessary
      elif not interval:
        interval = schema.config['intervals'].keys()[0]

      # No schema found, return an empty data set for each query
      # on that stat
      if not schema:
        for spec,transform in specs.items():
          rval.append( {
            'stat' : spec,
            'stat_name' : stat,
            'function' : transform[0],
            'target' : stat,  # graphite compatible key
            'datapoints' : []
          } )
        continue

      # Filter out the unique transforms
      transforms = specs.values()
      if transforms==[(None,None)]:
        transforms = None
      else:
        transforms = [ t[1] for t in transforms ]

      start = start or None
      end = end or None

      data = schema.timeseries.series(stat, interval,
        condense=condense, transform=transforms,
        fetch=fetch, process_row=process_row, join_rows=join_rows,
        start=start, end=end, steps=steps, collapse=collapse)

      # If there were any transforms, then that means there's a list to append
      # for each matching stat, else there's just a single value.
      if transforms:
        for spec,transform in specs.iteritems():
          # This transposition of the way in which kairos returns the
          # transforms and how torus presents it is most unfortunate.
          # In both cases I prefer the format for its given role.
          rval.append( {
            'stat' : spec,
            'stat_name' : stat,
            'function' : transform[0],
            'target' : stat,  # graphite compatible key
            'schema' : schema.name,
            'interval' : interval,
            'datapoints' : extract(data, transform[1]),
          } )
      else:
        rval.append( {
          'stat' : specs.keys()[0],
          'stat_name' : stat,
          'target' : stat,  # graphite compatible key
          'schema' : schema.name,
          'interval' : interval,
          'datapoints' : data,
        } )

    return rval
<commit_msg>Adding /insert endpoint to torus<commit_after>'''
Copyright (c) 2013, Agora Games, LLC All rights reserved.

https://github.com/agoragames/torus/blob/master/LICENSE.txt
'''

import re
import time

import ujson
from urlparse import *
from gevent.pywsgi import WSGIServer

from .util import parse_time
from .exceptions import *

FUNC_MATCH = re.compile('^(?P<func>[a-zA-Z0-9_]+)\((?P<stat>[^\)]+)\)$')

def extract(dct, transform):
  '''
  Recursively extract the transformed data from a given dictionary.
  '''
  # If we're at the point where we've found the transformed data, return it
  if transform in dct:
    return dct[transform]

  rval = type(dct)()
  for k,v in dct.iteritems():
    rval[k] = extract(v, transform)
  return rval

class Web(WSGIServer):
  '''
  Web server to mine data out of kairos.
  '''

  def __init__(self, **kwargs):
    '''
    Initialize with the given configuration and start the server.
    '''
    self.host = kwargs.get('host', '')
    self.port = kwargs.get('port', 8080)
    self._configuration = kwargs.get('configuration')
    super(Web,self).__init__( (self.host,int(self.port)), self.handle_request, log=None )

  def handle_request(self, env, start_response):
    cmd = env['PATH_INFO'][1:]
    if cmd.endswith('/'):
      cmd = cmd[:-1]
    params = parse_qs( env['QUERY_STRING'] )

    try:
      if cmd == 'series':
        result = self._series(params)

      elif cmd == 'list':
        result = self._list(params)

      elif cmd == 'properties':
        result = self._properties(params)

      elif cmd == 'insert':
        result = self._insert(params)

      else:
        raise HttpNotFound()

      start_response('200 OK', [('content-type','application/json')] )
      if result is not None:
        return [ ujson.dumps(result, double_precision=4) ]
      return []

    except HttpError as e:
      start_response( '%s %s'%(e.code, e.msg),
        [('content-type','application/json')] )
      return []

    except Exception as e:
      import traceback
      traceback.print_exc()
      start_response( '500 Internal Server Error',
        [('content-type','application/json')] )
      return []

    start_response( '404 Not Found', [('content-type','application/json')] )
    return []

  def _list(self, params):
    '''
    Return a list of all stored stat names.
    '''
    # Future versions may add an "extended" view that includes properties.
    schema_name = params.get('schema',[None])[0]
    rval = set()

    if schema_name:
      schema = self._configuration.schema(schema_name)
      if not schema:
        raise HttpNotFound()
      rval.update( schema.list() )
    else:
      for schema in self._configuration.schemas():
        rval.update( schema.list() )
    return sorted(rval)

  def _properties(self, params):
    '''
    Fetch the properties of a stat.
    '''
    rval = {}

    for stat in params['stat']:
      rval.setdefault( stat, {} )
      for schema in self._configuration.schemas(stat):
        rval[stat][schema.name] = schema.properties(stat)

    return rval

  def _insert(self, params):
    '''
    Insert a data point
    '''
    stat = params['stat'][0]
    value = params['value'][0]
    timestamp = long(float( params.get('timestamp',[time.time()])[0] ))

    self._configuration.process(stat, value, timestamp)

  def _series(self, params):
    '''
    Handle the data URL.
    '''
    rval = []

    format = params.setdefault('format',['graphite'])[0]
    condense = False
    fetch = None
    process_row = None
    join_rows = None

    # Force condensed data for graphite return
    if format=='graphite':
      condense = True
    else:
      condense = bool(params.get('condense',[False])[0])

    collapse = bool(params.get('collapse',[False])[0])

    # If start or end times are defined, process them
    start = params.get('start', [''])[0]
    end = params.get('end', [''])[0]
    if start:
      start = parse_time(start)
    if end:
      end = parse_time(end)

    steps = int(params.get('steps',[0])[0])
    schema_name = params.get('schema',[None])[0]
    interval = params.get('interval',[None])[0]

    # First assemble the unique stats and the functions.
    stat_queries = {}
    for stat_spec in params['stat']:
      func_match = FUNC_MATCH.match(stat_spec)
      if func_match:
        func_name = func_match.groupdict()['func']
        stat = func_match.groupdict()['stat']
      else:
        if format=='graphite':
          func_name = 'mean'
        else:
          func_name = None
        stat = stat_spec

      stat = tuple(stat.split(','))
      stat_queries.setdefault( stat, {} )

      # First process as a macro
      if func_name:
        macro = self._configuration.macro(func_name)
        if macro:
          format = macro.get( 'format', format )
          fetch = macro.get( 'fetch' )
          process_row = macro.get( 'process_row' )
          join_rows = macro.get( 'join_rows' )
          condense = macro.get( 'condense', condense )
          collapse = macro.get( 'collapse', collapse )
          start = macro.get( 'start', start )
          end = macro.get( 'end', end )
          steps = macro.get( 'steps', steps )
          func_name = macro.get( 'transform' )
          schema_name = macro.get( 'schema', schema_name )
          interval = macro.get( 'interval', interval )
          if start:
            start = parse_time(start)
          if end:
            end = parse_time(end)

      # If not a macro, or the macro has defined its own transform
      if func_name:
        func = self._configuration.transform(func_name) or func_name
        stat_queries[stat][stat_spec] = (func_name, func)
      else:
        # essentially a "null" transform, we'll get our data back
        stat_queries[stat][stat_spec] = (None, None)

    # For each unique stat, walk trough all the schemas until we find one that
    # matches the stat and has a matching interval if one is specified. If there
    # isn't one specified, then pick the first match and the first interval.
    for stat,specs in stat_queries.iteritems():
      schema = self._configuration.schema(schema_name)

      # If user-requested schema (or none) not found, try to find one.
      if not schema and not schema_name:
        schemas = self._configuration.schemas(stat)
        for schema in schemas:
          if interval in schema.config['intervals'].keys():
            break
        else:
          # No matching interval found, so if there were any schemas and the
          # user didn't define an interval, try to find one.
          if schema and not interval:
            interval = schema.config['intervals'].keys()[0]

      # If user-requested schema found, resolve interval if necessary
      elif not interval:
        interval = schema.config['intervals'].keys()[0]

      # No schema found, return an empty data set for each query
      # on that stat
      if not schema:
        for spec,transform in specs.items():
          rval.append( {
            'stat' : spec,
            'stat_name' : stat,
            'function' : transform[0],
            'target' : stat,  # graphite compatible key
            'datapoints' : []
          } )
        continue

      # Filter out the unique transforms
      transforms = specs.values()
      if transforms==[(None,None)]:
        transforms = None
      else:
        transforms = [ t[1] for t in transforms ]

      start = start or None
      end = end or None

      data = schema.timeseries.series(stat, interval,
        condense=condense, transform=transforms,
        fetch=fetch, process_row=process_row, join_rows=join_rows,
        start=start, end=end, steps=steps, collapse=collapse)

      # If there were any transforms, then that means there's a list to append
      # for each matching stat, else there's just a single value.
      if transforms:
        for spec,transform in specs.iteritems():
          # This transposition of the way in which kairos returns the
          # transforms and how torus presents it is most unfortunate.
          # In both cases I prefer the format for its given role.
          rval.append( {
            'stat' : spec,
            'stat_name' : stat,
            'function' : transform[0],
            'target' : stat,  # graphite compatible key
            'schema' : schema.name,
            'interval' : interval,
            'datapoints' : extract(data, transform[1]),
          } )
      else:
        rval.append( {
          'stat' : specs.keys()[0],
          'stat_name' : stat,
          'target' : stat,  # graphite compatible key
          'schema' : schema.name,
          'interval' : interval,
          'datapoints' : data,
        } )

    return rval
<|endoftext|>


<commit_before>#!/usr/bin/env python

from __future__ import print_function

import pickle
import sklearn
import numpy as np
import glob
import os
import neukrill_net.utils as utils
import neukrill_net.image_processing as image_processing
import neukrill_net.bagofwords as bagofwords

import sklearn.preprocessing
import sklearn.ensemble
import sklearn.linear_model
import sklearn.cross_validation
import sklearn.dummy
from sklearn.externals import joblib
import sklearn.metrics

def main():

    # this should be parsed from json, but hardcoded for now
    bow_options = {'verbose':True, 'n_features_max':500, 'patch_size':31, 'clusteralgo':'kmeans', 'n_clusters':300, 'random_seed':42}

    # Load the settings, providing
    settings = utils.Settings('settings.json')

    # Load the raw data
    print('Loading the raw training data')
    rawdata, labels = utils.load_rawdata(settings.image_fnames, classes=settings.classes)

    # Encode the labels
    label_encoder = sklearn.preprocessing.LabelEncoder()
    y = label_encoder.fit_transform(labels)

    # Probably not the best classifier
    clf = sklearn.linear_model.LogisticRegression()

    cv = sklearn.cross_validation.StratifiedShuffleSplit(y)

    bow = bagofwords.Bow(**bow_options)
    sample = np.random.random_integers(0, len(rawdata)-1, size=(3000)) # Subsample so we can do this in sensible time
    bow.build_vocabulary([rawdata[i] for i in sample])
    #bow.build_vocabulary(rawdata)
    print('Bagging words for raw training data')
    X = [bow.compute_image_bow(img) for img in rawdata]
    X = np.vstack(X)

    # Try cross-validating
    print('Cross-validating')
    results = []
    for train, test in cv:
        # Make a new BOW encoding
        #bow = bagofwords.Bow(**bow_options)
        #bow.build_vocabulary([rawdata[i] for i in train])
        #X = [bow.compute_image_bow(img) for img in rawdata]

        clf.fit(X[train], y[train])
        p = clf.predict_proba(X[test])
        results.append(sklearn.metrics.log_loss(y[test], p))

    print(results)
    print('CV average = {}'.format(np.mean(results)))

    # Train on the whole thing and save model for later
    #bow = bagofwords.Bow(**bow_options)
    #bow.build_vocabulary(rawdata)
    #X = [bow.compute_image_bow(img) for img in rawdata]

    clf.fit(X,y)

    print('Loading the raw test data')
    rawtest, names = utils.load_rawdata(settings.image_fnames)
    print('Bagging words for raw test data')
    X2 = [bow.compute_image_bow(img) for img in rawdata]
    X2 = np.vstack(X2)

    p = clf.predict_proba(X2)

    utils.write_predictions('submission_bow_initial.csv', p, names, settings)

if __name__=='__main__':
    main()
<commit_msg>Working for initial BOW submission<commit_after>#!/usr/bin/env python

from __future__ import print_function

import pickle
import sklearn
import numpy as np
import glob
import os
import neukrill_net.utils as utils
import neukrill_net.image_processing as image_processing
import neukrill_net.bagofwords as bagofwords

import sklearn.preprocessing
import sklearn.ensemble
import sklearn.linear_model
import sklearn.cross_validation
import sklearn.dummy
from sklearn.externals import joblib
import sklearn.metrics

def main():

    # this should be parsed from json, but hardcoded for now
    bow_options = {'verbose':True, 'normalise_hist':False, 'n_features_max':100, 'patch_size':15, 'clusteralgo':'kmeans', 'n_clusters':20, 'random_seed':42}

    # Load the settings, providing
    settings = utils.Settings('settings.json')

    # Load the raw data
    print('Loading the raw training data')
    rawdata, labels = utils.load_rawdata(settings.image_fnames, classes=settings.classes)

    # Encode the labels
    label_encoder = sklearn.preprocessing.LabelEncoder()
    y = label_encoder.fit_transform(labels)

    # Probably not the best classifier
    clf = sklearn.linear_model.LogisticRegression()

    cv = sklearn.cross_validation.StratifiedShuffleSplit(y)

    bow = bagofwords.Bow(**bow_options)
    sample = np.random.random_integers(0, len(rawdata)-1, size=(1000)) # Subsample so we can do this in sensible time
    bow.build_vocabulary([rawdata[i] for i in sample])
    #bow.build_vocabulary(rawdata)
    print('Bagging words for raw training data')
    X = [bow.compute_image_bow(img) for img in rawdata]
    X = np.vstack(X)

    # Try cross-validating
    print('Cross-validating')
    results = []
    for train, test in cv:
        # Make a new BOW encoding
        #bow = bagofwords.Bow(**bow_options)
        #bow.build_vocabulary([rawdata[i] for i in train])
        #X = [bow.compute_image_bow(img) for img in rawdata]

        clf.fit(X[train], y[train])
        p = clf.predict_proba(X[test])
        res = sklearn.metrics.log_loss(y[test], p)
        print(res)
        results.append(res)

    print(results)
    print('CV average = {}'.format(np.mean(results)))

    # Train on the whole thing and save model for later
    #bow = bagofwords.Bow(**bow_options)
    #bow.build_vocabulary(rawdata)
    #X = [bow.compute_image_bow(img) for img in rawdata]

    clf.fit(X,y)

    print('Loading the raw test data')
    rawtest, names = utils.load_rawdata(settings.image_fnames)
    print('Bagging words for raw test data')
    X2 = [bow.compute_image_bow(img) for img in rawtest]
    X2 = np.vstack(X2)

    p = clf.predict_proba(X2)

    utils.write_predictions('submission_bow_initial.csv', p, names, settings)

if __name__=='__main__':
    main()
<|endoftext|>


<commit_before># __init__.py
#
# Copyright(c) Exequiel Ceasar Navarrete <esnavarrete1@up.edu.ph>
# Licensed under MIT
# Version 0.0.0

import os
from flask import Flask, jsonify
from flask_environments import Environments
from flask_sqlalchemy import SQLAlchemy
from flask_marshmallow import Marshmallow

# instantiate the application
app = Flask(__name__)
env = Environments(app)
env.from_yaml(os.path.join(os.getcwd(), 'conf/main.yml'))
db = SQLAlchemy(app)
ma = Marshmallow(app)

@app.errorhandler(404)
def page_not_found(e):
  return jsonify(error=404, text=str(e.description), success=False)

# Flask Views
from app.views import regions, crops, provinces

# Flask Blueprints
app.register_blueprint(regions.mod)
app.register_blueprint(crops.mod)
app.register_blueprint(provinces.mod)

@app.route("/")
def main():
  return "Welcome to the API!"

<commit_msg>Setup application caching<commit_after># __init__.py
#
# Copyright(c) Exequiel Ceasar Navarrete <esnavarrete1@up.edu.ph>
# Licensed under MIT
# Version 0.0.0

import os
import sys
from flask import Flask, jsonify
from flask_environments import Environments
from flask_sqlalchemy import SQLAlchemy
from flask_marshmallow import Marshmallow
from flask_caching import Cache
from oauth2client.service_account import ServiceAccountCredentials

# instantiate the application
app = Flask(__name__)
env = Environments(app)
env.from_yaml(os.path.join(os.getcwd(), 'conf/main.yml'))
db = SQLAlchemy(app)
ma = Marshmallow(app)

# setup caching
cache = Cache(app, config={
  'CACHE_TYPE': app.config['CACHE']['TYPE'],
  'CACHE_DIR': os.path.join(os.getcwd(), app.config['CACHE']['DIRECTORY'])
})

@app.errorhandler(404)
def page_not_found(e):
  return jsonify(error=404, text=str(e.description), success=False)

# Flask Views
from app.views import regions, crops, provinces

# Flask Blueprints
app.register_blueprint(regions.mod)
app.register_blueprint(crops.mod)
app.register_blueprint(provinces.mod)

@app.route("/")
def main():
  return "Welcome to the API!"

<|endoftext|>