<commit_before>#!/usr/bin/python

from multiprocessing import Pool
import numpy as np, pandas as pd
import os

# Set defaults
dataDir = '/afs/cbs.mpg.de/projects/mar005_lsd-lemon-surf/probands'
fsDir = '/afs/cbs.mpg.de/projects/mar004_lsd-lemon-preproc/freesurfer'
out_file = '/scr/liberia1/data/individual_dist_label/res_individual_dist_label_lh_20150714.txt'

### !!! hemi needs to be changed within function !!! ###

# ----------------------------------- functions -----------------------------------

def run_individual_dist_label(subject):

    hemi = 'lh'
    import os, glob, subprocess, h5py
    import numpy as np, pandas as pd, nibabel as nib
    import nipype.interfaces.freesurfer as fs
    from surfer import Brain
    from sklearn.utils.arpack import eigsh

    dataDir = '/afs/cbs.mpg.de/projects/mar005_lsd-lemon-surf/probands'
    fsDir = '/afs/cbs.mpg.de/projects/mar004_lsd-lemon-preproc/freesurfer'
    outDir = '/scr/liberia1/data/individual_dist_label'

    def img2disc(data, foci=False, labelfile=False, hemi='lh', filename='temp.png'):
        brain = Brain('fsaverage5', hemi, 'inflated', curv=False)
        brain.add_data(data, data.min(), data.max(), colormap="spectral", alpha=0.6)
        if labelfile:
            brain.add_label(labelfile, borders=True, color='grey')
        if foci:
            brain.add_foci(foci, coords_as_verts=True, scale_factor=.7, color='black')
        brain.save_montage(filename, order=['lat', 'med'], orientation='h', border_size=10)

    def runFiedler(conn):
        # https://github.com/margulies/topography
        # prep for embedding
        K = (conn + 1) / 2.
        v = np.sqrt(np.sum(K, axis=1))
        A = K/(v[:, None] * v[None, :])
        del K
        A = np.squeeze(A * [A > 0])
        # diffusion embedding
        n_components_embedding = 2
        lambdas, vectors = eigsh(A, k=n_components_embedding+1)
        del A
        lambdas = lambdas[::-1]
        vectors = vectors[:, ::-1]
        psi = vectors/vectors[:, 0][:, None]
        lambdas = lambdas[1:] / (1 - lambdas[1:])
        embedding = psi[:, 1:(n_components_embedding + 1 + 1)] * lambdas[:n_components_embedding+1][None, :]
        return embedding

    def runMasking(data, hemi):
        mask = np.zeros((10242))
        for label in [39, 40, 46, 47, 49, 50, 51, 68, 85, 86]:
            label = np.sort(nib.freesurfer.io.read_label(glob.glob('%s/fsaverage5/label/*%s*label*' % (fsDir, hemi))[label]))
            mask[label] = 1
        masked = data * mask
        return masked

    def runSmoothing(data, hemi, subject):
        temp1 = './temp1_%s.mgz' % subject
        temp2 = './temp2_%s.mgz' % subject
        img = np.expand_dims(data, axis=0)
        img = np.expand_dims(img, axis=0)
        img = nib.freesurfer.mghformat.MGHImage(img.astype(float32), affine=None)
        img.to_filename(temp1)
        smoothing = fs.SurfaceSmooth(subjects_dir=fsDir,
                                     subject_id='fsaverage5',
                                     in_file=temp1,
                                     out_file=temp2,
                                     hemi=hemi,
                                     fwhm=5,
                                     cortex=True,
                                     terminal_output='none')
        smoothing.run()
        out = nib.load(temp2).get_data().squeeze()
        os.remove(temp1)
        os.remove(temp2)
        return out

    def runExtrema(data, hemi, subject):
        temp1 = './temp_%s.mgz' % subject
        temp2 = './temp_%s.log' % subject
        thmin = (abs(data).max() - 1.3*abs(data).std())
        cluster = np.array([x if x > thmin else 0 for x in abs(data)])
        cluster_img = np.expand_dims(cluster, axis=0)
        cluster_img = np.expand_dims(cluster_img, axis=0)
        cluster_img = nib.freesurfer.mghformat.MGHImage(cluster_img.astype(float32), affine=None)
        cluster_img.to_filename(temp1)
        cml = 'mri_surfcluster --in %s --subject fsaverage5 --hemi %s --thmin %s --annot aparc.a2009s --sum %s' % (temp1, hemi, thmin, temp2)
        subprocess.call(cml, shell=True)
        extrema_log = pd.read_csv(temp2, skiprows=34, skipinitialspace=21, header=None, dtype={0:np.str})
        extrema_vertices = [int(extrema_log[0].iloc[i][15:25]) for i in range(len(extrema_log))]
        os.remove(temp1)
        os.remove(temp2)
        return extrema_vertices

    # file names and location
    corr_file1 = '%s/%s/correlation_maps/%s_lsd_corr_1ab_fsa5_%s.npy' % (dataDir, subject, subject, hemi)
    corr_file2 = '%s/%s/correlation_maps/%s_lsd_corr_2ab_fsa5_%s.npy' % (dataDir, subject, subject, hemi)
    dist_file = '%s/%s/distance_maps/%s_%s_geoDist_fsa5.mat' % (dataDir, subject, subject, hemi)
    parietal_label_file = '%s/%s/labels/fsa5/%s.G_pariet_inf-Angular_fsa5.label' % (dataDir, subject, hemi)
    temporal_label_file = '%s/%s/labels/fsa5/%s.Pole_temporal_fsa5.label' % (dataDir, subject, hemi)
    V1_label_file = '%s/%s/labels/fsa5/%s.S_calcarine_fsa5.label' % (dataDir, subject, hemi)
    A1_label_file = '%s/%s/labels/fsa5/%s.G_temp_sup-G_T_transv_fsa5.label' % (dataDir, subject, hemi)
    fiedler_file = '%s/fiedler/%s_fiedler_%s' % (outDir, subject, hemi)
    peak_img_file = '%s/qc/%s_fiedler_dmnExtrema_%s.png' % (outDir, subject, hemi)

    try:
        #if not False in [os.path.isfile(i) for i in [corr_file, dist_file, parietal_label_file, temporal_label_file, V1_label_file, A1_label_file]]:
        # read in data
        cort = np.sort(nib.freesurfer.io.read_label('%s/fsaverage5/label/%s.cortex.label' % (fsDir, hemi)))
        corr1 = np.load(corr_file1)
        corr2 = np.load(corr_file2)
        corr = (corr1+corr2) /2
        with h5py.File(dist_file, 'r') as f:
            dist = f['dataAll'][()]
        parietal_vertices = np.sort(nib.freesurfer.io.read_label(parietal_label_file))
        temppole_vertices = np.sort(nib.freesurfer.io.read_label(temporal_label_file))
        V1_vertices = np.sort(nib.freesurfer.io.read_label(V1_label_file))
        A1_vertices = np.sort(nib.freesurfer.io.read_label(A1_label_file))

        # local extrema in fiedler vector
        fiedler = np.zeros(len(corr))
        fiedler[cort] = runFiedler(corr[cort, :][:, cort])[:,0]
        del corr
        f_smoothed = runSmoothing(fiedler, hemi, subject)
        f_masked = runMasking(f_smoothed, hemi)
        f_extrema_vertices = runExtrema(f_masked, hemi, subject)

        # distances
        dist_extrema_2_parietal = [np.mean(dist[parietal_vertices, i]) for i in f_extrema_vertices]
        parietal_peak_vertex = f_extrema_vertices[dist_extrema_2_parietal.index(min(dist_extrema_2_parietal))]
        dist_extrema_2_temporal = [np.mean(dist[temppole_vertices, i]) for i in f_extrema_vertices]
        temporal_peak_vertex = f_extrema_vertices[dist_extrema_2_temporal.index(min(dist_extrema_2_temporal))]

        # save standardized fiedler
        if fiedler[parietal_peak_vertex] < 0:
            f_stand = -fiedler
        else:
            f_stand = fiedler

        np.save(fiedler_file, f_stand)
        img2disc(f_stand, foci=[parietal_peak_vertex, temporal_peak_vertex], hemi=hemi, filename=peak_img_file)

        # return results
        V1_vertices = nib.freesurfer.io.read_label('%s/%s/labels/fsa5/%s.S_calcarine_fsa5.label' % (dataDir, subject, hemi))
        A1_vertices = nib.freesurfer.io.read_label('%s/%s/labels/fsa5/%s.G_temp_sup-G_T_transv_fsa5.label' % (dataDir, subject, hemi))
        V1_parietal = dist[V1_vertices, parietal_peak_vertex].mean()
        A1_parietal = dist[A1_vertices, parietal_peak_vertex].mean()
        V1_temporal = dist[V1_vertices, temporal_peak_vertex].mean()
        A1_temporal = dist[A1_vertices, temporal_peak_vertex].mean()
        return subject, hemi, V1_parietal, A1_parietal, V1_temporal, A1_temporal

    except:
        return subject, hemi, None, None, None, None
        pass

# --------------------------------------------------------------------------------------------------------------------------------------------------

### run serially ###

subjects = [sub for sub in os.listdir(dataDir) if os.path.isdir(os.path.join(dataDir, sub))]
output_dict = {}
res = []

for subject in subjects:
    res.append(run_individual_dist_label(subject))

output_dict['subject'], output_dict['hemi'], output_dict['V1_parietal'], output_dict['A1_parietal'], output_dict['V1_temporal'], output_dict['A1_temporal'] = np.array(res).T
pd.DataFrame(output_dict).to_csv(out_file, sep='\t', index=False, columns=['subject', 'hemi', 'V1_parietal', 'A1_parietal', 'V1_temporal', 'A1_temporal'])

### run in parallel ### (not good for qc screenshots though)

#p = Pool(20)
#res = p.map(run_individual_dist_label, subjects)
#output_dict = {}
#output_dict['subject'], output_dict['hemi'], output_dict['V1_parietal'], output_dict['A1_parietal'], output_dict['V1_temporal'], output_dict['A1_temporal'] = np.array(res).T
#pd.DataFrame(output_dict).to_csv(out_file, sep='\t', index=False, columns=['subject', 'hemi', 'V1_parietal', 'A1_parietal', 'V1_temporal', 'A1_temporal'])<commit_msg>added new diffusion embedding<commit_after>#!/usr/bin/python

from multiprocessing import Pool
import numpy as np, pandas as pd
import os
import pandas as pd

# Set defaults
dataDir = '/afs/cbs.mpg.de/projects/mar005_lsd-lemon-surf/probands'
fsDir = '/afs/cbs.mpg.de/projects/mar004_lsd-lemon-preproc/freesurfer'
subjects_file = '/scr/liberia1/data/individual_dist_label/subjects_rest_surf_1a1b2a2b_dist_incl_150714.txt'
out_file = '/scr/liberia1/data/individual_dist_label/res_individual_dist_label_nnb%s_%s_20150722.txt'

hemis = ['lh', 'rh']
n_neighbors_list = [50, 100, 150]

### !!! hemi needs to be changed within function !!! ###

# ----------------------------------- functions -----------------------------------

def run_individual_dist_label(subject, hemi, n_neighbors):

    import os, glob, subprocess, h5py
    import numpy as np, pandas as pd, nibabel as nib
    import nipype.interfaces.freesurfer as fs
    from surfer import Brain
    from sklearn.utils.arpack import eigsh

    dataDir = '/afs/cbs.mpg.de/projects/mar005_lsd-lemon-surf/probands'
    fsDir = '/afs/cbs.mpg.de/projects/mar004_lsd-lemon-preproc/freesurfer'
    outDir = '/scr/liberia1/data/individual_dist_label'

    def img2disc(data, foci_all=False, foci_dmn=False, labelfile=False, hemi='lh', filename='temp.png'):
        brain = Brain('fsaverage5', hemi, 'inflated', curv=False)
        brain.add_data(data, data.min(), data.max(), colormap="spectral", alpha=0.6)
        if labelfile:
            brain.add_label(labelfile, borders=True, color='grey')
        if foci_all:
            brain.add_foci(foci_all, coords_as_verts=True, scale_factor=.5, color='black')
        if foci_dmn:
            brain.add_foci(foci_dmn, coords_as_verts=True, scale_factor=.7, color='blue')
        brain.save_montage(filename, order=['lat', 'med'], orientation='h', border_size=10)

    # old
    def runFiedler(conn):
        # https://github.com/margulies/topography
        # prep for embedding
        K = (conn + 1) / 2.
        v = np.sqrt(np.sum(K, axis=1))
        A = K/(v[:, None] * v[None, :])
        del K
        A = np.squeeze(A * [A > 0])
        # diffusion embedding
        n_components_embedding = 2
        lambdas, vectors = eigsh(A, k=n_components_embedding+1)
        del A
        lambdas = lambdas[::-1]
        vectors = vectors[:, ::-1]
        psi = vectors/vectors[:, 0][:, None]
        lambdas = lambdas[1:] / (1 - lambdas[1:])
        embedding = psi[:, 1:(n_components_embedding + 1 + 1)] * lambdas[:n_components_embedding+1][None, :]
        return embedding

    # new
    def runAllFiedler(corr, cort, hemi='lh', n_neighbors=50):

        def compute_diffusion_map(L, alpha=0.5, n_components=None, diffusion_time=0, verbose=False):
            # from https://github.com/satra/mapalign/blob/master/mapalign/embed.py
            import numpy as np
            import scipy.sparse as sps

            from sklearn.manifold.spectral_embedding_ import _graph_is_connected

            use_sparse = False
            if sps.issparse(L):
                use_sparse = True

            if not _graph_is_connected(L):
                raise ValueError('Graph is disconnected')

            if verbose:
                print 'checked conditions'

            ndim = L.shape[0]
            L_alpha = L.copy()
            if alpha > 0:
                if verbose:
                    print 'step2'
                # Step 2
                d = np.array(L_alpha.sum(axis=1)).flatten()
                d_alpha = np.power(d, -alpha)
                if use_sparse:
                    L_alpha.data *= d_alpha[L_alpha.indices]
                    L_alpha = sps.csr_matrix(L_alpha.transpose().toarray())
                    L_alpha.data *= d_alpha[L_alpha.indices]
                    L_alpha = sps.csr_matrix(L_alpha.transpose().toarray())
                else:
                    L_alpha = d_alpha[:, None] * L_alpha * d_alpha[None, :]

            # Step 3
            if verbose:
                print 'step 3'
            d_alpha = np.power(np.array(L_alpha.sum(axis=1)).flatten(), -1)
            if use_sparse:
                L_alpha.data *= d_alpha[L_alpha.indices]
            else:
                L_alpha = d_alpha[:, None] * L_alpha

            M = L_alpha

            from sklearn.utils.arpack import eigsh, eigs

            # Step 4
            if verbose:
                print 'step 4'
            func = eigs
            if n_components is not None:
                lambdas, vectors = func(M, k=n_components + 1)
            else:
                lambdas, vectors = func(M, k=max(2, int(np.sqrt(ndim))))
            del M

            if func == eigsh:
                lambdas = lambdas[::-1]
                vectors = vectors[:, ::-1]
            else:
                lambdas = np.real(lambdas)
                vectors = np.real(vectors)
                lambda_idx = np.argsort(lambdas)[::-1]
                lambdas = lambdas[lambda_idx]
                vectors = vectors[:, lambda_idx]

            # Step 5
            if verbose:
                print 'step 5'
            psi = vectors/vectors[:, [0]]
            if diffusion_time == 0:
                lambdas = lambdas[1:] / (1 - lambdas[1:])
            else:
                lambdas = lambdas[1:] ** float(diffusion_time)
            lambda_ratio = lambdas/lambdas[0]
            threshold = max(0.05, lambda_ratio[-1])

            n_components_auto = np.amax(np.nonzero(lambda_ratio > threshold)[0])
            n_components_auto = min(n_components_auto, ndim)
            if n_components is None:
                n_components = n_components_auto
            embedding = psi[:, 1:(n_components + 1)] * lambdas[:n_components][None, :]

            result = dict(lambdas=lambdas, vectors=vectors,
                          n_components=n_components, diffusion_time=diffusion_time,
                          n_components_auto=n_components_auto)
            return embedding, result

        def thresh_neighbors(C, n_neighbors=50):
            import scipy.sparse as sps
            # K = exp(C)
            K = (C + 1) / 2.
            idx = np.argsort(K, axis=1)
            col = idx[:, -n_neighbors:].flatten()
            row = (np.array(range(K.shape[0]))[:, None] * np.ones((1, n_neighbors))).flatten().astype(int)
            A1 = sps.csr_matrix((np.ones((len(row))), (row, col)), shape=K.shape)
            A1 = (A1 + A1.transpose())# > 0
            idx1 = A1.nonzero()
            K = sps.csr_matrix((K.flat[idx1[0]*A1.shape[1] + idx1[1]], A1.indices, A1.indptr))
            #K = sps.csr_matrix((K.flat[row*K.shape[0] + col], (row, col)), shape=K.shape)
            del A1
            return K

        A = corr[cort, :][:, cort]
        L = thresh_neighbors(A, n_neighbors=n_neighbors)
        embedding, result = compute_diffusion_map(L,
                                                  alpha=0.5,
                                                  n_components=2,
                                                  diffusion_time=0)
        comp1 = np.zeros(len(corr))
        comp1[cort] = embedding.T[0]
        comp2 = np.zeros(len(corr))
        comp2[cort] = embedding.T[1]
        return comp1, comp2

    def runMasking(data, hemi):
        mask = np.zeros((10242))
        for label in [39, 40, 46, 47, 49, 50, 51, 68, 85, 86]:
            label = np.sort(nib.freesurfer.io.read_label(glob.glob('%s/fsaverage5/label/*%s*label*' % (fsDir, hemi))[label]))
            mask[label] = 1
        masked = data * mask
        return masked

    def runSmoothing(data, hemi, subject):
        temp1 = './temp1_%s.mgz' % subject
        temp2 = './temp2_%s.mgz' % subject
        img = np.expand_dims(data, axis=0)
        img = np.expand_dims(img, axis=0)
        img = nib.freesurfer.mghformat.MGHImage(img.astype(float32), affine=None)
        img.to_filename(temp1)
        smoothing = fs.SurfaceSmooth(subjects_dir=fsDir,
                                     subject_id='fsaverage5',
                                     in_file=temp1,
                                     out_file=temp2,
                                     hemi=hemi,
                                     fwhm=20,
                                     cortex=True,
                                     terminal_output='none')
        smoothing.run()
        out = nib.load(temp2).get_data().squeeze()
        os.remove(temp1)
        os.remove(temp2)
        return out

    def runExtrema(data, hemi, subject):
        temp1 = './temp_%s.mgz' % subject
        temp2 = './temp_%s.log' % subject
        thmin = (abs(data).max() - 1.3*abs(data).std())
        cluster = np.array([x if x > thmin else 0 for x in abs(data)])
        cluster_img = np.expand_dims(cluster, axis=0)
        cluster_img = np.expand_dims(cluster_img, axis=0)
        cluster_img = nib.freesurfer.mghformat.MGHImage(cluster_img.astype(float32), affine=None)
        cluster_img.to_filename(temp1)
        cml = 'mri_surfcluster --in %s --subject fsaverage5 --hemi %s --thmin %s --annot aparc.a2009s --sum %s' % (temp1, hemi, thmin, temp2)
        subprocess.call(cml, shell=True)
        extrema_log = pd.read_csv(temp2, skiprows=34, skipinitialspace=21, header=None, dtype={0:np.str})
        extrema_vertices = [int(extrema_log[0].iloc[i][15:25]) for i in range(len(extrema_log))]
        os.remove(temp1)
        os.remove(temp2)
        return extrema_vertices

    # file names and location
    corr_file1 = '%s/%s/correlation_maps/%s_lsd_corr_1ab_fsa5_%s.npy' % (dataDir, subject, subject, hemi)
    corr_file2 = '%s/%s/correlation_maps/%s_lsd_corr_2ab_fsa5_%s.npy' % (dataDir, subject, subject, hemi)
    dist_file = '%s/%s/distance_maps/%s_%s_geoDist_fsa5.mat' % (dataDir, subject, subject, hemi)
    parietal_label_file = '%s/%s/labels/fsa5/%s.G_pariet_inf-Angular_fsa5.label' % (dataDir, subject, hemi)
    temporal_label_file = '%s/%s/labels/fsa5/%s.Pole_temporal_fsa5.label' % (dataDir, subject, hemi)
    V1_label_file = '%s/%s/labels/fsa5/%s.S_calcarine_fsa5.label' % (dataDir, subject, hemi)
    A1_label_file = '%s/%s/labels/fsa5/%s.G_temp_sup-G_T_transv_fsa5.label' % (dataDir, subject, hemi)
    fiedler_file = '%s/fiedler/%s_fiedler_nnb%s_%s' % (outDir, subject, n_neighbors, hemi)
    comp2_file = '%s/fiedler/%s_comp2_nnb%s_%s' % (outDir, subject, n_neighbors, hemi)
    peak_img_file = '%s/qc/%s_fiedler_dmnExtrema_nnb%s_%s.png' % (outDir, subject, n_neighbors, hemi)

    try:
        #if not False in [os.path.isfile(i) for i in [corr_file, dist_file, parietal_label_file, temporal_label_file, V1_label_file, A1_label_file]]:
        # read in data
        cort = np.sort(nib.freesurfer.io.read_label('%s/fsaverage5/label/%s.cortex.label' % (fsDir, hemi)))
        corr1 = np.load(corr_file1)
        corr2 = np.load(corr_file2)
        corr = (corr1+corr2) /2
        with h5py.File(dist_file, 'r') as f:
            dist = f['dataAll'][()]
        parietal_vertices = np.sort(nib.freesurfer.io.read_label(parietal_label_file))
        temppole_vertices = np.sort(nib.freesurfer.io.read_label(temporal_label_file))
        V1_vertices = np.sort(nib.freesurfer.io.read_label(V1_label_file))
        A1_vertices = np.sort(nib.freesurfer.io.read_label(A1_label_file))

        # local extrema in fiedler vector
        #fiedler = np.zeros(len(corr))
        #fiedler[cort] = runFiedler(corr[cort, :][:, cort])[:,0]
        print 'before embedding'
        fiedler, comp2 = runAllFiedler(corr, cort, hemi, n_neighbors=n_neighbors)
        print 'after embedding'
        del corr
        f_smoothed = runSmoothing(fiedler, hemi, subject)
        f_masked = runMasking(f_smoothed, hemi)
        f_extrema_vertices = runExtrema(f_masked, hemi, subject)

        # distances
        dist_extrema_2_parietal = [np.mean(dist[parietal_vertices, i]) for i in f_extrema_vertices]
        parietal_peak_vertex = f_extrema_vertices[dist_extrema_2_parietal.index(min(dist_extrema_2_parietal))]
        dist_extrema_2_temporal = [np.mean(dist[temppole_vertices, i]) for i in f_extrema_vertices]
        temporal_peak_vertex = f_extrema_vertices[dist_extrema_2_temporal.index(min(dist_extrema_2_temporal))]

        # save standardized fiedler
        if fiedler[parietal_peak_vertex] < 0:
            f_stand = -fiedler
        else:
            f_stand = fiedler

        np.save(fiedler_file, f_stand)
        np.save(comp2_file, comp2)
        img2disc(f_stand, foci_all=f_extrema_vertices, foci_dmn=[parietal_peak_vertex, temporal_peak_vertex], hemi=hemi, filename=peak_img_file)

        # return results
        V1_vertices = nib.freesurfer.io.read_label('%s/%s/labels/fsa5/%s.S_calcarine_fsa5.label' % (dataDir, subject, hemi))
        A1_vertices = nib.freesurfer.io.read_label('%s/%s/labels/fsa5/%s.G_temp_sup-G_T_transv_fsa5.label' % (dataDir, subject, hemi))
        V1_parietal = dist[V1_vertices, parietal_peak_vertex].mean()
        A1_parietal = dist[A1_vertices, parietal_peak_vertex].mean()
        V1_temporal = dist[V1_vertices, temporal_peak_vertex].mean()
        A1_temporal = dist[A1_vertices, temporal_peak_vertex].mean()
        return subject, hemi, V1_parietal, A1_parietal, V1_temporal, A1_temporal

    except:
        return subject, hemi, None, None, None, None
        pass

# --------------------------------------------------------------------------------------------------------------------------------------------------

### run serially ###

subjects = pd.read_csv(subjects_file, header=None, converters={0:str})[0].tolist()

for hemi in hemis:
    for n_neighbors in n_neighbors_list:
        output_dict = {}
        res = []
        for subject in subjects:
            res.append(run_individual_dist_label(subject, hemi, n_neighbors))

        output_dict['subject'], output_dict['hemi'], output_dict['V1_parietal'], output_dict['A1_parietal'], output_dict['V1_temporal'], output_dict['A1_temporal'] = np.array(res).T

        pd.DataFrame(output_dict).to_csv(out_file % (n_neighbors, hemi), sep='\t', index=False, columns=['subject', 'hemi', 'V1_parietal', 'A1_parietal', 'V1_temporal', 'A1_temporal'])

### run in parallel ### (not good for qc screenshots though)

#p = Pool(20)
#res = p.map(run_individual_dist_label, subjects)
#output_dict = {}
#output_dict['subject'], output_dict['hemi'], output_dict['V1_parietal'], output_dict['A1_parietal'], output_dict['V1_temporal'], output_dict['A1_temporal'] = np.array(res).T
#pd.DataFrame(output_dict).to_csv(out_file, sep='\t', index=False, columns=['subject', 'hemi', 'V1_parietal', 'A1_parietal', 'V1_temporal', 'A1_temporal'])<|endoftext|>


<commit_before>import appleseed as asr
import appleseed.studio as studio
from appleseed.studio import ui
from appleseed.studio.Qt import QtWidgets
from appleseed.textureconverter import *

import os
import sys
import logging

logging.basicConfig(level=logging.INFO, stream=sys.stdout)

def register():
    menu = ui.find_or_create_menu("Plugins")

    act = QtWidgets.QAction("Convert textures", menu)
    act.triggered.connect(convert_all_textures_to_tx)

    menu.addAction(act)

def convert_all_textures_to_tx():
    def _find_maketx():
        root_path = studio.get_root_path()
        maketx_path = os.path.join(root_path, 'bin', 'maketx')

        if os.path.exists(maketx_path):
            return maketx_path
        else:
            raise Exception('maketx binary is not found')

    project = studio.current_project()

    if project is None:
        raise Exception('No project is opened')

    scene = project.get_scene()
    textures = get_textures(scene)

    tx_converter = TextureConverter(_find_maketx())

    for texture in textures:
        if texture.get_model() != 'disk_texture_2d':
            continue

        texture_parameters = texture.get_parameters()

        texture_path = texture_parameters['filename']
        texture_full_path = get_full_path(texture_path, project)

        if texture_full_path.endswith('.tx'):
            logging.debug('Skipped conversion of {}'.format(texture_full_path))
            continue

        new_texture_full_path = tx_converter.convert(texture_full_path)

        if new_texture_full_path is None:
            logging.info('Skipped conversion of {}'.format(texture_full_path))
        else:
            new_texture_path = os.path.join(os.path.dirname(texture_path),
                                            os.path.basename(new_texture_full_path))

            texture_parameters['filename'] = new_texture_path
            texture.set_parameters(texture_parameters)
            studio.set_project_dirty()
            logging.info('{} converted to {}'.format(texture_path, new_texture_path))

def get_textures(container):
    assert isinstance(container, asr.BaseGroup)

    textures = list(container.textures())

    assemblies = container.assemblies()
    for key in assemblies:
        textures += get_textures(assemblies[key])

    return textures

def get_full_path(texture_path, project):
    if os.path.isabs(texture_path):
        return texture_path
    else:
        return project.qualify_path(texture_path)
<commit_msg>Quick fix to return instead of raise exception<commit_after>import appleseed as asr
import appleseed.studio as studio
from appleseed.studio import ui
from appleseed.studio.Qt import QtWidgets
from appleseed.textureconverter import *

import os
import sys
import logging

logging.basicConfig(level=logging.INFO, stream=sys.stdout)

def register():
    menu = ui.find_or_create_menu("Plugins")

    act = QtWidgets.QAction("Convert textures", menu)
    act.triggered.connect(convert_all_textures_to_tx)

    menu.addAction(act)

def convert_all_textures_to_tx():
    def _find_maketx():
        root_path = studio.get_root_path()
        maketx_path = os.path.join(root_path, 'bin', 'maketx')

        if os.path.exists(maketx_path):
            return maketx_path
        else:
            raise Exception('maketx binary is not found')

    project = studio.current_project()

    if project is None:
        return

    scene = project.get_scene()
    textures = get_textures(scene)

    tx_converter = TextureConverter(_find_maketx())

    for texture in textures:
        if texture.get_model() != 'disk_texture_2d':
            continue

        texture_parameters = texture.get_parameters()

        texture_path = texture_parameters['filename']
        texture_full_path = get_full_path(texture_path, project)

        if texture_full_path.endswith('.tx'):
            logging.debug('Skipped conversion of {}'.format(texture_full_path))
            continue

        new_texture_full_path = tx_converter.convert(texture_full_path)

        if new_texture_full_path is None:
            logging.info('Skipped conversion of {}'.format(texture_full_path))
        else:
            new_texture_path = os.path.join(os.path.dirname(texture_path),
                                            os.path.basename(new_texture_full_path))

            texture_parameters['filename'] = new_texture_path
            texture.set_parameters(texture_parameters)
            studio.set_project_dirty()
            logging.info('{} converted to {}'.format(texture_path, new_texture_path))

def get_textures(container):
    assert isinstance(container, asr.BaseGroup)

    textures = list(container.textures())

    assemblies = container.assemblies()
    for key in assemblies:
        textures += get_textures(assemblies[key])

    return textures

def get_full_path(texture_path, project):
    if os.path.isabs(texture_path):
        return texture_path
    else:
        return project.qualify_path(texture_path)
<|endoftext|>


<commit_before># Copyright 2015 The Swarming Authors. All rights reserved.
# Use of this source code is governed by the Apache v2.0 license that can be
# found in the LICENSE file.

"""Android specific utility functions.

This file serves as an API to bot_config.py. bot_config.py can be replaced on
the server to allow additional server-specific functionality.
"""

import logging
import os
import re
import subprocess
import sys

# This file must be imported from swarming_bot.zip (or with '..' in sys.path).
# libusb1 must have been put in the path already.

import adb
import adb.adb_commands

# Find abs path to third_party/ dir in the bot root dir.
import third_party
THIRD_PARTY_DIR = os.path.dirname(os.path.abspath(third_party.__file__))
sys.path.insert(0, os.path.join(THIRD_PARTY_DIR, 'rsa'))
sys.path.insert(0, os.path.join(THIRD_PARTY_DIR, 'pyasn1'))

import rsa

from pyasn1.codec.der import decoder
from pyasn1.type import univ
from rsa import pkcs1

### Private stuff.

# python-rsa lib hashes all messages it signs. ADB does it already, we just
# need to slap a signature on top of already hashed message. Introduce "fake"
# hashing algo for this.
class _Accum(object):
  def __init__(self):
    self._buf = ''
  def update(self, msg):
    self._buf += msg
  def digest(self):
    return self._buf
pkcs1.HASH_METHODS['SHA-1-PREHASHED'] = _Accum
pkcs1.HASH_ASN1['SHA-1-PREHASHED'] = pkcs1.HASH_ASN1['SHA-1']

# Set when ADB is initialized. It contains one or multiple key used to
# authenticate to Android debug protocol (adb).
_ADB_KEYS = None

# Cache of /system/build.prop on Android devices connected to this host.
_BUILD_PROP_ANDROID = {}

def _dumpsys(cmd, arg):
  out = cmd.Shell('dumpsys ' + arg).decode('utf-8', 'replace')
  if out.startswith('Can\'t find service: '):
    return None
  return out

def _parcel_to_list(lines):
  """Parses 'service call' output."""
  out = []
  for line in lines:
    match = re.match(
        '  0x[0-9a-f]{8}\\: ([0-9a-f ]{8}) ([0-9a-f ]{8}) ([0-9a-f ]{8}) '
        '([0-9a-f ]{8}) \'.{16}\'\\)?', line)
    if not match:
      break
    for i in xrange(1, 5):
      group = match.group(i)
      char = group[4:8]
      if char != '    ':
        out.append(char)
      char = group[0:4]
      if char != '    ':
        out.append(char)
  return out

### Public API.

def initialize(pub_key, priv_key):
  """Initialize Android support through adb.

  You can steal pub_key, priv_key pair from ~/.android/adbkey and
  ~/.android/adbkey.pub.
  """
  global _ADB_KEYS
  assert bool(pub_key) == bool(priv_key)
  if _ADB_KEYS is None:
    _ADB_KEYS = []
    if pub_key:
      try:
        _ADB_KEYS.append(PythonRSASigner(pub_key, priv_key))
      except ValueError as exc:
        logging.warning('Ignoring adb private key: %s', exc)

    # Try to add local adb keys if available.
    path = os.path.expanduser('~/.android/adbkey')
    if os.path.isfile(path) and os.path.isfile(path+'.pub'):
      with open(path + '.pub', 'rb') as f:
        pub = f.read()
      with open(path, 'rb') as f:
        priv = f.read()
      try:
        _ADB_KEYS.append(PythonRSASigner(pub, priv))
      except ValueError as exc:
        logging.warning('Ignoring adb private key %s: %s', path, exc)

    if not _ADB_KEYS:
      return False
  else:
    if pub_key:
      logging.warning('initialize() was called repeatedly: ignoring keys')
  return bool(_ADB_KEYS)

class PythonRSASigner(object):
  """Implements adb_protocol.AuthSigner using http://stuvel.eu/rsa."""
  def __init__(self, pub, priv):
    self.priv_key = load_rsa_private_key(priv)
    self.pub_key = pub

  def Sign(self, data):
    return rsa.sign(data, self.priv_key, 'SHA-1-PREHASHED')

  def GetPublicKey(self):
    return self.pub_key

def load_rsa_private_key(pem):
  """PEM encoded PKCS#8 private key -> rsa.PrivateKey."""
  # ADB uses private RSA keys in pkcs#8 format. 'rsa' library doesn't support
  # them natively. Do some ASN unwrapping to extract naked RSA key
  # (in der-encoded form). See https://www.ietf.org/rfc/rfc2313.txt.
  # Also http://superuser.com/a/606266.
  try:
    der = rsa.pem.load_pem(pem, 'PRIVATE KEY')
    keyinfo, _ = decoder.decode(der)
    if keyinfo[1][0] != univ.ObjectIdentifier('1.2.840.113549.1.1.1'):
        raise ValueError('Not a DER-encoded OpenSSL private RSA key')
    private_key_der = keyinfo[2].asOctets()
  except IndexError:
    raise ValueError('Not a DER-encoded OpenSSL private RSA key')
  return rsa.PrivateKey.load_pkcs1(private_key_der, format='DER')

def kill_adb():
  """adb sucks. Kill it with fire."""
  if not adb:
    return
  try:
    subprocess.call(['adb', 'kill-server'])
  except OSError:
    pass
  subprocess.call(['killall', '--exact', 'adb'])

def get_devices():
  """Returns the list of devices available.

  Caller MUST call close_devices(cmds) on the return value.

  Returns one of:
    - dict of {serial_number: adb.adb_commands.AdbCommands}. The value may be
      None if there was an Auth failure.
    - None if adb is unavailable.
  """
  if not adb:
    return None

  cmds = {}
  generator = adb.adb_commands.AdbCommands.Devices()
  while True:
    try:
      # Use manual iterator handling instead of "for handle in generator" to
      # catch USB exception explicitly.
      handle = generator.next()
    except adb.common.usb1.USBErrorOther as e:
      # This happens if the user is not in group plugdev.
      continue
    except StopIteration:
      break

    try:
      handle.Open()
    except adb.common.usb1.USBErrorNoDevice as e:
      logging.warning(
          'Got USBErrorNoDevice for %s: %s', handle.usb_info, e)
      # Do not kill adb, it just means the USB host is likely resetting and the
      # device is temporarily unavailable.We can't use handle.serial_number
      # since this communicates with the device.
      # TODO(maruel): In practice we'd like to retry for a few seconds.
      cmds[handle.port_path] = None
      continue
    except adb.common.usb1.USBErrorBusy as e:
      logging.warning(
          'Got USBErrorBusy for %s. Killing adb: %s', handle.usb_info, e)
      kill_adb()
      try:
        # If it throws again, it probably means another process
        # holds a handle to the USB ports or group acl (plugdev) hasn't been
        # setup properly.
        handle.Open()
      except adb.common.usb1.USBErrorBusy as e:
        logging.warning(
            'USB port for %s is already open (and failed to kill ADB) '
            'Try rebooting the host: %s', handle.usb_info, e)
        cmds[handle.serial_number] = None
        continue
    except adb.common.usb1.USBErrorAccess as e:
      # Do not try to use serial_number, since we can't even access the port.
      logging.warning(
          'Try rebooting the host: %s: %s', handle.port_path, e)
      cmds['/'.join(map(str, handle.port_path))] = None
      continue

    try:
      # Give 10s for the user to accept the dialog. The best design would be to
      # do a quick check with timeout=100ms and only if the first failed, try
      # again with a long timeout. The goal is not to hang the bots for several
      # minutes when all the devices are unauthenticated.
      # TODO(maruel): A better fix would be to change python-adb to continue the
      # authentication dance from where it stopped. This is left as a follow up.
      cmd = adb.adb_commands.AdbCommands.Connect(
          handle, banner='swarming', rsa_keys=_ADB_KEYS, auth_timeout_ms=10000)
    except adb.usb_exceptions.DeviceAuthError as e:
      logging.warning('AUTH FAILURE: %s: %s', handle.usb_info, e)
      cmd = None
    except adb.usb_exceptions.ReadFailedError as e:
      logging.warning('READ FAILURE: %s: %s', handle.usb_info, e)
      cmd = None
    except adb.usb_exceptions.WriteFailedError as e:
      logging.warning('WRITE FAILURE: %s: %s', handle.usb_info, e)
      cmd = None
    except ValueError as e:
      logging.warning(
          'Trying unpluging and pluging it back: %s: %s', handle.usb_info, e)
      cmd = None

    try:
      serial = handle.serial_number
    except adb.common.libusb1.USBError:
      serial = handle.usb_info
    cmds[serial] = cmd

  # Remove any /system/build.prop cache so if a device is disconnect, reflashed
  # then reconnected, it will likely be refresh properly. The main concern is
  # that the bot didn't have the time to loop once while this is being done.
  # Restarting the bot works fine too.
  for key in _BUILD_PROP_ANDROID.keys():
    if key not in cmds:
      _BUILD_PROP_ANDROID.pop(key)
  return cmds

def close_devices(devices):
  """Closes all devices opened by get_devices()."""
  for device in (devices or {}).itervalues():
    if device:
      device.Close()

class CpuScalingGovernor(object):
  """List of valid CPU scaling governor values."""
  ON_DEMAND = 'ondemand'
  PERFORMANCE = 'performance'
  CONSERVATIVE = 'conservative'
  POWER_SAVE = 'powersave'
  USER_DEFINED = 'userspace'

  @classmethod
  def is_valid(cls, name):
    return name in [getattr(cls, m) for m in dir(cls) if m[0].isupper()]

def set_cpu_scaling_governor(cmd, governor):
  """Sets the CPU scaling governor to the one specified."""
  assert CpuScalingGovernor.is_valid(governor), governor
  cmd.Shell(
      'echo "%s">/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor' %
      governor)

def set_cpu_scaling(cmd, speed):
  """Enforces strict CPU speed and disable the CPU scaling governor."""
  assert isinstance(speed, int), speed
  assert 10000 <= speed <= 10000000, speed
  set_cpu_scaling_governor(cmd, CpuScalingGovernor.USER_DEFINED)
  cmd.Shell(
      'echo "%d">/sys/devices/system/cpu/cpu0/cpufreq/scaling_setspeed' %
      speed)

def get_build_prop(cmd):
  """Returns the system properties for a device.

  This isn't really changing through the lifetime of a bot. One corner case is
  when the device is flashed or disconnected.
  """
  if cmd.handle.serial_number not in _BUILD_PROP_ANDROID:
    properties = {}
    try:
      out = cmd.Shell('cat /system/build.prop').decode('utf-8')
    except adb.usb_exceptions.ReadFailedError:
      # It's a bit annoying because it means timeout_ms was wasted. Blacklist
      # the device until it is disconnected and reconnected.
      properties = None
    else:
      for line in out.splitlines():
        if line.startswith(u'#') or not line:
          continue
        key, value = line.split(u'=', 1)
        properties[key] = value
    _BUILD_PROP_ANDROID[cmd.handle.serial_number] = properties
  return _BUILD_PROP_ANDROID[cmd.handle.serial_number]

def get_temp(cmd):
  """Returns the device's 2 temperatures if available."""
  temps = []
  for i in xrange(2):
    try:
      temps.append(
          int(cmd.Shell('cat /sys/class/thermal/thermal_zone%d/temp' % i)))
    except ValueError:
      # On other devices, the only real way to read it is via Java
      # developer.android.com/guide/topics/sensors/sensors_environment.html
      pass
  return temps

def get_battery(cmd):
  """Returns details about the battery's state."""
  props = {}
  out = _dumpsys(cmd, 'battery')
  if not out:
    return props
  for line in out.splitlines():
    if line.endswith(u':'):
      continue
    # On Android 4.1.2, it uses "voltage:123" instead of "voltage: 123".
    key, value = line.split(u':', 2)
    props[key.lstrip()] = value.strip()
  out = {u'power': []}
  if props[u'AC powered'] == u'true':
    out[u'power'].append(u'AC')
  if props[u'USB powered'] == u'true':
    out[u'power'].append(u'USB')
  if props.get(u'Wireless powered') == u'true':
    out[u'power'].append(u'Wireless')
  for key in (u'health', u'level', u'status', u'temperature', u'voltage'):
    out[key] = int(props[key])
  return out

def get_cpu_scale(cmd):
  """Returns the CPU scaling factor."""
  mapping = {
    'cpuinfo_max_freq': u'max',
    'cpuinfo_min_freq': u'min',
    'scaling_cur_freq': u'cur',
    'scaling_governor': u'governor',
  }
  return {
    v: cmd.Shell('cat /sys/devices/system/cpu/cpu0/cpufreq/' + k)
    for k, v in mapping.iteritems()
  }

def get_uptime(cmd):
  """Returns the device's uptime in second."""
  return float(cmd.Shell('cat /proc/uptime').split()[1])

def get_disk(cmd):
  """Returns details about the battery's state."""
  props = {}
  out = _dumpsys(cmd, 'diskstats')
  if not out:
    return props
  for line in out.splitlines():
    if line.endswith(u':'):
      continue
    key, value = line.split(u': ', 2)
    match = re.match(u'^(\d+)K / (\d+)K.*', value)
    if match:
      props[key.lstrip()] = {
        'free_mb': round(float(match.group(1)) / 1024., 1),
        'size_mb': round(float(match.group(2)) / 1024., 1),
      }
  return {
    u'cache': props[u'Cache-Free'],
    u'data': props[u'Data-Free'],
    u'system': props[u'System-Free'],
  }

def get_imei(cmd):
  """Returns the phone's IMEI."""
  # Android <5.0.
  out = _dumpsys(cmd, 'iphonesubinfo')
  match = re.search('  Device ID = (.+)$', out)
  if match:
    return match.group(1)

  # Android >= 5.0.
  lines = cmd.Shell('service call iphonesubinfo 1').splitlines()
  if len(lines) >= 4 and lines[0] == 'Result: Parcel(':
    # Process the UTF-16 string.
    chars = _parcel_to_list(lines[1:])[4:-1]
    return u''.join(map(unichr, (int(i, 16) for i in chars)))

def get_ip(cmd):
  """Returns the IP address."""
  # If ever needed, read wifi.interface from /system/build.prop if a device
  # doesn't use wlan0.
  return cmd.Shell('getprop dhcp.wlan0.ipaddress').strip()
<commit_msg>Fix the invalid fix in 48463b27df61649ab.<commit_after># Copyright 2015 The Swarming Authors. All rights reserved.
# Use of this source code is governed by the Apache v2.0 license that can be
# found in the LICENSE file.

"""Android specific utility functions.

This file serves as an API to bot_config.py. bot_config.py can be replaced on
the server to allow additional server-specific functionality.
"""

import logging
import os
import re
import subprocess
import sys

# This file must be imported from swarming_bot.zip (or with '..' in sys.path).
# libusb1 must have been put in the path already.

import adb
import adb.adb_commands

# Find abs path to third_party/ dir in the bot root dir.
import third_party
THIRD_PARTY_DIR = os.path.dirname(os.path.abspath(third_party.__file__))
sys.path.insert(0, os.path.join(THIRD_PARTY_DIR, 'rsa'))
sys.path.insert(0, os.path.join(THIRD_PARTY_DIR, 'pyasn1'))

import rsa

from pyasn1.codec.der import decoder
from pyasn1.type import univ
from rsa import pkcs1

### Private stuff.

# python-rsa lib hashes all messages it signs. ADB does it already, we just
# need to slap a signature on top of already hashed message. Introduce "fake"
# hashing algo for this.
class _Accum(object):
  def __init__(self):
    self._buf = ''
  def update(self, msg):
    self._buf += msg
  def digest(self):
    return self._buf
pkcs1.HASH_METHODS['SHA-1-PREHASHED'] = _Accum
pkcs1.HASH_ASN1['SHA-1-PREHASHED'] = pkcs1.HASH_ASN1['SHA-1']

# Set when ADB is initialized. It contains one or multiple key used to
# authenticate to Android debug protocol (adb).
_ADB_KEYS = None

# Cache of /system/build.prop on Android devices connected to this host.
_BUILD_PROP_ANDROID = {}

def _dumpsys(cmd, arg):
  out = cmd.Shell('dumpsys ' + arg).decode('utf-8', 'replace')
  if out.startswith('Can\'t find service: '):
    return None
  return out

def _parcel_to_list(lines):
  """Parses 'service call' output."""
  out = []
  for line in lines:
    match = re.match(
        '  0x[0-9a-f]{8}\\: ([0-9a-f ]{8}) ([0-9a-f ]{8}) ([0-9a-f ]{8}) '
        '([0-9a-f ]{8}) \'.{16}\'\\)?', line)
    if not match:
      break
    for i in xrange(1, 5):
      group = match.group(i)
      char = group[4:8]
      if char != '    ':
        out.append(char)
      char = group[0:4]
      if char != '    ':
        out.append(char)
  return out

### Public API.

def initialize(pub_key, priv_key):
  """Initialize Android support through adb.

  You can steal pub_key, priv_key pair from ~/.android/adbkey and
  ~/.android/adbkey.pub.
  """
  global _ADB_KEYS
  assert bool(pub_key) == bool(priv_key)
  if _ADB_KEYS is None:
    _ADB_KEYS = []
    if pub_key:
      try:
        _ADB_KEYS.append(PythonRSASigner(pub_key, priv_key))
      except ValueError as exc:
        logging.warning('Ignoring adb private key: %s', exc)

    # Try to add local adb keys if available.
    path = os.path.expanduser('~/.android/adbkey')
    if os.path.isfile(path) and os.path.isfile(path+'.pub'):
      with open(path + '.pub', 'rb') as f:
        pub = f.read()
      with open(path, 'rb') as f:
        priv = f.read()
      try:
        _ADB_KEYS.append(PythonRSASigner(pub, priv))
      except ValueError as exc:
        logging.warning('Ignoring adb private key %s: %s', path, exc)

    if not _ADB_KEYS:
      return False
  else:
    if pub_key:
      logging.warning('initialize() was called repeatedly: ignoring keys')
  return bool(_ADB_KEYS)

class PythonRSASigner(object):
  """Implements adb_protocol.AuthSigner using http://stuvel.eu/rsa."""
  def __init__(self, pub, priv):
    self.priv_key = load_rsa_private_key(priv)
    self.pub_key = pub

  def Sign(self, data):
    return rsa.sign(data, self.priv_key, 'SHA-1-PREHASHED')

  def GetPublicKey(self):
    return self.pub_key

def load_rsa_private_key(pem):
  """PEM encoded PKCS#8 private key -> rsa.PrivateKey."""
  # ADB uses private RSA keys in pkcs#8 format. 'rsa' library doesn't support
  # them natively. Do some ASN unwrapping to extract naked RSA key
  # (in der-encoded form). See https://www.ietf.org/rfc/rfc2313.txt.
  # Also http://superuser.com/a/606266.
  try:
    der = rsa.pem.load_pem(pem, 'PRIVATE KEY')
    keyinfo, _ = decoder.decode(der)
    if keyinfo[1][0] != univ.ObjectIdentifier('1.2.840.113549.1.1.1'):
        raise ValueError('Not a DER-encoded OpenSSL private RSA key')
    private_key_der = keyinfo[2].asOctets()
  except IndexError:
    raise ValueError('Not a DER-encoded OpenSSL private RSA key')
  return rsa.PrivateKey.load_pkcs1(private_key_der, format='DER')

def kill_adb():
  """adb sucks. Kill it with fire."""
  if not adb:
    return
  try:
    subprocess.call(['adb', 'kill-server'])
  except OSError:
    pass
  subprocess.call(['killall', '--exact', 'adb'])

def get_devices():
  """Returns the list of devices available.

  Caller MUST call close_devices(cmds) on the return value.

  Returns one of:
    - dict of {serial_number: adb.adb_commands.AdbCommands}. The value may be
      None if there was an Auth failure.
    - None if adb is unavailable.
  """
  if not adb:
    return None

  cmds = {}
  generator = adb.adb_commands.AdbCommands.Devices()
  while True:
    try:
      # Use manual iterator handling instead of "for handle in generator" to
      # catch USB exception explicitly.
      handle = generator.next()
    except adb.common.usb1.USBErrorOther as e:
      # This happens if the user is not in group plugdev.
      continue
    except StopIteration:
      break

    try:
      handle.Open()
    except adb.common.usb1.USBErrorNoDevice as e:
      logging.warning(
          'Got USBErrorNoDevice for %s: %s', handle.usb_info, e)
      # Do not kill adb, it just means the USB host is likely resetting and the
      # device is temporarily unavailable.We can't use handle.serial_number
      # since this communicates with the device.
      # TODO(maruel): In practice we'd like to retry for a few seconds.
      cmds['/'.join(map(str, handle.port_path))] = None
      continue
    except adb.common.usb1.USBErrorBusy as e:
      logging.warning(
          'Got USBErrorBusy for %s. Killing adb: %s', handle.usb_info, e)
      kill_adb()
      try:
        # If it throws again, it probably means another process
        # holds a handle to the USB ports or group acl (plugdev) hasn't been
        # setup properly.
        handle.Open()
      except adb.common.usb1.USBErrorBusy as e:
        logging.warning(
            'USB port for %s is already open (and failed to kill ADB) '
            'Try rebooting the host: %s', handle.usb_info, e)
        cmds[handle.serial_number] = None
        continue
    except adb.common.usb1.USBErrorAccess as e:
      # Do not try to use serial_number, since we can't even access the port.
      logging.warning(
          'Try rebooting the host: %s: %s', handle.port_path, e)
      cmds['/'.join(map(str, handle.port_path))] = None
      continue

    try:
      # Give 10s for the user to accept the dialog. The best design would be to
      # do a quick check with timeout=100ms and only if the first failed, try
      # again with a long timeout. The goal is not to hang the bots for several
      # minutes when all the devices are unauthenticated.
      # TODO(maruel): A better fix would be to change python-adb to continue the
      # authentication dance from where it stopped. This is left as a follow up.
      cmd = adb.adb_commands.AdbCommands.Connect(
          handle, banner='swarming', rsa_keys=_ADB_KEYS, auth_timeout_ms=10000)
    except adb.usb_exceptions.DeviceAuthError as e:
      logging.warning('AUTH FAILURE: %s: %s', handle.usb_info, e)
      cmd = None
    except adb.usb_exceptions.ReadFailedError as e:
      logging.warning('READ FAILURE: %s: %s', handle.usb_info, e)
      cmd = None
    except adb.usb_exceptions.WriteFailedError as e:
      logging.warning('WRITE FAILURE: %s: %s', handle.usb_info, e)
      cmd = None
    except ValueError as e:
      logging.warning(
          'Trying unpluging and pluging it back: %s: %s', handle.usb_info, e)
      cmd = None

    try:
      serial = handle.serial_number
    except adb.common.libusb1.USBError:
      serial = handle.usb_info
    cmds[serial] = cmd

  # Remove any /system/build.prop cache so if a device is disconnect, reflashed
  # then reconnected, it will likely be refresh properly. The main concern is
  # that the bot didn't have the time to loop once while this is being done.
  # Restarting the bot works fine too.
  for key in _BUILD_PROP_ANDROID.keys():
    if key not in cmds:
      _BUILD_PROP_ANDROID.pop(key)
  return cmds

def close_devices(devices):
  """Closes all devices opened by get_devices()."""
  for device in (devices or {}).itervalues():
    if device:
      device.Close()

class CpuScalingGovernor(object):
  """List of valid CPU scaling governor values."""
  ON_DEMAND = 'ondemand'
  PERFORMANCE = 'performance'
  CONSERVATIVE = 'conservative'
  POWER_SAVE = 'powersave'
  USER_DEFINED = 'userspace'

  @classmethod
  def is_valid(cls, name):
    return name in [getattr(cls, m) for m in dir(cls) if m[0].isupper()]

def set_cpu_scaling_governor(cmd, governor):
  """Sets the CPU scaling governor to the one specified."""
  assert CpuScalingGovernor.is_valid(governor), governor
  cmd.Shell(
      'echo "%s">/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor' %
      governor)

def set_cpu_scaling(cmd, speed):
  """Enforces strict CPU speed and disable the CPU scaling governor."""
  assert isinstance(speed, int), speed
  assert 10000 <= speed <= 10000000, speed
  set_cpu_scaling_governor(cmd, CpuScalingGovernor.USER_DEFINED)
  cmd.Shell(
      'echo "%d">/sys/devices/system/cpu/cpu0/cpufreq/scaling_setspeed' %
      speed)

def get_build_prop(cmd):
  """Returns the system properties for a device.

  This isn't really changing through the lifetime of a bot. One corner case is
  when the device is flashed or disconnected.
  """
  if cmd.handle.serial_number not in _BUILD_PROP_ANDROID:
    properties = {}
    try:
      out = cmd.Shell('cat /system/build.prop').decode('utf-8')
    except adb.usb_exceptions.ReadFailedError:
      # It's a bit annoying because it means timeout_ms was wasted. Blacklist
      # the device until it is disconnected and reconnected.
      properties = None
    else:
      for line in out.splitlines():
        if line.startswith(u'#') or not line:
          continue
        key, value = line.split(u'=', 1)
        properties[key] = value
    _BUILD_PROP_ANDROID[cmd.handle.serial_number] = properties
  return _BUILD_PROP_ANDROID[cmd.handle.serial_number]

def get_temp(cmd):
  """Returns the device's 2 temperatures if available."""
  temps = []
  for i in xrange(2):
    try:
      temps.append(
          int(cmd.Shell('cat /sys/class/thermal/thermal_zone%d/temp' % i)))
    except ValueError:
      # On other devices, the only real way to read it is via Java
      # developer.android.com/guide/topics/sensors/sensors_environment.html
      pass
  return temps

def get_battery(cmd):
  """Returns details about the battery's state."""
  props = {}
  out = _dumpsys(cmd, 'battery')
  if not out:
    return props
  for line in out.splitlines():
    if line.endswith(u':'):
      continue
    # On Android 4.1.2, it uses "voltage:123" instead of "voltage: 123".
    key, value = line.split(u':', 2)
    props[key.lstrip()] = value.strip()
  out = {u'power': []}
  if props[u'AC powered'] == u'true':
    out[u'power'].append(u'AC')
  if props[u'USB powered'] == u'true':
    out[u'power'].append(u'USB')
  if props.get(u'Wireless powered') == u'true':
    out[u'power'].append(u'Wireless')
  for key in (u'health', u'level', u'status', u'temperature', u'voltage'):
    out[key] = int(props[key])
  return out

def get_cpu_scale(cmd):
  """Returns the CPU scaling factor."""
  mapping = {
    'cpuinfo_max_freq': u'max',
    'cpuinfo_min_freq': u'min',
    'scaling_cur_freq': u'cur',
    'scaling_governor': u'governor',
  }
  return {
    v: cmd.Shell('cat /sys/devices/system/cpu/cpu0/cpufreq/' + k)
    for k, v in mapping.iteritems()
  }

def get_uptime(cmd):
  """Returns the device's uptime in second."""
  return float(cmd.Shell('cat /proc/uptime').split()[1])

def get_disk(cmd):
  """Returns details about the battery's state."""
  props = {}
  out = _dumpsys(cmd, 'diskstats')
  if not out:
    return props
  for line in out.splitlines():
    if line.endswith(u':'):
      continue
    key, value = line.split(u': ', 2)
    match = re.match(u'^(\d+)K / (\d+)K.*', value)
    if match:
      props[key.lstrip()] = {
        'free_mb': round(float(match.group(1)) / 1024., 1),
        'size_mb': round(float(match.group(2)) / 1024., 1),
      }
  return {
    u'cache': props[u'Cache-Free'],
    u'data': props[u'Data-Free'],
    u'system': props[u'System-Free'],
  }

def get_imei(cmd):
  """Returns the phone's IMEI."""
  # Android <5.0.
  out = _dumpsys(cmd, 'iphonesubinfo')
  match = re.search('  Device ID = (.+)$', out)
  if match:
    return match.group(1)

  # Android >= 5.0.
  lines = cmd.Shell('service call iphonesubinfo 1').splitlines()
  if len(lines) >= 4 and lines[0] == 'Result: Parcel(':
    # Process the UTF-16 string.
    chars = _parcel_to_list(lines[1:])[4:-1]
    return u''.join(map(unichr, (int(i, 16) for i in chars)))

def get_ip(cmd):
  """Returns the IP address."""
  # If ever needed, read wifi.interface from /system/build.prop if a device
  # doesn't use wlan0.
  return cmd.Shell('getprop dhcp.wlan0.ipaddress').strip()
<|endoftext|>


<commit_before>from subprocess import Popen, PIPE, CalledProcessError
import logging
import os
import shutil

logger = logging.getLogger(__name__)

def extend_env(extra_env):
    """
    Copies and extends the current environment with the values present in
    `extra_env`.
    """
    env = os.environ.copy()
    env.update(extra_env)
    return env

def get_env_str(env):
    """
    Gets a string representation of a dict as though it contained environment
    variable values.
    """
    return ' '.join("{0}='{1}'".format(k, v) for k, v in env.items())

def pipe_commands(cmds, extra_env=None, show_stderr=False, show_last_stdout=False):
    """
    Executes the list of commands piping each one into the next.
    """
    env = extend_env(extra_env) if extra_env else None
    env_str = (get_env_str(extra_env) + ' ') if extra_env else ''
    cmd_strs = [env_str + ' '.join(cmd) for cmd in cmds]

    logger.info('Running `{0}`'.format(' | '.join(cmd_strs)))

    with open('/dev/null', 'w') as NULL:
        # Start processes
        processes = []
        last_i = len(cmds) - 1
        for i, (cmd_str, cmd) in enumerate(zip(cmd_strs, cmds)):
            if i == last_i:
                p_stdout = None if show_last_stdout else NULL
            else:
                p_stdout = PIPE
            p_stdin = processes[-1][1].stdout if processes else None
            p_stderr = None if show_stderr else NULL

            p = Popen(cmd, env=env, stdout=p_stdout, stdin=p_stdin, stderr=p_stderr)
            processes.append((cmd_str, p))

        # Close processes
        for cmd_str, p in processes:
            if p.stdout:
                p.stdout.close()
            if p.wait() != 0:
                raise CalledProcessError(cmd=cmd_str, returncode=p.returncode)

def pipe_commands_to_file(cmds, path, extra_env=None, show_stderr=False):
    """
    Executes the list of commands piping each one into the next and writing
    stdout of the last process into a file at the given path.
    """
    env = extend_env(extra_env) if extra_env else None
    env_str = (get_env_str(extra_env) + ' ') if extra_env else ''
    cmd_strs = [env_str + ' '.join(cmd) for cmd in cmds]

    logger.info('Saving output of `{0}`'.format(' | '.join(cmd_strs)))

    with open('/dev/null', 'w') as NULL:
        # Start processes
        processes = []
        for cmd_str, cmd in zip(cmd_strs, cmds):
            p_stdin = processes[-1][1].stdout if processes else None
            p_stderr = None if show_stderr else NULL

            p = Popen(cmd, env=env, stdout=PIPE, stdin=p_stdin, stderr=p_stderr)
            processes.append((cmd_str, p))

        p_last = processes[-1][1]

        with open(path, 'w') as f:
            shutil.copyfileobj(p_last.stdout, f)

            # Close processes
            for cmd_str, p in processes:
                p.stdout.close()
                if p.wait() != 0:
                    raise CalledProcessError(cmd=cmd_str, returncode=p.returncode)
<commit_msg>Open the output file in binary mode<commit_after>from subprocess import Popen, PIPE, CalledProcessError
import logging
import os
import shutil

logger = logging.getLogger(__name__)

def extend_env(extra_env):
    """
    Copies and extends the current environment with the values present in
    `extra_env`.
    """
    env = os.environ.copy()
    env.update(extra_env)
    return env

def get_env_str(env):
    """
    Gets a string representation of a dict as though it contained environment
    variable values.
    """
    return ' '.join("{0}='{1}'".format(k, v) for k, v in env.items())

def pipe_commands(cmds, extra_env=None, show_stderr=False, show_last_stdout=False):
    """
    Executes the list of commands piping each one into the next.
    """
    env = extend_env(extra_env) if extra_env else None
    env_str = (get_env_str(extra_env) + ' ') if extra_env else ''
    cmd_strs = [env_str + ' '.join(cmd) for cmd in cmds]

    logger.info('Running `{0}`'.format(' | '.join(cmd_strs)))

    with open('/dev/null', 'w') as NULL:
        # Start processes
        processes = []
        last_i = len(cmds) - 1
        for i, (cmd_str, cmd) in enumerate(zip(cmd_strs, cmds)):
            if i == last_i:
                p_stdout = None if show_last_stdout else NULL
            else:
                p_stdout = PIPE
            p_stdin = processes[-1][1].stdout if processes else None
            p_stderr = None if show_stderr else NULL

            p = Popen(cmd, env=env, stdout=p_stdout, stdin=p_stdin, stderr=p_stderr)
            processes.append((cmd_str, p))

        # Close processes
        for cmd_str, p in processes:
            if p.stdout:
                p.stdout.close()
            if p.wait() != 0:
                raise CalledProcessError(cmd=cmd_str, returncode=p.returncode)

def pipe_commands_to_file(cmds, path, extra_env=None, show_stderr=False):
    """
    Executes the list of commands piping each one into the next and writing
    stdout of the last process into a file at the given path.
    """
    env = extend_env(extra_env) if extra_env else None
    env_str = (get_env_str(extra_env) + ' ') if extra_env else ''
    cmd_strs = [env_str + ' '.join(cmd) for cmd in cmds]

    logger.info('Saving output of `{0}`'.format(' | '.join(cmd_strs)))

    with open('/dev/null', 'w') as NULL:
        # Start processes
        processes = []
        for cmd_str, cmd in zip(cmd_strs, cmds):
            p_stdin = processes[-1][1].stdout if processes else None
            p_stderr = None if show_stderr else NULL

            p = Popen(cmd, env=env, stdout=PIPE, stdin=p_stdin, stderr=p_stderr)
            processes.append((cmd_str, p))

        p_last = processes[-1][1]

        with open(path, 'wb') as f:
            shutil.copyfileobj(p_last.stdout, f)

            # Close processes
            for cmd_str, p in processes:
                p.stdout.close()
                if p.wait() != 0:
                    raise CalledProcessError(cmd=cmd_str, returncode=p.returncode)
<|endoftext|>


<commit_before>from scipy.stats import shapiro
import scipy.stats as stats
import numpy as np
import matplotlib.pyplot as plt
from nilearn.plotting import plot_stat_map
from nilearn import image
import nibable as nib
import linear_modeling
import normal_assumption

img = nib.load("../../../data/sub001/BOLD/task003_run001/filtered_func_data_mni.nii.gz")
data = img.get_data()

######### Get n_trs and voxel shape
n_trs = data.shape[-1]
vol_shape = data.shape[:-1]

######### Construct design matrix:
# Load the convolution files.
design_mat = np.ones((n_trs, 9))
files = ('1', '2', '3', '4', '5', '6')
for file in files:
    design_mat[:, (int(file) - 1)] = np.loadtxt('../../../data/convo_prep/task001_run001_cond00'
                                          + file + '_conv.txt')
# adding the linear drifter terms to the design matrix
linear_drift = np.linspace(-1, 1, n_trs)
design_mat[:, 6] = linear_drift
quadratic_drift = linear_drift ** 2
quadratic_drift -= np.mean(quadratic_drift)
design_mat[:, 7] = quadratic_drift

######### we take the mean volume (over time), and do a histogram of the values
mean_vol1 = np.mean(data, axis=-1)

# mask out the outer-brain noise using mean volumes over time.
in_brain_mask = mean_vol > 5000
# We can use this 3D mask to index into our 4D dataset.
# This selects all the voxel time-courses for voxels within the brain
# (as defined by the mask)
in_brain_tcs = data[in_brain_mask, :]

######### Lastly, do t test on betas:
y = in_brain_tcs.T
X = design_mat

beta, MRSS, df = linear_modeling.beta_est(y,X)

errors = y - X.dot(beta)

pval = sw(errors)

#The return, pval should have shape (221783,1) which is a column vector.
# Reshape p values

p_val = np.ones(vol_shape + (pval.shape[1],))
p_val[in_brain_mask, :] = pval

# smoothing
fmri_img = image.smooth_img('../../../data/sub001/BOLD/task001_run001/filtered_func_data_mni.nii.gz', fwhm=6)
mean_img = image.mean_img(fmri_img)
# Thresholding
p_val = np.ones(vol_shape + (pval.shape[1],))
p_val[in_brain_mask, :] = pval

log_p_values = -np.log10(p_val[..., 0])
log_p_values[np.isnan(log_p_values)] = 0.
log_p_values[log_p_values > 10.] = 10.
log_p_values[log_p_values < -np.log10(0.05/137)] = 0
plot_stat_map(nibabel.Nifti1Image(log_p_values, img.get_affine()),
              mean_img, title='SW Test p-values', annotate=False,
              colorbar=True)
<commit_msg>minor change<commit_after>from scipy.stats import shapiro
import scipy.stats as stats
import numpy as np
import matplotlib.pyplot as plt
from nilearn.plotting import plot_stat_map
from nilearn import image
import nibable as nib
import linear_modeling
import normal_assumption

img = nib.load("../../../data/sub001/BOLD/task003_run001/filtered_func_data_mni.nii.gz")
data = img.get_data()

######### Get n_trs and voxel shape
n_trs = data.shape[-1]
vol_shape = data.shape[:-1]

######### Construct design matrix:
# Load the convolution files.
design_mat = np.ones((n_trs, 9))
files = ('1', '2', '3', '4', '5', '6')
for file in files:
    design_mat[:, (int(file) - 1)] = np.loadtxt('../../../data/convo_prep/task001_run001_cond00'
                                          + file + '_conv.txt')
# adding the linear drifter terms to the design matrix
linear_drift = np.linspace(-1, 1, n_trs)
design_mat[:, 6] = linear_drift
quadratic_drift = linear_drift ** 2
quadratic_drift -= np.mean(quadratic_drift)
design_mat[:, 7] = quadratic_drift

######### we take the mean volume (over time), and do a histogram of the values
mean_vol1 = np.mean(data, axis=-1)

# mask out the outer-brain noise using mean volumes over time.
in_brain_mask = mean_vol > 5000
# We can use this 3D mask to index into our 4D dataset.
# This selects all the voxel time-courses for voxels within the brain
# (as defined by the mask)
in_brain_tcs = data[in_brain_mask, :]

######### Lastly, do t test on betas:
y = in_brain_tcs.T
X = design_mat

beta, MRSS, df = linear_modeling.beta_est(y,X)

errors = y - X.dot(beta)

pval = sw(errors)

#The return, pval should have shape (221783,1) which is a column vector.
# Reshape p values

p_val = np.ones(vol_shape + (pval.shape[1],))
p_val[in_brain_mask, :] = pval

# smoothing
fmri_img = image.smooth_img('../../../data/sub001/BOLD/task001_run001/filtered_func_data_mni.nii.gz', fwhm=6)
mean_img = image.mean_img(fmri_img)
# Thresholding
p_val = np.ones(vol_shape + (pval.shape[1],))
p_val[in_brain_mask, :] = pval

log_p_values = -np.log10(p_val[..., 0])
log_p_values[np.isnan(log_p_values)] = 0.
log_p_values[log_p_values > 10.] = 10.
log_p_values[log_p_values < -np.log10(0.05/137)] = 0
plot_stat_map(nib.Nifti1Image(log_p_values, img.get_affine()),
              mean_img, title='SW Test p-values', annotate=False,
              colorbar=True)
<|endoftext|>


<commit_before>"""Common settings and globals."""

from os.path import abspath, basename, dirname, join, normpath
from sys import path

########## PATH CONFIGURATION
# Absolute filesystem path to the Django project directory:
DJANGO_ROOT = dirname(dirname(abspath(__file__)))

# Absolute filesystem path to the top-level project folder:
SITE_ROOT = dirname(DJANGO_ROOT)
# Site name:
SITE_NAME = basename(DJANGO_ROOT)

# Add our project to our pythonpath, this way we don't need to type our project
# name in our dotted import paths:
path.append(DJANGO_ROOT)
########## END PATH CONFIGURATION

########## DEBUG CONFIGURATION
# See: https://docs.djangoproject.com/en/dev/ref/settings/#debug
DEBUG = False

# See: https://docs.djangoproject.com/en/dev/ref/settings/#template-debug
TEMPLATE_DEBUG = DEBUG
########## END DEBUG CONFIGURATION

########## MANAGER CONFIGURATION
# See: https://docs.djangoproject.com/en/dev/ref/settings/#admins
ADMINS = (
    ('Your Name', 'your_email@example.com'),
)

# See: https://docs.djangoproject.com/en/dev/ref/settings/#managers
MANAGERS = ADMINS
########## END MANAGER CONFIGURATION

########## DATABASE CONFIGURATION
# See: https://docs.djangoproject.com/en/dev/ref/settings/#databases
DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.',
        'NAME': '',
        'USER': '',
        'PASSWORD': '',
        'HOST': '',
        'PORT': '',
    }
}
########## END DATABASE CONFIGURATION

########## GENERAL CONFIGURATION
# See: https://docs.djangoproject.com/en/dev/ref/settings/#time-zone
TIME_ZONE = 'Europe/London'

# See: https://docs.djangoproject.com/en/dev/ref/settings/#language-code
LANGUAGE_CODE = 'en-gb'

# See: https://docs.djangoproject.com/en/dev/ref/settings/#site-id
SITE_ID = 1

# See: https://docs.djangoproject.com/en/dev/ref/settings/#use-i18n
USE_I18N = True

# See: https://docs.djangoproject.com/en/dev/ref/settings/#use-l10n
USE_L10N = True
########## END GENERAL CONFIGURATION

########## MEDIA CONFIGURATION
# See: https://docs.djangoproject.com/en/dev/ref/settings/#media-root
MEDIA_ROOT = normpath(join(SITE_ROOT, 'media'))

# See: https://docs.djangoproject.com/en/dev/ref/settings/#media-url
MEDIA_URL = ''
########## END MEDIA CONFIGURATION

########## STATIC FILE CONFIGURATION
# See: https://docs.djangoproject.com/en/dev/ref/settings/#static-root
STATIC_ROOT = ''

# See: https://docs.djangoproject.com/en/dev/ref/settings/#static-url
STATIC_URL = '/static/'

# See: https://docs.djangoproject.com/en/dev/ref/contrib/staticfiles/#std:setting-STATICFILES_DIRS
STATICFILES_DIRS = (
    normpath(join(SITE_ROOT, 'static')),
)

# See: https://docs.djangoproject.com/en/dev/ref/contrib/staticfiles/#staticfiles-finders
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
)
########## END STATIC FILE CONFIGURATION

########## SECRET CONFIGURATION                                                          p
# See: https://docs.djangoproject.com/en/dev/ref/settings/#secret-key
# Note: This key only used for development and testing.
SECRET_KEY = r"%umk1jscu2-85mmiiio(9*kyy3@!ro*6v6j^o=&g7qt6(*vr_j"
########## END SECRET CONFIGURATION

########## TEMPLATE CONFIGURATION
# See: https://docs.djangoproject.com/en/dev/ref/settings/#template-context-processors
TEMPLATE_CONTEXT_PROCESSORS = (
    'django.contrib.auth.context_processors.auth',
    'django.core.context_processors.debug',
    'django.core.context_processors.i18n',
    'django.core.context_processors.media',
    'django.core.context_processors.static',
    'django.core.context_processors.tz',
    'django.contrib.messages.context_processors.messages',
    'django.core.context_processors.request',
)

# See: https://docs.djangoproject.com/en/dev/ref/settings/#template-loaders
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
)

# See: https://docs.djangoproject.com/en/dev/ref/settings/#template-dirs
TEMPLATE_DIRS = (
    normpath(join(SITE_ROOT, 'templates')),
)
########## END TEMPLATE CONFIGURATION

########## MIDDLEWARE CONFIGURATION
# See: https://docs.djangoproject.com/en/dev/ref/settings/#middleware-classes
MIDDLEWARE_CLASSES = (
    # Use GZip compression to reduce bandwidth.
    'django.middleware.gzip.GZipMiddleware',

    # Default Django middleware.
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'core.middleware.SubdomainMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
)
########## END MIDDLEWARE CONFIGURATION

########## URL CONFIGURATION
# See: https://docs.djangoproject.com/en/dev/ref/settings/#root-urlconf
ROOT_URLCONF = '%s.urls' % SITE_NAME
########## END URL CONFIGURATION

########## APP CONFIGURATION
DJANGO_APPS = (
    # Default Django apps:
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',

    # Useful template tags:
    'django.contrib.humanize',

    # Admin panel and documentation:
    'django.contrib.admin',
    'django.contrib.admindocs',

)

THIRD_PARTY_APPS = (
    # Database migration helpers:
    'south',
)

# Apps specific for this project go here.
LOCAL_APPS = (
    'companies',
    'companysettings',
    'openings',
    'public_jobs',
    'accounts',
    'public',
    'applications',
    'core' # Contains views/methods used in other apps, not an app itself
)

# See: https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps
INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS
########## END APP CONFIGURATION

########## LOGGING CONFIGURATION
# See: https://docs.djangoproject.com/en/dev/ref/settings/#logging
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'formatters': {
        'verbose': {
            'format': '%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s'
        },
        'simple': {
            'format': '%(levelname)s %(message)s'
        },
        },
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'null': {
            'level':'DEBUG',
            'class':'django.utils.log.NullHandler',
            },
        'console':{
            'level': 'DEBUG',
            'class': 'logging.StreamHandler',
            'formatter': 'simple'
        },
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler',
            'include_html': True,
        }
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
            }
    }
}
########## END LOGGING CONFIGURATION

########## WSGI CONFIGURATION
# See: https://docs.djangoproject.com/en/dev/ref/settings/#wsgi-application
WSGI_APPLICATION = 'wsgi.application'
########## END WSGI CONFIGURATION

########## DJANGO REGISTRATION
LOGIN_REDIRECT_URL = "/"

ABSOLUTE_URL_OVERRIDES = {
    'auth.user': lambda u: "/companies/create/"
}
########## END DJANGO REGISTRATION

########## CUSTOM USER
AUTH_USER_MODEL = 'accounts.CustomUser'
ACCOUNT_ACTIVATION_DAYS = 7
DEFAULT_FROM_EMAIL = 'Hyrodactil'
LOGIN_URL = '/login/'
SKIP_ACTIVATION = False

SITE_URL = "hyrodactil.com"
SESSION_COOKIE_DOMAIN = "." + SITE_URL
<commit_msg>core: Changed SITE_URL to hizard.com<commit_after>"""Common settings and globals."""

from os.path import abspath, basename, dirname, join, normpath
from sys import path

########## PATH CONFIGURATION
# Absolute filesystem path to the Django project directory:
DJANGO_ROOT = dirname(dirname(abspath(__file__)))

# Absolute filesystem path to the top-level project folder:
SITE_ROOT = dirname(DJANGO_ROOT)
# Site name:
SITE_NAME = basename(DJANGO_ROOT)

# Add our project to our pythonpath, this way we don't need to type our project
# name in our dotted import paths:
path.append(DJANGO_ROOT)
########## END PATH CONFIGURATION

########## DEBUG CONFIGURATION
# See: https://docs.djangoproject.com/en/dev/ref/settings/#debug
DEBUG = False

# See: https://docs.djangoproject.com/en/dev/ref/settings/#template-debug
TEMPLATE_DEBUG = DEBUG
########## END DEBUG CONFIGURATION

########## MANAGER CONFIGURATION
# See: https://docs.djangoproject.com/en/dev/ref/settings/#admins
ADMINS = (
    ('Your Name', 'your_email@example.com'),
)

# See: https://docs.djangoproject.com/en/dev/ref/settings/#managers
MANAGERS = ADMINS
########## END MANAGER CONFIGURATION

########## DATABASE CONFIGURATION
# See: https://docs.djangoproject.com/en/dev/ref/settings/#databases
DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.',
        'NAME': '',
        'USER': '',
        'PASSWORD': '',
        'HOST': '',
        'PORT': '',
    }
}
########## END DATABASE CONFIGURATION

########## GENERAL CONFIGURATION
# See: https://docs.djangoproject.com/en/dev/ref/settings/#time-zone
TIME_ZONE = 'Europe/London'

# See: https://docs.djangoproject.com/en/dev/ref/settings/#language-code
LANGUAGE_CODE = 'en-gb'

# See: https://docs.djangoproject.com/en/dev/ref/settings/#site-id
SITE_ID = 1

# See: https://docs.djangoproject.com/en/dev/ref/settings/#use-i18n
USE_I18N = True

# See: https://docs.djangoproject.com/en/dev/ref/settings/#use-l10n
USE_L10N = True
########## END GENERAL CONFIGURATION

########## MEDIA CONFIGURATION
# See: https://docs.djangoproject.com/en/dev/ref/settings/#media-root
MEDIA_ROOT = normpath(join(SITE_ROOT, 'media'))

# See: https://docs.djangoproject.com/en/dev/ref/settings/#media-url
MEDIA_URL = ''
########## END MEDIA CONFIGURATION

########## STATIC FILE CONFIGURATION
# See: https://docs.djangoproject.com/en/dev/ref/settings/#static-root
STATIC_ROOT = ''

# See: https://docs.djangoproject.com/en/dev/ref/settings/#static-url
STATIC_URL = '/static/'

# See: https://docs.djangoproject.com/en/dev/ref/contrib/staticfiles/#std:setting-STATICFILES_DIRS
STATICFILES_DIRS = (
    normpath(join(SITE_ROOT, 'static')),
)

# See: https://docs.djangoproject.com/en/dev/ref/contrib/staticfiles/#staticfiles-finders
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
)
########## END STATIC FILE CONFIGURATION

########## SECRET CONFIGURATION                                                          p
# See: https://docs.djangoproject.com/en/dev/ref/settings/#secret-key
# Note: This key only used for development and testing.
SECRET_KEY = r"%umk1jscu2-85mmiiio(9*kyy3@!ro*6v6j^o=&g7qt6(*vr_j"
########## END SECRET CONFIGURATION

########## TEMPLATE CONFIGURATION
# See: https://docs.djangoproject.com/en/dev/ref/settings/#template-context-processors
TEMPLATE_CONTEXT_PROCESSORS = (
    'django.contrib.auth.context_processors.auth',
    'django.core.context_processors.debug',
    'django.core.context_processors.i18n',
    'django.core.context_processors.media',
    'django.core.context_processors.static',
    'django.core.context_processors.tz',
    'django.contrib.messages.context_processors.messages',
    'django.core.context_processors.request',
)

# See: https://docs.djangoproject.com/en/dev/ref/settings/#template-loaders
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
)

# See: https://docs.djangoproject.com/en/dev/ref/settings/#template-dirs
TEMPLATE_DIRS = (
    normpath(join(SITE_ROOT, 'templates')),
)
########## END TEMPLATE CONFIGURATION

########## MIDDLEWARE CONFIGURATION
# See: https://docs.djangoproject.com/en/dev/ref/settings/#middleware-classes
MIDDLEWARE_CLASSES = (
    # Use GZip compression to reduce bandwidth.
    'django.middleware.gzip.GZipMiddleware',

    # Default Django middleware.
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'core.middleware.SubdomainMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
)
########## END MIDDLEWARE CONFIGURATION

########## URL CONFIGURATION
# See: https://docs.djangoproject.com/en/dev/ref/settings/#root-urlconf
ROOT_URLCONF = '%s.urls' % SITE_NAME
########## END URL CONFIGURATION

########## APP CONFIGURATION
DJANGO_APPS = (
    # Default Django apps:
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',

    # Useful template tags:
    'django.contrib.humanize',

    # Admin panel and documentation:
    'django.contrib.admin',
    'django.contrib.admindocs',

)

THIRD_PARTY_APPS = (
    # Database migration helpers:
    'south',
)

# Apps specific for this project go here.
LOCAL_APPS = (
    'companies',
    'companysettings',
    'openings',
    'public_jobs',
    'accounts',
    'public',
    'applications',
    'core' # Contains views/methods used in other apps, not an app itself
)

# See: https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps
INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS
########## END APP CONFIGURATION

########## LOGGING CONFIGURATION
# See: https://docs.djangoproject.com/en/dev/ref/settings/#logging
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'formatters': {
        'verbose': {
            'format': '%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s'
        },
        'simple': {
            'format': '%(levelname)s %(message)s'
        },
        },
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'null': {
            'level':'DEBUG',
            'class':'django.utils.log.NullHandler',
            },
        'console':{
            'level': 'DEBUG',
            'class': 'logging.StreamHandler',
            'formatter': 'simple'
        },
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler',
            'include_html': True,
        }
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
            }
    }
}
########## END LOGGING CONFIGURATION

########## WSGI CONFIGURATION
# See: https://docs.djangoproject.com/en/dev/ref/settings/#wsgi-application
WSGI_APPLICATION = 'wsgi.application'
########## END WSGI CONFIGURATION

########## DJANGO REGISTRATION
LOGIN_REDIRECT_URL = "/"

ABSOLUTE_URL_OVERRIDES = {
    'auth.user': lambda u: "/companies/create/"
}
########## END DJANGO REGISTRATION

########## CUSTOM USER
AUTH_USER_MODEL = 'accounts.CustomUser'
ACCOUNT_ACTIVATION_DAYS = 7
DEFAULT_FROM_EMAIL = 'Hyrodactil'
LOGIN_URL = '/login/'
SKIP_ACTIVATION = False

SITE_URL = "hizard.com"
SESSION_COOKIE_DOMAIN = "." + SITE_URL
<|endoftext|>


<commit_before>import vigra
import numpy as np
import h5py
import logging
from hytra.pluginsystem.plugin_manager import TrackingPluginManager
from hytra.core.ilastik_project_options import IlastikProjectOptions

def getLogger():
    ''' logger to be used in this module '''
    return logging.getLogger(__name__)

class RandomForestClassifier:
    """
    A random forest (RF) classifier wraps a list of RFs as used in ilastik,
    and allows to read the RFs trained by ilastik, as well as which features were selected.
    """

    def __init__(self, classifierPath=None, ilpFilename=None, ilpOptions=IlastikProjectOptions(), selectedFeatures=[]):
        """
        Construct a random forest by either loading it from file (`classifierPath` and `ilpFilename` must be given),
        or an empty untrained random forest with specified `selectedFeatures`
        """
        self._options = ilpOptions
        self._classifierPath = classifierPath
        self._ilpFilename = ilpFilename
        if ilpFilename is not None and classifierPath is not None:
            self._randomForests = self._readRandomForests()
            self.selectedFeatures = self._readSelectedFeatures()
        else:
            self._randomForests = []
            self.selectedFeatures = selectedFeatures

    def _readRandomForests(self):
        """
        Read in a list of random forests at a given location in the hdf5 file
        """
        with h5py.File(self._ilpFilename, 'r') as h5file:
            if self._classifierPath == '/':
                fullPath = '/' + self._options.classifierForestsGroupName
            else:
                fullPath = '/'.join([self._classifierPath, self._options.classifierForestsGroupName])
            randomForests = []
            getLogger().info(" Attempting to read {} classifier(s) in {} from {}".format(
                len([key for key in h5file[fullPath].keys() if 'Forest' in key]), self._ilpFilename, fullPath))

            for k in h5file[fullPath].keys():
                if 'Forest' in k:
                    getLogger().info(" Reading forest: {}".format( str('/'.join([fullPath, k])) ) )
                    rf = vigra.learning.RandomForest(str(self._ilpFilename), str('/'.join([fullPath, k])))
                    randomForests.append(rf)
            return randomForests

    def _readSelectedFeatures(self):
        """
        Read which features were selected when training this RF
        """
        with h5py.File(self._ilpFilename, 'r') as h5file:
            if self._classifierPath == '/':
                fullPath = '/' + self._options.selectedFeaturesGroupName
            else:
                fullPath = '/'.join([self._classifierPath, self._options.selectedFeaturesGroupName])
            featureNameList = []

            for feature_group_name in h5file[fullPath].keys():
                feature_group = h5file[fullPath][feature_group_name]
                for feature in feature_group.keys():
                    # discard squared distances feature
                    if feature == 'ChildrenRatio_SquaredDistances':
                        continue

                    # if feature == 'Coord<Principal<Kurtosis>>':
                    #     feature = 'Coord<Principal<Kurtosis> >'
                    # elif feature == 'Coord<Principal<Skewness>>':
                    #     feature = 'Coord<Principal<Skewness> >'

                    featureNameList.append(feature)
            return featureNameList

    def extractFeatureVector(self, featureDict):
        """
        Extract the vector(s) of required features from the given feature dictionary,
        by concatenating the columns of the selected features into a matrix of new features, one row per object
        """
        featureVectors = None
        for f in self.selectedFeatures:
            if f not in featureDict:
                raise AssertionError("Feature '{}' not present in object features!".format(f))
            vec = featureDict[f]
            if len(vec.shape) == 1:
                vec = np.expand_dims(vec, axis=1)
            if featureVectors is None:
                featureVectors = vec
            else:
                if len(vec.shape) == 3:
                    for row in range(vec.shape[2]):
                        featureVectors = np.hstack([featureVectors, vec[..., row]])
                elif len(vec.shape) > 3:
                    raise ValueError("Cannot deal with features of more than two dimensions yet")
                else:
                    featureVectors = np.hstack([featureVectors, vec])

        return featureVectors

    def predictProbabilities(self, features, featureDict=None):
        """
        Given a matrix of features, where each row represents one object and each column is a specific feature,
        this method predicts the probabilities for all classes that this RF knows.

        If features=None but a featureDict is given, the selected features for this random forest are automatically extracted
        """
        assert (len(self._randomForests) > 0)

        # make sure features are good
        if features is None and featureDict is not None:
            features = self.extractFeatureVector(featureDict)
        assert (len(features.shape) == 2)
        # assert(features.shape[1] == self._randomForests[0].featureCount())
        if not features.shape[1] == self._randomForests[0].featureCount():
            getLogger().error(
                "Cannot predict from features of shape {} if {} features are expected".format(features.shape,
                      self._randomForests[0].featureCount()))
            print(features)
            raise AssertionError()

        # predict by summing the probabilities of all the given random forests (not in parallel - not optimized for speed)
        probabilities = np.zeros((features.shape[0], self._randomForests[0].labelCount()))
        for rf in self._randomForests:
            probabilities += rf.predictProbabilities(features.astype('float32'))

        return probabilities

    def train(self, featureMatrix, labels):
        """
        Train the random forest given feature matrix and labels
        """
        getLogger().info(
            "Training classifier from {} positive and {} negative labels".format(
                np.count_nonzero(np.asarray(labels)), len(labels) - np.count_nonzero(np.asarray(labels))))
        getLogger().info("Training classifier from a feature vector of length {}".format(featureMatrix.shape))

        self._randomForests = [vigra.learning.RandomForest()]
        oob = self._randomForests[0].learnRF(
            np.asarray(featureMatrix).astype("float32"),
            (np.asarray(labels)).astype("uint32").reshape(-1, 1))
        getLogger().info("RF trained with OOB Error {}".format(oob))

    def save(self, outputFilename=None, classifierPath='/'):
        """
        Save the random forest to a HDF5 file into a specified path inside the HDF5 file.

        Pass in `None` for both parameters to use the values specified in the constructor.

        """
        if outputFilename is None:
            outputFilename = self._ilpFilename
        assert(outputFilename is not None)

        if classifierPath is None:
            classifierPath = self._classifierPath

        if classifierPath == '/':
            fullPath = '/' + self._options.classifierForestsGroupName + '/Forest0000'
        else:
            fullPath = '/'.join([classifierPath, self._options.classifierForestsGroupName, 'Forest0000'])
        self._randomForests[0].writeHDF5(outputFilename, pathInFile=fullPath)

        # write selected features
        with h5py.File(outputFilename, 'r+') as f:
            featureNamesH5 = f.create_group('SelectedFeatures')
            featureNamesH5 = featureNamesH5.create_group('Standard Object Features')
            for feature in self.selectedFeatures:
                featureNamesH5.create_group(feature)<commit_msg>Make saving the random forest more robust<commit_after>import vigra
import numpy as np
import h5py
import os
import logging
from hytra.pluginsystem.plugin_manager import TrackingPluginManager
from hytra.core.ilastik_project_options import IlastikProjectOptions

def getLogger():
    ''' logger to be used in this module '''
    return logging.getLogger(__name__)

class RandomForestClassifier:
    """
    A random forest (RF) classifier wraps a list of RFs as used in ilastik,
    and allows to read the RFs trained by ilastik, as well as which features were selected.
    """

    def __init__(self, classifierPath=None, ilpFilename=None, ilpOptions=IlastikProjectOptions(), selectedFeatures=[]):
        """
        Construct a random forest by either loading it from file (`classifierPath` and `ilpFilename` must be given),
        or an empty untrained random forest with specified `selectedFeatures`
        """
        self._options = ilpOptions
        self._classifierPath = classifierPath
        self._ilpFilename = ilpFilename
        if ilpFilename is not None and classifierPath is not None:
            self._randomForests = self._readRandomForests()
            self.selectedFeatures = self._readSelectedFeatures()
        else:
            self._randomForests = []
            self.selectedFeatures = selectedFeatures

    def _readRandomForests(self):
        """
        Read in a list of random forests at a given location in the hdf5 file
        """
        with h5py.File(self._ilpFilename, 'r') as h5file:
            if self._classifierPath == '/':
                fullPath = '/' + self._options.classifierForestsGroupName
            else:
                fullPath = '/'.join([self._classifierPath, self._options.classifierForestsGroupName])
            randomForests = []
            getLogger().info(" Attempting to read {} classifier(s) in {} from {}".format(
                len([key for key in h5file[fullPath].keys() if 'Forest' in key]), self._ilpFilename, fullPath))

            for k in h5file[fullPath].keys():
                if 'Forest' in k:
                    getLogger().info(" Reading forest: {}".format( str('/'.join([fullPath, k])) ) )
                    rf = vigra.learning.RandomForest(str(self._ilpFilename), str('/'.join([fullPath, k])))
                    randomForests.append(rf)
            return randomForests

    def _readSelectedFeatures(self):
        """
        Read which features were selected when training this RF
        """
        with h5py.File(self._ilpFilename, 'r') as h5file:
            if self._classifierPath == '/':
                fullPath = '/' + self._options.selectedFeaturesGroupName
            else:
                fullPath = '/'.join([self._classifierPath, self._options.selectedFeaturesGroupName])
            featureNameList = []

            for feature_group_name in h5file[fullPath].keys():
                feature_group = h5file[fullPath][feature_group_name]
                for feature in feature_group.keys():
                    # discard squared distances feature
                    if feature == 'ChildrenRatio_SquaredDistances':
                        continue

                    # if feature == 'Coord<Principal<Kurtosis>>':
                    #     feature = 'Coord<Principal<Kurtosis> >'
                    # elif feature == 'Coord<Principal<Skewness>>':
                    #     feature = 'Coord<Principal<Skewness> >'

                    featureNameList.append(feature)
            return featureNameList

    def extractFeatureVector(self, featureDict):
        """
        Extract the vector(s) of required features from the given feature dictionary,
        by concatenating the columns of the selected features into a matrix of new features, one row per object
        """
        featureVectors = None
        for f in self.selectedFeatures:
            if f not in featureDict:
                raise AssertionError("Feature '{}' not present in object features!".format(f))
            vec = featureDict[f]
            if len(vec.shape) == 1:
                vec = np.expand_dims(vec, axis=1)
            if featureVectors is None:
                featureVectors = vec
            else:
                if len(vec.shape) == 3:
                    for row in range(vec.shape[2]):
                        featureVectors = np.hstack([featureVectors, vec[..., row]])
                elif len(vec.shape) > 3:
                    raise ValueError("Cannot deal with features of more than two dimensions yet")
                else:
                    featureVectors = np.hstack([featureVectors, vec])

        return featureVectors

    def predictProbabilities(self, features, featureDict=None):
        """
        Given a matrix of features, where each row represents one object and each column is a specific feature,
        this method predicts the probabilities for all classes that this RF knows.

        If features=None but a featureDict is given, the selected features for this random forest are automatically extracted
        """
        assert (len(self._randomForests) > 0)

        # make sure features are good
        if features is None and featureDict is not None:
            features = self.extractFeatureVector(featureDict)
        assert (len(features.shape) == 2)
        # assert(features.shape[1] == self._randomForests[0].featureCount())
        if not features.shape[1] == self._randomForests[0].featureCount():
            getLogger().error(
                "Cannot predict from features of shape {} if {} features are expected".format(features.shape,
                      self._randomForests[0].featureCount()))
            print(features)
            raise AssertionError()

        # predict by summing the probabilities of all the given random forests (not in parallel - not optimized for speed)
        probabilities = np.zeros((features.shape[0], self._randomForests[0].labelCount()))
        for rf in self._randomForests:
            probabilities += rf.predictProbabilities(features.astype('float32'))

        return probabilities

    def train(self, featureMatrix, labels):
        """
        Train the random forest given feature matrix and labels
        """
        getLogger().info(
            "Training classifier from {} positive and {} negative labels".format(
                np.count_nonzero(np.asarray(labels)), len(labels) - np.count_nonzero(np.asarray(labels))))
        getLogger().info("Training classifier from a feature vector of length {}".format(featureMatrix.shape))

        self._randomForests = [vigra.learning.RandomForest()]
        oob = self._randomForests[0].learnRF(
            np.asarray(featureMatrix).astype("float32"),
            (np.asarray(labels)).astype("uint32").reshape(-1, 1))
        getLogger().info("RF trained with OOB Error {}".format(oob))

    def save(self, outputFilename=None, classifierPath='/'):
        """
        Save the random forest to a HDF5 file into a specified path inside the HDF5 file.

        Pass in `None` for both parameters to use the values specified in the constructor.

        """
        if outputFilename is None:
            outputFilename = self._ilpFilename
        assert(outputFilename is not None)

        if classifierPath is None:
            classifierPath = self._classifierPath

        if classifierPath == '/':
            fullPath = '/' + os.path.join(self._options.classifierForestsGroupName, 'Forest0000')
        else:
            fullPath = os.path.join(classifierPath, self._options.classifierForestsGroupName, 'Forest0000')
        self._randomForests[0].writeHDF5(outputFilename, pathInFile=fullPath)

        if classifierPath == '/':
            selectedFeaturesPath = 'SelectedFeatures'
        else:
            selectedFeaturesPath = os.path.join(classifierPath, 'SelectedFeatures')

        # write selected features
        with h5py.File(outputFilename, 'r+') as f:
            if selectedFeaturesPath in f:
                del f[selectedFeaturesPath]
            featureNamesH5 = f.create_group(selectedFeaturesPath)
            featureNamesH5 = featureNamesH5.create_group('Standard Object Features')
            for feature in self.selectedFeatures:
                featureNamesH5.create_group(feature)<|endoftext|>