<commit_before>import argparse
import pandas as pd

def parse_arguments():
    info = 'Format prob2020 output into summarized features'
    parser = argparse.ArgumentParser(description=info)
    help_str = 'simmulate_summary output'
    parser.add_argument('-s', '--summary',
                        type=str, required=True,
                        help=help_str)
    help_str = 'TSG output from probabilistic 20/20'
    parser.add_argument('-tsg-test', '--tsg-test',
                        type=str, required=True,
                        help=help_str)
    help_str = 'Oncogene output from probabilistic 20/20'
    parser.add_argument('-og-test', '--og-test',
                        type=str, required=True,
                        help=help_str)
    help_str = 'simulate_non_silent_ratio output'
    parser.add_argument('-n', '--non-silent',
                        type=str,
                        default=None,
                        help=help_str)
    help_str = 'Mutsigcv covariate features'
    parser.add_argument('-c', '--covariates',
                        type=str, required=True,
                        help=help_str)
    help_str = 'Output feature file for 20/20+'
    parser.add_argument('-o', '--output',
                        type=str, required=True,
                        help=help_str)
    args = parser.parse_args()
    return vars(args)

def process_features(df, non_silent_df=None):
    """Processes mutation consequence types from probabilistic 20/20.
    """
    # rename column headers
    rename_dict = {'silent snv': 'silent'}
    df = df.rename(columns=rename_dict)

    # calculate the mean/std for the entire cohort
    if non_silent_df is not None:
        non_silent_df = non_silent_df.rename(columns=lambda x: x.replace(' count', ''))
        non_sil_cols = ['nonsense', 'silent', 'splice site', 'lost stop',
                        'missense', 'lost start']
        total_cts = non_silent_df[non_sil_cols].sum(axis=1)
        mycounts = non_silent_df[non_sil_cols]
        nonsil_to_silent = (df['non-silent snv']+df['inframe indel']+df['frameshift indel']).astype(float)/(df['silent']+1)
        miss_to_silent = (non_silent_df['missense']).astype(float)/(non_silent_df['silent']+1)
        norm_cts = mycounts.div(total_cts.astype(float), axis=0)
        norm_cts = norm_cts.fillna(0.0)
        lost_start_stop = norm_cts['lost stop'] + norm_cts['lost start']
        norm_cts = norm_cts.drop(['lost start', 'lost stop'], axis=1)
        norm_cts['lost start and stop'] = lost_start_stop
        norm_cts['missense to silent'] = miss_to_silent
        norm_cts['non-silent to silent'] = nonsil_to_silent
        feature_means = norm_cts.mean()
        feature_stdev = norm_cts.std()

    # get nonsilent/silent
    nonsilent_to_silent = (df['non-silent snv']+df['inframe indel']+df['frameshift indel']).astype(float)/(df['silent']+1)

    # drop id col
    df = df.drop(['ID', 'non-silent snv'], axis=1)

    # handle mutation counts
    count_cols = ['silent', 'nonsense', 'lost stop', 'lost start', 'missense',
                  'recurrent missense', 'splice site', 'inframe indel', 'frameshift indel']
    mycounts = df[count_cols]
    df['missense'] -= df['recurrent missense']

    # calculate features
    miss_to_silent = (df['missense']+df['recurrent missense']).astype(float)/(df['silent']+1)

    # normalize out of total mutations
    total_cts = df[count_cols].sum(axis=1)
    norm_cts = mycounts.div(total_cts.astype(float), axis=0)
    norm_cts = norm_cts.fillna(0.0)

    # combine lost stop and lost start
    lost_start_stop = norm_cts['lost stop'] + norm_cts['lost start']
    df = df.drop(['lost start', 'lost stop'], axis=1)
    norm_cts = norm_cts.drop(['lost start', 'lost stop'], axis=1)
    count_cols.pop(3)
    count_cols.pop(2)

    df[count_cols] = norm_cts
    df['lost start and stop'] = lost_start_stop
    df['missense to silent'] = miss_to_silent
    df['non-silent to silent'] = nonsilent_to_silent

    # normalize
    if non_silent_df is not None:
        cols = ['nonsense', 'silent', 'splice site', 'lost start and stop',
                'missense', 'missense to silent', 'non-silent to silent']
        normed = (df[cols] - feature_means) #/ feature_stdev.astype(float)
        df[normed.columns] = normed

    return df

def main(opts):
    # read in prob 20/20 files
    count_df = pd.read_csv(opts['summary'], sep='\t')
    tsg_test_df = pd.read_csv(opts['tsg_test'], sep='\t')
    og_test_df = pd.read_csv(opts['og_test'], sep='\t')
    covar_df = pd.read_csv(opts['covariates'], sep='\t')
    og_test_df = og_test_df.rename(columns={'gene':'Gene'})
    tsg_test_df = tsg_test_df.rename(columns={'gene':'Gene'})

    # use cohort simulation if provided
    if opts['non_silent']:
        non_sil_df = pd.read_csv(opts['non_silent'], sep='\t')
    else:
        non_sil_df = None

    # make feature matrix
    feature_df = process_features(count_df, non_sil_df)
    tsg_test_cols = ['Gene', 'deleterious p-value']
    feature_df = pd.merge(feature_df, tsg_test_df[tsg_test_cols],
                          how='left', on='Gene')
    og_test_cols = ['Gene', 'entropy p-value']
    feature_df = pd.merge(feature_df, og_test_df[og_test_cols],
                          how='left', on='Gene')

    # add covariate feature columns
    covar_cols = ['gene',
                  'expression_CCLE',
                  'replication_time',
                  #'noncoding_mutation_rate',
                  'HiC_compartment',
                  ]
    covar_df = covar_df[covar_cols].rename(columns={'gene': 'Gene'})
    feature_df = pd.merge(feature_df, covar_df,
                          how='left', on='Gene')

    # fill na values
    rename_dict = {'Gene': 'gene'}
    feature_df = feature_df.rename(columns=rename_dict)
    feature_df = feature_df.fillna(feature_df.mean())

    feature_df.to_csv(opts['output'], sep='\t', index=False)

if __name__ == "__main__":
    opts = parse_arguments()
    main(opts)
<commit_msg>Updated feature generation script<commit_after>import argparse
import pandas as pd
import numpy as np
import IPython

def parse_arguments():
    info = 'Format prob2020 output into summarized features'
    parser = argparse.ArgumentParser(description=info)
    help_str = 'simmulate_summary output'
    parser.add_argument('-s', '--summary',
                        type=str, required=True,
                        help=help_str)
    help_str = 'TSG output from probabilistic 20/20'
    parser.add_argument('-tsg-test', '--tsg-test',
                        type=str, required=True,
                        help=help_str)
    help_str = 'Oncogene output from probabilistic 20/20'
    parser.add_argument('-og-test', '--og-test',
                        type=str, required=True,
                        help=help_str)
    help_str = 'simulate_non_silent_ratio output'
    parser.add_argument('-n', '--non-silent',
                        type=str,
                        default=None,
                        help=help_str)
    help_str = 'Mutsigcv covariate features'
    parser.add_argument('-c', '--covariates',
                        type=str, required=True,
                        help=help_str)
    help_str = 'Output feature file for 20/20+'
    parser.add_argument('-o', '--output',
                        type=str, required=True,
                        help=help_str)
    args = parser.parse_args()
    return vars(args)

def process_features(df, non_silent_df=None):
    """Processes mutation consequence types from probabilistic 20/20.
    """
    # rename column headers
    rename_dict = {'silent snv': 'silent'}
    df = df.rename(columns=rename_dict)

    # calculate the mean/std for the entire cohort
    if non_silent_df is not None:
        non_silent_df = non_silent_df.rename(columns=lambda x: x.replace(' count', ''))
        non_sil_cols = ['nonsense', 'silent', 'splice site', 'lost stop',
                        'missense', 'lost start']
        total_cts = non_silent_df[non_sil_cols].sum(axis=1)
        mycounts = non_silent_df[non_sil_cols]
        nonsil_to_silent = (df['non-silent snv']+df['inframe indel']+df['frameshift indel']).astype(float)/(df['silent']+1)
        miss_to_silent = (non_silent_df['missense']).astype(float)/(non_silent_df['silent']+1)
        norm_cts = mycounts.div(total_cts.astype(float), axis=0)
        norm_cts = norm_cts.fillna(0.0)
        lost_start_stop = norm_cts['lost stop'] + norm_cts['lost start']
        norm_cts = norm_cts.drop(['lost start', 'lost stop'], axis=1)
        norm_cts['lost start and stop'] = lost_start_stop
        norm_cts['missense to silent'] = miss_to_silent
        norm_cts['non-silent to silent'] = nonsil_to_silent
        feature_means = norm_cts.mean()
        feature_stdev = norm_cts.std()

    # get nonsilent/silent
    nonsilent_to_silent = (df['non-silent snv']+df['inframe indel']+df['frameshift indel']).astype(float)/(df['silent']+1)

    # process score information
    if 'Total Missense MGAEntropy' in df.columns:
        num_mis = df['missense'].copy()
        #num_mis[num_mis==0] = 1
        df['Mean Missense MGAEntropy'] = np.nan
        df.loc[num_mis!=0, 'Mean Missense MGAEntropy'] = df['Total Missense MGAEntropy'][num_mis!=0] / num_mis[num_mis!=0]
        df['Mean Missense MGAEntropy'] = df['Mean Missense MGAEntropy'].fillna(df['Mean Missense MGAEntropy'].max())
        del df['Total Missense MGAEntropy']
    if 'Total Missense VEST Score' in df.columns:
        sum_cols = ['Total Missense VEST Score', 'lost stop',
                    'lost start', 'splice site', 'frameshift indel', 'inframe indel',
                    'nonsense']
        all_muts = ['non-silent snv', 'silent', 'inframe indel', 'frameshift indel']
        tot_vest_score = df[sum_cols].sum(axis=1).astype(float)
        num_muts = df[all_muts].sum(axis=1).astype(float)
        df['Mean VEST Score'] = tot_vest_score / num_muts
        #df['Missense VEST Score'] = df['Total Missense VEST Score'] / num_muts
        #df['VEST normalized missense position entropy'] = df['normalized missense position entropy'] * (1.-df['Missense VEST Score'])
        del df['Total Missense VEST Score']

    # drop id col
    df = df.drop(['ID', 'non-silent snv'], axis=1)

    # handle mutation counts
    count_cols = ['silent', 'nonsense', 'lost stop', 'lost start', 'missense',
                  'recurrent missense', 'splice site', 'inframe indel', 'frameshift indel']
    mycounts = df[count_cols]
    df['missense'] -= df['recurrent missense']

    # calculate features
    miss_to_silent = (df['missense']+df['recurrent missense']).astype(float)/(df['silent']+1)

    # normalize out of total mutations
    total_cts = df[count_cols].sum(axis=1)
    norm_cts = mycounts.div(total_cts.astype(float), axis=0)
    norm_cts = norm_cts.fillna(0.0)

    # combine lost stop and lost start
    lost_start_stop = norm_cts['lost stop'] + norm_cts['lost start']
    df = df.drop(['lost start', 'lost stop'], axis=1)
    norm_cts = norm_cts.drop(['lost start', 'lost stop'], axis=1)
    count_cols.pop(3)
    count_cols.pop(2)

    df[count_cols] = norm_cts
    df['lost start and stop'] = lost_start_stop
    df['missense to silent'] = miss_to_silent
    df['non-silent to silent'] = nonsilent_to_silent

    # normalize
    if non_silent_df is not None:
        cols = ['nonsense', 'silent', 'splice site', 'lost start and stop',
                'missense', 'missense to silent', 'non-silent to silent']
        normed = (df[cols] - feature_means) #/ feature_stdev.astype(float)
        df[normed.columns] = normed

    return df

def main(opts):
    # read in prob 20/20 files
    count_df = pd.read_csv(opts['summary'], sep='\t')
    tsg_test_df = pd.read_csv(opts['tsg_test'], sep='\t')
    og_test_df = pd.read_csv(opts['og_test'], sep='\t')
    covar_df = pd.read_csv(opts['covariates'], sep='\t')
    og_test_df = og_test_df.rename(columns={'gene':'Gene'})
    tsg_test_df = tsg_test_df.rename(columns={'gene':'Gene'})

    # use cohort simulation if provided
    if opts['non_silent']:
        non_sil_df = pd.read_csv(opts['non_silent'], sep='\t')
    else:
        non_sil_df = None

    # make feature matrix
    feature_df = process_features(count_df, non_sil_df)
    tsg_test_cols = ['Gene', 'inactivating p-value']
    feature_df = pd.merge(feature_df, tsg_test_df[tsg_test_cols],
                          how='left', on='Gene')
    og_test_cols = ['Gene', 'entropy p-value', 'vest p-value', 'combined p-value']
    feature_df = pd.merge(feature_df, og_test_df[og_test_cols],
                          how='left', on='Gene')

    # add covariate feature columns
    covar_cols = ['gene',
                  'expression_CCLE',
                  'replication_time',
                  #'noncoding_mutation_rate',
                  'HiC_compartment',
                  ]
    covar_df = covar_df[covar_cols].rename(columns={'gene': 'Gene'})
    feature_df = pd.merge(feature_df, covar_df,
                          how='left', on='Gene')

    # fill na values
    rename_dict = {'Gene': 'gene'}
    feature_df = feature_df.rename(columns=rename_dict)
    feature_df = feature_df.fillna(feature_df.mean())

    feature_df.to_csv(opts['output'], sep='\t', index=False)

if __name__ == "__main__":
    opts = parse_arguments()
    main(opts)
<|endoftext|>


<commit_before>"""Common path names."""

# TODO: Get rid of messy globbing by by explicit data file naming.
# TODO: Move all functions to a class.

from __future__ import absolute_import, division, print_function

from .types import ImageMode, Path

def _fmt_tspec(tspec):
    parts = filter(None, [tspec.method, tspec.winsize, tspec.feature])
    return '-'.join(str(x) for x in parts)

class Paths(object):
    def __init__(self, mode):
        self.mode = ImageMode(mode)

    def samplelist(self, samplelist):
        """Return path to samplelist."""
        d = dict(m=self.mode, l=samplelist)
        return Path('patients') / 'patients_{m[0]}_{l}.txt'.format(**d)

    def pmap(self, case=None, scan=None, fmt='dicom'):
        """Return path to pmap."""
        if 'std' in self.mode or self.mode == 'T2-fitted':
            fmt = 'h5'  # TODO: Temporary redirection.
        d = dict(m=self.mode, m_=self.mode[:2], c=case, s=scan)
        if fmt == 'h5':
            path = 'images/{m_}'
            if case is not None and scan is not None:
                path += '/{c}-{s}.h5'
            return Path(path.format(**d))
        elif fmt == 'dicom':
            if len(self.mode) == 1:
                path = 'dicoms/{m[0]}_*'
            else:
                path = 'dicoms/{m[1]}_*'
            if case is not None and scan is not None:
                if self.mode == 'DWI':
                    path += '/{c}_hB_{s}.zip'
                elif len(self.mode) == 1:
                    path += '/{c}_{s}*'
                else:
                    path += '/{c}_*_{s}/{c}_*_{s}*_{m[2]}.zip'
            path, = Path().glob(path.format(**d))
            return path
        else:
            raise ValueError('Unknown format: {}'.format(fmt))

    def subregion(self, case=None, scan=None):
        """Return path to subregion file. XXX: Obsolete."""
        d = dict(m=self.mode, c=case, s=scan)
        path = Path('subregions') / '{m[0]}'.format(**d)
        if case is not None and scan is not None:
            path /= '{c}_{s}_subregion10.txt'.format(**d)
        return path

    def mask(self, masktype, case, scan, lesion=None, algparams=()):
        """Return path and deps of masks of different types."""
        if masktype == 'all':
            return None
        d = dict(m=self.mode, mt=masktype, c=case, s=scan, l=lesion,
                 ap_='_'.join(algparams))
        do_glob = True
        if masktype == 'prostate':
            path = 'masks_{mt}/{m[0]}/{c}_*_{s}*.h5'
        elif masktype == 'lesion':
            path = ('masks_{mt}/'
                    '{m[0]}/PCa_masks_{m[0]}_lesion{l}/{c}_*{s}_{m[0]}.h5')
        elif masktype in ('CA', 'N'):
            path = 'masks_roi/{m[0]}/{c}_*_{s}_D_{mt}.h5'
        elif masktype == 'auto':
            path = 'masks_{mt}/{m}/{ap_}/{c}_{s}_auto.mask'
            do_glob = False  # Don't require existence, can be generated.
        else:
            raise Exception('Unknown mask type: {mt}'.format(**d))
        path = path.format(**d)
        if do_glob:
            path, = Path().glob(path)
        return Path(path)

    def roi(self, masktype, case=None, scan=None, lesion=None, algparams=()):
        """Return whole ROI path or part of it."""
        if masktype == 'image':
            return self.pmap(case, scan)  # No ROI, but the whole image.
        d = dict(m=self.mode, mt=masktype, c=case, s=scan, l=lesion,
                 ap_='_'.join(algparams))
        path = 'rois_{mt}/{m}'
        if algparams:
            path += '/{ap_}'
        if case is not None and scan is not None:
            if masktype == 'prostate':
                path += '/{c}-{s}.h5'
            elif masktype == 'lesion':
                path += '/{c}-{s}-{l}.h5'
            else:
                path += '/{c}_x_x_{s}_{m}_{mt}.txt'
        return Path(path.format(**d))

    def texture(self, case, scan, lesion, masktype, slices, portion, tspec,
                algparams=(), voxel='mean'):
        """Return path to texture file."""
        if tspec is not None and tspec.method == 'raw':
            # 'Raw' texture method is actually just the source image.
            return self.pmap(case, scan)
        d = dict(m=self.mode, c=case, s=scan, l=lesion, mt=masktype,
                 slices=slices, portion=portion, tspec=tspec,
                 ap_='_'.join(algparams), vx=voxel, ext='txt')
        if voxel == 'all':
            d['ext'] = 'h5'
        path = 'texture_{mt}/{m}_{slices}_{portion}_{vx}'
        if masktype == 'auto':
            path += '/{ap_}'
        path += '/{c}_{s}_{l}'
        if tspec is not None:
            path += '_{tspec.method}-{tspec.winsize}'
        path += '.{ext}'
        return Path(path.format(**d))

    def std_cfg(self):
        """Return path to standardization configuration file."""
        return Path('stdcfg_{m}.txt'.format(m=self.mode))

    def histogram(self, roi, samplelist):
        """Return path to histogram plot."""
        return Path('histograms/{m}_{r}_{s}.png'.format(m=self.mode, r=roi,
                                                        s=samplelist))

    def grid(self, case, scan, mt, tspec, fmt='txt'):
        """Return path to the first of the grid files.

        FIXME: The first element in the resulting tuple no more exists as file.
        """
        path = Path('grid_{mt}/{m}'.format(mt=mt, m=self.mode))
        if tspec is not None:
            path /= _fmt_tspec(tspec)
        if case is not None and scan is not None:
            path /= '{c}-{s}.{f}'.format(c=case, s=scan, f=fmt)
        target = '{r}-0{e}'.format(r=path.stem, e=path.suffix)
        return path, target

# The rest are for compatibility.

def samplelist_path(mode, samplelist):
    return str(Paths(mode).samplelist(samplelist))

def pmap_path(mode, **kwargs):
    return str(Paths(mode).pmap(**kwargs))

def subregion_path(mode, **kwargs):
    return str(Paths(mode).subregion(**kwargs))

def mask_path(mode, masktype, case, scan, **kwargs):
    return str(Paths(mode).mask(masktype, case, scan, **kwargs))

def roi_path(mode, masktype, **kwargs):
    return str(Paths(mode).roi(masktype, **kwargs))

def texture_path(mode, case, scan, lesion, masktype, slices, portion, tspec,
                 **kwargs):
    return str(Paths(mode).texture(case, scan, lesion, masktype, slices,
                                   portion, tspec, **kwargs))

def std_cfg_path(mode):
    return str(Paths(mode).std_cfg())

def histogram_path(mode, roi, samplelist):
    return str(Paths(mode).histogram(roi, samplelist))

def grid_path(mode, case, scan, mt, tspec, **kwargs):
    return tuple(str(x) for x in Paths(mode).grid(case, scan, mt, tspec,
                                                  **kwargs))
<commit_msg>use images/ instead of dicoms/<commit_after>"""Common path names."""

# TODO: Get rid of messy globbing by by explicit data file naming.
# TODO: Move all functions to a class.

from __future__ import absolute_import, division, print_function

from .types import ImageMode, Path

def _fmt_tspec(tspec):
    parts = filter(None, [tspec.method, tspec.winsize, tspec.feature])
    return '-'.join(str(x) for x in parts)

class Paths(object):
    def __init__(self, mode):
        self.mode = ImageMode(mode)

    def samplelist(self, samplelist):
        """Return path to samplelist."""
        d = dict(m=self.mode, l=samplelist)
        return Path('patients') / 'patients_{m[0]}_{l}.txt'.format(**d)

    def pmap(self, case=None, scan=None, fmt='dicom'):
        """Return path to pmap."""
        if 'std' in self.mode or self.mode == 'T2-fitted':
            fmt = 'h5'  # TODO: Temporary redirection.
        d = dict(m=self.mode, m_=self.mode[:2], c=case, s=scan)
        path = 'images'
        if fmt == 'h5':
            path += '/{m_}'
            if case is not None and scan is not None:
                path += '/{c}-{s}.h5'
            return Path(path.format(**d))
        elif fmt == 'dicom':
            if len(self.mode) == 1:
                path += '/{m[0]}_*'
            else:
                path += '/{m[1]}_*'
            if case is not None and scan is not None:
                if self.mode == 'DWI':
                    path += '/{c}_hB_{s}.zip'
                elif len(self.mode) == 1:
                    path += '/{c}_{s}*'
                else:
                    path += '/{c}_*_{s}/{c}_*_{s}*_{m[2]}.zip'
            path, = Path().glob(path.format(**d))
            return path
        else:
            raise ValueError('Unknown format: {}'.format(fmt))

    def subregion(self, case=None, scan=None):
        """Return path to subregion file. XXX: Obsolete."""
        d = dict(m=self.mode, c=case, s=scan)
        path = Path('subregions') / '{m[0]}'.format(**d)
        if case is not None and scan is not None:
            path /= '{c}_{s}_subregion10.txt'.format(**d)
        return path

    def mask(self, masktype, case, scan, lesion=None, algparams=()):
        """Return path and deps of masks of different types."""
        if masktype == 'all':
            return None
        d = dict(m=self.mode, mt=masktype, c=case, s=scan, l=lesion,
                 ap_='_'.join(algparams))
        do_glob = True
        if masktype == 'prostate':
            path = 'masks_{mt}/{m[0]}/{c}_*_{s}*.h5'
        elif masktype == 'lesion':
            path = ('masks_{mt}/'
                    '{m[0]}/PCa_masks_{m[0]}_lesion{l}/{c}_*{s}_{m[0]}.h5')
        elif masktype in ('CA', 'N'):
            path = 'masks_roi/{m[0]}/{c}_*_{s}_D_{mt}.h5'
        elif masktype == 'auto':
            path = 'masks_{mt}/{m}/{ap_}/{c}_{s}_auto.mask'
            do_glob = False  # Don't require existence, can be generated.
        else:
            raise Exception('Unknown mask type: {mt}'.format(**d))
        path = path.format(**d)
        if do_glob:
            path, = Path().glob(path)
        return Path(path)

    def roi(self, masktype, case=None, scan=None, lesion=None, algparams=()):
        """Return whole ROI path or part of it."""
        if masktype == 'image':
            return self.pmap(case, scan)  # No ROI, but the whole image.
        d = dict(m=self.mode, mt=masktype, c=case, s=scan, l=lesion,
                 ap_='_'.join(algparams))
        path = 'rois_{mt}/{m}'
        if algparams:
            path += '/{ap_}'
        if case is not None and scan is not None:
            if masktype == 'prostate':
                path += '/{c}-{s}.h5'
            elif masktype == 'lesion':
                path += '/{c}-{s}-{l}.h5'
            else:
                path += '/{c}_x_x_{s}_{m}_{mt}.txt'
        return Path(path.format(**d))

    def texture(self, case, scan, lesion, masktype, slices, portion, tspec,
                algparams=(), voxel='mean'):
        """Return path to texture file."""
        if tspec is not None and tspec.method == 'raw':
            # 'Raw' texture method is actually just the source image.
            return self.pmap(case, scan)
        d = dict(m=self.mode, c=case, s=scan, l=lesion, mt=masktype,
                 slices=slices, portion=portion, tspec=tspec,
                 ap_='_'.join(algparams), vx=voxel, ext='txt')
        if voxel == 'all':
            d['ext'] = 'h5'
        path = 'texture_{mt}/{m}_{slices}_{portion}_{vx}'
        if masktype == 'auto':
            path += '/{ap_}'
        path += '/{c}_{s}_{l}'
        if tspec is not None:
            path += '_{tspec.method}-{tspec.winsize}'
        path += '.{ext}'
        return Path(path.format(**d))

    def std_cfg(self):
        """Return path to standardization configuration file."""
        return Path('stdcfg_{m}.txt'.format(m=self.mode))

    def histogram(self, roi, samplelist):
        """Return path to histogram plot."""
        return Path('histograms/{m}_{r}_{s}.png'.format(m=self.mode, r=roi,
                                                        s=samplelist))

    def grid(self, case, scan, mt, tspec, fmt='txt'):
        """Return path to the first of the grid files.

        FIXME: The first element in the resulting tuple no more exists as file.
        """
        path = Path('grid_{mt}/{m}'.format(mt=mt, m=self.mode))
        if tspec is not None:
            path /= _fmt_tspec(tspec)
        if case is not None and scan is not None:
            path /= '{c}-{s}.{f}'.format(c=case, s=scan, f=fmt)
        target = '{r}-0{e}'.format(r=path.stem, e=path.suffix)
        return path, target

# The rest are for compatibility.

def samplelist_path(mode, samplelist):
    return str(Paths(mode).samplelist(samplelist))

def pmap_path(mode, **kwargs):
    return str(Paths(mode).pmap(**kwargs))

def subregion_path(mode, **kwargs):
    return str(Paths(mode).subregion(**kwargs))

def mask_path(mode, masktype, case, scan, **kwargs):
    return str(Paths(mode).mask(masktype, case, scan, **kwargs))

def roi_path(mode, masktype, **kwargs):
    return str(Paths(mode).roi(masktype, **kwargs))

def texture_path(mode, case, scan, lesion, masktype, slices, portion, tspec,
                 **kwargs):
    return str(Paths(mode).texture(case, scan, lesion, masktype, slices,
                                   portion, tspec, **kwargs))

def std_cfg_path(mode):
    return str(Paths(mode).std_cfg())

def histogram_path(mode, roi, samplelist):
    return str(Paths(mode).histogram(roi, samplelist))

def grid_path(mode, case, scan, mt, tspec, **kwargs):
    return tuple(str(x) for x in Paths(mode).grid(case, scan, mt, tspec,
                                                  **kwargs))
<|endoftext|>


<commit_before>import binascii
import struct
import math
import os
import sys

def unpack_values(data, length):
    # unpack data stream

    values = []

    pixel = 0
    ptr = 0

    while len(values) < length:

        delta = struct.unpack('b', data[ptr])[0]
        ptr += 1

        if delta != -128:
            pixel += delta
            values.append(pixel)
            continue

        delta = struct.unpack('<h', data[ptr:ptr + 2])[0]
        ptr += 2

        if delta != -32768:
            pixel += delta
            values.append(pixel)
            continue

        delta = struct.unpack('<i', data[ptr:ptr + 4])[0]
        ptr += 4

        if delta != -2147483648:
            pixel += delta
            values.append(pixel)
            continue

        delta = struct.unpack('<q', data[ptr:ptr + 8])[0]
        ptr += 8
        pixel += delta
        values.append(pixel)

    return values

def unpackbyteoffset(filename):
    data = open(filename, 'r').read()

    start_tag = binascii.unhexlify('0c1a04d5')

    data_offset = data.find(start_tag) + 4

    header = data[:data.find(start_tag)].split('\n')

    fast = 0
    slow = 0
    length = 0

    for record in header:
        if 'X-Binary-Size-Fastest-Dimension' in record:
            fast = int(record.split()[-1])
        if 'X-Binary-Size-Second-Dimension' in record:
            slow = int(record.split()[-1])
        if 'X-Binary-Number-of-Elements' in record:
            length = int(record.split()[-1])

    assert(length == fast * slow)

    values = unpack_values(data[data_offset:], length)

    for i in range(slow):
        for j in range(fast):
            k = i * fast + j
            print i, j, values[k]

if __name__ == '__main__':

    unpackbyteoffset(sys.argv[1])
<commit_msg>Get a histogram: analysing cbfs from rescale..<commit_after>import binascii
import struct
import math
import os
import sys

def unpack_values(data, length):
    # unpack data stream

    values = []

    pixel = 0
    ptr = 0

    while len(values) < length:

        delta = struct.unpack('b', data[ptr])[0]
        ptr += 1

        if delta != -128:
            pixel += delta
            values.append(pixel)
            continue

        delta = struct.unpack('<h', data[ptr:ptr + 2])[0]
        ptr += 2

        if delta != -32768:
            pixel += delta
            values.append(pixel)
            continue

        delta = struct.unpack('<i', data[ptr:ptr + 4])[0]
        ptr += 4

        if delta != -2147483648:
            pixel += delta
            values.append(pixel)
            continue

        delta = struct.unpack('<q', data[ptr:ptr + 8])[0]
        ptr += 8
        pixel += delta
        values.append(pixel)

    return values

def unpackbyteoffset(filename):

    hist = [0 for j in range 65536]

    data = open(filename, 'r').read()

    start_tag = binascii.unhexlify('0c1a04d5')

    data_offset = data.find(start_tag) + 4

    header = data[:data.find(start_tag)].split('\n')

    fast = 0
    slow = 0
    length = 0

    for record in header:
        if 'X-Binary-Size-Fastest-Dimension' in record:
            fast = int(record.split()[-1])
        if 'X-Binary-Size-Second-Dimension' in record:
            slow = int(record.split()[-1])
        if 'X-Binary-Number-of-Elements' in record:
            length = int(record.split()[-1])

    assert(length == fast * slow)

    values = unpack_values(data[data_offset:], length)

    for i in range(slow):
        for j in range(fast):
            k = i * fast + j
            hist[values[k]] += 1

    return hist, max(values)

if __name__ == '__main__':

    hist, maximum = unpackbyteoffset(sys.argv[1])

    for j in range(maximum + 1):
        print j, histogram[j]
<|endoftext|>


<commit_before># -*- coding: utf8 -*-
# This file is part of PYBOSSA.
#
# Copyright (C) 2015 Scifabric LTD.
#
# PYBOSSA is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# PYBOSSA is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with PYBOSSA.  If not, see <http://www.gnu.org/licenses/>.

from pybossa.model.user import User
from . import BaseFactory, factory, user_repo

class UserFactory(BaseFactory):
    class Meta:
        model = User

    @classmethod
    def _create(cls, model_class, *args, **kwargs):
        user = model_class(*args, **kwargs)
        user_repo.save(user)
        return user

    id = factory.Sequence(lambda n: n)
    name = factory.Sequence(lambda n: u'user%d' % n)
    fullname = factory.Sequence(lambda n: u'User %d' % n)
    email_addr = factory.LazyAttribute(lambda usr: u'%s@test.com' % usr.name)
    locale = u'en'
    admin = False
    pro = False
    ldap = None
    subscribed = True
    privacy_mode = True
    consent = False
    api_key = factory.Sequence(lambda n: u'api-key%d' % n)
    info = dict(foo='bar', container='container',
                avatar='img.png',
                avatar_url='http://cdn.com/container/img.png')
<commit_msg>Add flag.<commit_after># -*- coding: utf8 -*-
# This file is part of PYBOSSA.
#
# Copyright (C) 2015 Scifabric LTD.
#
# PYBOSSA is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# PYBOSSA is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with PYBOSSA.  If not, see <http://www.gnu.org/licenses/>.

from pybossa.model.user import User
from . import BaseFactory, factory, user_repo

class UserFactory(BaseFactory):
    class Meta:
        model = User

    @classmethod
    def _create(cls, model_class, *args, **kwargs):
        user = model_class(*args, **kwargs)
        user_repo.save(user)
        return user

    id = factory.Sequence(lambda n: n)
    name = factory.Sequence(lambda n: u'user%d' % n)
    fullname = factory.Sequence(lambda n: u'User %d' % n)
    email_addr = factory.LazyAttribute(lambda usr: u'%s@test.com' % usr.name)
    locale = u'en'
    admin = False
    pro = False
    ldap = None
    subscribed = True
    privacy_mode = True
    restrict = False
    consent = False
    api_key = factory.Sequence(lambda n: u'api-key%d' % n)
    info = dict(foo='bar', container='container',
                avatar='img.png',
                avatar_url='http://cdn.com/container/img.png')
<|endoftext|>


<commit_before>""" Processor class define for Scan and Session."""

import fnmatch
import logging
import re
import os

from . import XnatUtils, task

__copyright__ = 'Copyright 2013 Vanderbilt University. All Rights Reserved'
__all__ = ['Processor', 'ScanProcessor', 'SessionProcessor']
# Logger for logs
LOGGER = logging.getLogger('dax')

class Processor(object):
    """ Base class for processor """
    def __init__(self, walltime_str, memreq_mb, spider_path,
                 version=None, ppn=1, suffix_proc='',
                 xsitype='proc:genProcData'):
        """
        Entry point of the Base class for processor.

        :param walltime_str: Amount of walltime to request for the process
        :param memreq_mb: Number of megabytes of memory to use
        :param spider_path: Fully qualified path to the spider to run
        :param version: Version of the spider
        :param ppn: Number of processors per not to use.
        :param suffix_proc: Processor suffix (if desired)
        :param xsitype: the XNAT xsiType.
        :return: None

        """
        self.walltime_str = walltime_str  # 00:00:00 format
        self.memreq_mb = memreq_mb   # memory required in megabytes
        # default values:
        self.version = "1.0.0"
        # Suffix
        if suffix_proc and suffix_proc[0] != '_':
            self.suffix_proc = '_%s' % suffix_proc
        else:
            self.suffix_proc = suffix_proc
        self.suffix_proc = re.sub('[^a-zA-Z0-9]', '_', self.suffix_proc)
        self.name = None
        self.spider_path = spider_path
        self.ppn = ppn
        self.xsitype = xsitype
        # getting name and version from spider_path
        self.set_spider_settings(spider_path, version)
        # if suffix_proc is empty, set it to "" for the spider call:
        if not suffix_proc:
            self.suffix_proc = ''

    # get the spider_path right with the version:
    def set_spider_settings(self, spider_path, version):
        """
        Method to set the spider version, path, and name from filepath

        :param spider_path: Fully qualified path and file of the spider
        :param version: version of the spider
        :return: None

        """
        if version:
            # get the proc_name
            proc_name = os.path.basename(spider_path)[7:-3]
            # remove any version if there is one
            proc_name = re.split("/*_v[0-9]/*", proc_name)[0]
            # setting the version and name of the spider
            self.version = version
            nformat = '''{procname}_v{version}{suffix}'''
            self.name = nformat.format(procname=proc_name,
                                       version=self.version.split('.')[0],
                                       suffix=self.suffix_proc)
            sformat = '''Spider_{procname}_v{version}.py'''
            spider_name = sformat.format(procname=proc_name,
                                         version=version.replace('.', '_'))
            self.spider_path = os.path.join(os.path.dirname(spider_path),
                                            spider_name)
        else:
            self.default_settings_spider(spider_path)

    def default_settings_spider(self, spider_path):
        """
        Get the default spider version and name

        :param spider_path: Fully qualified path and file of the spider
        :return: None

        """
        # set spider path
        self.spider_path = spider_path
        # set the name and the version of the spider
        if len(re.split("/*_v[0-9]/*", spider_path)) > 1:
            basename = os.path.basename(spider_path)
            self.version = basename[7:-3].split('_v')[-1].replace('_', '.')
            spidername = basename[7:-3]
            self.name = '''{procname}_v{version}{suffix}'''.format(
                procname=re.split("/*_v[0-9]/*", spidername)[0],
                version=self.version.split('.')[0], suffix=self.suffix_proc)
        else:
            self.name = os.path.basename(spider_path)[7:-3] + self.suffix_proc

    # has_inputs - does this object have the required inputs?
    # e.g. NIFTI format of the required scan type and quality
    #      and are there no conflicting inputs.
    # i.e. only 1 required by 2 found?
    # other arguments here, could be Proj/Subj/Sess/Scan/Assessor depending
    # on processor type?
    def has_inputs(self):
        """
        Check to see if the spider has all the inputs necessary to run.

        :raises: NotImplementedError if user does not override
        :return: None

        """
        raise NotImplementedError()

    # should_run - is the object of the proper object type?
    # e.g. is it a scan? and is it the required scan type?
    # e.g. is it a T1?
    # other arguments here, could be Proj/Subj/Sess/Scan/Assessor depending
    # on processor type?
    def should_run(self):
        """
        Responsible for determining if the assessor should shouw up in session.

        :raises: NotImplementedError if not overridden.
        :return: None

        """
        raise NotImplementedError()

    def build_cmds(self, cobj, dir):

        """
        Build the commands that will go in the PBS/SLURM script
        :raises: NotImplementedError if not overridden from base class.
        :return: None
        """
        raise NotImplementedError()

class ScanProcessor(Processor):
    """ Scan Processor class for processor on a scan on XNAT """
    def __init__(self, scan_types, walltime_str, memreq_mb, spider_path,
                 version=None, ppn=1, suffix_proc='', full_regex=False):
        """
        Entry point of the ScanProcessor Class.

        :param scan_types: Types of scans that the spider should run on
        :param walltime_str: Amount of walltime to request for the process
        :param memreq_mb: Amount of memory in MB to request for the process
        :param spider_path: Absolute path to the spider
        :param version: Version of the spider (taken from the file name)
        :param ppn: Number of processors per node to request
        :param suffix_proc: Processor suffix
        :param full_regex: use full regex
        :return: None

        """
        super(ScanProcessor, self).__init__(walltime_str, memreq_mb,
                                            spider_path, version, ppn,
                                            suffix_proc)
        self.full_regex = full_regex
        if isinstance(scan_types, list):
            self.scan_types = scan_types
        elif isinstance(scan_types, str):
            if scan_types == 'all':
                self.scan_types = 'all'
            else:
                self.scan_types = scan_types.split(',')
        else:
            self.scan_types = []

    def has_inputs(self):
        """
        Method to check and see that the process has all of the inputs
         that it needs to run.

        :raises: NotImplementedError if not overridden.
        :return: None

        """
        raise NotImplementedError()

    def get_assessor(self, cscan):
        """
        Returns the label of the assessor

        :param cscan: CachedImageScan object from XnatUtils
        :return: String of the assessor label

        """
        scan_dict = cscan.info()
        subj_label = scan_dict['subject_label']
        sess_label = scan_dict['session_label']
        proj_label = scan_dict['project_label']
        scan_label = scan_dict['scan_label']
        assr_name = '-x-'.join([proj_label, subj_label, sess_label, scan_label,
                                self.name])

        # Check if shared project:
        csess = cscan.parent()
        proj_shared = csess.has_shared_project()
        assr_name_shared = None
        if proj_shared is not None:
            assr_name_shared = '-x-'.join([proj_shared, subj_label, sess_label,
                                           scan_label, self.name])

        # Look for existing assessor
        p_assr = None
        assr_label = assr_name
        for assr in csess.assessors():
            if assr_name_shared is not None and \
               assr.info()['label'] == assr_name_shared:
                p_assr = assr
                assr_label = assr_name_shared
                break
            if assr.info()['label'] == assr_name:
                p_assr = assr
                break

        return p_assr, assr_label

    def get_task(self, intf, cscan, upload_dir):
        """
        Get the Task object

        :param intf: XNAT interface (pyxnat.Interface class)
        :param cscan: CachedImageScan object from XnatUtils
        :param upload_dir: the directory to put the processed data when the
         process is done
        :return: Task object

        """
        scan_dict = cscan.info()
        _, assessor_name = self.get_assessor(cscan)
        scan = XnatUtils.get_full_object(intf, scan_dict)
        assessor = scan.parent().assessor(assessor_name)
        return task.Task(self, assessor, upload_dir)

    def should_run(self, scan_dict):
        """
        Method to see if the assessor should appear in the session.

        :param scan_dict: Dictionary of information about the scan
        :return: True if it should run, false if it shouldn't

        """
        if self.scan_types == 'all':
            return True
        else:
            for expression in self.scan_types:
                regex = XnatUtils.extract_exp(expression, self.full_regex)
                if regex.match(scan_dict['scan_type']):
                    return True
            return False

class SessionProcessor(Processor):
    """ Session Processor class for processor on a session on XNAT """
    def __init__(self, walltime_str, memreq_mb, spider_path, version=None,
                 ppn=1, suffix_proc=''):
        """
        Entry point for the session processor

        :param walltime_str: Amount of walltime to request for the process
        :param memreq_mb: Amount of memory in MB to request for the process
        :param spider_path: Absolute path to the spider
        :param version: Version of the spider (taken from the file name)
        :param ppn: Number of processors per node to request
        :param suffix_proc: Processor suffix
        :return: None

        """
        super(SessionProcessor, self).__init__(walltime_str, memreq_mb,
                                               spider_path, version, ppn,
                                               suffix_proc)

    def has_inputs(self):
        """
        Check to see that the session has the required inputs to run.

        :raises: NotImplementedError if not overriden from base class.
        :return: None
        """
        raise NotImplementedError()

    def should_run(self, session_dict):
        """
        By definition, this should always run, so it just returns true
         with no checks

        :param session_dict: Dictionary of session information for
         XnatUtils.list_experiments()
        :return: True

        """
        return True

    def get_assessor(self, csess):
        """
        Get the name of the assessor

        :param csess: CachedImageSession from XnatUtils
        :return: String of the assessor label

        """
        session_dict = csess.info()
        proj_label = session_dict['project']
        subj_label = session_dict['subject_label']
        sess_label = session_dict['label']
        assr_name = '-x-'.join([proj_label, subj_label, sess_label, self.name])

        # Check if shared project:
        proj_shared = csess.has_shared_project()
        assr_name_shared = None
        if proj_shared is not None:
            assr_name_shared = '-x-'.join([proj_shared, subj_label, sess_label,
                                           self.name])

        # Look for existing assessor
        p_assr = None
        assr_label = assr_name
        for assr in csess.assessors():
            if assr_name_shared is not None and \
               assr.info()['label'] == assr_name_shared:
                p_assr = assr
                assr_label = assr_name_shared
                break
            if assr.info()['label'] == assr_name:
                p_assr = assr
                break

        return p_assr, assr_label

    def get_task(self, intf, csess, upload_dir):
        """
        Return the Task object

        :param intf: XNAT interface see pyxnat.Interface
        :param csess: CachedImageSession from XnatUtils
        :param upload_dir: directory to put the data after run on the node
        :return: Task object of the assessor

        """
        sess_info = csess.info()
        _, assessor_name = self.get_assessor(csess)
        session = XnatUtils.get_full_object(intf, sess_info)
        assessor = session.assessor(assessor_name)
        return task.Task(self, assessor, upload_dir)

def processors_by_type(proc_list):
    """
    Organize the processor types and return a list of session processors
     first, then scan

    :param proc_list: List of Processor classes from the DAX settings file
    :return: List of SessionProcessors, and list of ScanProcessors

    """
    sess_proc_list = list()
    scan_proc_list = list()

    # Build list of processors by type
    for proc in proc_list:
        if issubclass(proc.__class__, ScanProcessor):
            scan_proc_list.append(proc)
        elif issubclass(proc.__class__, SessionProcessor):
            sess_proc_list.append(proc)
        else:
            LOGGER.warn('unknown processor type: %s' % proc)

    return sess_proc_list, scan_proc_list
<commit_msg>adding back the get_assessor_name with changes for getting the right assessor with shared project<commit_after>""" Processor class define for Scan and Session."""

import fnmatch
import logging
import re
import os

from . import XnatUtils, task

__copyright__ = 'Copyright 2013 Vanderbilt University. All Rights Reserved'
__all__ = ['Processor', 'ScanProcessor', 'SessionProcessor']
# Logger for logs
LOGGER = logging.getLogger('dax')

class Processor(object):
    """ Base class for processor """
    def __init__(self, walltime_str, memreq_mb, spider_path,
                 version=None, ppn=1, suffix_proc='',
                 xsitype='proc:genProcData'):
        """
        Entry point of the Base class for processor.

        :param walltime_str: Amount of walltime to request for the process
        :param memreq_mb: Number of megabytes of memory to use
        :param spider_path: Fully qualified path to the spider to run
        :param version: Version of the spider
        :param ppn: Number of processors per not to use.
        :param suffix_proc: Processor suffix (if desired)
        :param xsitype: the XNAT xsiType.
        :return: None

        """
        self.walltime_str = walltime_str  # 00:00:00 format
        self.memreq_mb = memreq_mb   # memory required in megabytes
        # default values:
        self.version = "1.0.0"
        # Suffix
        if suffix_proc and suffix_proc[0] != '_':
            self.suffix_proc = '_%s' % suffix_proc
        else:
            self.suffix_proc = suffix_proc
        self.suffix_proc = re.sub('[^a-zA-Z0-9]', '_', self.suffix_proc)
        self.name = None
        self.spider_path = spider_path
        self.ppn = ppn
        self.xsitype = xsitype
        # getting name and version from spider_path
        self.set_spider_settings(spider_path, version)
        # if suffix_proc is empty, set it to "" for the spider call:
        if not suffix_proc:
            self.suffix_proc = ''

    # get the spider_path right with the version:
    def set_spider_settings(self, spider_path, version):
        """
        Method to set the spider version, path, and name from filepath

        :param spider_path: Fully qualified path and file of the spider
        :param version: version of the spider
        :return: None

        """
        if version:
            # get the proc_name
            proc_name = os.path.basename(spider_path)[7:-3]
            # remove any version if there is one
            proc_name = re.split("/*_v[0-9]/*", proc_name)[0]
            # setting the version and name of the spider
            self.version = version
            nformat = '''{procname}_v{version}{suffix}'''
            self.name = nformat.format(procname=proc_name,
                                       version=self.version.split('.')[0],
                                       suffix=self.suffix_proc)
            sformat = '''Spider_{procname}_v{version}.py'''
            spider_name = sformat.format(procname=proc_name,
                                         version=version.replace('.', '_'))
            self.spider_path = os.path.join(os.path.dirname(spider_path),
                                            spider_name)
        else:
            self.default_settings_spider(spider_path)

    def default_settings_spider(self, spider_path):
        """
        Get the default spider version and name

        :param spider_path: Fully qualified path and file of the spider
        :return: None

        """
        # set spider path
        self.spider_path = spider_path
        # set the name and the version of the spider
        if len(re.split("/*_v[0-9]/*", spider_path)) > 1:
            basename = os.path.basename(spider_path)
            self.version = basename[7:-3].split('_v')[-1].replace('_', '.')
            spidername = basename[7:-3]
            self.name = '''{procname}_v{version}{suffix}'''.format(
                procname=re.split("/*_v[0-9]/*", spidername)[0],
                version=self.version.split('.')[0], suffix=self.suffix_proc)
        else:
            self.name = os.path.basename(spider_path)[7:-3] + self.suffix_proc

    # has_inputs - does this object have the required inputs?
    # e.g. NIFTI format of the required scan type and quality
    #      and are there no conflicting inputs.
    # i.e. only 1 required by 2 found?
    # other arguments here, could be Proj/Subj/Sess/Scan/Assessor depending
    # on processor type?
    def has_inputs(self):
        """
        Check to see if the spider has all the inputs necessary to run.

        :raises: NotImplementedError if user does not override
        :return: None

        """
        raise NotImplementedError()

    # should_run - is the object of the proper object type?
    # e.g. is it a scan? and is it the required scan type?
    # e.g. is it a T1?
    # other arguments here, could be Proj/Subj/Sess/Scan/Assessor depending
    # on processor type?
    def should_run(self):
        """
        Responsible for determining if the assessor should shouw up in session.

        :raises: NotImplementedError if not overridden.
        :return: None

        """
        raise NotImplementedError()

    def build_cmds(self, cobj, dir):

        """
        Build the commands that will go in the PBS/SLURM script
        :raises: NotImplementedError if not overridden from base class.
        :return: None
        """
        raise NotImplementedError()

class ScanProcessor(Processor):
    """ Scan Processor class for processor on a scan on XNAT """
    def __init__(self, scan_types, walltime_str, memreq_mb, spider_path,
                 version=None, ppn=1, suffix_proc='', full_regex=False):
        """
        Entry point of the ScanProcessor Class.

        :param scan_types: Types of scans that the spider should run on
        :param walltime_str: Amount of walltime to request for the process
        :param memreq_mb: Amount of memory in MB to request for the process
        :param spider_path: Absolute path to the spider
        :param version: Version of the spider (taken from the file name)
        :param ppn: Number of processors per node to request
        :param suffix_proc: Processor suffix
        :param full_regex: use full regex
        :return: None

        """
        super(ScanProcessor, self).__init__(walltime_str, memreq_mb,
                                            spider_path, version, ppn,
                                            suffix_proc)
        self.full_regex = full_regex
        if isinstance(scan_types, list):
            self.scan_types = scan_types
        elif isinstance(scan_types, str):
            if scan_types == 'all':
                self.scan_types = 'all'
            else:
                self.scan_types = scan_types.split(',')
        else:
            self.scan_types = []

    def has_inputs(self):
        """
        Method to check and see that the process has all of the inputs
         that it needs to run.

        :raises: NotImplementedError if not overridden.
        :return: None

        """
        raise NotImplementedError()

    def get_assessor_name(self, cscan):
        """
        Returns the label of the assessor

        :param cscan: CachedImageScan object from XnatUtils
        :return: String of the assessor label

        """
        scan_dict = cscan.info()
        subj_label = scan_dict['subject_label']
        sess_label = scan_dict['session_label']
        proj_label = scan_dict['project_label']
        scan_label = scan_dict['scan_label']
        assr_name = '-x-'.join([proj_label, subj_label, sess_label, scan_label,
                                self.name])

        # Check if shared project:
        csess = cscan.parent()
        proj_shared = csess.has_shared_project()
        assr_name_shared = None
        if proj_shared is not None:
            assr_name_shared = '-x-'.join([proj_shared, subj_label, sess_label,
                                           scan_label, self.name])

        # Look for existing assessor
        assr_label = assr_name
        for assr in csess.assessors():
            if assr_name_shared is not None and \
               assr.info()['label'] == assr_name_shared:
                assr_label = assr_name_shared
                break
            if assr.info()['label'] == assr_name:
                break

        return assr_label

    def get_assessor(self, cscan):
        """
        Returns the assessor object depending on cscan and the assessor label.

        :param cscan: CachedImageScan object from XnatUtils
        :return: String of the assessor label

        """
        assessor_name = self.get_assessor_name(cscan)

        # Look for existing assessor
        csess = cscan.parent()
        p_assr = None
        for assr in csess.assessors():
            if assr.info()['label'] == assessor_name:
                p_assr = assr
                break

        return p_assr, assessor_name

    def get_task(self, intf, cscan, upload_dir):
        """
        Get the Task object

        :param intf: XNAT interface (pyxnat.Interface class)
        :param cscan: CachedImageScan object from XnatUtils
        :param upload_dir: the directory to put the processed data when the
         process is done
        :return: Task object

        """
        scan_dict = cscan.info()
        assessor_name = self.get_assessor_name(cscan)
        scan = XnatUtils.get_full_object(intf, scan_dict)
        assessor = scan.parent().assessor(assessor_name)
        return task.Task(self, assessor, upload_dir)

    def should_run(self, scan_dict):
        """
        Method to see if the assessor should appear in the session.

        :param scan_dict: Dictionary of information about the scan
        :return: True if it should run, false if it shouldn't

        """
        if self.scan_types == 'all':
            return True
        else:
            for expression in self.scan_types:
                regex = XnatUtils.extract_exp(expression, self.full_regex)
                if regex.match(scan_dict['scan_type']):
                    return True
            return False

class SessionProcessor(Processor):
    """ Session Processor class for processor on a session on XNAT """
    def __init__(self, walltime_str, memreq_mb, spider_path, version=None,
                 ppn=1, suffix_proc=''):
        """
        Entry point for the session processor

        :param walltime_str: Amount of walltime to request for the process
        :param memreq_mb: Amount of memory in MB to request for the process
        :param spider_path: Absolute path to the spider
        :param version: Version of the spider (taken from the file name)
        :param ppn: Number of processors per node to request
        :param suffix_proc: Processor suffix
        :return: None

        """
        super(SessionProcessor, self).__init__(walltime_str, memreq_mb,
                                               spider_path, version, ppn,
                                               suffix_proc)

    def has_inputs(self):
        """
        Check to see that the session has the required inputs to run.

        :raises: NotImplementedError if not overriden from base class.
        :return: None
        """
        raise NotImplementedError()

    def should_run(self, session_dict):
        """
        By definition, this should always run, so it just returns true
         with no checks

        :param session_dict: Dictionary of session information for
         XnatUtils.list_experiments()
        :return: True

        """
        return True

    def get_assessor_name(self, csess):
        """
        Returns the label of the assessor

        :param csess: CachedImageSession object from XnatUtils
        :return: String of the assessor label

        """
        session_dict = csess.info()
        proj_label = session_dict['project']
        subj_label = session_dict['subject_label']
        sess_label = session_dict['label']
        assr_name = '-x-'.join([proj_label, subj_label, sess_label, self.name])

        # Check if shared project:
        proj_shared = csess.has_shared_project()
        assr_name_shared = None
        if proj_shared is not None:
            assr_name_shared = '-x-'.join([proj_shared, subj_label, sess_label,
                                           self.name])

        # Look for existing assessor
        assr_label = assr_name
        for assr in csess.assessors():
            if assr_name_shared is not None and \
               assr.info()['label'] == assr_name_shared:
                assr_label = assr_name_shared
                break
            if assr.info()['label'] == assr_name:
                break

        return assr_label

    def get_assessor(self, csess):
        """
        Returns the assessor object depending on csess and the assessor label.

        :param csess: CachedImageSession object from XnatUtils
        :return: String of the assessor label

        """
        assessor_name = self.get_assessor_name(csess)

        # Look for existing assessor
        p_assr = None
        for assr in csess.assessors():
            if assr.info()['label'] == assessor_name:
                p_assr = assr
                break

        return p_assr, assessor_name

    def get_task(self, intf, csess, upload_dir):
        """
        Return the Task object

        :param intf: XNAT interface see pyxnat.Interface
        :param csess: CachedImageSession from XnatUtils
        :param upload_dir: directory to put the data after run on the node
        :return: Task object of the assessor

        """
        sess_info = csess.info()
        assessor_name = self.get_assessor_name(csess)
        session = XnatUtils.get_full_object(intf, sess_info)
        assessor = session.assessor(assessor_name)
        return task.Task(self, assessor, upload_dir)

def processors_by_type(proc_list):
    """
    Organize the processor types and return a list of session processors
     first, then scan

    :param proc_list: List of Processor classes from the DAX settings file
    :return: List of SessionProcessors, and list of ScanProcessors

    """
    sess_proc_list = list()
    scan_proc_list = list()

    # Build list of processors by type
    for proc in proc_list:
        if issubclass(proc.__class__, ScanProcessor):
            scan_proc_list.append(proc)
        elif issubclass(proc.__class__, SessionProcessor):
            sess_proc_list.append(proc)
        else:
            LOGGER.warn('unknown processor type: %s' % proc)

    return sess_proc_list, scan_proc_list
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
Windows Service module.
'''

# Import python libs
from __future__ import absolute_import
import salt.utils
import time
import logging
from subprocess import list2cmdline
from salt.ext.six.moves import zip
from salt.ext.six.moves import range
try:
    from shlex import quote as _cmd_quote  # pylint: disable=E0611
except ImportError:
    from pipes import quote as _cmd_quote

log = logging.getLogger(__name__)

# Define the module's virtual name
__virtualname__ = 'service'

BUFFSIZE = 5000
SERVICE_STOP_DELAY_SECONDS = 15
SERVICE_STOP_POLL_MAX_ATTEMPTS = 5

def __virtual__():
    '''
    Only works on Windows systems
    '''
    if salt.utils.is_windows():
        return __virtualname__
    return False

def has_powershell():
    '''
    Confirm if Powershell is available

    CLI Example:

    .. code-block:: bash

        salt '*' service.has_powershell
    '''
    return 'powershell' in __salt__['cmd.run'](
            ['where', 'powershell'], python_shell=False
        )

def get_enabled():
    '''
    Return the enabled services

    CLI Example:

    .. code-block:: bash

        salt '*' service.get_enabled
    '''
    return sorted([service for service in get_all() if enabled(service)])

def get_disabled():
    '''
    Return the disabled services

    CLI Example:

    .. code-block:: bash

        salt '*' service.get_disabled
    '''
    return sorted([service for service in get_all() if disabled(service)])

def available(name):
    '''
    Returns ``True`` if the specified service is available, otherwise returns
    ``False``.

    CLI Example:

    .. code-block:: bash

        salt '*' service.available <service name>
    '''
    return name in get_all()

def missing(name):
    '''
    The inverse of service.available.
    Returns ``True`` if the specified service is not available, otherwise returns
    ``False``.

    CLI Example:

    .. code-block:: bash

        salt '*' service.missing <service name>
    '''
    return name not in get_all()

def get_all():
    '''
    Return all installed services

    CLI Example:

    .. code-block:: bash

        salt '*' service.get_all
    '''
    ret = set()
    cmd = list2cmdline(['sc', 'query', 'type=', 'service', 'state=', 'all', 'bufsize=', str(BUFFSIZE)])
    lines = __salt__['cmd.run'](cmd).splitlines()
    for line in lines:
        if 'SERVICE_NAME:' in line:
            comps = line.split(':', 1)
            if not len(comps) > 1:
                continue
            ret.add(comps[1].strip())
    return sorted(ret)

def get_service_name(*args):
    '''
    The Display Name is what is displayed in Windows when services.msc is
    executed.  Each Display Name has an associated Service Name which is the
    actual name of the service.  This function allows you to discover the
    Service Name by returning a dictionary of Display Names and Service Names,
    or filter by adding arguments of Display Names.

    If no args are passed, return a dict of all services where the keys are the
    service Display Names and the values are the Service Names.

    If arguments are passed, create a dict of Display Names and Service Names

    CLI Example:

    .. code-block:: bash

        salt '*' service.get_service_name
        salt '*' service.get_service_name 'Google Update Service (gupdate)' 'DHCP Client'
    '''
    ret = {}
    services = []
    display_names = []
    cmd = ['sc', 'query', 'type=', 'service', 'state=', 'all', 'bufsize=', str(BUFFSIZE)]
    lines = __salt__['cmd.run'](cmd, python_shell=False).splitlines()
    for line in lines:
        if 'SERVICE_NAME:' in line:
            comps = line.split(':', 1)
            if not len(comps) > 1:
                continue
            services.append(comps[1].strip())
        if 'DISPLAY_NAME:' in line:
            comps = line.split(':', 1)
            if not len(comps) > 1:
                continue
            display_names.append(comps[1].strip())
    if len(services) == len(display_names):
        service_dict = dict(zip(display_names, services))
    else:
        return 'Service Names and Display Names mismatch'
    if len(args) == 0:
        return service_dict
    for arg in args:
        if arg in service_dict:
            ret[arg] = service_dict[arg]
    return ret

def start(name):
    '''
    Start the specified service

    CLI Example:

    .. code-block:: bash

        salt '*' service.start <service name>
    '''
    cmd = ['net', 'start', name]
    return not __salt__['cmd.retcode'](cmd, python_shell=False)

def stop(name):
    '''
    Stop the specified service

    CLI Example:

    .. code-block:: bash

        salt '*' service.stop <service name>
    '''
    # net stop issues a stop command and waits briefly (~30s), but will give
    # up if the service takes too long to stop with a misleading
    # "service could not be stopped" message and RC 0.

    cmd = ['net', 'stop', name]
    res = __salt__['cmd.run'](cmd, python_shell=False)
    if 'service was stopped' in res:
        return True

    # we requested a stop, but the service is still thinking about it.
    # poll for the real status
    for attempt in range(SERVICE_STOP_POLL_MAX_ATTEMPTS):
        if not status(name):
            return True
        log.debug('Waiting for %s to stop', name)
        time.sleep(SERVICE_STOP_DELAY_SECONDS)

    log.warning('Giving up on waiting for service `%s` to stop', name)
    return False

def restart(name):
    '''
    Restart the named service

    CLI Example:

    .. code-block:: bash

        salt '*' service.restart <service name>
    '''
    if has_powershell():
        if 'salt-minion' in name:
            create_win_salt_restart_task()
            return execute_salt_restart_task()
        cmd = 'Restart-Service {0}'.format(_cmd_quote(name))
        return not __salt__['cmd.retcode'](cmd, shell='powershell', python_shell=True)
    return stop(name) and start(name)

def create_win_salt_restart_task():
    '''

    Create a task in Windows task scheduler to enable restarting the salt-minion

    CLI Example:

    .. code-block:: bash

        salt '*' service.create_win_salt_restart_task()
    '''
    cmd = 'schtasks /RU "System" /Create /TN restart-salt-minion /TR "powershell Restart-Service salt-minion" /sc ONCE /sd 01/15/1975 /st 01:00 /F'

    return __salt__['cmd.run'](cmd)

def execute_salt_restart_task():
    '''
    Run the Windows Salt restart task

    CLI Example:

    .. code-block:: bash

        salt '*' service.execute_salt_restart_task()
    '''
    cmd = 'schtasks /Run /TN restart-salt-minion'
    return __salt__['cmd.run'](cmd)

def status(name, sig=None):
    '''
    Return the status for a service, returns the PID or an empty string if the
    service is running or not, pass a signature to use to find the service via
    ps

    CLI Example:

    .. code-block:: bash

        salt '*' service.status <service name> [service signature]
    '''
    cmd = ['sc', 'query', name]
    statuses = __salt__['cmd.run'](cmd, python_shell=False).splitlines()
    for line in statuses:
        if 'RUNNING' in line:
            return True
        elif 'STOP_PENDING' in line:
            return True
    return False

def getsid(name):
    '''
    Return the sid for this windows service

    CLI Example:

    .. code-block:: bash

        salt '*' service.getsid <service name>
    '''
    cmd = ['sc', 'showsid', name]
    lines = __salt__['cmd.run'](cmd, python_shell=False).splitlines()
    for line in lines:
        if 'SERVICE SID:' in line:
            comps = line.split(':', 1)
            if comps[1] > 1:
                return comps[1].strip()
            else:
                return None

def enable(name, **kwargs):
    '''
    Enable the named service to start at boot

    CLI Example:

    .. code-block:: bash

        salt '*' service.enable <service name>
    '''
    cmd = ['sc', 'config', name, 'start=', 'auto']
    return not __salt__['cmd.retcode'](cmd, python_shell=False)

def disable(name, **kwargs):
    '''
    Disable the named service to start at boot

    CLI Example:

    .. code-block:: bash

        salt '*' service.disable <service name>
    '''
    cmd = ['sc', 'config', name, 'start=', 'disabled']
    return not __salt__['cmd.retcode'](cmd, python_shell=False)

def enabled(name, **kwargs):
    '''
    Check to see if the named service is enabled to start on boot

    CLI Example:

    .. code-block:: bash

        salt '*' service.enabled <service name>
    '''
    cmd = list2cmdline(['sc', 'qc', name])
    lines = __salt__['cmd.run'](cmd).splitlines()
    for line in lines:
        if 'AUTO_START' in line:
            return True
    return False

def disabled(name):
    '''
    Check to see if the named service is disabled to start on boot

    CLI Example:

    .. code-block:: bash

        salt '*' service.disabled <service name>
    '''
    cmd = list2cmdline(['sc', 'qc', name])
    lines = __salt__['cmd.run'](cmd).splitlines()
    for line in lines:
        if 'DEMAND_START' in line:
            return True
        elif 'DISABLED' in line:
            return True
    return False
<commit_msg>Fixed service.restart for salt-minion<commit_after># -*- coding: utf-8 -*-
'''
Windows Service module.
'''

# Import python libs
from __future__ import absolute_import
import salt.utils
import time
import logging
from subprocess import list2cmdline
from salt.ext.six.moves import zip
from salt.ext.six.moves import range
try:
    from shlex import quote as _cmd_quote  # pylint: disable=E0611
except ImportError:
    from pipes import quote as _cmd_quote

log = logging.getLogger(__name__)

# Define the module's virtual name
__virtualname__ = 'service'

BUFFSIZE = 5000
SERVICE_STOP_DELAY_SECONDS = 15
SERVICE_STOP_POLL_MAX_ATTEMPTS = 5

def __virtual__():
    '''
    Only works on Windows systems
    '''
    if salt.utils.is_windows():
        return __virtualname__
    return False

def has_powershell():
    '''
    Confirm if Powershell is available

    CLI Example:

    .. code-block:: bash

        salt '*' service.has_powershell
    '''
    return 'powershell' in __salt__['cmd.run'](
            ['where', 'powershell'], python_shell=False
        )

def get_enabled():
    '''
    Return the enabled services

    CLI Example:

    .. code-block:: bash

        salt '*' service.get_enabled
    '''
    return sorted([service for service in get_all() if enabled(service)])

def get_disabled():
    '''
    Return the disabled services

    CLI Example:

    .. code-block:: bash

        salt '*' service.get_disabled
    '''
    return sorted([service for service in get_all() if disabled(service)])

def available(name):
    '''
    Returns ``True`` if the specified service is available, otherwise returns
    ``False``.

    CLI Example:

    .. code-block:: bash

        salt '*' service.available <service name>
    '''
    return name in get_all()

def missing(name):
    '''
    The inverse of service.available.
    Returns ``True`` if the specified service is not available, otherwise returns
    ``False``.

    CLI Example:

    .. code-block:: bash

        salt '*' service.missing <service name>
    '''
    return name not in get_all()

def get_all():
    '''
    Return all installed services

    CLI Example:

    .. code-block:: bash

        salt '*' service.get_all
    '''
    ret = set()
    cmd = list2cmdline(['sc', 'query', 'type=', 'service', 'state=', 'all', 'bufsize=', str(BUFFSIZE)])
    lines = __salt__['cmd.run'](cmd).splitlines()
    for line in lines:
        if 'SERVICE_NAME:' in line:
            comps = line.split(':', 1)
            if not len(comps) > 1:
                continue
            ret.add(comps[1].strip())
    return sorted(ret)

def get_service_name(*args):
    '''
    The Display Name is what is displayed in Windows when services.msc is
    executed.  Each Display Name has an associated Service Name which is the
    actual name of the service.  This function allows you to discover the
    Service Name by returning a dictionary of Display Names and Service Names,
    or filter by adding arguments of Display Names.

    If no args are passed, return a dict of all services where the keys are the
    service Display Names and the values are the Service Names.

    If arguments are passed, create a dict of Display Names and Service Names

    CLI Example:

    .. code-block:: bash

        salt '*' service.get_service_name
        salt '*' service.get_service_name 'Google Update Service (gupdate)' 'DHCP Client'
    '''
    ret = {}
    services = []
    display_names = []
    cmd = ['sc', 'query', 'type=', 'service', 'state=', 'all', 'bufsize=', str(BUFFSIZE)]
    lines = __salt__['cmd.run'](cmd, python_shell=False).splitlines()
    for line in lines:
        if 'SERVICE_NAME:' in line:
            comps = line.split(':', 1)
            if not len(comps) > 1:
                continue
            services.append(comps[1].strip())
        if 'DISPLAY_NAME:' in line:
            comps = line.split(':', 1)
            if not len(comps) > 1:
                continue
            display_names.append(comps[1].strip())
    if len(services) == len(display_names):
        service_dict = dict(zip(display_names, services))
    else:
        return 'Service Names and Display Names mismatch'
    if len(args) == 0:
        return service_dict
    for arg in args:
        if arg in service_dict:
            ret[arg] = service_dict[arg]
    return ret

def start(name):
    '''
    Start the specified service

    CLI Example:

    .. code-block:: bash

        salt '*' service.start <service name>
    '''
    cmd = ['net', 'start', name]
    return not __salt__['cmd.retcode'](cmd, python_shell=False)

def stop(name):
    '''
    Stop the specified service

    CLI Example:

    .. code-block:: bash

        salt '*' service.stop <service name>
    '''
    # net stop issues a stop command and waits briefly (~30s), but will give
    # up if the service takes too long to stop with a misleading
    # "service could not be stopped" message and RC 0.

    cmd = ['net', 'stop', name]
    res = __salt__['cmd.run'](cmd, python_shell=False)
    if 'service was stopped' in res:
        return True

    # we requested a stop, but the service is still thinking about it.
    # poll for the real status
    for attempt in range(SERVICE_STOP_POLL_MAX_ATTEMPTS):
        if not status(name):
            return True
        log.debug('Waiting for %s to stop', name)
        time.sleep(SERVICE_STOP_DELAY_SECONDS)

    log.warning('Giving up on waiting for service `%s` to stop', name)
    return False

def restart(name):
    '''
    Restart the named service

    CLI Example:

    .. code-block:: bash

        salt '*' service.restart <service name>
    '''
    if has_powershell():
        if 'salt-minion' in name:
            create_win_salt_restart_task()
            return execute_salt_restart_task()
        cmd = 'Restart-Service {0}'.format(_cmd_quote(name))
        return not __salt__['cmd.retcode'](cmd, shell='powershell', python_shell=True)
    return stop(name) and start(name)

def create_win_salt_restart_task():
    '''

    Create a task in Windows task scheduler to enable restarting the salt-minion

    CLI Example:

    .. code-block:: bash

        salt '*' service.create_win_salt_restart_task()
    '''
    cmd = 'schtasks /RU "System" /Create /TN restart-salt-minion /TR "powershell Restart-Service salt-minion" /sc ONCE /sd 01/15/1975 /st 01:00 /F'

    return __salt__['cmd.shell'](cmd)

def execute_salt_restart_task():
    '''
    Run the Windows Salt restart task

    CLI Example:

    .. code-block:: bash

        salt '*' service.execute_salt_restart_task()
    '''
    cmd = 'schtasks /Run /TN restart-salt-minion'
    return __salt__['cmd.shell'](cmd)

def status(name, sig=None):
    '''
    Return the status for a service, returns the PID or an empty string if the
    service is running or not, pass a signature to use to find the service via
    ps

    CLI Example:

    .. code-block:: bash

        salt '*' service.status <service name> [service signature]
    '''
    cmd = ['sc', 'query', name]
    statuses = __salt__['cmd.run'](cmd, python_shell=False).splitlines()
    for line in statuses:
        if 'RUNNING' in line:
            return True
        elif 'STOP_PENDING' in line:
            return True
    return False

def getsid(name):
    '''
    Return the sid for this windows service

    CLI Example:

    .. code-block:: bash

        salt '*' service.getsid <service name>
    '''
    cmd = ['sc', 'showsid', name]
    lines = __salt__['cmd.run'](cmd, python_shell=False).splitlines()
    for line in lines:
        if 'SERVICE SID:' in line:
            comps = line.split(':', 1)
            if comps[1] > 1:
                return comps[1].strip()
            else:
                return None

def enable(name, **kwargs):
    '''
    Enable the named service to start at boot

    CLI Example:

    .. code-block:: bash

        salt '*' service.enable <service name>
    '''
    cmd = ['sc', 'config', name, 'start=', 'auto']
    return not __salt__['cmd.retcode'](cmd, python_shell=False)

def disable(name, **kwargs):
    '''
    Disable the named service to start at boot

    CLI Example:

    .. code-block:: bash

        salt '*' service.disable <service name>
    '''
    cmd = ['sc', 'config', name, 'start=', 'disabled']
    return not __salt__['cmd.retcode'](cmd, python_shell=False)

def enabled(name, **kwargs):
    '''
    Check to see if the named service is enabled to start on boot

    CLI Example:

    .. code-block:: bash

        salt '*' service.enabled <service name>
    '''
    cmd = list2cmdline(['sc', 'qc', name])
    lines = __salt__['cmd.run'](cmd).splitlines()
    for line in lines:
        if 'AUTO_START' in line:
            return True
    return False

def disabled(name):
    '''
    Check to see if the named service is disabled to start on boot

    CLI Example:

    .. code-block:: bash

        salt '*' service.disabled <service name>
    '''
    cmd = list2cmdline(['sc', 'qc', name])
    lines = __salt__['cmd.run'](cmd).splitlines()
    for line in lines:
        if 'DEMAND_START' in line:
            return True
        elif 'DISABLED' in line:
            return True
    return False
<|endoftext|>


<commit_before>#!/usr/bin/env python
##
##  Copyright 2016 SRI International
##  See COPYING file distributed along with the package for the copyright and license terms.
##
"""
xnat_scans_filter.py
======================
This script filters the csv file generated using xnat_extractor.py. This filters
is based on records from XNAT where there is one row per scan.

xnat_scans_filter.py -i path/to/xnat.csv
"""

import os
import sys

import redcap
import pandas as pd

# Fields to retrieve from REDCap
fields = ['study_id', 'redcap_event_name', 'exclude', 'visit_ignore',
          'visit_date', 'mri_missing', 'mri_xnat_sid', 'mri_xnat_eids',
          'visit_notes']

# Forms that the fields come from in REDCap.
forms = ['mr_session_report', 'visit_date', 'demographics']

def get_project_entry(args=None):
    """
    Pulls the data from REDCap
    """
    # Get API key.
    summary_key_file = open(os.path.join(os.path.expanduser("~"),
                                         '.server_config',
                                         'redcap-dataentry-token'), 'r')
    summary_api_key = summary_key_file.read().strip()

    # Connect to API.
    project_entry = redcap.Project('https://ncanda.sri.com/redcap/api/',
                                   summary_api_key,
                                   verify_ssl=False)
    return project_entry

def data_entry_fields(fields, project, arm):
    """
    Gets the dataframe containing a specific arm from REDCap
    """
    # Get a dataframe of fields
    data_entry_raw = project.export_records(fields=fields,
                                            forms=forms,
                                            format='df',
                                            events=[arm])
    return data_entry_raw

def append_site_id_row(xnat_df, scans_df):
    scans_df['site_id'] = ''
    ids = xnat_df[['site_id', 'subject_id']]
    map = {r.subject_id: r.site_id for idx, r in ids.iterrows()}
    for idx, row in scans_df.iterrows():
        scans_df.site_id.loc[idx] = map.get(row.case)
    return scans_df

def is_in_redcap(rc_df, scans_df):
    """
    Checks if the scans missing in the pipeline are listed in REDCap and adds
    a column indicating as such.
    """
    scans_df['in_redcap'] = False
    scans_df['visit_ignore___yes'] = ''
    scans_df['visit_ignore_why'] = ''
    scans_df['visit_ignore_why_other'] = ''
    scans_df['visit_notes'] = ''
    scans_df['mri_missing'] = ''
    scans_df['mri_missing_why'] = ''
    scans_df['mri_missing_why_other'] = ''
    scans_df['mri_notes'] = ''
    rc_cases = rc_df[rc_df.mri_xnat_sid.isin(scans_df.case)]
    for idx, row in rc_cases.iterrows():
        scan_cases = scans_df[scans_df.case == row.mri_xnat_sid]
        scans_df.in_redcap.loc[scan_cases.index] = True
        # Visit info
        scans_df.visit_ignore___yes.loc[scan_cases.index] = row.visit_ignore___yes
        scans_df.visit_ignore_why.loc[scan_cases.index] = row.visit_ignore_why
        scans_df.visit_ignore_why_other.loc[scan_cases.index] = row.visit_ignore_why_other
        scans_df.visit_notes.loc[scan_cases.index] = row.visit_notes
        # Scan info
        scans_df.mri_missing.loc[scan_cases.index] = row.mri_missing
        scans_df.mri_missing_why.loc[scan_cases.index] = row.mri_missing_why
        scans_df.mri_missing_why_other.loc[scan_cases.index] = row.mri_missing_why_other
        scans_df.mri_notes.loc[scan_cases.index] = row.mri_notes
    return scans_df

def is_in_xnat(xnat_df, scans_df):
    """
    Checks XNAT for scans near the visit date recorded in REDCap
    """

def main(args=None):
    # Connect to REDCap
    project_entry = get_project_entry()

    # Get the visit dataframe
    project_df = data_entry_fields(fields, project_entry, args.event)

    # Get a list of all EIDs for the given visit
    xnat_eids = project_df['mri_xnat_eids'].tolist()

    # Read the csv file from xnat_extractor
    xnat_csv = pd.read_csv(args.infile)

    # Filter the XNAT records by the EIDs in REDCap
    # This provides a list of all the scans in XNAT that are also in REDCap
    filter_csv = xnat_csv[xnat_csv['experiment_id'].isin(xnat_eids)]

    # Iterate through scans missing in the pipeline and check:
    #   - they are present in the filtered REDCap list
    # present in the REDCap filter list.
    if args.missing_scans:
        list_missing_scans = pd.read_csv(args.missing_scans)
        missing_scans_df = append_site_id_row(xnat_csv, list_missing_scans)

        # Add columns indicating if there is data for this visit in redcap
        in_redcap = is_in_redcap(project_df, missing_scans_df)
        filter_csv = in_redcap

    # Write the results to disk
    filter_csv.to_csv(args.outfile)

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('-i', '--infile',
                        required=True,
                        help="Input csv file from xnat_extractor.py")
    parser.add_argument('-e', '--event',
                        choices=['baseline_visit_arm_1', '1y_visit_arm_1'],
                        default='1y_visit_arm_1')
    parser.add_argument('-m', '--missing-scans',
                        help="Output of list_missing_scans script.")
    parser.add_argument('-o', '--outfile',
                        default=os.path.join('/tmp', 'xnat_scans_filter.csv'))
    parser.add_argument('-v', '--verbose',
                        action='store_true',
                        help='Enable verbose reporting.')
    argv = parser.parse_args()
    sys.exit(main(args=argv))<commit_msg>fix: merge conflicts<commit_after>#!/usr/bin/env python
##
##  Copyright 2016 SRI International
##  See COPYING file distributed along with the package for the copyright and license terms.
##
"""
xnat_scans_filter.py
======================
This script filters the csv file generated using xnat_extractor.py. This filters
is based on records from XNAT where there is one row per scan.

xnat_scans_filter.py -i path/to/xnat.csv
"""

import os
import sys

import redcap
import pandas as pd

# Fields to retrieve from REDCap
fields = ['study_id', 'redcap_event_name', 'exclude', 'visit_ignore',
          'visit_date', 'mri_missing', 'mri_xnat_sid', 'mri_xnat_eids',
          'visit_notes']

# Forms that the fields come from in REDCap.
forms = ['mr_session_report', 'visit_date', 'demographics']

def get_project_entry(args=None):
    """
    Pulls the data from REDCap
    """
    # Get API key.
    summary_key_file = open(os.path.join(os.path.expanduser("~"),
                                         '.server_config',
                                         'redcap-dataentry-token'), 'r')
    summary_api_key = summary_key_file.read().strip()

    # Connect to API.
    project_entry = redcap.Project('https://ncanda.sri.com/redcap/api/',
                                   summary_api_key,
                                   verify_ssl=False)
    return project_entry

def data_entry_fields(fields, project, arm):
    """
    Gets the dataframe containing a specific arm from REDCap
    """
    # Get a dataframe of fields
    data_entry_raw = project.export_records(fields=fields,
                                            forms=forms,
                                            format='df',
                                            events=[arm])
    return data_entry_raw

def append_site_id_row(xnat_df, scans_df):
    scans_df['site_id'] = ''
    ids = xnat_df[['site_id', 'subject_id']]
    map = {r.subject_id: r.site_id for idx, r in ids.iterrows()}
    for idx, row in scans_df.iterrows():
        scans_df.site_id.loc[idx] = map.get(row.case)
    return scans_df

def is_in_redcap(rc_df, scans_df):
    """
    Checks if the scans missing in the pipeline are listed in REDCap and adds
    a column indicating as such.
    """
    scans_df['in_redcap'] = False
    scans_df['visit_ignore___yes'] = ''
    scans_df['visit_ignore_why'] = ''
    scans_df['visit_ignore_why_other'] = ''
    scans_df['visit_notes'] = ''
    scans_df['mri_missing'] = ''
    scans_df['mri_missing_why'] = ''
    scans_df['mri_missing_why_other'] = ''
    scans_df['mri_notes'] = ''
    rc_cases = rc_df[rc_df.mri_xnat_sid.isin(scans_df.case)]
    for idx, row in rc_cases.iterrows():
        scan_cases = scans_df[scans_df.case == row.mri_xnat_sid]
        scans_df.in_redcap.loc[scan_cases.index] = True
        # Visit info
        scans_df.visit_ignore___yes.loc[scan_cases.index] = row.visit_ignore___yes
        scans_df.visit_ignore_why.loc[scan_cases.index] = row.visit_ignore_why
        scans_df.visit_ignore_why_other.loc[scan_cases.index] = row.visit_ignore_why_other
        scans_df.visit_notes.loc[scan_cases.index] = row.visit_notes
        # Scan info
        scans_df.mri_missing.loc[scan_cases.index] = row.mri_missing
        scans_df.mri_missing_why.loc[scan_cases.index] = row.mri_missing_why
        scans_df.mri_missing_why_other.loc[scan_cases.index] = row.mri_missing_why_other
        scans_df.mri_notes.loc[scan_cases.index] = row.mri_notes
    return scans_df

def is_in_xnat(xnat_df, scans_df):
    """
    Checks XNAT for scans near the visit date recorded in REDCap
    """

def main(args=None):
    # Connect to REDCap
    project_entry = get_project_entry()

    # Get the visit dataframe
    project_df = data_entry_fields(fields, project_entry, args.event)

    # Get a list of all EIDs for the given visit
    xnat_eids = project_df['mri_xnat_eids'].tolist()

    # Read the csv file from xnat_extractor
    xnat_csv = pd.read_csv(args.infile)

    # Filter the XNAT records by the EIDs in REDCap
    # This provides a list of all the scans in XNAT that are also in REDCap
    filter_csv = xnat_csv[xnat_csv['experiment_id'].isin(xnat_eids)]

    # Iterate through scans missing in the pipeline and check:
    #   - they are present in the filtered REDCap list
    # present in the REDCap filter list.
    if args.missing_scans:
        list_missing_scans = pd.read_csv(args.missing_scans)
        missing_scans_df = append_site_id_row(xnat_csv, list_missing_scans)

        # Add columns indicating if there is data for this visit in redcap
        in_redcap = is_in_redcap(project_df, missing_scans_df)
        filter_csv = in_redcap

    # Write the results to disk
    filter_csv.to_csv(args.outfile)

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('-i', '--infile',
                        required=True,
                        help="Input csv file from xnat_extractor.py")
    parser.add_argument('-e', '--event',
                        choices=['baseline_visit_arm_1', '1y_visit_arm_1'],
                        default='1y_visit_arm_1')
    parser.add_argument('-m', '--missing-scans',
                        help="Output of list_missing_scans script.")
    parser.add_argument('-o', '--outfile',
                        default=os.path.join('/tmp', 'xnat_scans_filter.csv'))
    parser.add_argument('-v', '--verbose',
                        action='store_true',
                        help='Enable verbose reporting.')
    argv = parser.parse_args()
    sys.exit(main(args=argv))
<|endoftext|>