<commit_before># emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:

import math
import numpy as np
import os
import csv
from ATT.iofunc import iofiles
from ATT.algorithm import surf_tools
from ATT.algorithm import hcp_tools
import pandas as pd

HCP_TEST_DATA = 'e:\\projects\\genetic_imaging\\HCPdata\\HCP900\\'
data_out_file = 'E:\\projects\\genetic_imaging\\HCPdata\\data\\HCPExtracted\\'

class get_hcp_data(object):
    def __init__(self,stem_path):
        self.stem_path = stem_path
        subid = os.listdir(stem_path)
        self.subid = subid

    def motion_FD(self,path_list):
        power_rela_fd_mean = []
        dt_rela_fd_mean = []
        def cal_relaFD(re1, re2):
            """
            """
            relative_fd = 0
            for i in range(0, 3):
                relative_fd += abs(float(re1[i]) - float(re2[i]))
            for i in range(3, 6):
                relative_fd += 50 * math.pi / 150 * (abs(float(re1[i]) - float(re2[i])))

            return math.sqrt(relative_fd)

        def cal_relaFD_mean(ls):
            """
            """
            rela_fd = []
            for i in range(1, len(ls)):
                rela_fd.append(cal_relaFD(regre_LR[i], regre_LR[i - 1]))
            indi_fd_mean = np.mean(np.array(rela_fd))
            return indi_fd_mean

        for i in path_list:
            try:
                with open(i + '/tfMRI_WM_LR/Movement_Regressors.txt', 'r') as f:
                    s = f.readlines()
                    regre_LR = np.array([j.rstrip().split() for j in s])
                with open(i + '/tfMRI_WM_RL/Movement_Regressors.txt', 'r') as f:
                    s = f.readlines()
                    regre_RL = np.array([j.rstrip().split() for j in s])
                indi_FD_mean = (cal_relaFD_mean(regre_LR) + cal_relaFD_mean(regre_RL)) / 2
                power_rela_fd_mean.append(indi_FD_mean)
            except FileNotFoundError as e:
                print(e)
                power_rela_fd_mean.append([])
        for i in path_list:
            try:
                with open(i + '/tfMRI_WM_LR/Movement_Regressors_dt.txt', 'r') as f:
                    s = f.readlines()
                    tmp = np.array([j.rstrip().split() for j in s])
                    regre_LR = []
                    for j in tmp:
                        j = list(map(lambda x:float(x),j))
                        regre_LR.append(j)
                with open(i + '/tfMRI_WM_RL/Movement_Regressors_dt.txt', 'r') as f:
                    s = f.readlines()
                    tmp = np.array([j.rstrip().split() for j in s])
                    regre_RL = []
                    for j in tmp:
                        j = list(map(lambda x: float(x), j))
                        regre_RL.append(j)

                indi_FD_mean = (np.mean(np.array(regre_RL))+np.mean(np.array(regre_LR)))/2
                dt_rela_fd_mean.append(indi_FD_mean)
            except FileNotFoundError as e:
                print(e)
                dt_rela_fd_mean.append([])

        return power_rela_fd_mean,dt_rela_fd_mean

    def get_brainsize(self,path_list):
        etiv_size_list = []
        for i in path_list:
            try:
                with open(i, 'r') as f:
                    s = f.readlines()[33][-21:-7]
                etiv_size_list.append(float(s))
            except FileNotFoundError as e:
                print(e)
                etiv_size_list.append([])
        return etiv_size_list

    def motion_RMS(self,path_list):
        relative_meanRMS = []
        for i in path_list:
            try:
                with open(i + '\\tfMRI_WM_LR\\Movement_RelativeRMS_mean.txt', 'r') as f:
                    relative_LR = f.readline()
                with open(i + '\\tfMRI_WM_RL\\Movement_RelativeRMS_mean.txt', 'r') as f:
                    relative_RL = f.readline()
                relative_meanRMS.append((float(relative_LR) + float(relative_RL) / 2))
            except FileNotFoundError as e:
                print(e)
                relative_meanRMS.append([])
        return relative_meanRMS
    def get_file_path_list(self,file_type):
        """
        generate a list containing needed file path
        file_type: func
                   stru
                   other
        """

        if file_type == 'func':
            catagory = ['tfMRI_WM_hp200_s12_level2.feat','','','','','','']
            catagory = int(input('''
    please input the number of  catagoty you want:
    1.*.hp200_s12_level12.feat
    2.*.hp200_s2_level2.feat
    3.*.hp200_s2_level2_MSMALL.feat
    4.*.hp200_s4_level2.feat
    5.*.hp200_s4_level2_MSMALL.feat
    6.**.hp200_s8_level2.feat
    --
            '''))
            if catagory == 1:
                self.catagory = 'hp200_s12_level12.feat'
                FUNC_STEM_PATH= 'MNINonLinear/Results/tfMRI_WM/tfMRI_WM_hp200_s12_level2.feat/'
            elif catagory == 2:
                self.catagory = 'hp200_s2_level2.feat'
                FUNC_STEM_PATH= 'MNINonLinear/Results/tfMRI_WM/tfMRI_WM_hp200_s2_level2.feat/'
            elif catagory == 3:
                self.catagory = 'hp200_s2_level2_MSMALL.feat'
                FUNC_STEM_PATH= 'MNINonLinear/Results/tfMRI_WM/tfMRI_WM_hp200_s2_level2_MSMALL.feat/'
            elif catagory == 4:
                self.catagory = 'hp200_s4_level2.feat'
                FUNC_STEM_PATH= 'MNINonLinear/Results/tfMRI_WM/tfMRI_WM_hp200_s4_level2.feat/'
            elif catagory == 5:
                self.catagory = 'hp200_s4_level2_MSMALL.feat'
                FUNC_STEM_PATH= 'MNINonLinear/Results/tfMRI_WM/tfMRI_WM_hp200_s4_level2_MSMALL.feat/'
            elif catagory == 6:
                self.catagory = 'hp200_s8_level2.feat'
                FUNC_STEM_PATH= 'MNINonLinear/Results/tfMRI_WM/tfMRI_WM_hp200_s8_level2.feat/'

            data_type = int(input('please input the number of your data type:\n:1.t value 2.beta\n--'))
            contrast_type = int(input('''
            please input the contrast type
            1.body-avg
            2.face-avg
            3.place-avg
            4.tool-avg
            '''))
            if data_type == 1:

                if  contrast_type == 1:
                    FILE_NAME = 'GrayordinatesStats/cope19.feat/tstat1.dtseries.nii'
                elif  contrast_type == 2:
                    FILE_NAME = 'GrayordinatesStats/cope20.feat/tstat1.dtseries.nii'
                elif  contrast_type == 3:
                    FILE_NAME = 'GrayordinatesStats/cope21.feat/tstat1.dtseries.nii'
                elif  contrast_type == 4:
                    FILE_NAME = 'GrayordinatesStats/cope22.feat/tstat1.dtseries.nii'
                path_list = [HCP_TEST_DATA+i+'/'+FUNC_STEM_PATH+ FILE_NAME for i in self.subid]

            if data_type == 2:
                if  contrast_type == 1:
                    FILE_NAME = 'GrayordinatesStats/cope19.feat/cope1.dtseries.nii'
                elif  contrast_type == 2:
                    FILE_NAME = 'GrayordinatesStats/cope20.feat/cope1.dtseries.nii'
                elif  contrast_type == 3:
                    FILE_NAME = 'GrayordinatesStats/cope21.feat/cope1.dtseries.nii'
                elif  contrast_type == 4:
                    FILE_NAME = 'GrayordinatesStats/cope22.feat/cope1.dtseries.nii'
                path_list = [self.stem_path+i+'/'+FUNC_STEM_PATH+ FILE_NAME for i in self.subid]
        elif file_type == 'stru':
            self.catagory = ''
    #        stru_type = ['*.MyelinMap_MSMAll.32k_fs_LR.dscalar.nii',
    #        '*.curvature_MSMAll.32k_fs_LR.dscalar.nii','*.thickness_MSMAll.32k_fs_LR.dscalar.nii']
            stru_type_num = int(input('''
    please input the structure file you need (not number)
    1.*.MyelinMap_MSMAll.32k_fs_LR.dscalar.nii
    2.*.curvature_MSMAll.32k_fs_LR.dscalar.nii
    3.*.thickness_MSMAll.32k_fs_LR.dscalar.nii
    --
            '''))
            if stru_type_num == 1 :
                stru_type = '.MyelinMap_MSMAll.32k_fs_LR.dscalar.nii'
            if stru_type_num == 2 :
                stru_type = '.curvature_MSMAll.32k_fs_LR.dscalar.nii'
            if stru_type_num == 3 :
                stru_type = '.thickness_MSMAll.32k_fs_LR.dscalar.nii'
            brain_region_num = int(input('''
    please input number of your target region
    1. left hemisphere
    2. right hemishpere
    3. whole brain
    --
             '''))
            if brain_region_num ==3:
                path_list  = [self.stem_path+i+'/'+'MNINonLinear/fsaverage_LR32k/'+i+stru_type for i in self.subid]
            if brain_region_num ==1:
                path_list  = [self.stem_path+i+'/'+'MNINonLinear/fsaverage_LR32k/'+i+'.L'+stru_type for i in self.subid]
            if brain_region_num ==2:
                path_list  = [self.stem_path+i+'/'+'MNINonLinear/fsaverage_LR32k/'+i+'.R'+stru_type for i in self.subid]

        elif file_type == 'other':
            other_type = int(input('''
            please input the type of data you want
            1.Motion
            2.brain_size
            '''))
            if other_type == 1:
                self.other_type ='motion'
                path_list = [self.stem_path+i+'/'+'MNINonLinear/Results' for i in self.subid]
            elif other_type == 2:
                self.other_type = 'brain_size'
                path_list = [self.stem_path+i+'/'+'T1w/'+i+'/'+'stats/aseg.stats' for i in self.subid]
        else:
            raise Exception('please input the right file type: func, stru, other')
        return path_list

    def getsave_certain_data(self,file_type,label_data,output,output_path):
        '''
        '''
        path_list = self.get_file_path_list(file_type)
        data_list = []
        if file_type == 'func' or file_type=='stru':
            for i in path_list:
                try:
                    data = iofiles.CIFTI(i).read_cifti()
                    data_list.append(
                    surf_tools.get_signals(data, label_data))  # get roi mean value of each roi of each subject
                except FileNotFoundError as e:
                    print(e)
                    data_list.append([])

            out_file_name = output_path+file_type+'/'+self.catagory+'/'+output+'.csv'
        elif file_type == 'other':
            data_list = []
            if self.other_type == 'motion':
                data_list.append(self.motion_RMS(path_list))
                data_list.append(self.motion_FD(path_list)[0])
                data_list.append(self.motion_FD(path_list)[1])
            elif self.other_type == 'brain_size':
                data_list.append(self.get_brainsize(path_list))

            data_list=np.array(data_list).T

            out_file_name = output_path+file_type+'/'+output+'.csv'

        pd_data_list = pd.DataFrame(data_list,index=self.subid)
        pd_data_list.to_csv(out_file_name)
        # with open(out_file_name,'w',newline = '') as f:
        #         f_csv = csv.writer(f)path
        #         f_csv.writerows(data_list)1
if __name__=='__main__':
    # labelpath = 'E:\projects\genetic_imaging\HCPdata\VanEssenMap\HCP_PhaseTwo\Q1-Q6_RelatedParcellation210\MNINonLinear\\fsaverage_LR32k\\Q1-Q6_RelatedParcellation210.CorticalAreas_dil_Final_Final_Areas_Group_Colors.32k_fs_LR.dlabel.nii'

    # labeldata =  iofiles.CIFTI(labelpath).read_cifti()
    getdata = get_hcp_data(HCP_TEST_DATA)
    pathlist = getdata.get_file_path_list('func')
    print(pathlist)
    # getdata.getsave_certain_data('other',labeldata,'brainsize_test',data_out_file)
    # print('now we are getting s2 beta value data')2
    # getdata.getsave_certain_data('func',labeldata,'beta_value',data_out_file)
    # print('now we are getting s2 msmall beta value data')
    # getdata.getsave_certain_data('func', labeldata, 'beta_value', data_out_file)
    # print('now we are getting s4 beta value data')
    # getdata.getsave_certain_data('func',labeldata,'beta_value',data_out_file)
    # print('now we are getting s4 msmall beta value data')
    # getdata.getsave_certain_data('func', labeldata, 'beta_value', data_out_file)
    # print('now we are getting s8 beta value data')
    # getdata.getsave_certain_data('func',labeldata,'beta_value',data_out_file)
    # print('now we are getting s12  beta value data')
    # getdata.getsave_certain_data('func', labeldata, 'beta_value', data_out_file)
    # print('now we are getting thickness data')
    # getdata.getsave_certain_data('stru', labeldata, 'thickness_msmall', data_out_file)
    # print('now we are getting curvature data')
    # getdata.getsave_certain_data('stru', labeldata, 'cruvature_msmall', data_out_file)
    # print('now we are getting myelinmap data')
    # getdata.getsave_certain_data('stru', labeldata, 'myelinmap_msmall', data_out_file)
    # print('now we are getting brain_size data')
    # getdata.getsave_certain_data('other', labeldata, 'brain_size', data_out_file)
    # print('now we are getting RMS and FD data')
    # getdata.getsave_certain_data('other', labeldata, 'head_motion', data_out_file)

<commit_msg>modify functions about calculating head motion,and add a time decorator to monitor running time<commit_after># emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:

import math
import numpy as np
import os
import csv
from ATT.iofunc import iofiles
from ATT.algorithm import surf_tools
import pandas as pd
from ATT.util import decorators
from memory_profiler import profile
from memory_profiler import memory_usage

HCP_TEST_DATA = 'e:\\projects\\genetic_imaging\\HCPdata\\HCP900\\'
data_out_file = 'E:\\projects\\genetic_imaging\\HCPdata\\data\\HCPExtracted\\'

class get_hcp_data(object):
    def __init__(self,stem_path):
        self.stem_path = stem_path
        subid = os.listdir(stem_path)
        self.subid = subid

    def motion_FD(self,path_list):
        power_rela_fd_mean = []
        dt_rela_fd_mean = []
        def cal_relaFD(re1, re2):
            """
            """
            relative_fd = 0
            for i in range(0, 3):
                relative_fd += abs(re1[i] - re2[i])
            for i in range(3, 6):
                relative_fd += 50 * math.pi / 150 * (abs(re1[i] - re2[i]))

            return math.sqrt(relative_fd)

        def cal_relaFD_mean(ls):
            """
            """
            ls_float = [list(map(lambda x: float(x), j)) for j in ls]
            rela_fd = []
            for i in range(1, len(ls_float)):
                rela_fd.append(cal_relaFD(ls_float[i], ls_float[i - 1]))
            indi_fd_mean = np.mean(np.array(rela_fd))
            return indi_fd_mean

        for i in path_list:
            try:
                with open(i + '/tfMRI_WM_LR/Movement_Regressors.txt', 'r') as f:
                    s = f.readlines()
                    regre_LR = np.array([j.rstrip().split() for j in s])
                with open(i + '/tfMRI_WM_RL/Movement_Regressors.txt', 'r') as f:
                    s = f.readlines()
                    regre_RL = np.array([j.rstrip().split() for j in s])
                indi_FD_mean = (cal_relaFD_mean(regre_LR) + cal_relaFD_mean(regre_RL)) / 2
                power_rela_fd_mean.append(indi_FD_mean)
            except IOError as e:
                print(e)
                power_rela_fd_mean.append([])
        for i in path_list:
            try:
                with open(i + '/tfMRI_WM_LR/Movement_Regressors_dt.txt', 'r') as f:
                    s = f.readlines()
                    regre_LR = regre_LR = np.array([j.rstrip().split() for j in s])
                with open(i + '/tfMRI_WM_RL/Movement_Regressors_dt.txt', 'r') as f:
                    s = f.readlines()
                    regre_RL = np.array([j.rstrip().split() for j in s])

                indi_FD_mean = (cal_relaFD_mean(regre_LR) + cal_relaFD_mean(regre_RL)) / 2
                dt_rela_fd_mean.append(indi_FD_mean)
            except IOError as e:
                print(e)
                dt_rela_fd_mean.append([])

        return power_rela_fd_mean,dt_rela_fd_mean

    def get_brainsize(self,path_list):
        etiv_size_list = []
        for i in path_list:
            try:
                with open(i, 'r') as f:
                    s = f.readlines()[33][-21:-7]
                etiv_size_list.append(float(s))
            except IOError as e:
                print(e)
                etiv_size_list.append([])
        return etiv_size_list

    def motion_RMS(self,path_list):
        relative_meanRMS = []
        for i in path_list:
            try:
                with open(i + '\\tfMRI_WM_LR\\Movement_RelativeRMS_mean.txt', 'r') as f:
                    relative_LR = f.readline()
                with open(i + '\\tfMRI_WM_RL\\Movement_RelativeRMS_mean.txt', 'r') as f:
                    relative_RL = f.readline()
                relative_meanRMS.append((float(relative_LR) + float(relative_RL) / 2))
            except IOError as e:
                print(e)
                relative_meanRMS.append([])
        return relative_meanRMS
    def get_file_path_list(self,file_type):
        """
        generate a list containing needed file path
        file_type: func
                   stru
                   other
        """

        if file_type == 'func':
            catagory = ['tfMRI_WM_hp200_s12_level2.feat','','','','','','']
            catagory = int(input('''
    please input the number of  catagoty you want:
    1.*.hp200_s12_level12.feat
    2.*.hp200_s2_level2.feat
    3.*.hp200_s2_level2_MSMALL.feat
    4.*.hp200_s4_level2.feat
    5.*.hp200_s4_level2_MSMALL.feat
    6.**.hp200_s8_level2.feat
    --
            '''))
            if catagory == 1:
                self.catagory = 'hp200_s12_level12.feat'
                FUNC_STEM_PATH= 'MNINonLinear/Results/tfMRI_WM/tfMRI_WM_hp200_s12_level2.feat/'
            elif catagory == 2:
                self.catagory = 'hp200_s2_level2.feat'
                FUNC_STEM_PATH= 'MNINonLinear/Results/tfMRI_WM/tfMRI_WM_hp200_s2_level2.feat/'
            elif catagory == 3:
                self.catagory = 'hp200_s2_level2_MSMALL.feat'
                FUNC_STEM_PATH= 'MNINonLinear/Results/tfMRI_WM/tfMRI_WM_hp200_s2_level2_MSMALL.feat/'
            elif catagory == 4:
                self.catagory = 'hp200_s4_level2.feat'
                FUNC_STEM_PATH= 'MNINonLinear/Results/tfMRI_WM/tfMRI_WM_hp200_s4_level2.feat/'
            elif catagory == 5:
                self.catagory = 'hp200_s4_level2_MSMALL.feat'
                FUNC_STEM_PATH= 'MNINonLinear/Results/tfMRI_WM/tfMRI_WM_hp200_s4_level2_MSMALL.feat/'
            elif catagory == 6:
                self.catagory = 'hp200_s8_level2.feat'
                FUNC_STEM_PATH= 'MNINonLinear/Results/tfMRI_WM/tfMRI_WM_hp200_s8_level2.feat/'

            data_type = int(input('please input the number of your data type:\n:1.t value 2.beta\n--'))
            contrast_type = int(input('''
            please input the contrast type
            1.body-avg
            2.face-avg
            3.place-avg
            4.tool-avg
            '''))
            if data_type == 1:

                if  contrast_type == 1:
                    FILE_NAME = 'GrayordinatesStats/cope19.feat/tstat1.dtseries.nii'
                elif  contrast_type == 2:
                    FILE_NAME = 'GrayordinatesStats/cope20.feat/tstat1.dtseries.nii'
                elif  contrast_type == 3:
                    FILE_NAME = 'GrayordinatesStats/cope21.feat/tstat1.dtseries.nii'
                elif  contrast_type == 4:
                    FILE_NAME = 'GrayordinatesStats/cope22.feat/tstat1.dtseries.nii'
                path_list = [self.stem_path+i+'/'+FUNC_STEM_PATH+ FILE_NAME for i in self.subid]

            if data_type == 2:
                if  contrast_type == 1:
                    FILE_NAME = 'GrayordinatesStats/cope19.feat/cope1.dtseries.nii'
                elif  contrast_type == 2:
                    FILE_NAME = 'GrayordinatesStats/cope20.feat/cope1.dtseries.nii'
                elif  contrast_type == 3:
                    FILE_NAME = 'GrayordinatesStats/cope21.feat/cope1.dtseries.nii'
                elif  contrast_type == 4:
                    FILE_NAME = 'GrayordinatesStats/cope22.feat/cope1.dtseries.nii'
                path_list = [self.stem_path+i+'/'+FUNC_STEM_PATH+ FILE_NAME for i in self.subid]
        elif file_type == 'stru':
            self.catagory = ''
    #        stru_type = ['*.MyelinMap_MSMAll.32k_fs_LR.dscalar.nii',
    #        '*.curvature_MSMAll.32k_fs_LR.dscalar.nii','*.thickness_MSMAll.32k_fs_LR.dscalar.nii']
            stru_type_num = int(input('''
    please input the structure file you need (not number)
    1.*.MyelinMap_MSMAll.32k_fs_LR.dscalar.nii
    2.*.curvature_MSMAll.32k_fs_LR.dscalar.nii
    3.*.thickness_MSMAll.32k_fs_LR.dscalar.nii
    --
            '''))
            if stru_type_num == 1 :
                stru_type = '.MyelinMap_MSMAll.32k_fs_LR.dscalar.nii'
            if stru_type_num == 2 :
                stru_type = '.curvature_MSMAll.32k_fs_LR.dscalar.nii'
            if stru_type_num == 3 :
                stru_type = '.thickness_MSMAll.32k_fs_LR.dscalar.nii'
            brain_region_num = int(input('''
    please input number of your target region
    1. left hemisphere
    2. right hemishpere
    3. whole brain
    --
             '''))
            if brain_region_num ==3:
                path_list  = [self.stem_path+i+'/'+'MNINonLinear/fsaverage_LR32k/'+i+stru_type for i in self.subid]
            if brain_region_num ==1:
                path_list  = [self.stem_path+i+'/'+'MNINonLinear/fsaverage_LR32k/'+i+'.L'+stru_type for i in self.subid]
            if brain_region_num ==2:
                path_list  = [self.stem_path+i+'/'+'MNINonLinear/fsaverage_LR32k/'+i+'.R'+stru_type for i in self.subid]

        elif file_type == 'other':
            other_type = int(input('''
            please input the type of data you want
            1.Motion
            2.brain_size
            '''))
            if other_type == 1:
                self.other_type ='motion'
                path_list = [self.stem_path+i+'/'+'MNINonLinear/Results' for i in self.subid]
            elif other_type == 2:
                self.other_type = 'brain_size'
                path_list = [self.stem_path+i+'/'+'T1w/'+i+'/'+'stats/aseg.stats' for i in self.subid]
        else:
            raise Exception('please input the right file type: func, stru, other')
        return path_list

    @decorators.timer
    #@profile
    def getsave_certain_data(self,file_type,label_data,output,output_path):
        '''
        '''
        path_list = self.get_file_path_list(file_type)
        data_list = []
        if file_type == 'func' or file_type=='stru':
            for i in path_list:
                try:
                    data = iofiles.CIFTI(i).read_cifti()
                    data_list.append(
                    surf_tools.get_signals(data, label_data))  # get roi mean value of each roi of each subject
                except IOError as e:
                    print(e)
                    data_list.append([])

            out_file_name = output_path+file_type+'/'+self.catagory+'/'+output+'.csv'
        elif file_type == 'other':
            data_list = []
            if self.other_type == 'motion':
                data_list.append(self.motion_RMS(path_list))
                data_list.append(self.motion_FD(path_list)[0])
                data_list.append(self.motion_FD(path_list)[1])
            elif self.other_type == 'brain_size':
                data_list.append(self.get_brainsize(path_list))

            data_list=np.array(data_list).T

            out_file_name = output_path+file_type+'/'+output+'.csv'

        pd_data_list = pd.DataFrame(data_list,index=self.subid)
        pd_data_list.to_csv(out_file_name)
        # with open(out_file_name,'w',newline = '') as f:
        #         f_csv = csv.writer(f)path
        #         f_csv.writerows(data_list)1
if __name__=='__main__':
    labelpath = 'E:\projects\genetic_imaging\HCPdata\VanEssenMap\HCP_PhaseTwo\Q1-Q6_RelatedParcellation210\MNINonLinear\\fsaverage_LR32k\\Q1-Q6_RelatedParcellation210.CorticalAreas_dil_Final_Final_Areas_Group_Colors.32k_fs_LR.dlabel.nii'

    labeldata =  iofiles.CIFTI(labelpath).read_cifti()
    getdata=get_hcp_data(HCP_TEST_DATA)
    # pathlist = getdata.get_file_path_list('other')
    # print(getdata.motion_FD(pathlist))
    # print(memory_usage(getdata.getsave_certain_data('func',labeldata,'testfortime',data_out_file)))
    getdata.getsave_certain_data('other', labeldata, 'testfortime', data_out_file)

<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-
#
# This file is part of I4P.
#
# I4P is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# I4P is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero Public License for more details.
#
# You should have received a copy of the GNU Affero Public License
# along with I4P.  If not, see <http://www.gnu.org/licenses/>.
#

from django.conf import settings

from piston.handler import BaseHandler

from apps.project_sheet.models import Answer, I4pProject, I4pProjectTranslation, Topic
from apps.project_sheet.utils import get_project_translations_from_parents

class I4pProjectTranslationHandler(BaseHandler):
    """
    Handler used to display informations about a project sheet.
    Use "project_id" GET parameter.
    """
    allowed_methods = ('GET',)
    model = I4pProjectTranslation

    def read(self, request, project_id=None):
        # TODO: Check if class attributes doesn't have problems with threads on production
        if project_id is None:
            language_code = request.GET.get('lang', 'en')
            if language_code not in dict(settings.LANGUAGES) :
                language_code = "en"
            page = int(request.GET.get('page', 1)) - 1
            self.__class__.fields = (
              'title',
              'baseline',
              ('project',(
                  ('location',(
                      'id',
                      'country'
                  )),
                  'best_of',
                  'status',
                  ('pictures',(
                      'id',
                      'thumb'
                  ))
              )),
            )
            # TODO: "pagination" in raw, change it to django standard
            projects = I4pProject.objects.all()[page*10:page*10+10]
            list_projects = get_project_translations_from_parents(projects, language_code, "en", True)
            return list_projects
        else:
            self.__class__.fields = (
              'about_section',
              'baseline',
              'callto_section',
              'completion_progress',
              'partners_section',
              ('project',(
                  'id',
                  ('location',(
                      'address',
                      'country'
                  )),
                  ('members',(
                      'fullname',
                      'username'
                  )),
                  ('objectives',(
                      'id',
                      'name'
                  )),
                  ('pictures',(
                      'author',
                      'created',
                      'desc',
                      'license',
                      'source',
                      'url'
                  )),
                  'questions',
                  ('references',(
                      'id',
                      'desc'
                  )),
                  ('videos',(
                      'id',
                      'video_url'
                  )),
                  'website'
              )),
              'themes',
              'title'
            )
            return I4pProjectTranslation.objects.get(pk=project_id)

    @classmethod
    def fullname(cls, model):
        return model.get_full_name()

    @classmethod
    def url(cls, model):
        return model.display.url

    @classmethod
    def questions(cls, model):
        questions = []
        for topic in Topic.objects.filter(site_topics=model.topics.all()):
            for question in topic.questions.all().order_by('weight'):
                answers = Answer.objects.filter(project=model.id, question=question)
                questions.append({
                    "question": question.content,
                    "answer": answers and answers[0].content or None
                })

        return questions

    @classmethod
    def thumb(cls, model):
        return model.thumbnail_image.url
<commit_msg>Displaying project details using specified language<commit_after># -*- encoding: utf-8 -*-
#
# This file is part of I4P.
#
# I4P is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# I4P is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero Public License for more details.
#
# You should have received a copy of the GNU Affero Public License
# along with I4P.  If not, see <http://www.gnu.org/licenses/>.
#

from django.conf import settings

from piston.handler import BaseHandler

from apps.project_sheet.models import Answer, Objective, I4pProject, I4pProjectTranslation, Topic
from apps.project_sheet.utils import get_project_translations_from_parents

class I4pProjectTranslationHandler(BaseHandler):
    """
    Handler used to display informations about a project sheet.
    Use "project_id" GET parameter.
    """
    allowed_methods = ('GET',)
    model = I4pProjectTranslation
    project = None

    def read(self, request, project_id=None):
        # TODO: Check if class attributes doesn't have problems with threads on production
        if project_id is None:
            language_code = request.GET.get('lang', 'en')
            if language_code not in dict(settings.LANGUAGES) :
                language_code = "en"
            page = int(request.GET.get('page', 1)) - 1
            self.__class__.fields = (
              'title',
              'baseline',
              ('project',(
                  ('location',(
                      'id',
                      'country'
                  )),
                  'best_of',
                  'status',
                  ('pictures',(
                      'id',
                      'thumb'
                  ))
              )),
            )
            # TODO: "pagination" in raw, change it to django standard
            projects = I4pProject.objects.all()[page*10:page*10+10]
            list_projects = get_project_translations_from_parents(projects, language_code, "en", True)
            return list_projects
        else:
            self.__class__.project = I4pProjectTranslation.objects.get(pk=project_id)
            self.__class__.fields = (
              'about_section',
              'baseline',
              'callto_section',
              'completion_progress',
              'partners_section',
              ('project',(
                  'id',
                  ('location',(
                      'address',
                      'country'
                  )),
                  ('members',(
                      'fullname',
                      'username'
                  )),
                  'objective',
                  ('pictures',(
                      'author',
                      'created',
                      'desc',
                      'license',
                      'source',
                      'url'
                  )),
                  'questions',
                  ('references',(
                      'id',
                      'desc'
                  )),
                  ('videos',(
                      'id',
                      'video_url'
                  )),
                  'website'
              )),
              'themes',
              'title'
            )
            return self.__class__.project

    @classmethod
    def fullname(cls, model):
        return model.get_full_name()

    @classmethod
    def url(cls, model):
        return model.display.url

    @classmethod
    def questions(cls, model):
        questions = []
        for topic in Topic.objects.filter(site_topics=model.topics.all()):
            for question in topic.questions.language(I4pProjectTranslationHandler.project.language_code).all().order_by('weight'):
                answers = Answer.objects.language(I4pProjectTranslationHandler.project.language_code).filter(project=model.id, question=question)
                questions.append({
                    "question": question.content,
                    "answer": answers and answers[0].content or None
                })

        return questions

    @classmethod
    def thumb(cls, model):
        return model.thumbnail_image.url

    # To get correct language for translated models
    @classmethod
    def objective(cls, model):
        objectives = model.objectives.language(I4pProjectTranslationHandler.project.language_code).all()
        objectives_list = []
        for objective in objectives:
            objectives_list.append({"name": objective.name})
        return objectives_list
<|endoftext|>


<commit_before>import re
from corehq.apps.api.object_fetch_api import CaseAttachmentAPI

from corehq.apps.api.domainapi import DomainAPI
from corehq.apps.api.redis_assets import RedisAssetsAPI
from corehq.apps.api.resources import v0_1, v0_2, v0_3, v0_4, v0_5, v0_6
from corehq.apps.commtrack.resources.v0_1 import ProductResource,\
    StockStatusResource, StockReportResource, FullStockTransactionResource
from corehq.apps.fixtures.resources.v0_1 import FixtureResource
from corehq.apps.locations.resources.v0_1 import LocationResource
from corehq.apps.reports.resources.v0_1 import ReportResource
from django.conf.urls.defaults import *
from django.http import HttpResponseNotFound
from tastypie.api import Api
from corehq.apps.api.es import XFormES
from dimagi.utils.decorators import inline

API_LIST = (
    ((0, 1), (
        v0_1.CommCareUserResource,
        v0_1.WebUserResource,
        v0_1.CommCareCaseResource,
        v0_1.XFormInstanceResource,
        FixtureResource,
        ReportResource,
    )),
    ((0, 2), (
        v0_1.CommCareUserResource,
        v0_1.WebUserResource,
        v0_2.CommCareCaseResource,
        v0_1.XFormInstanceResource,
        FixtureResource,
        ReportResource,
    )),
    ((0, 3), (
        v0_1.CommCareUserResource,
        v0_1.WebUserResource,
        v0_3.CommCareCaseResource,
        v0_3.XFormInstanceResource,
        FixtureResource,
        ReportResource,
    )),
    ((0, 4), (
        v0_1.CommCareUserResource,
        v0_1.WebUserResource,
        v0_4.ApplicationResource,
        v0_4.CommCareCaseResource,
        v0_4.GroupResource,
        v0_4.XFormInstanceResource,
        v0_4.RepeaterResource,
        v0_4.SingleSignOnResource,
        v0_4.HOPECaseResource,
        FixtureResource,
        ReportResource,
    )),
    ((0, 5), (
        v0_4.ApplicationResource,
        v0_4.CommCareCaseResource,
        v0_4.XFormInstanceResource,
        v0_4.RepeaterResource,
        v0_4.SingleSignOnResource,
        v0_4.HOPECaseResource,
        v0_5.CommCareUserResource,
        v0_5.WebUserResource,
        v0_5.GroupResource,
        v0_5.BulkUserResource,
        FixtureResource,
        ReportResource,
    )),
)

# eventually these will have to version too but this works for now
COMMTRACK_RESOURCES = (LocationResource, ProductResource, StockStatusResource,
                       FullStockTransactionResource, StockReportResource)

class CommCareHqApi(Api):
    def top_level(self, request, api_name=None, **kwargs):
        return HttpResponseNotFound()

        @property
        def urls(self):
            """
            Exactly copied from https://github.com/toastdriven/django-tastypie/blob/v0.9.11/tastypie/api.py#L84
            (BSD-licensed) and hotfixed for https://github.com/toastdriven/django-tastypie/issues/816
            """
            api_name_regex = re.escape(self.api_name)

            pattern_list = [
                url(r"^(?P<api_name>%s)%s$" % (api_name_regex, trailing_slash()), self.wrap_view('top_level'), name="api_%s_top_level" % self.api_name),
            ]

            for name in sorted(self._registry.keys()):
                self._registry[name].api_name = self.api_name
                pattern_list.append((r"^(?P<api_name>%s)/" % api_name_regex, include(self._registry[name].urls)))

            urlpatterns = self.override_urls() + patterns('',
                *pattern_list
            )
            return urlpatterns

@inline
def api_url_patterns():
    for version, resources in API_LIST:
        api = CommCareHqApi(api_name='v%d.%d' % version)
        for R in resources:
            api.register(R())
        for R in COMMTRACK_RESOURCES:
            api.register(R())
        yield (r'^', include(api.urls))
    yield url(r'^v0.1/xform_es/$', XFormES.as_domain_specific_view())
    # HACK: fix circular import here, to fix later
    try:
        from pact.api import PactAPI
    except ImportError:
        pass # maybe pact isn't installed
    for view_class in DomainAPI.__subclasses__():
        yield url(r'^custom/%s/v%s/$' % (view_class.api_name(), view_class.api_version()), view_class.as_view(), name="%s_%s" % (view_class.api_name(), view_class.api_version()))
    yield url(r'^case/attachment/(?P<case_id>[\w\-]+)/(?P<attachment_id>.*)/(?P<attachment_src>.*)$', CaseAttachmentAPI.as_view(), name="api_case_attachment")
    yield url(r'^redis_assets/$', RedisAssetsAPI.as_view())

urlpatterns = patterns('',
    *list(api_url_patterns))
<commit_msg>remove extra import<commit_after>import re
from corehq.apps.api.object_fetch_api import CaseAttachmentAPI

from corehq.apps.api.domainapi import DomainAPI
from corehq.apps.api.redis_assets import RedisAssetsAPI
from corehq.apps.api.resources import v0_1, v0_2, v0_3, v0_4, v0_5
from corehq.apps.commtrack.resources.v0_1 import ProductResource,\
    StockStatusResource, StockReportResource, FullStockTransactionResource
from corehq.apps.fixtures.resources.v0_1 import FixtureResource
from corehq.apps.locations.resources.v0_1 import LocationResource
from corehq.apps.reports.resources.v0_1 import ReportResource
from django.conf.urls.defaults import *
from django.http import HttpResponseNotFound
from tastypie.api import Api
from corehq.apps.api.es import XFormES
from dimagi.utils.decorators import inline

API_LIST = (
    ((0, 1), (
        v0_1.CommCareUserResource,
        v0_1.WebUserResource,
        v0_1.CommCareCaseResource,
        v0_1.XFormInstanceResource,
        FixtureResource,
        ReportResource,
    )),
    ((0, 2), (
        v0_1.CommCareUserResource,
        v0_1.WebUserResource,
        v0_2.CommCareCaseResource,
        v0_1.XFormInstanceResource,
        FixtureResource,
        ReportResource,
    )),
    ((0, 3), (
        v0_1.CommCareUserResource,
        v0_1.WebUserResource,
        v0_3.CommCareCaseResource,
        v0_3.XFormInstanceResource,
        FixtureResource,
        ReportResource,
    )),
    ((0, 4), (
        v0_1.CommCareUserResource,
        v0_1.WebUserResource,
        v0_4.ApplicationResource,
        v0_4.CommCareCaseResource,
        v0_4.GroupResource,
        v0_4.XFormInstanceResource,
        v0_4.RepeaterResource,
        v0_4.SingleSignOnResource,
        v0_4.HOPECaseResource,
        FixtureResource,
        ReportResource,
    )),
    ((0, 5), (
        v0_4.ApplicationResource,
        v0_4.CommCareCaseResource,
        v0_4.XFormInstanceResource,
        v0_4.RepeaterResource,
        v0_4.SingleSignOnResource,
        v0_4.HOPECaseResource,
        v0_5.CommCareUserResource,
        v0_5.WebUserResource,
        v0_5.GroupResource,
        v0_5.BulkUserResource,
        FixtureResource,
        ReportResource,
    )),
)

# eventually these will have to version too but this works for now
COMMTRACK_RESOURCES = (LocationResource, ProductResource, StockStatusResource,
                       FullStockTransactionResource, StockReportResource)

class CommCareHqApi(Api):
    def top_level(self, request, api_name=None, **kwargs):
        return HttpResponseNotFound()

        @property
        def urls(self):
            """
            Exactly copied from https://github.com/toastdriven/django-tastypie/blob/v0.9.11/tastypie/api.py#L84
            (BSD-licensed) and hotfixed for https://github.com/toastdriven/django-tastypie/issues/816
            """
            api_name_regex = re.escape(self.api_name)

            pattern_list = [
                url(r"^(?P<api_name>%s)%s$" % (api_name_regex, trailing_slash()), self.wrap_view('top_level'), name="api_%s_top_level" % self.api_name),
            ]

            for name in sorted(self._registry.keys()):
                self._registry[name].api_name = self.api_name
                pattern_list.append((r"^(?P<api_name>%s)/" % api_name_regex, include(self._registry[name].urls)))

            urlpatterns = self.override_urls() + patterns('',
                *pattern_list
            )
            return urlpatterns

@inline
def api_url_patterns():
    for version, resources in API_LIST:
        api = CommCareHqApi(api_name='v%d.%d' % version)
        for R in resources:
            api.register(R())
        for R in COMMTRACK_RESOURCES:
            api.register(R())
        yield (r'^', include(api.urls))
    yield url(r'^v0.1/xform_es/$', XFormES.as_domain_specific_view())
    # HACK: fix circular import here, to fix later
    try:
        from pact.api import PactAPI
    except ImportError:
        pass # maybe pact isn't installed
    for view_class in DomainAPI.__subclasses__():
        yield url(r'^custom/%s/v%s/$' % (view_class.api_name(), view_class.api_version()), view_class.as_view(), name="%s_%s" % (view_class.api_name(), view_class.api_version()))
    yield url(r'^case/attachment/(?P<case_id>[\w\-]+)/(?P<attachment_id>.*)/(?P<attachment_src>.*)$', CaseAttachmentAPI.as_view(), name="api_case_attachment")
    yield url(r'^redis_assets/$', RedisAssetsAPI.as_view())

urlpatterns = patterns('',
    *list(api_url_patterns))
<|endoftext|>


<commit_before>'''
The Saltutil module is used to manage the state of the salt minion itself. It
is used to manage minion modules as well as automate updates to the salt minion

:depends:   - esky Python module for update functionality
'''

# Import python libs
import os
import hashlib
import shutil
import signal
import logging
import fnmatch
import time
import sys

# Import salt libs
import salt.payload
import salt.state
import salt.client
from salt.exceptions import SaltReqTimeoutError
from salt._compat import string_types

# Import third party libs
try:
    import esky
    HAS_ESKY = True
except ImportError:
    HAS_ESKY = False

log = logging.getLogger(__name__)

def _sync(form, env=None):
    '''
    Sync the given directory in the given environment
    '''
    if env is None:
        # No environment passed, detect them based on gathering the top files
        # from the master
        env = 'base'
        st_ = salt.state.HighState(__opts__)
        top = st_.get_top()
        if top:
            env = st_.top_matches(top).keys()
    if isinstance(env, string_types):
        env = env.split(',')
    ret = []
    remote = set()
    source = os.path.join('salt://_{0}'.format(form))
    mod_dir = os.path.join(__opts__['extension_modules'], '{0}'.format(form))
    if not os.path.isdir(mod_dir):
        log.info('Creating module dir \'{0}\''.format(mod_dir))
        os.makedirs(mod_dir)
    for sub_env in env:
        log.info('Syncing {0} for environment \'{1}\''.format(form, sub_env))
        cache = []
        log.info('Loading cache from {0}, for {1})'.format(source, sub_env))
        cache.extend(__salt__['cp.cache_dir'](source, sub_env))
        local_cache_dir = os.path.join(
                __opts__['cachedir'],
                'files',
                sub_env,
                '_{0}'.format(form)
                )
        log.debug('Local cache dir: \'{0}\''.format(local_cache_dir))
        for fn_ in cache:
            if __opts__.get('file_client', '') == 'local':
                for fn_root in __opts__['file_roots'].get(sub_env, []):
                    if fn_.startswith(fn_root):
                        relpath = os.path.relpath(fn_, fn_root)
                        relpath = relpath[relpath.index('/') + 1:]
                        relname = os.path.splitext(relpath)[0].replace(
                                os.sep,
                                '.')
                        remote.add(relpath)
                        dest = os.path.join(mod_dir, relpath)
            else:
                relpath = os.path.relpath(fn_, local_cache_dir)
                relname = os.path.splitext(relpath)[0].replace(os.sep, '.')
                remote.add(relpath)
                dest = os.path.join(mod_dir, relpath)
            log.info('Copying \'{0}\' to \'{1}\''.format(fn_, dest))
            if os.path.isfile(dest):
                # The file is present, if the sum differs replace it
                srch = hashlib.md5(
                    salt.utils.fopen(fn_, 'r').read()
                ).hexdigest()
                dsth = hashlib.md5(
                    salt.utils.fopen(dest, 'r').read()
                ).hexdigest()
                if srch != dsth:
                    # The downloaded file differs, replace!
                    shutil.copyfile(fn_, dest)
                    ret.append('{0}.{1}'.format(form, relname))
            else:
                dest_dir = os.path.dirname(dest)
                if not os.path.isdir(dest_dir):
                    os.makedirs(dest_dir)
                shutil.copyfile(fn_, dest)
                ret.append('{0}.{1}'.format(form, relname))

    touched = bool(ret)
    if __opts__.get('clean_dynamic_modules', True):
        current = set(_listdir_recursively(mod_dir))
        for fn_ in current - remote:
            full = os.path.join(mod_dir, fn_)
            if os.path.isfile(full):
                touched = True
                os.remove(full)
        #cleanup empty dirs
        while True:
            emptydirs = _list_emptydirs(mod_dir)
            if not emptydirs:
                break
            for emptydir in emptydirs:
                touched = True
                os.rmdir(emptydir)
    #dest mod_dir is touched? trigger reload if requested
    if touched:
        mod_file = os.path.join(__opts__['cachedir'], 'module_refresh')
        with salt.utils.fopen(mod_file, 'a+') as ofile:
            ofile.write('')
    return ret

def _listdir_recursively(rootdir):
    file_list = []
    for root, dirs, files in os.walk(rootdir):
        for filename in files:
            relpath = os.path.relpath(root, rootdir).strip('.')
            file_list.append(os.path.join(relpath, filename))
    return file_list

def _list_emptydirs(rootdir):
    emptydirs = []
    for root, dirs, files in os.walk(rootdir):
        if not files and not dirs:
            emptydirs.append(root)
    return emptydirs

def update(version=None):
    '''
    Update the salt minion from the URL defined in opts['update_url']

    This feature requires the minion to be running a bdist_esky build.

    The version number is optional and will default to the most recent version
    available at opts['update_url'].

    Returns details about the transaction upon completion.

    CLI Example:

    .. code-block:: bash

        salt '*' saltutil.update 0.10.3
    '''
    if not HAS_ESKY:
        return 'Esky not available as import'
    if not getattr(sys, 'frozen', False):
        return 'Minion is not running an Esky build'
    if not __salt__['config.option']('update_url'):
        return '"update_url" not configured on this minion'
    app = esky.Esky(sys.executable, __opts__['update_url'])
    oldversion = __grains__['saltversion']
    try:
        if not version:
            version = app.find_update()
        if not version:
            return 'No updates available'
        app.fetch_version(version)
        app.install_version(version)
        app.cleanup()
    except Exception as err:
        return err
    restarted = {}
    for service in __opts__['update_restart_services']:
        restarted[service] = __salt__['service.restart'](service)
    return {'comment': 'Updated from {0} to {1}'.format(oldversion, version),
            'restarted': restarted}

def sync_modules(env=None, refresh=True):
    '''
    Sync the modules from the _modules directory on the salt master file
    server. This function is environment aware, pass the desired environment
    to grab the contents of the _modules directory, base is the default
    environment.

    CLI Example:

    .. code-block:: bash

        salt '*' saltutil.sync_modules
    '''
    ret = _sync('modules', env)
    if refresh:
        refresh_modules()
    return ret

def sync_states(env=None, refresh=True):
    '''
    Sync the states from the _states directory on the salt master file
    server. This function is environment aware, pass the desired environment
    to grab the contents of the _states directory, base is the default
    environment.

    CLI Example:

    .. code-block:: bash

        salt '*' saltutil.sync_states
    '''
    ret = _sync('states', env)
    if refresh:
        refresh_modules()
    return ret

def sync_grains(env=None, refresh=True):
    '''
    Sync the grains from the _grains directory on the salt master file
    server. This function is environment aware, pass the desired environment
    to grab the contents of the _grains directory, base is the default
    environment.

    CLI Example:

    .. code-block:: bash

        salt '*' saltutil.sync_grains
    '''
    ret = _sync('grains', env)
    if refresh:
        refresh_modules()
    return ret

def sync_renderers(env=None, refresh=True):
    '''
    Sync the renderers from the _renderers directory on the salt master file
    server. This function is environment aware, pass the desired environment
    to grab the contents of the _renderers directory, base is the default
    environment.

    CLI Example:

    .. code-block:: bash

        salt '*' saltutil.sync_renderers
    '''
    ret = _sync('renderers', env)
    if refresh:
        refresh_modules()
    return ret

def sync_returners(env=None, refresh=True):
    '''
    Sync the returners from the _returners directory on the salt master file
    server. This function is environment aware, pass the desired environment
    to grab the contents of the _returners directory, base is the default
    environment.

    CLI Example:

    .. code-block:: bash

        salt '*' saltutil.sync_returners
    '''
    ret = _sync('returners', env)
    if refresh:
        refresh_modules()
    return ret

def sync_outputters(env=None, refresh=True):
    '''
    Sync the outputters from the _outputters directory on the salt master file
    server. This function is environment aware, pass the desired environment
    to grab the contents of the _outputters directory, base is the default
    environment.

    CLI Example:

    .. code-block:: bash

        salt '*' saltutil.sync_outputters
    '''
    ret = _sync('outputters', env)
    if refresh:
        refresh_modules()
    return ret

def sync_all(env=None, refresh=True):
    '''
    Sync down all of the dynamic modules from the file server for a specific
    environment

    CLI Example:

    .. code-block:: bash

        salt '*' saltutil.sync_all
    '''
    log.debug('Syncing all')
    ret = {}
    ret['modules'] = sync_modules(env, False)
    ret['states'] = sync_states(env, False)
    ret['grains'] = sync_grains(env, False)
    ret['renderers'] = sync_renderers(env, False)
    ret['returners'] = sync_returners(env, False)
    ret['outputters'] = sync_outputters(env, False)
    if refresh:
        refresh_modules()
    return ret

def refresh_pillar():
    '''
    Signal the minion to refresh the pillar data.

    CLI Example:

    .. code-block:: bash

        salt '*' saltutil.refresh_pillar
    '''
    __salt__['event.fire']({}, 'pillar_refresh')

def refresh_modules():
    '''
    Signal the minion to refresh the module and grain data

    CLI Example:

    .. code-block:: bash

        salt '*' saltutil.refresh_modules
    '''
    __salt__['event.fire']({}, 'module_refresh')

def is_running(fun):
    '''
    If the named function is running return the data associated with it/them.
    The argument can be a glob

    CLI Example:

    .. code-block:: bash

        salt '*' saltutil.is_running state.highstate
    '''
    run = running()
    ret = []
    for data in run:
        if fnmatch.fnmatch(data.get('fun', ''), fun):
            ret.append(data)
    return ret

def running():
    '''
    Return the data on all running salt processes on the minion

    CLI Example:

    .. code-block:: bash

        salt '*' saltutil.running
    '''
    procs = __salt__['status.procs']()
    ret = []
    serial = salt.payload.Serial(__opts__)
    pid = os.getpid()
    proc_dir = os.path.join(__opts__['cachedir'], 'proc')
    if not os.path.isdir(proc_dir):
        return []
    for fn_ in os.listdir(proc_dir):
        path = os.path.join(proc_dir, fn_)
        with salt.utils.fopen(path, 'rb') as fp_:
            data = serial.loads(fp_.read())
        if not isinstance(data, dict):
            # Invalid serial object
            continue
        if not procs.get(str(data['pid'])):
            # The process is no longer running, clear out the file and
            # continue
            os.remove(path)
            continue
        if data.get('pid') == pid:
            continue
        ret.append(data)
    return ret

def find_job(jid):
    '''
    Return the data for a specific job id

    CLI Example:

    .. code-block:: bash

        salt '*' saltutil.find_job <job id>
    '''
    for data in running():
        if data['jid'] == jid:
            return data
    return {}

def signal_job(jid, sig):
    '''
    Sends a signal to the named salt job's process

    CLI Example:

    .. code-block:: bash

        salt '*' saltutil.signal_job <job id> 15
    '''
    for data in running():
        if data['jid'] == jid:
            try:
                os.kill(int(data['pid']), sig)
                return 'Signal {0} sent to job {1} at pid {2}'.format(
                        int(sig),
                        jid,
                        data['pid']
                        )
            except OSError:
                path = os.path.join(__opts__['cachedir'], 'proc', str(jid))
                if os.path.isfile(path):
                    os.remove(path)
                return ('Job {0} was not running and job data has been '
                        ' cleaned up').format(jid)
    return ''

def term_job(jid):
    '''
    Sends a termination signal (SIGTERM 15) to the named salt job's process

    CLI Example:

    .. code-block:: bash

        salt '*' saltutil.term_job <job id>
    '''
    return signal_job(jid, signal.SIGTERM)

def kill_job(jid):
    '''
    Sends a kill signal (SIGKILL 9) to the named salt job's process

    CLI Example:

    .. code-block:: bash

        salt '*' saltutil.kill_job <job id>
    '''
    return signal_job(jid, signal.SIGKILL)

def regen_keys():
    '''
    Used to regenerate the minion keys.

    CLI Example:

    .. code-block:: bash

        salt '*' saltutil.regen_keys
    '''
    for fn_ in os.listdir(__opts__['pki_dir']):
        path = os.path.join(__opts__['pki_dir'], fn_)
        try:
            os.remove(path)
        except os.error:
            pass
    time.sleep(60)
    sreq = salt.payload.SREQ(__opts__['master_uri'])
    auth = salt.crypt.SAuth(__opts__)

def revoke_auth():
    '''
    The minion sends a request to the master to revoke its own key.
    Note that the minion session will be revoked and the minion may
    not be able to return the result of this command back to the master.

    CLI Example:

    .. code-block:: bash

        salt '*' saltutil.revoke_auth
    '''
    sreq = salt.payload.SREQ(__opts__['master_uri'])
    auth = salt.crypt.SAuth(__opts__)
    tok = auth.gen_token('salt')
    load = {'cmd': 'revoke_auth',
            'id': __opts__['id'],
            'tok': tok}
    try:
        return auth.crypticle.loads(
                sreq.send('aes', auth.crypticle.dumps(load), 1))
    except SaltReqTimeoutError:
        return False
    return False

def cmd(tgt,
        fun,
        arg=(),
        timeout=None,
        expr_form='glob',
        ret='',
        kwarg=None,
        **kwargs):
    '''
    Assuming this minion is a master, execute a salt command

    CLI Example:

    .. code-block:: bash

        salt '*' saltutil.cmd
    '''
    local = salt.client.LocalClient(os.path.dirname(__opts__['conf_path']))
    ret = {}
    for ret_comp in local.cmd_iter(
            tgt,
            fun,
            arg,
            timeout,
            expr_form,
            ret,
            kwarg,
            **kwargs):
        ret.update(ret_comp)
    return ret
<commit_msg>Add ssh support to saltutil.cmd<commit_after>'''
The Saltutil module is used to manage the state of the salt minion itself. It
is used to manage minion modules as well as automate updates to the salt minion

:depends:   - esky Python module for update functionality
'''

# Import python libs
import os
import hashlib
import shutil
import signal
import logging
import fnmatch
import time
import sys

# Import salt libs
import salt.payload
import salt.state
import salt.client
from salt.exceptions import SaltReqTimeoutError
from salt._compat import string_types

# Import third party libs
try:
    import esky
    HAS_ESKY = True
except ImportError:
    HAS_ESKY = False

log = logging.getLogger(__name__)

def _sync(form, env=None):
    '''
    Sync the given directory in the given environment
    '''
    if env is None:
        # No environment passed, detect them based on gathering the top files
        # from the master
        env = 'base'
        st_ = salt.state.HighState(__opts__)
        top = st_.get_top()
        if top:
            env = st_.top_matches(top).keys()
    if isinstance(env, string_types):
        env = env.split(',')
    ret = []
    remote = set()
    source = os.path.join('salt://_{0}'.format(form))
    mod_dir = os.path.join(__opts__['extension_modules'], '{0}'.format(form))
    if not os.path.isdir(mod_dir):
        log.info('Creating module dir \'{0}\''.format(mod_dir))
        os.makedirs(mod_dir)
    for sub_env in env:
        log.info('Syncing {0} for environment \'{1}\''.format(form, sub_env))
        cache = []
        log.info('Loading cache from {0}, for {1})'.format(source, sub_env))
        cache.extend(__salt__['cp.cache_dir'](source, sub_env))
        local_cache_dir = os.path.join(
                __opts__['cachedir'],
                'files',
                sub_env,
                '_{0}'.format(form)
                )
        log.debug('Local cache dir: \'{0}\''.format(local_cache_dir))
        for fn_ in cache:
            if __opts__.get('file_client', '') == 'local':
                for fn_root in __opts__['file_roots'].get(sub_env, []):
                    if fn_.startswith(fn_root):
                        relpath = os.path.relpath(fn_, fn_root)
                        relpath = relpath[relpath.index('/') + 1:]
                        relname = os.path.splitext(relpath)[0].replace(
                                os.sep,
                                '.')
                        remote.add(relpath)
                        dest = os.path.join(mod_dir, relpath)
            else:
                relpath = os.path.relpath(fn_, local_cache_dir)
                relname = os.path.splitext(relpath)[0].replace(os.sep, '.')
                remote.add(relpath)
                dest = os.path.join(mod_dir, relpath)
            log.info('Copying \'{0}\' to \'{1}\''.format(fn_, dest))
            if os.path.isfile(dest):
                # The file is present, if the sum differs replace it
                srch = hashlib.md5(
                    salt.utils.fopen(fn_, 'r').read()
                ).hexdigest()
                dsth = hashlib.md5(
                    salt.utils.fopen(dest, 'r').read()
                ).hexdigest()
                if srch != dsth:
                    # The downloaded file differs, replace!
                    shutil.copyfile(fn_, dest)
                    ret.append('{0}.{1}'.format(form, relname))
            else:
                dest_dir = os.path.dirname(dest)
                if not os.path.isdir(dest_dir):
                    os.makedirs(dest_dir)
                shutil.copyfile(fn_, dest)
                ret.append('{0}.{1}'.format(form, relname))

    touched = bool(ret)
    if __opts__.get('clean_dynamic_modules', True):
        current = set(_listdir_recursively(mod_dir))
        for fn_ in current - remote:
            full = os.path.join(mod_dir, fn_)
            if os.path.isfile(full):
                touched = True
                os.remove(full)
        #cleanup empty dirs
        while True:
            emptydirs = _list_emptydirs(mod_dir)
            if not emptydirs:
                break
            for emptydir in emptydirs:
                touched = True
                os.rmdir(emptydir)
    #dest mod_dir is touched? trigger reload if requested
    if touched:
        mod_file = os.path.join(__opts__['cachedir'], 'module_refresh')
        with salt.utils.fopen(mod_file, 'a+') as ofile:
            ofile.write('')
    return ret

def _listdir_recursively(rootdir):
    file_list = []
    for root, dirs, files in os.walk(rootdir):
        for filename in files:
            relpath = os.path.relpath(root, rootdir).strip('.')
            file_list.append(os.path.join(relpath, filename))
    return file_list

def _list_emptydirs(rootdir):
    emptydirs = []
    for root, dirs, files in os.walk(rootdir):
        if not files and not dirs:
            emptydirs.append(root)
    return emptydirs

def update(version=None):
    '''
    Update the salt minion from the URL defined in opts['update_url']

    This feature requires the minion to be running a bdist_esky build.

    The version number is optional and will default to the most recent version
    available at opts['update_url'].

    Returns details about the transaction upon completion.

    CLI Example:

    .. code-block:: bash

        salt '*' saltutil.update 0.10.3
    '''
    if not HAS_ESKY:
        return 'Esky not available as import'
    if not getattr(sys, 'frozen', False):
        return 'Minion is not running an Esky build'
    if not __salt__['config.option']('update_url'):
        return '"update_url" not configured on this minion'
    app = esky.Esky(sys.executable, __opts__['update_url'])
    oldversion = __grains__['saltversion']
    try:
        if not version:
            version = app.find_update()
        if not version:
            return 'No updates available'
        app.fetch_version(version)
        app.install_version(version)
        app.cleanup()
    except Exception as err:
        return err
    restarted = {}
    for service in __opts__['update_restart_services']:
        restarted[service] = __salt__['service.restart'](service)
    return {'comment': 'Updated from {0} to {1}'.format(oldversion, version),
            'restarted': restarted}

def sync_modules(env=None, refresh=True):
    '''
    Sync the modules from the _modules directory on the salt master file
    server. This function is environment aware, pass the desired environment
    to grab the contents of the _modules directory, base is the default
    environment.

    CLI Example:

    .. code-block:: bash

        salt '*' saltutil.sync_modules
    '''
    ret = _sync('modules', env)
    if refresh:
        refresh_modules()
    return ret

def sync_states(env=None, refresh=True):
    '''
    Sync the states from the _states directory on the salt master file
    server. This function is environment aware, pass the desired environment
    to grab the contents of the _states directory, base is the default
    environment.

    CLI Example:

    .. code-block:: bash

        salt '*' saltutil.sync_states
    '''
    ret = _sync('states', env)
    if refresh:
        refresh_modules()
    return ret

def sync_grains(env=None, refresh=True):
    '''
    Sync the grains from the _grains directory on the salt master file
    server. This function is environment aware, pass the desired environment
    to grab the contents of the _grains directory, base is the default
    environment.

    CLI Example:

    .. code-block:: bash

        salt '*' saltutil.sync_grains
    '''
    ret = _sync('grains', env)
    if refresh:
        refresh_modules()
    return ret

def sync_renderers(env=None, refresh=True):
    '''
    Sync the renderers from the _renderers directory on the salt master file
    server. This function is environment aware, pass the desired environment
    to grab the contents of the _renderers directory, base is the default
    environment.

    CLI Example:

    .. code-block:: bash

        salt '*' saltutil.sync_renderers
    '''
    ret = _sync('renderers', env)
    if refresh:
        refresh_modules()
    return ret

def sync_returners(env=None, refresh=True):
    '''
    Sync the returners from the _returners directory on the salt master file
    server. This function is environment aware, pass the desired environment
    to grab the contents of the _returners directory, base is the default
    environment.

    CLI Example:

    .. code-block:: bash

        salt '*' saltutil.sync_returners
    '''
    ret = _sync('returners', env)
    if refresh:
        refresh_modules()
    return ret

def sync_outputters(env=None, refresh=True):
    '''
    Sync the outputters from the _outputters directory on the salt master file
    server. This function is environment aware, pass the desired environment
    to grab the contents of the _outputters directory, base is the default
    environment.

    CLI Example:

    .. code-block:: bash

        salt '*' saltutil.sync_outputters
    '''
    ret = _sync('outputters', env)
    if refresh:
        refresh_modules()
    return ret

def sync_all(env=None, refresh=True):
    '''
    Sync down all of the dynamic modules from the file server for a specific
    environment

    CLI Example:

    .. code-block:: bash

        salt '*' saltutil.sync_all
    '''
    log.debug('Syncing all')
    ret = {}
    ret['modules'] = sync_modules(env, False)
    ret['states'] = sync_states(env, False)
    ret['grains'] = sync_grains(env, False)
    ret['renderers'] = sync_renderers(env, False)
    ret['returners'] = sync_returners(env, False)
    ret['outputters'] = sync_outputters(env, False)
    if refresh:
        refresh_modules()
    return ret

def refresh_pillar():
    '''
    Signal the minion to refresh the pillar data.

    CLI Example:

    .. code-block:: bash

        salt '*' saltutil.refresh_pillar
    '''
    __salt__['event.fire']({}, 'pillar_refresh')

def refresh_modules():
    '''
    Signal the minion to refresh the module and grain data

    CLI Example:

    .. code-block:: bash

        salt '*' saltutil.refresh_modules
    '''
    __salt__['event.fire']({}, 'module_refresh')

def is_running(fun):
    '''
    If the named function is running return the data associated with it/them.
    The argument can be a glob

    CLI Example:

    .. code-block:: bash

        salt '*' saltutil.is_running state.highstate
    '''
    run = running()
    ret = []
    for data in run:
        if fnmatch.fnmatch(data.get('fun', ''), fun):
            ret.append(data)
    return ret

def running():
    '''
    Return the data on all running salt processes on the minion

    CLI Example:

    .. code-block:: bash

        salt '*' saltutil.running
    '''
    procs = __salt__['status.procs']()
    ret = []
    serial = salt.payload.Serial(__opts__)
    pid = os.getpid()
    proc_dir = os.path.join(__opts__['cachedir'], 'proc')
    if not os.path.isdir(proc_dir):
        return []
    for fn_ in os.listdir(proc_dir):
        path = os.path.join(proc_dir, fn_)
        with salt.utils.fopen(path, 'rb') as fp_:
            data = serial.loads(fp_.read())
        if not isinstance(data, dict):
            # Invalid serial object
            continue
        if not procs.get(str(data['pid'])):
            # The process is no longer running, clear out the file and
            # continue
            os.remove(path)
            continue
        if data.get('pid') == pid:
            continue
        ret.append(data)
    return ret

def find_job(jid):
    '''
    Return the data for a specific job id

    CLI Example:

    .. code-block:: bash

        salt '*' saltutil.find_job <job id>
    '''
    for data in running():
        if data['jid'] == jid:
            return data
    return {}

def signal_job(jid, sig):
    '''
    Sends a signal to the named salt job's process

    CLI Example:

    .. code-block:: bash

        salt '*' saltutil.signal_job <job id> 15
    '''
    for data in running():
        if data['jid'] == jid:
            try:
                os.kill(int(data['pid']), sig)
                return 'Signal {0} sent to job {1} at pid {2}'.format(
                        int(sig),
                        jid,
                        data['pid']
                        )
            except OSError:
                path = os.path.join(__opts__['cachedir'], 'proc', str(jid))
                if os.path.isfile(path):
                    os.remove(path)
                return ('Job {0} was not running and job data has been '
                        ' cleaned up').format(jid)
    return ''

def term_job(jid):
    '''
    Sends a termination signal (SIGTERM 15) to the named salt job's process

    CLI Example:

    .. code-block:: bash

        salt '*' saltutil.term_job <job id>
    '''
    return signal_job(jid, signal.SIGTERM)

def kill_job(jid):
    '''
    Sends a kill signal (SIGKILL 9) to the named salt job's process

    CLI Example:

    .. code-block:: bash

        salt '*' saltutil.kill_job <job id>
    '''
    return signal_job(jid, signal.SIGKILL)

def regen_keys():
    '''
    Used to regenerate the minion keys.

    CLI Example:

    .. code-block:: bash

        salt '*' saltutil.regen_keys
    '''
    for fn_ in os.listdir(__opts__['pki_dir']):
        path = os.path.join(__opts__['pki_dir'], fn_)
        try:
            os.remove(path)
        except os.error:
            pass
    time.sleep(60)
    sreq = salt.payload.SREQ(__opts__['master_uri'])
    auth = salt.crypt.SAuth(__opts__)

def revoke_auth():
    '''
    The minion sends a request to the master to revoke its own key.
    Note that the minion session will be revoked and the minion may
    not be able to return the result of this command back to the master.

    CLI Example:

    .. code-block:: bash

        salt '*' saltutil.revoke_auth
    '''
    sreq = salt.payload.SREQ(__opts__['master_uri'])
    auth = salt.crypt.SAuth(__opts__)
    tok = auth.gen_token('salt')
    load = {'cmd': 'revoke_auth',
            'id': __opts__['id'],
            'tok': tok}
    try:
        return auth.crypticle.loads(
                sreq.send('aes', auth.crypticle.dumps(load), 1))
    except SaltReqTimeoutError:
        return False
    return False

def cmd(tgt,
        fun,
        arg=(),
        timeout=None,
        expr_form='glob',
        ret='',
        kwarg=None,
        ssh=False,
        **kwargs):
    '''
    Assuming this minion is a master, execute a salt command

    CLI Example:

    .. code-block:: bash

        salt '*' saltutil.cmd
    '''
    if ssh:
        client = salt.client.SSHClient(
                os.path.dirname(__opts__['conf_path']))
    else:
        client = salt.client.LocalClient(
                os.path.dirname(__opts__['conf_path']))
    ret = {}
    for ret_comp in client.cmd_iter(
            tgt,
            fun,
            arg,
            timeout,
            expr_form,
            ret,
            kwarg,
            **kwargs):
        ret.update(ret_comp)
    return ret

def cmd_iter(tgt,
             fun,
             arg=(),
             timeout=None,
             expr_form='glob',
             ret='',
             kwarg=None,
             ssh=False,
             **kwargs):
    '''
    Assuming this minion is a master, execute a salt command

    CLI Example:

    .. code-block:: bash

        salt '*' saltutil.cmd
    '''
    if ssh:
        client = salt.client.SSHClient(
                os.path.dirname(__opts__['conf_path']))
    else:
        client = salt.client.LocalClient(
                os.path.dirname(__opts__['conf_path']))
    for ret in client.cmd_iter(
            tgt,
            fun,
            arg,
            timeout,
            expr_form,
            ret,
            kwarg,
            **kwargs):
        yield ret
<|endoftext|>


<commit_before># Lint as: python2, python3
# Copyright 2018 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
from __future__ import print_function
import argparse
import os
import plistlib
import re
import subprocess
import sys

# Regex with benign codesign messages that can be safely ignored.
# It matches the following bening outputs:
# * signed Mach-O thin
# * signed Mach-O universal
# * signed app bundle with Mach-O universal
# * signed bundle with Mach-O thin
# * replacing existing signature
_BENIGN_CODESIGN_OUTPUT_REGEX = re.compile(
    r'(signed.*Mach-O (universal|thin)|libswift.*\.dylib: replacing existing signature)'
)

def _check_output(args, inputstr=None):
  proc = subprocess.Popen(
      args,
      stdin=subprocess.PIPE,
      stdout=subprocess.PIPE,
      stderr=subprocess.PIPE)
  stdout, stderr = proc.communicate(input=inputstr)
  if proc.returncode != 0:
    # print the stdout and stderr, as the exception won't print it.
    print("ERROR:{stdout}\n\n{stderr}".format(stdout=stdout, stderr=stderr))
    raise subprocess.CalledProcessError(proc.returncode, args)
  return stdout, stderr

def plist_from_bytes(byte_content):
  try:
    return plistlib.loads(byte_content)
  except AttributeError:
    return plistlib.readPlistFromString(byte_content)

def _parse_mobileprovision_file(mobileprovision_file):
  """Reads and parses a mobileprovision file."""
  plist_xml = subprocess.check_output([
      "security",
      "cms",
      "-D",
      "-i",
      mobileprovision_file,
  ])
  return plist_from_bytes(plist_xml)

def _certificate_fingerprint(identity):
  """Extracts a fingerprint given identity in a mobileprovision file."""
  fingerprint, stderr = _check_output([
      "openssl",
      "x509",
      "-inform",
      "DER",
      "-noout",
      "-fingerprint",
  ],
                                      inputstr=identity)
  fingerprint = fingerprint.decode("utf-8").strip()
  fingerprint = fingerprint.replace("SHA1 Fingerprint=", "")
  fingerprint = fingerprint.replace(":", "")
  return fingerprint

def _get_identities_from_provisioning_profile(mpf):
  """Iterates through all the identities in a provisioning profile, lazily."""
  for identity in mpf["DeveloperCertificates"]:
    yield _certificate_fingerprint(identity.data)

def _find_codesign_identities(identity=None):
  """Finds code signing identities on the current system."""
  ids = []
  output, stderr = _check_output([
      "security",
      "find-identity",
      "-v",
      "-p",
      "codesigning",
  ])
  output = output.decode("utf-8").strip()
  pattern = "(?P<hash>[A-F0-9]{40})"
  if identity:
    pattern += r'\s+"(?P<full_name>.*?{}.*?)"'.format(re.escape(identity))
  regex = re.compile(pattern)
  for line in output.splitlines():
    # CSSMERR_TP_CERT_REVOKED comes from Security.framework/cssmerr.h
    if "CSSMERR_TP_CERT_REVOKED" in line:
      continue
    m = regex.search(line)
    if m:
      groups = m.groupdict()
      id = groups.get("full_name") or groups["hash"]
      ids.append(id)
  return ids

def _find_codesign_identity(mobileprovision):
  """Finds a valid identity on the system given a mobileprovision file."""
  mpf = _parse_mobileprovision_file(mobileprovision)
  ids_codesign = set(_find_codesign_identities())
  for id_mpf in _get_identities_from_provisioning_profile(mpf):
    if id_mpf in ids_codesign:
      return id_mpf

def _filter_codesign_output(codesign_output):
  """Filters the codesign output which can be extra verbose."""
  filtered_lines = []
  for line in codesign_output.split("\n"):
    if line and not _BENIGN_CODESIGN_OUTPUT_REGEX.search(line):
      filtered_lines.append(line)
  return "\n".join(filtered_lines)

def main(argv):
  parser = argparse.ArgumentParser(description="codesign wrapper")
  parser.add_argument(
      "--mobileprovision", type=str, help="mobileprovision file")
  parser.add_argument(
      "--codesign", required=True, type=str, help="path to codesign binary")
  parser.add_argument(
      "--identity", type=str, help="specific identity to sign with")
  args, codesign_args = parser.parse_known_args()
  identity = args.identity
  if identity is None:
    identity = _find_codesign_identity(args.mobileprovision)
  elif identity != "-":
    matching_identities = _find_codesign_identities(identity)
    if matching_identities:
      identity = matching_identities[0]
    else:
      print(
          "ERROR: No signing identity found for '{}'".format(identity),
          file=sys.stderr)
      return -1
  # No identity was found, fail
  if identity == None:
    print("ERROR: Unable to find an identity on the system matching the "\
        "ones in %s" % args.mobileprovision, file=sys.stderr)
    return 1
  stdout, stderr = _check_output([args.codesign, "-v", "--sign", identity] +
                                 codesign_args,)
  if stdout:
    filtered_stdout = _filter_codesign_output(stdout)
    if filtered_stdout:
      print(filtered_stdout)
  if stderr:
    filtered_stderr = _filter_codesign_output(stderr)
    if filtered_stderr:
      print(filtered_stderr)

if __name__ == '__main__':
  sys.exit(main(sys.argv))
<commit_msg>Use splitlines over explicit split.<commit_after># Lint as: python2, python3
# Copyright 2018 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
from __future__ import print_function
import argparse
import os
import plistlib
import re
import subprocess
import sys

# Regex with benign codesign messages that can be safely ignored.
# It matches the following bening outputs:
# * signed Mach-O thin
# * signed Mach-O universal
# * signed app bundle with Mach-O universal
# * signed bundle with Mach-O thin
# * replacing existing signature
_BENIGN_CODESIGN_OUTPUT_REGEX = re.compile(
    r'(signed.*Mach-O (universal|thin)|libswift.*\.dylib: replacing existing signature)'
)

def _check_output(args, inputstr=None):
  proc = subprocess.Popen(
      args,
      stdin=subprocess.PIPE,
      stdout=subprocess.PIPE,
      stderr=subprocess.PIPE)
  stdout, stderr = proc.communicate(input=inputstr)
  if proc.returncode != 0:
    # print the stdout and stderr, as the exception won't print it.
    print("ERROR:{stdout}\n\n{stderr}".format(stdout=stdout, stderr=stderr))
    raise subprocess.CalledProcessError(proc.returncode, args)
  return stdout, stderr

def plist_from_bytes(byte_content):
  try:
    return plistlib.loads(byte_content)
  except AttributeError:
    return plistlib.readPlistFromString(byte_content)

def _parse_mobileprovision_file(mobileprovision_file):
  """Reads and parses a mobileprovision file."""
  plist_xml = subprocess.check_output([
      "security",
      "cms",
      "-D",
      "-i",
      mobileprovision_file,
  ])
  return plist_from_bytes(plist_xml)

def _certificate_fingerprint(identity):
  """Extracts a fingerprint given identity in a mobileprovision file."""
  fingerprint, stderr = _check_output([
      "openssl",
      "x509",
      "-inform",
      "DER",
      "-noout",
      "-fingerprint",
  ],
                                      inputstr=identity)
  fingerprint = fingerprint.decode("utf-8").strip()
  fingerprint = fingerprint.replace("SHA1 Fingerprint=", "")
  fingerprint = fingerprint.replace(":", "")
  return fingerprint

def _get_identities_from_provisioning_profile(mpf):
  """Iterates through all the identities in a provisioning profile, lazily."""
  for identity in mpf["DeveloperCertificates"]:
    yield _certificate_fingerprint(identity.data)

def _find_codesign_identities(identity=None):
  """Finds code signing identities on the current system."""
  ids = []
  output, stderr = _check_output([
      "security",
      "find-identity",
      "-v",
      "-p",
      "codesigning",
  ])
  output = output.decode("utf-8").strip()
  pattern = "(?P<hash>[A-F0-9]{40})"
  if identity:
    pattern += r'\s+"(?P<full_name>.*?{}.*?)"'.format(re.escape(identity))
  regex = re.compile(pattern)
  for line in output.splitlines():
    # CSSMERR_TP_CERT_REVOKED comes from Security.framework/cssmerr.h
    if "CSSMERR_TP_CERT_REVOKED" in line:
      continue
    m = regex.search(line)
    if m:
      groups = m.groupdict()
      id = groups.get("full_name") or groups["hash"]
      ids.append(id)
  return ids

def _find_codesign_identity(mobileprovision):
  """Finds a valid identity on the system given a mobileprovision file."""
  mpf = _parse_mobileprovision_file(mobileprovision)
  ids_codesign = set(_find_codesign_identities())
  for id_mpf in _get_identities_from_provisioning_profile(mpf):
    if id_mpf in ids_codesign:
      return id_mpf

def _filter_codesign_output(codesign_output):
  """Filters the codesign output which can be extra verbose."""
  filtered_lines = []
  for line in codesign_output.splitlines():
    if line and not _BENIGN_CODESIGN_OUTPUT_REGEX.search(line):
      filtered_lines.append(line)
  return "\n".join(filtered_lines)

def main(argv):
  parser = argparse.ArgumentParser(description="codesign wrapper")
  parser.add_argument(
      "--mobileprovision", type=str, help="mobileprovision file")
  parser.add_argument(
      "--codesign", required=True, type=str, help="path to codesign binary")
  parser.add_argument(
      "--identity", type=str, help="specific identity to sign with")
  args, codesign_args = parser.parse_known_args()
  identity = args.identity
  if identity is None:
    identity = _find_codesign_identity(args.mobileprovision)
  elif identity != "-":
    matching_identities = _find_codesign_identities(identity)
    if matching_identities:
      identity = matching_identities[0]
    else:
      print(
          "ERROR: No signing identity found for '{}'".format(identity),
          file=sys.stderr)
      return -1
  # No identity was found, fail
  if identity == None:
    print("ERROR: Unable to find an identity on the system matching the "\
        "ones in %s" % args.mobileprovision, file=sys.stderr)
    return 1
  stdout, stderr = _check_output([args.codesign, "-v", "--sign", identity] +
                                 codesign_args,)
  if stdout:
    filtered_stdout = _filter_codesign_output(stdout)
    if filtered_stdout:
      print(filtered_stdout)
  if stderr:
    filtered_stderr = _filter_codesign_output(stderr)
    if filtered_stderr:
      print(filtered_stderr)

if __name__ == '__main__':
  sys.exit(main(sys.argv))
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import (division as _, print_function as _,
                absolute_import as _, unicode_literals as _)

# Version number
__version__ = "0.1"

# Was coronagraph imported from setup.py?
try:
    __CORONAGRAPH_SETUP__
except NameError:
    __CORONAGRAPH_SETUP__ = False

if not __CORONAGRAPH_SETUP__:
    # This is a regular coronagraph run
    from .teleplanstar import *
    from . import observe
    from .observe import *
    from .degrade_spec import *
    from . import imager
    from .convolve_spec import *
    from .count_rates import *
    from .count_rates_wrapper import *
    from . import noise_routines
    from .transits import *
<commit_msg>bumped version to 1.0 following completion of JOSS review<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import (division as _, print_function as _,
                absolute_import as _, unicode_literals as _)

# Version number
__version__ = "1.0"

# Was coronagraph imported from setup.py?
try:
    __CORONAGRAPH_SETUP__
except NameError:
    __CORONAGRAPH_SETUP__ = False

if not __CORONAGRAPH_SETUP__:
    # This is a regular coronagraph run
    from .teleplanstar import *
    from . import observe
    from .observe import *
    from .degrade_spec import *
    from . import imager
    from .convolve_spec import *
    from .count_rates import *
    from .count_rates_wrapper import *
    from . import noise_routines
    from .transits import *
<|endoftext|>


<commit_before>import sys
sys.path.append('../../')
from lib.cilok import urlEncode16,tokenuri,setTTL,keyuri
from lib.sampeu import getWMTS
from apps.models import calendar
from apps.templates import batik

class Controller(object):
	def home(self,uridt='null'):
		provinsi = 'malut'
		provloc = '127.867580, 0.641499'
		mapzoom = '9'
		kabkotcord = [
		'127.60705673,1.39740732',
		'128.35871739999993,0.4419543',
		# '125.9128434, -2.1216504',
		# '127.7237678, -1.5109015',
		# '127.8936663, 1.5074308',
		# '128.4849923, 1.3121235',
		# '128.4008357, 2.3656672',
		# '124.7740793, -1.8268344',
		'127.3613533, 0.7957999',#71
		'127.681228, 0.5060207'
		]
		listkabkot = [
		'%8201%','%8202%','%8203%','%8204%','%8205%','%8206%','%8207%','%8208%',
		'%8271%','%8272%'
		]
		batik.provinsi(provinsi,listkabkot,provloc,mapzoom,kabkotcord)
		cal = calendar.Calendar()
		dt = {}
		for kabkot in listkabkot:
			dt[kabkot]=cal.getYearCountKabKot(str(int(kabkot[1:3])),str(int(kabkot[3:5])),uridt)
		cal.close()
		dt['%WMTS%']=getWMTS()
		dt['%PERIODE%']=uridt
		dt['%LAMAN INDONESIA%']=urlEncode16(keyuri+'%peta%home'+'%'+uridt)
		dt['%TAHUN SEBELUMNYA%']=urlEncode16(keyuri+'%'+provinsi+'%home'+'%'+str(int(uridt)-1))
		dt['%TAHUN SETELAHNYA%']=urlEncode16(keyuri+'%'+provinsi+'%home'+'%'+str(int(uridt)+1))
		return dt<commit_msg>tambah pulau taliabu<commit_after>import sys
sys.path.append('../../')
from lib.cilok import urlEncode16,tokenuri,setTTL,keyuri
from lib.sampeu import getWMTS
from apps.models import calendar
from apps.templates import batik

class Controller(object):
	def home(self,uridt='null'):
		provinsi = 'malut'
		provloc = '127.867580, 0.641499'
		mapzoom = '9'
		kabkotcord = [
		'127.60705673,1.39740732',
		'128.35871739999993,0.4419543',
		# '125.9128434, -2.1216504',
		# '127.7237678, -1.5109015',
		# '127.8936663, 1.5074308',
		# '128.4849923, 1.3121235',
		# '128.4008357, 2.3656672',
		'124.7740793, -1.8268344',
		'127.3613533, 0.7957999',#71
		'127.681228, 0.5060207'
		]
		listkabkot = [
		'%8201%','%8202%','%8203%','%8204%','%8205%','%8206%','%8207%','%8208%',
		'%8271%','%8272%'
		]
		batik.provinsi(provinsi,listkabkot,provloc,mapzoom,kabkotcord)
		cal = calendar.Calendar()
		dt = {}
		for kabkot in listkabkot:
			dt[kabkot]=cal.getYearCountKabKot(str(int(kabkot[1:3])),str(int(kabkot[3:5])),uridt)
		cal.close()
		dt['%WMTS%']=getWMTS()
		dt['%PERIODE%']=uridt
		dt['%LAMAN INDONESIA%']=urlEncode16(keyuri+'%peta%home'+'%'+uridt)
		dt['%TAHUN SEBELUMNYA%']=urlEncode16(keyuri+'%'+provinsi+'%home'+'%'+str(int(uridt)-1))
		dt['%TAHUN SETELAHNYA%']=urlEncode16(keyuri+'%'+provinsi+'%home'+'%'+str(int(uridt)+1))
		return dt<|endoftext|>