<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
"""
FreeSurfer tools interfaces
~~~~~~~~~~~~~~~~~~~~~~~~~~~

Fetch some example data:

    >>> import os
    >>> from niworkflows import data
    >>> data_root = data.get_bids_examples(variant='BIDS-examples-1-enh-ds054')
    >>> os.chdir(data_root)

Disable warnings:

    >>> import niworkflows.nipype as nn
    >>> nn.logging.getLogger('interface').setLevel('ERROR')

"""

import os.path as op
import nibabel as nb
import numpy as np

from skimage import morphology as sim
from scipy.ndimage.morphology import binary_fill_holes
from nilearn.image import resample_to_img, new_img_like

from niworkflows.nipype.utils.filemanip import copyfile, filename_to_list, fname_presuffix
from niworkflows.nipype.interfaces.base import (
    isdefined, InputMultiPath, BaseInterfaceInputSpec, TraitedSpec, File, traits, Directory
)
from niworkflows.nipype.interfaces import freesurfer as fs
from niworkflows.nipype.interfaces.base import SimpleInterface
from niworkflows.nipype.interfaces.freesurfer.preprocess import ConcatenateLTA

class StructuralReference(fs.RobustTemplate):
    """ Variation on RobustTemplate that simply copies the source if a single
    volume is provided.

    >>> from fmriprep.utils.bids import collect_data
    >>> t1w = collect_data('ds114', '01')[0]['t1w']
    >>> template = StructuralReference()
    >>> template.inputs.in_files = t1w
    >>> template.inputs.auto_detect_sensitivity = True
    >>> template.cmdline  # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
    'mri_robust_template --satit --mov .../sub-01_ses-retest_T1w.nii.gz
        .../sub-01_ses-test_T1w.nii.gz --template mri_robust_template_out.mgz'

    """

    def _num_vols(self):
        n_files = len(self.inputs.in_files)
        if n_files != 1:
            return n_files

        img = nb.load(self.inputs.in_files[0])
        if len(img.shape) == 3:
            return 1

        return img.shape[3]

    @property
    def cmdline(self):
        if self._num_vols() == 1:
            return "echo Only one time point!"
        return super(StructuralReference, self).cmdline

    def _list_outputs(self):
        outputs = super(StructuralReference, self)._list_outputs()
        if self._num_vols() == 1:
            in_file = self.inputs.in_files[0]
            outputs['out_file'] = in_file
            if isdefined(outputs['transform_outputs']):
                transform_file = outputs['transform_outputs'][0]
                fs.utils.LTAConvert(in_lta='identity.nofile', source_file=in_file,
                                    target_file=in_file, out_lta=transform_file).run()
        return outputs

class MakeMidthicknessInputSpec(fs.utils.MRIsExpandInputSpec):
    graymid = InputMultiPath(desc='Existing graymid/midthickness file')

class MakeMidthickness(fs.MRIsExpand):
    """ Variation on MRIsExpand that checks for an existing midthickness/graymid
    surface, and copies if available.

    mris_expand is an expensive operation, so this avoids re-running it when the
    working directory is lost.
    If users provide their own midthickness/graymid file, we assume they have
    created it correctly.
    """
    input_spec = MakeMidthicknessInputSpec

    @property
    def cmdline(self):
        cmd = super(MakeMidthickness, self).cmdline
        if not isdefined(self.inputs.graymid) or len(self.inputs.graymid) < 1:
            return cmd

        # Possible graymid values inclue {l,r}h.{graymid,midthickness}
        # Prefer midthickness to graymid, require to be of the same hemisphere
        # as input
        source = None
        in_base = op.basename(self.inputs.in_file)
        mt = self._associated_file(in_base, 'midthickness')
        gm = self._associated_file(in_base, 'graymid')

        for surf in self.inputs.graymid:
            if op.basename(surf) == mt:
                source = surf
                break
            if op.basename(surf) == gm:
                source = surf

        if source is None:
            return cmd

        return "cp {} {}".format(source, self._list_outputs()['out_file'])

class FSInjectBrainExtractedInputSpec(BaseInterfaceInputSpec):
    subjects_dir = Directory(mandatory=True, desc='FreeSurfer SUBJECTS_DIR')
    subject_id = traits.Str(mandatory=True, desc='Subject ID')
    in_brain = File(mandatory=True, exists=True, desc='input file, part of a BIDS tree')

class FSInjectBrainExtractedOutputSpec(TraitedSpec):
    subjects_dir = Directory(desc='FreeSurfer SUBJECTS_DIR')
    subject_id = traits.Str(desc='Subject ID')

class FSInjectBrainExtracted(SimpleInterface):
    input_spec = FSInjectBrainExtractedInputSpec
    output_spec = FSInjectBrainExtractedOutputSpec

    def _run_interface(self, runtime):
        subjects_dir, subject_id = inject_skullstripped(
            self.inputs.subjects_dir,
            self.inputs.subject_id,
            self.inputs.in_brain)
        self._results['subjects_dir'] = subjects_dir
        self._results['subject_id'] = subject_id
        return runtime

class FSDetectInputsInputSpec(BaseInterfaceInputSpec):
    t1w_list = InputMultiPath(File(exists=True), mandatory=True,
                              desc='input file, part of a BIDS tree')
    t2w_list = InputMultiPath(File(exists=True), desc='input file, part of a BIDS tree')
    hires_enabled = traits.Bool(True, usedefault=True, desc='enable hi-resolution processing')

class FSDetectInputsOutputSpec(TraitedSpec):
    t2w = File(desc='reference T2w image')
    use_t2w = traits.Bool(desc='enable use of T2w downstream computation')
    hires = traits.Bool(desc='enable hi-res processing')
    mris_inflate = traits.Str(desc='mris_inflate argument')

class FSDetectInputs(SimpleInterface):
    input_spec = FSDetectInputsInputSpec
    output_spec = FSDetectInputsOutputSpec

    def _run_interface(self, runtime):
        t2w, self._results['hires'], mris_inflate = detect_inputs(
            self.inputs.t1w_list,
            hires_enabled=self.inputs.hires_enabled,
            t2w_list=self.inputs.t2w_list if isdefined(self.inputs.t2w_list) else None)

        self._results['use_t2w'] = t2w is not None
        if self._results['use_t2w']:
            self._results['t2w'] = t2w

        if self._results['hires']:
            self._results['mris_inflate'] = mris_inflate

        return runtime

class PatchedConcatenateLTA(ConcatenateLTA):
    """
    A temporarily patched version of ``fs.ConcatenateLTA`` to recover from
    `this bug <https://www.mail-archive.com/freesurfer@nmr.mgh.harvard.edu/msg55520.html>`_
    in FreeSurfer, that was
    `fixed here <https://github.com/freesurfer/freesurfer/pull/180>`_.

    The original FMRIPREP's issue is found
    `here <https://github.com/poldracklab/fmriprep/issues/768>`_.
    """

    def _list_outputs(self):
        outputs = super(ConcatenateLTA, self)._list_outputs()

        with open(outputs['out_file'], 'r') as f:
            lines = f.readlines()

        fixed = False
        newfile = []
        for line in lines:
            if line.startswith('filename = ') and len(line.strip("\n")) >= 255:
                fixed = True
                newfile.append('filename = path_too_long\n')
            else:
                newfile.append(line)

        if fixed:
            with open(outputs['out_file'], 'w') as f:
                f.write(''.join(newfile))
        return outputs

class RefineBrainMaskInputSpec(BaseInterfaceInputSpec):
    in_anat = File(exists=True, mandatory=True,
                   desc='input anatomical reference (INU corrected)')
    in_aseg = File(exists=True, mandatory=True,
                   desc='input ``aseg`` file, in NifTi format.')
    in_ants = File(exists=True, mandatory=True,
                   desc='brain tissue segmentation generated with antsBrainExtraction.sh')

class RefineBrainMaskOutputSpec(TraitedSpec):
    out_file = File(exists=True, desc='new mask')

class RefineBrainMask(SimpleInterface):
    input_spec = RefineBrainMaskInputSpec
    output_spec = RefineBrainMaskOutputSpec

    def _run_interface(self, runtime):

        self._results['out_file'] = fname_presuffix(
            self.inputs.in_anat, suffix='_rbrainmask', newpath=runtime.cwd)

        anatnii = nb.load(self.inputs.in_anat)
        msknii = nb.Nifti1Image(
            grow_mask(anatnii.get_data(),
                      nb.load(self.inputs.in_aseg).get_data(),
                      nb.load(self.inputs.in_ants).get_data(),
            ),
            anatnii.affine,
            anatnii.header
        )
        msknii.set_data_dtype(np.uint8)
        msknii.to_filename(self._results['out_file'])

        return runtime

def inject_skullstripped(subjects_dir, subject_id, skullstripped):
    mridir = op.join(subjects_dir, subject_id, 'mri')
    t1 = op.join(mridir, 'T1.mgz')
    bm_auto = op.join(mridir, 'brainmask.auto.mgz')
    bm = op.join(mridir, 'brainmask.mgz')

    if not op.exists(bm_auto):
        img = nb.load(t1)
        mask = nb.load(skullstripped)
        bmask = new_img_like(mask, mask.get_data() > 0)
        resampled_mask = resample_to_img(bmask, img, 'nearest')
        masked_image = new_img_like(img, img.get_data() * resampled_mask.get_data())
        masked_image.to_filename(bm_auto)

    if not op.exists(bm):
        copyfile(bm_auto, bm, copy=True, use_hardlink=True)

    return subjects_dir, subject_id

def detect_inputs(t1w_list, t2w_list=None, hires_enabled=True):
    t1w_list = filename_to_list(t1w_list)
    t2w_list = filename_to_list(t2w_list) if t2w_list is not None else []
    t1w_ref = nb.load(t1w_list[0])
    # Use high resolution preprocessing if voxel size < 1.0mm
    # Tolerance of 0.05mm requires that rounds down to 0.9mm or lower
    hires = hires_enabled and max(t1w_ref.header.get_zooms()) < 1 - 0.05

    t2w = None
    if t2w_list and max(nb.load(t2w_list[0]).header.get_zooms()) < 1.2:
        t2w = t2w_list[0]

    # https://surfer.nmr.mgh.harvard.edu/fswiki/SubmillimeterRecon
    mris_inflate = '-n 50' if hires else None
    return (t2w, hires, mris_inflate)

def refine_aseg(aseg, ball_size=4):
    """
    First step to reconcile ANTs' and FreeSurfer's brain masks.

    Here, the ``aseg.mgz`` mask from FreeSurfer is refined in two
    steps, using binary morphological operations:

      1. With a binary closing operation the sulci are included
         into the mask. This results in a smoother brain mask
         that does not exclude deep, wide sulci.

      2. Fill any holes (typically, there could be a hole next to
         the pineal gland and the corpora quadrigemina if the great
         cerebral brain is segmented out).

    """
    # Read aseg data
    bmask = aseg.copy()
    bmask[bmask > 0] = 1
    bmask = bmask.astype(np.uint8)

    # Morphological operations
    selem = sim.ball(ball_size)
    newmask = sim.binary_closing(bmask, selem)
    newmask = binary_fill_holes(newmask.astype(np.uint8), selem).astype(np.uint8)

    return newmask.astype(np.uint8)

def grow_mask(anat, aseg, ants_segs, ww=7, zval=2.0, bw=4):
    """
    Grow mask including pixels that have a high likelihood.
    GM tissue parameters are sampled in image patches of ``ww`` size.

    This is inspired on mindboggle's solution to the problem:
    https://github.com/nipy/mindboggle/blob/master/mindboggle/guts/segment.py#L1660

    """
    selem = sim.ball(bw)

    aseg[aseg == 42] = 3  # Collapse both hemispheres
    gm = anat.copy()
    gm[aseg != 3] = 0

    refined = refine_aseg(aseg)
    newrefmask = sim.binary_dilation(refined, selem) - refined
    indices = np.argwhere(newrefmask > 0)
    for pixel in indices:
        # When ATROPOS identified the pixel as GM, set and carry on
        if ants_segs[tuple(pixel)] == 3:
            refined[tuple(pixel)] = 1
            continue

        window = gm[
            pixel[0] - ww:pixel[0] + ww,
            pixel[1] - ww:pixel[1] + ww,
            pixel[2] - ww:pixel[2] + ww
        ]
        if np.any(window > 0):
            zstat = abs(anat[tuple(pixel)] - window[window > 0].mean()) / window[window > 0].std()
            refined[tuple(pixel)] = int(zstat < zval)

    refined = sim.binary_opening(refined, selem)
    return refined
<commit_msg>pacify flake8<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
"""
FreeSurfer tools interfaces
~~~~~~~~~~~~~~~~~~~~~~~~~~~

Fetch some example data:

    >>> import os
    >>> from niworkflows import data
    >>> data_root = data.get_bids_examples(variant='BIDS-examples-1-enh-ds054')
    >>> os.chdir(data_root)

Disable warnings:

    >>> import niworkflows.nipype as nn
    >>> nn.logging.getLogger('interface').setLevel('ERROR')

"""

import os.path as op
import nibabel as nb
import numpy as np

from skimage import morphology as sim
from scipy.ndimage.morphology import binary_fill_holes
from nilearn.image import resample_to_img, new_img_like

from niworkflows.nipype.utils.filemanip import copyfile, filename_to_list, fname_presuffix
from niworkflows.nipype.interfaces.base import (
    isdefined, InputMultiPath, BaseInterfaceInputSpec, TraitedSpec, File, traits, Directory
)
from niworkflows.nipype.interfaces import freesurfer as fs
from niworkflows.nipype.interfaces.base import SimpleInterface
from niworkflows.nipype.interfaces.freesurfer.preprocess import ConcatenateLTA

class StructuralReference(fs.RobustTemplate):
    """ Variation on RobustTemplate that simply copies the source if a single
    volume is provided.

    >>> from fmriprep.utils.bids import collect_data
    >>> t1w = collect_data('ds114', '01')[0]['t1w']
    >>> template = StructuralReference()
    >>> template.inputs.in_files = t1w
    >>> template.inputs.auto_detect_sensitivity = True
    >>> template.cmdline  # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
    'mri_robust_template --satit --mov .../sub-01_ses-retest_T1w.nii.gz
        .../sub-01_ses-test_T1w.nii.gz --template mri_robust_template_out.mgz'

    """

    def _num_vols(self):
        n_files = len(self.inputs.in_files)
        if n_files != 1:
            return n_files

        img = nb.load(self.inputs.in_files[0])
        if len(img.shape) == 3:
            return 1

        return img.shape[3]

    @property
    def cmdline(self):
        if self._num_vols() == 1:
            return "echo Only one time point!"
        return super(StructuralReference, self).cmdline

    def _list_outputs(self):
        outputs = super(StructuralReference, self)._list_outputs()
        if self._num_vols() == 1:
            in_file = self.inputs.in_files[0]
            outputs['out_file'] = in_file
            if isdefined(outputs['transform_outputs']):
                transform_file = outputs['transform_outputs'][0]
                fs.utils.LTAConvert(in_lta='identity.nofile', source_file=in_file,
                                    target_file=in_file, out_lta=transform_file).run()
        return outputs

class MakeMidthicknessInputSpec(fs.utils.MRIsExpandInputSpec):
    graymid = InputMultiPath(desc='Existing graymid/midthickness file')

class MakeMidthickness(fs.MRIsExpand):
    """ Variation on MRIsExpand that checks for an existing midthickness/graymid
    surface, and copies if available.

    mris_expand is an expensive operation, so this avoids re-running it when the
    working directory is lost.
    If users provide their own midthickness/graymid file, we assume they have
    created it correctly.
    """
    input_spec = MakeMidthicknessInputSpec

    @property
    def cmdline(self):
        cmd = super(MakeMidthickness, self).cmdline
        if not isdefined(self.inputs.graymid) or len(self.inputs.graymid) < 1:
            return cmd

        # Possible graymid values inclue {l,r}h.{graymid,midthickness}
        # Prefer midthickness to graymid, require to be of the same hemisphere
        # as input
        source = None
        in_base = op.basename(self.inputs.in_file)
        mt = self._associated_file(in_base, 'midthickness')
        gm = self._associated_file(in_base, 'graymid')

        for surf in self.inputs.graymid:
            if op.basename(surf) == mt:
                source = surf
                break
            if op.basename(surf) == gm:
                source = surf

        if source is None:
            return cmd

        return "cp {} {}".format(source, self._list_outputs()['out_file'])

class FSInjectBrainExtractedInputSpec(BaseInterfaceInputSpec):
    subjects_dir = Directory(mandatory=True, desc='FreeSurfer SUBJECTS_DIR')
    subject_id = traits.Str(mandatory=True, desc='Subject ID')
    in_brain = File(mandatory=True, exists=True, desc='input file, part of a BIDS tree')

class FSInjectBrainExtractedOutputSpec(TraitedSpec):
    subjects_dir = Directory(desc='FreeSurfer SUBJECTS_DIR')
    subject_id = traits.Str(desc='Subject ID')

class FSInjectBrainExtracted(SimpleInterface):
    input_spec = FSInjectBrainExtractedInputSpec
    output_spec = FSInjectBrainExtractedOutputSpec

    def _run_interface(self, runtime):
        subjects_dir, subject_id = inject_skullstripped(
            self.inputs.subjects_dir,
            self.inputs.subject_id,
            self.inputs.in_brain)
        self._results['subjects_dir'] = subjects_dir
        self._results['subject_id'] = subject_id
        return runtime

class FSDetectInputsInputSpec(BaseInterfaceInputSpec):
    t1w_list = InputMultiPath(File(exists=True), mandatory=True,
                              desc='input file, part of a BIDS tree')
    t2w_list = InputMultiPath(File(exists=True), desc='input file, part of a BIDS tree')
    hires_enabled = traits.Bool(True, usedefault=True, desc='enable hi-resolution processing')

class FSDetectInputsOutputSpec(TraitedSpec):
    t2w = File(desc='reference T2w image')
    use_t2w = traits.Bool(desc='enable use of T2w downstream computation')
    hires = traits.Bool(desc='enable hi-res processing')
    mris_inflate = traits.Str(desc='mris_inflate argument')

class FSDetectInputs(SimpleInterface):
    input_spec = FSDetectInputsInputSpec
    output_spec = FSDetectInputsOutputSpec

    def _run_interface(self, runtime):
        t2w, self._results['hires'], mris_inflate = detect_inputs(
            self.inputs.t1w_list,
            hires_enabled=self.inputs.hires_enabled,
            t2w_list=self.inputs.t2w_list if isdefined(self.inputs.t2w_list) else None)

        self._results['use_t2w'] = t2w is not None
        if self._results['use_t2w']:
            self._results['t2w'] = t2w

        if self._results['hires']:
            self._results['mris_inflate'] = mris_inflate

        return runtime

class PatchedConcatenateLTA(ConcatenateLTA):
    """
    A temporarily patched version of ``fs.ConcatenateLTA`` to recover from
    `this bug <https://www.mail-archive.com/freesurfer@nmr.mgh.harvard.edu/msg55520.html>`_
    in FreeSurfer, that was
    `fixed here <https://github.com/freesurfer/freesurfer/pull/180>`_.

    The original FMRIPREP's issue is found
    `here <https://github.com/poldracklab/fmriprep/issues/768>`_.
    """

    def _list_outputs(self):
        outputs = super(ConcatenateLTA, self)._list_outputs()

        with open(outputs['out_file'], 'r') as f:
            lines = f.readlines()

        fixed = False
        newfile = []
        for line in lines:
            if line.startswith('filename = ') and len(line.strip("\n")) >= 255:
                fixed = True
                newfile.append('filename = path_too_long\n')
            else:
                newfile.append(line)

        if fixed:
            with open(outputs['out_file'], 'w') as f:
                f.write(''.join(newfile))
        return outputs

class RefineBrainMaskInputSpec(BaseInterfaceInputSpec):
    in_anat = File(exists=True, mandatory=True,
                   desc='input anatomical reference (INU corrected)')
    in_aseg = File(exists=True, mandatory=True,
                   desc='input ``aseg`` file, in NifTi format.')
    in_ants = File(exists=True, mandatory=True,
                   desc='brain tissue segmentation generated with antsBrainExtraction.sh')

class RefineBrainMaskOutputSpec(TraitedSpec):
    out_file = File(exists=True, desc='new mask')

class RefineBrainMask(SimpleInterface):
    input_spec = RefineBrainMaskInputSpec
    output_spec = RefineBrainMaskOutputSpec

    def _run_interface(self, runtime):

        self._results['out_file'] = fname_presuffix(
            self.inputs.in_anat, suffix='_rbrainmask', newpath=runtime.cwd)

        anatnii = nb.load(self.inputs.in_anat)
        msknii = nb.Nifti1Image(
            grow_mask(anatnii.get_data(),
                      nb.load(self.inputs.in_aseg).get_data(),
                      nb.load(self.inputs.in_ants).get_data()),
            anatnii.affine,
            anatnii.header
        )
        msknii.set_data_dtype(np.uint8)
        msknii.to_filename(self._results['out_file'])

        return runtime

def inject_skullstripped(subjects_dir, subject_id, skullstripped):
    mridir = op.join(subjects_dir, subject_id, 'mri')
    t1 = op.join(mridir, 'T1.mgz')
    bm_auto = op.join(mridir, 'brainmask.auto.mgz')
    bm = op.join(mridir, 'brainmask.mgz')

    if not op.exists(bm_auto):
        img = nb.load(t1)
        mask = nb.load(skullstripped)
        bmask = new_img_like(mask, mask.get_data() > 0)
        resampled_mask = resample_to_img(bmask, img, 'nearest')
        masked_image = new_img_like(img, img.get_data() * resampled_mask.get_data())
        masked_image.to_filename(bm_auto)

    if not op.exists(bm):
        copyfile(bm_auto, bm, copy=True, use_hardlink=True)

    return subjects_dir, subject_id

def detect_inputs(t1w_list, t2w_list=None, hires_enabled=True):
    t1w_list = filename_to_list(t1w_list)
    t2w_list = filename_to_list(t2w_list) if t2w_list is not None else []
    t1w_ref = nb.load(t1w_list[0])
    # Use high resolution preprocessing if voxel size < 1.0mm
    # Tolerance of 0.05mm requires that rounds down to 0.9mm or lower
    hires = hires_enabled and max(t1w_ref.header.get_zooms()) < 1 - 0.05

    t2w = None
    if t2w_list and max(nb.load(t2w_list[0]).header.get_zooms()) < 1.2:
        t2w = t2w_list[0]

    # https://surfer.nmr.mgh.harvard.edu/fswiki/SubmillimeterRecon
    mris_inflate = '-n 50' if hires else None
    return (t2w, hires, mris_inflate)

def refine_aseg(aseg, ball_size=4):
    """
    First step to reconcile ANTs' and FreeSurfer's brain masks.

    Here, the ``aseg.mgz`` mask from FreeSurfer is refined in two
    steps, using binary morphological operations:

      1. With a binary closing operation the sulci are included
         into the mask. This results in a smoother brain mask
         that does not exclude deep, wide sulci.

      2. Fill any holes (typically, there could be a hole next to
         the pineal gland and the corpora quadrigemina if the great
         cerebral brain is segmented out).

    """
    # Read aseg data
    bmask = aseg.copy()
    bmask[bmask > 0] = 1
    bmask = bmask.astype(np.uint8)

    # Morphological operations
    selem = sim.ball(ball_size)
    newmask = sim.binary_closing(bmask, selem)
    newmask = binary_fill_holes(newmask.astype(np.uint8), selem).astype(np.uint8)

    return newmask.astype(np.uint8)

def grow_mask(anat, aseg, ants_segs, ww=7, zval=2.0, bw=4):
    """
    Grow mask including pixels that have a high likelihood.
    GM tissue parameters are sampled in image patches of ``ww`` size.

    This is inspired on mindboggle's solution to the problem:
    https://github.com/nipy/mindboggle/blob/master/mindboggle/guts/segment.py#L1660

    """
    selem = sim.ball(bw)

    aseg[aseg == 42] = 3  # Collapse both hemispheres
    gm = anat.copy()
    gm[aseg != 3] = 0

    refined = refine_aseg(aseg)
    newrefmask = sim.binary_dilation(refined, selem) - refined
    indices = np.argwhere(newrefmask > 0)
    for pixel in indices:
        # When ATROPOS identified the pixel as GM, set and carry on
        if ants_segs[tuple(pixel)] == 3:
            refined[tuple(pixel)] = 1
            continue

        window = gm[
            pixel[0] - ww:pixel[0] + ww,
            pixel[1] - ww:pixel[1] + ww,
            pixel[2] - ww:pixel[2] + ww
        ]
        if np.any(window > 0):
            zstat = abs(anat[tuple(pixel)] - window[window > 0].mean()) / window[window > 0].std()
            refined[tuple(pixel)] = int(zstat < zval)

    refined = sim.binary_opening(refined, selem)
    return refined
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
"""
FreeSurfer tools interfaces
~~~~~~~~~~~~~~~~~~~~~~~~~~~

"""

import os.path as op
import nibabel as nb

from nilearn.image import resample_to_img, new_img_like

from niworkflows.nipype.utils.filemanip import copyfile, filename_to_list
from niworkflows.nipype.interfaces.base import (
    isdefined, InputMultiPath, BaseInterfaceInputSpec, TraitedSpec, File, traits, Directory
)
from niworkflows.nipype.interfaces import freesurfer as fs

from niworkflows.interfaces.base import SimpleInterface

class StructuralReference(fs.RobustTemplate):
    """ Variation on RobustTemplate that simply copies the source if a single
    volume is provided. """
    @property
    def cmdline(self):
        cmd = super(StructuralReference, self).cmdline
        if len(self.inputs.in_files) > 1:
            return cmd

        img = nb.load(self.inputs.in_files[0])
        if len(img.shape) > 3 and img.shape[3] > 1:
            return cmd

        out_file = self._list_outputs()['out_file']
        copyfile(self.inputs.in_files[0], out_file)
        return "echo Only one time point!"

class MakeMidthicknessInputSpec(fs.utils.MRIsExpandInputSpec):
    graymid = InputMultiPath(desc='Existing graymid/midthickness file')

class MakeMidthickness(fs.MRIsExpand):
    """ Variation on RobustTemplate that simply copies the source if a single
    volume is provided. """
    input_spec = MakeMidthicknessInputSpec

    @property
    def cmdline(self):
        cmd = super(MakeMidthickness, self).cmdline
        if not isdefined(self.inputs.graymid) or len(self.inputs.graymid) < 1:
            return cmd

        # Possible graymid values inclue {l,r}h.{graymid,midthickness}
        # Prefer midthickness to graymid, require to be of the same hemisphere
        # as input
        source = None
        in_base = op.basename(self.inputs.in_file)
        mt = self._associated_file(in_base, 'midthickness')
        gm = self._associated_file(in_base, 'graymid')

        for surf in self.inputs.graymid:
            if op.basename(surf) == mt:
                source = surf
                break
            if op.basename(surf) == gm:
                source = surf

        if source is None:
            return cmd

        return "cp {} {}".format(source, self._list_outputs()['out_file'])

class FSInjectBrainExtractedInputSpec(BaseInterfaceInputSpec):
    subjects_dir = Directory(mandatory=True, desc='FreeSurfer SUBJECTS_DIR')
    subject_id = traits.Str(mandatory=True, desc='Subject ID')
    in_brain = File(mandatory=True, exists=True, desc='input file, part of a BIDS tree')

class FSInjectBrainExtractedOutputSpec(TraitedSpec):
    subjects_dir = Directory(desc='FreeSurfer SUBJECTS_DIR')
    subject_id = traits.Str(desc='Subject ID')

class FSInjectBrainExtracted(SimpleInterface):
    input_spec = FSInjectBrainExtractedInputSpec
    output_spec = FSInjectBrainExtractedOutputSpec

    def _run_interface(self, runtime):
        subjects_dir, subject_id = inject_skullstripped(
            self.inputs.subjects_dir,
            self.inputs.subject_id,
            self.inputs.in_brain)
        self._results['subjects_dir'] = subjects_dir
        self._results['subject_id'] = subject_id
        return runtime

class FSDetectInputsInputSpec(BaseInterfaceInputSpec):
    t1w_list = InputMultiPath(File(exists=True), mandatory=True,
                              desc='input file, part of a BIDS tree')
    t2w_list = InputMultiPath(File(exists=True), desc='input file, part of a BIDS tree')
    hires_enabled = traits.Bool(True, usedefault=True, desc='enable hi-resolution processing')

class FSDetectInputsOutputSpec(TraitedSpec):
    t2w = File(desc='reference T2w image')
    use_t2w = traits.Bool(desc='enable use of T2w downstream computation')
    hires = traits.Bool(desc='enable hi-res processing')
    mris_inflate = traits.Str(desc='mris_inflate argument')

class FSDetectInputs(SimpleInterface):
    input_spec = FSDetectInputsInputSpec
    output_spec = FSDetectInputsOutputSpec

    def _run_interface(self, runtime):
        t2w, self._results['hires'], mris_inflate = detect_inputs(
            self.inputs.t1w_list,
            hires_enabled=self.inputs.hires_enabled,
            t2w_list=self.inputs.t2w_list if isdefined(self.inputs.t2w_list) else None)

        self._results['use_t2w'] = t2w is not None
        if self._results['use_t2w']:
            self._results['t2w'] = t2w

        if self._results['hires']:
            self._results['mris_inflate'] = mris_inflate

        return runtime

def inject_skullstripped(subjects_dir, subject_id, skullstripped):
    mridir = op.join(subjects_dir, subject_id, 'mri')
    t1 = op.join(mridir, 'T1.mgz')
    bm_auto = op.join(mridir, 'brainmask.auto.mgz')
    bm = op.join(mridir, 'brainmask.mgz')

    if not op.exists(bm_auto):
        img = nb.load(t1)
        mask = nb.load(skullstripped)
        bmask = new_img_like(mask, mask.get_data() > 0)
        resampled_mask = resample_to_img(bmask, img, 'nearest')
        masked_image = new_img_like(img, img.get_data() * resampled_mask.get_data())
        masked_image.to_filename(bm_auto)

    if not op.exists(bm):
        copyfile(bm_auto, bm, copy=True, use_hardlink=True)

    return subjects_dir, subject_id

def detect_inputs(t1w_list, t2w_list=None, hires_enabled=True):
    t1w_list = filename_to_list(t1w_list)
    t2w_list = filename_to_list(t2w_list) if t2w_list is not None else []
    t1w_ref = nb.load(t1w_list[0])
    # Use high resolution preprocessing if voxel size < 1.0mm
    # Tolerance of 0.05mm requires that rounds down to 0.9mm or lower
    hires = hires_enabled and max(t1w_ref.header.get_zooms()) < 1 - 0.05

    t2w = None
    if t2w_list and max(nb.load(t2w_list[0]).header.get_zooms()) < 1.2:
        t2w = t2w_list[0]

    # https://surfer.nmr.mgh.harvard.edu/fswiki/SubmillimeterRecon
    mris_inflate = '-n 50' if hires else None
    return (t2w, hires, mris_inflate)
<commit_msg>DOCTEST: StructuralReference example<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
"""
FreeSurfer tools interfaces
~~~~~~~~~~~~~~~~~~~~~~~~~~~

Fetch some example data:

    >>> import os
    >>> from niworkflows import data
    >>> data_root = data.get_bids_examples(variant='BIDS-examples-1-enh-ds054')
    >>> os.chdir(data_root)

Disable warnings:

    >>> import niworkflows.nipype as nn
    >>> nn.logging.getLogger('interface').setLevel('ERROR')

"""

import os.path as op
import nibabel as nb

from nilearn.image import resample_to_img, new_img_like

from niworkflows.nipype.utils.filemanip import copyfile, filename_to_list
from niworkflows.nipype.interfaces.base import (
    isdefined, InputMultiPath, BaseInterfaceInputSpec, TraitedSpec, File, traits, Directory
)
from niworkflows.nipype.interfaces import freesurfer as fs

from niworkflows.interfaces.base import SimpleInterface

class StructuralReference(fs.RobustTemplate):
    """ Variation on RobustTemplate that simply copies the source if a single
    volume is provided.

    >>> from fmriprep.utils.bids import collect_data
    >>> t1w = collect_data('ds114', '01')[0]['t1w']
    >>> template = StructuralReference()
    >>> template.inputs.in_files = t1w
    >>> template.inputs.auto_detect_sensitivity = True
    >>> template.cmdline  # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
    'mri_robust_template --satit --mov .../sub-01_ses-retest_T1w.nii.gz
        .../sub-01_ses-test_T1w.nii.gz --template mri_robust_template_out.mgz'

    """
    @property
    def cmdline(self):
        cmd = super(StructuralReference, self).cmdline
        if len(self.inputs.in_files) > 1:
            return cmd

        img = nb.load(self.inputs.in_files[0])
        if len(img.shape) > 3 and img.shape[3] > 1:
            return cmd

        out_file = self._list_outputs()['out_file']
        copyfile(self.inputs.in_files[0], out_file)
        return "echo Only one time point!"

class MakeMidthicknessInputSpec(fs.utils.MRIsExpandInputSpec):
    graymid = InputMultiPath(desc='Existing graymid/midthickness file')

class MakeMidthickness(fs.MRIsExpand):
    """ Variation on RobustTemplate that simply copies the source if a single
    volume is provided. """
    input_spec = MakeMidthicknessInputSpec

    @property
    def cmdline(self):
        cmd = super(MakeMidthickness, self).cmdline
        if not isdefined(self.inputs.graymid) or len(self.inputs.graymid) < 1:
            return cmd

        # Possible graymid values inclue {l,r}h.{graymid,midthickness}
        # Prefer midthickness to graymid, require to be of the same hemisphere
        # as input
        source = None
        in_base = op.basename(self.inputs.in_file)
        mt = self._associated_file(in_base, 'midthickness')
        gm = self._associated_file(in_base, 'graymid')

        for surf in self.inputs.graymid:
            if op.basename(surf) == mt:
                source = surf
                break
            if op.basename(surf) == gm:
                source = surf

        if source is None:
            return cmd

        return "cp {} {}".format(source, self._list_outputs()['out_file'])

class FSInjectBrainExtractedInputSpec(BaseInterfaceInputSpec):
    subjects_dir = Directory(mandatory=True, desc='FreeSurfer SUBJECTS_DIR')
    subject_id = traits.Str(mandatory=True, desc='Subject ID')
    in_brain = File(mandatory=True, exists=True, desc='input file, part of a BIDS tree')

class FSInjectBrainExtractedOutputSpec(TraitedSpec):
    subjects_dir = Directory(desc='FreeSurfer SUBJECTS_DIR')
    subject_id = traits.Str(desc='Subject ID')

class FSInjectBrainExtracted(SimpleInterface):
    input_spec = FSInjectBrainExtractedInputSpec
    output_spec = FSInjectBrainExtractedOutputSpec

    def _run_interface(self, runtime):
        subjects_dir, subject_id = inject_skullstripped(
            self.inputs.subjects_dir,
            self.inputs.subject_id,
            self.inputs.in_brain)
        self._results['subjects_dir'] = subjects_dir
        self._results['subject_id'] = subject_id
        return runtime

class FSDetectInputsInputSpec(BaseInterfaceInputSpec):
    t1w_list = InputMultiPath(File(exists=True), mandatory=True,
                              desc='input file, part of a BIDS tree')
    t2w_list = InputMultiPath(File(exists=True), desc='input file, part of a BIDS tree')
    hires_enabled = traits.Bool(True, usedefault=True, desc='enable hi-resolution processing')

class FSDetectInputsOutputSpec(TraitedSpec):
    t2w = File(desc='reference T2w image')
    use_t2w = traits.Bool(desc='enable use of T2w downstream computation')
    hires = traits.Bool(desc='enable hi-res processing')
    mris_inflate = traits.Str(desc='mris_inflate argument')

class FSDetectInputs(SimpleInterface):
    input_spec = FSDetectInputsInputSpec
    output_spec = FSDetectInputsOutputSpec

    def _run_interface(self, runtime):
        t2w, self._results['hires'], mris_inflate = detect_inputs(
            self.inputs.t1w_list,
            hires_enabled=self.inputs.hires_enabled,
            t2w_list=self.inputs.t2w_list if isdefined(self.inputs.t2w_list) else None)

        self._results['use_t2w'] = t2w is not None
        if self._results['use_t2w']:
            self._results['t2w'] = t2w

        if self._results['hires']:
            self._results['mris_inflate'] = mris_inflate

        return runtime

def inject_skullstripped(subjects_dir, subject_id, skullstripped):
    mridir = op.join(subjects_dir, subject_id, 'mri')
    t1 = op.join(mridir, 'T1.mgz')
    bm_auto = op.join(mridir, 'brainmask.auto.mgz')
    bm = op.join(mridir, 'brainmask.mgz')

    if not op.exists(bm_auto):
        img = nb.load(t1)
        mask = nb.load(skullstripped)
        bmask = new_img_like(mask, mask.get_data() > 0)
        resampled_mask = resample_to_img(bmask, img, 'nearest')
        masked_image = new_img_like(img, img.get_data() * resampled_mask.get_data())
        masked_image.to_filename(bm_auto)

    if not op.exists(bm):
        copyfile(bm_auto, bm, copy=True, use_hardlink=True)

    return subjects_dir, subject_id

def detect_inputs(t1w_list, t2w_list=None, hires_enabled=True):
    t1w_list = filename_to_list(t1w_list)
    t2w_list = filename_to_list(t2w_list) if t2w_list is not None else []
    t1w_ref = nb.load(t1w_list[0])
    # Use high resolution preprocessing if voxel size < 1.0mm
    # Tolerance of 0.05mm requires that rounds down to 0.9mm or lower
    hires = hires_enabled and max(t1w_ref.header.get_zooms()) < 1 - 0.05

    t2w = None
    if t2w_list and max(nb.load(t2w_list[0]).header.get_zooms()) < 1.2:
        t2w = t2w_list[0]

    # https://surfer.nmr.mgh.harvard.edu/fswiki/SubmillimeterRecon
    mris_inflate = '-n 50' if hires else None
    return (t2w, hires, mris_inflate)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import gui
import human_player
import rules
import game

import time

from ai_player import *
from priority_filter import *
from priority_filter_2 import *
from blindness_filter import *
from evaluator import *

import pdb

import pstats, cProfile

class TwoTimer:
    def __init__(self):
        self.totals = [0.0, 0.0]
        self.current = 0

    def __enter__(self):
        self.start = time.clock()
        return self

    def __exit__(self, *args):
        self.end = time.clock()
        self.totals[self.current] += self.end - self.start
        self.current = 1 - self.current

    def __repr__(self):
        tot = self.totals
        return "B: %.2fs, W: %.2fs, B/W: %.2f" % (tot[0], tot[1], tot[0]/tot[1])

class Genome():
    """
    max_depth
    width (max_moves_per_depth_level)
    narrowing: 0, 1, .5, -0.5 (per depth level)
    line base
    capture_score_base
    take_score_base
    threat_score_base
    capture scaling
    line length scaling
    the value of the move
    max_time_per_move (proportion of remaining time)
    total game time
    search iterations - 0,1,2,3
    """
    def __init__(self):
        self.max_depth = 6
        self.mmpdl = 9
        self.narrowing = 0
        self.chokes = [(4,2)]
        self.filter2 = False

        self.capture_score_base = 300
        self.take_score_base = 100
        self.threat_score_base = 20
        self.captures_scale = [0, 1, 1, 1, 1, 1]
        self.length_factor = 27
        self.move_factor = 30
        self.blindness = 0
        self.sub = True

    def create_player(self, name):
        #pdb.set_trace()
        if self.filter2:
            sf = PriorityFilter2()
        else:
            sf = PriorityFilter()
        sf.set_max_moves_per_depth_level(mmpdl=self.mmpdl, narrowing=self.narrowing,
                chokes=self.chokes)
        if self.blindness > 0:
            bf = BlindnessFilter(sf)
            bf.set_blindness(self.blindness)
            sf = bf
        p = AIPlayer(sf, name=name)
        self.set_config(p)

        return p

    def set_config(self, player):
        uc = player.get_utility_calculator()
        uc.capture_score_base = self.capture_score_base
        uc.take_score_base = self.take_score_base
        uc.threat_score_base = self.threat_score_base
        uc.captures_scale = self.captures_scale
        uc.length_factor = self.length_factor
        uc.move_factor = self.move_factor
        uc.sub = self.sub

        self.max_depth_boost = 0
        # TODO: pf = player.get_priority_filter()

class MatchResults():
    def __init__(self):
        self.results = []
        self.bw_total = [0, 0, 0]
        self.dc_total = {"Defender":0, "Contender":0}
        self.total_ratio = 0.
        self.games_played = 0

    def __repr__(self):
        return "\n".join(self.results) + "\nB/W:" + str(self.bw_total) \
                + " won: " + str(self.dc_total) \
                + " C/D avg. time: " + str(self.total_ratio / self.games_played)

    def add(self, result):
        res_str, winner_colour, winner_name, ratio = result
        self.results.append(res_str)
        self.bw_total[winner_colour] += 1
        self.dc_total[str(winner_name)] += 1
        self.total_ratio += ratio
        self.games_played += 1

class Match():
    def __init__(self):
        #pdb.set_trace()
        self.genome1 = Genome()
        self.genome2 = Genome()

    def set_up(self, game_length):
        self.p1 = self.genome1.create_player("Defender")
        self.p2 = self.genome2.create_player("Contender")
        self.p1.set_max_depth(game_length + self.genome1.max_depth_boost)
        self.p2.set_max_depth(game_length + self.genome2.max_depth_boost)

    def play_one_game(self, board_size, p1, p2):
        r = rules.Rules(board_size, "standard")
        self.game = game.Game(r, p1, p2)
        #self.evaluator = Evaluator(UtilityCalculator(), self.game.current_state)

        tt = TwoTimer()

        while not self.game.finished():
            p = self.game.get_current_player()
            with tt:
                m = p.do_the_search()
                self.game.make_move(m)
                #print self.evaluator.utility()
        #pdb.set_trace()
        winner_name = self.game.winner_name()
        winner = self.game.winner()
        if p1.name == "Contender":
            ratio = tt.totals[0] / tt.totals[1]
        else:
            ratio = tt.totals[1] / tt.totals[0]

        print "Game was won by: %s %s" % (["?", "B", "W"][winner], winner_name)
        print tt

        return "%s vs. %s: %s (%sx%s %s) %s" % (p1.name, p2.name, winner_name,
                board_size, board_size, p1.max_depth, tt), winner, winner_name, ratio

    def play_some_games(self):

        #self.genome2.length_factor = 35
        #self.genome2.take_score_base = 110
        #self.genome2.capture_score_base = 310 # Try this again for high depth
        #self.genome2.threat_score_base = 25
        #self.genome1.blindness = 0.30
        #self.genome2.blindness = 0.40

        self.genome2.filter2 = True
        #self.genome2.narrowing = 3
        #self.genome2.max_depth += 2 # Setting max_depth here doesn't work
        #self.genome2.mmpdl = 15
        #self.genome1.chokes = []
        #self.genome2.chokes = [(4,5)]
        #self.genome2.chokes = [(4,3)]
        #self.genome2.chokes = [(2,2)]
        #self.genome1.max_depth_boost = 2
        #self.genome2.max_depth_boost = 2
        #self.genome2.captures_scale = [0, 1, 10, 100, 1000, 50]
        #self.genome2.captures_scale = [0, 0, 0, 0, 0, 0]
        #self.genome2.move_factor = 10000000

        #self.genome1.sub = False
        #self.genome2.sub = False
        #self.genome2.move_factor = 50
        #self.genome2.move_factor = 5

        results = MatchResults()
        for game_length in range(3, 4):
            for board_size in [9, 13, 19]:
            #for board_size in [9, 13, 16, 19]:
                for first_player in [0, 1]:
                    self.set_up(game_length)
                    players = [self.p1, self.p2]
                    second_player = 1 - first_player
                    res = self.play_one_game(board_size,
                                             players[first_player],
                                             players[second_player])
                    results.add(res)

        print results

import random

if __name__ == "__main__":
    random.seed()
    m = Match()
    '''
    m.play_some_games()
    '''
    cProfile.runctx("m.play_some_games()", globals(), locals(), "Profile.prof")

    s = pstats.Stats("Profile.prof")
    #s.strip_dirs().sort_stats("cumulative").print_stats(20) # or "time"
    s.strip_dirs().sort_stats("time").print_stats(20)
<commit_msg>First user of AIFactory<commit_after>#!/usr/bin/env python

import rules
import game

import time

from ai_player import *
from priority_filter import *
from priority_filter_2 import *
from blindness_filter import *
from evaluator import *
from ai_factory import *

import pdb

class TwoTimer:
    def __init__(self):
        self.totals = [0.0, 0.0]
        self.current = 0

    def __enter__(self):
        self.start = time.clock()
        return self

    def __exit__(self, *args):
        self.end = time.clock()
        self.totals[self.current] += self.end - self.start
        self.current = 1 - self.current

    def __repr__(self):
        tot = self.totals
        return "B: %.2fs, W: %.2fs, B/W: %.2f" % (tot[0], tot[1], tot[0]/tot[1])

class MatchResults():
    def __init__(self):
        self.results = []
        self.bw_total = [0, 0, 0]
        self.dc_total = {"Defender":0, "Contender":0}
        self.total_ratio = 0.
        self.games_played = 0

    def __repr__(self):
        return "\n".join(self.results) + "\nB/W:" + str(self.bw_total) \
                + " won: " + str(self.dc_total) \
                + " C/D avg. time: " + str(self.total_ratio / self.games_played)

    def add(self, result):
        res_str, winner_colour, winner_name, ratio = result
        self.results.append(res_str)
        self.bw_total[winner_colour] += 1
        self.dc_total[str(winner_name)] += 1
        self.total_ratio += ratio
        self.games_played += 1

class Match():
    def __init__(self):
        pdb.set_trace()
        self.genome1 = Genome("Defender")
        self.genome2 = Genome("Contender")

    def set_up(self, game_length):
        aif = AIFactory()
        self.genome1.max_depth = game_length
        self.genome2.max_depth = game_length
        self.p1 = aif.create_ai(self.genome1)
        self.p2 = aif.create_ai(self.genome2)

    def play_one_game(self, board_size, p1, p2):
        r = rules.Rules(board_size, "standard")
        self.game = game.Game(r, p1, p2)
        #self.evaluator = Evaluator(UtilityCalculator(), self.game.current_state)

        tt = TwoTimer()

        while not self.game.finished():
            p = self.game.get_current_player()
            with tt:
                m = p.do_the_search()
                self.game.make_move(m)
                #print self.evaluator.utility()
        #pdb.set_trace()
        winner_name = self.game.winner_name()
        winner = self.game.winner()
        if p1.name == "Contender":
            ratio = tt.totals[0] / tt.totals[1]
        else:
            ratio = tt.totals[1] / tt.totals[0]

        print "Game was won by: %s %s" % (["?", "B", "W"][winner], winner_name)
        print tt

        return "%s vs. %s: %s (%sx%s %s) %s" % (p1.name, p2.name, winner_name,
                board_size, board_size, p1.max_depth, tt), winner, winner_name, ratio

    def play_some_games(self):

        #self.genome2.length_factor = 35
        #self.genome2.take_score_base = 110
        #self.genome2.capture_score_base = 310 # Try this again for high depth
        #self.genome2.threat_score_base = 25
        #self.genome1.blindness = 0.30
        #self.genome2.blindness = 0.40

        self.genome2.filter2 = True
        #self.genome2.narrowing = 3
        #self.genome2.max_depth += 2 # Setting max_depth here doesn't work
        #self.genome2.mmpdl = 15
        #self.genome1.chokes = []
        #self.genome2.chokes = [(4,5)]
        #self.genome2.chokes = [(4,3)]
        #self.genome2.chokes = [(2,2)]
        #self.genome1.max_depth_boost = 2
        #self.genome2.max_depth_boost = 2
        #self.genome2.captures_scale = [0, 1, 10, 100, 1000, 50]
        #self.genome2.captures_scale = [0, 0, 0, 0, 0, 0]
        #self.genome2.move_factor = 10000000

        #self.genome1.sub = False
        #self.genome2.sub = False
        #self.genome2.move_factor = 50
        #self.genome2.move_factor = 5

        results = MatchResults()
        for game_length in range(3, 4):
            for board_size in [9, 13, 19]:
            #for board_size in [9, 13, 16, 19]:
                for first_player in [0, 1]:
                    self.set_up(game_length)
                    players = [self.p1, self.p2]
                    second_player = 1 - first_player
                    res = self.play_one_game(board_size,
                                             players[first_player],
                                             players[second_player])
                    results.add(res)

        print results

import random
import pstats, cProfile

if __name__ == "__main__":
    random.seed()
    m = Match()
    m.play_some_games()
    '''
    cProfile.runctx("m.play_some_games()", globals(), locals(), "Profile.prof")

    s = pstats.Stats("Profile.prof")
    #s.strip_dirs().sort_stats("cumulative").print_stats(20) # or "time"
    s.strip_dirs().sort_stats("time").print_stats(20)
    '''
<|endoftext|>


<commit_before>

import postProcessingFcVolMatchedModularDynamicPipeline as ppfc
# import preprocessingPipelineImprovedModular as prep
import preprocessingPipelineImprovedModularWOAnatDynamic as prep

import hypothesisTest as ht
import fdrBrainResultsModular as fdres
import os
import json
from os.path import join as opj
import itertools
from bids.grabbids import BIDSLayout

# ------------- Paths ----------------------------------------------------------------------------------------

path_cwd = os.getcwd()
path_split_list = path_cwd.split('/')
s = path_split_list[0:-1] # for getting to the parent dir of pwd
s = opj('/',*s) # *s converts list to path, # very important to add '/' in the begining so it is read as directory later

json_path = 'scripts/json/paths.json'
with open(json_path, 'rt') as fp:
    task_info = json.load(fp)

base_directory = opj(s,task_info["base_directory_for_results"])
motion_correction_bet_directory = task_info["motion_correction_bet_directory"]
parent_wf_directory = task_info["parent_wf_directory"]
functional_connectivity_directory = task_info["functional_connectivity_directory"]
# functional_connectivity_directory = 'temp_fc'
coreg_reg_directory = task_info["coreg_reg_directory"]
atlas_resize_reg_directory = task_info["atlas_resize_reg_directory"]
data_directory = opj(s,task_info["data_directory"])
datasink_name = task_info["datasink_name"]
fc_datasink_name = task_info["fc_datasink_name"]
# fc_datasink_name = 'temp_dataSink'
atlasPath = opj(s,task_info["atlas_path"]) # Standard Brainnetome path

brain_path = opj(base_directory,datasink_name,'preprocessed_brain_paths/brain_file_list.npy')
mask_path = opj(base_directory,datasink_name,'preprocessed_mask_paths/mask_file_list.npy')
atlas_path = opj(base_directory,datasink_name,'atlas_paths/atlas_file_list.npy') # Brainetome atlas in functional space
tr_path = opj(base_directory,datasink_name,'tr_paths/tr_list.npy')
motion_params_path = opj(base_directory,datasink_name,'motion_params_paths/motion_params_file_list.npy')

func2std_mat_path = opj(base_directory, datasink_name,'joint_xformation_matrix_paths/joint_xformation_matrix_file_list.npy')

MNI3mm_path = opj(base_directory,parent_wf_directory,motion_correction_bet_directory,coreg_reg_directory,'resample_mni/MNI152_T1_2mm_brain_resample.nii')

hypothesis_test_dir = opj(base_directory, task_info["hypothesis_test_dir"])

fdr_results_dir = task_info["fdr_results_dir"]

demographics_file_path = '/home1/varunk/Autism-Connectome-Analysis-brain_connectivity/notebooks/demographics.csv'
phenotype_file_path = '/home1/varunk/data/ABIDE1/RawDataBIDs/composite_phenotypic_file.csv'
# categoryInfo = '/home1/varunk/data/NYU_Cocaine-BIDS/grouping.csv'
categoryInfo = None

# ----------------------------------------------------------------------------------------------------------------------------------------------------------

vols = 120 # needed for volume matching. Tells how many volumes you need to keep.
number_of_skipped_volumes = 4
#  True volumes will be vols - number_of_skipped_volumes

num_proc = 4

number_of_subjects = -1
# number_of_subjects = 2 # Number of subjects you wish to work with

# ----------------------------- Getting Subjects -------------------------------
# ----------------------------------- BIDS -------------------------------------
layout = BIDSLayout(data_directory)

if number_of_subjects == -1:
    number_of_subjects = len(layout.get_subjects())

subject_list = (layout.get_subjects())[0:number_of_subjects]
# subject_list = list(map(int, subject_list))

# -----------------------------------File List----------------------------------
# group1FilesPath = ''
# group2FilesPath = ''
#
# group1FilesList = np.genfromtxt(group1FilesPath,dtype='unicode')
# group2FilesList = np.genfromtxt(group2FilesPath,dtype='unicode')
#
# fileList = group1FilesPath + group2FilesPath

# -----------------------------------------------------------------------------

paths = [json_path,
base_directory,
motion_correction_bet_directory,
parent_wf_directory,
functional_connectivity_directory,
coreg_reg_directory,
atlas_resize_reg_directory,
subject_list,
datasink_name,
fc_datasink_name,
atlasPath,
brain_path,
mask_path,
atlas_path,
tr_path,
motion_params_path,
func2std_mat_path,
MNI3mm_path,
demographics_file_path,
phenotype_file_path,
data_directory,
hypothesis_test_dir,
fdr_results_dir]

PREPROC = 1
POSTPROC = 1
HYPOTEST = 1
FDRES = 1

match = 1 # Age matching
applyFisher = True

# itr = (list(itertools.product([0, 1], repeat=3)))
itr = [(1,0,1,1,1)]
# itr = [(1,1,1,1,1)]
# ---------------------- Preprocess --------------------------------------------

ANAT = 1

itr_preproc = [1,1,0,1]
# itr_preproc = [0,0,0]
extract, slicetimer,motionOutliers, mcflirt= list(map(str, itr_preproc))
options_binary_string = extract+slicetimer+motionOutliers+mcflirt
if PREPROC == 1:
    print('Preprocessing')
    prep.main(paths,options_binary_string, ANAT, num_proc)

# ------------------------PostProcess------------------------------------------
if POSTPROC == 1:
    print('PostProcessing')

    for motion_param_regression, global_signal_regression, smoothing, band_pass_filtering, volCorrect in itr:
        combination = 'motionRegress' + str(int(motion_param_regression)) + \
         'global' + str(int(global_signal_regression)) + 'smoothing' + str(int(smoothing)) +\
         'filt' + str(int(band_pass_filtering))

        print("Combination: ",combination)
        functional_connectivity_directory =  combination
        print(motion_param_regression,  global_signal_regression, smoothing,band_pass_filtering)
        ppfc.main(paths, vols, motion_param_regression, global_signal_regression, band_pass_filtering, smoothing, volCorrect, \
        number_of_skipped_volumes, num_proc)

# ------------------- Hypothesis Test ------------------------------------------

if HYPOTEST == 1:
    print('Hypothesis Test')

    # itr = (list(itertools.product([0, 1], repeat=3)))
    #
    # itr = [(1,1,0,1,1)]

    bugs = ['51232','51233','51242','51243','51244','51245','51246','51247','51270','51310','50045', '51276', '50746', '50727', '51276']

    # bugs = []

    for motion_param_regression, global_signal_regression, smoothing,band_pass_filtering, volCorrect in itr:
        ht.main(paths, bugs,applyFisher,categoryInfo, match, motion_param_regression, global_signal_regression, band_pass_filtering, \
            smoothing, num_proc)

# -------------------- FDR and results -----------------------------------------

if FDRES == 1:
    print('FDR Correction and computing files for visualization of results')
    # itr = (list(itertools.product([0, 1], repeat=3)))

    # itr = [(1,1,0,1,1)]
    for params in itr:
        fdres.main(paths, params, num_proc = 7)
<commit_msg>Added log facility.<commit_after>import postProcessingFcVolMatchedModularDynamicPipeline as ppfc
# import preprocessingPipelineImprovedModular as prep
import preprocessingPipelineImprovedModularWOAnatDynamic as prep

import hypothesisTest as ht
import fdrBrainResultsModular as fdres
import os
import json
from os.path import join as opj
import itertools
from bids.grabbids import BIDSLayout
import time

# ------------- Paths ----------------------------------------------------------------------------------------

path_cwd = os.getcwd()
path_split_list = path_cwd.split('/')
s = path_split_list[0:-1] # for getting to the parent dir of pwd
s = opj('/',*s) # *s converts list to path, # very important to add '/' in the begining so it is read as directory later

json_path = 'scripts/json/paths.json'
with open(json_path, 'rt') as fp:
    task_info = json.load(fp)

base_directory = opj(s,task_info["base_directory_for_results"])
motion_correction_bet_directory = task_info["motion_correction_bet_directory"]
parent_wf_directory = task_info["parent_wf_directory"]
functional_connectivity_directory = task_info["functional_connectivity_directory"]
# functional_connectivity_directory = 'temp_fc'
coreg_reg_directory = task_info["coreg_reg_directory"]
atlas_resize_reg_directory = task_info["atlas_resize_reg_directory"]
data_directory = opj(s,task_info["data_directory"])
datasink_name = task_info["datasink_name"]
fc_datasink_name = task_info["fc_datasink_name"]
# fc_datasink_name = 'temp_dataSink'
atlasPath = opj(s,task_info["atlas_path"]) # Standard Brainnetome path

brain_path = opj(base_directory,datasink_name,'preprocessed_brain_paths/brain_file_list.npy')
mask_path = opj(base_directory,datasink_name,'preprocessed_mask_paths/mask_file_list.npy')
atlas_path = opj(base_directory,datasink_name,'atlas_paths/atlas_file_list.npy') # Brainetome atlas in functional space
tr_path = opj(base_directory,datasink_name,'tr_paths/tr_list.npy')
motion_params_path = opj(base_directory,datasink_name,'motion_params_paths/motion_params_file_list.npy')

func2std_mat_path = opj(base_directory, datasink_name,'joint_xformation_matrix_paths/joint_xformation_matrix_file_list.npy')

MNI3mm_path = opj(base_directory,parent_wf_directory,motion_correction_bet_directory,coreg_reg_directory,'resample_mni/MNI152_T1_2mm_brain_resample.nii')

hypothesis_test_dir = opj(base_directory, task_info["hypothesis_test_dir"])

fdr_results_dir = task_info["fdr_results_dir"]

score_corr_dir =  opj(base_directory,task_info["score_corr_dir"])

demographics_file_path = '/home1/varunk/Autism-Connectome-Analysis-brain_connectivity/notebooks/demographics.csv'
phenotype_file_path = '/home1/varunk/data/ABIDE1/RawDataBIDs/composite_phenotypic_file.csv'
# categoryInfo = '/home1/varunk/data/NYU_Cocaine-BIDS/grouping.csv'
categoryInfo = None

# --------------- Creating Log --------------------

# Get time

current_time = time.asctime( time.localtime(time.time()) )

log_path = opj(base_directory,"log.txt")
log = open(log_path, 'a')
print("-------------Starting the analysis at %s--------------\n"%(current_time))
log.write("-------------Starting the analysis at %s--------------\n"%(current_time))
log.flush()

# ----------------------------------------------------------------------------------------------------------------------------------------------------------

vols = 120 # needed for volume matching. Tells how many volumes you need to keep.
number_of_skipped_volumes = 4
#  True volumes will be vols - number_of_skipped_volumes

num_proc = 7

number_of_subjects = -1
# number_of_subjects = 7 # Number of subjects you wish to work with

log.write("Vols for matching: %s\n"%(vols))
log.write("Number_of_skipped_volumes: %s\n"%(number_of_skipped_volumes))
log.flush()
# ----------------------------- Getting Subjects -------------------------------
# ----------------------------------- BIDS -------------------------------------
layout = BIDSLayout(data_directory)

if number_of_subjects == -1:
    number_of_subjects = len(layout.get_subjects())

subject_list = (layout.get_subjects())[0:number_of_subjects]
# subject_list = list(map(int, subject_list))

# -----------------------------------File List----------------------------------
# group1FilesPath = ''
# group2FilesPath = ''
#
# group1FilesList = np.genfromtxt(group1FilesPath,dtype='unicode')
# group2FilesList = np.genfromtxt(group2FilesPath,dtype='unicode')
#
# fileList = group1FilesPath + group2FilesPath

# -----------------------------------------------------------------------------

paths = [json_path,
base_directory,
motion_correction_bet_directory,
parent_wf_directory,
functional_connectivity_directory,
coreg_reg_directory,
atlas_resize_reg_directory,
subject_list,
datasink_name,
fc_datasink_name,
atlasPath,
brain_path,
mask_path,
atlas_path,
tr_path,
motion_params_path,
func2std_mat_path,
MNI3mm_path,
demographics_file_path,
phenotype_file_path,
data_directory,
hypothesis_test_dir,
fdr_results_dir,
score_corr_dir
]

PREPROC = 0
POSTPROC = 0
HYPOTEST = 0
FDRES = 0
SCORE_CORR = 1

match = 1 # Age matching
applyFisher = True

# itr = (list(itertools.product([0, 1], repeat=3)))
itr = [(1,0,1,1,1)]
# itr = [(1,1,1,1,1)]

log.write("Operations:\n")
log.write("Preprocess = %s\n"%(PREPROC))
log.write("Postprocess = %s\n"%(POSTPROC))
log.write("Hypothesis Test = %s\n"%(HYPOTEST))
log.write("FDR correction and Vizualization = %s\n"%(FDRES))
log.write("Score-Connectivity Correlation  = %s\n"%(SCORE_CORR))
log.flush()

# ---------------------- Preprocess --------------------------------------------

ANAT = 1

itr_preproc = [1,1,0,1]
# itr_preproc = [0,0,0]
extract, slicetimer,motionOutliers, mcflirt= list(map(str, itr_preproc))
options_binary_string = extract+slicetimer+motionOutliers+mcflirt

log.write("Preprocessing Params\n")
log.write("Remove begining slices  = %s\n"%(extract))
log.write("Slice time correction  = %s\n"%(slicetimer))
log.write("Calculate motionOutliers  = %s\n"%(motionOutliers))
log.write("Do motion correction using McFLIRT  = %s\n"%(mcflirt))

log.flush()

if PREPROC == 1:
    print('Preprocessing')
    prep.main(paths,options_binary_string, ANAT, num_proc)

# ------------------------PostProcess------------------------------------------
if POSTPROC == 1:
    print('PostProcessing')

    for motion_param_regression, global_signal_regression, smoothing, band_pass_filtering, volCorrect in itr:
        combination = 'motionRegress' + str(int(motion_param_regression)) + \
         'global' + str(int(global_signal_regression)) + 'smoothing' + str(int(smoothing)) +\
         'filt' + str(int(band_pass_filtering))

        print("Combination: ",combination)
        functional_connectivity_directory =  combination
        print(motion_param_regression,  global_signal_regression, smoothing,band_pass_filtering)
        ppfc.main(paths, vols, motion_param_regression, global_signal_regression, band_pass_filtering, smoothing, volCorrect, \
        number_of_skipped_volumes, num_proc)

# ------------------- Hypothesis Test ------------------------------------------

bugs = ['51232','51233','51242','51243','51244','51245','51246','51247','51270','51310','50045', '51276', '50746', '50727', '51276']

if HYPOTEST == 1:
    print('Hypothesis Test')

    # itr = (list(itertools.product([0, 1], repeat=3)))
    #
    # itr = [(1,1,0,1,1)]

    # bugs = ['51232','51233','51242','51243','51244','51245','51246','51247','51270','51310','50045', '51276', '50746', '50727', '51276']

    # bugs = []

    for motion_param_regression, global_signal_regression, smoothing,band_pass_filtering, volCorrect in itr:
        ht.main(paths, bugs,applyFisher,categoryInfo, match, motion_param_regression, global_signal_regression, band_pass_filtering, \
            smoothing, num_proc)

# -------------------- FDR and results -----------------------------------------

if FDRES == 1:
    print('FDR Correction and computing files for visualization of results')
    # itr = (list(itertools.product([0, 1], repeat=3)))

    # itr = [(1,1,0,1,1)]
    for params in itr:
        fdres.main(paths, params, num_proc = 7)

if SCORE_CORR == 1:
    print('Calculating Correlation-Score correaltions')

    # itr = (list(itertools.product([0, 1], repeat=3)))
    #
    # itr = [(1,1,0,1,1)]

    # bugs = []

    for motion_param_regression, global_signal_regression, smoothing,band_pass_filtering, volCorrect in itr:
        ht.main(paths, bugs,applyFisher,categoryInfo, match, motion_param_regression, global_signal_regression, band_pass_filtering, \
            smoothing, num_proc)
<|endoftext|>


<commit_before>import os
import csv
import types
import sys
from time import time
from django.conf import settings
from django.db import connection, transaction
#from arches.app.models.entity import Entity
from arches.app.models.entity import Entity
from pprint import pprint
import collections

class Row(object):
    def __init__(self, *args):
        if len(args) == 0:
            self.resource_id = ''
            self.resourcetype = ''
            self.attributename = ''
            self.attributevalue = ''
            self.group_id = ''
        elif isinstance(args[0], list):
            self.resource_id = args[0][0].strip()
            self.resourcetype = args[0][1].strip()
            self.attributename = args[0][2].strip()
            self.attributevalue = args[0][3].strip()
            self.group_id = args[0][4].strip()

    def __repr__(self):
        return ('"%s, %s, %s, %s"') % (self.resource_id, self. resourcetype, self. attributename, self.attributevalue)

class Group(object):
    def __init__(self, *args):
        if len(args) == 0:
            self.resource_id = ''
            self.group_id = ''
            self.rows = []
        elif isinstance(args[0], list):
            self.resource_id = args[0][0].strip()
            self.group_id = args[0][4].strip()
            self.rows = []

class Resource(object):
    def __init__(self, *args):
        if len(args) == 0:
            self.resource_id = ''
            self.entitytypeid = ''
            self.groups = []
            self.nongroups = []
        elif isinstance(args[0], list):
            self.entitytypeid = args[0][1].strip()
            self.resource_id = args[0][0].strip()
            self.nongroups = []
            self.groups = []
    def appendrow(self, row, group_id=None):
        if group_id != None:
            for group in self.groups:
                if group.group_id == group_id:
                    group.rows.append(row)
        else:
           self.nongroups.append(row)

    def __str__(self):
        return '{0},{1}'.format(self.resource_id, self.entitytypeid)

class ResourceRelation(object):
    def __init__(self, *args):
        if len(args) == 0:
            self.resourceid1 = ''
            self.xref_entity_id1 = ''
            self.xref_value = ''
            self.resourceid2 = ''
            self.xref_entity_id2 = ''
            self.legacyid = ''
            self.relation_type = ''
            self.start_date = ''
            self.end_date = ''
    def __str__(self):
        return '{0},{1}'.format(self.start_date, self.end_date)

class DataLoader(object):

    def load(self, filepath, resource_relation_entitytypeids=None, truncate=True):
        start = time()
        resource_info = open(filepath, 'rb')
        rows = [line.split("|") for line in resource_info]
        resourceList = []
        resource_id = ''
        group_id = ''

        if truncate:
            cursor = connection.cursor()
            cursor.execute("""
                TRUNCATE data.entities CASCADE;
            """ )

        for row in rows:
            if rows.index(row) != 0:
                if (settings.LIMIT_ENTITY_TYPES_TO_LOAD == None or row[1].strip() in settings.LIMIT_ENTITY_TYPES_TO_LOAD):
                    if row[0].strip() != resource_id:
                        resource = Resource(row)
                        resourceList.append(resource)
                        resource_id = row[0].strip()

                    if row[4].strip() != '-1' and row[4].strip() != group_id:
                        resource.groups.append(Group(row))
                        group_id = row[4].strip()

                    if row[4].strip() == group_id:
                        resource.appendrow(Row(row), group_id=group_id)
                        #resourceList[len(resourceList)-1].groups[len(resourceList[len(resourceList)-1].groups)-1].rows.append(Row(row))

                    if row[4].strip() == '-1':
                        resource.appendrow(Row(row))
                        #resourceList[len(resourceList)-1].nongroups.append(Row(row))

        elapsed = (time() - start)
        print 'time to parse csv = %s' % (elapsed)
        return self.resource_list_to_entities(resourceList, resource_relation_entitytypeids, cursor)

    def resource_list_to_entities(self, resourceList, resource_relation_entitytypeids, cursor):
        start = time()
        ret = {'successfully_saved':0, 'successfully_indexed':0, 'failed_to_save':[], 'failed_to_index':[]}
        schema = None
        current_entitiy_type = None
        resource_relations = []

        for resource in resourceList:
            masterGraph = None
            entityData = []
            if current_entitiy_type != resource.entitytypeid:
                schema = Entity.get_mapping_schema(resource.entitytypeid)

            master_graph = self.build_master_graph(resource, schema)
            self.update_related_resources(master_graph, resourceList)

            master_graph.save(username=settings.ETL_USERNAME)
            resource.entityid = master_graph.entityid
            resource.legacyid = self.get_legacy_id(master_graph)

            # # Gather legacyids for all entities in resource_info.
            # if resource_relation_entitytypeids:

            #     def traverse_down(entity, dictionary):
            #         for child in entity.relatedentities:
            #             if child.value != '':
            #                 dictionary[child.entitytypeid] = child.value
            #             traverse_down(child, dictionary)
            #         return dictionary

            #    for item in master_graph.find_entities_by_type_id('EXTERNAL XREF.E42'):
            #         relation = ResourceRelation()
            #         relation.resourceid1 = master_graph.entityid
            #         relation.legacyid = master_graph.entitytypeid.split('.')[0] + ':' + item.value
            #         resource_relations.append(relation)

            #     # If an entity in resource_info has an Arches Resource.E1 entity, grab it's value and the rest of the xref branch.
            #     for item in master_graph.find_entities_by_type_id('ARCHES RESOURCE.E1'):
            #         xref_attributes = {}
            #         relation = ResourceRelation()
            #         relation.resourceid1 = master_graph.entityid
            #         relation.xref_entity_id1 = item.entityid
            #         relation.xref_value = item.value
            #         xref_dic = {}
            #         if len(item.relatedentities) > 0:
            #             xref_attributes = traverse_down(item, xref_dic)
            #             # parse xref branch attributes and add to relation object
            #             if 'ARCHES RESOURCE CROSS-REFERENCE RELATIONSHIP TYPE.E55' in xref_attributes:
            #                 relation.relation_type = xref_attributes['ARCHES RESOURCE CROSS-REFERENCE RELATIONSHIP TYPE.E55']
            #             if 'DATE RESOURCE ASSOCIATION STARTED.E50' in xref_attributes:
            #                 relation.start_date = xref_attributes['DATE RESOURCE ASSOCIATION STARTED.E50']
            #             if 'DATE RESOURCE ASSOCIATION ENDED.E50' in xref_attributes:
            #                 relation.end_date = xref_attributes['DATE RESOURCE ASSOCIATION ENDED.E50']
            #         resource_relations.append(relation)

            ret['successfully_saved'] += 1
            master_graph.index()
            ret['successfully_indexed'] += 1

        # Compare resourceRelation objects and cross populate xref_entity_id's and resourceid's if resource.legacyid == related_resource.xref_value
        if resource_relation_entitytypeids and len(resource_relations) > 0:
            self.save_related_resources(resource_relations, resource_relation_entitytypeids, cursor)

        elapsed = (time() - start)
        print 'total time to etl = %s' % (elapsed)
        print 'average time per entity = %s' % (elapsed/len(resourceList))
        print ret
        return ret

    def build_master_graph(self, resource, schema):
        master_graph = None
        entity_data = []
        for row in resource.nongroups:
            entity = Entity()
            entity.create_from_mapping(row.resourcetype, schema[row.attributename]['steps'], row.attributename, row.attributevalue)
            entity_data.append(entity)

        if len(entity_data) > 0:
            master_graph = entity_data[0]
            for mapping in entity_data[1:]:
                master_graph.merge(mapping)

        for group in resource.groups:
            entity_data2 = []
            for row in group.rows:
                entity = Entity()
                entity.create_from_mapping(row.resourcetype, schema[row.attributename]['steps'], row.attributename, row.attributevalue)
                entity_data2.append(entity)

            mapping_graph = entity_data2[0]
            for mapping in entity_data2[1:]:
                mapping_graph.merge(mapping)

            if master_graph == None:
                master_graph = mapping_graph
            else:
                node_type_to_merge_at = schema[row.attributename]['mergenodeid']
                master_graph.merge_at(mapping_graph, node_type_to_merge_at)

        return master_graph

    def get_legacy_id(self, entity):
        for item in entity.find_entities_by_type_id('EXTERNAL XREF.E42'):
            return item.value

    def update_related_resources(self, master_graph, resourceList):
        for item in master_graph.find_entities_by_type_id('ARCHES RESOURCE.E1'):
            for resource in resourceList:
                try:
                    if item.value == resource.legacyid:
                        item.value = resource.entityid
                except:pass

    def save_related_resources(self, resource_relations, resource_relation_entitytypeids, cursor):
        for resource in resource_relations:
            for resource2 in resource_relations:
                if resource2.legacyid and (resource.xref_value == resource2.legacyid):
                    resource.resourceid2 = resource2.resourceid1
                    resource.xref_entity_id2 = resource2.xref_entity_id1

            if resource.resourceid2:
                if resource.xref_entity_id2 == '':
                    resource.xref_entity_id2 = self.generate_uuid(cursor)

                    resource_entity = Entity().get(resource.resourceid2)

                    mapping_nodes = collections.OrderedDict([
                        ('ARCHES RESOURCE.E1', resource.resourceid1),
                        ('ARCHES RESOURCE CROSS-REFERENCE RELATIONSHIP TYPE.E55', resource.relation_type),
                        ('DATE RESOURCE ASSOCIATION STARTED.E50', resource.start_date),
                        ('DATE RESOURCE ASSOCIATION ENDED.E50', resource.end_date)
                        ])

                    if resource_entity.entitytypeid in resource_relation_entitytypeids:
                        entities_to_merge = []
                        for entity_type_id, entity_value in mapping_nodes.iteritems():
                            entity = Entity()
                            mapping_steps = self.get_mapping_steps(resource_entity, entity_type_id, cursor)
                            entity.create_from_mapping(resource_entity.entitytypeid, mapping_steps, entity_type_id, entity_value)
                            entities_to_merge.append(entity)

                        if len(entities_to_merge) > 0:
                            base_entity = entities_to_merge[0]
                            for entity in entities_to_merge[1:]:
                                base_entity.merge(entity)

                            resource_entity.merge(base_entity)

                            try:
                                resource_entity.save()
                                print 'successfully_saved'
                            except:
                                print 'failed to save'

                #pprint(JSONSerializer().serializeToPython(resource_entity, ensure_ascii=True, indent=4))

                entity = Entity()
                related_resource = entity.get(resource.resourceid1)
                for item in related_resource.find_entities_by_type_id('ARCHES RESOURCE.E1'):
                    if item.value == resource.xref_value:
                        item.value = resource.resourceid2
                        related_resource.save()
                    else:
                        print '''ERROR: '%s' is not a valid resource legacyid''' %(item.value)

    def generate_uuid(self, cursor):
        sql = """
            SELECT uuid_generate_v1mc()
        """
        cursor.execute(sql)
        return cursor.fetchone()[0]

    def get_mapping_steps(self, resource_entity, leaf_entity_type_id, cursor):
        sql = """
            SELECT mappingid FROM ontology.mappings WHERE entitytypeidfrom = '%s' AND entitytypeidto = '%s'
        """%(resource_entity.entitytypeid, leaf_entity_type_id)
        cursor.execute(sql)
        return resource_entity._get_mappings(cursor.fetchone()[0])

    def findmergenode(self, graph):
        """
        Logic to try and determine the proper node to merge multiple instances of the same types of data. Like address info.
        Currently unused.

        """

        rank = None
        highestnodewithvalue = None
        entitieswithvalues = []
        def findentitieswithvalues(entity):
            if entity.value != '':
                entitieswithvalues.append(entity)
        graph.traverse(findentitieswithvalues)

        if len(entitieswithvalues) == 0:
            raise Exception("Entity graph has no values")
        for entity in entitieswithvalues:
            if rank == None:
                rank = entity.get_rank()
            if entity.get_rank() <= rank:
                rank = entity.get_rank()
                highestnodewithvalue = entity

        rank = None
        highestforkingnode = None
        entitiesthatfork = []
        def findentitiesthatfork(entity):
            if len(entity.relatedentities) > 1:
                entitiesthatfork.append(entity)
        graph.traverse(findentitiesthatfork)

        for entity in entitiesthatfork:
            if rank == None:
                rank = entity.get_rank()
            if entity.get_rank() <= rank:
                rank = entity.get_rank()
                highestforkingnode = entity

        # LOGIC SECTION
        if highestforkingnode == None or highestnodewithvalue.get_rank() >= highestforkingnode.get_rank():
            if highestnodewithvalue.get_parent().get_rank() == 0:
                return highestnodewithvalue.get_parent()
            else:
                return highestnodewithvalue.get_parent().get_parent()

        if highestnodewithvalue.get_rank() < highestforkingnode.get_rank():
            if highestforkingnode.get_rank() == 0:
                return highestforkingnode
            else:
                return highestforkingnode.get_parent()

        return None

        #     groupNode =

        # 1.) Identify highest node with value;
        # 2.) Identify node at which the group has a common parent;

        # 3.) Take the node that is closer to the root
        #     a.) if that is that is the node with a value: split two parents up or root
        #     b.) else: split one parent up

<commit_msg>remove uneeded file<commit_after><|endoftext|>


<commit_before>__package__ = 'archivebox.parsers'
__description__ = 'URL list'

from typing import IO, Iterable
from datetime import datetime

from ..index.schema import Link
from ..util import (
    enforce_types
)

@enforce_types
def parse_url_list(text_file: IO[str], **_kwargs) -> Iterable[Link]:
    """Parse raw URLs from each line in a text file"""

    text_file.seek(0)
    for line in text_file.readlines():
        url = line.strip()
        if not url:
            continue

        yield Link(
            url=url,
            timestamp=str(datetime.now().timestamp()),
            title=None,
            tags=None,
            sources=[text_file.name],
        )

KEY = 'url_list'
NAME = 'URL List'
PARSER = parse_url_list
<commit_msg>only add url-list lines that are real urls<commit_after>__package__ = 'archivebox.parsers'
__description__ = 'URL list'

import re

from typing import IO, Iterable
from datetime import datetime

from ..index.schema import Link
from ..util import (
    enforce_types,
    URL_REGEX,
)

@enforce_types
def parse_url_list(text_file: IO[str], **_kwargs) -> Iterable[Link]:
    """Parse raw URLs from each line in a text file"""

    text_file.seek(0)
    for line in text_file.readlines():
        url = line.strip()
        if (not url) or not re.findall(URL_REGEX, url):
            continue

        yield Link(
            url=url,
            timestamp=str(datetime.now().timestamp()),
            title=None,
            tags=None,
            sources=[text_file.name],
        )

KEY = 'url_list'
NAME = 'URL List'
PARSER = parse_url_list
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from django import forms

from calebasse.actes.models import Act
from calebasse.ressources.models import ActType
from ajax_select import make_ajax_field

class ActSearchForm(forms.Form):
    STATES = (
            ('pointe', u'Pointés'),
            ('non-pointe', u'Non pointés'),
            ('valide', u'Validés'),
            ('absent-or-canceled', u'Absents ou annulés'),
            ('is-billable', u'Facturables'),
            ('non-invoicable', u'Non facturables'),
            ('switch-billable', u'Avec facturabilité inversée'),
            ('lost', u'Perdus'),
            ('pause-invoicing', u'En pause facturation'),
            ('invoiced', u'Facturés'),
            ('group', u'De groupe'),
#            ('current-invoicing', u'Facturation en cours')
            )

    INITIAL = [x[0] for x in STATES]

    last_name = forms.CharField(required=False)
    patient_record_id = forms.IntegerField(required=False)
    social_security_number = forms.CharField(required=False)

    doctor_name = forms.CharField(required=False)
    filters = forms.MultipleChoiceField(choices=STATES,
            widget=forms.CheckboxSelectMultiple)

class ActUpdate(forms.ModelForm):
    doctors = make_ajax_field(Act, 'doctors', 'intervenant', True)
    class Meta:
        model = Act
        fields = ('act_type', 'doctors', 'is_lost', 'pause', 'switch_billable', 'comment',
                'valide')
        widgets = {
                'comment': forms.Textarea(attrs={'cols': 52, 'rows': 4}),
                }

    def __init__(self, instance, service=None, **kwargs):
        super(ActUpdate, self).__init__(instance=instance, **kwargs)
        if instance.patient.service:
            self.fields['act_type'].queryset = \
                    ActType.objects.for_service(instance.patient.service) \
                    .order_by('name')
<commit_msg>actes: 'valide' act field must not be edited, replace 'Validés' by 'Présents' (fixes #3413).<commit_after># -*- coding: utf-8 -*-

from django import forms

from calebasse.actes.models import Act
from calebasse.ressources.models import ActType
from ajax_select import make_ajax_field

class ActSearchForm(forms.Form):
    STATES = (
            ('pointe', u'Pointés'),
            ('non-pointe', u'Non pointés'),
            ('valide', u'Présents'),
            ('absent-or-canceled', u'Absents ou annulés'),
            ('is-billable', u'Facturables'),
            ('non-invoicable', u'Non facturables'),
            ('switch-billable', u'Avec facturabilité inversée'),
            ('lost', u'Perdus'),
            ('pause-invoicing', u'En pause facturation'),
            ('invoiced', u'Facturés'),
            ('group', u'De groupe'),
#            ('current-invoicing', u'Facturation en cours')
            )

    INITIAL = [x[0] for x in STATES]

    last_name = forms.CharField(required=False)
    patient_record_id = forms.IntegerField(required=False)
    social_security_number = forms.CharField(required=False)

    doctor_name = forms.CharField(required=False)
    filters = forms.MultipleChoiceField(choices=STATES,
            widget=forms.CheckboxSelectMultiple)

class ActUpdate(forms.ModelForm):
    doctors = make_ajax_field(Act, 'doctors', 'intervenant', True)
    class Meta:
        model = Act
        fields = ('act_type', 'doctors', 'is_lost', 'pause',
            'switch_billable', 'comment', )
        widgets = {
                'comment': forms.Textarea(attrs={'cols': 52, 'rows': 4}),
                }

    def __init__(self, instance, service=None, **kwargs):
        super(ActUpdate, self).__init__(instance=instance, **kwargs)
        if instance.patient.service:
            self.fields['act_type'].queryset = \
                    ActType.objects.for_service(instance.patient.service) \
                    .order_by('name')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import functools
import hashlib
import logging
import threading
import binascii
from collections import OrderedDict

from celery.local import PromiseProxy
from gevent.pool import Pool

from website import settings

_local = threading.local()
logger = logging.getLogger(__name__)

def postcommit_queue():
    if not hasattr(_local, 'postcommit_queue'):
        _local.postcommit_queue = OrderedDict()
    return _local.postcommit_queue

def postcommit_before_request():
    _local.postcommit_queue = OrderedDict()

def postcommit_after_request(response, base_status_error_code=500):
    if response.status_code >= base_status_error_code:
        _local.postcommit_queue = OrderedDict()
        return response
    try:
        if postcommit_queue():
            number_of_threads = 30  # one db connection per greenlet, let's share
            pool = Pool(number_of_threads)
            for func in postcommit_queue().values():
                pool.spawn(func)
            pool.join(timeout=5.0, raise_error=True)  # 5 second timeout and reraise exceptions
    except AttributeError as ex:
        if not settings.DEBUG_MODE:
            logger.error('Post commit task queue not initialized: {}'.format(ex))
    return response

def enqueue_postcommit_task(fn, args, kwargs, once_per_request=True):
    # make a hash of the pertinent data
    raw = [fn.__name__, fn.__module__, args, kwargs]
    m = hashlib.md5()
    m.update('-'.join([x.__repr__() for x in raw]))
    key = m.hexdigest()

    if not once_per_request:
        # we want to run it once for every occurrence, add a random string
        key = '{}:{}'.format(key, binascii.hexlify(os.urandom(8)))
    postcommit_queue().update({key: functools.partial(fn, *args, **kwargs)})

handlers = {
    'before_request': postcommit_before_request,
    'after_request': postcommit_after_request,
}

def run_postcommit(once_per_request=True, celery=False):
    '''
    Delays function execution until after the request's transaction has been committed.
    !!!Tasks enqueued using this decorator **WILL NOT** run if the return status code is >= 500!!!
    Unless celery is marked True, then they run any way
    :return:
    '''
    def wrapper(func):
        # if we're local dev or running unit tests, run without queueing
        if settings.DEBUG_MODE:
            return func
        @functools.wraps(func)
        def wrapped(*args, **kwargs):
            if celery is True and isinstance(func, PromiseProxy):
                func.delay(*args, **kwargs)
            else:
                enqueue_postcommit_task(func, args, kwargs, once_per_request=once_per_request)
        return wrapped
    return wrapper
<commit_msg>Adding os import that would have made everything fail<commit_after># -*- coding: utf-8 -*-
import functools
import hashlib
import logging
import threading
import binascii
from collections import OrderedDict
import os

from celery.local import PromiseProxy
from gevent.pool import Pool

from website import settings

_local = threading.local()
logger = logging.getLogger(__name__)

def postcommit_queue():
    if not hasattr(_local, 'postcommit_queue'):
        _local.postcommit_queue = OrderedDict()
    return _local.postcommit_queue

def postcommit_before_request():
    _local.postcommit_queue = OrderedDict()

def postcommit_after_request(response, base_status_error_code=500):
    if response.status_code >= base_status_error_code:
        _local.postcommit_queue = OrderedDict()
        return response
    try:
        if postcommit_queue():
            number_of_threads = 30  # one db connection per greenlet, let's share
            pool = Pool(number_of_threads)
            for func in postcommit_queue().values():
                pool.spawn(func)
            pool.join(timeout=5.0, raise_error=True)  # 5 second timeout and reraise exceptions
    except AttributeError as ex:
        if not settings.DEBUG_MODE:
            logger.error('Post commit task queue not initialized: {}'.format(ex))
    return response

def enqueue_postcommit_task(fn, args, kwargs, once_per_request=True):
    # make a hash of the pertinent data
    raw = [fn.__name__, fn.__module__, args, kwargs]
    m = hashlib.md5()
    m.update('-'.join([x.__repr__() for x in raw]))
    key = m.hexdigest()

    if not once_per_request:
        # we want to run it once for every occurrence, add a random string
        key = '{}:{}'.format(key, binascii.hexlify(os.urandom(8)))
    postcommit_queue().update({key: functools.partial(fn, *args, **kwargs)})

handlers = {
    'before_request': postcommit_before_request,
    'after_request': postcommit_after_request,
}

def run_postcommit(once_per_request=True, celery=False):
    '''
    Delays function execution until after the request's transaction has been committed.
    !!!Tasks enqueued using this decorator **WILL NOT** run if the return status code is >= 500!!!
    Unless celery is marked True, then they run any way
    :return:
    '''
    def wrapper(func):
        # if we're local dev or running unit tests, run without queueing
        if settings.DEBUG_MODE:
            return func
        @functools.wraps(func)
        def wrapped(*args, **kwargs):
            if celery is True and isinstance(func, PromiseProxy):
                func.delay(*args, **kwargs)
            else:
                enqueue_postcommit_task(func, args, kwargs, once_per_request=once_per_request)
        return wrapped
    return wrapper
<|endoftext|>