<commit_before>#__BEGIN_LICENSE__
# Copyright (c) 2015, United States Government, as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All rights reserved.
#
# The xGDS platform is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0.
#
# Unless required by applicable law or agreed to in writing, software distributed
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.
#__END_LICENSE__

import datetime
import pytz

from django import forms
from django.conf import settings
from django.core.urlresolvers import reverse
from django.forms.fields import IntegerField, ChoiceField

from resumable.fields import ResumableFileField

from xgds_map_server.models import KmlMap, MapGroup, MapLayer, MapTile, MapCollection, MapSearch, MapDataTile
from xgds_data.models import Collection, RequestLog

# pylint: disable=C1001

class AbstractMapForm(forms.ModelForm):
    parent = forms.ModelChoiceField(queryset=MapGroup.objects.filter(deleted=False), empty_label=None, label="Folder")
    username = forms.CharField(required=False, widget=forms.HiddenInput())

    def getModel(self):
        return None

    def getExclude(self):
        return ['creator', 'modifier', 'creation_time', 'modification_time', 'deleted']

    def getParentGroup(self):
        mapGroupName = self.cleaned_data['parent']
        foundParents = MapGroup.objects.filter(name=mapGroupName)
        return foundParents[0]

    class Meta:
        abstract = True
#         model = self.getModel()
        widgets = {
            # Note: no practical way to retrieve max lengths from Map model
            'name': forms.TextInput(attrs={'size': 50}),
            'description': forms.Textarea(attrs={'cols': 50, 'rows': 7})
        }
        exclude = ['creator', 'modifier', 'creation_time', 'modification_time', 'deleted']

class MapForm(AbstractMapForm):
    class Meta(AbstractMapForm.Meta):
        model = KmlMap

class MapGroupForm(AbstractMapForm):
    class Meta(AbstractMapForm.Meta):
        model = MapGroup

class MapLayerForm(AbstractMapForm):
    class Meta(AbstractMapForm.Meta):
        model = MapLayer
        exclude = ['minLat', 'minLon', 'maxLat', 'maxLon', 'jsonFeatures', 'creator', 'modifier', 'creation_time', 'modification_time', 'deleted', 'defaultColor']

class MapLayerFromSelectedForm(AbstractMapForm):
    jsonFeatures = forms.CharField(widget=forms.HiddenInput())

    class Meta(AbstractMapForm.Meta):
        model = MapLayer
        exclude = ['minLat', 'minLon', 'maxLat', 'maxLon', 'creator', 'modifier', 'creation_time', 'modification_time', 'deleted', 'defaultColor']

class MapTileForm(AbstractMapForm):
    if settings.XGDS_MAP_SERVER_GDAL2TILES_ZOOM_LEVELS:
        minZoom = IntegerField(initial=12, label="Min Zoom")
        maxZoom = IntegerField(initial=20, label="Max Zoom", help_text="(UAV=23, Satellite=20)")

    resampleMethod = ChoiceField(choices=settings.XGDS_MAP_SERVER_GDAL_RESAMPLE_OPTIONS,
                                 label="Resampling")

    sourceFile = ResumableFileField(allowed_mimes=("image/tiff",),
                                    upload_url=lambda: reverse('uploadResumable'),
                                    chunks_dir=getattr(settings, 'FILE_UPLOAD_TEMP_DIR'),
                                    label="GeoTiff File"
                                    )

    def save(self, commit=True):
        instance = super(MapTileForm, self).save(commit=False)
        if not instance.creator:
            instance.creator = self.cleaned_data['username']
            instance.creation_time = datetime.datetime.now(pytz.utc)
        else:
            instance.modifier = self.cleaned_data['username']
            instance.modification_time = self.cleaned_data['username']
        instance.parent = self.getParentGroup()
        if commit:
            instance.save()

    class Meta(AbstractMapForm.Meta):
        model = MapTile
        exclude = ['creator', 'modifier', 'creation_time', 'modification_time', 'deleted', 'processed', 'minx', 'miny', 'maxx', 'maxy', 'resolutions']

class MapDataTileForm(MapTileForm):
    dataFile = ResumableFileField(allowed_mimes=("image/png",),
                                  upload_url=lambda: reverse('uploadResumable'),
                                  chunks_dir=getattr(settings, 'FILE_UPLOAD_TEMP_DIR'),
                                  label="Data File (png)"
                                  )

    legendFile = ResumableFileField(allowed_mimes=("image/png",),
                                  upload_url=lambda: reverse('uploadResumable'),
                                  chunks_dir=getattr(settings, 'FILE_UPLOAD_TEMP_DIR'),
                                  label="Legend File (png, vertical)"
                                  )

    class Meta(AbstractMapForm.Meta):
        model = MapDataTile
        exclude = ['creator', 'modifier', 'creation_time', 'modification_time', 'deleted', 'processed', 'minx', 'miny', 'maxx', 'maxy', 'resolutions']

class EditMapTileForm(AbstractMapForm):

    class Meta(AbstractMapForm.Meta):
        model = MapTile
        exclude = ['creator', 'modifier', 'creation_time', 'modification_time', 'deleted', 'processed', 'sourceFile', 'minx', 'miny', 'maxx', 'maxy', 'resolutions']

class EditMapDataTileForm(EditMapTileForm):
    class Meta(AbstractMapForm.Meta):
        model = MapDataTile
        exclude = ['creator', 'modifier', 'creation_time', 'modification_time', 'deleted', 'processed', 'sourceFile', 'minx', 'miny', 'maxx', 'maxy', 'resolutions']

class CollectionModelChoiceField(forms.ModelChoiceField):
    def label_from_instance(self, obj):
        return obj.name

class MapCollectionForm(AbstractMapForm):
    collection = CollectionModelChoiceField(queryset=Collection.objects.all().order_by('name'), empty_label=None)

    class Meta(AbstractMapForm.Meta):
        model = MapCollection

class MapSearchForm(AbstractMapForm):

    class Meta(AbstractMapForm.Meta):
        model = MapSearch
        exclude = ['creator', 'modifier', 'creation_time', 'modification_time', 'deleted', 'requestLog', 'locked', 'visible', 'mapBounded']

<commit_msg>updated forms to not show min/max lat long<commit_after>#__BEGIN_LICENSE__
# Copyright (c) 2015, United States Government, as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All rights reserved.
#
# The xGDS platform is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0.
#
# Unless required by applicable law or agreed to in writing, software distributed
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.
#__END_LICENSE__

import datetime
import pytz

from django import forms
from django.conf import settings
from django.core.urlresolvers import reverse
from django.forms.fields import IntegerField, ChoiceField

from resumable.fields import ResumableFileField

from xgds_map_server.models import KmlMap, MapGroup, MapLayer, MapTile, MapCollection, MapSearch, MapDataTile
from xgds_data.models import Collection, RequestLog

# pylint: disable=C1001

class AbstractMapForm(forms.ModelForm):
    parent = forms.ModelChoiceField(queryset=MapGroup.objects.filter(deleted=False), empty_label=None, label="Folder")
    username = forms.CharField(required=False, widget=forms.HiddenInput())

    def getModel(self):
        return None

    def getExclude(self):
        return ['creator', 'modifier', 'creation_time', 'modification_time', 'deleted', 'minLat', 'minLon', 'maxLat', 'maxLon']

    def getParentGroup(self):
        mapGroupName = self.cleaned_data['parent']
        foundParents = MapGroup.objects.filter(name=mapGroupName)
        return foundParents[0]

    class Meta:
        abstract = True
#         model = self.getModel()
        widgets = {
            # Note: no practical way to retrieve max lengths from Map model
            'name': forms.TextInput(attrs={'size': 50}),
            'description': forms.Textarea(attrs={'cols': 50, 'rows': 7})
        }
        exclude = ['creator', 'modifier', 'creation_time', 'modification_time', 'deleted', 'minLat', 'minLon', 'maxLat', 'maxLon']

class MapForm(AbstractMapForm):
    class Meta(AbstractMapForm.Meta):
        model = KmlMap

class MapGroupForm(AbstractMapForm):
    class Meta(AbstractMapForm.Meta):
        model = MapGroup

class MapLayerForm(AbstractMapForm):
    class Meta(AbstractMapForm.Meta):
        model = MapLayer
        exclude = ['minLat', 'minLon', 'maxLat', 'maxLon', 'jsonFeatures', 'creator', 'modifier', 'creation_time', 'modification_time', 'deleted', 'defaultColor']

class MapLayerFromSelectedForm(AbstractMapForm):
    jsonFeatures = forms.CharField(widget=forms.HiddenInput())

    class Meta(AbstractMapForm.Meta):
        model = MapLayer
        exclude = ['minLat', 'minLon', 'maxLat', 'maxLon', 'creator', 'modifier', 'creation_time', 'modification_time', 'deleted', 'defaultColor']

class MapTileForm(AbstractMapForm):
    if settings.XGDS_MAP_SERVER_GDAL2TILES_ZOOM_LEVELS:
        minZoom = IntegerField(initial=12, label="Min Zoom")
        maxZoom = IntegerField(initial=20, label="Max Zoom", help_text="(UAV=23, Satellite=20)")

    resampleMethod = ChoiceField(choices=settings.XGDS_MAP_SERVER_GDAL_RESAMPLE_OPTIONS,
                                 label="Resampling")

    sourceFile = ResumableFileField(allowed_mimes=("image/tiff",),
                                    upload_url=lambda: reverse('uploadResumable'),
                                    chunks_dir=getattr(settings, 'FILE_UPLOAD_TEMP_DIR'),
                                    label="GeoTiff File"
                                    )

    def save(self, commit=True):
        instance = super(MapTileForm, self).save(commit=False)
        if not instance.creator:
            instance.creator = self.cleaned_data['username']
            instance.creation_time = datetime.datetime.now(pytz.utc)
        else:
            instance.modifier = self.cleaned_data['username']
            instance.modification_time = self.cleaned_data['username']
        instance.parent = self.getParentGroup()
        if commit:
            instance.save()

    class Meta(AbstractMapForm.Meta):
        model = MapTile
        exclude = ['creator', 'modifier', 'creation_time', 'modification_time', 'deleted', 'processed', 'minx', 'miny', 'maxx', 'maxy', 'resolutions', 'minLat', 'minLon', 'maxLat', 'maxLon']

class MapDataTileForm(MapTileForm):
    dataFile = ResumableFileField(allowed_mimes=("image/png",),
                                  upload_url=lambda: reverse('uploadResumable'),
                                  chunks_dir=getattr(settings, 'FILE_UPLOAD_TEMP_DIR'),
                                  label="Data File (png)"
                                  )

    legendFile = ResumableFileField(allowed_mimes=("image/png",),
                                  upload_url=lambda: reverse('uploadResumable'),
                                  chunks_dir=getattr(settings, 'FILE_UPLOAD_TEMP_DIR'),
                                  label="Legend File (png, vertical)"
                                  )

    class Meta(AbstractMapForm.Meta):
        model = MapDataTile
        exclude = ['creator', 'modifier', 'creation_time', 'modification_time', 'deleted', 'processed', 'minx', 'miny', 'maxx', 'maxy', 'resolutions']

class EditMapTileForm(AbstractMapForm):

    class Meta(AbstractMapForm.Meta):
        model = MapTile
        exclude = ['creator', 'modifier', 'creation_time', 'modification_time', 'deleted', 'processed', 'sourceFile', 'minx', 'miny', 'maxx', 'maxy', 'resolutions']

class EditMapDataTileForm(EditMapTileForm):
    class Meta(AbstractMapForm.Meta):
        model = MapDataTile
        exclude = ['creator', 'modifier', 'creation_time', 'modification_time', 'deleted', 'processed', 'sourceFile', 'minx', 'miny', 'maxx', 'maxy', 'resolutions']

class CollectionModelChoiceField(forms.ModelChoiceField):
    def label_from_instance(self, obj):
        return obj.name

class MapCollectionForm(AbstractMapForm):
    collection = CollectionModelChoiceField(queryset=Collection.objects.all().order_by('name'), empty_label=None)

    class Meta(AbstractMapForm.Meta):
        model = MapCollection

class MapSearchForm(AbstractMapForm):

    class Meta(AbstractMapForm.Meta):
        model = MapSearch
        exclude = ['creator', 'modifier', 'creation_time', 'modification_time', 'deleted', 'requestLog', 'locked', 'visible', 'mapBounded']

<|endoftext|>


<commit_before>#
# Executes the rendering jobs on the cluster (based on Rhoana's driver).
# It takes a collection of tilespec files, each describing a montage of a single section,
# normalizaes each section, and renders its 2d image.
# The input is a directory with tilespec files in json format (each file for a single layer),
# and a workspace directory where the intermediate and result files will be located.
# Renders the full images as squared tiles in full resolution.
#

import sys
import os.path
import os
import subprocess
import datetime
import time
from itertools import product
from collections import defaultdict
import argparse
import glob
import json
from utils import path2url, create_dir, read_layer_from_file, parse_range
from job import Job

class NormalizeCoordinates(Job):
    def __init__(self, tiles_fname, output_dir, output_file, jar_file):
        Job.__init__(self)
        self.already_done = False
        self.tiles_fname = '"{0}"'.format(tiles_fname)
        self.output_dir = '-o "{0}"'.format(output_dir)
        self.jar_file = '-j "{0}"'.format(jar_file)
        self.dependencies = []
        self.memory = 3000
        self.time = 10
        self.is_java_job = True
        self.output = output_file
        #self.already_done = os.path.exists(self.output_file)

    def command(self):
        return ['python -u',
                os.path.join(os.environ['ALIGNER'], 'scripts', 'normalize_coordinates.py'),
                self.output_dir, self.jar_file, self.tiles_fname]

class RenderTiles2D(Job):
    def __init__(self, dependencies, tiles_fname, out_dir, tile_size, output_type, jar_file, output_pattern=None, blend_type=None, threads_num=1):
        Job.__init__(self)
        self.already_done = False
        self.tiles_fname = '"{0}"'.format(tiles_fname)
        self.output_dir = '-o "{0}"'.format(out_dir)
        self.jar_file = '-j "{0}"'.format(jar_file)
        self.tile_size = '-s {0}'.format(tile_size)
        self.threads = threads_num
        self.threads_str = '-t {0}'.format(threads_num)
        if blend_type is None:
            self.blend_type = ''
        else:
            self.blend_type = '-b {0}'.format(blend_type)
        self.output_type = '--output_type "{0}"'.format(output_type)
        if output_pattern is None:
            self.output_pattern = ''
        else:
            self.output_pattern = '--output_pattern "{0}"'.format(output_pattern)
        self.dependencies = dependencies
        self.memory = 14000
        self.time = 600
        self.is_java_job = True
        self.output = out_dir
        #self.already_done = os.path.exists(self.output_file)

    def command(self):
        return ['python -u',
                os.path.join(os.environ['ALIGNER'], 'scripts', 'render_tiles_2d.py'),
                self.output_dir, self.jar_file, self.tile_size, self.blend_type, self.output_type, self.output_pattern, self.threads_str, self.tiles_fname]

class CreateZoomedTiles(Job):
    def __init__(self, dependencies, in_dir, open_sea_dragon, processes_num=1):
        Job.__init__(self)
        self.already_done = False
        self.in_dir = '"{0}"'.format(in_dir)
        self.open_sea_dragon = ''
        if open_sea_dragon is True:
            self.open_sea_dragon = '-s'
        self.dependencies = dependencies
        self.threads = processes_num
        self.processes_str = '-p {0}'.format(processes_num)
        self.memory = 16000
        self.time = 500
        self.is_java_job = False
        self.output = os.path.join(in_dir, "1")
        #self.already_done = os.path.exists(self.output_file)

    def command(self):
        return ['python -u',
                os.path.join(os.environ['ALIGNER'], 'scripts', 'create_zoomed_tiles.py'),
                self.open_sea_dragon, self.processes_str, self.in_dir]

###############################
# Driver
###############################
if __name__ == '__main__':

    # Command line parser
    parser = argparse.ArgumentParser(description='Aligns (2d-elastic montaging) a given set of images using the SLURM cluster commands.')
    parser.add_argument('tiles_dir', metavar='tiles_dir', type=str,
                        help='a directory that contains a tile_spec files in json format')
    parser.add_argument('-w', '--workspace_dir', type=str,
                        help='a directory where the output files of the different stages will be kept (default: ./temp)',
                        default='./temp')
    parser.add_argument('-o', '--output_dir', type=str,
                        help='the directory where the output to be rendered in json format files will be stored (default: ./output)',
                        default='./output')
    parser.add_argument('-j', '--jar_file', type=str,
                        help='the jar file that includes the render (default: ../target/render-0.0.1-SNAPSHOT.jar)',
                        default='../target/render-0.0.1-SNAPSHOT.jar')
    parser.add_argument('-i', '--tile_size', type=int,
                        help='the size (square side) of each tile (default: 512)',
                        default=512)
    parser.add_argument('-s', '--skip_layers', type=str,
                        help='the range of layers (sections) that will not be processed e.g., "2,3,9-11,18" (default: no skipped sections)',
                        default=None)
    parser.add_argument('--avoid_mipmaps', action="store_true",
                        help='Do not create mipmaps after the full scale tiling')
    parser.add_argument('-b', '--blend_type', type=str,
                        help='the mosaics blending type',
                        default=None)
    parser.add_argument('--output_type', type=str,
                        help='The output type format',
                        default='jpg')
    parser.add_argument('--output_pattern', type=str,
                        help='The output file name pattern where "%row%col" will be replaced by "_tr[row]-tc[rol]_" with the row and column numbers',
                        default=None)
    parser.add_argument('-k', '--keeprunning', action='store_true',
                        help='Run all jobs and report cluster jobs execution stats')
    parser.add_argument('-m', '--multicore', action='store_true',
                        help='Run all jobs in blocks on multiple cores')
    parser.add_argument('-mk', '--multicore_keeprunning', action='store_true',
                        help='Run all jobs in blocks on multiple cores and report cluster jobs execution stats')

    args = parser.parse_args()

    assert 'ALIGNER' in os.environ
    #assert 'VIRTUAL_ENV' in os.environ

    # create a workspace directory if not found
    create_dir(args.workspace_dir)

    norm_dir = os.path.join(args.workspace_dir, "normalized")
    create_dir(norm_dir)
    create_dir(args.output_dir)

    # Get all input json files (one per section) into a dictionary {json_fname -> [filtered json fname, sift features file, etc.]}
    json_files = dict((jf, {}) for jf in (glob.glob(os.path.join(args.tiles_dir, '*.json'))))

    skipped_layers = parse_range(args.skip_layers)

    for f in json_files.keys():
        tiles_fname_basename = os.path.basename(f)
        tiles_fname_prefix = os.path.splitext(tiles_fname_basename)[0]

        # read the layer from the file
        layer = read_layer_from_file(f)

        # Check if we need to skip the layer
        if layer in skipped_layers:
            print "Skipping layer {}".format(layer)
            continue

        slayer = str(layer)

        job_normalize = None
        job_render = None
        job_tile = None

        # Normalize the json file
        norm_json = os.path.join(norm_dir, tiles_fname_basename)
        if not os.path.exists(norm_json):
            print "Normalizing layer: {0}".format(slayer)
            job_normalize = NormalizeCoordinates(f, norm_dir, norm_json, args.jar_file)

        # Render the normalized json file into tiles
        out_pattern = args.output_pattern
        if out_pattern is None:
            out_pattern = '{}%rowcolmontaged'.format(tiles_fname_prefix)

        out_dir = os.path.join(args.output_dir, out_pattern)
        create_dir(out_dir)
        out_0_dir = os.path.join(out_dir, "0")
        if not os.path.exists(out_0_dir):
            dependencies = [ ]
            if job_normalize != None:
                dependencies.append(job_normalize)
            job_render = RenderTiles2D(dependencies, norm_json, out_0_dir, args.tile_size, args.output_type,
                args.jar_file, output_pattern=out_pattern, blend_type=args.blend_type, threads_num=32)

        # Create zoomed tiles
        if not args.avoid_mipmaps:
            out_1_dir = os.path.join(out_dir, "1")
            if not os.path.exists(out_1_dir):
                dependencies = [ ]
                if job_normalize != None:
                    dependencies.append(job_normalize)
                if job_render != None:
                    dependencies.append(job_render)
                job_tile = CreateZoomedTiles(dependencies, out_dir, True, processes_num=16)

    # Run all jobs
    if args.keeprunning:
        Job.keep_running()
    elif args.multicore:
        # Bundle jobs for multicore nodes
        # if RUN_LOCAL:
        #     print "ERROR: --local cannot be used with --multicore (not yet implemented)."
        #     sys.exit(1)
        Job.multicore_run_all()
    elif args.multicore_keeprunning:
        # Bundle jobs for multicore nodes
        Job.multicore_keep_running()
    else:
        Job.run_all()

<commit_msg>Fixing output dir<commit_after>#
# Executes the rendering jobs on the cluster (based on Rhoana's driver).
# It takes a collection of tilespec files, each describing a montage of a single section,
# normalizaes each section, and renders its 2d image.
# The input is a directory with tilespec files in json format (each file for a single layer),
# and a workspace directory where the intermediate and result files will be located.
# Renders the full images as squared tiles in full resolution.
#

import sys
import os.path
import os
import subprocess
import datetime
import time
from itertools import product
from collections import defaultdict
import argparse
import glob
import json
from utils import path2url, create_dir, read_layer_from_file, parse_range
from job import Job

class NormalizeCoordinates(Job):
    def __init__(self, tiles_fname, output_dir, output_file, jar_file):
        Job.__init__(self)
        self.already_done = False
        self.tiles_fname = '"{0}"'.format(tiles_fname)
        self.output_dir = '-o "{0}"'.format(output_dir)
        self.jar_file = '-j "{0}"'.format(jar_file)
        self.dependencies = []
        self.memory = 3000
        self.time = 10
        self.is_java_job = True
        self.output = output_file
        #self.already_done = os.path.exists(self.output_file)

    def command(self):
        return ['python -u',
                os.path.join(os.environ['ALIGNER'], 'scripts', 'normalize_coordinates.py'),
                self.output_dir, self.jar_file, self.tiles_fname]

class RenderTiles2D(Job):
    def __init__(self, dependencies, tiles_fname, out_dir, tile_size, output_type, jar_file, output_pattern=None, blend_type=None, threads_num=1):
        Job.__init__(self)
        self.already_done = False
        self.tiles_fname = '"{0}"'.format(tiles_fname)
        self.output_dir = '-o "{0}"'.format(out_dir)
        self.jar_file = '-j "{0}"'.format(jar_file)
        self.tile_size = '-s {0}'.format(tile_size)
        self.threads = threads_num
        self.threads_str = '-t {0}'.format(threads_num)
        if blend_type is None:
            self.blend_type = ''
        else:
            self.blend_type = '-b {0}'.format(blend_type)
        self.output_type = '--output_type "{0}"'.format(output_type)
        if output_pattern is None:
            self.output_pattern = ''
        else:
            self.output_pattern = '--output_pattern "{0}"'.format(output_pattern)
        self.dependencies = dependencies
        self.memory = 14000
        self.time = 600
        self.is_java_job = True
        self.output = out_dir
        #self.already_done = os.path.exists(self.output_file)

    def command(self):
        return ['python -u',
                os.path.join(os.environ['ALIGNER'], 'scripts', 'render_tiles_2d.py'),
                self.output_dir, self.jar_file, self.tile_size, self.blend_type, self.output_type, self.output_pattern, self.threads_str, self.tiles_fname]

class CreateZoomedTiles(Job):
    def __init__(self, dependencies, in_dir, open_sea_dragon, processes_num=1):
        Job.__init__(self)
        self.already_done = False
        self.in_dir = '"{0}"'.format(in_dir)
        self.open_sea_dragon = ''
        if open_sea_dragon is True:
            self.open_sea_dragon = '-s'
        self.dependencies = dependencies
        self.threads = processes_num
        self.processes_str = '-p {0}'.format(processes_num)
        self.memory = 16000
        self.time = 500
        self.is_java_job = False
        self.output = os.path.join(in_dir, "1")
        #self.already_done = os.path.exists(self.output_file)

    def command(self):
        return ['python -u',
                os.path.join(os.environ['ALIGNER'], 'scripts', 'create_zoomed_tiles.py'),
                self.open_sea_dragon, self.processes_str, self.in_dir]

###############################
# Driver
###############################
if __name__ == '__main__':

    # Command line parser
    parser = argparse.ArgumentParser(description='Aligns (2d-elastic montaging) a given set of images using the SLURM cluster commands.')
    parser.add_argument('tiles_dir', metavar='tiles_dir', type=str,
                        help='a directory that contains a tile_spec files in json format')
    parser.add_argument('-w', '--workspace_dir', type=str,
                        help='a directory where the output files of the different stages will be kept (default: ./temp)',
                        default='./temp')
    parser.add_argument('-o', '--output_dir', type=str,
                        help='the directory where the output to be rendered in json format files will be stored (default: ./output)',
                        default='./output')
    parser.add_argument('-j', '--jar_file', type=str,
                        help='the jar file that includes the render (default: ../target/render-0.0.1-SNAPSHOT.jar)',
                        default='../target/render-0.0.1-SNAPSHOT.jar')
    parser.add_argument('-i', '--tile_size', type=int,
                        help='the size (square side) of each tile (default: 512)',
                        default=512)
    parser.add_argument('-s', '--skip_layers', type=str,
                        help='the range of layers (sections) that will not be processed e.g., "2,3,9-11,18" (default: no skipped sections)',
                        default=None)
    parser.add_argument('--avoid_mipmaps', action="store_true",
                        help='Do not create mipmaps after the full scale tiling')
    parser.add_argument('-b', '--blend_type', type=str,
                        help='the mosaics blending type',
                        default=None)
    parser.add_argument('--output_type', type=str,
                        help='The output type format',
                        default='jpg')
    parser.add_argument('--output_pattern', type=str,
                        help='The output file name pattern where "%row%col" will be replaced by "_tr[row]-tc[rol]_" with the row and column numbers',
                        default=None)
    parser.add_argument('-k', '--keeprunning', action='store_true',
                        help='Run all jobs and report cluster jobs execution stats')
    parser.add_argument('-m', '--multicore', action='store_true',
                        help='Run all jobs in blocks on multiple cores')
    parser.add_argument('-mk', '--multicore_keeprunning', action='store_true',
                        help='Run all jobs in blocks on multiple cores and report cluster jobs execution stats')

    args = parser.parse_args()

    assert 'ALIGNER' in os.environ
    #assert 'VIRTUAL_ENV' in os.environ

    # create a workspace directory if not found
    create_dir(args.workspace_dir)

    norm_dir = os.path.join(args.workspace_dir, "normalized")
    create_dir(norm_dir)
    create_dir(args.output_dir)

    # Get all input json files (one per section) into a dictionary {json_fname -> [filtered json fname, sift features file, etc.]}
    json_files = dict((jf, {}) for jf in (glob.glob(os.path.join(args.tiles_dir, '*.json'))))

    skipped_layers = parse_range(args.skip_layers)

    for f in json_files.keys():
        tiles_fname_basename = os.path.basename(f)
        tiles_fname_prefix = os.path.splitext(tiles_fname_basename)[0]

        # read the layer from the file
        layer = read_layer_from_file(f)

        # Check if we need to skip the layer
        if layer in skipped_layers:
            print "Skipping layer {}".format(layer)
            continue

        slayer = str(layer)

        job_normalize = None
        job_render = None
        job_tile = None

        # Normalize the json file
        norm_json = os.path.join(norm_dir, tiles_fname_basename)
        if not os.path.exists(norm_json):
            print "Normalizing layer: {0}".format(slayer)
            job_normalize = NormalizeCoordinates(f, norm_dir, norm_json, args.jar_file)

        # Render the normalized json file into tiles
        out_pattern = args.output_pattern
        if out_pattern is None:
            out_pattern = '{}%rowcolmontaged'.format(tiles_fname_prefix)

        out_dir = os.path.join(args.output_dir, tiles_fname_prefix)
        create_dir(out_dir)
        out_0_dir = os.path.join(out_dir, "0")
        if not os.path.exists(out_0_dir):
            dependencies = [ ]
            if job_normalize != None:
                dependencies.append(job_normalize)
            job_render = RenderTiles2D(dependencies, norm_json, out_0_dir, args.tile_size, args.output_type,
                args.jar_file, output_pattern=out_pattern, blend_type=args.blend_type, threads_num=32)

        # Create zoomed tiles
        if not args.avoid_mipmaps:
            out_1_dir = os.path.join(out_dir, "1")
            if not os.path.exists(out_1_dir):
                dependencies = [ ]
                if job_normalize != None:
                    dependencies.append(job_normalize)
                if job_render != None:
                    dependencies.append(job_render)
                job_tile = CreateZoomedTiles(dependencies, out_dir, True, processes_num=16)

    # Run all jobs
    if args.keeprunning:
        Job.keep_running()
    elif args.multicore:
        # Bundle jobs for multicore nodes
        # if RUN_LOCAL:
        #     print "ERROR: --local cannot be used with --multicore (not yet implemented)."
        #     sys.exit(1)
        Job.multicore_run_all()
    elif args.multicore_keeprunning:
        # Bundle jobs for multicore nodes
        Job.multicore_keep_running()
    else:
        Job.run_all()

<|endoftext|>


<commit_before># Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Utility classes to generate and manage a BuildFactory to be passed to a
builder dictionary as the 'factory' member, for each builder in c['builders'].

Specifically creates a base BuildFactory that will execute a gclient checkout
first."""

import os
import re

from master.factory.build_factory import BuildFactory
from master.factory import commands

import config

def ShouldRunTest(tests, name):
  """Returns True if |name| is an entry in |tests|."""
  if not tests:
    return False

  if name in tests:
    return True
  return False

def ShouldRunMatchingTest(tests, pattern):
  """Returns True if regex |pattern| matches an entry in |tests|."""
  if not tests:
    return False

  for test in tests:
    if re.match(pattern, test):
      return True
  return False

class GClientSolution(object):
  """Defines a GClient solution."""

  def __init__(self, svn_url, name=None, custom_deps_list=None,
               needed_components=None, custom_vars_list=None,
               custom_deps_file=None, safesync_url=None,
               managed=None):
    """ Initialize the GClient Solution.
    Params:
      svn_url: SVN path for this solution.
      name: Name for this solution. if None, it uses the last item in the path.
      custom_deps_list: Modifications to make on the DEPS file.
      needed_components: A map used to skip dependencies when a test is not run.
          The map key is the test name. The map value is an array containing the
          dependencies that are not needed when this test is not run.
      custom_vars_list: Modifications to make on the vars in the DEPS file.
      custom_deps_file: Change the default DEPS filename.
      safesync_url: Select to build based on a lkgr url.
      managed: Specify managed in .gclient file
    """
    self.svn_url = svn_url
    self.name = name
    self.custom_deps_list = (custom_deps_list or [])[:]
    self.custom_vars_list = (custom_vars_list or [])[:]
    self.custom_deps_file = custom_deps_file
    self.needed_components = (needed_components or {}).copy()
    self.safesync_url = safesync_url
    self.managed = managed

    if not self.name:
      self.name = svn_url.split('/')[-1]

  def GetSpec(self, tests=None):
    """Returns the specs for this solution.
    Params:
      tests: List of tests to run. This is required only when needed_components
             is not None.
    """

    final_custom_deps_list = self.custom_deps_list[:]
    # Extend the custom deps with everything that is not going to be used
    # in this factory
    if self.needed_components:
      for test, dependencies in self.needed_components.iteritems():
        if ShouldRunMatchingTest(tests, test):
          continue
        final_custom_deps_list.extend(dependencies)

    # Create the custom_deps string
    custom_deps = ''
    for dep in final_custom_deps_list:
      if dep[1] is None:
        dep_url = None
      else:
        dep_url = '"%s"' % dep[1]
      custom_deps += '"%s" : %s, ' % (dep[0], dep_url)

    custom_vars = ''
    for var in self.custom_vars_list:
      if var[1] is None:
        var_value = None
      else:
        var_value = '"%s"' % var[1]
      custom_vars += '"%s" : %s, ' % (var[0], var_value)

    extras = ''
    if self.custom_deps_file:
      extras += '"deps_file": "%s",' % self.custom_deps_file
    if self.safesync_url:
      extras += '"safesync_url": "%s"' % self.safesync_url
    if self.managed:
      extras += '"managed": %s' % self.managed

    # This must not contain any line breaks or other characters that would
    # require escaping on the command line, since it will be passed to gclient.
    spec = (
      '{ "name": "%s", '
        '"url": "%s", '
        '"custom_deps": {'
                          '%s'
                       '},'
        '"custom_vars": {'
                          '%s'
                       '},'
        '%s'
      '},' % (self.name, self.svn_url, custom_deps, custom_vars, extras)
    )
    return spec

class GClientFactory(object):
  """Encapsulates data and methods common to both (all) master.cfg files."""

  def __init__(self, build_dir, solutions, target_platform=None,
               nohooks_on_update=False, target_os=None, revision_mapping=None):
    self._build_dir = build_dir
    self._solutions = solutions
    self._target_platform = target_platform or 'win32'
    self._target_os = target_os
    self._nohooks_on_update = nohooks_on_update
    self._revision_mapping = revision_mapping

  def BuildGClientSpec(self, tests=None):
    spec = 'solutions = ['
    for solution in self._solutions:
      spec += solution.GetSpec(tests)
    spec += ']'

    if self._target_os:
      spec += ';target_os = ["' + self._target_os + '"]'

    return spec

  def BaseFactory(self, gclient_spec=None, official_release=False,
                  factory_properties=None, build_properties=None,
                  delay_compile_step=False, sudo_for_remove=False,
                  gclient_deps=None, slave_type=None, options=None):
    if gclient_spec is None:
      gclient_spec = self.BuildGClientSpec()
    factory_properties = factory_properties or {}
    factory = BuildFactory(build_properties)
    factory_cmd_obj = commands.FactoryCommands(factory,
        target_platform=self._target_platform)
    # First kill any svn.exe tasks so we can update in peace, and
    # afterwards use the checked-out script to kill everything else.
    if (self._target_platform == 'win32' and
        not factory_properties.get('no_kill')):
      factory_cmd_obj.AddSvnKillStep()
    script_solutions = None
    if (factory_properties.get('goma_canary') and
        factory_properties.get('slave_internal_url')):
      script_solutions = [GClientSolution(
          factory_properties.get('slave_internal_url'),
          name="slave.DEPS",
          custom_vars_list=[("goma_linux_revision", "HEAD"),
                            ("goma_mac_revision", "HEAD"),
                            ("goma_win_revision", "HEAD")])]
    factory_cmd_obj.AddUpdateScriptStep(
        gclient_jobs=factory_properties.get('update_scripts_gclient_jobs'),
        solutions=script_solutions)
    # Once the script is updated, the zombie processes left by the previous
    # run can be killed.
    if (self._target_platform == 'win32' and
        not factory_properties.get('no_kill')):
      factory_cmd_obj.AddTaskkillStep()

    env = factory_properties.get('gclient_env', {})
    # Allow gclient_deps to also come from the factory_properties.
    if gclient_deps == None:
      gclient_deps = factory_properties.get('gclient_deps', None)

    if gclient_deps == 'ios':
      gclient_spec += ';target_os = [\'ios\'];target_os_only = True'

    # Do we need to operate in magic blink mode?
    blink_config = factory_properties.get('blink_config')

    # Force the build checkout to be at some revision.  This may or may not
    # activate depending on its own criteria, but the expectation is that if
    # this does activate, it will emit a BOT_UPDATED file in the build/
    # directory to signal to the other gclient update steps to no-op.
    code_review_site = config.Master.Master4.code_review_site
    factory_cmd_obj.AddBotUpdateStep(env, gclient_spec, self._revision_mapping,
                                     server=code_review_site,
                                     blink_config=blink_config)

    # svn timeout is 2 min; we allow 5
    timeout = factory_properties.get('gclient_timeout')
    if official_release or factory_properties.get('nuke_and_pave'):
      no_gclient_branch = factory_properties.get('no_gclient_branch', False)
      factory_cmd_obj.AddClobberTreeStep(gclient_spec, env, timeout,
          gclient_deps=gclient_deps, gclient_nohooks=self._nohooks_on_update,
          no_gclient_branch=no_gclient_branch, options=options)
    else:
      # Revert the tree to a clean (unmodified) state.
      factory_cmd_obj.AddGClientRevertStep()
      if not delay_compile_step:
        self.AddUpdateStep(gclient_spec, factory_properties, factory,
                           slave_type, sudo_for_remove,
                           gclient_deps=gclient_deps, options=options,
                           blink_config=blink_config)
    return factory

  def BuildFactory(self, target='Release', clobber=False, tests=None, mode=None,
                   slave_type='BuilderTester', options=None,
                   compile_timeout=1200, build_url=None, project=None,
                   factory_properties=None, gclient_deps=None,
                   target_arch=None, skip_archive_steps=False):
    if factory_properties is None:
      factory_properties = {}
    factory_properties.setdefault('gclient_env', {})
    gclient_env = factory_properties['gclient_env']
    if options and '--build-tool=ninja' in options:
      gclient_env['GYP_GENERATORS'] = 'ninja'

    # Create the spec for the solutions
    gclient_spec = self.BuildGClientSpec(tests)

    # Initialize the factory with the basic steps.
    factory = self.BaseFactory(gclient_spec,
                               factory_properties=factory_properties,
                               slave_type=slave_type,
                               gclient_deps=gclient_deps, options=options)

    # Optional repository root (default: 'src').
    repository_root = factory_properties.get('repository_root', 'src')

    # Get the factory command object to create new steps to the factory.
    factory_cmd_obj = commands.FactoryCommands(factory, target,
                                               self._build_dir,
                                               self._target_platform,
                                               target_arch,
                                               repository_root)

    # Update clang if necessary.
    if ('clang=1' in gclient_env.get('GYP_DEFINES', '') or
        (self._target_platform != 'win32' and
         factory_properties.get('asan'))):
      factory_cmd_obj.AddUpdateClangStep()

    # Add a step to cleanup temporary files and data left from a previous run
    # to prevent the drives from becoming full over time.
    factory_cmd_obj.AddTempCleanupStep()

    # Update the NaCl SDK if needed
    if factory_properties.get('update_nacl_sdk'):
      factory_cmd_obj.AddUpdateNaClSDKStep(
          factory_properties['update_nacl_sdk'])

    # Add the compile step if needed.
    if slave_type in ['BuilderTester', 'Builder', 'Trybot', 'Indexer',
                      'TrybotBuilder']:
      if self._target_platform == 'win32':
        # Look for a solution named for its enclosing directory.
        project = project or os.path.basename(self._build_dir) + '.sln'

      factory_cmd_obj.AddCompileStep(
          project,
          clobber,
          mode=mode,
          options=options,
          timeout=compile_timeout,
          env=factory_properties.get('compile_env'))

    # Generate synthetic user profiles. Must run before AddZipBuild().
    if factory_properties.get('create_profiles'):
      # pylint: disable=W0212
      factory_cmd_obj.AddProfileCreationStep('small_profile')

    if not skip_archive_steps:
      # Archive the full output directory if the machine is a builder.
      if slave_type in ['Builder', 'TrybotBuilder']:
        if build_url:
          # There are some builders that are classified as builders, but only
          # because we only need to see if the patch compiles, and not
          # actually because triggered testers needs the build. So if we don't
          # supply a build_url, we assume that the build artifact doesn't
          # need to be uploaded and we can bypass this step.
          factory_cmd_obj.AddZipBuild(build_url,
                                      factory_properties=factory_properties)

      # Download the full output directory if the machine is a tester.
      if slave_type in ['Tester', 'TrybotTester']:
        factory_cmd_obj.AddExtractBuild()

    return factory

  # pylint: disable=R0201
  def TriggerFactory(self, factory, slave_type, factory_properties):
    """Add post steps on a build created by BuildFactory."""
    # Trigger any schedulers waiting on the build to complete.
    factory_properties = factory_properties or {}
    if factory_properties.get('trigger') is None:
      return

    trigger_name = factory_properties.get('trigger')

    # Allow overwriting default values for specified properties.
    set_properties = factory_properties.get('trigger_set_properties', {})

    # Propagate properties to the children if this is set in the factory.
    copy_properties = factory_properties.get('trigger_properties', [])
    factory.addStep(commands.CreateTriggerStep(
        trigger_name=trigger_name,
        trigger_copy_properties=copy_properties,
        trigger_set_properties=set_properties))

  def AddUpdateStep(self, gclient_spec, factory_properties, factory,
                    slave_type, sudo_for_remove=False, gclient_deps=None,
                    options=None, blink_config=False):
    if gclient_spec is None:
      gclient_spec = self.BuildGClientSpec()
    factory_properties = factory_properties or {}

    # Get the factory command object to add update step to the factory.
    factory_cmd_obj = commands.FactoryCommands(factory,
        target_platform=self._target_platform)

    # Get variables needed for the update.
    env = factory_properties.get('gclient_env', {})
    timeout = factory_properties.get('gclient_timeout')

    no_gclient_branch = factory_properties.get('no_gclient_branch', False)
    no_gclient_revision = factory_properties.get('no_gclient_revision', False)

    gclient_transitive = factory_properties.get('gclient_transitive', False)
    primary_repo = factory_properties.get('primary_repo', '')
    gclient_jobs = factory_properties.get('gclient_jobs')

    # Do not run gyp_chromium on testers.
    if slave_type in ('Tester',):
      env.update({'GYP_CHROMIUM_NO_ACTION': '1'})

    # Add the update step.
    factory_cmd_obj.AddUpdateStep(
        gclient_spec,
        env=env,
        timeout=timeout,
        sudo_for_remove=sudo_for_remove,
        gclient_deps=gclient_deps,
        gclient_nohooks=True,
        no_gclient_branch=no_gclient_branch,
        no_gclient_revision=no_gclient_revision,
        gclient_transitive=gclient_transitive,
        primary_repo=primary_repo,
        gclient_jobs=gclient_jobs,
        blink_config=blink_config)

    if slave_type in ('AnnotatedTrybot', 'CrosTrybot', 'Trybot', 'Bisect',
                      'TrybotTester', 'TrybotBuilder'):
      factory_cmd_obj.AddApplyIssueStep(
          timeout=timeout,
          server=config.Master.Master4.code_review_site,
          revision_mapping=self._revision_mapping)

    if not self._nohooks_on_update:
      factory_cmd_obj.AddRunHooksStep(env=env, timeout=timeout, options=options)
<commit_msg>Add ZipBuild archive for both factory parameter and factory property<commit_after># Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Utility classes to generate and manage a BuildFactory to be passed to a
builder dictionary as the 'factory' member, for each builder in c['builders'].

Specifically creates a base BuildFactory that will execute a gclient checkout
first."""

import os
import re

from master.factory.build_factory import BuildFactory
from master.factory import commands

import config

def ShouldRunTest(tests, name):
  """Returns True if |name| is an entry in |tests|."""
  if not tests:
    return False

  if name in tests:
    return True
  return False

def ShouldRunMatchingTest(tests, pattern):
  """Returns True if regex |pattern| matches an entry in |tests|."""
  if not tests:
    return False

  for test in tests:
    if re.match(pattern, test):
      return True
  return False

class GClientSolution(object):
  """Defines a GClient solution."""

  def __init__(self, svn_url, name=None, custom_deps_list=None,
               needed_components=None, custom_vars_list=None,
               custom_deps_file=None, safesync_url=None,
               managed=None):
    """ Initialize the GClient Solution.
    Params:
      svn_url: SVN path for this solution.
      name: Name for this solution. if None, it uses the last item in the path.
      custom_deps_list: Modifications to make on the DEPS file.
      needed_components: A map used to skip dependencies when a test is not run.
          The map key is the test name. The map value is an array containing the
          dependencies that are not needed when this test is not run.
      custom_vars_list: Modifications to make on the vars in the DEPS file.
      custom_deps_file: Change the default DEPS filename.
      safesync_url: Select to build based on a lkgr url.
      managed: Specify managed in .gclient file
    """
    self.svn_url = svn_url
    self.name = name
    self.custom_deps_list = (custom_deps_list or [])[:]
    self.custom_vars_list = (custom_vars_list or [])[:]
    self.custom_deps_file = custom_deps_file
    self.needed_components = (needed_components or {}).copy()
    self.safesync_url = safesync_url
    self.managed = managed

    if not self.name:
      self.name = svn_url.split('/')[-1]

  def GetSpec(self, tests=None):
    """Returns the specs for this solution.
    Params:
      tests: List of tests to run. This is required only when needed_components
             is not None.
    """

    final_custom_deps_list = self.custom_deps_list[:]
    # Extend the custom deps with everything that is not going to be used
    # in this factory
    if self.needed_components:
      for test, dependencies in self.needed_components.iteritems():
        if ShouldRunMatchingTest(tests, test):
          continue
        final_custom_deps_list.extend(dependencies)

    # Create the custom_deps string
    custom_deps = ''
    for dep in final_custom_deps_list:
      if dep[1] is None:
        dep_url = None
      else:
        dep_url = '"%s"' % dep[1]
      custom_deps += '"%s" : %s, ' % (dep[0], dep_url)

    custom_vars = ''
    for var in self.custom_vars_list:
      if var[1] is None:
        var_value = None
      else:
        var_value = '"%s"' % var[1]
      custom_vars += '"%s" : %s, ' % (var[0], var_value)

    extras = ''
    if self.custom_deps_file:
      extras += '"deps_file": "%s",' % self.custom_deps_file
    if self.safesync_url:
      extras += '"safesync_url": "%s"' % self.safesync_url
    if self.managed:
      extras += '"managed": %s' % self.managed

    # This must not contain any line breaks or other characters that would
    # require escaping on the command line, since it will be passed to gclient.
    spec = (
      '{ "name": "%s", '
        '"url": "%s", '
        '"custom_deps": {'
                          '%s'
                       '},'
        '"custom_vars": {'
                          '%s'
                       '},'
        '%s'
      '},' % (self.name, self.svn_url, custom_deps, custom_vars, extras)
    )
    return spec

class GClientFactory(object):
  """Encapsulates data and methods common to both (all) master.cfg files."""

  def __init__(self, build_dir, solutions, target_platform=None,
               nohooks_on_update=False, target_os=None, revision_mapping=None):
    self._build_dir = build_dir
    self._solutions = solutions
    self._target_platform = target_platform or 'win32'
    self._target_os = target_os
    self._nohooks_on_update = nohooks_on_update
    self._revision_mapping = revision_mapping

  def BuildGClientSpec(self, tests=None):
    spec = 'solutions = ['
    for solution in self._solutions:
      spec += solution.GetSpec(tests)
    spec += ']'

    if self._target_os:
      spec += ';target_os = ["' + self._target_os + '"]'

    return spec

  def BaseFactory(self, gclient_spec=None, official_release=False,
                  factory_properties=None, build_properties=None,
                  delay_compile_step=False, sudo_for_remove=False,
                  gclient_deps=None, slave_type=None, options=None):
    if gclient_spec is None:
      gclient_spec = self.BuildGClientSpec()
    factory_properties = factory_properties or {}
    factory = BuildFactory(build_properties)
    factory_cmd_obj = commands.FactoryCommands(factory,
        target_platform=self._target_platform)
    # First kill any svn.exe tasks so we can update in peace, and
    # afterwards use the checked-out script to kill everything else.
    if (self._target_platform == 'win32' and
        not factory_properties.get('no_kill')):
      factory_cmd_obj.AddSvnKillStep()
    script_solutions = None
    if (factory_properties.get('goma_canary') and
        factory_properties.get('slave_internal_url')):
      script_solutions = [GClientSolution(
          factory_properties.get('slave_internal_url'),
          name="slave.DEPS",
          custom_vars_list=[("goma_linux_revision", "HEAD"),
                            ("goma_mac_revision", "HEAD"),
                            ("goma_win_revision", "HEAD")])]
    factory_cmd_obj.AddUpdateScriptStep(
        gclient_jobs=factory_properties.get('update_scripts_gclient_jobs'),
        solutions=script_solutions)
    # Once the script is updated, the zombie processes left by the previous
    # run can be killed.
    if (self._target_platform == 'win32' and
        not factory_properties.get('no_kill')):
      factory_cmd_obj.AddTaskkillStep()

    env = factory_properties.get('gclient_env', {})
    # Allow gclient_deps to also come from the factory_properties.
    if gclient_deps == None:
      gclient_deps = factory_properties.get('gclient_deps', None)

    if gclient_deps == 'ios':
      gclient_spec += ';target_os = [\'ios\'];target_os_only = True'

    # Do we need to operate in magic blink mode?
    blink_config = factory_properties.get('blink_config')

    # Force the build checkout to be at some revision.  This may or may not
    # activate depending on its own criteria, but the expectation is that if
    # this does activate, it will emit a BOT_UPDATED file in the build/
    # directory to signal to the other gclient update steps to no-op.
    code_review_site = config.Master.Master4.code_review_site
    factory_cmd_obj.AddBotUpdateStep(env, gclient_spec, self._revision_mapping,
                                     server=code_review_site,
                                     blink_config=blink_config)

    # svn timeout is 2 min; we allow 5
    timeout = factory_properties.get('gclient_timeout')
    if official_release or factory_properties.get('nuke_and_pave'):
      no_gclient_branch = factory_properties.get('no_gclient_branch', False)
      factory_cmd_obj.AddClobberTreeStep(gclient_spec, env, timeout,
          gclient_deps=gclient_deps, gclient_nohooks=self._nohooks_on_update,
          no_gclient_branch=no_gclient_branch, options=options)
    else:
      # Revert the tree to a clean (unmodified) state.
      factory_cmd_obj.AddGClientRevertStep()
      if not delay_compile_step:
        self.AddUpdateStep(gclient_spec, factory_properties, factory,
                           slave_type, sudo_for_remove,
                           gclient_deps=gclient_deps, options=options,
                           blink_config=blink_config)
    return factory

  def BuildFactory(self, target='Release', clobber=False, tests=None, mode=None,
                   slave_type='BuilderTester', options=None,
                   compile_timeout=1200, build_url=None, project=None,
                   factory_properties=None, gclient_deps=None,
                   target_arch=None, skip_archive_steps=False):
    if factory_properties is None:
      factory_properties = {}
    factory_properties.setdefault('gclient_env', {})
    gclient_env = factory_properties['gclient_env']
    if options and '--build-tool=ninja' in options:
      gclient_env['GYP_GENERATORS'] = 'ninja'

    # Create the spec for the solutions
    gclient_spec = self.BuildGClientSpec(tests)

    # Initialize the factory with the basic steps.
    factory = self.BaseFactory(gclient_spec,
                               factory_properties=factory_properties,
                               slave_type=slave_type,
                               gclient_deps=gclient_deps, options=options)

    # Optional repository root (default: 'src').
    repository_root = factory_properties.get('repository_root', 'src')

    # Get the factory command object to create new steps to the factory.
    factory_cmd_obj = commands.FactoryCommands(factory, target,
                                               self._build_dir,
                                               self._target_platform,
                                               target_arch,
                                               repository_root)

    # Update clang if necessary.
    if ('clang=1' in gclient_env.get('GYP_DEFINES', '') or
        (self._target_platform != 'win32' and
         factory_properties.get('asan'))):
      factory_cmd_obj.AddUpdateClangStep()

    # Add a step to cleanup temporary files and data left from a previous run
    # to prevent the drives from becoming full over time.
    factory_cmd_obj.AddTempCleanupStep()

    # Update the NaCl SDK if needed
    if factory_properties.get('update_nacl_sdk'):
      factory_cmd_obj.AddUpdateNaClSDKStep(
          factory_properties['update_nacl_sdk'])

    # Add the compile step if needed.
    if slave_type in ['BuilderTester', 'Builder', 'Trybot', 'Indexer',
                      'TrybotBuilder']:
      if self._target_platform == 'win32':
        # Look for a solution named for its enclosing directory.
        project = project or os.path.basename(self._build_dir) + '.sln'

      factory_cmd_obj.AddCompileStep(
          project,
          clobber,
          mode=mode,
          options=options,
          timeout=compile_timeout,
          env=factory_properties.get('compile_env'))

    # Generate synthetic user profiles. Must run before AddZipBuild().
    if factory_properties.get('create_profiles'):
      # pylint: disable=W0212
      factory_cmd_obj.AddProfileCreationStep('small_profile')

    if not skip_archive_steps:
      # Archive the full output directory if the machine is a builder.
      if slave_type in ['Builder', 'TrybotBuilder']:
        if build_url or factory_properties.get('build_url'):
          # There are some builders that are classified as builders, but only
          # because we only need to see if the patch compiles, and not
          # actually because triggered testers needs the build. So if we don't
          # supply a build_url, we assume that the build artifact doesn't
          # need to be uploaded and we can bypass this step.
          factory_cmd_obj.AddZipBuild(build_url,
                                      factory_properties=factory_properties)

      # Download the full output directory if the machine is a tester.
      if slave_type in ['Tester', 'TrybotTester']:
        factory_cmd_obj.AddExtractBuild()

    return factory

  # pylint: disable=R0201
  def TriggerFactory(self, factory, slave_type, factory_properties):
    """Add post steps on a build created by BuildFactory."""
    # Trigger any schedulers waiting on the build to complete.
    factory_properties = factory_properties or {}
    if factory_properties.get('trigger') is None:
      return

    trigger_name = factory_properties.get('trigger')

    # Allow overwriting default values for specified properties.
    set_properties = factory_properties.get('trigger_set_properties', {})

    # Propagate properties to the children if this is set in the factory.
    copy_properties = factory_properties.get('trigger_properties', [])
    factory.addStep(commands.CreateTriggerStep(
        trigger_name=trigger_name,
        trigger_copy_properties=copy_properties,
        trigger_set_properties=set_properties))

  def AddUpdateStep(self, gclient_spec, factory_properties, factory,
                    slave_type, sudo_for_remove=False, gclient_deps=None,
                    options=None, blink_config=False):
    if gclient_spec is None:
      gclient_spec = self.BuildGClientSpec()
    factory_properties = factory_properties or {}

    # Get the factory command object to add update step to the factory.
    factory_cmd_obj = commands.FactoryCommands(factory,
        target_platform=self._target_platform)

    # Get variables needed for the update.
    env = factory_properties.get('gclient_env', {})
    timeout = factory_properties.get('gclient_timeout')

    no_gclient_branch = factory_properties.get('no_gclient_branch', False)
    no_gclient_revision = factory_properties.get('no_gclient_revision', False)

    gclient_transitive = factory_properties.get('gclient_transitive', False)
    primary_repo = factory_properties.get('primary_repo', '')
    gclient_jobs = factory_properties.get('gclient_jobs')

    # Do not run gyp_chromium on testers.
    if slave_type in ('Tester',):
      env.update({'GYP_CHROMIUM_NO_ACTION': '1'})

    # Add the update step.
    factory_cmd_obj.AddUpdateStep(
        gclient_spec,
        env=env,
        timeout=timeout,
        sudo_for_remove=sudo_for_remove,
        gclient_deps=gclient_deps,
        gclient_nohooks=True,
        no_gclient_branch=no_gclient_branch,
        no_gclient_revision=no_gclient_revision,
        gclient_transitive=gclient_transitive,
        primary_repo=primary_repo,
        gclient_jobs=gclient_jobs,
        blink_config=blink_config)

    if slave_type in ('AnnotatedTrybot', 'CrosTrybot', 'Trybot', 'Bisect',
                      'TrybotTester', 'TrybotBuilder'):
      factory_cmd_obj.AddApplyIssueStep(
          timeout=timeout,
          server=config.Master.Master4.code_review_site,
          revision_mapping=self._revision_mapping)

    if not self._nohooks_on_update:
      factory_cmd_obj.AddRunHooksStep(env=env, timeout=timeout, options=options)
<|endoftext|>


<commit_before># Copyright (c) 2011 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Set of utilities to add commands to a buildbot factory.

This is based on commands.py and adds Syzygy-specific commands."""

from buildbot.process.properties import WithProperties
from buildbot.steps import shell

from master.factory import commands

class _UrlStatusCommand(shell.ShellCommand):
  """A ShellCommand subclass that adorns its build status with a URL on success.
  """
  def __init__(self, extra_text=None, **kw):
    """Initialize the buildstep.

    Args:
         extra_text: a tuple of (name, url) to pass to addUrl on successful
            completion.
    """
    self._extra_text = extra_text
    shell.ShellCommand.__init__(self, **kw)

    # Record our argument for the factory.
    self.addFactoryArguments(extra_text=extra_text)

  def commandComplete(self, cmd):
    """On success, add the URL provided to our status."""
    if cmd.rc == 0 and self._extra_text:
      (name, url) = self._extra_text
      self.addURL(self.build.render(name), self.build.render(url))

class SyzygyCommands(commands.FactoryCommands):
  """Encapsulates methods to add Syzygy commands to a buildbot factory."""

  def __init__(self, factory=None, target=None, build_dir=None,
               target_platform=None, target_arch=None):
    commands.FactoryCommands.__init__(self, factory, target, build_dir,
                                      target_platform)

    self._arch = target_arch
    self._factory = factory

  def AddRandomizeChromeStep(self):
    # Randomization script path.
    script_path = self.PathJoin(self._build_dir, 'internal', 'build',
                                'randomize_chrome.py')
    command = [self._python, script_path,
               '--build-dir=%s' % self._build_dir,
               '--target=%s' % self._target,
               '--verbose']
    self.AddTestStep(shell.ShellCommand, 'Randomly Reorder Chrome', command)

  def AddBenchmarkChromeStep(self):
    # Benchmark script path.
    script_path = self.PathJoin(self._build_dir, 'internal', 'build',
                                'benchmark_chrome.py')
    command = [self._python, script_path,
               '--build-dir=%s' % self._build_dir,
               '--target=%s' % self._target,
               '--verbose']
    self.AddTestStep(shell.ShellCommand, 'Benchmark Chrome', command)

  def AddGenerateCoverage(self):
    # Coverage script path.
    script_path = self.PathJoin(self._build_dir, 'build',
                                'generate_coverage.py')
    command = [self._python,
               script_path,
               '--verbose',
               '--build-dir',
               self.PathJoin(self._build_dir, self._target)]
    self.AddTestStep(shell.ShellCommand, 'Capture Unittest Coverage', command)

    # Store the coverage results by the checkout revision.
    src_dir = self.PathJoin(self._build_dir, self._target, 'cov')
    dst_gs_url = WithProperties(
        'gs://syzygy-archive/builds/coverage/%(got_revision)s')
    url = WithProperties(
        'http://syzygy-archive.commondatastorage.googleapis.com/builds/'
           'coverage/%(got_revision)s/index.html')

    command = [self._python,
               self.PathJoin(self._script_dir, 'syzygy/gsutil_cp_dir.py'),
               src_dir,
               dst_gs_url,]

    self._factory.addStep(_UrlStatusCommand,
                          command=command,
                          extra_text=('Coverage Report', url),
                          name='archive',
                          description='Archive Coverage Report')

  def AddArchival(self):
    '''Adds steps to archive the build output for official builds.'''
    # Store the coverage results by the checkout revision.
    src_archive = self.PathJoin(self._build_dir, self._target, 'benchmark.zip')
    dst_gs_url = WithProperties(
        'gs://syzygy-archive/builds/official/%(got_revision)s/benchmark.zip')
    url = WithProperties(
        'http://syzygy-archive.commondatastorage.googleapis.com/builds/'
           'official/%(got_revision)s/benchmark.zip')

    command = [self._python,
               self.PathJoin(self._script_dir, 'syzygy/gsutil_cp_dir.py'),
               src_archive,
               dst_gs_url,]

    self._factory.addStep(_UrlStatusCommand,
                          command=command,
                          extra_text=('Build archive', url),
                          name='archive',
                          description='Build archive')
<commit_msg>More path munging to fix build to cope with new output path. Review URL: http://codereview.chromium.org/8890021<commit_after># Copyright (c) 2011 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Set of utilities to add commands to a buildbot factory.

This is based on commands.py and adds Syzygy-specific commands."""

from buildbot.process.properties import WithProperties
from buildbot.steps import shell

from master.factory import commands

class _UrlStatusCommand(shell.ShellCommand):
  """A ShellCommand subclass that adorns its build status with a URL on success.
  """
  def __init__(self, extra_text=None, **kw):
    """Initialize the buildstep.

    Args:
         extra_text: a tuple of (name, url) to pass to addUrl on successful
            completion.
    """
    self._extra_text = extra_text
    shell.ShellCommand.__init__(self, **kw)

    # Record our argument for the factory.
    self.addFactoryArguments(extra_text=extra_text)

  def commandComplete(self, cmd):
    """On success, add the URL provided to our status."""
    if cmd.rc == 0 and self._extra_text:
      (name, url) = self._extra_text
      self.addURL(self.build.render(name), self.build.render(url))

class SyzygyCommands(commands.FactoryCommands):
  """Encapsulates methods to add Syzygy commands to a buildbot factory."""

  def __init__(self, factory=None, target=None, build_dir=None,
               target_platform=None, target_arch=None):
    commands.FactoryCommands.__init__(self, factory, target, build_dir,
                                      target_platform)

    self._arch = target_arch
    self._factory = factory

  def AddRandomizeChromeStep(self):
    # Randomization script path.
    script_path = self.PathJoin(self._build_dir, '..', 'syzygy',
                                'internal', 'build', 'randomize_chrome.py')
    command = [self._python, script_path,
               '--build-dir=%s' % self._build_dir,
               '--target=%s' % self._target,
               '--verbose']
    self.AddTestStep(shell.ShellCommand, 'Randomly Reorder Chrome', command)

  def AddBenchmarkChromeStep(self):
    # Benchmark script path.
    script_path = self.PathJoin(self._build_dir, '..', 'syzygy',
                                'internal', 'build', 'benchmark_chrome.py')
    command = [self._python, script_path,
               '--build-dir=%s' % self._build_dir,
               '--target=%s' % self._target,
               '--verbose']
    self.AddTestStep(shell.ShellCommand, 'Benchmark Chrome', command)

  def AddGenerateCoverage(self):
    # Coverage script path.
    script_path = self.PathJoin(self._build_dir, '..', 'syzygy', 'build',
                                'generate_coverage.py')
    command = [self._python,
               script_path,
               '--verbose',
               '--build-dir',
               self.PathJoin(self._build_dir, self._target)]
    self.AddTestStep(shell.ShellCommand, 'Capture Unittest Coverage', command)

    # Store the coverage results by the checkout revision.
    src_dir = self.PathJoin(self._build_dir, self._target, 'cov')
    dst_gs_url = WithProperties(
        'gs://syzygy-archive/builds/coverage/%(got_revision)s')
    url = WithProperties(
        'http://syzygy-archive.commondatastorage.googleapis.com/builds/'
           'coverage/%(got_revision)s/index.html')

    command = [self._python,
               self.PathJoin(self._script_dir, 'syzygy/gsutil_cp_dir.py'),
               src_dir,
               dst_gs_url,]

    self._factory.addStep(_UrlStatusCommand,
                          command=command,
                          extra_text=('Coverage Report', url),
                          name='archive',
                          description='Archive Coverage Report')

  def AddArchival(self):
    '''Adds steps to archive the build output for official builds.'''
    # Store the coverage results by the checkout revision.
    src_archive = self.PathJoin(self._build_dir, self._target, 'benchmark.zip')
    dst_gs_url = WithProperties(
        'gs://syzygy-archive/builds/official/%(got_revision)s/benchmark.zip')
    url = WithProperties(
        'http://syzygy-archive.commondatastorage.googleapis.com/builds/'
           'official/%(got_revision)s/benchmark.zip')

    command = [self._python,
               self.PathJoin(self._script_dir, 'syzygy/gsutil_cp_dir.py'),
               src_archive,
               dst_gs_url,]

    self._factory.addStep(_UrlStatusCommand,
                          command=command,
                          extra_text=('Build archive', url),
                          name='archive',
                          description='Build archive')
<|endoftext|>


<commit_before>"""
MZ (2020 - 01 - 20)
Take files with results of the fit and put them in a fits file.
Some results from the naivefit are left in the npy format as they are
a supplementary results.

Columns:
object_id
membership_1
...
membership_N for N components
membership_bg.

Do the same for the components.
Apart from XYZUVW for the components,
add additional coordinates: (l, b, dist), and (ra, dec, dist)
"""
# TODO: This could later be incorporated into naivefit itself and be a default option

import numpy as np
from astropy.table import Table
from astropy import units as u
import string

import sys
#sys.path.insert(0, '../')
sys.path.insert(0, '/home/marusa/chronostar/chronostar/') # TODO: hardcoded
import readparam

filename_components = 'final_comps.npy' # The best step of walkers. Lnprob can oscillate when converged.
filename_membership = 'final_membership.npy'
fit_pars = readparam.readParam('/data/mash/marusa/chronostar_projects/scocen/epifit/fit_pars.log') # TODO: hardcoded...
filename_raw_input_data = fit_pars['data_table'] # Star IDs

### COMPONENTS
# Components (EXcludes background, but background must be 1-sum(memberships for all components))
#TODO: It would be great to add background membership as well, because then it is much more simple (using one column) to determine stars that are members of any of the components as opposed to the background.
comp = np.load(filename_components)
ncomps = len(comp)

# Compnames
abc=string.ascii_uppercase
compnames = [abc[i] for i in range(ncomps)]

tabcomps = Table([compnames, comp[:,0], comp[:,1], comp[:,2], comp[:,3], comp[:,4], comp[:,5], comp[:,6], comp[:,7], comp[:,8]], names=('comp_ID', 'X', 'Y', 'Z', 'U', 'V', 'W', 'dX', 'dV', 'Age'))
tabcomps['X'].unit = u.pc
tabcomps['Y'].unit = u.pc
tabcomps['Z'].unit = u.pc
tabcomps['dX'].unit = u.pc
tabcomps['U'].unit = u.km/u.s
tabcomps['V'].unit = u.km/u.s
tabcomps['W'].unit = u.km/u.s
tabcomps['dV'].unit = u.km/u.s
tabcomps['Age'].unit = u.Myr

tabcomps.write('final_comps.fits', format='fits')

### MEMBERSHIPS
# Raw data with IDs
tab_raw = Table.read(filename_raw_input_data)
ids = list(tab_raw['source_id'])
tab = Table((ids,), names=('source_id',))

# Membership
memb = np.load(filename_membership)
for i, c in enumerate(compnames):
    tab['membership%s'%c.replace('comp', '')] = memb[:,i]

tab.write('final_membership.fits', format='fits')

# TODO: what is this
#final_comps_bak.npy # get some pickle error
#final_med_and_spans.npy # related to components: these are actual estimates from the walkers with uncertainties!<commit_msg>updated script for npy-fits conversion<commit_after>"""
MZ (2020 - 01 - 20)
Take files with results of the fit and put them in a fits file.
Some results from the naivefit are left in the npy format as they are
a supplementary results.

Columns:
object_id
membership_1
...
membership_N for N components
membership_bg.

Do the same for the components.
Apart from XYZUVW for the components,
add additional coordinates: (l, b, dist), and (ra, dec, dist)
"""
# TODO: This could later be incorporated into naivefit itself and be a default option

import numpy as np
from astropy.table import Table
from astropy import units as u
import string

import os
import sys
#sys.path.insert(0, '../')
sys.path.insert(0, '/home/marusa/chronostar/chronostar/') # TODO: hardcoded
import readparam

if len(sys.argv)<3:
    print('Usage:')
    print('python prepare_fits_file_with_results.py path_to_the_results_folder component_folder')
    print('E.g. python prepare_fits_file_with_results.py results/ 15/')

root = sys.argv[1]
root_comp = sys.argv[2]

filename_components = os.path.join(root, root_comp, 'final_comps.npy') # The best step of walkers. Lnprob can oscillate when converged.
filename_membership =  os.path.join(root, root_comp, 'final_membership.npy')
fit_pars = readparam.readParam(os.path.join(root, 'fit_pars.log'))
filename_raw_input_data = fit_pars['data_table'] # Star IDs

### COMPONENTS
# Components (EXcludes background, but background must be 1-sum(memberships for all components))
#TODO: It would be great to add background membership as well, because then it is much more simple (using one column) to determine stars that are members of any of the components as opposed to the background.
comp = np.load(filename_components)
ncomps = len(comp)

# Compnames
abc=string.ascii_uppercase
compnames = [abc[i] for i in range(ncomps)]

tabcomps = Table([compnames, comp[:,0], comp[:,1], comp[:,2], comp[:,3], comp[:,4], comp[:,5], comp[:,6], comp[:,7], comp[:,8]], names=('comp_ID', 'X', 'Y', 'Z', 'U', 'V', 'W', 'dX', 'dV', 'Age'))
tabcomps['X'].unit = u.pc
tabcomps['Y'].unit = u.pc
tabcomps['Z'].unit = u.pc
tabcomps['dX'].unit = u.pc
tabcomps['U'].unit = u.km/u.s
tabcomps['V'].unit = u.km/u.s
tabcomps['W'].unit = u.km/u.s
tabcomps['dV'].unit = u.km/u.s
tabcomps['Age'].unit = u.Myr

tabcomps.write('final_comps.fits', format='fits')

### MEMBERSHIPS
# Raw data with IDs
tab_raw = Table.read(filename_raw_input_data)
ids = list(tab_raw['source_id'])
tab = Table((ids,), names=('source_id',))

# Membership
memb = np.load(filename_membership)
for i, c in enumerate(compnames):
    tab['membership%s'%c.replace('comp', '')] = memb[:,i]

tab.write('final_membership.fits', format='fits')

# TODO: what is this
#final_comps_bak.npy # get some pickle error
#final_med_and_spans.npy # related to components: these are actual estimates from the walkers with uncertainties!<|endoftext|>


<commit_before>
#!/usr/bin/env python

##
##  See COPYING file distributed along with the ncanda-data-integration package
##  for the copyright and license terms
##
"""
XNAT Sessions Report

Check for valid scanning sessions and time windows by first caching all the
XNAT session XML files and then parsing these files for necessary info. Note
that to create the XML file cache you need to run with --update

Example
=======
- When running for the first time run
  ./xnat_sessions_report.py --update
  so that the cach (located at experimentsdir) is created

- Update the cache (stored in experimentsdir) and generate the baseline report
  ./xnat_sessions_report.py --update --baseline

- Use the existing cache to extract 10 in the followup window
 ./xnat_sessions_report.py --num_extract 10 --min 180 --max 540
"""

import os

import pandas as pd

import xnat_extractor as xe

verbose = None

def get_scan_type_pairs(modality):
    """
    Get a dictionary of series description based on modality
    :param modality: str (anatomy, diffusion, functional)
    :return: dict
    """
    scan_type_pairs = dict(scan1=None, scan2=None)
    if modality == 'anatomy':
        t1_scan_types = ['ncanda-t1spgr-v1', 'ncanda-mprage-v1']
        t2_scan_types = ['ncanda-t2fse-v1']
        scan_type_pairs.update(scan1=t1_scan_types,
                               scan2=t2_scan_types)
    elif modality == 'diffusion':
        pepolar = ['ncanda-dti6b500pepolar-v1']
        dwi = ['ncanda-dti60b1000-v1']
        scan_type_pairs.update(scan1=pepolar,
                               scan2=dwi)
    elif modality == 'functional':
        fmri = ['ncanda-rsfmri-v1']
        fieldmap = ['ncanda-grefieldmap-v1']
        scan_type_pairs.update(scan1=fmri,
                               scan2=fieldmap)
    return scan_type_pairs

def main(args=None):
    # TODO: Handle when T1 and T2 are in separate session (i.e., rescan)
    if args.update:
        config = xe.get_config(args.config)
        session = xe.get_xnat_session(config)
        xe.extract_experiment_xml(config, session,
                                  args.experimentsdir, args.num_extract)

    # extract info from the experiment XML files
    experiment = xe.get_experiments_dir_info(args.experimentsdir)
    # Scan specific information
    scan = xe.get_experiments_dir_scan_info(args.experimentsdir)
    # Session info
    reading = xe.get_experiments_dir_reading_info(args.experimentsdir)
    df = xe.merge_experiments_scans_reading(experiment, scan, reading)

    # exclude phantoms, including the traveling human phantoms
    site_id_pattern = '[A-E]-[0-9]{5}-[MF]-[0-9]'
    df = df[df.site_id.str.contains(site_id_pattern)]

    # exclude subjects not part of study
    df = df[df['subject_id'] != 'NCANDA_S00127']

    if args.unknown :
        print "Sessions that have not yet been quality controlled"
        for eid in df.experiment_id.drop_duplicates():
            eid_df = df[df.experiment_id == eid]
            eid_df = eid_df[~pd.isnull(eid_df['quality'])]
            if not len(eid_df[eid_df['quality'] != 'unknown']) :
                print eid
        sys.exit()
    if args.ignore_window or args.session_notes or args.scan_notes :
        if args.usable :
            df = df[df['quality'] == 'usable']

        columns = ['site_id', 'subject_id', 'experiment_id', 'experiment_date','excludefromanalysis']
        if args.ignore_window or args.scan_notes :
            columns = columns + ['scan_id', 'scan_type', 'quality']
            if args.scan_notes :
                columns = columns + [ 'scan_note']

        if args.session_notes :
            columns = columns + [ 'note' ]

        result = df[columns]

        # print result
    else :
        df.loc[:, 'experiment_date'] = df.experiment_date.astype('datetime64')
        result = pd.DataFrame()
        for subject_id in df.subject_id.drop_duplicates():
            subject_df = df[df.subject_id == subject_id]

            # find the earliest exam date for each given subject
            grouping = subject_df.groupby('subject_id')
            baseline_date = grouping['experiment_date'].nsmallest(1)
            baseline_df = subject_df[subject_df.experiment_date == baseline_date[0]]

            # Find window for follow-up
            day_min = pd.datetools.Day(n=args.min)
            day_max = pd.datetools.Day(n=args.max)
            followup_min = baseline_df.experiment_date + day_min
            followup_max = baseline_df.experiment_date + day_max

            df_min = subject_df.experiment_date > followup_min[0]
            df_max = subject_df.experiment_date < followup_max[0]
            followup_df = subject_df[df_min & df_max]

            # Included followup sessions slightly outside window
            included = ['NCANDA_E02615', 'NCANDA_E02860']
            included_df = subject_df[subject_df.experiment_id.isin(included)]
            if included_df.shape[0]:
                followup_df = included_df

            # Create report for baseline visit
            if args.baseline:
                followup_df = baseline_df

            # filter for specific scan types

            scan_type_pairs = get_scan_type_pairs(args.modality)
            scan1 = scan_type_pairs.get('scan1')
            scan2 = scan_type_pairs.get('scan2')
            scan1_df = followup_df[followup_df.scan_type.isin(scan1)]
            scan2_df = followup_df[followup_df.scan_type.isin(scan2)]

            # Filter quality column
            if args.usable :
                scan1_selected = scan1_df[scan1_df.quality == 'usable']
                scan2_selected = scan2_df[scan2_df.quality == 'usable']
            else :
                scan1_selected = scan1_df
                scan2_selected = scan2_df

            # report columns
            columns = ['site_id', 'subject_id', 'experiment_id', 'experiment_date',
                       'excludefromanalysis', 'note', 'scan_type', 'quality',
                       'scan_note']
            scan1_recs = scan1_selected.loc[:, columns].to_records(index=False)
            scan2_recs = scan2_selected.loc[:, columns].to_records(index=False)

            scan1_report = pd.DataFrame(scan1_recs,
                                        index=scan1_selected.experiment_id)
            scan2_report = pd.DataFrame(scan2_recs,
                                        index=scan2_selected.experiment_id)

            scan1_scan2_report = scan1_report.join(scan2_report[['scan_type',
                                                                 'quality',
                                                                 'scan_note']],
                                                   lsuffix='_scan1',
                                                   rsuffix='_scan2',
                                                   how='inner')
            if scan1_scan2_report.shape[0]:
                result = result.append(scan1_scan2_report)
    #
    # Write out results
    #

    # Remove any duplicate rows due to extra usable scan types (i.e., fieldmaps)
    result = result.drop_duplicates()
    result.to_csv(args.outfile, index=False)

if __name__ == "__main__":
    import sys
    import argparse

    parser = argparse.ArgumentParser(prog='xnat_sessions_report.py',
                                     description=__doc__)
    parser.add_argument('-c', '--config',
                        type=str,
                        default=os.path.join(os.path.expanduser('~'),
                                             '.server_config', 'ncanda.cfg'))
    parser.add_argument('-b', '--baseline',
                        action='store_true',
                        help='Create report for baseline visit.')
    parser.add_argument('-e', '--experimentsdir',
                        type=str,
                        default='/tmp/experiments',
                        help='Name of experiments xml directory')
    parser.add_argument('-m', '--modality',
                        type=str,
                        default='anatomy',
                        choices=['anatomy', 'diffusion', 'functional'],
                        help='Name of experiments xml directory')
    parser.add_argument('--min',
                        type=int,
                        default=180,
                        help='Minimum days from baseline (to specify followup '
                             '1y, only impacts final report but not -u option)')
    parser.add_argument('--max',
                        type=int,
                        default=540,
                        help='Maximum days from baseline (to specify followup '
                             '1y, only impacts final report but not -u option)')
    parser.add_argument('--ignore-window',
                        action='store_true',
                        help='Just list sessions regardless of window')
    parser.add_argument('--usable',
                        action='store_true',
                        help='Only list scans with usable image quality')
    parser.add_argument('--unknown',
                        action='store_true',
                        help='Only list sessions that have unknown scans, i.e. have not been reviewed')
    parser.add_argument('--session-notes',
                        action='store_true',
                        help='create report with session notes')
    parser.add_argument('--scan-notes',
                        action='store_true',
                        help='include scan notes in the report')
    parser.add_argument('-o', '--outfile',
                        type=str,
                        default='/tmp/usability_report.csv',
                        help='Name of csv file to write.')
    parser.add_argument('-n', '--num_extract',
                        type=int,
                        help='Number of sessions to extract (only works in '
                             'connection with -u)')
    parser.add_argument('-u', '--update',
                        action='store_true',
                        help='Update the cache of xml files')
    parser.add_argument('-v', '--verbose',
                        action='store_true',
                        help='Print verbose output.')
    argv = parser.parse_args()
    verbose = argv.verbose
    xe.verbose = argv.verbose

    sys.exit(main(args=argv))
<commit_msg>Now also lists individual scans that have not been reviewed<commit_after>
#!/usr/bin/env python

##
##  See COPYING file distributed along with the ncanda-data-integration package
##  for the copyright and license terms
##
"""
XNAT Sessions Report

Check for valid scanning sessions and time windows by first caching all the
XNAT session XML files and then parsing these files for necessary info. Note
that to create the XML file cache you need to run with --update

Example
=======
- When running for the first time run
  ./xnat_sessions_report.py --update
  so that the cach (located at experimentsdir) is created

- Update the cache (stored in experimentsdir) and generate the baseline report
  ./xnat_sessions_report.py --update --baseline

- Use the existing cache to extract 10 in the followup window
 ./xnat_sessions_report.py --num_extract 10 --min 180 --max 540
"""

import os

import pandas as pd

import xnat_extractor as xe

verbose = None

def get_scan_type_pairs(modality):
    """
    Get a dictionary of series description based on modality
    :param modality: str (anatomy, diffusion, functional)
    :return: dict
    """
    scan_type_pairs = dict(scan1=None, scan2=None)
    if modality == 'anatomy':
        t1_scan_types = ['ncanda-t1spgr-v1', 'ncanda-mprage-v1']
        t2_scan_types = ['ncanda-t2fse-v1']
        scan_type_pairs.update(scan1=t1_scan_types,
                               scan2=t2_scan_types)
    elif modality == 'diffusion':
        pepolar = ['ncanda-dti6b500pepolar-v1']
        dwi = ['ncanda-dti60b1000-v1']
        scan_type_pairs.update(scan1=pepolar,
                               scan2=dwi)
    elif modality == 'functional':
        fmri = ['ncanda-rsfmri-v1']
        fieldmap = ['ncanda-grefieldmap-v1']
        scan_type_pairs.update(scan1=fmri,
                               scan2=fieldmap)
    return scan_type_pairs

def main(args=None):
    # TODO: Handle when T1 and T2 are in separate session (i.e., rescan)
    if args.update:
        config = xe.get_config(args.config)
        session = xe.get_xnat_session(config)
        xe.extract_experiment_xml(config, session,
                                  args.experimentsdir, args.num_extract)

    # extract info from the experiment XML files
    experiment = xe.get_experiments_dir_info(args.experimentsdir)
    # Scan specific information
    scan = xe.get_experiments_dir_scan_info(args.experimentsdir)
    # Session info
    reading = xe.get_experiments_dir_reading_info(args.experimentsdir)
    df = xe.merge_experiments_scans_reading(experiment, scan, reading)

    # exclude phantoms, including the traveling human phantoms
    site_id_pattern = '[A-E]-[0-9]{5}-[MF]-[0-9]'
    df = df[df.site_id.str.contains(site_id_pattern)]

    # exclude subjects not part of study
    df = df[df['subject_id'] != 'NCANDA_S00127']

    if args.unknown :
        print "Sessions that have not yet been quality controlled"
        scanCheckList = pd.DataFrame()
        required_scans = ['ncanda-mprage-v1','ncanda-t1spgr-v1','ncanda-t2fse-v1','ncanda-dti6b500pepolar-v1','ncanda-dti30b400-v1','ncanda-dti60b1000-v1','ncanda-grefieldmap-v1','ncanda-rsfmri-v1']

        for eid in df.experiment_id.drop_duplicates():
            eid_df = df[df.experiment_id == eid]
            eid_df = eid_df[~pd.isnull(eid_df['quality'])]
            if not len(eid_df[eid_df['quality'] != 'unknown']) :
                print eid
            else :
                unknownScanDF = eid_df[eid_df['quality'] == 'unknown']
                mandatoryCheck = unknownScanDF[unknownScanDF['scan_type'].isin(required_scans)]
                if len(mandatoryCheck) :
                    scanCheckList = scanCheckList.append(mandatoryCheck)

        print " "
        print "Mandatory scans that have not yet been quality controlled"
        print scanCheckList['scan_type']

        sys.exit()

    if args.ignore_window or args.session_notes or args.scan_notes :
        if args.usable :
            df = df[df['quality'] == 'usable']

        columns = ['site_id', 'subject_id', 'experiment_id', 'experiment_date','excludefromanalysis']
        if args.ignore_window or args.scan_notes :
            columns = columns + ['scan_id', 'scan_type', 'quality']
            if args.scan_notes :
                columns = columns + [ 'scan_note']

        if args.session_notes :
            columns = columns + [ 'note' ]

        result = df[columns]

        # print result
    else :
        df.loc[:, 'experiment_date'] = df.experiment_date.astype('datetime64')
        result = pd.DataFrame()
        for subject_id in df.subject_id.drop_duplicates():
            subject_df = df[df.subject_id == subject_id]

            # find the earliest exam date for each given subject
            grouping = subject_df.groupby('subject_id')
            baseline_date = grouping['experiment_date'].nsmallest(1)
            baseline_df = subject_df[subject_df.experiment_date == baseline_date[0]]

            # Find window for follow-up
            day_min = pd.datetools.Day(n=args.min)
            day_max = pd.datetools.Day(n=args.max)
            followup_min = baseline_df.experiment_date + day_min
            followup_max = baseline_df.experiment_date + day_max

            df_min = subject_df.experiment_date > followup_min[0]
            df_max = subject_df.experiment_date < followup_max[0]
            followup_df = subject_df[df_min & df_max]

            # Included followup sessions slightly outside window
            included = ['NCANDA_E02615', 'NCANDA_E02860']
            included_df = subject_df[subject_df.experiment_id.isin(included)]
            if included_df.shape[0]:
                followup_df = included_df

            # Create report for baseline visit
            if args.baseline:
                followup_df = baseline_df

            # filter for specific scan types

            scan_type_pairs = get_scan_type_pairs(args.modality)
            scan1 = scan_type_pairs.get('scan1')
            scan2 = scan_type_pairs.get('scan2')
            scan1_df = followup_df[followup_df.scan_type.isin(scan1)]
            scan2_df = followup_df[followup_df.scan_type.isin(scan2)]

            # Filter quality column
            if args.usable :
                scan1_selected = scan1_df[scan1_df.quality == 'usable']
                scan2_selected = scan2_df[scan2_df.quality == 'usable']
            else :
                scan1_selected = scan1_df
                scan2_selected = scan2_df

            # report columns
            columns = ['site_id', 'subject_id', 'experiment_id', 'experiment_date',
                       'excludefromanalysis', 'note', 'scan_type', 'quality',
                       'scan_note']
            scan1_recs = scan1_selected.loc[:, columns].to_records(index=False)
            scan2_recs = scan2_selected.loc[:, columns].to_records(index=False)

            scan1_report = pd.DataFrame(scan1_recs,
                                        index=scan1_selected.experiment_id)
            scan2_report = pd.DataFrame(scan2_recs,
                                        index=scan2_selected.experiment_id)

            scan1_scan2_report = scan1_report.join(scan2_report[['scan_type',
                                                                 'quality',
                                                                 'scan_note']],
                                                   lsuffix='_scan1',
                                                   rsuffix='_scan2',
                                                   how='inner')
            if scan1_scan2_report.shape[0]:
                result = result.append(scan1_scan2_report)
    #
    # Write out results
    #

    # Remove any duplicate rows due to extra usable scan types (i.e., fieldmaps)
    result = result.drop_duplicates()
    result.to_csv(args.outfile, index=False)

if __name__ == "__main__":
    import sys
    import argparse

    parser = argparse.ArgumentParser(prog='xnat_sessions_report.py',
                                     description=__doc__)
    parser.add_argument('-c', '--config',
                        type=str,
                        default=os.path.join(os.path.expanduser('~'),
                                             '.server_config', 'ncanda.cfg'))
    parser.add_argument('-b', '--baseline',
                        action='store_true',
                        help='Create report for baseline visit.')
    parser.add_argument('-e', '--experimentsdir',
                        type=str,
                        default='/tmp/experiments',
                        help='Name of experiments xml directory')
    parser.add_argument('-m', '--modality',
                        type=str,
                        default='anatomy',
                        choices=['anatomy', 'diffusion', 'functional'],
                        help='Name of experiments xml directory')
    parser.add_argument('--min',
                        type=int,
                        default=180,
                        help='Minimum days from baseline (to specify followup '
                             '1y, only impacts final report but not -u option)')
    parser.add_argument('--max',
                        type=int,
                        default=540,
                        help='Maximum days from baseline (to specify followup '
                             '1y, only impacts final report but not -u option)')
    parser.add_argument('--ignore-window',
                        action='store_true',
                        help='Just list sessions regardless of window')
    parser.add_argument('--usable',
                        action='store_true',
                        help='Only list scans with usable image quality')
    parser.add_argument('--unknown',
                        action='store_true',
                        help='Only list sessions that have unknown scans, i.e. have not been reviewed')
    parser.add_argument('--session-notes',
                        action='store_true',
                        help='create report with session notes')
    parser.add_argument('--scan-notes',
                        action='store_true',
                        help='include scan notes in the report')
    parser.add_argument('-o', '--outfile',
                        type=str,
                        default='/tmp/usability_report.csv',
                        help='Name of csv file to write.')
    parser.add_argument('-n', '--num_extract',
                        type=int,
                        help='Number of sessions to extract (only works in '
                             'connection with -u)')
    parser.add_argument('-u', '--update',
                        action='store_true',
                        help='Update the cache of xml files')
    parser.add_argument('-v', '--verbose',
                        action='store_true',
                        help='Print verbose output.')
    argv = parser.parse_args()
    verbose = argv.verbose
    xe.verbose = argv.verbose

    sys.exit(main(args=argv))
<|endoftext|>