<commit_before># -*- coding: utf-8 -*-
# (C) Copyright 2014 Voyager Search
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#  http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from tasks.utils import status
from tasks.utils import task_utils
from tasks import _

status_writer = status.Writer()
import arcpy

def execute(request):
    """Creates a GeoPDF.
    :param request: json as a dict.
    """
    added_to_map = 0
    errors = 0
    skipped = 0

    parameters = request['params']
    num_results, response_index = task_utils.get_result_count(parameters)
    input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'])
    if num_results > task_utils.CHUNK_SIZE:
        status_writer.send_state(status.STAT_FAILED, 'Reduce results to 25 or less.')
        return

    map_template = task_utils.get_parameter_value(parameters, 'map_template', 'value')
    base_map = task_utils.get_parameter_value(parameters, 'base_map', 'value')
    map_title = task_utils.get_parameter_value(parameters, 'map_title', 'value')
    attribute_setting = task_utils.get_parameter_value(parameters, 'attribute_settings', 'value')
    author = task_utils.get_parameter_value(parameters, 'map_author', 'value')
    try:
        map_view = task_utils.get_parameter_value(parameters, 'map_view', 'extent')
    except KeyError:
        map_view = None
        pass

    temp_folder = os.path.join(request['folder'], 'temp')
    if not os.path.exists(temp_folder):
        os.makedirs(temp_folder)

    if base_map == 'NONE':
        base_layer = None
    else:
        base_layer = arcpy.mapping.Layer(os.path.join(os.path.dirname(os.path.dirname(__file__)), 'supportfiles', 'basemaps', '{0}.lyr'.format(base_map)))
    mxd_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'supportfiles', 'frame', map_template)
    mxd = arcpy.mapping.MapDocument(mxd_path)
    data_frame = arcpy.mapping.ListDataFrames(mxd)[0]

    layers = []
    for item in input_items:
        try:
            # Is the item a mxd data frame.
            map_frame_name = task_utils.get_data_frame_name(item)
            if map_frame_name:
                item = item.split('|')[0].strip()

            dsc = arcpy.Describe(item)
            if dsc.dataType == 'Layer':
                layers.append(arcpy.mapping.Layer(dsc.catalogPath))

            elif dsc.dataType == 'FeatureClass' or dsc.dataType == 'ShapeFile':
                feature_layer = arcpy.MakeFeatureLayer_management(item, os.path.basename(item))
                layer_file = arcpy.SaveToLayerFile_management(
                    feature_layer,
                    os.path.join(temp_folder, '{0}.lyr'.format(os.path.basename(item)))
                )
                layers.append(arcpy.mapping.Layer(layer_file.getOutput(0)))

            elif dsc.dataType == 'FeatureDataset':
                arcpy.env.workspace = item
                for fc in arcpy.ListFeatureClasses():
                    layer_file = arcpy.SaveToLayerFile_management(arcpy.MakeFeatureLayer_management(fc, fc + '_layer'),
                                                                  os.path.join(temp_folder, '{0}.lyr'.format(fc)))
                    layer = arcpy.mapping.Layer(layer_file.getOutput(0))
                    layer.name = fc
                    layers.append(layer)

            elif dsc.dataType == 'RasterDataset':
                raster_layer = arcpy.MakeRasterLayer_management(item, os.path.basename(item))
                layer_file = arcpy.SaveToLayerFile_management(
                    raster_layer,
                    os.path.join(temp_folder, '{0}.lyr'.format(os.path.basename(item)))
                )
                layers.append(arcpy.mapping.Layer(layer_file.getOutput(0)))

            elif dsc.catalogPath.endswith('.kml') or dsc.catalogPath.endswith('.kmz'):
                name = os.path.splitext(dsc.name)[0]
                arcpy.KMLToLayer_conversion(dsc.catalogPath, temp_folder, name)
                layers.append(arcpy.mapping.Layer(os.path.join(temp_folder, '{0}.lyr'.format(name))))

            elif dsc.dataType == 'MapDocument':
                input_mxd = arcpy.mapping.MapDocument(item)
                if map_frame_name:
                    df = arcpy.mapping.ListDataFrames(input_mxd, map_frame_name)[0]
                    layers = arcpy.mapping.ListLayers(input_mxd, data_frame=df)
                else:
                    layers = arcpy.mapping.ListLayers(input_mxd)

            if layers:
                for layer in layers:
                    status_writer.send_status(_('Adding layer {0}...').format(layer.name))
                    arcpy.mapping.AddLayer(data_frame, layer)
                    added_to_map += 1
                    layers = []
            else:
                status_writer.send_status(_('Invalid input type: {0}').format(item))
                skipped += 1
        except Exception as ex:
            status_writer.send_status(_('FAIL: {0}').format(repr(ex)))
            errors += 1
            pass

    if map_view:
        extent = map_view.split(' ')
        new_extent = data_frame.extent
        new_extent.XMin, new_extent.YMin = float(extent[0]), float(extent[1])
        new_extent.XMax, new_extent.YMax = float(extent[2]), float(extent[3])
        data_frame.extent = new_extent
    else:
        data_frame.zoomToSelectedFeatures()

    # Update text elements in map template.
    date_element = arcpy.mapping.ListLayoutElements(mxd, 'TEXT_ELEMENT', 'date')
    if date_element:
        date_element[0].text = 'Date: {0}'.format(task_utils.get_local_date())

    title_element = arcpy.mapping.ListLayoutElements(mxd, 'TEXT_ELEMENT', 'title')
    if title_element:
        title_element[0].text = map_title

    author_element = arcpy.mapping.ListLayoutElements(mxd, 'TEXT_ELEMENT', 'author')
    if author_element:
        author_element[0].text = '{0} {1}'.format(author_element[0].text, author)

    if map_template in ('ANSI_D_LND.mxd', 'ANSI_E_LND.mxd'):
        coord_elements = arcpy.mapping.ListLayoutElements(mxd, 'TEXT_ELEMENT', 'x*')
        coord_elements += arcpy.mapping.ListLayoutElements(mxd, 'TEXT_ELEMENT', 'y*')
        if coord_elements:
            for e in coord_elements:
                new_text = e.text
                if e.name == 'xmin':
                    dms = task_utils.dd_to_dms(data_frame.extent.XMin)
                    if data_frame.extent.XMin > 0:
                        new_text = new_text.replace('W', 'E')
                elif e.name == 'xmax':
                    dms = task_utils.dd_to_dms(data_frame.extent.XMax)
                    if data_frame.extent.XMax > 0:
                        new_text = new_text.replace('W', 'E')
                elif e.name == 'ymin':
                    dms = task_utils.dd_to_dms(data_frame.extent.YMin)
                    if data_frame.extent.YMin < 0:
                        new_text = new_text.replace('N', 'S')
                elif e.name == 'ymax':
                    if data_frame.extent.YMax < 0:
                        new_text = new_text.replace('N', 'S')
                    dms = task_utils.dd_to_dms(data_frame.extent.YMax)

                new_text = new_text.replace('d', str(dms[0]))
                new_text = new_text.replace('m', str(dms[1]))
                new_text = new_text.replace('s', str(dms[2]))
                e.text = new_text

    # Do this now so it does not affect zoom level or extent.
    if base_layer:
        status_writer.send_status(_('Adding basemap {0}...').format(base_map))
        arcpy.mapping.AddLayer(data_frame, base_layer, 'BOTTOM')

    if added_to_map > 0:
        status_writer.send_status(_('Exporting to PDF...'))
        arcpy.mapping.ExportToPDF(mxd,
                                  os.path.join(request['folder'], 'output.pdf'),
                                  layers_attributes=attribute_setting)
        # Create a thumbnail size PNG of the mxd.
        task_utils.make_thumbnail(mxd, os.path.join(request['folder'], '_thumb.png'), False)
    else:
        status_writer.send_state(status.STAT_FAILED, _('No results can be exported to PDF'))
        task_utils.report(os.path.join(request['folder'], '_report.json'), added_to_map, skipped, skipped, 0)
        return

    # Update state if necessary.
    if skipped > 0 or errors > 0:
        status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped + errors))
    task_utils.report(os.path.join(request['folder'], '_report.json'), added_to_map, skipped, errors)
<commit_msg>VG-2774: Fix for inputs with the same name<commit_after># -*- coding: utf-8 -*-
# (C) Copyright 2014 Voyager Search
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#  http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from tasks.utils import status
from tasks.utils import task_utils
from tasks import _

status_writer = status.Writer()
import arcpy

def execute(request):
    """Creates a GeoPDF.
    :param request: json as a dict.
    """
    added_to_map = 0
    errors = 0
    skipped = 0

    parameters = request['params']
    num_results, response_index = task_utils.get_result_count(parameters)
    input_items = task_utils.get_input_items(parameters[response_index]['response']['docs'])
    if num_results > task_utils.CHUNK_SIZE:
        status_writer.send_state(status.STAT_FAILED, 'Reduce results to 25 or less.')
        return

    map_template = task_utils.get_parameter_value(parameters, 'map_template', 'value')
    base_map = task_utils.get_parameter_value(parameters, 'base_map', 'value')
    map_title = task_utils.get_parameter_value(parameters, 'map_title', 'value')
    attribute_setting = task_utils.get_parameter_value(parameters, 'attribute_settings', 'value')
    author = task_utils.get_parameter_value(parameters, 'map_author', 'value')
    try:
        map_view = task_utils.get_parameter_value(parameters, 'map_view', 'extent')
    except KeyError:
        map_view = None
        pass

    temp_folder = os.path.join(request['folder'], 'temp')
    if not os.path.exists(temp_folder):
        os.makedirs(temp_folder)

    if base_map == 'NONE':
        base_layer = None
    else:
        base_layer = arcpy.mapping.Layer(os.path.join(os.path.dirname(os.path.dirname(__file__)), 'supportfiles', 'basemaps', '{0}.lyr'.format(base_map)))
    mxd_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'supportfiles', 'frame', map_template)
    mxd = arcpy.mapping.MapDocument(mxd_path)
    data_frame = arcpy.mapping.ListDataFrames(mxd)[0]

    layers = []
    all_layers = []
    for i, item in enumerate(input_items, 1):
        try:
            # Is the item a mxd data frame.
            map_frame_name = task_utils.get_data_frame_name(item)
            if map_frame_name:
                item = item.split('|')[0].strip()

            dsc = arcpy.Describe(item)
            if dsc.dataType == 'Layer':
                layers.append(arcpy.mapping.Layer(dsc.catalogPath))

            elif dsc.dataType == 'FeatureClass' or dsc.dataType == 'ShapeFile':
                if os.path.basename(item) in [l.name for l in all_layers]:
                    layer_name = '{0}_{1}'.format(os.path.basename(item), i)
                else:
                    layer_name = os.path.basename(item)
                feature_layer = arcpy.MakeFeatureLayer_management(item, layer_name)
                layer_file = arcpy.SaveToLayerFile_management(
                    feature_layer,
                    os.path.join(temp_folder, '{0}.lyr'.format(layer_name))
                )
                layers.append(arcpy.mapping.Layer(layer_file.getOutput(0)))
                all_layers.append(arcpy.mapping.Layer(layer_file.getOutput(0)))

            elif dsc.dataType == 'FeatureDataset':
                arcpy.env.workspace = item
                for fc in arcpy.ListFeatureClasses():
                    layer_file = arcpy.SaveToLayerFile_management(arcpy.MakeFeatureLayer_management(fc, '{0}_{1}'.format(fc, i)),
                                                                  os.path.join(temp_folder, '{0}_{1}.lyr'.format(fc, i)))
                    layer = arcpy.mapping.Layer(layer_file.getOutput(0))
                    layer.name = fc
                    layers.append(layer)
                    all_layers.append(layer)

            elif dsc.dataType == 'RasterDataset':
                if os.path.basename(item) in [l.name for l in all_layers]:
                    layer_name = '{0}_{1}'.format(os.path.basename(item), i)
                else:
                    layer_name = os.path.basename(item)
                raster_layer = arcpy.MakeRasterLayer_management(item, layer_name)
                layer_file = arcpy.SaveToLayerFile_management(
                    raster_layer,
                    os.path.join(temp_folder, '{0}.lyr'.format(layer_name))
                )
                layers.append(arcpy.mapping.Layer(layer_file.getOutput(0)))
                all_layers.append(arcpy.mapping.Layer(layer_file.getOutput(0)))

            elif dsc.catalogPath.endswith('.kml') or dsc.catalogPath.endswith('.kmz'):
                if not os.path.splitext(dsc.name)[0] in layers:
                    name = os.path.splitext(dsc.name)[0]
                else:
                    name = '{0}_{1}'.format(os.path.splitext(dsc.name)[0], i)
                arcpy.KMLToLayer_conversion(dsc.catalogPath, temp_folder, name)
                layers.append(arcpy.mapping.Layer(os.path.join(temp_folder, '{0}.lyr'.format(name))))
                all_layers.append(arcpy.mapping.Layer(os.path.join(temp_folder, '{0}.lyr'.format(name))))

            elif dsc.dataType == 'MapDocument':
                input_mxd = arcpy.mapping.MapDocument(item)
                if map_frame_name:
                    df = arcpy.mapping.ListDataFrames(input_mxd, map_frame_name)[0]
                    layers = arcpy.mapping.ListLayers(input_mxd, data_frame=df)
                else:
                    layers = arcpy.mapping.ListLayers(input_mxd)

            if layers:
                for layer in layers:
                    status_writer.send_status(_('Adding layer {0}...').format(layer.name))
                    arcpy.mapping.AddLayer(data_frame, layer)
                    added_to_map += 1
                    layers = []
            else:
                status_writer.send_status(_('Invalid input type: {0}').format(item))
                skipped += 1
        except Exception as ex:
            status_writer.send_status(_('FAIL: {0}').format(repr(ex)))
            errors += 1
            pass

    if map_view:
        extent = map_view.split(' ')
        new_extent = data_frame.extent
        new_extent.XMin, new_extent.YMin = float(extent[0]), float(extent[1])
        new_extent.XMax, new_extent.YMax = float(extent[2]), float(extent[3])
        data_frame.extent = new_extent
    else:
        data_frame.zoomToSelectedFeatures()

    # Update text elements in map template.
    date_element = arcpy.mapping.ListLayoutElements(mxd, 'TEXT_ELEMENT', 'date')
    if date_element:
        date_element[0].text = 'Date: {0}'.format(task_utils.get_local_date())

    title_element = arcpy.mapping.ListLayoutElements(mxd, 'TEXT_ELEMENT', 'title')
    if title_element:
        title_element[0].text = map_title

    author_element = arcpy.mapping.ListLayoutElements(mxd, 'TEXT_ELEMENT', 'author')
    if author_element:
        author_element[0].text = '{0} {1}'.format(author_element[0].text, author)

    if map_template in ('ANSI_D_LND.mxd', 'ANSI_E_LND.mxd'):
        coord_elements = arcpy.mapping.ListLayoutElements(mxd, 'TEXT_ELEMENT', 'x*')
        coord_elements += arcpy.mapping.ListLayoutElements(mxd, 'TEXT_ELEMENT', 'y*')
        if coord_elements:
            for e in coord_elements:
                new_text = e.text
                if e.name == 'xmin':
                    dms = task_utils.dd_to_dms(data_frame.extent.XMin)
                    if data_frame.extent.XMin > 0:
                        new_text = new_text.replace('W', 'E')
                elif e.name == 'xmax':
                    dms = task_utils.dd_to_dms(data_frame.extent.XMax)
                    if data_frame.extent.XMax > 0:
                        new_text = new_text.replace('W', 'E')
                elif e.name == 'ymin':
                    dms = task_utils.dd_to_dms(data_frame.extent.YMin)
                    if data_frame.extent.YMin < 0:
                        new_text = new_text.replace('N', 'S')
                elif e.name == 'ymax':
                    if data_frame.extent.YMax < 0:
                        new_text = new_text.replace('N', 'S')
                    dms = task_utils.dd_to_dms(data_frame.extent.YMax)

                new_text = new_text.replace('d', str(dms[0]))
                new_text = new_text.replace('m', str(dms[1]))
                new_text = new_text.replace('s', str(dms[2]))
                e.text = new_text

    # Do this now so it does not affect zoom level or extent.
    if base_layer:
        status_writer.send_status(_('Adding basemap {0}...').format(base_map))
        arcpy.mapping.AddLayer(data_frame, base_layer, 'BOTTOM')

    if added_to_map > 0:
        status_writer.send_status(_('Exporting to PDF...'))
        arcpy.mapping.ExportToPDF(mxd,
                                  os.path.join(request['folder'], 'output.pdf'),
                                  layers_attributes=attribute_setting)
        # Create a thumbnail size PNG of the mxd.
        task_utils.make_thumbnail(mxd, os.path.join(request['folder'], '_thumb.png'), False)
    else:
        status_writer.send_state(status.STAT_FAILED, _('No results can be exported to PDF'))
        task_utils.report(os.path.join(request['folder'], '_report.json'), added_to_map, skipped, skipped, 0)
        return

    # Update state if necessary.
    if skipped > 0 or errors > 0:
        status_writer.send_state(status.STAT_WARNING, _('{0} results could not be processed').format(skipped + errors))
    task_utils.report(os.path.join(request['folder'], '_report.json'), added_to_map, skipped, errors)
<|endoftext|>


<commit_before>#!/usr/bin/env python

#Copyright (C) 2011 by Glenn Hickey
#
#Released under the MIT license, see LICENSE.txt
#!/usr/bin/env python

"""Wrapper to run the cactus_workflow progressively, using the input species tree as a guide

tree.   The --buildMAF and --joinMAF options can be used to create and merge MAFs from the

cacti.   The --buildReference option is removed, as references *are always* built when

--setupAndBuildAlignments is specified.  If the kyoto tycoon DB is used, it is best to use the helper

script, preKtserverDbs.py to set up the server.
"""

import os
import xml.etree.ElementTree as ET
import math
from optparse import OptionParser
from collections import deque
import random
from itertools import izip
from shutil import move
import copy
from time import sleep

from sonLib.bioio import getTempFile
from sonLib.bioio import printBinaryTree
from sonLib.bioio import system

from jobTree.src.bioio import getLogLevelString
from jobTree.src.bioio import logger
from jobTree.src.bioio import setLoggingFromOptions

from cactus.shared.common import cactusRootPath

from jobTree.scriptTree.target import Target
from jobTree.scriptTree.stack import Stack

from cactus.pipeline.cactus_workflow import CactusSetupPhase
from cactus.pipeline.cactus_workflow import CactusPhylogenyPhase
from cactus.pipeline.cactus_workflow import CactusReferencePhase
from cactus.pipeline.cactus_workflow import CactusFacesPhase
from cactus.pipeline.cactus_workflow import getOptionalAttrib

from cactus.progressive.multiCactusProject import MultiCactusProject
from cactus.progressive.multiCactusTree import MultiCactusTree
from cactus.progressive.experimentWrapper import ExperimentWrapper
from cactus.progressive.ktserverLauncher import KtserverLauncher
from cactus.progressive.mafFilter import removeOutgroupFromMaf
from cactus.progressive.schedule import Schedule

class ProgressiveDown(Target):
    def __init__(self, options, project, event, schedule):
        Target.__init__(self)
        self.options = options
        self.project = project
        self.event = event
        self.schedule = schedule

    def run(self):
        logger.info("Progressive Down: " + self.event)

        expXml = ET.parse(self.project.expMap[self.event]).getroot()
        experiment = ExperimentWrapper(expXml)

        if self.options.recursive:
            deps = self.schedule.deps(self.event)
            for child in deps:
                if child in self.project.expMap:
                    self.addChildTarget(ProgressiveDown(self.options,
                                                        self.project, child,
                                                        self.schedule))

        self.setFollowOnTarget(ProgressiveUp(self.options, self.project,
                               experiment, self.event))

class ProgressiveUp(Target):
    def __init__(self, options, project, experiment, event):
        Target.__init__(self)
        self.options = options
        self.project = project
        self.experiment = experiment
        self.event = event

    def run(self):
        logger.info("Progressive Up: " + self.event)

        # delete database files if --setupAndBuildAlignments
        if self.options.setupAndBuildAlignments:
            dbPath = os.path.join(self.experiment.getDbDir(),
                                  self.experiment.getDbName())
            seqPath = os.path.join(self.experiment.getDbDir(), "sequences")
            if os.path.isfile(dbPath):
                os.remove(dbPath)
            if os.path.isfile(seqPath):
                os.remove(seqPath)

        ktserver = None
        if self.experiment.getDbType() == "kyoto_tycoon":
            ktserver = KtserverLauncher()
            ktserver.spawnServer(self.experiment)

        # get parameters that cactus_workflow stuff wants
        wfOpts, sequences = getWorkflowParams(self.options, self.experiment)

        if self.options.setupAndBuildAlignments:
            self.addChildTarget(CactusSetupPhase(wfOpts, sequences))
            logger.info("Going to create alignments and define the cactus tree")

        self.setFollowOnTarget(ExtractReference(self.options,
                                                self.project,
                                                self.experiment,
                                                self.event,
                                                ktserver))

class ExtractReference(Target):
    def __init__(self, options, project, experiment, event, ktserver):
        Target.__init__(self)
        self.options = options
        self.project = project
        self.experiment = experiment
        self.event = event
        self.ktserver = ktserver

    def run(self):
        if self.options.buildReference:
            logger.info("Starting Reference Extract Phase")

            cmdLine = "cactus_getReferenceSeq --cactusDisk \'%s\' \
             --flowerName 0 \
            --referenceEventString %s --outputFile %s --logLevel %s" % \
            (self.experiment.getDiskDatabaseString(), self.event,
             self.experiment.getReferencePath(), getLogLevelString())

            system(cmdLine)

        self.setFollowOnTarget(BuildMAF(self.options, self.project,
                                        self.experiment,
                                        self.event, self.ktserver))

class BuildMAF(Target):
    def __init__(self, options, project, experiment, event, ktserver):
        Target.__init__(self)
        self.options = options
        self.project = project
        self.experiment = experiment
        self.event = event
        self.ktserver = ktserver

    def run(self):
        if self.options.buildMAF:
            logger.info("Starting MAF Build phase")

            cmdLine = "cactus_MAFGenerator --cactusDisk \'%s\' --flowerName 0 \
            --outputFile %s --logLevel %s" % \
            (self.experiment.getDiskDatabaseString(),
             self.experiment.getMAFPath(), getLogLevelString())

            system(cmdLine)
            removeOutgroupFromMaf(self.experiment.getMAFPath(),
                                  self.experiment.getOutgroupName())

        self.setFollowOnTarget(JoinMAF(self.options, self.project,
                                       self.experiment,
                                       self.event, self.ktserver))

class JoinMAF(Target):
    def __init__(self, options, project, experiment, event, ktserver):
        Target.__init__(self)
        self.options = options
        self.project = project
        self.experiment = experiment
        self.event = event
        self.ktserver = ktserver

    def run(self):
        # don't need the ktserver anymore, so we kill it
        if self.ktserver:
            self.ktserver.killServer()

        if self.options.joinMAF:
            logger.info("Starting MAF Join phase")

            rootRefName = self.experiment.getReferenceNameFromConfig()
            rootIsTreeMAF = False
            for child, path in self.experiment.seqMap.items():
                if child in self.project.expMap and\
                child != self.experiment.getOutgroupName():
                    childExpPath = self.project.expMap[child]
                    childExpXML = ET.parse(childExpPath).getroot()
                    childExp = ExperimentWrapper(childExpXML)
                    childRefName = childExp.getReferenceNameFromConfig()
                    cmdline = "mafJoin -maxBlkWidth=128 -maxInputBlkWidth=1000 "
                    if not rootIsTreeMAF:
                        cmdline += "-treelessRoot1=%s " % rootRefName
                    cmdline += "-treelessRoot2=%s " % childRefName
                    cmdline = "%s %s %s %s %s_tmp.maf" % (cmdline, childRefName,
                                                  self.experiment.getMAFPath(),
                                                  childExp.getMAFPath(),
                                                  self.experiment.getMAFPath())
                    system(cmdline)

                    move("%s_tmp.maf" % self.experiment.getMAFPath(),
                         self.experiment.getMAFPath())
                    rootIsTreeMAF = True

# copy and pasted from cactus_workflow.py
def getWorkflowParams(baseOptions, experiment):
    options = copy.deepcopy(baseOptions)
    options.experimentFile = experiment.xmlRoot
    #Get the database string
    options.cactusDiskDatabaseString = ET.tostring(options.experimentFile.find("cactus_disk").find("st_kv_database_conf"))
    #Get the species tree
    options.speciesTree = options.experimentFile.attrib["species_tree"]
    #Parse the config file which contains all the program options
    if options.experimentFile.attrib["config"] == "default":
        options.experimentFile.attrib["config"] = os.path.join(cactusRootPath(), "pipeline", "cactus_workflow_config.xml")
    else:
        logger.info("Using user specified experiment file")
    #Get the config file for the experiment
    options.config = ET.parse(options.experimentFile.attrib["config"]).getroot()
    #Get the sequences
    sequences = options.experimentFile.attrib["sequences"].split()
    #Get any list of 'required species' for the blocks of the cactus.
    options.requiredSpecies = getOptionalAttrib(options.experimentFile, "required_species")
    options.singleCopySpecies = getOptionalAttrib(options.experimentFile, "single_copy_species")
    logger.info("Parsed the XML options file")
    return options, sequences

def main():
    usage = "usage: %prog [options] <multicactus project>"
    description = "Progressive version of cactus_workflow"
    parser = OptionParser(usage=usage, description=description)
    Stack.addJobTreeOptions(parser)

    parser.add_option("--setupAndBuildAlignments", dest="setupAndBuildAlignments", action="store_true",
                      help="Setup and build alignments then normalise the resulting structure", default=False)

    parser.add_option("--buildMAF", dest="buildMAF", action="store_true",
                     help="Create a MAF file from the cactus [default=false]", default=False)

    parser.add_option("--joinMAF", dest="joinMAF", action="store_true",
                     help="Progressively join all cactus MAFs[default=false]", default=False)

    parser.add_option("--recursive", dest="recursive",
                      help="Recursively process subjobs [default=True]", default="True")

    parser.add_option("--event", dest="event",
                      help="Target event to process [default=root]", default=None)

    options, args = parser.parse_args()
    setLoggingFromOptions(options)

    # cannot align without building reference and vice versa in progressive mode
    options.buildReference = options.setupAndBuildAlignments
    # forget about this stuff for now
    options.buildTrees = False
    options.buildFaces = False

    options.recursive = not options.recursive.lower() == "false"

    if len(args) != 1:
        parser.print_help()
        raise RuntimeError("Unrecognised input arguments: %s" % " ".join(args))

    project = MultiCactusProject()
    project.readXML(args[0])
#    check no longer works, thanks to outgroup
#    project.check()
    schedule = Schedule()
    schedule.compute(project)
    if options.event == None:
        options.event = project.mcTree.tree.iD
    assert options.event in project.expMap

    baseTarget = ProgressiveDown(options, project, options.event, schedule)
    Stack(baseTarget).startJobTree(options)

def _test():
    import doctest
    return doctest.testmod()

if __name__ == '__main__':
    from cactus.progressive.cactus_progressive import *
    _test()
    main()<commit_msg>update for changes to schedule class<commit_after>#!/usr/bin/env python

#Copyright (C) 2011 by Glenn Hickey
#
#Released under the MIT license, see LICENSE.txt
#!/usr/bin/env python

"""Wrapper to run the cactus_workflow progressively, using the input species tree as a guide

tree.   The --buildMAF and --joinMAF options can be used to create and merge MAFs from the

cacti.   The --buildReference option is removed, as references *are always* built when

--setupAndBuildAlignments is specified.  If the kyoto tycoon DB is used, it is best to use the helper

script, preKtserverDbs.py to set up the server.
"""

import os
import xml.etree.ElementTree as ET
import math
from optparse import OptionParser
from collections import deque
import random
from itertools import izip
from shutil import move
import copy
from time import sleep

from sonLib.bioio import getTempFile
from sonLib.bioio import printBinaryTree
from sonLib.bioio import system

from jobTree.src.bioio import getLogLevelString
from jobTree.src.bioio import logger
from jobTree.src.bioio import setLoggingFromOptions

from cactus.shared.common import cactusRootPath

from jobTree.scriptTree.target import Target
from jobTree.scriptTree.stack import Stack

from cactus.pipeline.cactus_workflow import CactusSetupPhase
from cactus.pipeline.cactus_workflow import CactusPhylogenyPhase
from cactus.pipeline.cactus_workflow import CactusReferencePhase
from cactus.pipeline.cactus_workflow import CactusFacesPhase
from cactus.pipeline.cactus_workflow import getOptionalAttrib

from cactus.progressive.multiCactusProject import MultiCactusProject
from cactus.progressive.multiCactusTree import MultiCactusTree
from cactus.progressive.experimentWrapper import ExperimentWrapper
from cactus.progressive.ktserverLauncher import KtserverLauncher
from cactus.progressive.mafFilter import removeOutgroupFromMaf
from cactus.progressive.schedule import Schedule

class ProgressiveDown(Target):
    def __init__(self, options, project, event, schedule):
        Target.__init__(self)
        self.options = options
        self.project = project
        self.event = event
        self.schedule = schedule

    def run(self):
        logger.info("Progressive Down: " + self.event)

        expXml = ET.parse(self.project.expMap[self.event]).getroot()
        experiment = ExperimentWrapper(expXml)

        if self.options.recursive:
            deps = self.schedule.deps(self.event)
            for child in deps:
                if child in self.project.expMap:
                    self.addChildTarget(ProgressiveDown(self.options,
                                                        self.project, child,
                                                        self.schedule))

        self.setFollowOnTarget(ProgressiveUp(self.options, self.project,
                               experiment, self.event))

class ProgressiveUp(Target):
    def __init__(self, options, project, experiment, event):
        Target.__init__(self)
        self.options = options
        self.project = project
        self.experiment = experiment
        self.event = event

    def run(self):
        logger.info("Progressive Up: " + self.event)

        # delete database files if --setupAndBuildAlignments
        if self.options.setupAndBuildAlignments:
            dbPath = os.path.join(self.experiment.getDbDir(),
                                  self.experiment.getDbName())
            seqPath = os.path.join(self.experiment.getDbDir(), "sequences")
            if os.path.isfile(dbPath):
                os.remove(dbPath)
            if os.path.isfile(seqPath):
                os.remove(seqPath)

        ktserver = None
        if self.experiment.getDbType() == "kyoto_tycoon":
            ktserver = KtserverLauncher()
            ktserver.spawnServer(self.experiment)

        # get parameters that cactus_workflow stuff wants
        wfOpts, sequences = getWorkflowParams(self.options, self.experiment)

        if self.options.setupAndBuildAlignments:
            self.addChildTarget(CactusSetupPhase(wfOpts, sequences))
            logger.info("Going to create alignments and define the cactus tree")

        self.setFollowOnTarget(ExtractReference(self.options,
                                                self.project,
                                                self.experiment,
                                                self.event,
                                                ktserver))

class ExtractReference(Target):
    def __init__(self, options, project, experiment, event, ktserver):
        Target.__init__(self)
        self.options = options
        self.project = project
        self.experiment = experiment
        self.event = event
        self.ktserver = ktserver

    def run(self):
        if self.options.buildReference:
            logger.info("Starting Reference Extract Phase")

            cmdLine = "cactus_getReferenceSeq --cactusDisk \'%s\' \
             --flowerName 0 \
            --referenceEventString %s --outputFile %s --logLevel %s" % \
            (self.experiment.getDiskDatabaseString(), self.event,
             self.experiment.getReferencePath(), getLogLevelString())

            system(cmdLine)

        self.setFollowOnTarget(BuildMAF(self.options, self.project,
                                        self.experiment,
                                        self.event, self.ktserver))

class BuildMAF(Target):
    def __init__(self, options, project, experiment, event, ktserver):
        Target.__init__(self)
        self.options = options
        self.project = project
        self.experiment = experiment
        self.event = event
        self.ktserver = ktserver

    def run(self):
        if self.options.buildMAF:
            logger.info("Starting MAF Build phase")

            cmdLine = "cactus_MAFGenerator --cactusDisk \'%s\' --flowerName 0 \
            --outputFile %s --logLevel %s" % \
            (self.experiment.getDiskDatabaseString(),
             self.experiment.getMAFPath(), getLogLevelString())

            system(cmdLine)
            removeOutgroupFromMaf(self.experiment.getMAFPath(),
                                  self.experiment.getOutgroupName())

        self.setFollowOnTarget(JoinMAF(self.options, self.project,
                                       self.experiment,
                                       self.event, self.ktserver))

class JoinMAF(Target):
    def __init__(self, options, project, experiment, event, ktserver):
        Target.__init__(self)
        self.options = options
        self.project = project
        self.experiment = experiment
        self.event = event
        self.ktserver = ktserver

    def run(self):
        # don't need the ktserver anymore, so we kill it
        if self.ktserver:
            self.ktserver.killServer()

        if self.options.joinMAF:
            logger.info("Starting MAF Join phase")

            rootRefName = self.experiment.getReferenceNameFromConfig()
            rootIsTreeMAF = False
            for child, path in self.experiment.seqMap.items():
                if child in self.project.expMap and\
                child != self.experiment.getOutgroupName():
                    childExpPath = self.project.expMap[child]
                    childExpXML = ET.parse(childExpPath).getroot()
                    childExp = ExperimentWrapper(childExpXML)
                    childRefName = childExp.getReferenceNameFromConfig()
                    cmdline = "mafJoin -maxBlkWidth=128 -maxInputBlkWidth=1000 "
                    if not rootIsTreeMAF:
                        cmdline += "-treelessRoot1=%s " % rootRefName
                    cmdline += "-treelessRoot2=%s " % childRefName
                    cmdline = "%s %s %s %s %s_tmp.maf" % (cmdline, childRefName,
                                                  self.experiment.getMAFPath(),
                                                  childExp.getMAFPath(),
                                                  self.experiment.getMAFPath())
                    system(cmdline)

                    move("%s_tmp.maf" % self.experiment.getMAFPath(),
                         self.experiment.getMAFPath())
                    rootIsTreeMAF = True

# copy and pasted from cactus_workflow.py
def getWorkflowParams(baseOptions, experiment):
    options = copy.deepcopy(baseOptions)
    options.experimentFile = experiment.xmlRoot
    #Get the database string
    options.cactusDiskDatabaseString = ET.tostring(options.experimentFile.find("cactus_disk").find("st_kv_database_conf"))
    #Get the species tree
    options.speciesTree = options.experimentFile.attrib["species_tree"]
    #Parse the config file which contains all the program options
    if options.experimentFile.attrib["config"] == "default":
        options.experimentFile.attrib["config"] = os.path.join(cactusRootPath(), "pipeline", "cactus_workflow_config.xml")
    else:
        logger.info("Using user specified experiment file")
    #Get the config file for the experiment
    options.config = ET.parse(options.experimentFile.attrib["config"]).getroot()
    #Get the sequences
    sequences = options.experimentFile.attrib["sequences"].split()
    #Get any list of 'required species' for the blocks of the cactus.
    options.requiredSpecies = getOptionalAttrib(options.experimentFile, "required_species")
    options.singleCopySpecies = getOptionalAttrib(options.experimentFile, "single_copy_species")
    logger.info("Parsed the XML options file")
    return options, sequences

def main():
    usage = "usage: %prog [options] <multicactus project>"
    description = "Progressive version of cactus_workflow"
    parser = OptionParser(usage=usage, description=description)
    Stack.addJobTreeOptions(parser)

    parser.add_option("--setupAndBuildAlignments", dest="setupAndBuildAlignments", action="store_true",
                      help="Setup and build alignments then normalise the resulting structure", default=False)

    parser.add_option("--buildMAF", dest="buildMAF", action="store_true",
                     help="Create a MAF file from the cactus [default=false]", default=False)

    parser.add_option("--joinMAF", dest="joinMAF", action="store_true",
                     help="Progressively join all cactus MAFs[default=false]", default=False)

    parser.add_option("--recursive", dest="recursive",
                      help="Recursively process subjobs [default=True]", default="True")

    parser.add_option("--event", dest="event",
                      help="Target event to process [default=root]", default=None)

    options, args = parser.parse_args()
    setLoggingFromOptions(options)

    # cannot align without building reference and vice versa in progressive mode
    options.buildReference = options.setupAndBuildAlignments
    # forget about this stuff for now
    options.buildTrees = False
    options.buildFaces = False

    options.recursive = not options.recursive.lower() == "false"

    if len(args) != 1:
        parser.print_help()
        raise RuntimeError("Unrecognised input arguments: %s" % " ".join(args))

    project = MultiCactusProject()
    project.readXML(args[0])
    schedule = Schedule()
    schedule.loadProject(project)
    schedule.compute()
    if options.event == None:
        options.event = project.mcTree.getRootName()
    assert options.event in project.expMap

    baseTarget = ProgressiveDown(options, project, options.event, schedule)
    Stack(baseTarget).startJobTree(options)

def _test():
    import doctest
    return doctest.testmod()

if __name__ == '__main__':
    from cactus.progressive.cactus_progressive import *
    _test()
    main()<|endoftext|>


<commit_before>"Multiclass Scatterplot Module."

import numpy as np
from ..table.nary import NAry
from ..stats import MCHistogram2D, Sample
from ..table.range_query_2d import RangeQuery2d
from ..utils.errors import ProgressiveError
from ..core.utils import is_notebook, get_physical_base
from ..io import Variable, VirtualVariable
from itertools import chain

from collections import defaultdict

class _DataClass(object):
    def __init__(self, name, group, x_column, y_column, approximate=False,
                 scheduler=None, **kwds):
        self.name = name
        self._group = group
        self.x_column = x_column
        self.y_column = y_column
        self._approximate = approximate
        self._scheduler = scheduler
        self.input_module = None
        self.input_slot = None
        self.min = None
        self.max = None
        self.histogram2d = None
        self.heatmap = None
        self.min_value = None
        self.max_value = None
        self.sample = None
        self.range_query_2d = None

    def scheduler(self):
        return self._scheduler

    def create_dependent_modules(self, input_module, input_slot,
                                 histogram2d=None, heatmap=None,
                                 **kwds):
        if self.input_module is not None:
            return self
        scheduler = self.scheduler()
        with scheduler:
            self.input_module = input_module
            self.input_slot = input_slot
            range_query_2d = RangeQuery2d(column_x=self.x_column,
                                          column_y=self.y_column,
                                          group=self._group,
                                          scheduler=scheduler,
                                          approximate=self._approximate)
            range_query_2d.create_dependent_modules(input_module,
                                                    input_slot,
                                                    min_value=False,
                                                    max_value=False)
            self.min_value = Variable(group=self._group, scheduler=scheduler)
            self.min_value.input.like = range_query_2d.min.output.table
            range_query_2d.input.lower = self.min_value.output.table
            self.max_value = Variable(group=self._group, scheduler=scheduler)
            self.max_value.input.like = range_query_2d.max.output.table
            range_query_2d.input.upper = self.max_value.output.table
            if histogram2d is None:
                histogram2d = MCHistogram2D(self.x_column, self.y_column,
                                            group=self._group,
                                            scheduler=scheduler)
            histogram2d.input.data = range_query_2d.output.table
            if self.sample == 'default':
                self.sample = Sample(samples=100, group=self._group,
                                     scheduler=scheduler)
            # elif sample is None and select is None:
            #    raise ProgressiveError("Scatterplot needs a select module")
            # if sample is not None:
            #    sample.input.table = range_query_2d.output.table
            if isinstance(self.sample, Sample):
                self.sample.input.table = range_query_2d.output.table
            self.histogram2d = histogram2d
            # self.sample = sample
            # self.select = select
            self.min = range_query_2d.min.output.table
            self.max = range_query_2d.max.output.table
            self.range_query_2d = range_query_2d
        scatterplot = self
        return scatterplot

class MCScatterPlot(NAry):
    "Module executing multiclass."
    def __init__(self, classes, x_label="x", y_label="y", approximate=False,
                 **kwds):
        """Multiclass ...
        """
        super(MCScatterPlot, self).__init__(**kwds)
        self._classes = classes  # TODO: check it ...
        self._x_label = x_label
        self._y_label = y_label
        self._approximate = approximate
        self._json_cache = None
        self.input_module = None
        self.input_slot = None
        self._data_class_dict = {}
        self.min_value = None
        self.max_value = None
        self._ipydata = is_notebook()
        self.hist_tensor = None
        self.sample_tensor = None

    def forget_changes(self, input_slot):
        changes = False
        if input_slot.deleted.any():
            input_slot.deleted.next()
            changes = True
        if input_slot.created.any():
            input_slot.created.next()
            changes = True
        if input_slot.updated.any():
            input_slot.updated.next()
            changes = True
        return changes

    def is_visualization(self):
        return True

    def get_visualization(self):
        return "mcscatterplot"

    def predict_step_size(self, duration):
        return 1

    def group_inputs(self):
        """
        Group inputs by classes using meta field on slots
        """
        ret = defaultdict(dict)
        changes = False
        for name in self.inputs:
            input_slot = self.get_input_slot(name)
            if input_slot is None:
                continue
            meta = input_slot.meta
            if meta is None:
                continue
            input_type = meta['inp']
            class_ = meta['class_']
            if input_type not in ('hist', 'sample'):
                raise ValueError('{} not in [hist, sample]'.format(input_type))
            changes |= self.forget_changes(input_slot)
            ret[class_].update({
                input_type: (input_slot, meta['x'], meta['y'])
            })
        return changes, ret

    def build_heatmap(self, inp, domain, plan):
        inp_table = inp.data()
        if inp_table is None:
            return None
        #with inp_table.lock:
        last = inp_table.last()
        if last is None:
            return None
        row = last.to_dict()
        json_ = {}
        if not (np.isnan(row['xmin']) or np.isnan(row['xmax'])
                or np.isnan(row['ymin']) or np.isnan(row['ymax'])):
            data = row['array']
            json_['bounds'] = (row['xmin'], row['ymin'],
                               row['xmax'], row['ymax'])
            if self._ipydata:
                assert isinstance(plan, int)
                json_['binnedPixels'] = plan
                self.hist_tensor[:,:,plan] = row['array']
            else:
                data = np.copy(row['array'])
                json_['binnedPixels'] = data
            json_['range'] = [np.min(data), np.max(data)]
            json_['count'] = np.sum(data)
            json_['value'] = domain
            return json_
        return None

    def make_json(self, json):
        buffers = []
        domain = []
        samples = []
        count = 0
        xmin = ymin = - np.inf
        xmax = ymax = np.inf
        changes, grouped_inputs = self.group_inputs()
        if self._ipydata and self.hist_tensor is None:
            z = len(grouped_inputs)
            for sl in grouped_inputs.values():
                hi = sl['hist'][0]
                xbins = hi.output_module.params.xbins
                ybins = hi.output_module.params.ybins
                self.hist_tensor = np.zeros((xbins, ybins, z), dtype='int32')
                sam = sl['sample'][0]
                nsam = sam.output_module.params.samples
                self.sample_tensor = np.zeros((nsam, 2, z), dtype='float32')
                break
        for i, (cname, inputs) in enumerate(grouped_inputs.items()):
            hist_input = inputs['hist'][0]
            buff = self.build_heatmap(hist_input, cname, i)
            if buff is None:
                return json
            xmin_, ymin_, xmax_, ymax_ = buff.pop('bounds')
            xmin = max(xmin, xmin_)
            ymin = max(ymin, ymin_)
            xmax = min(xmax, xmax_)
            ymax = min(ymax, ymax_)
            buffers.append(buff)
            count += buff['count']
            domain.append(cname)
            sample_input = inputs['sample'][0]
            select = sample_input.data()
            x_column, y_column = inputs['sample'][1],  inputs['sample'][2]
            if self._ipydata:
                smpl = []
                if select is not None:
                    assert self.sample_tensor.shape[0] == len(select)
                    ph_x = get_physical_base(select[x_column])
                    self.sample_tensor[:,0,i] = ph_x.loc[select[x_column].index.values]
                    ph_y = get_physical_base(select[y_column])
                    self.sample_tensor[:,1,i] = ph_y.loc[select[y_column].index.values]

            else:
                smpl = select.to_json(orient='split', columns=[x_column, y_column]) if select is not None else []
            samples.append(smpl)

        # TODO: check consistency among classes (e.g. same xbin, ybin etc.)
        xbins, ybins = self.hist_tensor.shape[:-1] if self._ipydata else buffers[0]['binnedPixels'].shape
        encoding = {
            "x": {
                "bin": {
                    "maxbins": xbins
                },
                "aggregate": "count",
                "field": self._x_label,
                "type": "quantitative",
                "scale": {
                    "domain": [
                            -7,
                        7
                    ],
                    "range": [
                        0,
                        xbins
                    ]
                }
            },
            "z": {
                "field": "category",
                "type": "nominal",
                "scale": {
                    "domain": domain
                }
            },
            "y": {
                "bin": {
                    "maxbins": ybins
                },
                "aggregate": "count",
                "field": self._y_label,
                "type": "quantitative",
                "scale": {
                    "domain": [
                            -7,
                        7
                    ],
                    "range": [
                        0,
                        ybins
                    ]
                }
            }
        }
        source = {"program": "progressivis",
                  "type": "python",
                  "rows": count
        }
        json['chart'] = dict(buffers=buffers, encoding=encoding, source=source)
        json['bounds'] = dict(xmin=xmin, ymin=ymin, xmax=xmax, ymax=ymax)
        s_data = []
        for i, s in enumerate(samples):
            if not s: continue
            d = s['data']
            for row in d:
                row.append(i)
            s_data.extend(d)
        json['sample'] = dict(data=s_data, index=list(range(len(s_data))))
        json['columns'] = [self._x_label, self._y_label]
        if self._ipydata:
            json['hist_tensor'] = self.hist_tensor
            json['sample_tensor'] = self.sample_tensor
        return json

    def run_step(self, run_number, step_size, howlong):
        for name in self.get_input_slot_multiple():
            slot = self.get_input_slot(name)
            # slot.update(run_number)
            if slot.has_buffered():
                slot.clear_buffers()
                # slot.created.next()
                # slot.updated.next()
                # slot.deleted.next()
                self._json_cache = None
                #print("SLOT has buffered", slot)
        return self._return_run_step(self.state_blocked, steps_run=0)

    def run(self, run_number):
        super(MCScatterPlot, self).run(run_number)
        if self._ipydata:
            return
        if self._json_cache is not None:
            return
        self._json_cache = self._to_json_impl()

    def to_json(self, short=False):
        if self._json_cache:
            return self._json_cache
        self._json_cache = self._to_json_impl(short)
        return self._json_cache

    def _to_json_impl(self, short=False):
        self.image = None
        json = super(MCScatterPlot, self).to_json(short, with_speed=False)
        if short:
            return json
        return self.make_json(json)

    def create_dependent_modules(self, input_module, input_slot,
                                 sample='default', **kwds):
        self.input_module = input_module
        self.input_slot = input_slot
        scheduler = self.scheduler()
        with scheduler:
            self.min_value = VirtualVariable([self._x_label, self._y_label],
                                             scheduler=scheduler)
            self.max_value = VirtualVariable([self._x_label, self._y_label],
                                             scheduler=scheduler)
            for cl in self._classes:
                if isinstance(cl, tuple):
                    self._add_class(*cl)
                else:
                    self._add_class(**cl)
            self._finalize()

    def __getitem__(self, key):
        return self._data_class_dict[key]

    def _finalize(self):
        for dc in self._data_class_dict.values():
            for dc2 in self._data_class_dict.values():
                x, y = dc2.x_column, dc2.y_column
                dc.histogram2d.input['table', ('min', x, y)] = dc2.range_query_2d.output.min
                dc.histogram2d.input['table', ('max', x, y)] = dc2.range_query_2d.output.max

    def _add_class(self, name, x_column, y_column, sample='default'):
        if self.input_module is None or self.input_slot is None:
            raise ProgressiveError("You have to create "
                                   "the dependent modules first!")
        data_class = _DataClass(name, self.name, x_column,
                                y_column,
                                approximate=self._approximate,
                                scheduler=self._scheduler)
        data_class.sample = sample
        data_class.create_dependent_modules(self.input_module, self.input_slot)
        col_translation = {self._x_label: x_column, self._y_label: y_column}
        hist_meta = dict(inp='hist', class_=name, **col_translation)
        sample_meta = dict(inp='sample', class_=name, **col_translation)
        self.input['table', hist_meta] = data_class.histogram2d.output.table
        self.input['table', sample_meta] = data_class.sample.output.table
        self._data_class_dict[name] = data_class
        self.min_value.subscribe(data_class.min_value, col_translation)
        self.max_value.subscribe(data_class.max_value, col_translation)

    def get_starving_mods(self):
        return chain(*[(s.histogram2d, s.sample)
                       for s in self._data_class_dict.values()])
<commit_msg>provisional fix<commit_after>"Multiclass Scatterplot Module."

import numpy as np
from ..table.nary import NAry
from ..stats import MCHistogram2D, Sample
from ..table.range_query_2d import RangeQuery2d
from ..utils.errors import ProgressiveError
from ..core.utils import is_notebook, get_physical_base
from ..io import Variable, VirtualVariable
from itertools import chain

from collections import defaultdict

class _DataClass(object):
    def __init__(self, name, group, x_column, y_column, approximate=False,
                 scheduler=None, **kwds):
        self.name = name
        self._group = group
        self.x_column = x_column
        self.y_column = y_column
        self._approximate = approximate
        self._scheduler = scheduler
        self.input_module = None
        self.input_slot = None
        self.min = None
        self.max = None
        self.histogram2d = None
        self.heatmap = None
        self.min_value = None
        self.max_value = None
        self.sample = None
        self.range_query_2d = None

    def scheduler(self):
        return self._scheduler

    def create_dependent_modules(self, input_module, input_slot,
                                 histogram2d=None, heatmap=None,
                                 **kwds):
        if self.input_module is not None:
            return self
        scheduler = self.scheduler()
        with scheduler:
            self.input_module = input_module
            self.input_slot = input_slot
            range_query_2d = RangeQuery2d(column_x=self.x_column,
                                          column_y=self.y_column,
                                          group=self._group,
                                          scheduler=scheduler,
                                          approximate=self._approximate)
            range_query_2d.create_dependent_modules(input_module,
                                                    input_slot,
                                                    min_value=False,
                                                    max_value=False)
            self.min_value = Variable(group=self._group, scheduler=scheduler)
            self.min_value.input.like = range_query_2d.min.output.table
            range_query_2d.input.lower = self.min_value.output.table
            self.max_value = Variable(group=self._group, scheduler=scheduler)
            self.max_value.input.like = range_query_2d.max.output.table
            range_query_2d.input.upper = self.max_value.output.table
            if histogram2d is None:
                histogram2d = MCHistogram2D(self.x_column, self.y_column,
                                            group=self._group,
                                            scheduler=scheduler)
            histogram2d.input.data = range_query_2d.output.table
            if self.sample == 'default':
                self.sample = Sample(samples=100, group=self._group,
                                     scheduler=scheduler)
            # elif sample is None and select is None:
            #    raise ProgressiveError("Scatterplot needs a select module")
            # if sample is not None:
            #    sample.input.table = range_query_2d.output.table
            if isinstance(self.sample, Sample):
                self.sample.input.table = range_query_2d.output.table
            self.histogram2d = histogram2d
            # self.sample = sample
            # self.select = select
            self.min = range_query_2d.min.output.table
            self.max = range_query_2d.max.output.table
            self.range_query_2d = range_query_2d
        scatterplot = self
        return scatterplot

class MCScatterPlot(NAry):
    "Module executing multiclass."
    def __init__(self, classes, x_label="x", y_label="y", approximate=False,
                 **kwds):
        """Multiclass ...
        """
        super(MCScatterPlot, self).__init__(**kwds)
        self._classes = classes  # TODO: check it ...
        self._x_label = x_label
        self._y_label = y_label
        self._approximate = approximate
        self._json_cache = None
        self.input_module = None
        self.input_slot = None
        self._data_class_dict = {}
        self.min_value = None
        self.max_value = None
        self._ipydata = is_notebook()
        self.hist_tensor = None
        self.sample_tensor = None

    def forget_changes(self, input_slot):
        changes = False
        if input_slot.deleted.any():
            input_slot.deleted.next()
            changes = True
        if input_slot.created.any():
            input_slot.created.next()
            changes = True
        if input_slot.updated.any():
            input_slot.updated.next()
            changes = True
        return changes

    def is_visualization(self):
        return True

    def get_visualization(self):
        return "mcscatterplot"

    def predict_step_size(self, duration):
        return 1

    def group_inputs(self):
        """
        Group inputs by classes using meta field on slots
        """
        ret = defaultdict(dict)
        changes = False
        for name in self.inputs:
            input_slot = self.get_input_slot(name)
            if input_slot is None:
                continue
            meta = input_slot.meta
            if meta is None:
                continue
            input_type = meta['inp']
            class_ = meta['class_']
            if input_type not in ('hist', 'sample'):
                raise ValueError('{} not in [hist, sample]'.format(input_type))
            changes |= self.forget_changes(input_slot)
            ret[class_].update({
                input_type: (input_slot, meta['x'], meta['y'])
            })
        return changes, ret

    def build_heatmap(self, inp, domain, plan):
        inp_table = inp.data()
        if inp_table is None:
            return None
        #with inp_table.lock:
        last = inp_table.last()
        if last is None:
            return None
        row = last.to_dict()
        json_ = {}
        if not (np.isnan(row['xmin']) or np.isnan(row['xmax'])
                or np.isnan(row['ymin']) or np.isnan(row['ymax'])):
            data = row['array']
            json_['bounds'] = (row['xmin'], row['ymin'],
                               row['xmax'], row['ymax'])
            if self._ipydata:
                assert isinstance(plan, int)
                json_['binnedPixels'] = plan
                self.hist_tensor[:,:,plan] = row['array']
            else:
                data = np.copy(row['array'])
                json_['binnedPixels'] = data
            json_['range'] = [np.min(data), np.max(data)]
            json_['count'] = np.sum(data)
            json_['value'] = domain
            return json_
        return None

    def make_json(self, json):
        buffers = []
        domain = []
        samples = []
        count = 0
        xmin = ymin = - np.inf
        xmax = ymax = np.inf
        changes, grouped_inputs = self.group_inputs()
        if self._ipydata and self.hist_tensor is None:
            z = len(grouped_inputs)
            for sl in grouped_inputs.values():
                hi = sl['hist'][0]
                xbins = hi.output_module.params.xbins
                ybins = hi.output_module.params.ybins
                self.hist_tensor = np.zeros((xbins, ybins, z), dtype='int32')
                sam = sl['sample'][0]
                nsam = sam.output_module.params.samples
                self.sample_tensor = np.zeros((nsam, 2, z), dtype='float32')
                break
        for i, (cname, inputs) in enumerate(grouped_inputs.items()):
            hist_input = inputs['hist'][0]
            buff = self.build_heatmap(hist_input, cname, i)
            if buff is None:
                return json
            xmin_, ymin_, xmax_, ymax_ = buff.pop('bounds')
            xmin = max(xmin, xmin_)
            ymin = max(ymin, ymin_)
            xmax = min(xmax, xmax_)
            ymax = min(ymax, ymax_)
            buffers.append(buff)
            count += buff['count']
            domain.append(cname)
            sample_input = inputs['sample'][0]
            select = sample_input.data()
            x_column, y_column = inputs['sample'][1],  inputs['sample'][2]
            if self._ipydata:
                smpl = []
                if select is not None:
                    assert self.sample_tensor.shape[0] == len(select)
                    ph_x = get_physical_base(select[x_column])
                    self.sample_tensor[:,0,i] = ph_x.loc[select[x_column].index.values]
                    ph_y = get_physical_base(select[y_column])
                    self.sample_tensor[:,1,i] = ph_y.loc[select[y_column].index.values]

            else:
                smpl = select.to_json(orient='split', columns=[x_column, y_column]) if select is not None else []
            samples.append(smpl)

        # TODO: check consistency among classes (e.g. same xbin, ybin etc.)
        xbins, ybins = self.hist_tensor.shape[:-1] if self._ipydata else buffers[0]['binnedPixels'].shape
        encoding = {
            "x": {
                "bin": {
                    "maxbins": xbins
                },
                "aggregate": "count",
                "field": self._x_label,
                "type": "quantitative",
                "scale": {
                    "domain": [
                            -7,
                        7
                    ],
                    "range": [
                        0,
                        xbins
                    ]
                }
            },
            "z": {
                "field": "category",
                "type": "nominal",
                "scale": {
                    "domain": domain
                }
            },
            "y": {
                "bin": {
                    "maxbins": ybins
                },
                "aggregate": "count",
                "field": self._y_label,
                "type": "quantitative",
                "scale": {
                    "domain": [
                            -7,
                        7
                    ],
                    "range": [
                        0,
                        ybins
                    ]
                }
            }
        }
        source = {"program": "progressivis",
                  "type": "python",
                  "rows": count
        }
        json['chart'] = dict(buffers=buffers, encoding=encoding, source=source)
        json['bounds'] = dict(xmin=xmin, ymin=ymin, xmax=xmax, ymax=ymax)
        s_data = []
        for i, s in enumerate(samples):
            if not s: continue
            d = s['data']
            for row in d:
                row.append(i)
            s_data.extend(d)
        json['sample'] = dict(data=s_data, index=list(range(len(s_data))))
        json['columns'] = [self._x_label, self._y_label]
        if self._ipydata:
            json['hist_tensor'] = self.hist_tensor
            json['sample_tensor'] = self.sample_tensor
        return json

    def run_step(self, run_number, step_size, howlong):
        for name in self.get_input_slot_multiple():
            slot = self.get_input_slot(name)
            # slot.update(run_number)
            if slot.has_buffered():
                slot.clear_buffers()
                # slot.created.next()
                # slot.updated.next()
                # slot.deleted.next()
                #print("SLOT has buffered", slot)
        self._json_cache = None
        return self._return_run_step(self.state_blocked, steps_run=0)

    def run(self, run_number):
        super(MCScatterPlot, self).run(run_number)
        if self._ipydata:
            return
        if self._json_cache is not None:
            return
        self._json_cache = self._to_json_impl()

    def to_json(self, short=False):
        if self._json_cache:
            return self._json_cache
        self._json_cache = self._to_json_impl(short)
        return self._json_cache

    def _to_json_impl(self, short=False):
        self.image = None
        json = super(MCScatterPlot, self).to_json(short, with_speed=False)
        if short:
            return json
        return self.make_json(json)

    def create_dependent_modules(self, input_module, input_slot,
                                 sample='default', **kwds):
        self.input_module = input_module
        self.input_slot = input_slot
        scheduler = self.scheduler()
        with scheduler:
            self.min_value = VirtualVariable([self._x_label, self._y_label],
                                             scheduler=scheduler)
            self.max_value = VirtualVariable([self._x_label, self._y_label],
                                             scheduler=scheduler)
            for cl in self._classes:
                if isinstance(cl, tuple):
                    self._add_class(*cl)
                else:
                    self._add_class(**cl)
            self._finalize()

    def __getitem__(self, key):
        return self._data_class_dict[key]

    def _finalize(self):
        for dc in self._data_class_dict.values():
            for dc2 in self._data_class_dict.values():
                x, y = dc2.x_column, dc2.y_column
                dc.histogram2d.input['table', ('min', x, y)] = dc2.range_query_2d.output.min
                dc.histogram2d.input['table', ('max', x, y)] = dc2.range_query_2d.output.max

    def _add_class(self, name, x_column, y_column, sample='default'):
        if self.input_module is None or self.input_slot is None:
            raise ProgressiveError("You have to create "
                                   "the dependent modules first!")
        data_class = _DataClass(name, self.name, x_column,
                                y_column,
                                approximate=self._approximate,
                                scheduler=self._scheduler)
        data_class.sample = sample
        data_class.create_dependent_modules(self.input_module, self.input_slot)
        col_translation = {self._x_label: x_column, self._y_label: y_column}
        hist_meta = dict(inp='hist', class_=name, **col_translation)
        sample_meta = dict(inp='sample', class_=name, **col_translation)
        self.input['table', hist_meta] = data_class.histogram2d.output.table
        self.input['table', sample_meta] = data_class.sample.output.table
        self._data_class_dict[name] = data_class
        self.min_value.subscribe(data_class.min_value, col_translation)
        self.max_value.subscribe(data_class.max_value, col_translation)

    def get_starving_mods(self):
        return chain(*[(s.histogram2d, s.sample)
                       for s in self._data_class_dict.values()])
<|endoftext|>


<commit_before>#!/usr/bin/env python2
__author__ = "Colin Wee"
__date__ = "2015-02-17"
__version__ = "0.2"

import sys, os
import datetime as dt, string
import ConfigParser

DATA_DIR = "resources"
TEMPLATE_FNAME = "template_mod.tex"
INFO_FNAME = "profile.cfg"
INFO_TITLE = "Student-Specific Information"
INFO_FIELDS = {
    "Name"        : "studentName",
    "Andrew ID"   : "andrewID",
    "Lab Section" : "labSection"
}

################################## Utilities ##################################

def formalDateToday():
    """Return a string representing today's date."""
    return dt.date.today().strftime("%B %d, %Y")

def digitsOnly(s):
    """Return True if s only contains digits only and False otherwise."""
    return (s == ''.join([c for c in s if c in string.digits]))

def YNInput(prompt=""):
    """Robustly accept yes/no input from user."""
    while True:
        cmd = raw_input("%sY/N --> " % prompt).upper()
        if cmd == "Y": return True
        elif cmd == "N": return False
        print "Invalid input. Try again."

def rawInputWithCheck(prompt):
    """Accept user input while allowing for correction of mistakes."""
    proceed = False
    i = None
    while not(proceed):
        i = raw_input(prompt)
        print "Is this correct?"
        print ' '*3, repr(i)
        proceed = YNInput(' '*2)
    return i

############################# Manage User Profile #############################

def initProfile():
    """
    Completes first-time setup of %s by taking user input.
    """ % os.path.join(DATA_DIR, INFO_FNAME)
    print "First time setup! You will only have to do this once."
    P = ConfigParser.RawConfigParser() # profile is a long word
    P.optionxform = str # preserve case in options
    P.add_section(INFO_TITLE)
    for field in INFO_FIELDS.keys():
        P.set(INFO_TITLE, field, rawInputWithCheck("\nEnter %s --> " % field))
    with open(os.path.join(DATA_DIR, INFO_FNAME), 'w') as F:
        P.write(F)

def readProfile():
    """Reads existing user info from %s""" os.path.join(DATA_DIR, INFO_FNAME)
    P = ConfigParser.RawConfigParser() # profile is a long word
    P.optionxform = str # preserve case in options
    P.read(os.path.join(DATA_DIR, INFO_FNAME))
    return dict(P.items(INFO_TITLE))

################################ Main Routine #################################

def main(argc, argv):
    print '#'*16, "15-150 Homework LaTeX Generator by Colin Wee", '#'*17, '\n'

    if ((argc != 2) or not(digitsOnly(argv[1]))):
        sys.exit("usage: %s <homework number>" % argv[0])

    hwNum = int(argv[1])
    outfile = "hw%02d.tex" % hwNum

    if not(os.access(os.path.join(DATA_DIR, TEMPLATE_FNAME), os.R_OK)):
        sys.exit("Error! Could not find my modded template at %s" %
                 os.path.join(DATA_DIR, TEMPLATE_FNAME))
    elif (os.path.isfile(outfile) and
          not(YNInput("%s already exists. Overwrite? " % outfile))):
        print "Exiting..."
        return 0
    elif not(os.access(os.path.join(DATA_DIR, INFO_FNAME), os.R_OK)):
        initProfile()

    profile = readProfile()

    with open(os.path.join(DATA_DIR, TEMPLATE_FNAME), 'r') as T:
        template = T.read()
        with open(outfile, 'w') as F:
            # I apologize for these lines. Python's string
            for field in INFO_FIELDS.keys():
                template = template.replace(INFO_FIELDS[field], profile[field])
            template = template.replace("homeworkNumber", str(hwNum))
            F.write(template.replace("currentDate", formalDateToday()))
    print "File %r successfully generated" % outfile
    return 0

if __name__ == "__main__":
    main(len(sys.argv), sys.argv)
<commit_msg>replaced multiple calls to os.path.join with global constants added a comment that apparently went missing<commit_after>#!/usr/bin/env python2
__author__ = "Colin Wee"
__date__ = "2015-02-17"
__version__ = "0.2"

import sys, os
import datetime as dt, string
import ConfigParser

DATA_DIR = "resources"
TEMPLATE_FNAME = "template_mod.tex"
INFO_FNAME = "profile.cfg"
INFO_TITLE = "Student-Specific Information"
INFO_FIELDS = {
    "Name"        : "studentName",
    "Andrew ID"   : "andrewID",
    "Lab Section" : "labSection"
}

INFO_RELPATH = os.path.join(DATA_DIR, INFO_FNAME)
TEMPLATE_RELPATH = os.path.join(DATA_DIR, TEMPLATE_FNAME)

################################## Utilities ##################################

def formalDateToday():
    """Return a string representing today's date."""
    return dt.date.today().strftime("%B %d, %Y")

def digitsOnly(s):
    """Return True if s only contains digits only and False otherwise."""
    return (s == ''.join([c for c in s if c in string.digits]))

def YNInput(prompt=""):
    """Robustly accept yes/no input from user."""
    while True:
        cmd = raw_input("%sY/N --> " % prompt).upper()
        if cmd == "Y": return True
        elif cmd == "N": return False
        print "Invalid input. Try again."

def rawInputWithCheck(prompt):
    """Accept user input while allowing for correction of mistakes."""
    proceed = False
    i = None
    while not(proceed):
        i = raw_input(prompt)
        print "Is this correct?"
        print ' '*3, repr(i)
        proceed = YNInput(' '*2)
    return i

############################# Manage User Profile #############################

def initProfile():
    """Completes first-time setup of %s by taking user input.""" % INFO_RELPATH
    print "First time setup! You will only have to do this once."
    P = ConfigParser.RawConfigParser() # profile is a long word
    P.optionxform = str # preserve case in options
    P.add_section(INFO_TITLE)
    for field in INFO_FIELDS.keys():
        P.set(INFO_TITLE, field, rawInputWithCheck("\nEnter %s --> " % field))
    with open(INFO_RELPATH, 'w') as F:
        P.write(F)

def readProfile():
    """Reads existing user info from %s""" % INFO_RELPATH
    P = ConfigParser.RawConfigParser() # profile is a long word
    P.optionxform = str # preserve case in options
    P.read(INFO_RELPATH)
    return dict(P.items(INFO_TITLE))

################################ Main Routine #################################

def main(argc, argv):
    print '#'*16, "15-150 Homework LaTeX Generator by Colin Wee", '#'*17, '\n'

    if ((argc != 2) or not(digitsOnly(argv[1]))):
        sys.exit("usage: %s <homework number>" % argv[0])

    hwNum = int(argv[1])
    outfile = "hw%02d.tex" % hwNum

    if not(os.access(TEMPLATE_RELPATH, os.R_OK)):
        sys.exit("Error! Could not find my modded template at %s" %
                 TEMPLATE_RELPATH)
    elif (os.path.isfile(outfile) and
          not(YNInput("%s already exists. Overwrite? " % outfile))):
        print "Exiting..."
        return 0
    elif not(os.access(INFO_RELPATH, os.R_OK)):
        initProfile()

    profile = readProfile()

    with open(TEMPLATE_RELPATH, 'r') as T:
        template = T.read()
        with open(outfile, 'w') as F:
            # I apologize for these lines
            # Python's string formatting is all-or-nothing
            for field in INFO_FIELDS.keys():
                template = template.replace(INFO_FIELDS[field], profile[field])
            template = template.replace("homeworkNumber", str(hwNum))
            F.write(template.replace("currentDate", formalDateToday()))
    print "File %r successfully generated" % outfile
    return 0

if __name__ == "__main__":
    main(len(sys.argv), sys.argv)
<|endoftext|>


<commit_before>import numpy as np
import scipy.ndimage as ndi
from ..filters import rank_order

def peak_local_max(image, min_distance=1, threshold_abs=None,
                   threshold_rel=None, exclude_border=1, indices=True,
                   num_peaks=np.inf, footprint=None, labels=None):
    """Find peaks in an image as coordinate list or boolean mask.

    Peaks are the local maxima in a region of `2 * min_distance + 1`
    (i.e. peaks are separated by at least `min_distance`).

    If peaks are flat (i.e. multiple adjacent pixels have identical
    intensities), the coordinates of all such pixels are returned.

    If both `threshold_abs` and `threshold_rel` are provided, the maximum
    of the two is chosen as the minimum intensity threshold of peaks.

    Parameters
    ----------
    image : ndarray
        Input image.
    min_distance : int, optional
        Minimum number of pixels separating peaks in a region of `2 *
        min_distance + 1` (i.e. peaks are separated by at least
        `min_distance`).
        To find the maximum number of peaks, use `min_distance=1`.
    threshold_abs : float, optional
        Minimum intensity of peaks. By default, the absolute threshold is
        the minimum intensity of the image.
    threshold_rel : float, optional
        Minimum intensity of peaks, calculated as `max(image) * threshold_rel`.
    exclude_border : int, optional
        If nonzero, `exclude_border` excludes peaks from
        within `exclude_border`-pixels of the border of the image.
    indices : bool, optional
        If True, the output will be an array representing peak
        coordinates.  If False, the output will be a boolean array shaped as
        `image.shape` with peaks present at True elements.
    num_peaks : int, optional
        Maximum number of peaks. When the number of peaks exceeds `num_peaks`,
        return `num_peaks` peaks based on highest peak intensity.
    footprint : ndarray of bools, optional
        If provided, `footprint == 1` represents the local region within which
        to search for peaks at every point in `image`.  Overrides
        `min_distance` (also for `exclude_border`).
    labels : ndarray of ints, optional
        If provided, each unique region `labels == value` represents a unique
        region to search for peaks. Zero is reserved for background.

    Returns
    -------
    output : ndarray or ndarray of bools

        * If `indices = True`  : (row, column, ...) coordinates of peaks.
        * If `indices = False` : Boolean array shaped like `image`, with peaks
          represented by True values.

    Notes
    -----
    The peak local maximum function returns the coordinates of local peaks
    (maxima) in an image. A maximum filter is used for finding local maxima.
    This operation dilates the original image. After comparison of the dilated
    and original image, this function returns the coordinates or a mask of the
    peaks where the dilated image equals the original image.

    Examples
    --------
    >>> img1 = np.zeros((7, 7))
    >>> img1[3, 4] = 1
    >>> img1[3, 2] = 1.5
    >>> img1
    array([[ 0. ,  0. ,  0. ,  0. ,  0. ,  0. ,  0. ],
           [ 0. ,  0. ,  0. ,  0. ,  0. ,  0. ,  0. ],
           [ 0. ,  0. ,  0. ,  0. ,  0. ,  0. ,  0. ],
           [ 0. ,  0. ,  1.5,  0. ,  1. ,  0. ,  0. ],
           [ 0. ,  0. ,  0. ,  0. ,  0. ,  0. ,  0. ],
           [ 0. ,  0. ,  0. ,  0. ,  0. ,  0. ,  0. ],
           [ 0. ,  0. ,  0. ,  0. ,  0. ,  0. ,  0. ]])

    >>> peak_local_max(img1, min_distance=1)
    array([[3, 2],
           [3, 4]])

    >>> peak_local_max(img1, min_distance=2)
    array([[3, 2]])

    >>> img2 = np.zeros((20, 20, 20))
    >>> img2[10, 10, 10] = 1
    >>> peak_local_max(img2, exclude_border=0)
    array([[10, 10, 10]])

    """

    out = np.zeros_like(image, dtype=np.bool)

    # In the case of labels, recursively build and return an output
    # operating on each label separately
    if labels is not None:
        label_values = np.unique(labels)
        # Reorder label values to have consecutive integers (no gaps)
        if np.any(np.diff(label_values) != 1):
            mask = labels >= 1
            labels[mask] = 1 + rank_order(labels[mask])[0].astype(labels.dtype)
        labels = labels.astype(np.int32)

        # New values for new ordering
        label_values = np.unique(labels)
        for label in label_values[label_values != 0]:
            maskim = (labels == label)
            out += peak_local_max(image * maskim, min_distance=min_distance,
                                  threshold_abs=threshold_abs,
                                  threshold_rel=threshold_rel,
                                  exclude_border=exclude_border,
                                  indices=False, num_peaks=np.inf,
                                  footprint=footprint, labels=None)

        if indices is True:
            return np.transpose(out.nonzero())
        else:
            return out.astype(np.bool)

    if np.all(image == image.flat[0]):
        if indices is True:
            return np.empty((0, 2), np.int)
        else:
            return out

    # Non maximum filter
    if footprint is not None:
        image_max = ndi.maximum_filter(image, footprint=footprint,
                                       mode='constant')
    else:
        size = 2 * min_distance + 1
        image_max = ndi.maximum_filter(image, size=size, mode='constant')
    mask = image == image_max

    if exclude_border and (footprint is not None):
        # zero out the image borders
        for i in range(mask.ndim):
            mask = mask.swapaxes(0, i)
            remove = (footprint.shape[i] if footprint is not None
                      else 2 * exclude_border)
            mask[:remove // 2] = mask[-remove // 2:] = False
            mask = mask.swapaxes(0, i)

    # find top peak candidates above a threshold
    thresholds = []
    if threshold_abs is None:
        threshold_abs = image.min()
    thresholds.append(threshold_abs)
    if threshold_rel is not None:
        thresholds.append(threshold_rel * image.max())
    if thresholds:
        mask &= image > max(thresholds)

    # get coordinates of peaks
    coordinates = np.transpose(mask.nonzero())

    if coordinates.shape[0] > num_peaks:
        intensities = image.flat[np.ravel_multi_index(coordinates.transpose(),
                                                      image.shape)]
        idx_maxsort = np.argsort(intensities)[::-1]
        coordinates = coordinates[idx_maxsort][:num_peaks]

    if indices is True:
        return coordinates
    else:
        nd_indices = tuple(coordinates.T)
        out[nd_indices] = True
        return out
<commit_msg>Added backwards compatibility for `exclude_border`<commit_after>import numpy as np
import scipy.ndimage as ndi
from ..filters import rank_order

def peak_local_max(image, min_distance=1, threshold_abs=None,
                   threshold_rel=None, exclude_border=1, indices=True,
                   num_peaks=np.inf, footprint=None, labels=None):
    """Find peaks in an image as coordinate list or boolean mask.

    Peaks are the local maxima in a region of `2 * min_distance + 1`
    (i.e. peaks are separated by at least `min_distance`).

    If peaks are flat (i.e. multiple adjacent pixels have identical
    intensities), the coordinates of all such pixels are returned.

    If both `threshold_abs` and `threshold_rel` are provided, the maximum
    of the two is chosen as the minimum intensity threshold of peaks.

    Parameters
    ----------
    image : ndarray
        Input image.
    min_distance : int, optional
        Minimum number of pixels separating peaks in a region of `2 *
        min_distance + 1` (i.e. peaks are separated by at least
        `min_distance`).
        To find the maximum number of peaks, use `min_distance=1`.
    threshold_abs : float, optional
        Minimum intensity of peaks. By default, the absolute threshold is
        the minimum intensity of the image.
    threshold_rel : float, optional
        Minimum intensity of peaks, calculated as `max(image) * threshold_rel`.
    exclude_border : int, optional
        If nonzero, `exclude_border` excludes peaks from
        within `exclude_border`-pixels of the border of the image.
    indices : bool, optional
        If True, the output will be an array representing peak
        coordinates.  If False, the output will be a boolean array shaped as
        `image.shape` with peaks present at True elements.
    num_peaks : int, optional
        Maximum number of peaks. When the number of peaks exceeds `num_peaks`,
        return `num_peaks` peaks based on highest peak intensity.
    footprint : ndarray of bools, optional
        If provided, `footprint == 1` represents the local region within which
        to search for peaks at every point in `image`.  Overrides
        `min_distance` (also for `exclude_border`).
    labels : ndarray of ints, optional
        If provided, each unique region `labels == value` represents a unique
        region to search for peaks. Zero is reserved for background.

    Returns
    -------
    output : ndarray or ndarray of bools

        * If `indices = True`  : (row, column, ...) coordinates of peaks.
        * If `indices = False` : Boolean array shaped like `image`, with peaks
          represented by True values.

    Notes
    -----
    The peak local maximum function returns the coordinates of local peaks
    (maxima) in an image. A maximum filter is used for finding local maxima.
    This operation dilates the original image. After comparison of the dilated
    and original image, this function returns the coordinates or a mask of the
    peaks where the dilated image equals the original image.

    Examples
    --------
    >>> img1 = np.zeros((7, 7))
    >>> img1[3, 4] = 1
    >>> img1[3, 2] = 1.5
    >>> img1
    array([[ 0. ,  0. ,  0. ,  0. ,  0. ,  0. ,  0. ],
           [ 0. ,  0. ,  0. ,  0. ,  0. ,  0. ,  0. ],
           [ 0. ,  0. ,  0. ,  0. ,  0. ,  0. ,  0. ],
           [ 0. ,  0. ,  1.5,  0. ,  1. ,  0. ,  0. ],
           [ 0. ,  0. ,  0. ,  0. ,  0. ,  0. ,  0. ],
           [ 0. ,  0. ,  0. ,  0. ,  0. ,  0. ,  0. ],
           [ 0. ,  0. ,  0. ,  0. ,  0. ,  0. ,  0. ]])

    >>> peak_local_max(img1, min_distance=1)
    array([[3, 2],
           [3, 4]])

    >>> peak_local_max(img1, min_distance=2)
    array([[3, 2]])

    >>> img2 = np.zeros((20, 20, 20))
    >>> img2[10, 10, 10] = 1
    >>> peak_local_max(img2, exclude_border=0)
    array([[10, 10, 10]])

    """

    if type(exclude_border) == bool:
      exclude_border = min_distance if exclude_border else 0

    out = np.zeros_like(image, dtype=np.bool)

    # In the case of labels, recursively build and return an output
    # operating on each label separately
    if labels is not None:
        label_values = np.unique(labels)
        # Reorder label values to have consecutive integers (no gaps)
        if np.any(np.diff(label_values) != 1):
            mask = labels >= 1
            labels[mask] = 1 + rank_order(labels[mask])[0].astype(labels.dtype)
        labels = labels.astype(np.int32)

        # New values for new ordering
        label_values = np.unique(labels)
        for label in label_values[label_values != 0]:
            maskim = (labels == label)
            out += peak_local_max(image * maskim, min_distance=min_distance,
                                  threshold_abs=threshold_abs,
                                  threshold_rel=threshold_rel,
                                  exclude_border=exclude_border,
                                  indices=False, num_peaks=np.inf,
                                  footprint=footprint, labels=None)

        if indices is True:
            return np.transpose(out.nonzero())
        else:
            return out.astype(np.bool)

    if np.all(image == image.flat[0]):
        if indices is True:
            return np.empty((0, 2), np.int)
        else:
            return out

    # Non maximum filter
    if footprint is not None:
        image_max = ndi.maximum_filter(image, footprint=footprint,
                                       mode='constant')
    else:
        size = 2 * min_distance + 1
        image_max = ndi.maximum_filter(image, size=size, mode='constant')
    mask = image == image_max

    if exclude_border and (footprint is not None):
        # zero out the image borders
        for i in range(mask.ndim):
            mask = mask.swapaxes(0, i)
            remove = (footprint.shape[i] if footprint is not None
                      else 2 * exclude_border)
            mask[:remove // 2] = mask[-remove // 2:] = False
            mask = mask.swapaxes(0, i)

    # find top peak candidates above a threshold
    thresholds = []
    if threshold_abs is None:
        threshold_abs = image.min()
    thresholds.append(threshold_abs)
    if threshold_rel is not None:
        thresholds.append(threshold_rel * image.max())
    if thresholds:
        mask &= image > max(thresholds)

    # get coordinates of peaks
    coordinates = np.transpose(mask.nonzero())

    if coordinates.shape[0] > num_peaks:
        intensities = image.flat[np.ravel_multi_index(coordinates.transpose(),
                                                      image.shape)]
        idx_maxsort = np.argsort(intensities)[::-1]
        coordinates = coordinates[idx_maxsort][:num_peaks]

    if indices is True:
        return coordinates
    else:
        nd_indices = tuple(coordinates.T)
        out[nd_indices] = True
        return out
<|endoftext|>


<commit_before>"""
RVT_fixme
detects a task list in your project folder
and reminds you of errors/warnings to fix
"""

import System
from System import EventHandler
from Autodesk.Revit.DB.Events import DocumentOpenedEventArgs
import sys
import os.path as op
import datetime
startup_path = op.dirname(__file__)
sys.path.append(startup_path)

def event_handler_function(sender, args):
    print(15*"-" + "event_handler file opened" + 15*"-")
    now_utc = str(datetime.datetime.utcnow())
    log_path = "d:/delme/model_open_log.txt"
    doc_path = doc.PathName
    model_name = op.basename(doc_path)
    model_path = op.dirname(doc_path)
    model_central_name = model_name.split("_"+user)[0]
    assume_fix_tasks_dir = op.join(model_path, "RVT_fixme")
    ini = op.join(assume_fix_tasks_dir, "fixme_{}.ini".format(model_central_name))
    if op.exists(ini):
        print("- found corresponding RVT_fixme to this model.")
        # print(doc_path)
        # print("file was opened at {0}".format(now_utc))
        # print("ini found at: {0}".format(ini))
        if doc.IsWorkshared:
            import on_ws_model_opened
            print("- workshared model found at:\n- {0}".format(doc_path))
            # print("attempt reload")
            # reload(on_ws_model_opened)
            # print("after_reload")
            on_ws_model_opened.connect_to_rvt(ini)
        with open(log_path, "a") as model_log:
            model_log.write("ws file was opened at \n- {0}\n-".format(now_utc))

__revit__.Application.DocumentOpened += EventHandler[DocumentOpenedEventArgs](event_handler_function)
rvt_user = __revit__.Application.Username

doc = None
uidoc = None

print("-"*59)
print(" | dear {0} welcome to RVT_fixme!!".format(rvt_user))
print("-"*59)
<commit_msg>startup - bcf todo add<commit_after>"""
RVT_fixme
detects a task list in your project folder
and reminds you of errors/warnings to fix
"""

import System
from System import EventHandler
from Autodesk.Revit.DB.Events import DocumentOpenedEventArgs
import sys
import os.path as op
import datetime
startup_path = op.dirname(__file__)
sys.path.append(startup_path)

# TODO add support for reading bcf/bcfzip guids, comments

def event_handler_function(sender, args):
    print(15*"-" + "event_handler file opened" + 15*"-")
    now_utc = str(datetime.datetime.utcnow())
    log_path = "d:/delme/model_open_log.txt"
    doc_path = doc.PathName
    model_name = op.basename(doc_path)
    model_path = op.dirname(doc_path)
    model_central_name = model_name.split("_"+user)[0]
    assume_fix_tasks_dir = op.join(model_path, "RVT_fixme")
    ini = op.join(assume_fix_tasks_dir, "fixme_{}.ini".format(model_central_name))
    if op.exists(ini):
        print("- found corresponding RVT_fixme to this model.")
        # print(doc_path)
        # print("file was opened at {0}".format(now_utc))
        # print("ini found at: {0}".format(ini))
        if doc.IsWorkshared:
            import on_ws_model_opened
            print("- workshared model found at:\n- {0}".format(doc_path))
            # print("attempt reload")
            # reload(on_ws_model_opened)
            # print("after_reload")
            on_ws_model_opened.connect_to_rvt(ini)
        with open(log_path, "a") as model_log:
            model_log.write("ws file was opened at \n- {0}\n-".format(now_utc))

__revit__.Application.DocumentOpened += EventHandler[DocumentOpenedEventArgs](event_handler_function)
rvt_user = __revit__.Application.Username

doc = None
uidoc = None

print("-"*59)
print(" | dear {0} welcome to RVT_fixme!!".format(rvt_user))
print("-"*59)
<|endoftext|>