<commit_before>
'''
Created on Jan 8, 2012

Large refactor to notebook Bokeh plotting 5 Jan, 2015

@author: cryan@bbn.com

A simple interactive tool plotting pulse sequences for visual inspection

Copyright 2012,2013,2014,2015 Raytheon BBN Technologies

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
'''

import os.path
import bokeh.plotting as bk
from bokeh.embed import notebook_div

from IPython.html import widgets
from IPython.display import display

import numpy as np

from mm import multimethod
from QGL.APSPattern import read_APS_file
from QGL.APS2Pattern import read_APS2_file
from QGL.TekPattern import read_Tek_awg_file
from instruments.AWGs import APS, APS2, Tek5014, Tek7000

import uuid, tempfile

import Libraries

#TODO: handle console plotting to static html file
def in_ipynb():
    try:
        cfg = get_ipython().config
        if cfg['IPKernelApp']['parent_appname'] == 'ipython-notebook':
            return True
        else:
            return False
    except NameError:
        return False

#Only load if in notebook
if in_ipynb():
    bk.output_notebook()

# define sequence reader dispatch on awg type
@multimethod(Tek5014, unicode)
def read_sequence_file(awg, filename):
    return read_Tek_awg_file(filename)

@multimethod(Tek7000, unicode)
def read_sequence_file(awg, filename):
    return read_Tek_awg_file(filename)

@multimethod(APS, unicode)
def read_sequence_file(awg, filename):
    return read_APS_file(filename)

@multimethod(APS2, unicode)
def read_sequence_file(awg, filename):
    return read_APS2_file(filename)

def all_zero_seqs(seqs):
    return all([np.all(seq == 0) for seq in seqs])

def plot_pulse_files(fileNames, firstSeqNum=0):
    '''
    plot_pulse_files(fileNames, firstSeqNum=0)

    Helper function to plot a list of AWG files.  In an iPython notebook the plots will be in line with
    dynamic updating.  For iPython consoles a static html file will be generated with the firstSeqNum.
    '''
    #If we only go one filename turn it into a list
    if isinstance(fileNames, str):
        fileNames = [fileNames]

    wfs = {}
    dataDict = {}
    lineNames = []
    title = ""

    for fileName in sorted(fileNames):

        #Assume a naming convention path/to/file/SequenceName-AWGName.h5
        AWGName = (os.path.split(os.path.splitext(fileName)[0])[1]).split('-')[1]
        #Strip any _ suffix
        if '_' in AWGName:
            AWGName = AWGName[:AWGName.index('_')]

        title += os.path.split(os.path.splitext(fileName)[0])[1] + "; "

        wfs[AWGName] = read_sequence_file(Libraries.instrumentLib[AWGName], fileName)

        for (k,seqs) in sorted(wfs[AWGName].items()):
            if not all_zero_seqs(seqs):
                lineNames.append(AWGName + '-' + k)
                dataDict[lineNames[-1] + "_x"] = np.arange(len(seqs[firstSeqNum]))
                dataDict[lineNames[-1]] = seqs[firstSeqNum] + 2*(len(lineNames)-1)

    #Remove trailing semicolon from title
    title = title[:-2]

    source = bk.ColumnDataSource(data=dataDict)
    figH = bk.figure(title=title, plot_width=1000, y_range=(-1,len(dataDict)+1))

    #Colorbrewer2 qualitative Set3 (http://colorbrewer2.org)
    colours = ["#8dd3c7", "#ffffb3", "#bebada", "#fb8072", "#80b1d3", "#fdb462",
                 "#b3de69", "#fccde5", "#d9d9d9", "#bc80bd", "#ccebc5", "#ffed6f"]

    for ct,k in enumerate(lineNames):
        figH.line(k+"_x", k, source=source, color=colours[ct%len(colours)], line_width=2, legend=k)

    if in_ipynb():
        #Setup inline plotting with slider updating of data

        def update_plot(_, seqNum):
            for ct,k in enumerate(lineNames):
                AWGName, chName = k.split('-')
                source.data[k+"_x"] = np.arange(len(wfs[AWGName][chName][seqNum-1]))
                source.data[k] = wfs[AWGName][chName][seqNum-1] + 2*ct
            source.push_notebook()

        #widgets.interact(update_plot, seqNum=(1, len(wfs[AWGName]["ch1"])), div=widgets.HTMLWidget(value=notebook_div(figH)))

        slider = widgets.IntSliderWidget(value=firstSeqNum+1, min=1, max=len(wfs[AWGName]["ch1"]), step=1, description="Sequence:")
        slider.on_trait_change(update_plot, 'value')
        plotBox = widgets.HTMLWidget(value=notebook_div(figH))
        appBox = widgets.ContainerWidget()
        appBox.children = [slider, plotBox]
        display(appBox)

    else:

        #Otherwise dump to a static file

        bk.output_file(os.path.join(tempfile.gettempdir(), str(uuid.uuid4()) + ".html"))
        bk.show(figH)
<commit_msg>Put how many sequences total next to sequence number slider<commit_after>
'''
Created on Jan 8, 2012

Large refactor to notebook Bokeh plotting 5 Jan, 2015

@author: cryan@bbn.com

A simple interactive tool plotting pulse sequences for visual inspection

Copyright 2012,2013,2014,2015 Raytheon BBN Technologies

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
'''

import os.path
import bokeh.plotting as bk
from bokeh.embed import notebook_div

from IPython.html import widgets
from IPython.display import display

import numpy as np

from mm import multimethod
from QGL.APSPattern import read_APS_file
from QGL.APS2Pattern import read_APS2_file
from QGL.TekPattern import read_Tek_awg_file
from instruments.AWGs import APS, APS2, Tek5014, Tek7000

import uuid, tempfile

import Libraries

#TODO: handle console plotting to static html file
def in_ipynb():
    try:
        cfg = get_ipython().config
        if cfg['IPKernelApp']['parent_appname'] == 'ipython-notebook':
            return True
        else:
            return False
    except NameError:
        return False

#Only load if in notebook
if in_ipynb():
    bk.output_notebook()

# define sequence reader dispatch on awg type
@multimethod(Tek5014, unicode)
def read_sequence_file(awg, filename):
    return read_Tek_awg_file(filename)

@multimethod(Tek7000, unicode)
def read_sequence_file(awg, filename):
    return read_Tek_awg_file(filename)

@multimethod(APS, unicode)
def read_sequence_file(awg, filename):
    return read_APS_file(filename)

@multimethod(APS2, unicode)
def read_sequence_file(awg, filename):
    return read_APS2_file(filename)

def all_zero_seqs(seqs):
    return all([np.all(seq == 0) for seq in seqs])

def plot_pulse_files(fileNames, firstSeqNum=0):
    '''
    plot_pulse_files(fileNames, firstSeqNum=0)

    Helper function to plot a list of AWG files.  In an iPython notebook the plots will be in line with
    dynamic updating.  For iPython consoles a static html file will be generated with the firstSeqNum.
    '''
    #If we only go one filename turn it into a list
    if isinstance(fileNames, str):
        fileNames = [fileNames]

    wfs = {}
    dataDict = {}
    lineNames = []
    title = ""

    for fileName in sorted(fileNames):

        #Assume a naming convention path/to/file/SequenceName-AWGName.h5
        AWGName = (os.path.split(os.path.splitext(fileName)[0])[1]).split('-')[1]
        #Strip any _ suffix
        if '_' in AWGName:
            AWGName = AWGName[:AWGName.index('_')]

        title += os.path.split(os.path.splitext(fileName)[0])[1] + "; "

        wfs[AWGName] = read_sequence_file(Libraries.instrumentLib[AWGName], fileName)

        for (k,seqs) in sorted(wfs[AWGName].items()):
            if not all_zero_seqs(seqs):
                lineNames.append(AWGName + '-' + k)
                dataDict[lineNames[-1] + "_x"] = np.arange(len(seqs[firstSeqNum]))
                dataDict[lineNames[-1]] = seqs[firstSeqNum] + 2*(len(lineNames)-1)

    #Remove trailing semicolon from title
    title = title[:-2]

    source = bk.ColumnDataSource(data=dataDict)
    figH = bk.figure(title=title, plot_width=1000, y_range=(-1,len(dataDict)+1))

    #Colorbrewer2 qualitative Set3 (http://colorbrewer2.org)
    colours = ["#8dd3c7", "#ffffb3", "#bebada", "#fb8072", "#80b1d3", "#fdb462",
                 "#b3de69", "#fccde5", "#d9d9d9", "#bc80bd", "#ccebc5", "#ffed6f"]

    for ct,k in enumerate(lineNames):
        figH.line(k+"_x", k, source=source, color=colours[ct%len(colours)], line_width=2, legend=k)

    if in_ipynb():
        #Setup inline plotting with slider updating of data

        def update_plot(_, seqNum):
            for ct,k in enumerate(lineNames):
                AWGName, chName = k.split('-')
                source.data[k+"_x"] = np.arange(len(wfs[AWGName][chName][seqNum-1]))
                source.data[k] = wfs[AWGName][chName][seqNum-1] + 2*ct
            source.push_notebook()

        #widgets.interact(update_plot, seqNum=(1, len(wfs[AWGName]["ch1"])), div=widgets.HTMLWidget(value=notebook_div(figH)))

        slider = widgets.IntSliderWidget(value=firstSeqNum+1, min=1, max=len(wfs[AWGName]["ch1"]), step=1, description="Sequence (of {}):".format(len(seqs)))
        slider.on_trait_change(update_plot, 'value')
        plotBox = widgets.HTMLWidget(value=notebook_div(figH))
        appBox = widgets.ContainerWidget()
        appBox.children = [slider, plotBox]
        display(appBox)

    else:

        #Otherwise dump to a static file

        bk.output_file(os.path.join(tempfile.gettempdir(), str(uuid.uuid4()) + ".html"))
        bk.show(figH)
<|endoftext|>


<commit_before>import click
import json
import os
import time
from hodor.gme import upload_file, poll_asset_processing, poll_layer_publishing
from pprintpp import pprint as pp
from os import walk
from retries import retries
from hodor.cli import pass_context

# @TODO Implement multithreading processing:
#   * Using Pool.apply_async()
#   * Just continue to dump logs to stdout at the moment
#   * How to handle multithreading uploading?
#   * We only need ctx's methods - so don't pass, just import?

@click.group(short_help="Create assets in Google Maps Engine")
# https://google-api-python-client.googlecode.com/hg/docs/epy/apiclient.http.MediaFileUpload-class.html
@click.option('--chunk-size', default=10485760,
              help='File chunk size in bytes. Defaults to chunks of 10mb. Pass -1 to use native streaming in Python instead.')
@click.option('--asset-processing-wait', default=90,
              help='Length of time in minutes we should wait for an asset to process before giving up. Defaults to 90 minutes.')
@pass_context
def cli(ctx, chunk_size, asset_processing_wait):
  ctx.chunk_size = chunk_size
  ctx.processing_timeout_mins = asset_processing_wait

@cli.command()
@click.option('--mosaic-id',
              help="The raster mosaic to add the newly created raster(s) to.")
@click.option('--process-mosaic/--no-process-mosaic', default=False,
              help="Whether the given raster mosaic should be processed after the raster(s) are added. Defaults to false.")
@click.option('--layer-id',
              help="The raster layer to add the newly created raster(s) to. Not compatible with mosaic options.")
@click.argument('configstore', type=click.Path(exists=True, resolve_path=True))
@pass_context
def raster(ctx, mosaic_id, process_mosaic, layer_id, configstore):
  """
  Create a new raster asset in Google Maps Engine

  Parameters
  ----------
  ctx : Context
    A Click Context object.
  configstore : Click.Path
    A pointer to the store of one or more asset configuration files.
  """

  rasters = hodor_uploader(ctx, "raster", configstore)

  for raster in rasters:
    # Optionally, add it to an existing raster mosaic
    if mosaic_id is not None:
      ctx.service.rasterCollections().rasters().batchInsert(id=mosaic_id, body={"ids": [raster["id"]]}).execute()
      ctx.log("Asset '%s' added to mosaic '%s'" % (raster["id"], mosaic_id))

      if process_mosaic == True:
        ctx.service.rasterCollections().process(id=mosaic_id).execute()
        ctx.log("Mosaic '%s' processing started" % (mosaic_id))

        # Poll until raster has processed
        poll_asset_processing(ctx, mosaic_id, ctx.service.rasterCollections())

    # Optionally, add it to an existing layer
    elif layer_id is not None:
      layer = ctx.service.layers().get(id=layer_id).execute()

      if layer['datasourceType'] != "image":
        raise Exception("Layer datasourceType is not 'image'.")
      if len(layer['datasources']) >= 100:
        raise Exception("The GME API currently only allows us to patch layers <= 100 datasources. Sad Keanu :(")

      ctx.service.layers().patch(id=layer_id, body={
        "datasources": layer['datasources'] + [raster["id"]]
      }).execute()
      ctx.log("Asset '%s' added to layer '%s'" % (raster["id"], layer_id))

@cli.command()
@click.argument('configstore', type=click.Path(exists=True, resolve_path=True))
@pass_context
def vector(ctx, configstore):
  """
  Create a new vector asset in Google Maps Engine

  Parameters
  ----------
  ctx : Context
    A Click Context object.
  configstore : Click.Path
    A pointer to the store of one or more asset configuration files.
  """

  hodor_uploader(ctx, "vector", configstore)

def hodor_uploader(ctx, asset_type, configstore):
  """
  Core uploader functionality for anything that adopts the
  Hodor-esque configstore system.

  Parameters
  ----------
  ctx : Context
    A Click Context object
  asset_type : str
    The GME asset type represented. Possible values: vector, raster
  configstore : Click.Path
    A pointer to the store of one or more asset configuration files.

  Returns
  -------
  list
    A list of assetIds for the assets created.
  """
  @retries(10)
  def create_asset(ctx, resource, config):
    return resource.upload(body=config).execute()

  # Init resource
  resource = ctx.service.tables() if asset_type == "vector" else ctx.service.rasters()

  # Process all available configfiles
  asset_configs = hodor_config_builder(configstore)
  asset_ids = []
  for asset_dir, configs in asset_configs.iteritems():
    payloaddir = os.path.join(asset_dir, "payload")

    for config in configs:
      # Create skeleton asset
      response = create_asset(ctx, resource, config)
      ctx.log("Asset '%s' created with id %s" % (response['name'], response['id']))

      # Upload the payload files
      start_time = time.time()
      for f in config['files']:
        upload_file(ctx, response['id'], asset_type, os.path.join(payloaddir, f['filename']), ctx.chunk_size)
      ctx.log("All uploads completed and took %s minutes" % (round((time.time() - start_time) / 60, 2)))

      # Poll until asset has processed
      poll_asset_processing(ctx, response['id'], resource)

      # Optionally, create any layers the user has provided config for
      if asset_type == "vector" and os.path.isfile(os.path.join(asset_dir, "layers.json")):
        layer_creator(ctx, response, os.path.join(asset_dir, "layers.json"))

      asset_ids.append(response["id"])
  return asset_ids

def hodor_config_builder(configstore):
  """
  Manages initialisation of the Hodor-esque config system.

  Parameters
  ----------
  configstore : Click.Path
    A pointer to the store of one or more asset configuration files.

  Returns
  -------
  dict
    An object representing the config files used to create the asset(s).
  {
    '/path/to/asset1': [{..asset config...}],
    '/path/to/asset2': [{..asset config...}]
  }
  """

  # Fetch all configfiles in the configstore
  configfiles = []
  if os.path.basename(configstore) == "config.json":
    # Just a single config.json file
    configfiles.append(configstore)
  else:
    # Recursively walk the directory tree finding all config.json files
    for (dirpath, dirnames, filenames) in walk(configstore):
      for f in filenames:
        if f == "config.json":
          configfiles.append(os.path.join(dirpath, f))

  # Create GME asset config blocks for all available assets
  # in the available configfiles
  assets = {}
  for cf in configfiles:
    with open(cf, "r") as f:
      config = json.load(f)

    # Handle bulk loads where the use has multiple files of the same
    # type in their payload dir. e.g.
    # /payload
    #   file1.tif
    #   file1.tfw
    #   file2.tif
    #   file2.tfw
    # Would be treated as two assets: file1 and file2
    if "name" in config and config["name"] == "{fileName}":
      # Group all payload files by their base filename.
      # NB: This fails for files with two extensions e.g. file.tif.tfw
      payloaddir = os.path.join(os.path.dirname(cf), "payload")
      grouped_payload = {}
      for (dirpath, dirnames, filenames) in walk(payloaddir):
        for f in filenames:
          basename = os.path.splitext(f)[0]
          if basename not in grouped_payload:
            grouped_payload[basename] = []
          grouped_payload[basename].append(f)
        break

      for name, payload in grouped_payload.iteritems():
        asset_config = config.copy()
        asset_config["name"] = name
        asset_config["files"] = [{'filename': f} for f in payload]

        asset_dir = os.path.dirname(cf)
        if asset_dir not in assets:
          assets[asset_dir] = []
        assets[asset_dir].append(asset_config)

    # Just a regular single asset upload
    else:
      # Backwards compatibility for Aaron who is supplying the files in the config already
      if "files" not in config:
        payloaddir = os.path.join(os.path.dirname(cf), "payload")
        for (dirpath, dirnames, filenames) in walk(payloaddir):
          config['files'] = [{'filename': f} for f in filenames if f != ".DS_Store"]
          break
      assets[os.path.dirname(cf)] = [config]
  return assets

def layer_creator(ctx, asset, configfile):
  """Creates layers from an asset according to
  user-supplied config.

  ctx : Context
    A Click Context object.
  asset : dict
    A vector or raster asset in GME.
  configfile : str
    Path to a layers.json file.
  """
  @retries(10)
  def create_layer(ctx, config):
    return ctx.service.layers().create(body=config, process=True).execute()

  with open(configfile, "r") as f:
    config = json.load(f)

  for layer in config["layers"]:
    layer["projectId"] = asset["projectId"]
    layer["datasources"] = [{"id": asset["id"]}]

    # Patch layers.json config with the optional styleFile and infoWindowFile parameters
    if "styleFile" in layer:
      styleFile = os.path.join(os.path.dirname(configfile.name), layer["styleFile"])
      if os.path.exists(styleFile):
        with open(styleFile) as f:
          layer["style"] = json.load(f)
        del layer["styleFile"]

    if "infoWindowFile" in layer:
      if "style" in layer:
        infoWindowFile = os.path.join(os.path.dirname(configfile.name), layer["infoWindowFile"])
        if os.path.exists(infoWindowFile):
          with open(infoWindowFile) as f:
            layer["style"]["featureInfo"]["content"] = f.read()
      del layer["infoWindowFile"]

    # Create layer
    layer = create_layer(ctx, layer)
    ctx.log("Layer '%s' created with id %s" % (layer['name'], layer['id']))

    # Poll until asset has processed
    poll_asset_processing(ctx, layer['id'], ctx.service.layers())
<commit_msg>Fix for featureInfo key issue<commit_after>import click
import json
import os
import time
from hodor.gme import upload_file, poll_asset_processing, poll_layer_publishing
from pprintpp import pprint as pp
from os import walk
from retries import retries
from hodor.cli import pass_context

# @TODO Implement multithreading processing:
#   * Using Pool.apply_async()
#   * Just continue to dump logs to stdout at the moment
#   * How to handle multithreading uploading?
#   * We only need ctx's methods - so don't pass, just import?

@click.group(short_help="Create assets in Google Maps Engine")
# https://google-api-python-client.googlecode.com/hg/docs/epy/apiclient.http.MediaFileUpload-class.html
@click.option('--chunk-size', default=10485760,
              help='File chunk size in bytes. Defaults to chunks of 10mb. Pass -1 to use native streaming in Python instead.')
@click.option('--asset-processing-wait', default=90,
              help='Length of time in minutes we should wait for an asset to process before giving up. Defaults to 90 minutes.')
@pass_context
def cli(ctx, chunk_size, asset_processing_wait):
  ctx.chunk_size = chunk_size
  ctx.processing_timeout_mins = asset_processing_wait

@cli.command()
@click.option('--mosaic-id',
              help="The raster mosaic to add the newly created raster(s) to.")
@click.option('--process-mosaic/--no-process-mosaic', default=False,
              help="Whether the given raster mosaic should be processed after the raster(s) are added. Defaults to false.")
@click.option('--layer-id',
              help="The raster layer to add the newly created raster(s) to. Not compatible with mosaic options.")
@click.argument('configstore', type=click.Path(exists=True, resolve_path=True))
@pass_context
def raster(ctx, mosaic_id, process_mosaic, layer_id, configstore):
  """
  Create a new raster asset in Google Maps Engine

  Parameters
  ----------
  ctx : Context
    A Click Context object.
  configstore : Click.Path
    A pointer to the store of one or more asset configuration files.
  """

  rasters = hodor_uploader(ctx, "raster", configstore)

  for raster in rasters:
    # Optionally, add it to an existing raster mosaic
    if mosaic_id is not None:
      ctx.service.rasterCollections().rasters().batchInsert(id=mosaic_id, body={"ids": [raster["id"]]}).execute()
      ctx.log("Asset '%s' added to mosaic '%s'" % (raster["id"], mosaic_id))

      if process_mosaic == True:
        ctx.service.rasterCollections().process(id=mosaic_id).execute()
        ctx.log("Mosaic '%s' processing started" % (mosaic_id))

        # Poll until raster has processed
        poll_asset_processing(ctx, mosaic_id, ctx.service.rasterCollections())

    # Optionally, add it to an existing layer
    elif layer_id is not None:
      layer = ctx.service.layers().get(id=layer_id).execute()

      if layer['datasourceType'] != "image":
        raise Exception("Layer datasourceType is not 'image'.")
      if len(layer['datasources']) >= 100:
        raise Exception("The GME API currently only allows us to patch layers <= 100 datasources. Sad Keanu :(")

      ctx.service.layers().patch(id=layer_id, body={
        "datasources": layer['datasources'] + [raster["id"]]
      }).execute()
      ctx.log("Asset '%s' added to layer '%s'" % (raster["id"], layer_id))

@cli.command()
@click.argument('configstore', type=click.Path(exists=True, resolve_path=True))
@pass_context
def vector(ctx, configstore):
  """
  Create a new vector asset in Google Maps Engine

  Parameters
  ----------
  ctx : Context
    A Click Context object.
  configstore : Click.Path
    A pointer to the store of one or more asset configuration files.
  """

  hodor_uploader(ctx, "vector", configstore)

def hodor_uploader(ctx, asset_type, configstore):
  """
  Core uploader functionality for anything that adopts the
  Hodor-esque configstore system.

  Parameters
  ----------
  ctx : Context
    A Click Context object
  asset_type : str
    The GME asset type represented. Possible values: vector, raster
  configstore : Click.Path
    A pointer to the store of one or more asset configuration files.

  Returns
  -------
  list
    A list of assetIds for the assets created.
  """
  @retries(10)
  def create_asset(ctx, resource, config):
    return resource.upload(body=config).execute()

  # Init resource
  resource = ctx.service.tables() if asset_type == "vector" else ctx.service.rasters()

  # Process all available configfiles
  asset_configs = hodor_config_builder(configstore)
  asset_ids = []
  for asset_dir, configs in asset_configs.iteritems():
    payloaddir = os.path.join(asset_dir, "payload")

    for config in configs:
      # Create skeleton asset
      response = create_asset(ctx, resource, config)
      ctx.log("Asset '%s' created with id %s" % (response['name'], response['id']))

      # Upload the payload files
      start_time = time.time()
      for f in config['files']:
        upload_file(ctx, response['id'], asset_type, os.path.join(payloaddir, f['filename']), ctx.chunk_size)
      ctx.log("All uploads completed and took %s minutes" % (round((time.time() - start_time) / 60, 2)))

      # Poll until asset has processed
      poll_asset_processing(ctx, response['id'], resource)

      # Optionally, create any layers the user has provided config for
      if asset_type == "vector" and os.path.isfile(os.path.join(asset_dir, "layers.json")):
        layer_creator(ctx, response, os.path.join(asset_dir, "layers.json"))

      asset_ids.append(response["id"])
  return asset_ids

def hodor_config_builder(configstore):
  """
  Manages initialisation of the Hodor-esque config system.

  Parameters
  ----------
  configstore : Click.Path
    A pointer to the store of one or more asset configuration files.

  Returns
  -------
  dict
    An object representing the config files used to create the asset(s).
  {
    '/path/to/asset1': [{..asset config...}],
    '/path/to/asset2': [{..asset config...}]
  }
  """

  # Fetch all configfiles in the configstore
  configfiles = []
  if os.path.basename(configstore) == "config.json":
    # Just a single config.json file
    configfiles.append(configstore)
  else:
    # Recursively walk the directory tree finding all config.json files
    for (dirpath, dirnames, filenames) in walk(configstore):
      for f in filenames:
        if f == "config.json":
          configfiles.append(os.path.join(dirpath, f))

  # Create GME asset config blocks for all available assets
  # in the available configfiles
  assets = {}
  for cf in configfiles:
    with open(cf, "r") as f:
      config = json.load(f)

    # Handle bulk loads where the use has multiple files of the same
    # type in their payload dir. e.g.
    # /payload
    #   file1.tif
    #   file1.tfw
    #   file2.tif
    #   file2.tfw
    # Would be treated as two assets: file1 and file2
    if "name" in config and config["name"] == "{fileName}":
      # Group all payload files by their base filename.
      # NB: This fails for files with two extensions e.g. file.tif.tfw
      payloaddir = os.path.join(os.path.dirname(cf), "payload")
      grouped_payload = {}
      for (dirpath, dirnames, filenames) in walk(payloaddir):
        for f in filenames:
          basename = os.path.splitext(f)[0]
          if basename not in grouped_payload:
            grouped_payload[basename] = []
          grouped_payload[basename].append(f)
        break

      for name, payload in grouped_payload.iteritems():
        asset_config = config.copy()
        asset_config["name"] = name
        asset_config["files"] = [{'filename': f} for f in payload]

        asset_dir = os.path.dirname(cf)
        if asset_dir not in assets:
          assets[asset_dir] = []
        assets[asset_dir].append(asset_config)

    # Just a regular single asset upload
    else:
      # Backwards compatibility for Aaron who is supplying the files in the config already
      if "files" not in config:
        payloaddir = os.path.join(os.path.dirname(cf), "payload")
        for (dirpath, dirnames, filenames) in walk(payloaddir):
          config['files'] = [{'filename': f} for f in filenames if f != ".DS_Store"]
          break
      assets[os.path.dirname(cf)] = [config]
  return assets

def layer_creator(ctx, asset, configfile):
  """Creates layers from an asset according to
  user-supplied config.

  ctx : Context
    A Click Context object.
  asset : dict
    A vector or raster asset in GME.
  configfile : str
    Path to a layers.json file.
  """
  @retries(10)
  def create_layer(ctx, config):
    return ctx.service.layers().create(body=config, process=True).execute()

  with open(configfile, "r") as f:
    config = json.load(f)

  for layer in config["layers"]:
    layer["projectId"] = asset["projectId"]
    layer["datasources"] = [{"id": asset["id"]}]

    # Patch layers.json config with the optional styleFile and infoWindowFile parameters
    if "styleFile" in layer:
      styleFile = os.path.join(os.path.dirname(configfile.name), layer["styleFile"])
      if os.path.exists(styleFile):
        with open(styleFile) as f:
          layer["style"] = json.load(f)
        del layer["styleFile"]

    if "infoWindowFile" in layer:
      if "style" in layer:
        infoWindowFile = os.path.join(os.path.dirname(configfile.name), layer["infoWindowFile"])
        if os.path.exists(infoWindowFile):
          with open(infoWindowFile) as f:
            if "featureInfo" not in layer["style"]:
              layer["style"]["featureInfo"] = {}
            layer["style"]["featureInfo"]["content"] = f.read()
      del layer["infoWindowFile"]

    # Create layer
    layer = create_layer(ctx, layer)
    ctx.log("Layer '%s' created with id %s" % (layer['name'], layer['id']))

    # Poll until asset has processed
    poll_asset_processing(ctx, layer['id'], ctx.service.layers())
<|endoftext|>


<commit_before># A generic Python debugger base class.
# This class takes care of details of the trace facility;
# a derived class should implement user interaction.
# There are two debuggers based upon this:
# 'pdb', a text-oriented debugger not unlike dbx or gdb;
# and 'wdb', a window-oriented debugger.
# And of course... you can roll your own!

import sys

BdbQuit = 'bdb.BdbQuit' # Exception to give up completely

class Bdb: # Basic Debugger

	def init(self):
		self.breaks = {}
		return self

	def reset(self):
		self.botframe = None
		self.stopframe = None
		self.returnframe = None
		self.quitting = 0

	def trace_dispatch(self, frame, event, arg):
		if self.quitting:
			return # None
		if event == 'line':
			return self.dispatch_line(frame)
		if event == 'call':
			return self.dispatch_call(frame, arg)
		if event == 'return':
			return self.dispatch_return(frame, arg)
		if event == 'exception':
			return self.dispatch_exception(frame, arg)
		print 'bdb.Bdb.dispatch: unknown debugging event:', `event`
		return self.trace_dispatch

	def dispatch_line(self, frame):
		if self.stop_here(frame) or self.break_here(frame):
			self.user_line(frame)
			if self.quitting: raise BdbQuit
		return self.trace_dispatch

	def dispatch_call(self, frame, arg):
		frame.f_locals['__args__'] = arg
		if self.botframe is None:
			# First call of dispatch since reset()
			self.botframe = frame
			return self.trace_dispatch
		if not (self.stop_here(frame) or self.break_anywhere(frame)):
			# No need to trace this function
			return # None
		self.user_call(frame, arg)
		if self.quitting: raise BdbQuit
		return self.trace_dispatch

	def dispatch_return(self, frame, arg):
		if self.stop_here(frame) or frame == self.returnframe:
			self.user_return(frame, arg)
			if self.quitting: raise BdbQuit

	def dispatch_exception(self, frame, arg):
		if self.stop_here(frame):
			self.user_exception(frame, arg)
			if self.quitting: raise BdbQuit
		return self.trace_dispatch

	# Normally derived classes don't override the following
	# functions, but they may if they want to redefine the
	# definition of stopping and breakpoints.

	def stop_here(self, frame):
		if self.stopframe is None:
			return 1
		if frame is self.stopframe:
			return 1
		while frame is not None and frame is not self.stopframe:
			if frame is self.botframe:
				return 1
			frame = frame.f_back
		return 0

	def break_here(self, frame):
		if not self.breaks.has_key(frame.f_code.co_filename):
			return 0
		if not frame.f_lineno in \
				self.breaks[frame.f_code.co_filename]:
			return 0
		return 1

	def break_anywhere(self, frame):
		return self.breaks.has_key(frame.f_code.co_filename)

	# Derived classes should override the user_* functions
	# to gain control.

	def user_call(self, frame, argument_list):
		# This function is called when there is the remote possibility
		# that we ever need to stop in this function
		pass

	def user_line(self, frame):
		# This function is called when we stop or break at this line
		pass

	def user_return(self, frame, return_value):
		# This function is called when a return trap is set here
		pass

	def user_exception(self, frame, (exc_type, exc_value, exc_traceback)):
		# This function is called if an exception occurs,
		# but only if we are to stop at or just below this level
		pass

	# Derived classes and clients can call the following functions
	# to affect the stepping state.

	def set_step(self):
		# Stop after one line of code
		self.stopframe = None
		self.returnframe = None
		self.quitting = 0

	def set_next(self, frame):
		# Stop on the next line in or below the given frame
		self.stopframe = frame
		self.returnframe = None
		self.quitting = 0

	def set_return(self, frame):
		# Stop when returning from the given frame
		self.stopframe = frame.f_back
		self.returnframe = frame
		self.quitting = 0

	def set_continue(self):
		# Don't stop except at breakpoints or when finished
		self.stopframe = self.botframe
		self.returnframe = None
		self.quitting = 0

	def set_quit(self):
		self.stopframe = self.botframe
		self.returnframe = None
		self.quitting = 1
		sys.trace = None
		del sys.trace

	# Derived classes and clients can call the following functions
	# to manipulate breakpoints.  These functions return an
	# error message is something went wrong, None if all is well.
	# Call self.get_*break*() to see the breakpoints.

	def set_break(self, filename, lineno):
		import linecache # Import as late as possible
		line = linecache.getline(filename, lineno)
		if not line:
			return 'That line does not exist!'
		if not self.breaks.has_key(filename):
			self.breaks[filename] = []
		list = self.breaks[filename]
		if lineno in list:
			return 'There is already a breakpoint there!'
		list.append(lineno)

	def clear_break(self, filename, lineno):
		if not self.breaks.has_key(filename):
			return 'There are no breakpoints in that file!'
		if lineno not in self.breaks[filename]:
			return 'There is no breakpoint there!'
		self.breaks[filename].remove(lineno)
		if not self.breaks[filename]:
			del self.breaks[filename]

	def clear_all_file_breaks(self, filename):
		if not self.breaks.has_key(filename):
			return 'There are no breakpoints in that file!'
		del self.breaks[filename]

	def clear_all_breaks(self, filename, lineno):
		if not self.breaks:
			return 'There are no breakpoints!'
		self.breaks = {}

	def get_break(self, filename, lineno):
		return self.breaks.has_key(filename) and \
			lineno in self.breaks[filename]

	def get_file_breaks(self, filename):
		if self.breaks.has_key(filename):
			return self.breaks[filename]
		else:
			return []

	def get_all_breaks(self):
		return self.breaks

	# Derived classes and clients can call the following function
	# to get a data structure representing a stack trace.

	def get_stack(self, f, t):
		stack = []
		if t and t.tb_frame is f:
			t = t.tb_next
		while f is not None:
			stack.append((f, f.f_lineno))
			if f is self.botframe:
				break
			f = f.f_back
		stack.reverse()
		i = max(0, len(stack) - 1)
		while t is not None:
			stack.append((t.tb_frame, t.tb_lineno))
			t = t.tb_next
		return stack, i

	# The following two functions can be called by clients to use
	# a debugger to debug a statement, given as a string.

	def run(self, cmd):
		modname = self.writetempfile(cmd)
		self.runctx('import ' + modname + '\n', {}, {})

	def runctx(self, cmd, globals, locals):
		self.reset()
		sys.trace = self.trace_dispatch
		try:
			exec(cmd + '\n', globals, locals)
		except BdbQuit:
			pass
		finally:
			self.quitting = 1
			sys.trace = None
			del sys.trace
		# XXX What to do if the command finishes normally?

	def writetempfile(self, cmd):
		import os
		modname = 'bdb' + `os.getpid()`
		filename = '/tmp/' + modname + '.py'
		f = open(filename, 'w')
		f.write(cmd + '\n')
		f.close()
		import sys
		if sys.modules.has_key(modname): del sys.modules[modname]
		if '/tmp' not in sys.path: sys.path.insert(0, '/tmp')
		return modname

# -------------------- testing --------------------

class Tdb(Bdb):
	def user_call(self, frame, args):
		import codehack
		name = codehack.getcodename(frame.f_code)
		if not name: name = '???'
		print '+++ call', name, args
	def user_line(self, frame):
		import linecache, string, codehack
		name = codehack.getcodename(frame.f_code)
		if not name: name = '???'
		fn = frame.f_code.co_filename
		line = linecache.getline(fn, frame.f_lineno)
		print '+++', fn, frame.f_lineno, name, ':', string.strip(line)
	def user_return(self, frame, retval):
		print '+++ return', retval
	def user_exception(self, frame, exc_stuff):
		print '+++ exception', exc_stuff
		self.set_continue()

def foo(n):
	print 'foo(', n, ')'
	x = bar(n*10)
	print 'bar returned', x

def bar(a):
	print 'bar(', a, ')'
	return a/2

def test():
	import linecache
	linecache.checkcache()
	t = Tdb().init()
	t.run('import bdb; bdb.foo(10)')
<commit_msg>Added a method "format_stack_entry" to be used by derived classes. Forget about the temp file.<commit_after># A generic Python debugger base class.
# This class takes care of details of the trace facility;
# a derived class should implement user interaction.
# There are two debuggers based upon this:
# 'pdb', a text-oriented debugger not unlike dbx or gdb;
# and 'wdb', a window-oriented debugger.
# And of course... you can roll your own!

import sys

BdbQuit = 'bdb.BdbQuit' # Exception to give up completely

class Bdb: # Basic Debugger

	def init(self):
		self.breaks = {}
		return self

	def reset(self):
		self.botframe = None
		self.stopframe = None
		self.returnframe = None
		self.quitting = 0

	def trace_dispatch(self, frame, event, arg):
		if self.quitting:
			return # None
		if event == 'line':
			return self.dispatch_line(frame)
		if event == 'call':
			return self.dispatch_call(frame, arg)
		if event == 'return':
			return self.dispatch_return(frame, arg)
		if event == 'exception':
			return self.dispatch_exception(frame, arg)
		print 'bdb.Bdb.dispatch: unknown debugging event:', `event`
		return self.trace_dispatch

	def dispatch_line(self, frame):
		if self.stop_here(frame) or self.break_here(frame):
			self.user_line(frame)
			if self.quitting: raise BdbQuit
		return self.trace_dispatch

	def dispatch_call(self, frame, arg):
		frame.f_locals['__args__'] = arg
		if self.botframe is None:
			# First call of dispatch since reset()
			self.botframe = frame
			return self.trace_dispatch
		if not (self.stop_here(frame) or self.break_anywhere(frame)):
			# No need to trace this function
			return # None
		self.user_call(frame, arg)
		if self.quitting: raise BdbQuit
		return self.trace_dispatch

	def dispatch_return(self, frame, arg):
		if self.stop_here(frame) or frame == self.returnframe:
			self.user_return(frame, arg)
			if self.quitting: raise BdbQuit

	def dispatch_exception(self, frame, arg):
		if self.stop_here(frame):
			self.user_exception(frame, arg)
			if self.quitting: raise BdbQuit
		return self.trace_dispatch

	# Normally derived classes don't override the following
	# functions, but they may if they want to redefine the
	# definition of stopping and breakpoints.

	def stop_here(self, frame):
		if self.stopframe is None:
			return 1
		if frame is self.stopframe:
			return 1
		while frame is not None and frame is not self.stopframe:
			if frame is self.botframe:
				return 1
			frame = frame.f_back
		return 0

	def break_here(self, frame):
		if not self.breaks.has_key(frame.f_code.co_filename):
			return 0
		if not frame.f_lineno in \
				self.breaks[frame.f_code.co_filename]:
			return 0
		return 1

	def break_anywhere(self, frame):
		return self.breaks.has_key(frame.f_code.co_filename)

	# Derived classes should override the user_* functions
	# to gain control.

	def user_call(self, frame, argument_list):
		# This function is called when there is the remote possibility
		# that we ever need to stop in this function
		pass

	def user_line(self, frame):
		# This function is called when we stop or break at this line
		pass

	def user_return(self, frame, return_value):
		# This function is called when a return trap is set here
		pass

	def user_exception(self, frame, (exc_type, exc_value, exc_traceback)):
		# This function is called if an exception occurs,
		# but only if we are to stop at or just below this level
		pass

	# Derived classes and clients can call the following functions
	# to affect the stepping state.

	def set_step(self):
		# Stop after one line of code
		self.stopframe = None
		self.returnframe = None
		self.quitting = 0

	def set_next(self, frame):
		# Stop on the next line in or below the given frame
		self.stopframe = frame
		self.returnframe = None
		self.quitting = 0

	def set_return(self, frame):
		# Stop when returning from the given frame
		self.stopframe = frame.f_back
		self.returnframe = frame
		self.quitting = 0

	def set_continue(self):
		# Don't stop except at breakpoints or when finished
		self.stopframe = self.botframe
		self.returnframe = None
		self.quitting = 0

	def set_quit(self):
		self.stopframe = self.botframe
		self.returnframe = None
		self.quitting = 1
		sys.trace = None
		del sys.trace

	# Derived classes and clients can call the following functions
	# to manipulate breakpoints.  These functions return an
	# error message is something went wrong, None if all is well.
	# Call self.get_*break*() to see the breakpoints.

	def set_break(self, filename, lineno):
		import linecache # Import as late as possible
		line = linecache.getline(filename, lineno)
		if not line:
			return 'That line does not exist!'
		if not self.breaks.has_key(filename):
			self.breaks[filename] = []
		list = self.breaks[filename]
		if lineno in list:
			return 'There is already a breakpoint there!'
		list.append(lineno)

	def clear_break(self, filename, lineno):
		if not self.breaks.has_key(filename):
			return 'There are no breakpoints in that file!'
		if lineno not in self.breaks[filename]:
			return 'There is no breakpoint there!'
		self.breaks[filename].remove(lineno)
		if not self.breaks[filename]:
			del self.breaks[filename]

	def clear_all_file_breaks(self, filename):
		if not self.breaks.has_key(filename):
			return 'There are no breakpoints in that file!'
		del self.breaks[filename]

	def clear_all_breaks(self, filename, lineno):
		if not self.breaks:
			return 'There are no breakpoints!'
		self.breaks = {}

	def get_break(self, filename, lineno):
		return self.breaks.has_key(filename) and \
			lineno in self.breaks[filename]

	def get_file_breaks(self, filename):
		if self.breaks.has_key(filename):
			return self.breaks[filename]
		else:
			return []

	def get_all_breaks(self):
		return self.breaks

	# Derived classes and clients can call the following function
	# to get a data structure representing a stack trace.

	def get_stack(self, f, t):
		stack = []
		if t and t.tb_frame is f:
			t = t.tb_next
		while f is not None:
			stack.append((f, f.f_lineno))
			if f is self.botframe:
				break
			f = f.f_back
		stack.reverse()
		i = max(0, len(stack) - 1)
		while t is not None:
			stack.append((t.tb_frame, t.tb_lineno))
			t = t.tb_next
		return stack, i

	#

	def format_stack_entry(self, (frame, lineno)):
		import codehack, linecache, repr, string
		filename = frame.f_code.co_filename
		s = filename + '(' + `lineno` + ')'
		s = s + codehack.getcodename(frame.f_code)
		if frame.f_locals.has_key('__args__'):
			args = frame.f_locals['__args__']
			if args is not None:
				s = s + repr.repr(args)
		if frame.f_locals.has_key('__return__'):
			rv = frame.f_locals['__return__']
			s = s + '->'
			s = s + repr.repr(rv)
		line = linecache.getline(filename, lineno)
		if line: s = s + ': ' + string.strip(line)
		return s

	# The following two functions can be called by clients to use
	# a debugger to debug a statement, given as a string.

	def run(self, cmd):
		import __main__
		dict = __main__.__dict__
		self.runctx(cmd, dict, dict)

	def runctx(self, cmd, globals, locals):
		self.reset()
		sys.trace = self.trace_dispatch
		try:
			exec(cmd + '\n', globals, locals)
		except BdbQuit:
			pass
		finally:
			self.quitting = 1
			sys.trace = None
			del sys.trace
		# XXX What to do if the command finishes normally?

# -------------------- testing --------------------

class Tdb(Bdb):
	def user_call(self, frame, args):
		import codehack
		name = codehack.getcodename(frame.f_code)
		if not name: name = '???'
		print '+++ call', name, args
	def user_line(self, frame):
		import linecache, string, codehack
		name = codehack.getcodename(frame.f_code)
		if not name: name = '???'
		fn = frame.f_code.co_filename
		line = linecache.getline(fn, frame.f_lineno)
		print '+++', fn, frame.f_lineno, name, ':', string.strip(line)
	def user_return(self, frame, retval):
		print '+++ return', retval
	def user_exception(self, frame, exc_stuff):
		print '+++ exception', exc_stuff
		self.set_continue()

def foo(n):
	print 'foo(', n, ')'
	x = bar(n*10)
	print 'bar returned', x

def bar(a):
	print 'bar(', a, ')'
	return a/2

def test():
	import linecache
	linecache.checkcache()
	t = Tdb().init()
	t.run('import bdb; bdb.foo(10)')
<|endoftext|>


<commit_before># Filename: test_aanet.py
# pylint: disable=locally-disabled,C0111,R0904,C0301,C0103,W0212
from os.path import join, dirname
import time
import numpy as np

import km3pipe as kp
from km3pipe.testing import TestCase, patch, Mock, skipif, skip
from km3pipe.io.aanet import AanetPump, MetaParser
from km3pipe.core import Pipeline, Module, Pump

try:
    import aa
    NO_AANET = False
except ImportError:
    import sys
    sys.modules['ROOT'] = Mock()
    sys.modules['aa'] = Mock()
    NO_AANET = True
else:
    import ROOT

__author__ = "Tamas Gal"
__copyright__ = "Copyright 2018, Tamas Gal and the KM3NeT collaboration."
__credits__ = []
__license__ = "MIT"
__maintainer__ = "Tamas Gal"
__email__ = "tgal@km3net.de"
__status__ = "Development"

TEST_DATA_DIR = join(dirname(__file__), "../../kp-data/test_data")

@skipif(NO_AANET, reason="No aanet environment found.")
class TestAanetPump(TestCase):
    def setUp(self):
        self.fname = join(TEST_DATA_DIR, 'sea_data.root')
        self.pump = AanetPump(filename=self.fname)

    def test_reading_hits(self):
        data = {
            'hit_lengths': [48, 71, 22, 44, 35],
            'tot_sums': [1252, 1948, 562, 1089, 915],
            'mean_times': [
                56379691.1875, 65447948.25352113, 51971419.40909091,
                38274188.06818182, 73676555.94285715
            ],
            'channel_id_sums': [556, 958, 372, 593, 433]
        }

        blob_counter = 0
        for idx, blob in enumerate(self.pump):
            blob_counter += 1
            hits = blob['Hits']
            assert len(hits) == data['hit_lengths'][idx]
            self.assertAlmostEqual(data['mean_times'][idx], np.mean(hits.time))
            assert data['tot_sums'][idx] == np.sum(hits.tot)
            assert data['channel_id_sums'][idx] == np.sum(hits.channel_id)
        assert 5 == blob_counter

    def test_event_info(self):
        data = {
            'run_ids': [4143] * 5,
            'event_ids': [0, 1, 2, 3, 4],
            'frame_indices': [60, 41, 80, 61, 110],
            'mc_ids': [59, 40, 79, 60, 109],
            'timestamps': [
                1551549606, 1551549604, 1551549608, 1551549606, 1551549611
            ],
            'nanoseconds': [0, 100000000, 0, 100000000, 0],
            'trigger_counters': [0, 0, 1, 1, 2],
            'trigger_masks': [22, 22, 6, 22, 6],
            'overlays': [6, 14, 2, 6, 3],
        }
        blob_counter = 0
        for idx, blob in enumerate(self.pump):
            blob_counter += 1
            ei = blob['EventInfo'][0]
            assert data['run_ids'][idx] == ei.run_id
            assert data['event_ids'][idx] == ei.event_id
            assert data['frame_indices'][idx] == ei.frame_index
            assert data['timestamps'][idx] == ei.timestamp
            assert data['trigger_counters'][idx] == ei.trigger_counter
            assert data['overlays'][idx] == ei.overlays
            assert data['trigger_masks'][idx] == ei.trigger_mask
            assert data['nanoseconds'][idx] == ei.nanoseconds
        assert 5 == blob_counter

    def test_aanet_and_hdf5_conformity_with_converted_files(self):
        aanet_pump = AanetPump(filename=join(TEST_DATA_DIR, 'mupage.root'))
        hdf5_pump = kp.io.HDF5Pump(
            filename=join(TEST_DATA_DIR, 'mupage.root.h5')
        )

        blob_counter = 0
        for aanet_blob, hdf5_blob in zip(aanet_pump, hdf5_pump):
            blob_counter += 1
            keys = ['Hits', 'McHits', 'McTracks', 'EventInfo', 'RawHeader']
            for key in keys:
                aanet_data = aanet_blob[key]
                hdf5_data = hdf5_blob[key]
                assert aanet_data.dtype == hdf5_data.dtype

                for attr in aanet_data.dtype.names:
                    # special case for RawHeader, since everything is a string
                    if key == 'RawHeader':
                        self.assertTupleEqual(
                            tuple(aanet_data[attr]), tuple(hdf5_data[attr])
                        )
                        continue
                    # special case for arrays which are all NaNs
                    if np.all(np.isnan(aanet_data[attr])):
                        assert np.all(np.isnan(hdf5_data[attr]))
                        continue
                    # otherwise just compare them
                    assert np.allclose(aanet_data[attr], hdf5_data[attr])
        assert 3 == blob_counter

    def test_against_single_file_freeze(self):
        start_time = time.time()

        aanet_pump = AanetPump(filename=join(TEST_DATA_DIR, 'mupage.root'))
        assert 10 > (time.time() - start_time)

    def test_reading_hits_from_multiple_files(self):
        # never mix files like this, but it's OK for a test ;)
        aanet_pump = AanetPump(
            filenames=[
                join(TEST_DATA_DIR, 'sea_data.root'),
                join(TEST_DATA_DIR, 'mupage.root'),
                join(TEST_DATA_DIR, 'sea_data.root')
            ]
        )
        blob_counter = 0
        for blob in aanet_pump:
            blob_counter += 1

        assert 13 == blob_counter

    def test_reading_hits_from_multiple_mupage_files(self):
        aanet_pump = AanetPump(
            filenames=[
                join(TEST_DATA_DIR, 'mupage.root'),
                join(TEST_DATA_DIR, 'mupage.root')
            ]
        )
        blob_counter = 0
        for blob in aanet_pump:
            blob_counter += 1

        assert 6 == blob_counter

    def test_reading_hits_from_multiple_corsika_files(self):
        aanet_pump = AanetPump(
            filenames=[
                join(
                    TEST_DATA_DIR,
                    'mcv5.0.DAT004340.propa.sirene.jte.jchain.aanet.4340.root'
                ),
                join(
                    TEST_DATA_DIR,
                    'mcv5.0.DAT004340.propa.sirene.jte.jchain.aanet.4340.root'
                )
            ]
        )
        blob_counter = 0
        for blob in aanet_pump:
            blob_counter += 1

        assert 6 == blob_counter

    def test_reading_multiple_corsika_files_pipeline(self):
        class Tester(kp.Module):
            def configure(self):
                self.counter = 0

            def process(self, blob):
                self.counter = self.counter + 1
                print(self.counter)
                return blob

            def finish(self):
                assert (self.counter == 6)

        pipe = Pipeline(timeit=False)
        pipe.attach(
            AanetPump,
            filenames=[
                join(
                    TEST_DATA_DIR,
                    'mcv5.0.DAT004340.propa.sirene.jte.jchain.aanet.4340.root'
                ),
                join(
                    TEST_DATA_DIR,
                    'mcv5.0.DAT004340.propa.sirene.jte.jchain.aanet.4340.root'
                )
            ]
        )
        pipe.attach(Tester)
        pipe.drain()

class TestMetaParser(TestCase):
    def test_init(self):
        MetaParser()

    def test_parse_string_of_single_entry(self):
        string = b"A 123\nA 1.2.3\nA KM3NET\nA a\nA b\nA Linux"
        mp = MetaParser()
        mp.parse_string(string)

        assert 1 == len(mp.meta)

        assert b'A' == mp.meta[0]['application_name']
        assert b'123' == mp.meta[0]['revision']
        assert b'A a\nA b' == mp.meta[0]['command']

    def test_parse_string_of_multiple_entries(self):
        string = (
            b"A 123\nA 1.2.3\nA KM3NET\nA a\nA b\nA Linux\n"
            b"B 456\nB 4.5.6\nB KM3NET\nB c\nB Linux"
        )
        mp = MetaParser()
        mp.parse_string(string)

        assert 2 == len(mp.meta)

        assert b'A' == mp.meta[0]['application_name']
        assert b'123' == mp.meta[0]['revision']
        assert b'1.2.3' == mp.meta[0]['root_version']
        assert b'A a\nA b' == mp.meta[0]['command']

        assert b'B' == mp.meta[1]['application_name']
        assert b'456' == mp.meta[1]['revision']
        assert b'4.5.6' == mp.meta[1]['root_version']
        assert b'B c' == mp.meta[1]['command']

    def test_parse_testfile(self):
        fname = join(TEST_DATA_DIR, 'jprintmeta.log')
        with open(fname, 'rb') as fobj:
            string = fobj.read()
        mp = MetaParser()
        mp.parse_string(string)
        assert 7 == len(mp.meta)

        assert b'JEvt' == mp.meta[0]['application_name']
        assert b'9912' == mp.meta[0]['revision']
        assert b'5.34/23' == mp.meta[0]['root_version']
        assert mp.meta[0]['command'].startswith(b'JEvt /pbs/throng/km3net')
        assert mp.meta[0]['command'].endswith(b'2 --!')

        assert b'JEnergy' == mp.meta[1]['application_name']
        assert b'9912' == mp.meta[1]['revision']
        assert b'5.34/23' == mp.meta[1]['root_version']
        assert mp.meta[1]['command'].startswith(b'JEnergy /pbs/throng/km3net')
        assert mp.meta[1]['command'].endswith(b'1 --!')

        assert b'JTriggerEfficiency' == mp.meta[-1]['application_name']
        assert b'8519' == mp.meta[-1]['revision']
        assert b'5.34/23' == mp.meta[-1]['root_version']
        assert mp.meta[-1]['command'].startswith(
            b'JTriggerEfficiency /pbs/throng/km3net'
        )
        assert mp.meta[-1]['command'].endswith(b'326 --!')

    def test_get_table(self):
        string = (
            b"A 123\nA 1.2.3\nA KM3NET\nA a\nA b\nA Linux\n"
            b"B 456\nB 4.5.6\nB KM3NET\nB c\nB Linux"
        )
        mp = MetaParser()
        mp.parse_string(string)
        tab = mp.get_table()
        assert 2 == len(tab)
        assert '/meta' == tab.h5loc
        assert tab.h5singleton
        assert b'B c' == tab['command'][1]
        assert np.dtype('S7') == tab['command'].dtype

    def test_empty_string(self):
        mp = MetaParser()
        mp.parse_string('')
        assert mp.get_table() is None
<commit_msg>Update tests for aanet header<commit_after># Filename: test_aanet.py
# pylint: disable=locally-disabled,C0111,R0904,C0301,C0103,W0212
from os.path import join, dirname
import time
import numpy as np

import km3pipe as kp
from km3pipe.testing import TestCase, patch, Mock, skipif, skip
from km3pipe.io.aanet import AanetPump, MetaParser
from km3pipe.core import Pipeline, Module, Pump

try:
    import aa
    NO_AANET = False
except ImportError:
    import sys
    sys.modules['ROOT'] = Mock()
    sys.modules['aa'] = Mock()
    NO_AANET = True
else:
    import ROOT

__author__ = "Tamas Gal"
__copyright__ = "Copyright 2018, Tamas Gal and the KM3NeT collaboration."
__credits__ = []
__license__ = "MIT"
__maintainer__ = "Tamas Gal"
__email__ = "tgal@km3net.de"
__status__ = "Development"

TEST_DATA_DIR = join(dirname(__file__), "../../kp-data/test_data")

@skipif(NO_AANET, reason="No aanet environment found.")
class TestAanetPump(TestCase):
    def setUp(self):
        self.fname = join(TEST_DATA_DIR, 'sea_data.root')
        self.pump = AanetPump(filename=self.fname)

    def test_reading_hits(self):
        data = {
            'hit_lengths': [48, 71, 22, 44, 35],
            'tot_sums': [1252, 1948, 562, 1089, 915],
            'mean_times': [
                56379691.1875, 65447948.25352113, 51971419.40909091,
                38274188.06818182, 73676555.94285715
            ],
            'channel_id_sums': [556, 958, 372, 593, 433]
        }

        blob_counter = 0
        for idx, blob in enumerate(self.pump):
            blob_counter += 1
            hits = blob['Hits']
            assert len(hits) == data['hit_lengths'][idx]
            self.assertAlmostEqual(data['mean_times'][idx], np.mean(hits.time))
            assert data['tot_sums'][idx] == np.sum(hits.tot)
            assert data['channel_id_sums'][idx] == np.sum(hits.channel_id)
        assert 5 == blob_counter

    def test_event_info(self):
        data = {
            'run_ids': [4143] * 5,
            'event_ids': [0, 1, 2, 3, 4],
            'frame_indices': [60, 41, 80, 61, 110],
            'mc_ids': [59, 40, 79, 60, 109],
            'timestamps': [
                1551549606, 1551549604, 1551549608, 1551549606, 1551549611
            ],
            'nanoseconds': [0, 100000000, 0, 100000000, 0],
            'trigger_counters': [0, 0, 1, 1, 2],
            'trigger_masks': [22, 22, 6, 22, 6],
            'overlays': [6, 14, 2, 6, 3],
        }
        blob_counter = 0
        for idx, blob in enumerate(self.pump):
            blob_counter += 1
            ei = blob['EventInfo'][0]
            assert data['run_ids'][idx] == ei.run_id
            assert data['event_ids'][idx] == ei.event_id
            assert data['frame_indices'][idx] == ei.frame_index
            assert data['timestamps'][idx] == ei.timestamp
            assert data['trigger_counters'][idx] == ei.trigger_counter
            assert data['overlays'][idx] == ei.overlays
            assert data['trigger_masks'][idx] == ei.trigger_mask
            assert data['nanoseconds'][idx] == ei.nanoseconds
        assert 5 == blob_counter

    def test_aanet_and_hdf5_conformity_with_converted_files(self):
        aanet_pump = AanetPump(filename=join(TEST_DATA_DIR, 'mupage.root'))
        hdf5_pump = kp.io.HDF5Pump(
            filename=join(TEST_DATA_DIR, 'mupage.root.h5')
        )

        blob_counter = 0
        for aanet_blob, hdf5_blob in zip(aanet_pump, hdf5_pump):
            blob_counter += 1
            keys = ['Hits', 'McHits', 'McTracks', 'EventInfo']
            for key in keys:
                aanet_data = aanet_blob[key]
                hdf5_data = hdf5_blob[key]
                assert aanet_data.dtype == hdf5_data.dtype

                for attr in aanet_data.dtype.names:
                    # special case for arrays which are all NaNs
                    if np.all(np.isnan(aanet_data[attr])):
                        assert np.all(np.isnan(hdf5_data[attr]))
                        continue
                    # otherwise just compare them
                    assert np.allclose(aanet_data[attr], hdf5_data[attr])

            aanet_header = aanet_blob['Header']
            hdf5_header = hdf5_blob['Header']
            self.assertTupleEqual(aanet_header.seed, hdf5_header.seed)
            self.assertTupleEqual(aanet_header.norma, hdf5_header.norma)
            self.assertTupleEqual(
                aanet_header.coord_origin, hdf5_header.coord_origin
            )
            self.assertTupleEqual(aanet_header.physics, hdf5_header.physics)
            self.assertTupleEqual(aanet_header.propag, hdf5_header.propag)
            self.assertTupleEqual(aanet_header.livetime, hdf5_header.livetime)
            self.assertTupleEqual(aanet_header.cut_in, hdf5_header.cut_in)
            self.assertTupleEqual(aanet_header.can, hdf5_header.can)
            self.assertTupleEqual(
                aanet_header.seabottom, hdf5_header.seabottom
            )
            self.assertTupleEqual(
                aanet_header.start_run, hdf5_header.start_run
            )
            self.assertTupleEqual(aanet_header.simul, hdf5_header.simul)
        assert 3 == blob_counter

    def test_against_single_file_freeze(self):
        start_time = time.time()

        aanet_pump = AanetPump(filename=join(TEST_DATA_DIR, 'mupage.root'))
        assert 10 > (time.time() - start_time)

    def test_reading_hits_from_multiple_files(self):
        # never mix files like this, but it's OK for a test ;)
        aanet_pump = AanetPump(
            filenames=[
                join(TEST_DATA_DIR, 'sea_data.root'),
                join(TEST_DATA_DIR, 'mupage.root'),
                join(TEST_DATA_DIR, 'sea_data.root')
            ]
        )
        blob_counter = 0
        for blob in aanet_pump:
            blob_counter += 1

        assert 13 == blob_counter

    def test_reading_hits_from_multiple_mupage_files(self):
        aanet_pump = AanetPump(
            filenames=[
                join(TEST_DATA_DIR, 'mupage.root'),
                join(TEST_DATA_DIR, 'mupage.root')
            ]
        )
        blob_counter = 0
        for blob in aanet_pump:
            blob_counter += 1

        assert 6 == blob_counter

    def test_reading_hits_from_multiple_corsika_files(self):
        aanet_pump = AanetPump(
            filenames=[
                join(
                    TEST_DATA_DIR,
                    'mcv5.0.DAT004340.propa.sirene.jte.jchain.aanet.4340.root'
                ),
                join(
                    TEST_DATA_DIR,
                    'mcv5.0.DAT004340.propa.sirene.jte.jchain.aanet.4340.root'
                )
            ]
        )
        blob_counter = 0
        for blob in aanet_pump:
            blob_counter += 1

        assert 6 == blob_counter

    def test_reading_multiple_corsika_files_pipeline(self):
        class Tester(kp.Module):
            def configure(self):
                self.counter = 0

            def process(self, blob):
                self.counter = self.counter + 1
                print(self.counter)
                return blob

            def finish(self):
                assert (self.counter == 6)

        pipe = Pipeline(timeit=False)
        pipe.attach(
            AanetPump,
            filenames=[
                join(
                    TEST_DATA_DIR,
                    'mcv5.0.DAT004340.propa.sirene.jte.jchain.aanet.4340.root'
                ),
                join(
                    TEST_DATA_DIR,
                    'mcv5.0.DAT004340.propa.sirene.jte.jchain.aanet.4340.root'
                )
            ]
        )
        pipe.attach(Tester)
        pipe.drain()

class TestMetaParser(TestCase):
    def test_init(self):
        MetaParser()

    def test_parse_string_of_single_entry(self):
        string = b"A 123\nA 1.2.3\nA KM3NET\nA a\nA b\nA Linux"
        mp = MetaParser()
        mp.parse_string(string)

        assert 1 == len(mp.meta)

        assert b'A' == mp.meta[0]['application_name']
        assert b'123' == mp.meta[0]['revision']
        assert b'A a\nA b' == mp.meta[0]['command']

    def test_parse_string_of_multiple_entries(self):
        string = (
            b"A 123\nA 1.2.3\nA KM3NET\nA a\nA b\nA Linux\n"
            b"B 456\nB 4.5.6\nB KM3NET\nB c\nB Linux"
        )
        mp = MetaParser()
        mp.parse_string(string)

        assert 2 == len(mp.meta)

        assert b'A' == mp.meta[0]['application_name']
        assert b'123' == mp.meta[0]['revision']
        assert b'1.2.3' == mp.meta[0]['root_version']
        assert b'A a\nA b' == mp.meta[0]['command']

        assert b'B' == mp.meta[1]['application_name']
        assert b'456' == mp.meta[1]['revision']
        assert b'4.5.6' == mp.meta[1]['root_version']
        assert b'B c' == mp.meta[1]['command']

    def test_parse_testfile(self):
        fname = join(TEST_DATA_DIR, 'jprintmeta.log')
        with open(fname, 'rb') as fobj:
            string = fobj.read()
        mp = MetaParser()
        mp.parse_string(string)
        assert 7 == len(mp.meta)

        assert b'JEvt' == mp.meta[0]['application_name']
        assert b'9912' == mp.meta[0]['revision']
        assert b'5.34/23' == mp.meta[0]['root_version']
        assert mp.meta[0]['command'].startswith(b'JEvt /pbs/throng/km3net')
        assert mp.meta[0]['command'].endswith(b'2 --!')

        assert b'JEnergy' == mp.meta[1]['application_name']
        assert b'9912' == mp.meta[1]['revision']
        assert b'5.34/23' == mp.meta[1]['root_version']
        assert mp.meta[1]['command'].startswith(b'JEnergy /pbs/throng/km3net')
        assert mp.meta[1]['command'].endswith(b'1 --!')

        assert b'JTriggerEfficiency' == mp.meta[-1]['application_name']
        assert b'8519' == mp.meta[-1]['revision']
        assert b'5.34/23' == mp.meta[-1]['root_version']
        assert mp.meta[-1]['command'].startswith(
            b'JTriggerEfficiency /pbs/throng/km3net'
        )
        assert mp.meta[-1]['command'].endswith(b'326 --!')

    def test_get_table(self):
        string = (
            b"A 123\nA 1.2.3\nA KM3NET\nA a\nA b\nA Linux\n"
            b"B 456\nB 4.5.6\nB KM3NET\nB c\nB Linux"
        )
        mp = MetaParser()
        mp.parse_string(string)
        tab = mp.get_table()
        assert 2 == len(tab)
        assert '/meta' == tab.h5loc
        assert tab.h5singleton
        assert b'B c' == tab['command'][1]
        assert np.dtype('S7') == tab['command'].dtype

    def test_empty_string(self):
        mp = MetaParser()
        mp.parse_string('')
        assert mp.get_table() is None
<|endoftext|>


<commit_before># Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *
import os
import shutil

class Mumax(MakefilePackage, CudaPackage):
    """GPU accelerated micromagnetic simulator."""

    homepage = "http://mumax.github.io"
    url      = "https://github.com/mumax/3/archive/3.10beta.tar.gz"

    version('3.10beta', sha256='f20fbd90a4b531fe5a0d8acc3d4505a092a5e426f5f53218a22a87d445daf0e9')

    variant('cuda', default=True,
            description='Use CUDA; must be true')
    variant('cuda_arch', 'N/A',
            description='Mumax will build GPU kernels that it supports')
    variant('gnuplot', default=False,
            description='Use gnuplot for graphs')

    depends_on('cuda')
    depends_on('go', type='build')
    depends_on('gnuplot', type='run', when='+gnuplot')

    conflicts('~cuda', msg='mumax requires cuda')

    patch('https://github.com/mumax/3/commit/2cf5c9a6985c9eb16a124c6bd96aed75b4a30c24.patch',
          sha256='a43b2ca6c9f9edfb1fd6d916a599f85a57c8bb3f9ee38148b1988fd82feec8ad')

    @property
    def gopath(self):
        return self.stage.path

    @property
    def mumax_gopath_dir(self):
        return join_path(self.gopath, 'src/github.com/mumax/3')

    def do_stage(self, mirror_only=False):
        super(Mumax, self).do_stage(mirror_only)
        if not os.path.exists(self.mumax_gopath_dir):
            # Need to move source to $GOPATH and then symlink the original
            # stage directory
            shutil.move(self.stage.source_path, self.mumax_gopath_dir)
            force_symlink(self.mumax_gopath_dir, self.stage.source_path)

    # filter out targets that do not exist
    def edit(self, spec, prefix):
        filter_file(r'(^ln -sf .*)', r'#\1', 'make.bash')
        filter_file(r'(^\(cd test)', r'#\1', 'make.bash')

    def setup_build_environment(self, env):
        env.prepend_path('GOPATH', self.gopath)

    def install(self, spec, prefix):
        make()
        with working_dir(self.gopath):
            install_tree('bin', prefix.bin)
<commit_msg>Add mumax-3.10 release version (#18740)<commit_after># Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *
import os
import shutil

class Mumax(MakefilePackage, CudaPackage):
    """GPU accelerated micromagnetic simulator."""

    homepage = "http://mumax.github.io"
    url      = "https://github.com/mumax/3/archive/v3.10.tar.gz"

    maintainers = ['glennpj']

    version('3.10',
            sha256='42c858661cec3896685ff4babea11e711f71fd6ea37d20c2bed7e4a918702caa',
            preferred=True)
    version('3.10beta',
            sha256='f20fbd90a4b531fe5a0d8acc3d4505a092a5e426f5f53218a22a87d445daf0e9',
            url='https://github.com/mumax/3/archive/3.10beta.tar.gz')

    variant('cuda', default=True,
            description='Use CUDA; must be true')
    variant('gnuplot', default=False,
            description='Use gnuplot for graphs')

    depends_on('cuda')
    depends_on('go', type='build')
    depends_on('gnuplot', type='run', when='+gnuplot')

    conflicts('~cuda', msg='mumax requires cuda')

    patch('https://github.com/mumax/3/commit/2cf5c9a6985c9eb16a124c6bd96aed75b4a30c24.patch',
          sha256='a43b2ca6c9f9edfb1fd6d916a599f85a57c8bb3f9ee38148b1988fd82feec8ad',
          when='@3.10beta')

    @property
    def cuda_arch(self):
        cuda_arch = ' '.join(self.spec.variants['cuda_arch'].value)
        if cuda_arch == 'none':
            raise InstallError(
                'Must select at least one value for cuda_arch'
            )
        return cuda_arch

    @property
    def gopath(self):
        return self.stage.path

    @property
    def mumax_gopath_dir(self):
        return join_path(self.gopath, 'src/github.com/mumax/3')

    def do_stage(self, mirror_only=False):
        super(Mumax, self).do_stage(mirror_only)
        if not os.path.exists(self.mumax_gopath_dir):
            # Need to move source to $GOPATH and then symlink the original
            # stage directory
            shutil.move(self.stage.source_path, self.mumax_gopath_dir)
            force_symlink(self.mumax_gopath_dir, self.stage.source_path)

    # filter out targets that do not exist
    def edit(self, spec, prefix):
        filter_file(r'(^all: cudakernels) hooks$', r'\1', 'Makefile')

    @when('@3.10beta')
    def edit(self, spec, prefix):
        filter_file(r'(^ln -sf .*)', r'#\1', 'make.bash')
        filter_file(r'(^\(cd test)', r'#\1', 'make.bash')
        filter_file(r'(for cc in ).*(; do)', r'\1{0}\2'.format(self.cuda_arch),
                    'cuda/make.bash')

    def setup_build_environment(self, env):
        env.prepend_path('GOPATH', self.gopath)
        env.set('CUDA_CC', self.cuda_arch)

    def install(self, spec, prefix):
        make()
        with working_dir(self.gopath):
            install_tree('bin', prefix.bin)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Filename: qrunprocessor.py
# Author: Tamas Gal <tgal@km3net.de>
"""
=========================================
Create a summary for a given list of runs
=========================================

Use this batch runner to process a given list of run numbers with a
script, which takes a `-o` to create a summary file, which has the name
of the processed file + a given suffix.
The actual call will look like this, with a bit of copy safeness:

    /abs/path/to/SCRIPT run.root -o /abs/path/to/OUTPUT_PATH+SUFFIX

Before constructing the job scripts, the ``OUTPUT_PATH`` will be traversed
to find files which have already been converted to avoid multiple conversions.

.. code-block:: console

    Usage:
        qrunprocessor [options] DET_ID RUN_LIST OUTPUT_PATH SCRIPT
        qrunprocessor (-h | --help)

    Options:
        DET_ID         Detector ID (e.g. 29).
        RUN_LIST       Path to the file containing the space separated run IDs.
        OUTPUT_PATH    Path to store the individual summary files.
        SCRIPT         The script to fire up.
        -s SUFFIX      The suffix, appended by SCRIPT [default: .summary.h5].
        -n N_FILES     Number of files to process per job [default: 10].
        -e ET          Estimated walltime per file in minutes [default: 15].
        -f FSIZE       Estimated filesystem size for a job [default: 12G].
        -m VMEM        Estimated vmem for a job [default: 8G].
        -j JOBNAME     The name of the submitted jobs [default: qrunprocessor].
        -l LOG_PATH    Path of the job log files [default: qlogs].
        -q             Dryrun: don't submit jobs, just print the job script.
        -h --help      Show this screen.

"""
__author__ = "Tamas Gal"
__email__ = "tgal@km3net.de"
__version__ = "1.0"

def main():
    from docopt import docopt
    args = docopt(__doc__, version=__version__)

    from glob import glob
    import os
    from os.path import basename, join, abspath
    import pathlib
    import time
    import km3pipe as kp
    from km3pipe.tools import chunks, iexists

    try:
        from tqdm import tqdm
    except ImportError:
        tqdm = lambda x: x

    cprint = kp.logger.get_printer('qrunprocessor')
    log = kp.logger.get_logger('qrunprocessor')

    CWD = os.getcwd()
    RUN_LIST = join(CWD, args['RUN_LIST'])
    OUTPUT_PATH = join(CWD, args['OUTPUT_PATH'])
    SCRIPT = abspath(args['SCRIPT'])
    SUFFIX = args['-s']
    DET_ID = int(args['DET_ID'])
    ET_PER_FILE = int(args['-e']) * 60    # [s]
    FILES_PER_JOB = int(args['-n'])
    FSIZE = args['-f']
    VMEM = args['-m']
    LOG_PATH = args['-l']
    JOB_NAME = args['-j']
    DRYRUN = args['-q']

    pathlib.Path(OUTPUT_PATH).mkdir(parents=True, exist_ok=True)

    with open(RUN_LIST, 'r') as fobj:
        run_numbers = [int(run) for run in fobj.read().split()]

    irods_files = []
    for run in run_numbers:
        irods_path = kp.tools.irods_filepath(DET_ID, run)
        irods_files.append(irods_path)

    processed_files = [
        basename(f) for f in glob(join(OUTPUT_PATH, '*{}'.format(SUFFIX)))
    ]

    rem_files = []
    for irods_file in irods_files:
        if basename(irods_file) + SUFFIX not in processed_files:
            rem_files.append(irods_file)

    cprint("Checking if files are accessible on iRODS")
    missing_files_on_irods = 0
    for rem_file in tqdm(rem_files):
        if not iexists(rem_file):
            log.error(
                "Skipping file, since not found on iRODS: {}".format(rem_file)
            )
            rem_files.remove(rem_file)
            missing_files_on_irods += 1

    cprint(
        "{} runs in total, {} already processed ({} missing on iRODS).".format(
            len(irods_files), len(processed_files), missing_files_on_irods
        )
    )
    cprint("Proceeding with the remaining {} files.".format(len(rem_files)))

    s = kp.shell.Script()

    for job_id, file_chunk in enumerate(chunks(rem_files, FILES_PER_JOB)):
        n_files = len(file_chunk)
        s.add("echo Creating run summary for {} files".format(n_files))
        s.add("cd $TMPDIR; mkdir -p $USER; cd $USER")
        s.add("echo")

        for ipath in file_chunk:
            fname = basename(ipath)
            s.separator(' ')
            s.separator('=')
            s.echo("Processing {}:".format(fname))
            s.add('pwd')
            s.iget(ipath)
            s.add('ls -al {}'.format(fname))
            s.add('JPrintTree -f {}'.format(fname))
            out_fname = fname + SUFFIX
            out_fpath = join(OUTPUT_PATH, out_fname)
            tmp_fname = out_fname + '.copying'
            tmp_fpath = join(OUTPUT_PATH, tmp_fname)
            s.add("{} {} -o {}".format(SCRIPT, fname, out_fname))
            s.cp(out_fname, tmp_fpath)
            s.add("rm {}".format(out_fname))
            s.mv(tmp_fpath, out_fpath)
            s.echo("File '{}' processed.".format(fname))
            s.separator('-')

        walltime = time.strftime(
            '%H:%M:%S', time.gmtime(ET_PER_FILE * n_files)
        )

        kp.shell.qsub(
            s,
            '{}_{}'.format(JOB_NAME, job_id),
            walltime=walltime,
            fsize=FSIZE,
            vmem=VMEM,
            log_path=LOG_PATH,
            irods=True,
            dryrun=DRYRUN
        )

        if DRYRUN:
            break

        s.clear()

if __name__ == "__main__":
    main()
<commit_msg>Remove Jpp dependency<commit_after>#!/usr/bin/env python
# Filename: qrunprocessor.py
# Author: Tamas Gal <tgal@km3net.de>
"""
=========================================
Create a summary for a given list of runs
=========================================

Use this batch runner to process a given list of run numbers with a
script, which takes a `-o` to create a summary file, which has the name
of the processed file + a given suffix.
The actual call will look like this, with a bit of copy safeness:

    /abs/path/to/SCRIPT run.root -o /abs/path/to/OUTPUT_PATH+SUFFIX

Before constructing the job scripts, the ``OUTPUT_PATH`` will be traversed
to find files which have already been converted to avoid multiple conversions.

.. code-block:: console

    Usage:
        qrunprocessor [options] DET_ID RUN_LIST OUTPUT_PATH SCRIPT
        qrunprocessor (-h | --help)

    Options:
        DET_ID         Detector ID (e.g. 29).
        RUN_LIST       Path to the file containing the space separated run IDs.
        OUTPUT_PATH    Path to store the individual summary files.
        SCRIPT         The script to fire up.
        -s SUFFIX      The suffix, appended by SCRIPT [default: .summary.h5].
        -n N_FILES     Number of files to process per job [default: 10].
        -e ET          Estimated walltime per file in minutes [default: 15].
        -f FSIZE       Estimated filesystem size for a job [default: 12G].
        -m VMEM        Estimated vmem for a job [default: 8G].
        -j JOBNAME     The name of the submitted jobs [default: qrunprocessor].
        -l LOG_PATH    Path of the job log files [default: qlogs].
        -q             Dryrun: don't submit jobs, just print the job script.
        -h --help      Show this screen.

"""
__author__ = "Tamas Gal"
__email__ = "tgal@km3net.de"
__version__ = "1.0"

def main():
    from docopt import docopt
    args = docopt(__doc__, version=__version__)

    from glob import glob
    import os
    from os.path import basename, join, abspath
    import pathlib
    import time
    import km3pipe as kp
    from km3pipe.tools import chunks, iexists

    try:
        from tqdm import tqdm
    except ImportError:

        def tqdm(x):
            return x

    cprint = kp.logger.get_printer('qrunprocessor')
    log = kp.logger.get_logger('qrunprocessor')

    CWD = os.getcwd()
    RUN_LIST = join(CWD, args['RUN_LIST'])
    OUTPUT_PATH = join(CWD, args['OUTPUT_PATH'])
    SCRIPT = abspath(args['SCRIPT'])
    SUFFIX = args['-s']
    DET_ID = int(args['DET_ID'])
    ET_PER_FILE = int(args['-e']) * 60    # [s]
    FILES_PER_JOB = int(args['-n'])
    FSIZE = args['-f']
    VMEM = args['-m']
    LOG_PATH = args['-l']
    JOB_NAME = args['-j']
    DRYRUN = args['-q']

    pathlib.Path(OUTPUT_PATH).mkdir(parents=True, exist_ok=True)

    with open(RUN_LIST, 'r') as fobj:
        run_numbers = [int(run) for run in fobj.read().split()]

    irods_files = []
    for run in run_numbers:
        irods_path = kp.tools.irods_filepath(DET_ID, run)
        irods_files.append(irods_path)

    processed_files = [
        basename(f) for f in glob(join(OUTPUT_PATH, '*{}'.format(SUFFIX)))
    ]

    rem_files = []
    for irods_file in irods_files:
        if basename(irods_file) + SUFFIX not in processed_files:
            rem_files.append(irods_file)

    cprint("Checking if files are accessible on iRODS")
    missing_files_on_irods = 0
    for rem_file in tqdm(rem_files):
        if not iexists(rem_file):
            log.error(
                "Skipping file, since not found on iRODS: {}".format(rem_file)
            )
            rem_files.remove(rem_file)
            missing_files_on_irods += 1

    cprint(
        "{} runs in total, {} already processed ({} missing on iRODS).".format(
            len(irods_files), len(processed_files), missing_files_on_irods
        )
    )
    cprint("Proceeding with the remaining {} files.".format(len(rem_files)))

    s = kp.shell.Script()

    for job_id, file_chunk in enumerate(chunks(rem_files, FILES_PER_JOB)):
        n_files = len(file_chunk)
        s.add("echo Creating run summary for {} files".format(n_files))
        s.add("cd $TMPDIR; mkdir -p $USER; cd $USER")
        s.add("echo")

        for ipath in file_chunk:
            fname = basename(ipath)
            s.separator(' ')
            s.separator('=')
            s.echo("Processing {}:".format(fname))
            s.add('pwd')
            s.iget(ipath)
            s.add('ls -al {}'.format(fname))
            s.add('km3pipe --version')
            s.add('KPrintTree -f {}'.format(fname))
            out_fname = fname + SUFFIX
            out_fpath = join(OUTPUT_PATH, out_fname)
            tmp_fname = out_fname + '.copying'
            tmp_fpath = join(OUTPUT_PATH, tmp_fname)
            s.add("{} {} -o {}".format(SCRIPT, fname, out_fname))
            s.cp(out_fname, tmp_fpath)
            s.add("rm {}".format(out_fname))
            s.mv(tmp_fpath, out_fpath)
            s.echo("File '{}' processed.".format(fname))
            s.separator('-')

        walltime = time.strftime(
            '%H:%M:%S', time.gmtime(ET_PER_FILE * n_files)
        )

        kp.shell.qsub(
            s,
            '{}_{}'.format(JOB_NAME, job_id),
            walltime=walltime,
            fsize=FSIZE,
            vmem=VMEM,
            log_path=LOG_PATH,
            irods=True,
            dryrun=DRYRUN
        )

        if DRYRUN:
            break

        s.clear()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Various decorator utilities

.. codeauthor:: Joe Decapo <joe@polka.cat>

"""

import sys

from termcolor import cprint

import clowder.util.formatting as fmt
from clowder.git.project_repo import ProjectRepo
from clowder.util.connectivity import is_offline

def clowder_required(func):
    """If no clowder repo, print clowder not found message and exit"""

    def wrapper(*args, **kwargs):
        """Wrapper"""

        _validate_clowder_repo_exists(args[0].clowder_repo)
        return func(*args, **kwargs)

    return wrapper

def initialize_skip_projects(func):
    """Initialize skip projects to empty list if necessary"""

    def wrapper(*args, **kwargs):
        """Wrapper"""

        if kwargs['skip'] is None:
            kwargs['skip'] = []
        return func(*args, **kwargs)

    return wrapper

def project_repo_exists(func):
    """If no git repo exists, print error message and exit"""

    def wrapper(*args, **kwargs):
        """Wrapper"""

        instance = args[0]
        if not ProjectRepo.existing_git_repository(instance.full_path()):
            cprint(" - Project repo is missing", 'red')
            return
        return func(*args, **kwargs)

    return wrapper

def print_clowder_repo_status(func):
    """Print clowder repo status"""

    def wrapper(*args, **kwargs):
        """Wrapper"""

        instance = args[0]
        instance.clowder_repo.print_status()
        return func(*args, **kwargs)

    return wrapper

def print_clowder_repo_status_fetch(func):
    """Print clowder repo status"""

    def wrapper(*args, **kwargs):
        """Wrapper"""

        instance = args[0]
        instance.clowder_repo.print_status(fetch=True)
        return func(*args, **kwargs)

    return wrapper

def network_connection_required(func):
    """If no network connection, print offline message and exit"""

    def wrapper(*args, **kwargs):
        """Wrapper"""

        if is_offline():
            print(fmt.offline_error())
            sys.exit(1)
        return func(*args, **kwargs)

    return wrapper

def valid_clowder_yaml_required(func):
    """If clowder.yaml is invalid, print invalid yaml message and exit"""

    def wrapper(*args, **kwargs):
        """Wrapper"""

        instance = args[0]
        _validate_clowder_repo_exists(instance.clowder_repo)
        if instance.invalid_yaml:
            print(fmt.invalid_yaml_error())
            print(fmt.error(instance.error))
            sys.exit(1)
        return func(*args, **kwargs)

    return wrapper

def _validate_clowder_repo_exists(repo):
    """If clowder repo doesn't exist, print message and exit"""

    if repo is None:
        cprint(' - No clowder found in the current directory\n', 'red')
        sys.exit(1)<commit_msg>Add Sphinx documentation to validate_clowder_repo method<commit_after># -*- coding: utf-8 -*-
"""Various decorator utilities

.. codeauthor:: Joe Decapo <joe@polka.cat>

"""

import sys

from termcolor import cprint

import clowder.util.formatting as fmt
from clowder.git.project_repo import ProjectRepo
from clowder.util.connectivity import is_offline

def clowder_required(func):
    """If no clowder repo, print clowder not found message and exit"""

    def wrapper(*args, **kwargs):
        """Wrapper"""

        _validate_clowder_repo_exists(args[0].clowder_repo)
        return func(*args, **kwargs)

    return wrapper

def initialize_skip_projects(func):
    """Initialize skip projects to empty list if necessary"""

    def wrapper(*args, **kwargs):
        """Wrapper"""

        if kwargs['skip'] is None:
            kwargs['skip'] = []
        return func(*args, **kwargs)

    return wrapper

def project_repo_exists(func):
    """If no git repo exists, print error message and exit"""

    def wrapper(*args, **kwargs):
        """Wrapper"""

        instance = args[0]
        if not ProjectRepo.existing_git_repository(instance.full_path()):
            cprint(" - Project repo is missing", 'red')
            return
        return func(*args, **kwargs)

    return wrapper

def print_clowder_repo_status(func):
    """Print clowder repo status"""

    def wrapper(*args, **kwargs):
        """Wrapper"""

        instance = args[0]
        instance.clowder_repo.print_status()
        return func(*args, **kwargs)

    return wrapper

def print_clowder_repo_status_fetch(func):
    """Print clowder repo status"""

    def wrapper(*args, **kwargs):
        """Wrapper"""

        instance = args[0]
        instance.clowder_repo.print_status(fetch=True)
        return func(*args, **kwargs)

    return wrapper

def network_connection_required(func):
    """If no network connection, print offline message and exit"""

    def wrapper(*args, **kwargs):
        """Wrapper"""

        if is_offline():
            print(fmt.offline_error())
            sys.exit(1)
        return func(*args, **kwargs)

    return wrapper

def valid_clowder_yaml_required(func):
    """If clowder.yaml is invalid, print invalid yaml message and exit"""

    def wrapper(*args, **kwargs):
        """Wrapper"""

        instance = args[0]
        _validate_clowder_repo_exists(instance.clowder_repo)
        if instance.invalid_yaml:
            print(fmt.invalid_yaml_error())
            print(fmt.error(instance.error))
            sys.exit(1)
        return func(*args, **kwargs)

    return wrapper

def _validate_clowder_repo_exists(repo):
    """If clowder repo doesn't exist, print message and exit

    :param ClowderRepo repo: Repo to check
    :return:
    """

    if repo is None:
        cprint(' - No clowder found in the current directory\n', 'red')
        sys.exit(1)
<|endoftext|>


<commit_before>import re
import xml.etree.ElementTree as ET

from shapely.affinity import scale

class SVGImage:
    def __init__(self, width: int, height: int, scale: float=1):
        self.width = width
        self.height = height
        self.scale = scale
        self.g = ET.Element('g', {})
        self.defs = ET.Element('defs')
        self.def_i = 0

        blur_filter = ET.Element('filter', {'id': 'wallblur'})
        blur_filter.append(ET.Element('feGaussianBlur',
                                      {'in': 'SourceGraphic',
                                       'stdDeviation': str(int(0.7 * self.scale))}))
        self.defs.append(blur_filter)

    def get_element(self):
        root = ET.Element('svg', {
            'width': str(self.width*self.scale),
            'height': str(self.height*self.scale),
            'xmlns:svg': 'http://www.w3.org/2000/svg',
            'xmlns': 'http://www.w3.org/2000/svg',
            'xmlns:xlink': 'http://www.w3.org/1999/xlink',
        })
        root.append(self.defs)
        root.append(self.g)
        return root

    def get_xml(self):
        return ET.tostring(self.get_element()).decode()

    def new_defid(self):
        defid = 's'+str(self.def_i)
        self.def_i += 1
        return defid

    def _trim_decimals(self, data):
        return re.sub(r'([0-9]+)\.0', r'\1', re.sub(r'([0-9]+\.[0-9])[0-9]+', r'\1', data))

    def _create_geometry(self, geometry):
        geometry = scale(geometry, xfact=1, yfact=-1, origin=(self.width / 2, self.height / 2))
        geometry = scale(geometry, xfact=self.scale, yfact=self.scale, origin=(0, 0))
        element = ET.fromstring(self._trim_decimals(geometry.svg(0, '#FFFFFF')))
        if element.tag != 'g':
            new_element = ET.Element('g')
            new_element.append(element)
            element = new_element
        paths = element.findall('polyline')
        if len(paths) == 0:
            paths = element.findall('path')
        for path in paths:
            path.attrib.pop('opacity', None)
            path.attrib.pop('fill', None)
            path.attrib.pop('fill-rule', None)
            path.attrib.pop('stroke', None)
            path.attrib.pop('stroke-width', None)
        return element

    def register_geometry(self, geometry, defid=None, as_clip_path=False, comment=None):
        if defid is None:
            defid = self.new_defid()

        element = self._create_geometry(geometry)

        if as_clip_path:
            element.tag = 'clipPath'
        element.set('id', defid)
        self.defs.append(element)
        return defid

    def add_clip_path(self, *geometries, inverted=False, subtract=False, defid=None):
        if defid is None:
            defid = self.new_defid()

        clippath = ET.Element('clipPath', {'id': defid})
        clippath.append(ET.Element('use', {'xlink:href': '#' + geometries[0]}))
        self.defs.append(clippath)
        return defid

    def add_geometry(self, geometry=None, fill_color=None, fill_opacity=None, opacity=None, filter=None,
                     stroke_width=0.0, stroke_color=None, stroke_opacity=None, stroke_linejoin=None, clip_path=None):
        if geometry is not None:
            if not geometry:
                return
            if isinstance(geometry, str):
                element = ET.Element('use', {'xlink:href': '#'+geometry})
            else:
                element = self._create_geometry(geometry)
        else:
            element = ET.Element('rect', {'width': '100%', 'height': '100%'})
        element.set('fill', fill_color or 'none')
        if fill_opacity:
            element.set('fill-opacity', str(fill_opacity)[:4])
        if stroke_width:
            element.set('stroke-width', self._trim_decimals(str(stroke_width * self.scale)))
        if stroke_color:
            element.set('stroke', stroke_color)
        if stroke_opacity:
            element.set('stroke-opacity', str(stroke_opacity)[:4])
        if stroke_linejoin:
            element.set('stroke-linejoin', stroke_linejoin)
        if opacity:
            element.set('opacity', str(opacity)[:4])
        if filter:
            element.set('filter', 'url(#'+filter+')')
        if clip_path:
            element.set('clip-path', 'url(#'+clip_path+')')

        self.g.append(element)
        return element
<commit_msg>fix svg-rendering of geometries that contain polylines and paths<commit_after>import re
import xml.etree.ElementTree as ET
from itertools import chain

from shapely.affinity import scale

class SVGImage:
    def __init__(self, width: int, height: int, scale: float=1):
        self.width = width
        self.height = height
        self.scale = scale
        self.g = ET.Element('g', {})
        self.defs = ET.Element('defs')
        self.def_i = 0

        blur_filter = ET.Element('filter', {'id': 'wallblur'})
        blur_filter.append(ET.Element('feGaussianBlur',
                                      {'in': 'SourceGraphic',
                                       'stdDeviation': str(int(0.7 * self.scale))}))
        self.defs.append(blur_filter)

    def get_element(self):
        root = ET.Element('svg', {
            'width': str(self.width*self.scale),
            'height': str(self.height*self.scale),
            'xmlns:svg': 'http://www.w3.org/2000/svg',
            'xmlns': 'http://www.w3.org/2000/svg',
            'xmlns:xlink': 'http://www.w3.org/1999/xlink',
        })
        root.append(self.defs)
        root.append(self.g)
        return root

    def get_xml(self):
        return ET.tostring(self.get_element()).decode()

    def new_defid(self):
        defid = 's'+str(self.def_i)
        self.def_i += 1
        return defid

    def _trim_decimals(self, data):
        return re.sub(r'([0-9]+)\.0', r'\1', re.sub(r'([0-9]+\.[0-9])[0-9]+', r'\1', data))

    def _create_geometry(self, geometry):
        geometry = scale(geometry, xfact=1, yfact=-1, origin=(self.width / 2, self.height / 2))
        geometry = scale(geometry, xfact=self.scale, yfact=self.scale, origin=(0, 0))
        element = ET.fromstring(self._trim_decimals(geometry.svg(0, '#FFFFFF')))
        if element.tag != 'g':
            new_element = ET.Element('g')
            new_element.append(element)
            element = new_element

        for elem in chain(element.findall('polyline'), element.findall('path')):
            elem.attrib.pop('opacity', None)
            elem.attrib.pop('fill', None)
            elem.attrib.pop('fill-rule', None)
            elem.attrib.pop('stroke', None)
            elem.attrib.pop('stroke-width', None)
        return element

    def register_geometry(self, geometry, defid=None, as_clip_path=False, comment=None):
        if defid is None:
            defid = self.new_defid()

        element = self._create_geometry(geometry)

        if as_clip_path:
            element.tag = 'clipPath'
        element.set('id', defid)
        self.defs.append(element)
        return defid

    def add_clip_path(self, *geometries, inverted=False, subtract=False, defid=None):
        if defid is None:
            defid = self.new_defid()

        clippath = ET.Element('clipPath', {'id': defid})
        clippath.append(ET.Element('use', {'xlink:href': '#' + geometries[0]}))
        self.defs.append(clippath)
        return defid

    def add_geometry(self, geometry=None, fill_color=None, fill_opacity=None, opacity=None, filter=None,
                     stroke_width=0.0, stroke_color=None, stroke_opacity=None, stroke_linejoin=None, clip_path=None):
        if geometry is not None:
            if not geometry:
                return
            if isinstance(geometry, str):
                element = ET.Element('use', {'xlink:href': '#'+geometry})
            else:
                element = self._create_geometry(geometry)
        else:
            element = ET.Element('rect', {'width': '100%', 'height': '100%'})
        element.set('fill', fill_color or 'none')
        if fill_opacity:
            element.set('fill-opacity', str(fill_opacity)[:4])
        if stroke_width:
            element.set('stroke-width', self._trim_decimals(str(stroke_width * self.scale)))
        if stroke_color:
            element.set('stroke', stroke_color)
        if stroke_opacity:
            element.set('stroke-opacity', str(stroke_opacity)[:4])
        if stroke_linejoin:
            element.set('stroke-linejoin', stroke_linejoin)
        if opacity:
            element.set('opacity', str(opacity)[:4])
        if filter:
            element.set('filter', 'url(#'+filter+')')
        if clip_path:
            element.set('clip-path', 'url(#'+clip_path+')')

        self.g.append(element)
        return element
<|endoftext|>