<commit_before>"""
    Block definitions for all known level formats
    Reader for block definitions stored in .yaml files (based on Minecraft X-Ray)
"""
from __future__ import absolute_import
from logging import getLogger
import os
import traceback
from os.path import join
from collections import defaultdict, namedtuple
import json

import numpy
import re

log = getLogger(__name__)
import logging
logging.basicConfig(level=logging.INFO)

MCYDecreasing = 0
MCYIncreasing = 1
MCZDecreasing = 2 #south
MCZIncreasing = 3 #north
MCXDecreasing = 4 #west
MCXIncreasing = 5 #east

class BlockType(namedtuple("_BlockType", "ID, meta, blocktypeSet")):
    """
    Value object representing an (id, data, blocktypeSet) tuple.
    Accessing its attributes will return the corresponding elements
    of its parent material's json arrays.

    """

    def __repr__(self):
        try:
            names = " (%s (%s%s)" % (self.displayName, self.internalName, self.blockState)
        except:
            names = ""
        return "<BlockType (%d:%d)%s>" % (self.ID, self.meta, names)

    def __str__(self):
        return "%s (%s%s) [%s:%s]" % (self.displayName, self.internalName, self.blockState, self.ID, self.meta)

    def __cmp__(self, other):
        if not isinstance(other, BlockType):
            return -1
        if None in (self, other):
            return -1

        key = lambda a: (a.internalName, a.blockState)
        return cmp(key(self), key(other))

    def __getattr__(self, attr):
        return self.blocktypeSet.getBlockTypeAttr(self, attr)

id_limit = 4096

def openResource(filename):
    path = join(os.path.dirname(__file__), filename)
    if not os.path.exists(path):
        try:
            # py2exe, .egg
            import pkg_resources
            return pkg_resources.resource_stream(__name__, filename)
        except ImportError as e:
            log.exception("pkg_resources not available")
            raise

    return file(path)

class BlockTypeSet(object):
    defaultColor = (0xc9, 0x77, 0xf0, 0xff)

    def __init__(self, defaultName="Unused Block", idMapping=None):
        object.__init__(self)
        self.allBlocks = []
        self.blockJsons = {}
        self.IDsByState = {}  # internalName[blockstates] -> (id, meta)
        self.statesByID = {}  # (id, meta) -> internalName[blockstates]

        self.IDsByName = {}  # internalName -> id
        self.namesByID = {}  # id -> internalName

        self.defaultBlockstates = {}  # internalName -> [blockstates]

        self.defaults = {
            'displayName': defaultName,
            'opacity': 15,
            'brightness': 0,
            'internalName': 'UNKNOWN_NAME',
            'blockState': '[UNKNOWN_STATE]',
            'unlocalizedName': 'name.unknown',
            'opaqueCube': True,
            'renderType': 3,  # Model block - defaults to question mark box
            'unknown': False,  # False for blocks loaded from builtin .json, True for FML IDs, False for blocks configured in editor
        }

        self.aka = defaultdict(lambda: "")

        self.brightness = numpy.zeros(id_limit, dtype='uint8')
        self.brightness[:] = self.defaults['brightness']
        self.opacity = numpy.zeros(id_limit, dtype='uint8')
        self.opacity[:] = self.defaults['opacity']
        self.renderColor = numpy.zeros((id_limit, 16, 3), dtype='uint8')
        self.renderColor[:] = 0xFF
        self.mapColor = numpy.zeros((id_limit, 16, 3), dtype='uint8')
        self.mapColor[:] = 0xFF

        self.opaqueCube = numpy.ones((id_limit, ), dtype='uint8')
        self.opaqueCube[0] = 0

        self.name = "Unnamed Set"
        self.namePrefix = "minecraft:"

    def getBlockTypeAttr(self, block, attr):
        """
        Called when accessing an attribute of a BlockType.
        :param block:
        :type block:
        :param attr:
        :type attr:
        :return:
        :rtype:
        """

        nameAndState = self.statesByID.get((block.ID, block.meta))
        if nameAndState is None:
            nameAndState = self.statesByID.get((block.ID, 0))
        if nameAndState is None:
            return self.defaults[attr]

        if attr == "internalName":
            return self._splitInternalName(nameAndState)[0]

        if attr == "blockState":
            return self._splitInternalName(nameAndState)[1]

        blockJson = self.blockJsons[nameAndState]
        if attr == "json":
            return blockJson
        if attr == "resourcePath":
            resourcePath = blockJson.get(attr)
            if resourcePath is None:
                return block.internalName.replace(self.namePrefix, "")

        retval = blockJson.get(attr)
        if retval is None:
            retval = self.defaults[attr]

        return retval

    def __repr__(self):
        return "<BlockTypeSet ({0})>".format(self.name)

    def get(self, key, default=None):
        try:
            return self[key]
        except KeyError:
            return default

    def __len__(self):
        return len(self.allBlocks)

    def __iter__(self):
        return iter(self.allBlocks)

    def _splitInternalName(self, nameAndState):
        if '[' in nameAndState:
            idx = nameAndState.index('[')
            internalName, blockState = nameAndState[:idx], nameAndState[idx:]
        else:
            internalName = nameAndState
            blockState = ""
        return internalName, blockState

    def __getitem__(self, nameAndState):
        """

        Possible uses:
            # block ID
            level.blocktypes[120]

            # block ID, meta
            level.blocktypes[120, 0]

            # returns Air
            level.blocktypes["air"]

            # returns default state of Powered Rail
            level.blocktypes["golden_rail"]

            # returns Powered Rail with given state
            level.blocktypes["golden_rail[powered=false,shape=north_south]"]

        :type nameAndState: str
        """

        if isinstance(nameAndState, tuple):
            internalName, blockState = nameAndState
            if isinstance(internalName, basestring):
                if ":" not in internalName:
                    internalName = self.namePrefix + internalName
                ID, meta = self.IDsByState[internalName + blockState]
            else:  # (ID, meta)
                ID, meta = nameAndState

        elif isinstance(nameAndState, basestring):
            if ":" not in nameAndState:
                nameAndState = self.namePrefix + nameAndState
            if nameAndState not in self.IDsByState:
                if nameAndState in self.defaultBlockstates:
                    nameAndState += self.defaultBlockstates[nameAndState]
            ID, meta = self.IDsByState[nameAndState]
        else:  # block ID
            ID = nameAndState
            meta = 0

        return BlockType(ID, meta, self)

    def blocksMatching(self, name):
        name = name.lower()
        return [v for v in self.allBlocks if name in v.displayName.lower() or name in v.aka.lower()]

    def addIDMappingFromJSON(self, entries):
        for ID, meta, nameAndState in entries:
            internalName, blockState = self._splitInternalName(nameAndState)
            self.IDsByState[nameAndState] = ID, meta
            self.IDsByName[internalName] = ID
        self.statesByID = {v: k for (k, v) in self.IDsByState.iteritems()}
        self.namesByID = {v: k for (k, v) in self.IDsByName.iteritems()}

    def addBlockIDsFromSchematicTag(self, mapping):
        for ID, tag in mapping.iteritems():
            try:
                ID = int(ID)
            except ValueError:
                log.error("Can't parse %s as an integer.")
                continue

            name = tag.value
            if name not in self.IDsByName:
                self.IDsByName[name] = ID
                self.namesByID[ID] = name

    def addItemIDsFromSchematicTag(self, mapping):
        pass

    def addBlocksFromJSON(self, blockJson):
        for block in blockJson:
            try:
                self.addJsonBlock(block)
            except (KeyError, ValueError) as e:
                log.warn(u"Exception while parsing block: %r", e)
                traceback.print_exc()
                log.warn(u"Block dict: \n%s", block)
                raise

        for block in self.allBlocks:
            if block.internalName not in self.defaultBlockstates:
                self.defaultBlockstates[block.internalName] = block.blockState

    def addJsonBlock(self, blockJson):
        """
        Adds a block to this blockset using the keys and values in `block`.

        Required keys:
            internalName - String used by Minecraft to identify this block

        :type blockJson: dict
        :return:
        :rtype:
        """

        ID = blockJson.get("id")
        internalName = blockJson.get("internalName")
        if ID is None and internalName is None:
            raise ValueError("Block definition must have either `id` or `internalName` attribute.")

        #log.debug("Adding block: \n%s", json.dumps(blockJson, indent=2))
        nameAndState = blockJson.get("blockState", internalName)
        internalName, blockState = self._splitInternalName(nameAndState)

        IDmeta = self.IDsByState.get(nameAndState)
        if IDmeta is None:
            log.info("No ID mapping for %s, skipping...", internalName)
            return
        ID, meta = IDmeta
        self.allBlocks.append(BlockType(ID, meta, self))

        oldJson = self.blockJsons.get(internalName + blockState)
        if oldJson is None:
            oldJson = self.blockJsons[internalName + blockState] = blockJson
        else:
            oldJson.update(blockJson)

        blockJson = oldJson
        del oldJson

        if blockJson.get("defaultState"):
            self.defaultBlockstates[internalName] = blockState

        for key in [
            'opaqueCube',
            'brightness',
            'opacity',
        ]:  # does not have data axis
            if key in blockJson:
                array = getattr(self, key)
                array[ID] = blockJson[key]

        color = blockJson.get('renderColor')
        if color is not None:
            self.renderColor[ID, meta or slice(None)] = ((color >> 16) & 0xff,
                                                              (color >> 8) & 0xff,
                                                              color & 0xff)

        color = blockJson.get('materialMapColor')
        if color is not None:
            self.mapColor[ID, meta or slice(None)] = ((color >> 16) & 0xff,
                                                           (color >> 8) & 0xff,
                                                           color & 0xff)

_cachedJsons = {}

def getJsonFile(filename):
    if filename in _cachedJsons:
        return _cachedJsons[filename]

    f = openResource(filename)
    try:
        s = f.read()
        #log.info(u"Loading block ID mapping from (%s) %s", len(s), f)
        entries = json.loads(s)
        _cachedJsons[filename] = entries
        return entries
    except EnvironmentError as e:
        log.error(u"Exception while loading JSON from %s: %s", f, e)
        traceback.print_exc()

class PCBlockTypeSet(BlockTypeSet):
    def __init__(self):
        super(PCBlockTypeSet, self).__init__()
        self.name = "Alpha"
        self.addIDMappingFromJSON(getJsonFile("idmapping_raw.json"))
        self.addIDMappingFromJSON(getJsonFile("idmapping.json"))
        self.addBlocksFromJSON(getJsonFile("minecraft_raw.json"))
        self.addBlocksFromJSON(getJsonFile("minecraft.json"))

pc_blocktypes = PCBlockTypeSet()

# def printStaticDefs(blocktypes):
#     varNames = set()
#     for block in blocktypes.allBlocks:
#         blockJson = block.json
#         name = blockJson['displayName']
#         #internalName = blockJson['internalName']
#         nameAndState = blockJson['blockState']
#         internalName, blockState = blocktypes._splitInternalName(nameAndState)
#         variableName = re.sub(r"[^\w]", '', name)
#         varName = variableName
#         i=2
#         while varName in varNames:
#             varName = "%s%d" % (variableName, i)
#             i += 1
#         varNames.add(varName)
#         print("self.%s = self['%s', '%s']" % (varName, internalName, blockState))
#     raise SystemExit
# printStaticDefs(pc_blocktypes)

blocktypes_named = {"Alpha": pc_blocktypes}

def blocktypeConverter(destTypes, sourceTypes):
    """

    :param destTypes:
    :type destTypes: BlockTypeSet
    :param sourceTypes:
    :type sourceTypes: BlockTypeSet
    :return:
    :rtype:
    """
    idTable = numpy.arange(0, id_limit, dtype=numpy.uint16)
    for name, ID in sourceTypes.IDsByName.iteritems():
        if name in destTypes.IDsByName:
            try:
                idTable[ID] = destTypes.IDsByName[name]
            except IndexError:
                log.error("Can't insert %s->%s into %s (%s???) ??", ID, destTypes.IDsByName[name], idTable.shape, type(ID))
                raise

    def _convert(ID, meta):
        return idTable[ID], meta

    return _convert

<commit_msg>Add default values to BlockType used for rendering. Remove special case for resourcePath (it should only point to valid states files or be None)<commit_after>"""
    Block definitions for all known level formats
    Reader for block definitions stored in .yaml files (based on Minecraft X-Ray)
"""
from __future__ import absolute_import
from logging import getLogger
import os
import traceback
from os.path import join
from collections import defaultdict, namedtuple
import json

import numpy
import re

log = getLogger(__name__)
import logging
logging.basicConfig(level=logging.INFO)

MCYDecreasing = 0
MCYIncreasing = 1
MCZDecreasing = 2 #south
MCZIncreasing = 3 #north
MCXDecreasing = 4 #west
MCXIncreasing = 5 #east

class BlockType(namedtuple("_BlockType", "ID, meta, blocktypeSet")):
    """
    Value object representing an (id, data, blocktypeSet) tuple.
    Accessing its attributes will return the corresponding elements
    of its parent material's json arrays.

    """

    def __repr__(self):
        try:
            names = " (%s (%s%s)" % (self.displayName, self.internalName, self.blockState)
        except:
            names = ""
        return "<BlockType (%d:%d)%s>" % (self.ID, self.meta, names)

    def __str__(self):
        return "%s (%s%s) [%s:%s]" % (self.displayName, self.internalName, self.blockState, self.ID, self.meta)

    def __cmp__(self, other):
        if not isinstance(other, BlockType):
            return -1
        if None in (self, other):
            return -1

        key = lambda a: (a.internalName, a.blockState)
        return cmp(key(self), key(other))

    def __getattr__(self, attr):
        return self.blocktypeSet.getBlockTypeAttr(self, attr)

id_limit = 4096

def openResource(filename):
    path = join(os.path.dirname(__file__), filename)
    if not os.path.exists(path):
        try:
            # py2exe, .egg
            import pkg_resources
            return pkg_resources.resource_stream(__name__, filename)
        except ImportError as e:
            log.exception("pkg_resources not available")
            raise

    return file(path)

class BlockTypeSet(object):
    defaultColor = (0xc9, 0x77, 0xf0, 0xff)

    def __init__(self, defaultName="Unused Block", idMapping=None):
        object.__init__(self)
        self.allBlocks = []
        self.blockJsons = {}
        self.IDsByState = {}  # internalName[blockstates] -> (id, meta)
        self.statesByID = {}  # (id, meta) -> internalName[blockstates]

        self.IDsByName = {}  # internalName -> id
        self.namesByID = {}  # id -> internalName

        self.defaultBlockstates = {}  # internalName -> [blockstates]

        self.defaults = {
            'displayName': defaultName,
            'opacity': 15,
            'brightness': 0,
            'internalName': 'UNKNOWN_NAME',
            'blockState': '[UNKNOWN_STATE]',
            'unlocalizedName': 'name.unknown',
            'opaqueCube': True,
            'resourcePath': None,  # Name of states file in assets/minecraft/blockstates/
            'resourceVariant': None,  # Name of variant from above file to use for this block type
            'forcedModel': None,  # Name of model file in assets/minecraft/models/block, forced by user configured blocks
            'forcedModelTextures': None,  # Mapping of texture variables to full texture paths, forced by user configured blocks
            'forcedModelRotation': None,  # Mapping of 'x', 'y', 'z' to 0, 90, 180 etc to rotate model. for non-forced blocks this is specified in the states file.
            'forcedRotationFlags': None,  # One or more of 'north', 'south' etc to use when rotating this block
            'renderType': 3,  # Model block - defaults to question mark box
            'unknown': False,  # False for blocks loaded from builtin .json, True for FML IDs, False for blocks configured in editor
            'color': 0xffffff,
        }

        self.aka = defaultdict(lambda: "")

        self.brightness = numpy.zeros(id_limit, dtype='uint8')
        self.brightness[:] = self.defaults['brightness']
        self.opacity = numpy.zeros(id_limit, dtype='uint8')
        self.opacity[:] = self.defaults['opacity']
        self.renderColor = numpy.zeros((id_limit, 16, 3), dtype='uint8')
        self.renderColor[:] = 0xFF
        self.mapColor = numpy.zeros((id_limit, 16, 3), dtype='uint8')
        self.mapColor[:] = 0xFF

        self.opaqueCube = numpy.ones((id_limit, ), dtype='uint8')
        self.opaqueCube[0] = 0

        self.name = "Unnamed Set"
        self.namePrefix = "minecraft:"

    def getBlockTypeAttr(self, block, attr):
        """
        Called when accessing an attribute of a BlockType.
        :param block:
        :type block:
        :param attr:
        :type attr:
        :return:
        :rtype:
        """

        nameAndState = self.statesByID.get((block.ID, block.meta))
        if nameAndState is None:
            nameAndState = self.statesByID.get((block.ID, 0))
        if nameAndState is None:
            return self.defaults[attr]

        if attr == "internalName":
            return self._splitInternalName(nameAndState)[0]

        if attr == "blockState":
            return self._splitInternalName(nameAndState)[1]

        blockJson = self.blockJsons[nameAndState]
        if attr == "json":
            return blockJson

        retval = blockJson.get(attr)
        if retval is None:
            retval = self.defaults[attr]

        return retval

    def __repr__(self):
        return "<BlockTypeSet ({0})>".format(self.name)

    def get(self, key, default=None):
        try:
            return self[key]
        except KeyError:
            return default

    def __len__(self):
        return len(self.allBlocks)

    def __iter__(self):
        return iter(self.allBlocks)

    def _splitInternalName(self, nameAndState):
        if '[' in nameAndState:
            idx = nameAndState.index('[')
            internalName, blockState = nameAndState[:idx], nameAndState[idx:]
        else:
            internalName = nameAndState
            blockState = ""
        return internalName, blockState

    def __getitem__(self, nameAndState):
        """

        Possible uses:
            # block ID
            level.blocktypes[120]

            # block ID, meta
            level.blocktypes[120, 0]

            # returns Air
            level.blocktypes["air"]

            # returns default state of Powered Rail
            level.blocktypes["golden_rail"]

            # returns Powered Rail with given state
            level.blocktypes["golden_rail[powered=false,shape=north_south]"]

        :type nameAndState: str
        """

        if isinstance(nameAndState, tuple):
            internalName, blockState = nameAndState
            if isinstance(internalName, basestring):
                if ":" not in internalName:
                    internalName = self.namePrefix + internalName
                ID, meta = self.IDsByState[internalName + blockState]
            else:  # (ID, meta)
                ID, meta = nameAndState

        elif isinstance(nameAndState, basestring):
            if ":" not in nameAndState:
                nameAndState = self.namePrefix + nameAndState
            if nameAndState not in self.IDsByState:
                if nameAndState in self.defaultBlockstates:
                    nameAndState += self.defaultBlockstates[nameAndState]
            ID, meta = self.IDsByState[nameAndState]
        else:  # block ID
            ID = nameAndState
            meta = 0

        return BlockType(ID, meta, self)

    def blocksMatching(self, name):
        name = name.lower()
        return [v for v in self.allBlocks if name in v.displayName.lower() or name in v.aka.lower()]

    def addIDMappingFromJSON(self, entries):
        for ID, meta, nameAndState in entries:
            internalName, blockState = self._splitInternalName(nameAndState)
            self.IDsByState[nameAndState] = ID, meta
            self.IDsByName[internalName] = ID
        self.statesByID = {v: k for (k, v) in self.IDsByState.iteritems()}
        self.namesByID = {v: k for (k, v) in self.IDsByName.iteritems()}

    def addBlockIDsFromSchematicTag(self, mapping):
        for ID, tag in mapping.iteritems():
            try:
                ID = int(ID)
            except ValueError:
                log.error("Can't parse %s as an integer.")
                continue

            name = tag.value
            if name not in self.IDsByName:
                self.IDsByName[name] = ID
                self.namesByID[ID] = name

    def addItemIDsFromSchematicTag(self, mapping):
        pass

    def addBlocksFromJSON(self, blockJson):
        for block in blockJson:
            try:
                self.addJsonBlock(block)
            except (KeyError, ValueError) as e:
                log.warn(u"Exception while parsing block: %r", e)
                traceback.print_exc()
                log.warn(u"Block dict: \n%s", block)
                raise

        for block in self.allBlocks:
            if block.internalName not in self.defaultBlockstates:
                self.defaultBlockstates[block.internalName] = block.blockState

    def addJsonBlock(self, blockJson):
        """
        Adds a block to this blockset using the keys and values in `block`.

        Required keys:
            internalName - String used by Minecraft to identify this block

        :type blockJson: dict
        :return:
        :rtype:
        """

        ID = blockJson.get("id")
        internalName = blockJson.get("internalName")
        if ID is None and internalName is None:
            raise ValueError("Block definition must have either `id` or `internalName` attribute.")

        #log.debug("Adding block: \n%s", json.dumps(blockJson, indent=2))
        nameAndState = blockJson.get("blockState", internalName)
        internalName, blockState = self._splitInternalName(nameAndState)

        IDmeta = self.IDsByState.get(nameAndState)
        if IDmeta is None:
            log.info("No ID mapping for %s, skipping...", internalName)
            return
        ID, meta = IDmeta
        self.allBlocks.append(BlockType(ID, meta, self))

        oldJson = self.blockJsons.get(internalName + blockState)
        if oldJson is None:
            oldJson = self.blockJsons[internalName + blockState] = blockJson
        else:
            oldJson.update(blockJson)

        blockJson = oldJson
        del oldJson

        if blockJson.get("defaultState"):
            self.defaultBlockstates[internalName] = blockState

        for key in [
            'opaqueCube',
            'brightness',
            'opacity',
        ]:  # does not have data axis
            if key in blockJson:
                array = getattr(self, key)
                array[ID] = blockJson[key]

        color = blockJson.get('renderColor')
        if color is not None:
            self.renderColor[ID, meta or slice(None)] = ((color >> 16) & 0xff,
                                                              (color >> 8) & 0xff,
                                                              color & 0xff)

        color = blockJson.get('materialMapColor')
        if color is not None:
            self.mapColor[ID, meta or slice(None)] = ((color >> 16) & 0xff,
                                                           (color >> 8) & 0xff,
                                                           color & 0xff)

_cachedJsons = {}

def getJsonFile(filename):
    if filename in _cachedJsons:
        return _cachedJsons[filename]

    f = openResource(filename)
    try:
        s = f.read()
        #log.info(u"Loading block ID mapping from (%s) %s", len(s), f)
        entries = json.loads(s)
        _cachedJsons[filename] = entries
        return entries
    except EnvironmentError as e:
        log.error(u"Exception while loading JSON from %s: %s", f, e)
        traceback.print_exc()

class PCBlockTypeSet(BlockTypeSet):
    def __init__(self):
        super(PCBlockTypeSet, self).__init__()
        self.name = "Alpha"
        self.addIDMappingFromJSON(getJsonFile("idmapping_raw.json"))
        self.addIDMappingFromJSON(getJsonFile("idmapping.json"))
        self.addBlocksFromJSON(getJsonFile("minecraft_raw.json"))
        self.addBlocksFromJSON(getJsonFile("minecraft.json"))

pc_blocktypes = PCBlockTypeSet()

# def printStaticDefs(blocktypes):
#     varNames = set()
#     for block in blocktypes.allBlocks:
#         blockJson = block.json
#         name = blockJson['displayName']
#         #internalName = blockJson['internalName']
#         nameAndState = blockJson['blockState']
#         internalName, blockState = blocktypes._splitInternalName(nameAndState)
#         variableName = re.sub(r"[^\w]", '', name)
#         varName = variableName
#         i=2
#         while varName in varNames:
#             varName = "%s%d" % (variableName, i)
#             i += 1
#         varNames.add(varName)
#         print("self.%s = self['%s', '%s']" % (varName, internalName, blockState))
#     raise SystemExit
# printStaticDefs(pc_blocktypes)

blocktypes_named = {"Alpha": pc_blocktypes}

def blocktypeConverter(destTypes, sourceTypes):
    """

    :param destTypes:
    :type destTypes: BlockTypeSet
    :param sourceTypes:
    :type sourceTypes: BlockTypeSet
    :return:
    :rtype:
    """
    idTable = numpy.arange(0, id_limit, dtype=numpy.uint16)
    for name, ID in sourceTypes.IDsByName.iteritems():
        if name in destTypes.IDsByName:
            try:
                idTable[ID] = destTypes.IDsByName[name]
            except IndexError:
                log.error("Can't insert %s->%s into %s (%s???) ??", ID, destTypes.IDsByName[name], idTable.shape, type(ID))
                raise

    def _convert(ID, meta):
        return idTable[ID], meta

    return _convert

<|endoftext|>


<commit_before>"""
OTC Pad Device.
Entry point for device code.

Usage:
  main.py [<port>] [--no-display]

Options:
  --no-display    Don't initialize the display.
"""
from docopt import docopt

import rpcserver
import rpcmethods
import confirm

if __name__ == "__main__":
    arguments = docopt(__doc__)
    port = arguments["<port>"]
    port = int(port) if port else 9051
    display_enabled = not arguments["--no-display"]

    if display_enabled:
        print "Starting confirmation display process."
        csc = confirm.ConfirmScreenController(fullscreen=False)
        rpcmethods.csc = csc
        csc.start()
    else:
        print "Starting dummy confirmer. (Always says yes)"
        csc = confirm.DummyConfirmScreenController()
        rpcmethods.csc = csc

    print "Starting rpc server."
    rpcserver.app.run(host="0.0.0.0", port=9051, debug=False)

    print "Shutting down."
    if csc:
        csc.shutdown()
<commit_msg>fix port switch<commit_after>"""
OTC Pad Device.
Entry point for device code.

Usage:
  main.py [<port>] [--no-display]

Options:
  --no-display    Don't initialize the display.
"""
from docopt import docopt

import rpcserver
import rpcmethods
import confirm

if __name__ == "__main__":
    arguments = docopt(__doc__)
    port = arguments["<port>"]
    port = int(port) if port else 9051
    display_enabled = not arguments["--no-display"]

    if display_enabled:
        print "Starting confirmation display process."
        csc = confirm.ConfirmScreenController(fullscreen=False)
        rpcmethods.csc = csc
        csc.start()
    else:
        print "Starting dummy confirmer. (Always says yes)"
        csc = confirm.DummyConfirmScreenController()
        rpcmethods.csc = csc

    print "Starting rpc server."
    rpcserver.app.run(host="0.0.0.0", port=port, debug=False)

    print "Shutting down."
    if csc:
        csc.shutdown()
<|endoftext|>


<commit_before>import re

from mitmproxy import exceptions
from mitmproxy import filt

class Replace:
    def __init__(self):
        self.lst = []

    def configure(self, options, updated):
        """
            .replacements is a list of tuples (fpat, rex, s):

            fpatt: a string specifying a filter pattern.
            rex: a regular expression, as bytes.
            s: the replacement string, as bytes
        """
        lst = []
        for fpatt, rex, s in options.replacements:
            cpatt = filt.parse(fpatt)
            if not cpatt:
                raise exceptions.OptionsError(
                    "Invalid filter pattern: %s" % fpatt
                )
            try:
                re.compile(rex)
            except re.error as e:
                raise exceptions.OptionsError(
                    "Invalid regular expression: %s - %s" % (rex, str(e))
                )
            lst.append((rex, s, cpatt))
        self.lst = lst

    def execute(self, f):
        for rex, s, cpatt in self.lst:
            if cpatt(f):
                if f.response:
                    f.response.replace(rex, s, re.DOTALL)
                else:
                    f.request.replace(rex, s, re.DOTALL)

    def request(self, flow):
        if not flow.reply.has_message:
            self.execute(flow)

    def response(self, flow):
        if not flow.reply.has_message:
            self.execute(flow)
<commit_msg>Convert to flags=value for future compatibility<commit_after>import re

from mitmproxy import exceptions
from mitmproxy import filt

class Replace:
    def __init__(self):
        self.lst = []

    def configure(self, options, updated):
        """
            .replacements is a list of tuples (fpat, rex, s):

            fpatt: a string specifying a filter pattern.
            rex: a regular expression, as bytes.
            s: the replacement string, as bytes
        """
        lst = []
        for fpatt, rex, s in options.replacements:
            cpatt = filt.parse(fpatt)
            if not cpatt:
                raise exceptions.OptionsError(
                    "Invalid filter pattern: %s" % fpatt
                )
            try:
                re.compile(rex)
            except re.error as e:
                raise exceptions.OptionsError(
                    "Invalid regular expression: %s - %s" % (rex, str(e))
                )
            lst.append((rex, s, cpatt))
        self.lst = lst

    def execute(self, f):
        for rex, s, cpatt in self.lst:
            if cpatt(f):
                if f.response:
                    f.response.replace(rex, s, flags=re.DOTALL)
                else:
                    f.request.replace(rex, s, flags=re.DOTALL)

    def request(self, flow):
        if not flow.reply.has_message:
            self.execute(flow)

    def response(self, flow):
        if not flow.reply.has_message:
            self.execute(flow)
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Illustrative GUI driving a small subset of PyAbel methods

# code adapted from http://matplotlib.org/examples/user_interfaces/embedding_in_tk.html

import numpy as np

import matplotlib
matplotlib.use('TkAgg')

from numpy import arange, sin, pi
from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg,\
                                              NavigationToolbar2TkAgg

from matplotlib.figure import Figure

from abel.hansenlaw import iabel_hansenlaw
from abel.tools.vmi import calculate_speeds, find_image_center_by_slice

import sys
if sys.version_info[0] < 3:
    import Tkinter as tk
else:
    import tkinter as tk
from tkFileDialog import askopenfilename

# ------------

root = tk.Tk()
root.wm_title("Simple GUI PyAbel")

f = Figure(figsize=(5, 4), dpi=100)
a = f.add_subplot(111)

def _display():
    global IM, canvas
    f.clf()
    a = f.add_subplot(111)
    a.imshow(IM, vmin=0)
    canvas.show()

def _getfilename():
    global IM
    fn = askopenfilename()
    IM = np.loadtxt(fn)
    _display()

def _center():
    global IM
    IM, offset = find_image_center_by_slice(IM)
    _display()

def _transform():
    global IM, canvas
    AIM = iabel_hansenlaw(IM)
    f.clf()
    a = f.add_subplot(111)
    a.imshow(AIM, vmin=0, vmax=AIM.max()/2)
    canvas.show()

def _speed():
    global IM, canvas
    AIM = iabel_hansenlaw(IM)
    speed, radial = calculate_speeds(AIM)
    f.clf()
    a = f.add_subplot(111)
    a.plot(radial, speed/speed[50:].max())
    a.axis(xmax=500,ymin=-0.05)
    canvas.show()

def _quit():
    root.quit()     # stops mainloop
    root.destroy()  # this is necessary on Windows to prevent
                    # Fatal Python Error: PyEval_RestoreThread: NULL tstate

# buttons
tk.Button(master=root, text='Load image file', command=_getfilename).pack(anchor=tk.W)
tk.Button(master=root, text='center image', command=_center).pack(anchor=tk.N)
tk.Button(master=root, text='raw image', command=_display).pack(anchor=tk.N)
tk.Button(master=root, text='inverse Abel tansform', command=_transform).pack(anchor=tk.N)
tk.Button(master=root, text='speed distribution', command=_speed).pack(anchor=tk.N)
tk.Button(master=root, text='Quit', command=_quit).pack(anchor=tk.SW)

# a tk.DrawingArea
canvas = FigureCanvasTkAgg(f, master=root)
a.annotate("load image file", (0.5, 0.5), horizontalalignment="center")
canvas.show()
canvas.get_tk_widget().pack(side=tk.TOP, fill=tk.BOTH, expand=1)

toolbar = NavigationToolbar2TkAgg(canvas, root)
toolbar.update()
canvas._tkcanvas.pack(side=tk.TOP, fill=tk.BOTH, expand=1)

tk.mainloop()
<commit_msg>pep8 fixes for example_simple_GUI.py<commit_after>#!/usr/bin/env python

# Illustrative GUI driving a small subset of PyAbel methods

# code adapted from
# http://matplotlib.org/examples/user_interfaces/embedding_in_tk.html

import numpy as np

from abel.hansenlaw import iabel_hansenlaw
from abel.tools.vmi import calculate_speeds, find_image_center_by_slice

import sys
if sys.version_info[0] < 3:
    import Tkinter as tk
else:
    import tkinter as tk
from tkFileDialog import askopenfilename

import matplotlib
matplotlib.use('TkAgg')
from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg,\
                                              NavigationToolbar2TkAgg
from matplotlib.figure import Figure

# ------------

root = tk.Tk()
root.wm_title("Simple GUI PyAbel")

f = Figure(figsize=(5, 4), dpi=100)
a = f.add_subplot(111)

def _display():
    global IM, canvas
    f.clf()
    a = f.add_subplot(111)
    a.imshow(IM, vmin=0)
    canvas.show()

def _getfilename():
    global IM
    fn = askopenfilename()
    IM = np.loadtxt(fn)
    _display()

def _center():
    global IM
    IM, offset = find_image_center_by_slice(IM)
    _display()

def _transform():
    global IM, canvas
    AIM = iabel_hansenlaw(IM)
    f.clf()
    a = f.add_subplot(111)
    a.imshow(AIM, vmin=0, vmax=AIM.max()/2)
    canvas.show()

def _speed():
    global IM, canvas
    AIM = iabel_hansenlaw(IM)
    speed, radial = calculate_speeds(AIM)
    f.clf()
    a = f.add_subplot(111)
    a.plot(radial, speed/speed[50:].max())
    a.axis(xmax=500, ymin=-0.05)
    canvas.show()

def _quit():
    root.quit()     # stops mainloop
    root.destroy()  # this is necessary on Windows to prevent
                    # Fatal Python Error: PyEval_RestoreThread: NULL tstate

# buttons
tk.Button(master=root, text='Load image file', command=_getfilename)\
   .pack(anchor=tk.W)
tk.Button(master=root, text='center image', command=_center).pack(anchor=tk.N)
tk.Button(master=root, text='raw image', command=_display).pack(anchor=tk.N)
tk.Button(master=root, text='inverse Abel tansform', command=_transform)\
   .pack(anchor=tk.N)
tk.Button(master=root, text='speed distribution', command=_speed)\
   .pack(anchor=tk.N)
tk.Button(master=root, text='Quit', command=_quit).pack(anchor=tk.SW)

# a tk.DrawingArea
canvas = FigureCanvasTkAgg(f, master=root)
a.annotate("load image file", (0.5, 0.5), horizontalalignment="center")
canvas.show()
canvas.get_tk_widget().pack(side=tk.TOP, fill=tk.BOTH, expand=1)

toolbar = NavigationToolbar2TkAgg(canvas, root)
toolbar.update()
canvas._tkcanvas.pack(side=tk.TOP, fill=tk.BOTH, expand=1)

tk.mainloop()
<|endoftext|>


<commit_before># Copyright 2014-2015 Boxkite Inc.

# This file is part of the DataCats package and is released under
# the terms of the GNU Affero General Public License version 3.0.
# See LICENSE.txt or http://www.fsf.org/licensing/licenses/agpl-3.0.html

import re

NAME_RE = r'[a-z][a-z0-9]*$'
DATACATS_NAME_RE = r'[a-z][a-z0-9]{4,}$'

def valid_name(n):
    """
    Return True for project names that may be used locally
    """
    return bool(re.match(NAME_RE, n))

def valid_datacats_name(n):
    """
    Return True for project names that may be deployed to datacats site
    """
    return bool(re.match(DATACATS_NAME_RE, n))
<commit_msg>docstring fix<commit_after># Copyright 2014-2015 Boxkite Inc.

# This file is part of the DataCats package and is released under
# the terms of the GNU Affero General Public License version 3.0.
# See LICENSE.txt or http://www.fsf.org/licensing/licenses/agpl-3.0.html

import re

NAME_RE = r'[a-z][a-z0-9]*$'
DATACATS_NAME_RE = r'[a-z][a-z0-9]{4,}$'

def valid_name(n):
    """
    Return True for project names that may be used locally
    """
    return bool(re.match(NAME_RE, n))

def valid_datacats_name(n):
    """
    Return True for project names that may be deployed to DataCats.com
    """
    return bool(re.match(DATACATS_NAME_RE, n))
<|endoftext|>


<commit_before>#!/usr/bin/env python

'''
@file prototype/hdf/test/test_hdf_array_iterator.py
@author Swarbhanu Chatterjee
@test prototype.hdf.test.test_hdf_array_iterator.py test suite for hdf_array_iterator.py
'''

import os
from prototype.hdf.hdf_array_iterator import acquire_data

from nose.plugins.attrib import attr
from pyon.util.int_test import IonIntegrationTestCase
from pyon.core.exception import NotFound, BadRequest
import h5py, numpy

@attr('INT', group='dm')
class HDFArrayIteratorTest_1d(IonIntegrationTestCase):

    def setUp(self):

        #--------------------------------------------------------------------
        # Create an hdf file for testing
        #--------------------------------------------------------------------

        self.salinity = [0,]*3
        self.temperature = [0,]*3
        self.pressure = [0,]*3

        self.salinity[0] = numpy.arange(50)
        self.salinity[1] = numpy.arange(50) + 50
        self.salinity[2] = numpy.arange(50) + 100

        self.temperature[0] = numpy.random.normal(size=50)
        self.temperature[1] = numpy.random.normal(size=50)
        self.temperature[2] = numpy.random.normal(size=50)

        self.pressure[0] = numpy.random.uniform(low=0.0, high=1.0, size=50)
        self.pressure[1] = numpy.random.uniform(low=0.0, high=1.0, size=50)
        self.pressure[2] = numpy.random.uniform(low=0.0, high=1.0, size=50)

        self.fnames = ['data1.hdf5', 'data2.hdf5', 'data3.hdf5']

        for fname, s, t, p in zip(self.fnames, self.salinity, self.temperature, self.pressure):
            file = h5py.File(fname, 'w')

            grp1 = file.create_group('fields')
            dset1 = grp1.create_dataset("salinity", data=s)
            dset2 = grp1.create_dataset("temperature", data=t)
            dset3 = grp1.create_dataset("pressure", data=p)

            file.close()

    def tearDown(self):
        """
        Cleanup. Delete Subscription, Stream, Process Definition
        """

        for fname in self.fnames:
            os.remove(fname)

    def test_concatenate_size(self):

        #--------------------------------------------------------------------------------------
        # Test with a length greater than the virtual dataset
        #--------------------------------------------------------------------------------------

        generator = acquire_data(hdf_files = self.fnames,
            var_names = ['temperature', 'salinity'],
            concatenate_size = 26,
            bounds = (slice(63,120))
        )

        # assert the result...
        out = generator.next()

        temperature = numpy.concatenate((self.temperature[0],self.temperature[1],self.temperature[2]), axis = 0)
        salinity = numpy.concatenate((self.salinity[0],self.salinity[1],self.salinity[2]), axis = 0)

        self.assertEquals(str(out['temperature']['values']), str(temperature[63:89]))
        self.assertEquals(str(out['salinity']['values']), str(salinity[63:89]) )

        #---------------------------------------------------------------------------------------------------
        # Test with a length less than the virtual dataset such that mod(length, concatenate_size) == 0
        #---------------------------------------------------------------------------------------------------

        generator = acquire_data(hdf_files = self.fnames,
            var_names = ['temperature', 'salinity'],
            concatenate_size = 175,
            bounds = (slice(63,120))
        )

        # assert the result...
        out = generator.next()

        temperature = numpy.concatenate((self.temperature[0],self.temperature[1],self.temperature[2]), axis = 0)
        salinity = numpy.concatenate((self.salinity[0],self.salinity[1],self.salinity[2]), axis = 0)

        self.assertEquals(str(out['temperature']['values']), str(temperature[63:120]))
        self.assertEquals(str(out['salinity']['values']), str(salinity[63:120]) )

        #---------------------------------------------------------------------------------------------------
        # Test with a length less than the virtual dataset such that mod(length, concatenate_size) != 0
        #---------------------------------------------------------------------------------------------------

        generator = acquire_data(hdf_files = self.fnames,
            var_names = ['temperature', 'salinity'],
            concatenate_size = 26,
            bounds = (slice(63,120))
        )

        # assert the result...
        out = generator.next()

        temperature = numpy.concatenate((self.temperature[0],self.temperature[1],self.temperature[2]), axis = 0)
        salinity = numpy.concatenate((self.salinity[0],self.salinity[1],self.salinity[2]), axis = 0)

        self.assertEquals(str(out['temperature']['values']), str(temperature[63:89]))
        self.assertEquals(str(out['salinity']['values']), str(salinity[63:89]) )

    def test_var_names(self):

        # Test with no names
        # assert an error?

        generator = acquire_data(hdf_files = self.fnames,
            var_names =  None,
            concatenate_size = 26,
            bounds = (slice(63,120))
        )

        with self.assertRaises(NotFound):
            out = generator.next()

        #---------------------------------------------------------------------------------------------------
        # Test with all names
        #---------------------------------------------------------------------------------------------------

        generator = acquire_data(hdf_files = self.fnames,
            var_names =  ['temperature', 'salinity', 'pressure'],
            concatenate_size = 26,
            bounds = (slice(63,120))
        )
        out = generator.next()

        # assert result

        temperature = numpy.concatenate((self.temperature[0],self.temperature[1],self.temperature[2]), axis = 0)
        salinity = numpy.concatenate((self.salinity[0],self.salinity[1],self.salinity[2]), axis = 0)
        pressure = numpy.concatenate((self.pressure[0],self.pressure[1],self.pressure[2]), axis = 0)

        self.assertEquals(str(out['temperature']['values']), str(temperature[63:89]))
        self.assertEquals(str(out['salinity']['values']), str(salinity[63:89]) )
        self.assertEquals(str(out['pressure']['values']), str(pressure[63:89]))

        #---------------------------------------------------------------------------------------------------
        # Test with some names
        #---------------------------------------------------------------------------------------------------

        generator = acquire_data(hdf_files = self.fnames,
            var_names =  ['temperature', 'salinity'],
            concatenate_size = 26,
            bounds = (slice(63,120))
        )
        out = generator.next()

        # assert result

        temperature = numpy.concatenate((self.temperature[0],self.temperature[1],self.temperature[2]), axis = 0)
        salinity = numpy.concatenate((self.salinity[0],self.salinity[1],self.salinity[2]), axis = 0)

        self.assertEquals(str(out['temperature']['values']), str(temperature[63:89]))
        self.assertEquals(str(out['salinity']['values']), str(salinity[63:89]) )

        #---------------------------------------------------------------------------------------------------
        # Test with name not in dataset
        #---------------------------------------------------------------------------------------------------

        generator = acquire_data(hdf_files = self.fnames,
            var_names =  ['biological_quotient'],
            concatenate_size = 26,
            bounds = (slice(63,120))
        )

        # assert an error

        with self.assertRaises(NotFound):
            out = generator.next()

    def test_bounds(self):

        # Test with bad input not a slice and not a tuple...

        generator = acquire_data(hdf_files = self.fnames,
            var_names =  ['temperature', 'salinity'],
            concatenate_size = 26,
            bounds = 's'
        )

        # Assert an error.

        with self.assertRaises(BadRequest):
            out = generator.next()

        #@todo can we make the error more transparent to the use - easier to correct their mistake?

        # Test with 2 tuple of slices on the 1d dataset

        generator = acquire_data(hdf_files = self.fnames,
            var_names =  ['temperature', 'salinity'],
            concatenate_size = 26,
            bounds = (slice(63,120), slice(63,120))
        )

        # Assert an error.

        with self.assertRaises(BadRequest):
            out = generator.next()

        # Test with bounds greater than the dataset length

        generator = acquire_data(hdf_files = self.fnames,
            var_names =  ['temperature', 'salinity'],
            concatenate_size = 200,
            bounds = (slice(0,200))
        )

        out = generator.next()

        # Assert result is the whole dataset

        temperature = numpy.concatenate((self.temperature[0],self.temperature[1],self.temperature[2]), axis = 0)
        salinity = numpy.concatenate((self.salinity[0],self.salinity[1],self.salinity[2]), axis = 0)

        self.assertEquals(str(out['temperature']['values']), str(temperature))
        self.assertEquals(str(out['salinity']['values']), str(salinity) )

        # Test with normal bounds slice
        # assert result

        # Test with no bounds
        # assert result

        # Test with concatenate larger than bounds slice
        # Assert result

        # Test with concatenate smaller than bounds slice
        # assert result

        pass

    def test_files_close(self):

        # Test that files are closed when method compelete normally
        ## How?

        # Test that files are close when acquire data fails too!
        ## how?
        pass

@attr('INT', group='dm')
class HDFArrayIteratorTest_2d(IonIntegrationTestCase):

    def setUp(self):

        import h5py, numpy

        #--------------------------------------------------------------------
        # Create an hdf file for testing
        #--------------------------------------------------------------------

        self.salinity = [0,]*3
        self.temperature = [0,]*3
        self.pressure = [0,]*3

        self.salinity[0] = numpy.arange(50).reshape(5,10)
        self.salinity[1] = numpy.arange(50).reshape(5,10) + 50
        self.salinity[2] = numpy.arange(50).reshape(5,10) + 100

        self.temperature[0] = numpy.random.normal(size=50).reshape(5,10)
        self.temperature[1] = numpy.random.normal(size=50).reshape(5,10)
        self.temperature[2] = numpy.random.normal(size=50).reshape(5,10)

        self.pressure[0] = numpy.random.uniform(low=0.0, high=1.0, size=50).reshape(5,10)
        self.pressure[1] = numpy.random.uniform(low=0.0, high=1.0, size=50).reshape(5,10)
        self.pressure[2] = numpy.random.uniform(low=0.0, high=1.0, size=50).reshape(5,10)

        self.fnames = ['data1.hdf5', 'data2.hdf5', 'data3.hdf5']

        for fname, s, t, p in zip(self.fnames, self.salinity, self.temperature, self.pressure):
            file = h5py.File(fname, 'w')

            grp1 = file.create_group('fields')
            dset1 = grp1.create_dataset("salinity", data=s)
            dset2 = grp1.create_dataset("temperature", data=t)
            dset3 = grp1.create_dataset("pressure", data=p)

            file.close()

    def tearDown(self):
        """
        Cleanup. Delete Subscription, Stream, Process Definition
        """

        for fname in self.fnames:
            os.remove(fname)

    def simple_test(self):

        generator = acquire_data(hdf_files = self.fnames,
            var_names = ['temperature', 'salinity'],
            concatenate_size = 25,
            bounds = None #(slice(63,150))
        )

        for d in generator:
            print 'Hello!', d

<commit_msg>Added all remaining test methods for testing hdf_array_iterator in the 1d case. The 1d case tests all working<commit_after>#!/usr/bin/env python

'''
@file prototype/hdf/test/test_hdf_array_iterator.py
@author Swarbhanu Chatterjee
@test prototype.hdf.test.test_hdf_array_iterator.py test suite for hdf_array_iterator.py
'''

import os
from prototype.hdf.hdf_array_iterator import acquire_data

from nose.plugins.attrib import attr
from pyon.util.int_test import IonIntegrationTestCase
from pyon.core.exception import NotFound, BadRequest
import h5py, numpy

@attr('INT', group='dm')
class HDFArrayIteratorTest_1d(IonIntegrationTestCase):

    def setUp(self):

        #--------------------------------------------------------------------
        # Create an hdf file for testing
        #--------------------------------------------------------------------

        self.salinity = [0,]*3
        self.temperature = [0,]*3
        self.pressure = [0,]*3

        self.salinity[0] = numpy.arange(50)
        self.salinity[1] = numpy.arange(50) + 50
        self.salinity[2] = numpy.arange(50) + 100

        self.temperature[0] = numpy.random.normal(size=50)
        self.temperature[1] = numpy.random.normal(size=50)
        self.temperature[2] = numpy.random.normal(size=50)

        self.pressure[0] = numpy.random.uniform(low=0.0, high=1.0, size=50)
        self.pressure[1] = numpy.random.uniform(low=0.0, high=1.0, size=50)
        self.pressure[2] = numpy.random.uniform(low=0.0, high=1.0, size=50)

        self.fnames = ['data1.hdf5', 'data2.hdf5', 'data3.hdf5']

        for fname, s, t, p in zip(self.fnames, self.salinity, self.temperature, self.pressure):
            file = h5py.File(fname, 'w')

            grp1 = file.create_group('fields')
            dset1 = grp1.create_dataset("salinity", data=s)
            dset2 = grp1.create_dataset("temperature", data=t)
            dset3 = grp1.create_dataset("pressure", data=p)

            file.close()

    def tearDown(self):
        """
        Cleanup. Delete Subscription, Stream, Process Definition
        """

        for fname in self.fnames:
            os.remove(fname)

    def test_concatenate_size(self):

        #--------------------------------------------------------------------------------------
        # Test with a length greater than the virtual dataset
        #--------------------------------------------------------------------------------------

        generator = acquire_data(hdf_files = self.fnames,
            var_names = ['temperature', 'salinity'],
            concatenate_size = 26,
            bounds = (slice(63,120))
        )

        # assert the result...
        out = generator.next()

        temperature = numpy.concatenate((self.temperature[0],self.temperature[1],self.temperature[2]), axis = 0)
        salinity = numpy.concatenate((self.salinity[0],self.salinity[1],self.salinity[2]), axis = 0)

        self.assertEquals(str(out['temperature']['values']), str(temperature[63:89]))
        self.assertEquals(str(out['salinity']['values']), str(salinity[63:89]) )

        #---------------------------------------------------------------------------------------------------
        # Test with a length less than the virtual dataset such that mod(length, concatenate_size) == 0
        #---------------------------------------------------------------------------------------------------

        generator = acquire_data(hdf_files = self.fnames,
            var_names = ['temperature', 'salinity'],
            concatenate_size = 175,
            bounds = (slice(63,120))
        )

        # assert the result...
        out = generator.next()

        temperature = numpy.concatenate((self.temperature[0],self.temperature[1],self.temperature[2]), axis = 0)
        salinity = numpy.concatenate((self.salinity[0],self.salinity[1],self.salinity[2]), axis = 0)

        self.assertEquals(str(out['temperature']['values']), str(temperature[63:120]))
        self.assertEquals(str(out['salinity']['values']), str(salinity[63:120]) )

        #---------------------------------------------------------------------------------------------------
        # Test with a length less than the virtual dataset such that mod(length, concatenate_size) != 0
        #---------------------------------------------------------------------------------------------------

        generator = acquire_data(hdf_files = self.fnames,
            var_names = ['temperature', 'salinity'],
            concatenate_size = 26,
            bounds = (slice(63,120))
        )

        # assert the result...
        out = generator.next()

        temperature = numpy.concatenate((self.temperature[0],self.temperature[1],self.temperature[2]), axis = 0)
        salinity = numpy.concatenate((self.salinity[0],self.salinity[1],self.salinity[2]), axis = 0)

        self.assertEquals(str(out['temperature']['values']), str(temperature[63:89]))
        self.assertEquals(str(out['salinity']['values']), str(salinity[63:89]) )

    def test_var_names(self):

        # Test with no names
        # assert an error?

        generator = acquire_data(hdf_files = self.fnames,
            var_names =  None,
            concatenate_size = 26,
            bounds = (slice(63,120))
        )

        with self.assertRaises(NotFound):
            out = generator.next()

        #---------------------------------------------------------------------------------------------------
        # Test with all names
        #---------------------------------------------------------------------------------------------------

        generator = acquire_data(hdf_files = self.fnames,
            var_names =  ['temperature', 'salinity', 'pressure'],
            concatenate_size = 26,
            bounds = (slice(63,120))
        )
        out = generator.next()

        # assert result

        temperature = numpy.concatenate((self.temperature[0],self.temperature[1],self.temperature[2]), axis = 0)
        salinity = numpy.concatenate((self.salinity[0],self.salinity[1],self.salinity[2]), axis = 0)
        pressure = numpy.concatenate((self.pressure[0],self.pressure[1],self.pressure[2]), axis = 0)

        self.assertEquals(str(out['temperature']['values']), str(temperature[63:89]))
        self.assertEquals(str(out['salinity']['values']), str(salinity[63:89]) )
        self.assertEquals(str(out['pressure']['values']), str(pressure[63:89]))

        #---------------------------------------------------------------------------------------------------
        # Test with some names
        #---------------------------------------------------------------------------------------------------

        generator = acquire_data(hdf_files = self.fnames,
            var_names =  ['temperature', 'salinity'],
            concatenate_size = 26,
            bounds = (slice(63,120))
        )
        out = generator.next()

        # assert result

        temperature = numpy.concatenate((self.temperature[0],self.temperature[1],self.temperature[2]), axis = 0)
        salinity = numpy.concatenate((self.salinity[0],self.salinity[1],self.salinity[2]), axis = 0)

        self.assertEquals(str(out['temperature']['values']), str(temperature[63:89]))
        self.assertEquals(str(out['salinity']['values']), str(salinity[63:89]) )

        #---------------------------------------------------------------------------------------------------
        # Test with name not in dataset
        #---------------------------------------------------------------------------------------------------

        generator = acquire_data(hdf_files = self.fnames,
            var_names =  ['biological_quotient'],
            concatenate_size = 26,
            bounds = (slice(63,120))
        )

        # assert an error

        with self.assertRaises(NotFound):
            out = generator.next()

    def test_bounds(self):

        # Test with bad input not a slice and not a tuple...

        generator = acquire_data(hdf_files = self.fnames,
            var_names =  ['temperature', 'salinity'],
            concatenate_size = 26,
            bounds = 's'
        )

        # Assert an error.

        with self.assertRaises(BadRequest):
            out = generator.next()

        #@todo can we make the error more transparent to the use - easier to correct their mistake?

        # Test with 2 tuple of slices on the 1d dataset

        generator = acquire_data(hdf_files = self.fnames,
            var_names =  ['temperature', 'salinity'],
            concatenate_size = 26,
            bounds = (slice(63,120), slice(63,120))
        )

        # Assert an error.

        with self.assertRaises(BadRequest):
            out = generator.next()

        # Test with bounds greater than the dataset length

        generator = acquire_data(hdf_files = self.fnames,
            var_names =  ['temperature', 'salinity'],
            concatenate_size = 200,
            bounds = (slice(0,200))
        )

        out = generator.next()

        # Assert result is the whole dataset

        temperature = numpy.concatenate((self.temperature[0],self.temperature[1],self.temperature[2]), axis = 0)
        salinity = numpy.concatenate((self.salinity[0],self.salinity[1],self.salinity[2]), axis = 0)

        self.assertEquals(str(out['temperature']['values']), str(temperature))
        self.assertEquals(str(out['salinity']['values']), str(salinity) )

        # Test with normal bounds slice

        generator = acquire_data(hdf_files = self.fnames,
            var_names =  ['temperature', 'salinity', 'pressure'],
            concatenate_size = 60,
            bounds = (slice(30,80))
        )
        out = generator.next()

        # assert result

        temperature = numpy.concatenate((self.temperature[0],self.temperature[1],self.temperature[2]), axis = 0)
        salinity = numpy.concatenate((self.salinity[0],self.salinity[1],self.salinity[2]), axis = 0)
        pressure = numpy.concatenate((self.pressure[0],self.pressure[1],self.pressure[2]), axis = 0)

        self.assertEquals(str(out['temperature']['values']), str(temperature[30:80]))
        self.assertEquals(str(out['salinity']['values']), str(salinity[30:80]) )
        self.assertEquals(str(out['pressure']['values']), str(pressure[30:80]))

        # Test with no bounds
        generator = acquire_data(hdf_files = self.fnames,
            var_names =  ['temperature', 'salinity', 'pressure'],
            concatenate_size = 60,
            bounds = None
        )
        out = generator.next()

        # assert result

        temperature = numpy.concatenate((self.temperature[0],self.temperature[1],self.temperature[2]), axis = 0)
        salinity = numpy.concatenate((self.salinity[0],self.salinity[1],self.salinity[2]), axis = 0)
        pressure = numpy.concatenate((self.pressure[0],self.pressure[1],self.pressure[2]), axis = 0)

        self.assertEquals(str(out['temperature']['values']), str(temperature[0:60]))
        self.assertEquals(str(out['salinity']['values']), str(salinity[0:60]) )
        self.assertEquals(str(out['pressure']['values']), str(pressure[0:60]))

        # Test with concatenate larger than bounds slice

        generator = acquire_data(hdf_files = self.fnames,
            var_names =  ['temperature', 'salinity', 'pressure'],
            concatenate_size = 60,
            bounds = (slice(30,50))
        )
        out = generator.next()

        # assert result

        temperature = numpy.concatenate((self.temperature[0],self.temperature[1],self.temperature[2]), axis = 0)
        salinity = numpy.concatenate((self.salinity[0],self.salinity[1],self.salinity[2]), axis = 0)
        pressure = numpy.concatenate((self.pressure[0],self.pressure[1],self.pressure[2]), axis = 0)

        self.assertEquals(str(out['temperature']['values']), str(temperature[30:50]))
        self.assertEquals(str(out['salinity']['values']), str(salinity[30:50]) )
        self.assertEquals(str(out['pressure']['values']), str(pressure[30:50]))

        # Test with concatenate smaller than bounds slice

        generator = acquire_data(hdf_files = self.fnames,
            var_names =  ['temperature', 'salinity', 'pressure'],
            concatenate_size = 10,
            bounds = (slice(30,100))
        )
        out = generator.next()

        # assert result

        temperature = numpy.concatenate((self.temperature[0],self.temperature[1],self.temperature[2]), axis = 0)
        salinity = numpy.concatenate((self.salinity[0],self.salinity[1],self.salinity[2]), axis = 0)
        pressure = numpy.concatenate((self.pressure[0],self.pressure[1],self.pressure[2]), axis = 0)

        self.assertEquals(str(out['temperature']['values']), str(temperature[30:40]))
        self.assertEquals(str(out['salinity']['values']), str(salinity[30:40]) )
        self.assertEquals(str(out['pressure']['values']), str(pressure[30:40]))

    def test_files_close(self):

        # Test that files are closed when method compelete normally
        ## How?

        # Test that files are close when acquire data fails too!
        ## how?
        pass

@attr('INT', group='dm')
class HDFArrayIteratorTest_2d(IonIntegrationTestCase):

    def setUp(self):

        import h5py, numpy

        #--------------------------------------------------------------------
        # Create an hdf file for testing
        #--------------------------------------------------------------------

        self.salinity = [0,]*3
        self.temperature = [0,]*3
        self.pressure = [0,]*3

        self.salinity[0] = numpy.arange(50).reshape(5,10)
        self.salinity[1] = numpy.arange(50).reshape(5,10) + 50
        self.salinity[2] = numpy.arange(50).reshape(5,10) + 100

        self.temperature[0] = numpy.random.normal(size=50).reshape(5,10)
        self.temperature[1] = numpy.random.normal(size=50).reshape(5,10)
        self.temperature[2] = numpy.random.normal(size=50).reshape(5,10)

        self.pressure[0] = numpy.random.uniform(low=0.0, high=1.0, size=50).reshape(5,10)
        self.pressure[1] = numpy.random.uniform(low=0.0, high=1.0, size=50).reshape(5,10)
        self.pressure[2] = numpy.random.uniform(low=0.0, high=1.0, size=50).reshape(5,10)

        self.fnames = ['data1.hdf5', 'data2.hdf5', 'data3.hdf5']

        for fname, s, t, p in zip(self.fnames, self.salinity, self.temperature, self.pressure):
            file = h5py.File(fname, 'w')

            grp1 = file.create_group('fields')
            dset1 = grp1.create_dataset("salinity", data=s)
            dset2 = grp1.create_dataset("temperature", data=t)
            dset3 = grp1.create_dataset("pressure", data=p)

            file.close()

    def tearDown(self):
        """
        Cleanup. Delete Subscription, Stream, Process Definition
        """

        for fname in self.fnames:
            os.remove(fname)

    def simple_test(self):

        generator = acquire_data(hdf_files = self.fnames,
            var_names = ['temperature', 'salinity'],
            concatenate_size = 25,
            bounds = None #(slice(63,150))
        )

        for d in generator:
            print 'Hello!', d

<|endoftext|>


<commit_before>import re
import sys
import shlex
import random
import datetime

from pandajedi.jedicore import Interaction
from TaskRefinerBase import TaskRefinerBase

from pandaserver.dataservice import DataServiceUtils

# brokerage for ATLAS production
class AtlasProdTaskRefiner (TaskRefinerBase):

    # constructor
    def __init__(self, taskBufferIF, ddmIF):
        TaskRefinerBase.__init__(self, taskBufferIF, ddmIF)

    # extract common parameters
    def extractCommon(self,jediTaskID,taskParamMap,workQueueMapper,splitRule):
        tmpLog = self.tmpLog
        # set ddmBackEnd
        if not 'ddmBackEnd' in taskParamMap:
            taskParamMap['ddmBackEnd'] = 'rucio'
        # get number of unprocessed events for event service
        autoEsConversion = False
        if 'esConvertible' in taskParamMap and taskParamMap['esConvertible'] == True:
            maxPrio = self.taskBufferIF.getConfigValue('taskrefiner', 'AES_MAXTASKPRIORITY', 'jedi', 'atlas')
            minPrio = self.taskBufferIF.getConfigValue('taskrefiner', 'AES_MINTASKPRIORITY', 'jedi', 'atlas')
            if maxPrio is not None and maxPrio < taskParamMap['taskPriority']:
                pass
            elif minPrio is not None and minPrio > taskParamMap['taskPriority']:
                pass
            else:
                nEvents, lastTaskTime = self.taskBufferIF.getNumUnprocessedEvents_JEDI('atlas', 'managed',
                                                                                {'processingType': 'simul',
                                                                                'eventService': 1})
                tmpLog.debug('check for ES nUnprocessedEvents={0} lastTaskTime={1}'.format(nEvents, lastTaskTime))
                # not chane many tasks at once
                if lastTaskTime is not None and lastTaskTime < datetime.datetime.utcnow() - datetime.timedelta(minutes=5):
                    # get threshold
                    minNumEvents = self.taskBufferIF.getConfigValue('taskrefiner', 'AES_EVENTPOOLSIZE', 'jedi', 'atlas')
                    if minNumEvents is not None and nEvents < minNumEvents:
                        autoEsConversion = True
                        tmpLog.debug('converted to ES for eventPool={0}'.format(minNumEvents))
        # add ES paramsters
        if ('esFraction' in taskParamMap and taskParamMap['esFraction'] > 0) or autoEsConversion:
            tmpStr  = '<PANDA_ES_ONLY>--eventService=True</PANDA_ES_ONLY>'
            taskParamMap['jobParameters'].append({'type':'constant',
                                                  'value':tmpStr})
            if 'nEventsPerWorker' not in taskParamMap and (taskParamMap['esFraction'] > random.random() or autoEsConversion):
                taskParamMap['nEventsPerWorker'] = 1
                if 'nEsConsumers' not in taskParamMap:
                    tmpVal = self.taskBufferIF.getConfigValue('taskrefiner', 'AES_NESCONSUMERS', 'jedi', 'atlas')
                    if tmpVal is None:
                        tmpVal = 1
                    taskParamMap['nEsConsumers'] = tmpVal
                if 'nSitesPerJob' not in taskParamMap:
                    tmpVal = self.taskBufferIF.getConfigValue('taskrefiner', 'AES_NSITESPERJOB', 'jedi', 'atlas')
                    if tmpVal is not None:
                        taskParamMap['nSitesPerJob'] = tmpVal
                if 'mergeEsOnOS' not in taskParamMap:
                    taskParamMap['mergeEsOnOS'] = True
                if 'maxAttemptES' not in taskParamMap:
                    taskParamMap['maxAttemptES'] = 10
                if 'skipScout' not in taskParamMap:
                    taskParamMap['skipScout'] = True
        TaskRefinerBase.extractCommon(self,jediTaskID,taskParamMap,workQueueMapper,splitRule)

    # main
    def doRefine(self,jediTaskID,taskParamMap):
        # make logger
        tmpLog = self.tmpLog
        tmpLog.debug('start taskType={0}'.format(self.taskSpec.taskType))
        try:
            # basic refine
            self.doBasicRefine(taskParamMap)
            # set nosplit+repeat for DBR
            for datasetSpec in self.inSecDatasetSpecList:
                if DataServiceUtils.isDBR(datasetSpec.datasetName):
                    datasetSpec.attributes = 'repeat,nosplit'
            # enable consistency check
            if not self.taskSpec.parent_tid in [None,self.taskSpec.jediTaskID]:
                for datasetSpec in self.inMasterDatasetSpec:
                    if datasetSpec.isMaster() and datasetSpec.type == 'input':
                        datasetSpec.enableCheckConsistency()
            # append attempt number
            for tmpKey,tmpOutTemplateMapList in self.outputTemplateMap.iteritems():
                for tmpOutTemplateMap in tmpOutTemplateMapList:
                    outFileTemplate = tmpOutTemplateMap['filenameTemplate']
                    if re.search('\.\d+$',outFileTemplate) == None and not outFileTemplate.endswith('.panda.um'):
                        tmpOutTemplateMap['filenameTemplate'] = outFileTemplate + '.1'
            # extract input datatype
            datasetTypeListIn = []
            for datasetSpec in self.inMasterDatasetSpec+self.inSecDatasetSpecList:
                datasetType = DataServiceUtils.getDatasetType(datasetSpec.datasetName)
                if not datasetType in ['',None]:
                    datasetTypeListIn.append(datasetType)
            # extract datatype and set destination if nessesary
            datasetTypeList = []
            for datasetSpec in self.outDatasetSpecList:
                datasetType = DataServiceUtils.getDatasetType(datasetSpec.datasetName)
                if not datasetType in ['',None]:
                    datasetTypeList.append(datasetType)
            # set numThrottled to use the task throttling mechanism
            if not 'noThrottle' in taskParamMap:
                self.taskSpec.numThrottled = 0
            # set to register datasets
            self.taskSpec.setToRegisterDatasets()
            # set transient to parent datasets
            if self.taskSpec.processingType in ['merge'] and not self.taskSpec.parent_tid in [None,self.taskSpec.jediTaskID]:
                # get parent
                tmpStat,parentTaskSpec = self.taskBufferIF.getTaskDatasetsWithID_JEDI(self.taskSpec.parent_tid,None,False)
                if tmpStat and parentTaskSpec != None:
                    # set transient to parent datasets
                    metaData = {'transient':True}
                    for datasetSpec in parentTaskSpec.datasetSpecList:
                        if datasetSpec.type in ['log','output']:
                            datasetType = DataServiceUtils.getDatasetType(datasetSpec.datasetName)
                            if not datasetType in ['',None] and datasetType in datasetTypeList and datasetType in datasetTypeListIn:
                                tmpLog.info('set metadata={0} to parent jediTaskID={1}:datasetID={2}:Name={3}'.format(str(metaData),
                                                                                self.taskSpec.parent_tid,
                                                                                datasetSpec.datasetID,
                                                                                datasetSpec.datasetName))
                                for metadataName,metadaValue in metaData.iteritems():
                                    self.ddmIF.getInterface(self.taskSpec.vo).setDatasetMetadata(datasetSpec.datasetName,
                                                                                metadataName,metadaValue)
        except:
            errtype,errvalue = sys.exc_info()[:2]
            tmpLog.error('doBasicRefine failed with {0}:{1}'.format(errtype.__name__,errvalue))
            raise errtype,errvalue
        tmpLog.debug('done')
        return self.SC_SUCCEEDED

    # insert string
    def insertString(self,paramName,tmpStr,origStr):
        items = shlex.split(origStr,posix=False)
        newStr = ''
        for item in items:
            if not paramName in item:
                newStr += item
            else:
                newStr += item[:-1]
                newStr += tmpStr
                newStr += item[-1]
            newStr += ' '
        return newStr

<commit_msg>setting coreCount=0 to converted AES tasks<commit_after>import re
import sys
import shlex
import random
import datetime

from pandajedi.jedicore import Interaction
from TaskRefinerBase import TaskRefinerBase

from pandaserver.dataservice import DataServiceUtils

# brokerage for ATLAS production
class AtlasProdTaskRefiner (TaskRefinerBase):

    # constructor
    def __init__(self, taskBufferIF, ddmIF):
        TaskRefinerBase.__init__(self, taskBufferIF, ddmIF)

    # extract common parameters
    def extractCommon(self,jediTaskID,taskParamMap,workQueueMapper,splitRule):
        tmpLog = self.tmpLog
        # set ddmBackEnd
        if not 'ddmBackEnd' in taskParamMap:
            taskParamMap['ddmBackEnd'] = 'rucio'
        # get number of unprocessed events for event service
        autoEsConversion = False
        if 'esConvertible' in taskParamMap and taskParamMap['esConvertible'] == True:
            maxPrio = self.taskBufferIF.getConfigValue('taskrefiner', 'AES_MAXTASKPRIORITY', 'jedi', 'atlas')
            minPrio = self.taskBufferIF.getConfigValue('taskrefiner', 'AES_MINTASKPRIORITY', 'jedi', 'atlas')
            if maxPrio is not None and maxPrio < taskParamMap['taskPriority']:
                pass
            elif minPrio is not None and minPrio > taskParamMap['taskPriority']:
                pass
            else:
                nEvents, lastTaskTime = self.taskBufferIF.getNumUnprocessedEvents_JEDI('atlas', 'managed',
                                                                                {'processingType': 'simul',
                                                                                'eventService': 1})
                tmpLog.debug('check for ES nUnprocessedEvents={0} lastTaskTime={1}'.format(nEvents, lastTaskTime))
                # not chane many tasks at once
                if lastTaskTime is not None and lastTaskTime < datetime.datetime.utcnow() - datetime.timedelta(minutes=5):
                    # get threshold
                    minNumEvents = self.taskBufferIF.getConfigValue('taskrefiner', 'AES_EVENTPOOLSIZE', 'jedi', 'atlas')
                    if minNumEvents is not None and nEvents < minNumEvents:
                        autoEsConversion = True
                        tmpLog.debug('converted to ES for eventPool={0}'.format(minNumEvents))
        # add ES paramsters
        if ('esFraction' in taskParamMap and taskParamMap['esFraction'] > 0) or autoEsConversion:
            tmpStr  = '<PANDA_ES_ONLY>--eventService=True</PANDA_ES_ONLY>'
            taskParamMap['jobParameters'].append({'type':'constant',
                                                  'value':tmpStr})
            if 'nEventsPerWorker' not in taskParamMap and (taskParamMap['esFraction'] > random.random() or autoEsConversion):
                taskParamMap['nEventsPerWorker'] = 1
                if 'nEsConsumers' not in taskParamMap:
                    tmpVal = self.taskBufferIF.getConfigValue('taskrefiner', 'AES_NESCONSUMERS', 'jedi', 'atlas')
                    if tmpVal is None:
                        tmpVal = 1
                    taskParamMap['nEsConsumers'] = tmpVal
                if 'nSitesPerJob' not in taskParamMap:
                    tmpVal = self.taskBufferIF.getConfigValue('taskrefiner', 'AES_NSITESPERJOB', 'jedi', 'atlas')
                    if tmpVal is not None:
                        taskParamMap['nSitesPerJob'] = tmpVal
                if 'mergeEsOnOS' not in taskParamMap:
                    taskParamMap['mergeEsOnOS'] = True
                if 'maxAttemptES' not in taskParamMap:
                    taskParamMap['maxAttemptES'] = 10
                taskParamMap['coreCount'] = 0
        TaskRefinerBase.extractCommon(self,jediTaskID,taskParamMap,workQueueMapper,splitRule)

    # main
    def doRefine(self,jediTaskID,taskParamMap):
        # make logger
        tmpLog = self.tmpLog
        tmpLog.debug('start taskType={0}'.format(self.taskSpec.taskType))
        try:
            # basic refine
            self.doBasicRefine(taskParamMap)
            # set nosplit+repeat for DBR
            for datasetSpec in self.inSecDatasetSpecList:
                if DataServiceUtils.isDBR(datasetSpec.datasetName):
                    datasetSpec.attributes = 'repeat,nosplit'
            # enable consistency check
            if not self.taskSpec.parent_tid in [None,self.taskSpec.jediTaskID]:
                for datasetSpec in self.inMasterDatasetSpec:
                    if datasetSpec.isMaster() and datasetSpec.type == 'input':
                        datasetSpec.enableCheckConsistency()
            # append attempt number
            for tmpKey,tmpOutTemplateMapList in self.outputTemplateMap.iteritems():
                for tmpOutTemplateMap in tmpOutTemplateMapList:
                    outFileTemplate = tmpOutTemplateMap['filenameTemplate']
                    if re.search('\.\d+$',outFileTemplate) == None and not outFileTemplate.endswith('.panda.um'):
                        tmpOutTemplateMap['filenameTemplate'] = outFileTemplate + '.1'
            # extract input datatype
            datasetTypeListIn = []
            for datasetSpec in self.inMasterDatasetSpec+self.inSecDatasetSpecList:
                datasetType = DataServiceUtils.getDatasetType(datasetSpec.datasetName)
                if not datasetType in ['',None]:
                    datasetTypeListIn.append(datasetType)
            # extract datatype and set destination if nessesary
            datasetTypeList = []
            for datasetSpec in self.outDatasetSpecList:
                datasetType = DataServiceUtils.getDatasetType(datasetSpec.datasetName)
                if not datasetType in ['',None]:
                    datasetTypeList.append(datasetType)
            # set numThrottled to use the task throttling mechanism
            if not 'noThrottle' in taskParamMap:
                self.taskSpec.numThrottled = 0
            # set to register datasets
            self.taskSpec.setToRegisterDatasets()
            # set transient to parent datasets
            if self.taskSpec.processingType in ['merge'] and not self.taskSpec.parent_tid in [None,self.taskSpec.jediTaskID]:
                # get parent
                tmpStat,parentTaskSpec = self.taskBufferIF.getTaskDatasetsWithID_JEDI(self.taskSpec.parent_tid,None,False)
                if tmpStat and parentTaskSpec != None:
                    # set transient to parent datasets
                    metaData = {'transient':True}
                    for datasetSpec in parentTaskSpec.datasetSpecList:
                        if datasetSpec.type in ['log','output']:
                            datasetType = DataServiceUtils.getDatasetType(datasetSpec.datasetName)
                            if not datasetType in ['',None] and datasetType in datasetTypeList and datasetType in datasetTypeListIn:
                                tmpLog.info('set metadata={0} to parent jediTaskID={1}:datasetID={2}:Name={3}'.format(str(metaData),
                                                                                self.taskSpec.parent_tid,
                                                                                datasetSpec.datasetID,
                                                                                datasetSpec.datasetName))
                                for metadataName,metadaValue in metaData.iteritems():
                                    self.ddmIF.getInterface(self.taskSpec.vo).setDatasetMetadata(datasetSpec.datasetName,
                                                                                metadataName,metadaValue)
        except:
            errtype,errvalue = sys.exc_info()[:2]
            tmpLog.error('doBasicRefine failed with {0}:{1}'.format(errtype.__name__,errvalue))
            raise errtype,errvalue
        tmpLog.debug('done')
        return self.SC_SUCCEEDED

    # insert string
    def insertString(self,paramName,tmpStr,origStr):
        items = shlex.split(origStr,posix=False)
        newStr = ''
        for item in items:
            if not paramName in item:
                newStr += item
            else:
                newStr += item[:-1]
                newStr += tmpStr
                newStr += item[-1]
            newStr += ' '
        return newStr

<|endoftext|>