<commit_before>#
#

# Copyright (C) 2006, 2007, 2008, 2013 Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""Fake hypervisor

"""

import os
import os.path
import logging

from ganeti import utils
from ganeti import constants
from ganeti import errors
from ganeti import objects
from ganeti import pathutils
from ganeti.hypervisor import hv_base

class FakeHypervisor(hv_base.BaseHypervisor):
  """Fake hypervisor interface.

  This can be used for testing the ganeti code without having to have
  a real virtualisation software installed.

  """
  PARAMETERS = {
    constants.HV_MIGRATION_MODE: hv_base.MIGRATION_MODE_CHECK,
    }

  CAN_MIGRATE = True

  _ROOT_DIR = pathutils.RUN_DIR + "/fake-hypervisor"

  def __init__(self):
    hv_base.BaseHypervisor.__init__(self)
    utils.EnsureDirs([(self._ROOT_DIR, constants.RUN_DIRS_MODE)])

  def ListInstances(self, hvparams=None):
    """Get the list of running instances.

    """
    return os.listdir(self._ROOT_DIR)

  def GetInstanceInfo(self, instance_name, hvparams=None):
    """Get instance properties.

    @type instance_name: string
    @param instance_name: the instance name
    @type hvparams: dict of strings
    @param hvparams: hvparams to be used with this instance

    @return: tuple of (name, id, memory, vcpus, stat, times)

    """
    file_name = self._InstanceFile(instance_name)
    if not os.path.exists(file_name):
      return None
    try:
      fh = open(file_name, "r")
      try:
        inst_id = fh.readline().strip()
        memory = utils.TryConvert(int, fh.readline().strip())
        vcpus = utils.TryConvert(int, fh.readline().strip())
        stat = hv_base.HvInstanceState.RUNNING
        times = 0
        return (instance_name, inst_id, memory, vcpus, stat, times)
      finally:
        fh.close()
    except IOError as err:
      raise errors.HypervisorError("Failed to list instance %s: %s" %
                                   (instance_name, err))

  def GetAllInstancesInfo(self, hvparams=None):
    """Get properties of all instances.

    @type hvparams: dict of strings
    @param hvparams: hypervisor parameter
    @return: list of tuples (name, id, memory, vcpus, stat, times)

    """
    data = []
    for file_name in os.listdir(self._ROOT_DIR):
      try:
        fh = open(utils.PathJoin(self._ROOT_DIR, file_name), "r")
        inst_id = "-1"
        memory = 0
        vcpus = 1
        stat = hv_base.HvInstanceState.SHUTDOWN
        times = -1
        try:
          inst_id = fh.readline().strip()
          memory = utils.TryConvert(int, fh.readline().strip())
          vcpus = utils.TryConvert(int, fh.readline().strip())
          stat = hv_base.HvInstanceState.RUNNING
          times = 0
        finally:
          fh.close()
        data.append((file_name, inst_id, memory, vcpus, stat, times))
      except IOError as err:
        raise errors.HypervisorError("Failed to list instances: %s" % err)
    return data

  @classmethod
  def _InstanceFile(cls, instance_name):
    """Compute the instance file for an instance name.

    """
    return utils.PathJoin(cls._ROOT_DIR, instance_name)

  def _IsAlive(self, instance_name):
    """Checks if an instance is alive.

    """
    file_name = self._InstanceFile(instance_name)
    return os.path.exists(file_name)

  def _MarkUp(self, instance, memory):
    """Mark the instance as running.

    This does no checks, which should be done by its callers.

    """
    file_name = self._InstanceFile(instance.name)
    fh = file(file_name, "w")
    try:
      fh.write("0\n%d\n%d\n" %
               (memory,
                instance.beparams[constants.BE_VCPUS]))
    finally:
      fh.close()

  def _MarkDown(self, instance_name):
    """Mark the instance as running.

    This does no checks, which should be done by its callers.

    """
    file_name = self._InstanceFile(instance_name)
    utils.RemoveFile(file_name)

  def StartInstance(self, instance, block_devices, startup_paused):
    """Start an instance.

    For the fake hypervisor, it just creates a file in the base dir,
    creating an exception if it already exists. We don't actually
    handle race conditions properly, since these are *FAKE* instances.

    """
    if self._IsAlive(instance.name):
      raise errors.HypervisorError("Failed to start instance %s: %s" %
                                   (instance.name, "already running"))
    try:
      self._MarkUp(instance, self._InstanceStartupMemory(instance))
    except IOError as err:
      raise errors.HypervisorError("Failed to start instance %s: %s" %
                                   (instance.name, err))

  def StopInstance(self, instance, force=False, retry=False, name=None,
                   timeout=None):
    """Stop an instance.

    For the fake hypervisor, this just removes the file in the base
    dir, if it exist, otherwise we raise an exception.

    """
    assert(timeout is None or force is not None)

    if name is None:
      name = instance.name
    if not self._IsAlive(name):
      raise errors.HypervisorError("Failed to stop instance %s: %s" %
                                   (name, "not running"))
    self._MarkDown(name)

  def RebootInstance(self, instance):
    """Reboot an instance.

    For the fake hypervisor, this does nothing.

    """
    return

  def BalloonInstanceMemory(self, instance, mem):
    """Balloon an instance memory to a certain value.

    @type instance: L{objects.Instance}
    @param instance: instance to be accepted
    @type mem: int
    @param mem: actual memory size to use for instance runtime

    """
    if not self._IsAlive(instance.name):
      raise errors.HypervisorError("Failed to balloon memory for %s: %s" %
                                   (instance.name, "not running"))
    try:
      self._MarkUp(instance, mem)
    except EnvironmentError as err:
      raise errors.HypervisorError("Failed to balloon memory for %s: %s" %
                                   (instance.name, utils.ErrnoOrStr(err)))

  def GetNodeInfo(self, hvparams=None):
    """Return information about the node.

    See L{BaseHypervisor.GetLinuxNodeInfo}.

    """
    result = self.GetLinuxNodeInfo()
    # substract running instances
    all_instances = self.GetAllInstancesInfo()
    result["memory_free"] -= min(result["memory_free"],
                                 sum([row[2] for row in all_instances]))
    return result

  @classmethod
  def GetInstanceConsole(cls, instance, primary_node, node_group,
                         hvparams, beparams):
    """Return information for connecting to the console of an instance.

    """
    return objects.InstanceConsole(instance=instance.name,
                                   kind=constants.CONS_MESSAGE,
                                   message=("Console not available for fake"
                                            " hypervisor"))

  def Verify(self, hvparams=None):
    """Verify the hypervisor.

    For the fake hypervisor, it just checks the existence of the base
    dir.

    @type hvparams: dict of strings
    @param hvparams: hypervisor parameters to be verified against; not used
      for fake hypervisors

    @return: Problem description if something is wrong, C{None} otherwise

    """
    if os.path.exists(self._ROOT_DIR):
      return None
    else:
      return "The required directory '%s' does not exist" % self._ROOT_DIR

  @classmethod
  def PowercycleNode(cls, hvparams=None):
    """Fake hypervisor powercycle, just a wrapper over Linux powercycle.

    @type hvparams: dict of strings
    @param hvparams: hypervisor params to be used on this node

    """
    cls.LinuxPowercycle()

  def AcceptInstance(self, instance, info, target):
    """Prepare to accept an instance.

    @type instance: L{objects.Instance}
    @param instance: instance to be accepted
    @type info: string
    @param info: instance info, not used
    @type target: string
    @param target: target host (usually ip), on this node

    """
    if self._IsAlive(instance.name):
      raise errors.HypervisorError("Can't accept instance, already running")

  def MigrateInstance(self, cluster_name, instance, target, live):
    """Migrate an instance.

    @type cluster_name: string
    @param cluster_name: name of the cluster
    @type instance: L{objects.Instance}
    @param instance: the instance to be migrated
    @type target: string
    @param target: hostname (usually ip) of the target node
    @type live: boolean
    @param live: whether to do a live or non-live migration

    """
    logging.debug("Fake hypervisor migrating %s to %s (live=%s)",
                  instance, target, live)

  def FinalizeMigrationDst(self, instance, info, success):
    """Finalize the instance migration on the target node.

    For the fake hv, this just marks the instance up.

    @type instance: L{objects.Instance}
    @param instance: instance whose migration is being finalized
    @type info: string/data (opaque)
    @param info: migration information, from the source node
    @type success: boolean
    @param success: whether the migration was a success or a failure

    """
    if success:
      self._MarkUp(instance, self._InstanceStartupMemory(instance))
    else:
      # ensure it's down
      self._MarkDown(instance.name)

  def FinalizeMigrationSource(self, instance, success, live):
    """Finalize the instance migration on the source node.

    @type instance: L{objects.Instance}
    @param instance: the instance that was migrated
    @type success: bool
    @param success: whether the migration succeeded or not
    @type live: bool
    @param live: whether the user requested a live migration or not

    """
    # pylint: disable=W0613
    if success:
      self._MarkDown(instance.name)

  def GetMigrationStatus(self, instance):
    """Get the migration status

    The fake hypervisor migration always succeeds.

    @type instance: L{objects.Instance}
    @param instance: the instance that is being migrated
    @rtype: L{objects.MigrationStatus}
    @return: the status of the current migration (one of
             L{constants.HV_MIGRATION_VALID_STATUSES}), plus any additional
             progress info that can be retrieved from the hypervisor

    """
    return objects.MigrationStatus(status=constants.HV_MIGRATION_COMPLETED)
<commit_msg>hv_fake: replace file() with open()<commit_after>#
#

# Copyright (C) 2006, 2007, 2008, 2013 Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""Fake hypervisor

"""

import os
import os.path
import logging

from ganeti import utils
from ganeti import constants
from ganeti import errors
from ganeti import objects
from ganeti import pathutils
from ganeti.hypervisor import hv_base

class FakeHypervisor(hv_base.BaseHypervisor):
  """Fake hypervisor interface.

  This can be used for testing the ganeti code without having to have
  a real virtualisation software installed.

  """
  PARAMETERS = {
    constants.HV_MIGRATION_MODE: hv_base.MIGRATION_MODE_CHECK,
    }

  CAN_MIGRATE = True

  _ROOT_DIR = pathutils.RUN_DIR + "/fake-hypervisor"

  def __init__(self):
    hv_base.BaseHypervisor.__init__(self)
    utils.EnsureDirs([(self._ROOT_DIR, constants.RUN_DIRS_MODE)])

  def ListInstances(self, hvparams=None):
    """Get the list of running instances.

    """
    return os.listdir(self._ROOT_DIR)

  def GetInstanceInfo(self, instance_name, hvparams=None):
    """Get instance properties.

    @type instance_name: string
    @param instance_name: the instance name
    @type hvparams: dict of strings
    @param hvparams: hvparams to be used with this instance

    @return: tuple of (name, id, memory, vcpus, stat, times)

    """
    file_name = self._InstanceFile(instance_name)
    if not os.path.exists(file_name):
      return None
    try:
      fh = open(file_name, "r")
      try:
        inst_id = fh.readline().strip()
        memory = utils.TryConvert(int, fh.readline().strip())
        vcpus = utils.TryConvert(int, fh.readline().strip())
        stat = hv_base.HvInstanceState.RUNNING
        times = 0
        return (instance_name, inst_id, memory, vcpus, stat, times)
      finally:
        fh.close()
    except IOError as err:
      raise errors.HypervisorError("Failed to list instance %s: %s" %
                                   (instance_name, err))

  def GetAllInstancesInfo(self, hvparams=None):
    """Get properties of all instances.

    @type hvparams: dict of strings
    @param hvparams: hypervisor parameter
    @return: list of tuples (name, id, memory, vcpus, stat, times)

    """
    data = []
    for file_name in os.listdir(self._ROOT_DIR):
      try:
        fh = open(utils.PathJoin(self._ROOT_DIR, file_name), "r")
        inst_id = "-1"
        memory = 0
        vcpus = 1
        stat = hv_base.HvInstanceState.SHUTDOWN
        times = -1
        try:
          inst_id = fh.readline().strip()
          memory = utils.TryConvert(int, fh.readline().strip())
          vcpus = utils.TryConvert(int, fh.readline().strip())
          stat = hv_base.HvInstanceState.RUNNING
          times = 0
        finally:
          fh.close()
        data.append((file_name, inst_id, memory, vcpus, stat, times))
      except IOError as err:
        raise errors.HypervisorError("Failed to list instances: %s" % err)
    return data

  @classmethod
  def _InstanceFile(cls, instance_name):
    """Compute the instance file for an instance name.

    """
    return utils.PathJoin(cls._ROOT_DIR, instance_name)

  def _IsAlive(self, instance_name):
    """Checks if an instance is alive.

    """
    file_name = self._InstanceFile(instance_name)
    return os.path.exists(file_name)

  def _MarkUp(self, instance, memory):
    """Mark the instance as running.

    This does no checks, which should be done by its callers.

    """
    file_name = self._InstanceFile(instance.name)
    fh = open(file_name, "w")
    try:
      fh.write("0\n%d\n%d\n" %
               (memory,
                instance.beparams[constants.BE_VCPUS]))
    finally:
      fh.close()

  def _MarkDown(self, instance_name):
    """Mark the instance as running.

    This does no checks, which should be done by its callers.

    """
    file_name = self._InstanceFile(instance_name)
    utils.RemoveFile(file_name)

  def StartInstance(self, instance, block_devices, startup_paused):
    """Start an instance.

    For the fake hypervisor, it just creates a file in the base dir,
    creating an exception if it already exists. We don't actually
    handle race conditions properly, since these are *FAKE* instances.

    """
    if self._IsAlive(instance.name):
      raise errors.HypervisorError("Failed to start instance %s: %s" %
                                   (instance.name, "already running"))
    try:
      self._MarkUp(instance, self._InstanceStartupMemory(instance))
    except IOError as err:
      raise errors.HypervisorError("Failed to start instance %s: %s" %
                                   (instance.name, err))

  def StopInstance(self, instance, force=False, retry=False, name=None,
                   timeout=None):
    """Stop an instance.

    For the fake hypervisor, this just removes the file in the base
    dir, if it exist, otherwise we raise an exception.

    """
    assert(timeout is None or force is not None)

    if name is None:
      name = instance.name
    if not self._IsAlive(name):
      raise errors.HypervisorError("Failed to stop instance %s: %s" %
                                   (name, "not running"))
    self._MarkDown(name)

  def RebootInstance(self, instance):
    """Reboot an instance.

    For the fake hypervisor, this does nothing.

    """
    return

  def BalloonInstanceMemory(self, instance, mem):
    """Balloon an instance memory to a certain value.

    @type instance: L{objects.Instance}
    @param instance: instance to be accepted
    @type mem: int
    @param mem: actual memory size to use for instance runtime

    """
    if not self._IsAlive(instance.name):
      raise errors.HypervisorError("Failed to balloon memory for %s: %s" %
                                   (instance.name, "not running"))
    try:
      self._MarkUp(instance, mem)
    except EnvironmentError as err:
      raise errors.HypervisorError("Failed to balloon memory for %s: %s" %
                                   (instance.name, utils.ErrnoOrStr(err)))

  def GetNodeInfo(self, hvparams=None):
    """Return information about the node.

    See L{BaseHypervisor.GetLinuxNodeInfo}.

    """
    result = self.GetLinuxNodeInfo()
    # substract running instances
    all_instances = self.GetAllInstancesInfo()
    result["memory_free"] -= min(result["memory_free"],
                                 sum([row[2] for row in all_instances]))
    return result

  @classmethod
  def GetInstanceConsole(cls, instance, primary_node, node_group,
                         hvparams, beparams):
    """Return information for connecting to the console of an instance.

    """
    return objects.InstanceConsole(instance=instance.name,
                                   kind=constants.CONS_MESSAGE,
                                   message=("Console not available for fake"
                                            " hypervisor"))

  def Verify(self, hvparams=None):
    """Verify the hypervisor.

    For the fake hypervisor, it just checks the existence of the base
    dir.

    @type hvparams: dict of strings
    @param hvparams: hypervisor parameters to be verified against; not used
      for fake hypervisors

    @return: Problem description if something is wrong, C{None} otherwise

    """
    if os.path.exists(self._ROOT_DIR):
      return None
    else:
      return "The required directory '%s' does not exist" % self._ROOT_DIR

  @classmethod
  def PowercycleNode(cls, hvparams=None):
    """Fake hypervisor powercycle, just a wrapper over Linux powercycle.

    @type hvparams: dict of strings
    @param hvparams: hypervisor params to be used on this node

    """
    cls.LinuxPowercycle()

  def AcceptInstance(self, instance, info, target):
    """Prepare to accept an instance.

    @type instance: L{objects.Instance}
    @param instance: instance to be accepted
    @type info: string
    @param info: instance info, not used
    @type target: string
    @param target: target host (usually ip), on this node

    """
    if self._IsAlive(instance.name):
      raise errors.HypervisorError("Can't accept instance, already running")

  def MigrateInstance(self, cluster_name, instance, target, live):
    """Migrate an instance.

    @type cluster_name: string
    @param cluster_name: name of the cluster
    @type instance: L{objects.Instance}
    @param instance: the instance to be migrated
    @type target: string
    @param target: hostname (usually ip) of the target node
    @type live: boolean
    @param live: whether to do a live or non-live migration

    """
    logging.debug("Fake hypervisor migrating %s to %s (live=%s)",
                  instance, target, live)

  def FinalizeMigrationDst(self, instance, info, success):
    """Finalize the instance migration on the target node.

    For the fake hv, this just marks the instance up.

    @type instance: L{objects.Instance}
    @param instance: instance whose migration is being finalized
    @type info: string/data (opaque)
    @param info: migration information, from the source node
    @type success: boolean
    @param success: whether the migration was a success or a failure

    """
    if success:
      self._MarkUp(instance, self._InstanceStartupMemory(instance))
    else:
      # ensure it's down
      self._MarkDown(instance.name)

  def FinalizeMigrationSource(self, instance, success, live):
    """Finalize the instance migration on the source node.

    @type instance: L{objects.Instance}
    @param instance: the instance that was migrated
    @type success: bool
    @param success: whether the migration succeeded or not
    @type live: bool
    @param live: whether the user requested a live migration or not

    """
    # pylint: disable=W0613
    if success:
      self._MarkDown(instance.name)

  def GetMigrationStatus(self, instance):
    """Get the migration status

    The fake hypervisor migration always succeeds.

    @type instance: L{objects.Instance}
    @param instance: the instance that is being migrated
    @rtype: L{objects.MigrationStatus}
    @return: the status of the current migration (one of
             L{constants.HV_MIGRATION_VALID_STATUSES}), plus any additional
             progress info that can be retrieved from the hypervisor

    """
    return objects.MigrationStatus(status=constants.HV_MIGRATION_COMPLETED)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Module for interacting with lectio.dk.
"""

__version__ = (0, 3, 1)

from .session import Session  # noqa
from .timetable import Period  # noqa
from .assignment import Assignment  # noqa
<commit_msg>Version: 0.3.2<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Module for interacting with lectio.dk.
"""

__version__ = (0, 3, 2)

from .session import Session  # noqa
from .timetable import Period  # noqa
from .assignment import Assignment  # noqa
<|endoftext|>


<commit_before>class LesionAnalysis:
    """
    class for the prostate lesion analysis b/w ARFI and histology
    """

    def __init__(self, pnum):
        self.root = '/luscinia/ProstateStudy/invivo/Patient%s' % pnum
        self.arfi_ios = '%s/ARFI_Index_Lesion_IOS.txt' % self.root
        self.hist_lesions = '%s/Histology/HistologyLesions.txt' % self.root
        self.read_arfi()
        self.read_histology()
        self.check_index_exact_match()
        self.check_index_nn_match()
        self.check_arfi_benign_match('atrophy')
        self.check_arfi_benign_match('bph')

    def read_arfi(self):
        """
        read ARFI lesion IOS data
        """
        with open(self.arfi_ios, 'r') as f:
            arfiios = f.readlines()

        self.arfi = {}
        for lesion in arfiios:
            lesion = lesion[:-1]
            self.arfi[lesion.split(', ')[0]] = lesion.split(', ')[1]

    def read_histology(self):
        """
        head histology pca, atrophy and bph lesions
        """
        with open(self.hist_lesions, 'r') as f:
            histread = f.readlines()

        self.histology = {}
        for lesion in histread:
            lesion = lesion[:-1]
            # make sure we hav)e a properly-formatted histology file
            if not any([x in lesion for x in ['pca', 'bph', 'atrophy']]):
                print "WARNING: Malformed histology lesion file."
            # there can be multiple pca lesions
            if 'pca' in lesion:
                if 'pca' not in self.histology:
                    self.histology['pca'] = [lesion.split(', ')[1:]]
                else:
                    self.histology['pca'].append(lesion.split(', ')[1:])
            else:
                self.histology[lesion.split(',')[0]] = lesion.split(', ')[1:]

    @staticmethod
    def nearest_neighbor(region):
        """
        extract the set of nearest neighbor regions
        """
        nn27dict = dict(nn1p=set(['2p', '2a', '1a', '7p', '7a', '3p']),
                        nn2p=set(['1p', '1a', '2a', '4p']),
                        nn3p=set(['4p', '3a', '9p', '5p', '1p']),
                        nn4p=set(['4a', '3p', '3a', '2p', '6p']),
                        nn5p=set(['6p', '11p', '5a', '15as', '3p', '6p']),
                        nn6p=set(['5p', '6a', '5a', '4p']),
                        nn7p=set(['1p', '8p', '7a', '8a', '1a', '9p']),
                        nn8p=set(['7p', '8a', '10p', '7a']),
                        nn9p=set(['10p', '3p', '9a', '7a', '11p']),
                        nn10p=set(['9p', '10a', '9a', '8p', '12p']),
                        nn11p=set(['12p', '5p', '11a', '15as', '12a', '9p']),
                        nn12p=set(['11p', '12a', '11a', '10p']),
                        nn1a=set(['13as', '2a', '7a', '3a']),
                        nn2a=set(['1a', '13as', '2p', '4a']),
                        nn3a=set(['5a', '4p', '3p', '14as', '5a', '1p']),
                        nn4a=set(['3a', '14as', '4p', '2a', '6a']),
                        nn5a=set(['15as', '6a', '3a', '5p']),
                        nn6a=set(['5a', '15as', '6p', '4a']),
                        nn7a=set(['1a', '13as', '8a', '7p', '9a']),
                        nn8a=set(['8p', '7a', '13as', '10a']),
                        nn9a=set(['10a', '14as', '7a', '11a']),
                        nn10a=set(['10p', '8a', '12a', '9a']),
                        nn11a=set(['12a', '15as', '12p', '9a']),
                        nn12a=set(['11a', '15as', '12p', '10a']),
                        nn13as=set(['1a', '7a', '8a', '2a', '14as']),
                        nn14as=set(['3a', '9a', '4a', '10a', '13as', '15a']),
                        nn15as=set(['6a', '12a', '5a', '11a', '14as']))

        nearest_neighbors = nn27dict['nn%s' % region]

        return nearest_neighbors

    def check_index_exact_match(self):
        """
        check for an exact match b/w ARFI and histology index lesions
        """
        arfi_index = self.arfi.keys()[0]
        hist_index = self.histology['pca'][0][0]

        if arfi_index == hist_index:
            self.index_exact_match = True
        else:
            self.index_exact_match = False

    def check_index_nn_match(self):
        """
        check for nearest-neightbor match b/w ARFI and histology index lesions
        """
        self.arfi_index = self.arfi.keys()[0]
        self.hist_index = self.histology['pca'][0][0]
        self.hist_index_nn = self.nearest_neighbor(self.hist_index)

        if self.arfi_index in self.hist_index_nn:
            self.index_nn_match = True
        else:
            self.index_nn_match = False

    def check_arfi_benign_match(self, benign):
        """
        check if atrophy or bph is present in the exact or nearest neighbor
        region to an ARFI lesion

        INPUT: benign expected to be either 'atrophy' or 'bph'

        EXAMPLE check_arfi_benign_match(self, 'atrophy')
        """
        try:
            benign_regions = self.histology[benign][1:]
            if any([x in benign_regions for x in self.arfi.keys()]):
                setattr(self, 'arfi_%s_match' % benign, True)
            else:
                setattr(self, 'arfi_%s_match' % benign, False)
        except KeyError:
            setattr(self, 'arfi_%s_match' % benign, False)
<commit_msg>print_analysis() method; more graceful handling of non-existing ARFI or histology data<commit_after>class LesionAnalysis:
    """
    class for the prostate lesion analysis b/w ARFI and histology
    """

    def __init__(self, pnum):
        self.pnum = pnum
        self.root = '/luscinia/ProstateStudy/invivo/Patient%s' % self.pnum
        self.arfi_ios = '%s/ARFI_Index_Lesion_IOS.txt' % self.root
        self.hist_lesions = '%s/Histology/HistologyLesions.txt' % self.root
        self.read_arfi()
        self.read_histology()
        self.check_index_exact_match()
        self.check_index_nn_match()
        self.check_arfi_benign_match('atrophy')
        self.check_arfi_benign_match('bph')

    def read_arfi(self):
        """
        read ARFI lesion IOS data
        """
        self.arfi = {}
        try:
            with open(self.arfi_ios, 'r') as f:
                arfiios = f.readlines()

            if 'None' not in arfiios:
                for lesion in arfiios:
                    lesion = lesion[:-1]
                    self.arfi[lesion.split(', ')[0]] = lesion.split(', ')[1]
            else:
                self.arfi[None] = None
        except IOError:
            print "%s does not exist" % self.arfi_ios
            self.arfi[None] = None

    def read_histology(self):
        """
        head histology pca, atrophy and bph lesions
        """
        self.histology = {}
        try:
            with open(self.hist_lesions, 'r') as f:
                histread = f.readlines()

            for lesion in histread:
                lesion = lesion[:-1]
                # make sure we hav)e a properly-formatted histology file
                if not any([x in lesion for x in ['pca', 'bph', 'atrophy']]):
                    print "WARNING: Malformed histology lesion file."
                # there can be multiple pca lesions
                if 'pca' in lesion:
                    if 'pca' not in self.histology:
                        self.histology['pca'] = [lesion.split(', ')[1:]]
                    else:
                        self.histology['pca'].append(lesion.split(', ')[1:])
                else:
                    self.histology[lesion.split(', ')[0]] = lesion.split(', ')[1:]
        except IOError:
            print "%s does not exist" % self.hist_lesions
            self.histology[None] = None

    @staticmethod
    def nearest_neighbor(region):
        """
        extract the set of nearest neighbor regions
        """
        nn27dict = dict(nn1p=set(['2p', '2a', '1a', '7p', '7a', '3p']),
                        nn2p=set(['1p', '1a', '2a', '4p']),
                        nn3p=set(['4p', '3a', '9p', '5p', '1p']),
                        nn4p=set(['4a', '3p', '3a', '2p', '6p']),
                        nn5p=set(['6p', '11p', '5a', '15as', '3p', '6p']),
                        nn6p=set(['5p', '6a', '5a', '4p']),
                        nn7p=set(['1p', '8p', '7a', '8a', '1a', '9p']),
                        nn8p=set(['7p', '8a', '10p', '7a']),
                        nn9p=set(['10p', '3p', '9a', '7a', '11p']),
                        nn10p=set(['9p', '10a', '9a', '8p', '12p']),
                        nn11p=set(['12p', '5p', '11a', '15as', '12a', '9p']),
                        nn12p=set(['11p', '12a', '11a', '10p']),
                        nn1a=set(['13as', '2a', '7a', '3a']),
                        nn2a=set(['1a', '13as', '2p', '4a']),
                        nn3a=set(['5a', '4p', '3p', '14as', '5a', '1p']),
                        nn4a=set(['3a', '14as', '4p', '2a', '6a']),
                        nn5a=set(['15as', '6a', '3a', '5p']),
                        nn6a=set(['5a', '15as', '6p', '4a']),
                        nn7a=set(['1a', '13as', '8a', '7p', '9a']),
                        nn8a=set(['8p', '7a', '13as', '10a']),
                        nn9a=set(['10a', '14as', '7a', '11a']),
                        nn10a=set(['10p', '8a', '12a', '9a']),
                        nn11a=set(['12a', '15as', '12p', '9a']),
                        nn12a=set(['11a', '15as', '12p', '10a']),
                        nn13as=set(['1a', '7a', '8a', '2a', '14as']),
                        nn14as=set(['3a', '9a', '4a', '10a', '13as', '15a']),
                        nn15as=set(['6a', '12a', '5a', '11a', '14as']))

        nearest_neighbors = nn27dict['nn%s' % region]

        return nearest_neighbors

    def check_index_exact_match(self):
        """
        check for an exact match b/w ARFI and histology index lesions
        """
        arfi_index = self.arfi.keys()[0]
        hist_index = self.histology['pca'][0][0]

        if arfi_index == hist_index:
            self.index_exact_match = True
        else:
            self.index_exact_match = False

    def check_index_nn_match(self):
        """
        check for nearest-neightbor match b/w ARFI and histology index lesions
        """
        self.arfi_index = self.arfi.keys()[0]
        self.hist_index = self.histology['pca'][0][0]
        self.hist_index_nn = self.nearest_neighbor(self.hist_index)

        if self.arfi_index in self.hist_index_nn:
            self.index_nn_match = True
        else:
            self.index_nn_match = False

    def check_arfi_benign_match(self, benign):
        """
        check if atrophy or bph is present in the exact or nearest neighbor
        region to an ARFI lesion

        INPUT: benign expected to be either 'atrophy' or 'bph'

        EXAMPLE check_arfi_benign_match(self, 'atrophy')
        """
        try:
            benign_regions = self.histology[benign][1:]
            if any([x in benign_regions for x in self.arfi.keys()]):
                setattr(self, 'arfi_%s_match' % benign, True)
            else:
                setattr(self, 'arfi_%s_match' % benign, False)
        except KeyError:
            setattr(self, 'arfi_%s_match' % benign, False)

    def print_analysis(self):
        """
        print analysis results to the terminal
        """
        print "================= PATIENT %s =================" % self.pnum
        if None in self.arfi or None in self.histology:
            print "Incomplete dataset; not included in analysis."
        else:
            print "Index lesion EXACT match:\t\t%s" % self.index_exact_match
            print "Index lesion NEAREST NEIGHBOR match:\t%s" % self.index_nn_match
            print "ARFI:Atrophy Match:\t\t\t%s" % self.arfi_atrophy_match
            print "ARFI:BPH Match:\t\t\t\t%s" % self.arfi_atrophy_match
<|endoftext|>


<commit_before>from . import bindings
from .nvpair import ptr_to_dict
from .handle import LibZFSHandle
from .utils.conversion import boolean_t

libzfs = bindings.libzfs
ffi = bindings.ffi

pool_state_t = bindings['pool_state_t']
zpool_errata_t = bindings['zpool_errata_t']
zpool_prop_t = bindings['zpool_prop_t']
zpool_status_t = bindings['zpool_status_t']
zprop_source_t = bindings['zprop_source_t']
ZPOOL_MAXNAMELEN = bindings['ZPOOL_MAXNAMELEN']
ZPOOL_CONFIG_POOL_NAME = bindings['ZPOOL_CONFIG_POOL_NAME']
ZPOOL_CONFIG_POOL_GUID = bindings['ZPOOL_CONFIG_POOL_GUID']

class ZPoolProperties(object):
    def __init__(self, pool):
        self._pool = pool

    @LibZFSHandle.requires_refcount
    def items(self):
        # We add another claim to the handle, to ensure that it remains open during the length of this generator
        with LibZFSHandle():
            for prop in zpool_prop_t:
                if prop >= zpool_prop_t.ZPOOL_NUM_PROPS:
                    continue
                value, source = self._get_prop(prop)
                yield prop, value

    @LibZFSHandle.requires_refcount
    def sources(self):
        # We add another claim to the handle, to ensure that it remains open during the length of this generator
        with LibZFSHandle():
            for prop in zpool_prop_t:
                if prop >= zpool_prop_t.ZPOOL_NUM_PROPS:
                    continue
                value, source = self._get_prop(prop)
                yield prop, source

    @LibZFSHandle.requires_refcount
    def _get_prop(self, key, literal = True):
        holder = ffi.new('char [%s]' % ZPOOL_MAXNAMELEN)
        sourceholder = ffi.new('zprop_source_t *')
        literal = boolean_t(literal)

        val = libzfs.zpool_get_prop_literal(self._pool._handle, int(key), holder,
                                            ZPOOL_MAXNAMELEN, sourceholder, literal)
        if not bool(val):
            value = ffi.string(holder)
            source = zprop_source_t(sourceholder[0])
            return value, source
        return None, None

    def __getitem__(self, key):
        try:
            key = zpool_prop_t(key)
        except ValueError:
            raise KeyError(key)
        value, source = self._get_prop(key)
        if value is None:
            raise KeyError(key)
        return value

    def __setitem__(self, key, value):
        # TODO: writing properties
        pass

class ZPool(object):
    _name = None
    _state = None
    _config = None
    _old_config = None
    _refreshed = False

    _status = None
    _status_extra = None
    _errata = None

    def __init__(self, handle):
        self._handle = handle
        self.properties = ZPoolProperties(self)

    @property
    def hdl(self):
        return self._handle

    @property
    def name(self):
        if self._name is None:
            self._name = ffi.string(libzfs.zpool_get_name(self.hdl))
        return self._name

    @property
    def state(self):
        if self._state is None:
            state = libzfs.zpool_get_state(self.hdl)
            self._state = pool_state_t(state)
        return self._state

    def _get_status(self):
        if self._status is not None:
            return
        msgid = ffi.new('char **')
        errata = ffi.new('zpool_errata_t *')

        reason = libzfs.zpool_get_status(self.hdl, msgid, errata)

        self._status = zpool_status_t(reason)
        if msgid[0] == ffi.NULL:
            self._status_extra = ''
        else:
            self._status_extra = ffi.string(msgid[0])
        self._errata = zpool_errata_t(errata[0])

    @property
    def status_extra(self):
        self._get_status()
        return self._status_extra

    @property
    def status(self):
        self._get_status()
        return self._status

    @property
    def errata(self):
        self._get_status()
        return self._errata

    @property
    def config(self):
        if self._config is None:
            config = libzfs.zpool_get_config(self.hdl, ffi.NULL)
            self._config = ptr_to_dict(config, free=False)
        return self._config

    @property
    def old_config(self):
        if self._refreshed is False:
            # Judging by zfs/cmd/zpool/zpool_main.c, we should ignore the 'old config'
            #  the first time we refresh the iostats.
            # The internals (zfs/lib/libzfs/libzfs_config.c) should probably clarify more
            #  on why.. for now we'll mimic what we see.
            return dict()
        if self._old_config is None:
            old_config = libzfs.zpool_get_old_config(self.hdl)
            self._old_config = ptr_to_dict(old_config, free=False)
        return self._old_config

    @LibZFSHandle.requires_refcount
    def refresh_stats(self):
        missing = ffi.new("boolean_t *")
        success = libzfs.zpool_refresh_stats(self.hdl, missing) == 0

        missing = bool(missing[0])

        success = success and not missing
        if success:
            self._config = None
            self._old_config = None
            self._refreshed = True

        return success

    def __repr__(self):
        return "<ZPool: %s: %s>" % (self.name, self.state.name)

    @classmethod
    def list(cls):
        pools = []

        @ffi.callback('zpool_iter_f')
        def _callback(handle, arg=None):
            zpool = ZPool(handle)
            pools.append(zpool)
            return 0

        with LibZFSHandle() as hdl:
            libzfs.zpool_iter(hdl, _callback, ffi.NULL)

        return pools

    @classmethod
    def get(cls, name=None, guid=None):
        if guid:
            guid = int(guid)

        with LibZFSHandle():
            pools = cls.list()

        if name:
            pools = [pool for pool in pools if pool.config.get(ZPOOL_CONFIG_POOL_NAME) == name]
        if guid:
            pools = [pool for pool in pools if pool.config.get(ZPOOL_CONFIG_POOL_GUID) == guid]

        if len(pools) == 1:
            return pools[0]
        raise KeyError("Could not find %s matching query" % cls.__name__)
<commit_msg>-Change: require refcount on ZPool.list/get.<commit_after>from . import bindings
from .nvpair import ptr_to_dict
from .handle import LibZFSHandle
from .utils.conversion import boolean_t

libzfs = bindings.libzfs
ffi = bindings.ffi

pool_state_t = bindings['pool_state_t']
zpool_errata_t = bindings['zpool_errata_t']
zpool_prop_t = bindings['zpool_prop_t']
zpool_status_t = bindings['zpool_status_t']
zprop_source_t = bindings['zprop_source_t']
ZPOOL_MAXNAMELEN = bindings['ZPOOL_MAXNAMELEN']
ZPOOL_CONFIG_POOL_NAME = bindings['ZPOOL_CONFIG_POOL_NAME']
ZPOOL_CONFIG_POOL_GUID = bindings['ZPOOL_CONFIG_POOL_GUID']

class ZPoolProperties(object):
    def __init__(self, pool):
        self._pool = pool

    @LibZFSHandle.requires_refcount
    def items(self):
        # We add another claim to the handle, to ensure that it remains open during the length of this generator
        with LibZFSHandle():
            for prop in zpool_prop_t:
                if prop >= zpool_prop_t.ZPOOL_NUM_PROPS:
                    continue
                value, source = self._get_prop(prop)
                yield prop, value

    @LibZFSHandle.requires_refcount
    def sources(self):
        # We add another claim to the handle, to ensure that it remains open during the length of this generator
        with LibZFSHandle():
            for prop in zpool_prop_t:
                if prop >= zpool_prop_t.ZPOOL_NUM_PROPS:
                    continue
                value, source = self._get_prop(prop)
                yield prop, source

    @LibZFSHandle.requires_refcount
    def _get_prop(self, key, literal = True):
        holder = ffi.new('char [%s]' % ZPOOL_MAXNAMELEN)
        sourceholder = ffi.new('zprop_source_t *')
        literal = boolean_t(literal)

        val = libzfs.zpool_get_prop_literal(self._pool._handle, int(key), holder,
                                            ZPOOL_MAXNAMELEN, sourceholder, literal)
        if not bool(val):
            value = ffi.string(holder)
            source = zprop_source_t(sourceholder[0])
            return value, source
        return None, None

    def __getitem__(self, key):
        try:
            key = zpool_prop_t(key)
        except ValueError:
            raise KeyError(key)
        value, source = self._get_prop(key)
        if value is None:
            raise KeyError(key)
        return value

    def __setitem__(self, key, value):
        # TODO: writing properties
        pass

class ZPool(object):
    _name = None
    _state = None
    _config = None
    _old_config = None
    _refreshed = False

    _status = None
    _status_extra = None
    _errata = None

    def __init__(self, handle):
        self._handle = handle
        self.properties = ZPoolProperties(self)

    @property
    def hdl(self):
        return self._handle

    @property
    def name(self):
        if self._name is None:
            self._name = ffi.string(libzfs.zpool_get_name(self.hdl))
        return self._name

    @property
    def state(self):
        if self._state is None:
            state = libzfs.zpool_get_state(self.hdl)
            self._state = pool_state_t(state)
        return self._state

    def _get_status(self):
        if self._status is not None:
            return
        msgid = ffi.new('char **')
        errata = ffi.new('zpool_errata_t *')

        reason = libzfs.zpool_get_status(self.hdl, msgid, errata)

        self._status = zpool_status_t(reason)
        if msgid[0] == ffi.NULL:
            self._status_extra = ''
        else:
            self._status_extra = ffi.string(msgid[0])
        self._errata = zpool_errata_t(errata[0])

    @property
    def status_extra(self):
        self._get_status()
        return self._status_extra

    @property
    def status(self):
        self._get_status()
        return self._status

    @property
    def errata(self):
        self._get_status()
        return self._errata

    @property
    def config(self):
        if self._config is None:
            config = libzfs.zpool_get_config(self.hdl, ffi.NULL)
            self._config = ptr_to_dict(config, free=False)
        return self._config

    @property
    def old_config(self):
        if self._refreshed is False:
            # Judging by zfs/cmd/zpool/zpool_main.c, we should ignore the 'old config'
            #  the first time we refresh the iostats.
            # The internals (zfs/lib/libzfs/libzfs_config.c) should probably clarify more
            #  on why.. for now we'll mimic what we see.
            return dict()
        if self._old_config is None:
            old_config = libzfs.zpool_get_old_config(self.hdl)
            self._old_config = ptr_to_dict(old_config, free=False)
        return self._old_config

    @LibZFSHandle.requires_refcount
    def refresh_stats(self):
        missing = ffi.new("boolean_t *")
        success = libzfs.zpool_refresh_stats(self.hdl, missing) == 0

        missing = bool(missing[0])

        success = success and not missing
        if success:
            self._config = None
            self._old_config = None
            self._refreshed = True

        return success

    def __repr__(self):
        return "<ZPool: %s: %s>" % (self.name, self.state.name)

    @classmethod
    @LibZFSHandle.requires_refcount
    def list(cls):
        pools = []

        @ffi.callback('zpool_iter_f')
        def _callback(handle, arg=None):
            zpool = ZPool(handle)
            pools.append(zpool)
            return 0

        with LibZFSHandle() as hdl:
            libzfs.zpool_iter(hdl, _callback, ffi.NULL)

        return pools

    @classmethod
    @LibZFSHandle.requires_refcount
    def get(cls, name=None, guid=None):
        if guid:
            guid = int(guid)

        with LibZFSHandle():
            pools = cls.list()

        if name:
            pools = [pool for pool in pools if pool.config.get(ZPOOL_CONFIG_POOL_NAME) == name]
        if guid:
            pools = [pool for pool in pools if pool.config.get(ZPOOL_CONFIG_POOL_GUID) == guid]

        if len(pools) == 1:
            return pools[0]
        raise KeyError("Could not find %s matching query" % cls.__name__)
<|endoftext|>


<commit_before>#! /usr/bin/env python

# Victor Calderon
# Vanderbilt University
# Project: Digits dataset for VandyAstroML project

"""
Description: Decision-tree algorithm on the dataset "Digits" from Scikit-learn.
"""
# modules for Python2/3
from __future__ import division, print_function, absolute_import

# Importing modules
import numpy as num
import random
from sklearn import datasets
from collections import Counter

class Best_Choices():
		def __init_(self, results=None, true_tree=None, false_tree=None,col=-1 ):
			self.results    = results # Dictionary of results
			self.true_tree  = None    # Subtree for the `true` tree
			self.false_tree = None    # SubTree for the `false` tree
			self.col        = -1      # Chosen column number

def splitting_branches(data, ncol, val):
	"""
	Separatates `ncol` from data into two branches, depending on the conditions
	based on `val`

	Parameters
	----------
	data: array_like
		N-dimensional array with the data

	ncol: int
		column number used to categorized the two output sets (arrays).

	val: string, int, or float
		value used to categorize `data`

	Returns
	-------
	set_arrs: array_like
		array containing both sets, i.e. arr1 and arr2, based on the criteria of
		`val`.
	"""
	# Checking if value is an integer, a float, or something else
	if isinstance(val, int) or
		val_type = lambda data: data[ncol] >= val # for numbers
	else:
		val_type = lambda data: data[ncol] == val # for strings or other
	# Saving into arrays (and making sure they are lists, and not arrays)
	set_corr = [row for row in data if val_type(row)]
	set_fail = [row for row in data if not val_type(row)]
	set_arrs = (set_corr, set_fail)

	return set_arrs

def counter_of_targets(data):
	"""
	Calculates the total number of occurences for each of the possible values of
	the targets for each (sub)array.

	Parameters
	----------
	data: array_like
		N-dimensional array or 1-d array

	Returns
	-------
	Count_dict: dictionary
		Dictionary with possible values and their counts
	"""
	## Assuming it's a list
	Count_obj = Counter(num.array(data).T[-1])
	Count_dict = {}
	for key in Count_obj.keys():
		Count_dict[key] = Count_obj[key]

	return Count_dict

def entropy_calculation(data):
	"""
	Calculated the `entropy` of the dataset. This is a measure of how unpure
	the mix of `target` values is. Ideally, entropy==0.

	Parameters
	----------
	data_dict: dictionary
		dictionary containing counts for possible `target` values

	Returns
	-------
	ent_val: float
		value for the estimated entropy of the given `data`
	"""
	counts  = counter_of_targets(data)
	ent_val = 0.
	# Looping over all `target` values in `counts`
	for key in counts.keys():
		p_val = counts[key]/float(len(data))
		# Entropy formula
		ent_val -= p_val*num.log2(p_val)

	return ent_val

def Tree_training(data, estimator=entropy_calculation):
	"""
	This function iterates over all possible values for each of the columns
	of `data` and constructs the whole Decision Tree structure

	Parameters
	----------
	data: array_like
		N-dimensional array, with `targets` being the last column.

	estimator: function
		Estimator that determines the cost function of the data, i.e. how
		(un)pure `data` is

	Returns
	-------
	Best_vals: class
		This class contains information on the best values obtained through the
		Decision Tree constructed.
	"""

def main():
	## Loading dataset
	digits = datasets.load_digits()
	# Digits - Data
	digits_dat = digits['data']
	n_samples  = digits_dat.shape[0]
	n_features = digits_dat.shape[1]
	# Digits - target names
	digits_tnames = digits['target_names'].astype(int)
	# Digits - target values
	digits_target = digits['target']

	## Training and control samples
	# Selects 80 percent of sample as training sample
	# and the other 20 percent as the control sample.
	# Want to select 80 percent of all types of targets
	train_frac = 0.8
	# training set - indices
	train_idx  = []
	for target in digits_tnames:
		tnames_idx    = num.where(digits_target==target)[0]
		tnames_size   = int(tnames_idx.size*train_frac)
		tnames_sample = random.sample(tnames_idx, tnames_size)
		train_idx.extend(tnames_sample)
	train_idx = num.array(train_idx)
	# Control set - indices
	cont_idx  = set(range(len(digits_target))).difference(train_idx)
	cont_idx  = num.array(list(cont_idx))
	# Training set - dictionary
	train_dict = {}
	train_dict['data'  ] = digits_dat   [train_idx]
	train_dict['target'] = digits_target[train_idx]
	train_data_target    = num.column_stack((train_dict['data'],
		train_dict['target']))
	# Combining data and target
	# Control set - dictionary
	cont_dict = {}
	cont_dict['data'  ] = digits_dat   [cont_idx]
	cont_dict['target'] = digits_target[cont_idx]
	cont_data_target    = num.column_stack((cont_dict['data'],
		cont_dict['target']))

<commit_msg>Changed to spaces from tabs<commit_after>#! /usr/bin/env python

# Victor Calderon
# Vanderbilt University
# Project: Digits dataset for VandyAstroML project

"""
Description: Decision-tree algorithm on the dataset "Digits" from Scikit-learn.
"""
# modules for Python2/3
from __future__ import division, print_function, absolute_import

# Importing modules
import numpy as num
import random
from sklearn import datasets
from collections import Counter

class Best_Choices():
        def __init_(self, results=None, true_tree=None, false_tree=None,col=-1 ):
            self.results    = results # Dictionary of results
            self.true_tree  = None    # Subtree for the `true` tree
            self.false_tree = None    # SubTree for the `false` tree
            self.col        = -1      # Chosen column number

def splitting_branches(data, ncol, val):
    """
    Separatates `ncol` from data into two branches, depending on the conditions
    based on `val`

    Parameters
    ----------
    data: array_like
        N-dimensional array with the data

    ncol: int
        column number used to categorized the two output sets (arrays).

    val: string, int, or float
        value used to categorize `data`

    Returns
    -------
    set_arrs: array_like
        array containing both sets, i.e. arr1 and arr2, based on the criteria of
        `val`.
    """
    # Checking if value is an integer, a float, or something else
    if isinstance(val, int) or
        val_type = lambda data: data[ncol] >= val # for numbers
    else:
        val_type = lambda data: data[ncol] == val # for strings or other
    # Saving into arrays (and making sure they are lists, and not arrays)
    set_corr = [row for row in data if val_type(row)]
    set_fail = [row for row in data if not val_type(row)]
    set_arrs = (set_corr, set_fail)

    return set_arrs

def counter_of_targets(data):
    """
    Calculates the total number of occurences for each of the possible values of
    the targets for each (sub)array.

    Parameters
    ----------
    data: array_like
        N-dimensional array or 1-d array

    Returns
    -------
    Count_dict: dictionary
        Dictionary with possible values and their counts
    """
    ## Assuming it's a list
    Count_obj = Counter(num.array(data).T[-1])
    Count_dict = {}
    for key in Count_obj.keys():
    Count_dict[key] = Count_obj[key]

    return Count_dict

def entropy_calculation(data):
    """
    Calculated the `entropy` of the dataset. This is a measure of how unpure
    the mix of `target` values is. Ideally, entropy==0.

    Parameters
    ----------
    data_dict: dictionary
        dictionary containing counts for possible `target` values

    Returns
    -------
    ent_val: float
        value for the estimated entropy of the given `data`
    """
    counts  = counter_of_targets(data)
    ent_val = 0.
    # Looping over all `target` values in `counts`
    for key in counts.keys():
        p_val = counts[key]/float(len(data))
        # Entropy formula
        ent_val -= p_val*num.log2(p_val)

    return ent_val

def Tree_training(data, estimator=entropy_calculation):
    """
    This function iterates over all possible values for each of the columns
    of `data` and constructs the whole Decision Tree structure

    Parameters
    ----------
    data: array_like
        N-dimensional array, with `targets` being the last column.

    estimator: function
        Estimator that determines the cost function of the data, i.e. how
        (un)pure `data` is

    Returns
    -------
    Best_vals: class
        This class contains information on the best values obtained through the
        Decision Tree constructed.
    """

def main():
    ## Loading dataset
    digits = datasets.load_digits()
    # Digits - Data
    digits_dat = digits['data']
    n_samples  = digits_dat.shape[0]
    n_features = digits_dat.shape[1]
    # Digits - target names
    digits_tnames = digits['target_names'].astype(int)
    # Digits - target values
    digits_target = digits['target']

    ## Training and control samples
    # Selects 80 percent of sample as training sample
    # and the other 20 percent as the control sample.
    # Want to select 80 percent of all types of targets
    train_frac = 0.8
    # training set - indices
    train_idx  = []
    for target in digits_tnames:
        tnames_idx    = num.where(digits_target==target)[0]
        tnames_size   = int(tnames_idx.size*train_frac)
        tnames_sample = random.sample(tnames_idx, tnames_size)
        train_idx.extend(tnames_sample)
    train_idx = num.array(train_idx)
    # Control set - indices
    cont_idx  = set(range(len(digits_target))).difference(train_idx)
    cont_idx  = num.array(list(cont_idx))
    # Training set - dictionary
    train_dict = {}
    train_dict['data'  ] = digits_dat   [train_idx]
    train_dict['target'] = digits_target[train_idx]
    train_data_target    = num.column_stack((train_dict['data'],
        train_dict['target']))
    # Combining data and target
    # Control set - dictionary
    cont_dict = {}
    cont_dict['data'  ] = digits_dat   [cont_idx]
    cont_dict['target'] = digits_target[cont_idx]
    cont_data_target    = num.column_stack((cont_dict['data'],
        cont_dict['target']))

<|endoftext|>


<commit_before>#!/bin/sh
# -*- python -*-

################################################################################
# This file is python bilingual: The next line starts a comment in Python,
# and is a no-op in shell
""":"
# Find a suitable python interpreter (adapt for your specific needs)
for cmd in python3 python python2; do
   command -v > /dev/null $cmd && exec $cmd $0 "$@"
done

echo "Error: Could not find a valid python interpreter --> exiting!" >&2
exit 2
":"""
################################################################################

# Git Version: @git@

#-----------------------------------------------------------------------
# XALT: A tool that tracks users jobs and environments on a cluster.
# Copyright (C) 2013-2014 University of Texas at Austin
# Copyright (C) 2013-2014 University of Tennessee
#
# This library is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as
# published by the Free Software Foundation; either version 2.1 of
# the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser  General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free
# Software Foundation, Inc., 59 Temple Place, Suite 330,
# Boston, MA 02111-1307 USA
#-----------------------------------------------------------------------

from __future__  import print_function
from fnmatch     import fnmatch
from collections import OrderedDict
import os, sys, re, argparse, subprocess

class CmdLineOptions(object):
  """ Command line Options class """

  def __init__(self):
    """ Empty Ctor """
    pass

  def execute(self):
    """ Specify command line arguments and parse the command line"""
    parser = argparse.ArgumentParser()
    parser.add_argument("--lib64",    dest='lib64',    action="store",                     help="LIB64 install directory")
    parser.add_argument("--base",     dest='base',     action="store",                     help="base library")
    parser.add_argument("--real",     dest='real',     action="store",                     help="real library")
    parser.add_argument("--verbose",  dest='verbose',  action="store_true", default=False, help="Print actions")
    args = parser.parse_args()

    return args

def files_in_tree(path,pattern):
  fileA = []
  if (not os.path.isdir(path)):
    return fileA

  for root, dirs, files in os.walk(path):
    for name in files:
      fn = os.path.join(root, name)
      if (fnmatch(fn,pattern)):
        fileA.append(fn)
  return fileA

def main():

  args      = CmdLineOptions().execute()
  lib64_dir = args.lib64
  baseLib   = args.base
  realLib   = args.real
  verbose   = args.verbose

  baseBn    = os.path.basename(baseLib)
  dirNm     = os.path.realpath(os.path.dirname(baseLib))
  pattern   = os.path.join(dirNm,baseBn)+"*"
  fileA     = files_in_tree(dirNm, pattern)
  fileA.append(realLib)
  fileT     = {}
  for fn in fileA:
    fileT[fn] = True

  for fn in fileT:
    if (verbose): print("Storing "+fn+":")
    if (os.path.islink(fn)):
      newFn = os.readlink(fn)
      if (newFn.find('/') == -1):
        newFn = os.path.join(dirNm,newFn)
      if (os.path.isfile(newFn)):
        if (fileT[newFn]):
          cmd = "cp "+newFn+" "+lib64_dir
          if (verbose): print (" ",cmd)
          subprocess.call(cmd)
          fileT[newFn] = False
        cmd = "ln -sf "+os.path.basename(newFn)+" "+ os.path.join(lib64_dir,os.path.basename(fn))
        if (verbose): print (" ",cmd)
        subprocess.call(cmd)

      else:
        print ("Cannot deal w/link: ",newFn)
        os.exit(-1)
    else:
      if (fileT[fn]):
        cmd = "cp "+fn+" "+lib64_dir
        if (verbose): print (" ",cmd)
        subprocess.call(cmd)
        fileT[fn] = False

if ( __name__ == '__main__'): main()
<commit_msg>testing subprocess.call()<commit_after>#!/bin/sh
# -*- python -*-

################################################################################
# This file is python bilingual: The next line starts a comment in Python,
# and is a no-op in shell
""":"
# Find a suitable python interpreter (adapt for your specific needs)
for cmd in python3 python python2; do
   command -v > /dev/null $cmd && exec $cmd $0 "$@"
done

echo "Error: Could not find a valid python interpreter --> exiting!" >&2
exit 2
":"""
################################################################################

# Git Version: @git@

#-----------------------------------------------------------------------
# XALT: A tool that tracks users jobs and environments on a cluster.
# Copyright (C) 2013-2014 University of Texas at Austin
# Copyright (C) 2013-2014 University of Tennessee
#
# This library is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as
# published by the Free Software Foundation; either version 2.1 of
# the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser  General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free
# Software Foundation, Inc., 59 Temple Place, Suite 330,
# Boston, MA 02111-1307 USA
#-----------------------------------------------------------------------

from __future__  import print_function
from fnmatch     import fnmatch
from collections import OrderedDict
import os, sys, re, argparse, subprocess

class CmdLineOptions(object):
  """ Command line Options class """

  def __init__(self):
    """ Empty Ctor """
    pass

  def execute(self):
    """ Specify command line arguments and parse the command line"""
    parser = argparse.ArgumentParser()
    parser.add_argument("--lib64",    dest='lib64',    action="store",                     help="LIB64 install directory")
    parser.add_argument("--base",     dest='base',     action="store",                     help="base library")
    parser.add_argument("--real",     dest='real',     action="store",                     help="real library")
    parser.add_argument("--verbose",  dest='verbose',  action="store_true", default=False, help="Print actions")
    args = parser.parse_args()

    return args

def files_in_tree(path,pattern):
  fileA = []
  if (not os.path.isdir(path)):
    return fileA

  for root, dirs, files in os.walk(path):
    for name in files:
      fn = os.path.join(root, name)
      if (fnmatch(fn,pattern)):
        fileA.append(fn)
  return fileA

def main():

  args      = CmdLineOptions().execute()
  lib64_dir = args.lib64
  baseLib   = args.base
  realLib   = args.real
  verbose   = args.verbose

  baseBn    = os.path.basename(baseLib)
  dirNm     = os.path.realpath(os.path.dirname(baseLib))
  pattern   = os.path.join(dirNm,baseBn)+"*"
  fileA     = files_in_tree(dirNm, pattern)
  fileA.append(realLib)
  fileT     = {}
  for fn in fileA:
    fileT[fn] = True

  for fn in fileT:
    if (verbose): print("Storing "+fn+":")
    if (os.path.islink(fn)):
      newFn = os.readlink(fn)
      if (newFn.find('/') == -1):
        newFn = os.path.join(dirNm,newFn)
      if (os.path.isfile(newFn)):
        if (fileT[newFn]):
          cmd = "cp "+newFn+" "+lib64_dir
          if (verbose): print (" ",cmd)
          subprocess.call(cmd, shell=True)
          fileT[newFn] = False
        cmd = "ln -sf "+os.path.basename(newFn)+" "+ os.path.join(lib64_dir,os.path.basename(fn))
        if (verbose): print (" ",cmd)
        subprocess.call(cmd, shell=True)

      else:
        print ("Cannot deal w/link: ",newFn)
        os.exit(-1)
    else:
      if (fileT[fn]):
        cmd = "cp "+fn+" "+lib64_dir
        if (verbose): print (" ",cmd)
        subprocess.call(cmd, shell=True)
        fileT[fn] = False

if ( __name__ == '__main__'): main()
<|endoftext|>


<commit_before>import random

from pyglet import gl, graphics, clock
import numpy

from life import DENSITY, CELL_SIZE

class Creator:

    def __init__(self, window):
        self._window = window
        self._window.push_handlers(self)
        self.width = window.width // CELL_SIZE
        self.height = window.height // CELL_SIZE
        self.field = numpy.empty(
            shape=[self.width, self.height],
            dtype=int)
        for y in range(self.height):
            for x in range(self.width):
                self.field[x, y] = int(random.random() < DENSITY)

    def on_draw(self):
        self._window.clear()
        gl.glColor3ub(0, 255, 0)
        for y in range(self.height):
            for x in range(self.width):
                if self.field[x, y]:
                    x0 = x * CELL_SIZE
                    y0 = y * CELL_SIZE
                    x1 = x0 + CELL_SIZE - 1
                    y1 = y0 + CELL_SIZE - 1
                    graphics.draw(
                        4, gl.GL_QUADS,
                        ('v2i', (x0, y0, x1, y0, x1, y1, x0, y1)))
        clock.schedule_once(self._cycle, 0)

    def _cycle(self, dt):
        new_field = numpy.empty(
            shape=[self.width, self.height],
            dtype='int8')
        for y in range(self.height):
            for x in range(self.width):
                mass = sum(self.field[x - 1:x + 2:2, y - 1:y + 2:2].flatten())
                if self.field[x, y] and mass not in (2, 3,):
                    new_field[x, y] = 0
                elif not self.field[x, y] and mass == 3:
                    new_field[x, y] = 1
                else:
                    new_field[x, y] = self.field[x, y]
        self.field = new_field
<commit_msg>Use draw queue.<commit_after>import random

from pyglet import gl, graphics, clock
import numpy

from life import DENSITY, CELL_SIZE
from collections import deque

class Creator:

    def __init__(self, window):
        self._window = window
        self._window.push_handlers(self)
        self.width = window.width // CELL_SIZE
        self.height = window.height // CELL_SIZE
        self.field = numpy.empty(
            shape=[self.width, self.height],
            dtype=int)
        self._draw_queue = deque()
        for y in range(self.height):
            for x in range(self.width):
                value = int(random.random() < DENSITY)
                self.field[x, y] = value
                self._draw_queue.append((x, y, value))
        clock.schedule_once(self._cycle, 0)

    def on_draw(self):
        while self._draw_queue:
            x, y, value = self._draw_queue.popleft()
            x0 = x * CELL_SIZE
            y0 = y * CELL_SIZE
            x1 = x0 + CELL_SIZE - 1
            y1 = y0 + CELL_SIZE - 1
            graphics.draw(
                4, gl.GL_QUADS,
                ('v2i', (x0, y0, x1, y0, x1, y1, x0, y1)),
                ('c3B', (0, 255 * value, 0) * 4))

    def _cycle(self, dt):
        new_field = numpy.empty(
            shape=[self.width, self.height],
            dtype='int8')
        for y in range(self.height):
            for x in range(self.width):
                mass = sum(self.field[x - 1:x + 2:2, y - 1:y + 2:2].flatten())
                if self.field[x, y] and mass not in (2, 3,):
                    new_field[x, y] = 0
                    self._draw_queue.append((x, y, 0))
                elif not self.field[x, y] and mass == 3:
                    new_field[x, y] = 1
                    self._draw_queue.append((x, y, 1))
                else:
                    new_field[x, y] = self.field[x, y]
        self.field = new_field
        clock.schedule_once(self._cycle, 0)
<|endoftext|>


<commit_before>"""
State Class

Stores information from data read by the json
this is done instead of passing a deserialized dict to objects so that named values
can be used
"""

import os
import json

class State(object):
    """
    Represents the state updated by the API
    """
    # the unix time when the file has been last updated
    last_updated_time = 0
    # the path of the file to read the state from
    file_path = "unset"
    # colors 1 and 2, that must be tuples or lists w/ max value 255
    color1 = (255, 0, 0)
    color2 = (0, 0, 0)
    # if color1/2 are random
    random1 = False
    random2 = False
    # the pattern id to use
    pattern = 0
    # depending on the pattern, typically is how many LEDs
    # wide the pattern is
    length = 5
    # how long in milliseconds the pattern takes to show a difference
    delay = 10

    def __init__(self, file_path):
        self.file_path = file_path
        # load the initial state from the file
        self.__update_state()

    def check_update(self):
        """
        Checks the state file to see if has been updated,
        and if so, reads from the file.
        """
        try:
            time = os.path.getmtime(self.file_path)
            if time > self.last_updated_time:
                # file has been updated since the last time it was read
                self.__update_state()
                self.last_updated_time = time
        except OSError:
            # file not found
            pass

    def __update_state(self):
        """
        Updates the state of this object from the file, called
        when the file is updated.

        Does not perform any validation on the data, only may check
        that the values for types are valid
        """
        # read from the file
        with open(self.file_path) as f:
            data = json.load(f)
            self.color1 = tuple(data['color1'])
            self.color2 = tuple(data['color2'])
            self.color1 = (255, 0, 0)
            self.color2 = (0, 255, 0)
            self.random1 = bool(data['random1'])
            self.random2 = bool(data['random2'])
            self.pattern = int(data['pattern'])
            self.length = int(data['length'])
            self.delay = int(data['delay'])
<commit_msg>remove workaround code<commit_after>"""
State Class

Stores information from data read by the json
this is done instead of passing a deserialized dict to objects so that named values
can be used
"""

import os
import json

class State(object):
    """
    Represents the state updated by the API
    """
    # the unix time when the file has been last updated
    last_updated_time = 0
    # the path of the file to read the state from
    file_path = "unset"
    # colors 1 and 2, that must be tuples or lists w/ max value 255
    color1 = (255, 0, 0)
    color2 = (0, 0, 0)
    # if color1/2 are random
    random1 = False
    random2 = False
    # the pattern id to use
    pattern = 0
    # depending on the pattern, typically is how many LEDs
    # wide the pattern is
    length = 5
    # how long in milliseconds the pattern takes to show a difference
    delay = 10

    def __init__(self, file_path):
        self.file_path = file_path
        # load the initial state from the file
        self.__update_state()

    def check_update(self):
        """
        Checks the state file to see if has been updated,
        and if so, reads from the file.
        """
        try:
            time = os.path.getmtime(self.file_path)
            if time > self.last_updated_time:
                # file has been updated since the last time it was read
                self.__update_state()
                self.last_updated_time = time
        except OSError:
            # file not found
            pass

    def __update_state(self):
        """
        Updates the state of this object from the file, called
        when the file is updated.

        Does not perform any validation on the data, only may check
        that the values for types are valid
        """
        # read from the file
        with open(self.file_path) as f:
            data = json.load(f)
            self.color1 = tuple(data['color1'])
            self.color2 = tuple(data['color2'])
            self.random1 = bool(data['random1'])
            self.random2 = bool(data['random2'])
            self.pattern = int(data['pattern'])
            self.length = int(data['length'])
            self.delay = int(data['delay'])
<|endoftext|>


<commit_before># Copyright 2021 The DDSP Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Lint as: python3
"""Library of functions and layers of Directed Acyclical Graphs.

DAGLayer exists as an alternative to manually specifying the forward pass in
python. The advantage is that a variety of configurations can be
programmatically specified via external dependency injection, such as with the
`gin` library.
"""

from typing import Dict, Sequence, Tuple, Text, TypeVar

from absl import logging
from ddsp import core
import gin
import tensorflow.compat.v2 as tf

tfkl = tf.keras.layers

# Define Types.
TensorDict = Dict[Text, tf.Tensor]
KeyOrModule = TypeVar('KeyOrModule', Text, tf.Module)
Node = Tuple[KeyOrModule, Sequence[Text], Sequence[Text]]
DAG = Sequence[Node]

# Helper Functions for DAGs  ---------------------------------------------------
filter_by_value = lambda d, cond: dict(filter(lambda e: cond(e[1]), d.items()))
is_module = lambda v: isinstance(v, tf.Module)

# Duck typing.
is_loss = lambda v: hasattr(v, 'get_losses_dict')
is_processor = lambda v: hasattr(v, 'get_signal') and hasattr(v, 'get_controls')

def split_keras_kwargs(kwargs):
  """Strip keras specific kwargs."""
  keras_kwargs = {}
  for key in ['training', 'mask', 'name']:
    if kwargs.get(key) is not None:
      keras_kwargs[key] = kwargs.pop(key)
  return keras_kwargs, kwargs

# DAG and ProcessorGroup Classes -----------------------------------------------
@gin.register
class DAGLayer(tfkl.Layer):
  """String modules together."""

  def __init__(self, dag: DAG, **kwarg_modules):
    """Constructor.

    Args:
      dag: A directed acyclical graph in the form of a list of nodes. Each node
        has the form

        ['module', ['input_key', ...], ['output_key', ...]]

        'module': Module instance or string name of module. For example,
          'encoder' woud access the attribute `dag_layer.encoder`.
        'input_key': List of strings, nested keys in dictionary of dag outputs.
          For example, 'inputs/f0_hz' would access `outputs[inputs]['f0_hz']`.
          Inputs to the dag are wrapped in a `inputs` dict as shown in the
          example. This list is ordered and has one key per a module input
          argument. Each node's outputs are prefixed by their module name.
        'output_key': List of strings, keys for each return value of the module.
          For example, ['amps', 'freqs'] would have the module return a dict
          {'module_name': {'amps': return_value_0, 'freqs': return_value_1}}.
          If the module returns a dictionary, the keys of the dictionary will be
          used and these values (if provided) will be ignored.

        The graph is read sequentially and must be topologically sorted. This
        means that all inputs for a module must already be generated by earlier
        modules (or in the input dictionary).
      **kwarg_modules: A series of modules to add to DAGLayer. Each kwarg that
        is a tf.Module will be added as a property of the layer, so that it will
        be accessible as `dag_layer.kwarg`. Also, other keras kwargs such as
        'name' are split off before adding modules.
    """
    keras_kwargs, kwarg_modules = split_keras_kwargs(kwarg_modules)
    super().__init__(**keras_kwargs)

    # Create properties/submodules from other kwargs.
    modules = filter_by_value(kwarg_modules, is_module)

    # Remove modules from the dag, make properties of dag_layer.
    dag, dag_modules = self.format_dag(dag)
    # DAG is now just strings.
    self.dag = dag
    modules.update(dag_modules)

    # Make as propreties of DAGLayer to keep track of variables in checkpoints.
    self.module_names = list(modules.keys())
    for module_name, module in modules.items():
      setattr(self, module_name, module)

  @property
  def modules(self):
    """Module getter."""
    return [getattr(self, name) for name in self.module_names]

  @staticmethod
  def format_dag(dag):
    """Remove modules from dag, and replace with module names."""
    modules = {}
    dag = list(dag)  # Make mutable in case it's a tuple.
    for i, node in enumerate(dag):
      node = list(node)  # Make mutable in case it's a tuple.
      module = node[0]
      if is_module(module):
        # Strip module from the dag.
        modules[module.name] = module
        # Replace with module name.
        node[0] = module.name
      dag[i] = node
    return dag, modules

  def call(self, inputs: TensorDict, **kwargs) -> tf.Tensor:
    """Run dag for an input dictionary."""
    return self.run_dag(inputs, **kwargs)

  @gin.configurable(whitelist=['verbose'])  # For debugging.
  def run_dag(self,
              inputs: TensorDict,
              verbose: bool = True,
              **kwargs) -> TensorDict:
    """Connects and runs submodules of dag.

    Args:
      inputs: A dictionary of input tensors fed to the dag.
      verbose: Print out dag routing when running.
      **kwargs: Other kwargs to pass to submodules, such as keras kwargs.

    Returns:
      A nested dictionary of all the output tensors.
    """
    # Initialize the outputs with inputs to the dag.
    outputs = {'inputs': inputs}
    # TODO(jesseengel): Remove this cluttering of the base namespace. Only there
    # for backwards compatability.
    outputs.update(inputs)

    # Run through the DAG nodes in sequential order.
    for node in self.dag:
      # The first element of the node can be either a module or module_key.
      module_key, input_keys = node[0], node[1]
      module = getattr(self, module_key)
      # Optionally specify output keys if module does not return dict.
      output_keys = node[2] if len(node) > 2 else None

      # Get the inputs to the node.
      inputs = [core.nested_lookup(key, outputs) for key in input_keys]

      if verbose:
        shape = lambda d: tf.nest.map_structure(lambda x: list(x.shape), d)
        logging.info('Input to Module: %s\nKeys: %s\nIn: %s\n',
                     module_key, input_keys, shape(inputs))

      # Duck typing to avoid dealing with multiple inheritance of Group modules.
      if is_processor(module):
        # Processor modules.
        module_outputs = module(*inputs, return_outputs_dict=True, **kwargs)
      elif is_loss(module):
        # Loss modules.
        module_outputs = module.get_losses_dict(*inputs, **kwargs)
      else:
        # Network modules.
        module_outputs = module(*inputs, **kwargs)

      if not isinstance(module_outputs, dict):
        module_outputs = core.to_dict(module_outputs, output_keys)

      if verbose:
        logging.info('Output from Module: %s\nOut: %s\n',
                     module_key, shape(module_outputs))

      # Add module outputs to the dictionary.
      outputs[module_key] = module_outputs

    # Alias final module output as dag output.
    # 'out' is a reserved key for final dag output.
    outputs['out'] = module_outputs

    return outputs
<commit_msg>Internal change<commit_after># Copyright 2021 The DDSP Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Lint as: python3
"""Library of functions and layers of Directed Acyclical Graphs.

DAGLayer exists as an alternative to manually specifying the forward pass in
python. The advantage is that a variety of configurations can be
programmatically specified via external dependency injection, such as with the
`gin` library.
"""

from typing import Dict, Sequence, Tuple, Text, TypeVar

from absl import logging
from ddsp import core
import gin
import tensorflow.compat.v2 as tf

tfkl = tf.keras.layers

# Define Types.
TensorDict = Dict[Text, tf.Tensor]
KeyOrModule = TypeVar('KeyOrModule', Text, tf.Module)
Node = Tuple[KeyOrModule, Sequence[Text], Sequence[Text]]
DAG = Sequence[Node]

# Helper Functions for DAGs  ---------------------------------------------------
filter_by_value = lambda d, cond: dict(filter(lambda e: cond(e[1]), d.items()))
is_module = lambda v: isinstance(v, tf.Module)

# Duck typing.
is_loss = lambda v: hasattr(v, 'get_losses_dict')
is_processor = lambda v: hasattr(v, 'get_signal') and hasattr(v, 'get_controls')

def split_keras_kwargs(kwargs):
  """Strip keras specific kwargs."""
  keras_kwargs = {}
  for key in ['training', 'mask', 'name']:
    if kwargs.get(key) is not None:
      keras_kwargs[key] = kwargs.pop(key)
  return keras_kwargs, kwargs

# DAG and ProcessorGroup Classes -----------------------------------------------
@gin.register
class DAGLayer(tfkl.Layer):
  """String modules together."""

  def __init__(self, dag: DAG, **kwarg_modules):
    """Constructor.

    Args:
      dag: A directed acyclical graph in the form of a list of nodes. Each node
        has the form

        ['module', ['input_key', ...], ['output_key', ...]]

        'module': Module instance or string name of module. For example,
          'encoder' woud access the attribute `dag_layer.encoder`.
        'input_key': List of strings, nested keys in dictionary of dag outputs.
          For example, 'inputs/f0_hz' would access `outputs[inputs]['f0_hz']`.
          Inputs to the dag are wrapped in a `inputs` dict as shown in the
          example. This list is ordered and has one key per a module input
          argument. Each node's outputs are prefixed by their module name.
        'output_key': List of strings, keys for each return value of the module.
          For example, ['amps', 'freqs'] would have the module return a dict
          {'module_name': {'amps': return_value_0, 'freqs': return_value_1}}.
          If the module returns a dictionary, the keys of the dictionary will be
          used and these values (if provided) will be ignored.

        The graph is read sequentially and must be topologically sorted. This
        means that all inputs for a module must already be generated by earlier
        modules (or in the input dictionary).
      **kwarg_modules: A series of modules to add to DAGLayer. Each kwarg that
        is a tf.Module will be added as a property of the layer, so that it will
        be accessible as `dag_layer.kwarg`. Also, other keras kwargs such as
        'name' are split off before adding modules.
    """
    keras_kwargs, kwarg_modules = split_keras_kwargs(kwarg_modules)
    super().__init__(**keras_kwargs)

    # Create properties/submodules from other kwargs.
    modules = filter_by_value(kwarg_modules, is_module)

    # Remove modules from the dag, make properties of dag_layer.
    dag, dag_modules = self.format_dag(dag)
    # DAG is now just strings.
    self.dag = dag
    modules.update(dag_modules)

    # Make as propreties of DAGLayer to keep track of variables in checkpoints.
    self.module_names = list(modules.keys())
    for module_name, module in modules.items():
      setattr(self, module_name, module)

  @property
  def modules(self):
    """Module getter."""
    return [getattr(self, name) for name in self.module_names]

  @staticmethod
  def format_dag(dag):
    """Remove modules from dag, and replace with module names."""
    modules = {}
    dag = list(dag)  # Make mutable in case it's a tuple.
    for i, node in enumerate(dag):
      node = list(node)  # Make mutable in case it's a tuple.
      module = node[0]
      if is_module(module):
        # Strip module from the dag.
        modules[module.name] = module
        # Replace with module name.
        node[0] = module.name
      dag[i] = node
    return dag, modules

  def call(self, inputs: TensorDict, **kwargs) -> tf.Tensor:
    """Run dag for an input dictionary."""
    return self.run_dag(inputs, **kwargs)

  @gin.configurable(allowlist=['verbose'])  # For debugging.
  def run_dag(self,
              inputs: TensorDict,
              verbose: bool = True,
              **kwargs) -> TensorDict:
    """Connects and runs submodules of dag.

    Args:
      inputs: A dictionary of input tensors fed to the dag.
      verbose: Print out dag routing when running.
      **kwargs: Other kwargs to pass to submodules, such as keras kwargs.

    Returns:
      A nested dictionary of all the output tensors.
    """
    # Initialize the outputs with inputs to the dag.
    outputs = {'inputs': inputs}
    # TODO(jesseengel): Remove this cluttering of the base namespace. Only there
    # for backwards compatability.
    outputs.update(inputs)

    # Run through the DAG nodes in sequential order.
    for node in self.dag:
      # The first element of the node can be either a module or module_key.
      module_key, input_keys = node[0], node[1]
      module = getattr(self, module_key)
      # Optionally specify output keys if module does not return dict.
      output_keys = node[2] if len(node) > 2 else None

      # Get the inputs to the node.
      inputs = [core.nested_lookup(key, outputs) for key in input_keys]

      if verbose:
        shape = lambda d: tf.nest.map_structure(lambda x: list(x.shape), d)
        logging.info('Input to Module: %s\nKeys: %s\nIn: %s\n',
                     module_key, input_keys, shape(inputs))

      # Duck typing to avoid dealing with multiple inheritance of Group modules.
      if is_processor(module):
        # Processor modules.
        module_outputs = module(*inputs, return_outputs_dict=True, **kwargs)
      elif is_loss(module):
        # Loss modules.
        module_outputs = module.get_losses_dict(*inputs, **kwargs)
      else:
        # Network modules.
        module_outputs = module(*inputs, **kwargs)

      if not isinstance(module_outputs, dict):
        module_outputs = core.to_dict(module_outputs, output_keys)

      if verbose:
        logging.info('Output from Module: %s\nOut: %s\n',
                     module_key, shape(module_outputs))

      # Add module outputs to the dictionary.
      outputs[module_key] = module_outputs

    # Alias final module output as dag output.
    # 'out' is a reserved key for final dag output.
    outputs['out'] = module_outputs

    return outputs
<|endoftext|>