<commit_before>
"""
    Implements a new custom Distutils command for handling library
    configuration.

    The "configure" command here doesn't directly affect things like
    config.pxi; rather, it exists to provide a set of attributes that are
    used by the build_ext replacement in setup_build.py.

    Options from the command line and environment variables are stored
    between invocations in a pickle file.  This allows configuring the library
    once and e.g. calling "build" and "test" without recompiling everything
    or explicitly providing the same options every time.

    This module also contains the auto-detection logic for figuring out
    the currently installed HDF5 version.
"""

import os
import os.path as op
import re
import sys
import json

def load_stashed_config():
    """ Load settings dict from the pickle file """
    try:
        with open('h5config.json', 'r') as f:
            cfg = json.load(f)
        if not isinstance(cfg, dict):
            raise TypeError
    except Exception:
        return {}
    return cfg

def stash_config(dct):
    """Save settings dict to the pickle file."""
    with open('h5config.json', 'w') as f:
        json.dump(dct, f)

def validate_version(s):
    """Ensure that s contains an X.Y.Z format version string, or ValueError."""
    m = re.match('(\d+)\.(\d+)\.(\d+)$', s)
    if m:
        return tuple(int(x) for x in m.groups())
    raise ValueError(f"HDF5 version string {s!r} not in X.Y.Z format")

def mpi_enabled():
    return os.environ.get('HDF5_MPI') == "ON"

class BuildConfig:
    def __init__(self, hdf5_includedirs, hdf5_libdirs, hdf5_define_macros,
                 hdf5_version, mpi, ros3, direct_vfd):
        self.hdf5_includedirs = hdf5_includedirs
        self.hdf5_libdirs = hdf5_libdirs
        self.hdf5_define_macros = hdf5_define_macros
        self.hdf5_version = hdf5_version
        self.mpi = mpi
        self.ros3 = ros3
        self.direct_vfd = direct_vfd

    @classmethod
    def from_env(cls):
        mpi = mpi_enabled()
        h5_inc, h5_lib, h5_macros = cls._find_hdf5_compiler_settings(mpi)

        h5_version_s = os.environ.get('HDF5_VERSION')
        h5py_ros3 = os.environ.get('H5PY_ROS3')
        h5py_direct_vfd = os.environ.get('H5PY_DIRECT_VFD')

        if h5_version_s and not mpi and h5py_ros3 and h5py_direct_vfd:
            # if we know config, don't use wrapper, it may not be supported
            return cls(
                h5_inc, h5_lib, h5_macros, validate_version(h5_version_s), mpi,
                h5py_ros3 == '1', h5py_direct_vfd == '1')

        h5_wrapper = HDF5LibWrapper(h5_lib)
        if h5_version_s:
            h5_version = validate_version(h5_version_s)
        else:
            h5_version = h5_wrapper.autodetect_version()
            if mpi and not h5_wrapper.has_mpi_support():
                raise RuntimeError("MPI support not detected")

        if h5py_ros3:
            ros3 = h5py_ros3 == '1'
        else:
            ros3 = h5_wrapper.has_ros3_support()

        if h5py_direct_vfd:
            direct_vfd = h5py_direct_vfd == '1'
        else:
            direct_vfd = h5_wrapper.has_direct_vfd_support()

        return cls(h5_inc, h5_lib, h5_macros, h5_version, mpi, ros3, direct_vfd)

    @staticmethod
    def _find_hdf5_compiler_settings(mpi=False):
        """Get compiler settings from environment or pkgconfig.

        Returns (include_dirs, lib_dirs, define_macros)
        """
        hdf5 = os.environ.get('HDF5_DIR')
        hdf5_includedir = os.environ.get('HDF5_INCLUDEDIR')
        hdf5_libdir = os.environ.get('HDF5_LIBDIR')
        hdf5_pkgconfig_name = os.environ.get('HDF5_PKGCONFIG_NAME')

        if sum([
            bool(hdf5_includedir or hdf5_libdir),
            bool(hdf5),
            bool(hdf5_pkgconfig_name)
        ]) > 1:
            raise ValueError(
                "Specify only one of: HDF5 lib/include dirs, HDF5 prefix dir, "
                "or HDF5 pkgconfig name"
            )

        if hdf5_includedir or hdf5_libdir:
            inc_dirs = [hdf5_includedir] if hdf5_includedir else []
            lib_dirs = [hdf5_libdir] if hdf5_libdir else []
            return (inc_dirs, lib_dirs, [])

        # Specified a prefix dir (e.g. '/usr/local')
        if hdf5:
            inc_dirs = [op.join(hdf5, 'include')]
            lib_dirs = [op.join(hdf5, 'lib')]
            if sys.platform.startswith('win'):
                lib_dirs.append(op.join(hdf5, 'bin'))
            return (inc_dirs, lib_dirs, [])

        # Specified a name to be looked up in pkgconfig
        if hdf5_pkgconfig_name:
            import pkgconfig
            if not pkgconfig.exists(hdf5_pkgconfig_name):
                raise ValueError(
                    f"No pkgconfig information for {hdf5_pkgconfig_name}"
                )
            pc = pkgconfig.parse(hdf5_pkgconfig_name)
            return (pc['include_dirs'], pc['library_dirs'], pc['define_macros'])

        # Fallback: query pkgconfig for default hdf5 names
        import pkgconfig
        pc_name = 'hdf5-openmpi' if mpi else 'hdf5'
        pc = {}
        try:
            if pkgconfig.exists(pc_name):
                pc = pkgconfig.parse(pc_name)
        except EnvironmentError:
            if os.name != 'nt':
                print(
                    "Building h5py requires pkg-config unless the HDF5 path "
                    "is explicitly specified using the environment variable HDF5_DIR. "
                    "For more information and details, "
                    "see https://docs.h5py.org/en/stable/build.html#custom-installation", file=sys.stderr
                )
                raise

        return (
            pc.get('include_dirs', []),
            pc.get('library_dirs', []),
            pc.get('define_macros', []),
        )

    def as_dict(self):
        return {
            'hdf5_includedirs': self.hdf5_includedirs,
            'hdf5_libdirs': self.hdf5_libdirs,
            'hdf5_define_macros': self.hdf5_define_macros,
            'hdf5_version': list(self.hdf5_version),  # list() to match the JSON
            'mpi': self.mpi,
            'ros3': self.ros3,
            'direct_vfd': self.direct_vfd,
        }

    def changed(self):
        """Has the config changed since the last build?"""
        return self.as_dict() != load_stashed_config()

    def record_built(self):
        """Record config after a successful build"""
        stash_config(self.as_dict())

    def summarise(self):
        def fmt_dirs(l):
            return '\n'.join((['['] + [f'  {d!r}' for d in l] + [']'])) if l else '[]'

        print('*' * 80)
        print(' ' * 23 + "Summary of the h5py configuration")
        print('')
        print("HDF5 include dirs:", fmt_dirs(self.hdf5_includedirs))
        print("HDF5 library dirs:", fmt_dirs(self.hdf5_libdirs))
        print("     HDF5 Version:", repr(self.hdf5_version))
        print("      MPI Enabled:", self.mpi)
        print(" ROS3 VFD Enabled:", self.ros3)
        print("DIRECT VFD Enabled:", self.direct_vfd)
        print(" Rebuild Required:", self.changed())
        print('')
        print('*' * 80)

class HDF5LibWrapper:

    def __init__(self, libdirs):
        self._load_hdf5_lib(libdirs)

    def _load_hdf5_lib(self, libdirs):
        """
        Detect and load the HDF5 library.

        Raises an exception if anything goes wrong.

        libdirs: the library paths to search for the library
        """
        import ctypes

        # extra keyword args to pass to LoadLibrary
        load_kw = {}
        if sys.platform.startswith('darwin'):
            default_path = 'libhdf5.dylib'
            regexp = re.compile(r'^libhdf5.dylib')
        elif sys.platform.startswith('win'):
            default_path = 'hdf5.dll'
            regexp = re.compile(r'^hdf5.dll')
            if sys.version_info >= (3, 8):
                # To overcome "difficulty" loading the library on windows
                # https://bugs.python.org/issue42114
                load_kw['winmode'] = 0
        elif sys.platform.startswith('cygwin'):
            default_path = 'cyghdf5-200.dll'
            regexp = re.compile(r'^cyghdf5-\d+.dll$')
        else:
            default_path = 'libhdf5.so'
            regexp = re.compile(r'^libhdf5.so')

        path = None
        for d in libdirs:
            try:
                candidates = [x for x in os.listdir(d) if regexp.match(x)]
            except Exception:
                continue   # Skip invalid entries

            if len(candidates) != 0:
                candidates.sort(key=lambda x: len(x))   # Prefer libfoo.so to libfoo.so.X.Y.Z
                path = op.abspath(op.join(d, candidates[0]))
                break

        if path is None:
            path = default_path

        print("Loading library to get build settings and version:", path)

        self._lib_path = path

        if op.isabs(path) and not op.exists(path):
            raise FileNotFoundError(f"{path} is missing")

        try:
            lib = ctypes.CDLL(path, **load_kw)
        except Exception:
            print("error: Unable to load dependency HDF5, make sure HDF5 is installed properly")
            raise

        self._lib = lib

    def autodetect_version(self):
        """
        Detect the current version of HDF5, and return X.Y.Z version string.

        Raises an exception if anything goes wrong.
        """
        import ctypes
        from ctypes import byref

        major = ctypes.c_uint()
        minor = ctypes.c_uint()
        release = ctypes.c_uint()

        try:
            self._lib.H5get_libversion(byref(major), byref(minor), byref(release))
        except Exception:
            print("error: Unable to find HDF5 version")
            raise

        return int(major.value), int(minor.value), int(release.value)

    def load_function(self, func_name):
        try:
            return getattr(self._lib, func_name)
        except AttributeError:
            # No such function
            return None

    def has_functions(self, *func_names):
        for func_name in func_names:
            if self.load_function(func_name) is None:
                return False
        return True

    def has_mpi_support(self):
        return self.has_functions("H5Pget_fapl_mpio", "H5Pset_fapl_mpio")

    def has_ros3_support(self):
        return self.has_functions("H5Pget_fapl_ros3", "H5Pset_fapl_ros3")

    def has_direct_vfd_support(self):
        return self.has_functions("H5Pget_fapl_direct", "H5Pset_fapl_direct")
<commit_msg>setup: Fix hdf5 dll search in mingw<commit_after>
"""
    Implements a new custom Distutils command for handling library
    configuration.

    The "configure" command here doesn't directly affect things like
    config.pxi; rather, it exists to provide a set of attributes that are
    used by the build_ext replacement in setup_build.py.

    Options from the command line and environment variables are stored
    between invocations in a pickle file.  This allows configuring the library
    once and e.g. calling "build" and "test" without recompiling everything
    or explicitly providing the same options every time.

    This module also contains the auto-detection logic for figuring out
    the currently installed HDF5 version.
"""

import os
import os.path as op
import re
import sys
import json

def load_stashed_config():
    """ Load settings dict from the pickle file """
    try:
        with open('h5config.json', 'r') as f:
            cfg = json.load(f)
        if not isinstance(cfg, dict):
            raise TypeError
    except Exception:
        return {}
    return cfg

def stash_config(dct):
    """Save settings dict to the pickle file."""
    with open('h5config.json', 'w') as f:
        json.dump(dct, f)

def validate_version(s):
    """Ensure that s contains an X.Y.Z format version string, or ValueError."""
    m = re.match('(\d+)\.(\d+)\.(\d+)$', s)
    if m:
        return tuple(int(x) for x in m.groups())
    raise ValueError(f"HDF5 version string {s!r} not in X.Y.Z format")

def mpi_enabled():
    return os.environ.get('HDF5_MPI') == "ON"

class BuildConfig:
    def __init__(self, hdf5_includedirs, hdf5_libdirs, hdf5_define_macros,
                 hdf5_version, mpi, ros3, direct_vfd):
        self.hdf5_includedirs = hdf5_includedirs
        self.hdf5_libdirs = hdf5_libdirs
        self.hdf5_define_macros = hdf5_define_macros
        self.hdf5_version = hdf5_version
        self.mpi = mpi
        self.ros3 = ros3
        self.direct_vfd = direct_vfd

    @classmethod
    def from_env(cls):
        mpi = mpi_enabled()
        h5_inc, h5_lib, h5_macros = cls._find_hdf5_compiler_settings(mpi)

        h5_version_s = os.environ.get('HDF5_VERSION')
        h5py_ros3 = os.environ.get('H5PY_ROS3')
        h5py_direct_vfd = os.environ.get('H5PY_DIRECT_VFD')

        if h5_version_s and not mpi and h5py_ros3 and h5py_direct_vfd:
            # if we know config, don't use wrapper, it may not be supported
            return cls(
                h5_inc, h5_lib, h5_macros, validate_version(h5_version_s), mpi,
                h5py_ros3 == '1', h5py_direct_vfd == '1')

        h5_wrapper = HDF5LibWrapper(h5_lib)
        if h5_version_s:
            h5_version = validate_version(h5_version_s)
        else:
            h5_version = h5_wrapper.autodetect_version()
            if mpi and not h5_wrapper.has_mpi_support():
                raise RuntimeError("MPI support not detected")

        if h5py_ros3:
            ros3 = h5py_ros3 == '1'
        else:
            ros3 = h5_wrapper.has_ros3_support()

        if h5py_direct_vfd:
            direct_vfd = h5py_direct_vfd == '1'
        else:
            direct_vfd = h5_wrapper.has_direct_vfd_support()

        return cls(h5_inc, h5_lib, h5_macros, h5_version, mpi, ros3, direct_vfd)

    @staticmethod
    def _find_hdf5_compiler_settings(mpi=False):
        """Get compiler settings from environment or pkgconfig.

        Returns (include_dirs, lib_dirs, define_macros)
        """
        hdf5 = os.environ.get('HDF5_DIR')
        hdf5_includedir = os.environ.get('HDF5_INCLUDEDIR')
        hdf5_libdir = os.environ.get('HDF5_LIBDIR')
        hdf5_pkgconfig_name = os.environ.get('HDF5_PKGCONFIG_NAME')

        if sum([
            bool(hdf5_includedir or hdf5_libdir),
            bool(hdf5),
            bool(hdf5_pkgconfig_name)
        ]) > 1:
            raise ValueError(
                "Specify only one of: HDF5 lib/include dirs, HDF5 prefix dir, "
                "or HDF5 pkgconfig name"
            )

        if hdf5_includedir or hdf5_libdir:
            inc_dirs = [hdf5_includedir] if hdf5_includedir else []
            lib_dirs = [hdf5_libdir] if hdf5_libdir else []
            return (inc_dirs, lib_dirs, [])

        # Specified a prefix dir (e.g. '/usr/local')
        if hdf5:
            inc_dirs = [op.join(hdf5, 'include')]
            lib_dirs = [op.join(hdf5, 'lib')]
            if sys.platform.startswith('win'):
                lib_dirs.append(op.join(hdf5, 'bin'))
            return (inc_dirs, lib_dirs, [])

        # Specified a name to be looked up in pkgconfig
        if hdf5_pkgconfig_name:
            import pkgconfig
            if not pkgconfig.exists(hdf5_pkgconfig_name):
                raise ValueError(
                    f"No pkgconfig information for {hdf5_pkgconfig_name}"
                )
            pc = pkgconfig.parse(hdf5_pkgconfig_name)
            return (pc['include_dirs'], pc['library_dirs'], pc['define_macros'])

        # Fallback: query pkgconfig for default hdf5 names
        import pkgconfig
        pc_name = 'hdf5-openmpi' if mpi else 'hdf5'
        pc = {}
        try:
            if pkgconfig.exists(pc_name):
                pc = pkgconfig.parse(pc_name)
        except EnvironmentError:
            if os.name != 'nt':
                print(
                    "Building h5py requires pkg-config unless the HDF5 path "
                    "is explicitly specified using the environment variable HDF5_DIR. "
                    "For more information and details, "
                    "see https://docs.h5py.org/en/stable/build.html#custom-installation", file=sys.stderr
                )
                raise

        return (
            pc.get('include_dirs', []),
            pc.get('library_dirs', []),
            pc.get('define_macros', []),
        )

    def as_dict(self):
        return {
            'hdf5_includedirs': self.hdf5_includedirs,
            'hdf5_libdirs': self.hdf5_libdirs,
            'hdf5_define_macros': self.hdf5_define_macros,
            'hdf5_version': list(self.hdf5_version),  # list() to match the JSON
            'mpi': self.mpi,
            'ros3': self.ros3,
            'direct_vfd': self.direct_vfd,
        }

    def changed(self):
        """Has the config changed since the last build?"""
        return self.as_dict() != load_stashed_config()

    def record_built(self):
        """Record config after a successful build"""
        stash_config(self.as_dict())

    def summarise(self):
        def fmt_dirs(l):
            return '\n'.join((['['] + [f'  {d!r}' for d in l] + [']'])) if l else '[]'

        print('*' * 80)
        print(' ' * 23 + "Summary of the h5py configuration")
        print('')
        print("HDF5 include dirs:", fmt_dirs(self.hdf5_includedirs))
        print("HDF5 library dirs:", fmt_dirs(self.hdf5_libdirs))
        print("     HDF5 Version:", repr(self.hdf5_version))
        print("      MPI Enabled:", self.mpi)
        print(" ROS3 VFD Enabled:", self.ros3)
        print("DIRECT VFD Enabled:", self.direct_vfd)
        print(" Rebuild Required:", self.changed())
        print('')
        print('*' * 80)

class HDF5LibWrapper:

    def __init__(self, libdirs):
        self._load_hdf5_lib(libdirs)

    def _load_hdf5_lib(self, libdirs):
        """
        Detect and load the HDF5 library.

        Raises an exception if anything goes wrong.

        libdirs: the library paths to search for the library
        """
        import ctypes

        # extra keyword args to pass to LoadLibrary
        load_kw = {}
        if sys.platform.startswith('darwin'):
            default_path = 'libhdf5.dylib'
            regexp = re.compile(r'^libhdf5.dylib')
        elif sys.platform.startswith('win'):
            if 'MSC' in sys.version:
                default_path = 'hdf5.dll'
                regexp = re.compile(r'^hdf5.dll')
            else:
                default_path = 'libhdf5-0.dll'
                regexp = re.compile(r'^libhdf5-[0-9].dll')
            if sys.version_info >= (3, 8):
                # To overcome "difficulty" loading the library on windows
                # https://bugs.python.org/issue42114
                load_kw['winmode'] = 0
        elif sys.platform.startswith('cygwin'):
            default_path = 'cyghdf5-200.dll'
            regexp = re.compile(r'^cyghdf5-\d+.dll$')
        else:
            default_path = 'libhdf5.so'
            regexp = re.compile(r'^libhdf5.so')

        path = None
        for d in libdirs:
            try:
                candidates = [x for x in os.listdir(d) if regexp.match(x)]
            except Exception:
                continue   # Skip invalid entries

            if len(candidates) != 0:
                candidates.sort(key=lambda x: len(x))   # Prefer libfoo.so to libfoo.so.X.Y.Z
                path = op.abspath(op.join(d, candidates[0]))
                break

        if path is None:
            path = default_path

        print("Loading library to get build settings and version:", path)

        self._lib_path = path

        if op.isabs(path) and not op.exists(path):
            raise FileNotFoundError(f"{path} is missing")

        try:
            lib = ctypes.CDLL(path, **load_kw)
        except Exception:
            print("error: Unable to load dependency HDF5, make sure HDF5 is installed properly")
            raise

        self._lib = lib

    def autodetect_version(self):
        """
        Detect the current version of HDF5, and return X.Y.Z version string.

        Raises an exception if anything goes wrong.
        """
        import ctypes
        from ctypes import byref

        major = ctypes.c_uint()
        minor = ctypes.c_uint()
        release = ctypes.c_uint()

        try:
            self._lib.H5get_libversion(byref(major), byref(minor), byref(release))
        except Exception:
            print("error: Unable to find HDF5 version")
            raise

        return int(major.value), int(minor.value), int(release.value)

    def load_function(self, func_name):
        try:
            return getattr(self._lib, func_name)
        except AttributeError:
            # No such function
            return None

    def has_functions(self, *func_names):
        for func_name in func_names:
            if self.load_function(func_name) is None:
                return False
        return True

    def has_mpi_support(self):
        return self.has_functions("H5Pget_fapl_mpio", "H5Pset_fapl_mpio")

    def has_ros3_support(self):
        return self.has_functions("H5Pget_fapl_ros3", "H5Pset_fapl_ros3")

    def has_direct_vfd_support(self):
        return self.has_functions("H5Pget_fapl_direct", "H5Pset_fapl_direct")
<|endoftext|>


<commit_before># standard python modules
import re

# extensions
import vtk
import numpy

from icqBaseShape import BaseShape

class CompositeShape(BaseShape):

  def __init__(self,):
    """
    Constructor
    """
    BaseShape.__init__(self,)

    self.expression = ''
    self.argShapes = []

  def assemble(self, expression, shapes):
    """
    Assemble shapes into a composite object
    @param expression expression of the shape objects, $0 for shape 0, $1 for shape 1, etc.
           operations include + (union), * (intersection), and - (removal)
    """
    self.expression = expression
    self.argShapes = shapes
    # replace $1 by self.argShape[0]
    for i in range(len(self.argShapes)):
      self.expression = re.sub('\${0}'.format(i),
        'self.argShapes[{0}].evaluate(pts)'.format(i), self.expression)

  def evaluate(self, pts):
    """
    Evaluate the inside/outside function
    @param pts array of points
    """
    return eval(self.expression)

  def computeSurfaceMeshes(self, maxTriArea):
    """
    Compute the surface meshes
    @param maxTriArea maximum triangle area
    """

    # get the bounds
    loBound, hiBound = self._getBounds(maxTriArea)

    # get the optimal number of cells in each direction
    nx, ny, nz = self._getUniformGridResolution(loBound, hiBound, maxTriArea)
    nx1, ny1, nz1 = nx + 1, ny + 1, nz + 1

    x = loBound[0] + (hiBound[0] - loBound[0])*numpy.array([i for i in range(nx1)])/nx
    y = loBound[1] + (hiBound[1] - loBound[1])*numpy.array([j for j in range(ny1)])/ny
    z = loBound[2] + (hiBound[2] - loBound[2])*numpy.array([i for k in range(nz1)])/nz
    xxx = numpy.outer(numpy.ones((nz1, ny1)), x)
    yyy = numpy.outer(numpy.outer(numpy.ones((nz1,)), y), numpy.ones((nx1,)))
    zzz = numpy.outer(z, numpy.ones((ny1, nx1)))
    verts = numpy.zeros( (nz1*ny1*nx1, 3), numpy.float64 )
    verts[:, 0] = xxx.flat
    verts[:, 1] = yyy.flat
    verts[:, 2] = zzz.flat

    insideVals = self.evaluate(verts)

    # build the VTK contour pipeline
    xyz = vtk.vtkDoubleArray()
    pts = vtk.vtkPoints()
    data = vtk.vtkDoubleArray()
    grd = vtk.vtkStructuredGrid()
    cont = vtk.vtkContourFilter()

    numPoints = nx1*ny1*nz1
    xyz.SetNumberOfTuples(numPoints)
    xyz.SetNumberOfComponents(3)
    xyz.SetVoidArray(verts, numPoints*3, 1)

    pts.SetData(xyz)

    data.SetNumberOfTuples(numPoints)
    data.SetNumberOfComponents(1)
    data.SetVoidArray(insideVals, numPoints*3, 1)

    grd.SetDimensions(nz1, ny1, nx1)
    grd.SetPoints(pts)
    grd.GetPointData().SetScalars(data)

    if vtk.VTK_MAJOR_VERSION >= 6:
      cont.SetInputData(grd)
    else:
      cont.SetInput(grd)
    cont.SetValue(0, 0.0) # or should it be 0.5?
    cont.Update()

    surfPolyData = cont.GetOutput()

    # gather the boundary points
    surfPoints = surfPolyData.GetPoints()
    numSurfPoints = surfPoints.GetNumberOfPoints()
    self.points = numpy.zeros( (numSurfPoints, 3), numpy.float64 )
    for i in range(numSurfPoints):
      self.points[i, :] = surfPoints.GetPoint(i)

    # gather the boundary triangles
    numSurfCells = surfPolyData.GetNumberOfCells()
    cellArr = numpy.zeros( (numSurfCells, 3), numpy.int )
    for i in range(numSurfCells):
      cell = surfPolyData.GetCell(i)
      ptIds = cell.GetPointIds()
      npts = ptIds.GetNumberOfIds()
      pInds = [0 for j in range(npts)]
      for j in range(npts):
        pInds[j] = int(ptIds.GetId(j))
      cellArr[i, :] = pInds

    # single surface mesh in this case
    self.surfaceMeshes = [cellArr]

  def _getBounds(self, maxTriArea):
    """
    Compute the min/max corner points
    @param maxTriArea may need to compute the individual shape surface meshes and use maxTriArea
                      to do so
    """
    loBound = numpy.array([float('inf')] * 3)
    hiBound = numpy.array([-float('inf')] * 3)
    for shp in self.argShapes:
      points = shp.getPoints()
      if len(points) == 0:
        shp.computeSurfaceMeshes(maxTriArea=maxTriArea)
        points = shp.getPoints()
      loBound[0] = min(loBound[0], points[:, 0].min())
      loBound[1] = min(loBound[1], points[:, 1].min())
      loBound[2] = min(loBound[2], points[:, 2].min())
      hiBound[0] = max(hiBound[0], points[:, 0].min())
      hiBound[1] = max(hiBound[1], points[:, 1].min())
      hiBound[2] = max(hiBound[2], points[:, 2].min())
    return loBound, hiBound

  def _getUniformGridResolution(self, loBound, hiBound, maxTriArea):
    """
    Get the uniform grid's resolution
    @param loBound low corner of the grid
    @param hiBound high corner of the grid
    @param maxTriArea maximum surface triangle area
    """
    sr3 = numpy.sqrt(3.)
    nx = max(1, int(sr3*(hiBound[0] - loBound[0])**2/maxTriArea + 0.5))
    ny = max(1, int(sr3*(hiBound[1] - loBound[1])**2/maxTriArea + 0.5))
    nz = max(1, int(sr3*(hiBound[2] - loBound[2])**2/maxTriArea + 0.5))
    return nx, ny, nz

################################################################################
def test():

  from icqPrimitiveShape import PrimitiveShape
  from numpy import sin, cos, pi

  # create first sphere
  radius1 = 0.5
  origin1 = numpy.array([0.1, 0.2, 0.3])
  surfaceFunctions1 = [(lambda u,v: radius1*sin(pi*u)*cos(2*pi*v) + origin1[0],
                       lambda u,v: radius1*sin(pi*u)*sin(2*pi*v) + origin1[1],
                       lambda u,v: radius1*cos(pi*u) + origin1[2])]
  def evalFunction1(pts):
    xNorm = pts[:, 0] - origin1[0]
    yNorm = pts[:, 1] - origin1[1]
    zNorm = pts[:, 2] - origin1[2]
    return radius1*radius1 - xNorm**2 - yNorm**2 - zNorm**2 > 0

  s1 = PrimitiveShape()
  s1.setSurfaceFunctions(surfaceFunctions1)
  s1.setEvaluateFunction(evalFunction1)
  s1.computeSurfaceMeshes(maxTriArea=0.1)

  # create second sphere
  radius2 = 0.5
  origin2 = numpy.array([0.4, 0.5, 0.6])
  surfaceFunctions2 = [(lambda u,v: radius2*sin(pi*u)*cos(2*pi*v) + origin2[0],
                       lambda u,v: radius2*sin(pi*u)*sin(2*pi*v) + origin2[1],
                       lambda u,v: radius2*cos(pi*u) + origin2[2])]
  def evalFunction2(pts):
    xNorm = pts[:, 0] - origin2[0]
    yNorm = pts[:, 1] - origin2[1]
    zNorm = pts[:, 2] - origin2[2]
    return radius2*radius2 - xNorm**2 - yNorm**2 - zNorm**2 > 0

  # create another sphere
  s2 = PrimitiveShape()
  s2.setSurfaceFunctions(surfaceFunctions2)
  s2.setEvaluateFunction(evalFunction2)
  s2.computeSurfaceMeshes(maxTriArea=0.1)

  # assemble
  cs = CompositeShape()
  cs.assemble('$0 + $1', (s1, s2))
  cs.computeSurfaceMeshes(maxTriArea=0.1)
  cs.save('testCompositeShape.vtk')

if __name__ == '__main__': test()
<commit_msg> working on the composite object<commit_after># standard python modules
import re

# extensions
import vtk
import numpy

from icqBaseShape import BaseShape

class CompositeShape(BaseShape):

  def __init__(self,):
    """
    Constructor
    """
    BaseShape.__init__(self,)

    self.expression = ''
    self.argShapes = []

  def assemble(self, expression, shapes):
    """
    Assemble shapes into a composite object
    @param expression expression of the shape objects, $0 for shape 0, $1 for shape 1, etc.
           operations include + (union), * (intersection), and - (removal)
    """
    self.expression = expression
    self.argShapes = shapes
    # replace $1 by self.argShape[0]
    for i in range(len(self.argShapes)):
      self.expression = re.sub('\${0}'.format(i),
        'self.argShapes[{0}].evaluate(pts)'.format(i), self.expression)

  def evaluate(self, pts):
    """
    Evaluate the inside/outside function
    @param pts array of points
    """
    return eval(self.expression)

  def computeSurfaceMeshes(self, maxTriArea):
    """
    Compute the surface meshes
    @param maxTriArea maximum triangle area
    """

    # compute cell normals
    self.computeSurfaceNormals()

    #
    # find all the points inside the volume
    #

    # move point out by a small amount to avoid floating point comparision
    # issues
    eps = 1.e-6

    numFaces = len(self.surfaceMeshes)
    insidePointList = []
    totalNumPoints = 0
    loBound = numpy.array([float('inf')] * 3)
    hiBound = numpy.array([-float('inf')] * 3)
    for iFace in range(numFaces):
      face = self.surfaceMeshes[iFace]
      normals = self.surfaceNormals[iFace]
      verts = self.points[face]
      displVerts = verts + eps*normals
      inPts = verts[self.evaluate(displVerts) > 0]
      totalNumPoints += inPts.shape[0]
      insidePointList.append(inPts)
      xs, ys, zs = inPts[:, 0], inPts[:, 1], inPts[:, 2]
      loBound[0] = min(loBound[0], xs.min())
      loBound[1] = min(loBound[1], ys.min())
      loBound[2] = min(loBound[2], zs.min())
      hiBound[0] = max(hiBound[0], xs.max())
      hiBound[1] = max(hiBound[1], ys.max())
      hiBound[2] = max(hiBound[2], zs.max())

    # add more points by sampling the volume containing the boundary surfaces
    cellLength = numpy.sqrt(2*maxTriArea)
    ns = numpy.array([ int( (hiBound[i] - loBound[i])/cellLength + 0.5) for i in range(3) ])
    deltas = (hiBound - loBound)/ns
    xs = numpy.array([loBound[0] + i*deltas[0] for i in range(ns[0])])
    ys = numpy.array([loBound[1] + j*deltas[1] for j in range(ns[1])])
    zs = numpy.array([loBound[2] + k*deltas[2] for k in range(ns[2])])
    xx, yy, zz = RectilinearGrid(xs, ys, zs)
    npts = len(xx.flat)
    gridPts = numpy.zeros( (npts, 3), numpy.float64 )
    gridPts[:, 0] = xx.flat
    gridPts[:, 1] = yy.flat
    gridPts[:, 2] = zz.flat

    # only add the points inside
    gridPts = gridPts[ self.evaluate(gridPts) > 0 ]
    totalNumPoints += gridPts.shape[0]
    insidePointList.append(gridPts)

    # create a single list of points
    insidePoints = numpy.zeros( (totalNumPoints, 3), numpy.float64 )
    iBeg = 0
    for pts in insidePointList:
      iEnd = iBeg + pts.shape[0]
      insidePoints[iBeg:iEnd, :] = pts
      iBeg = iEnd

    # tessellate
    tri = Triangulation()
    tri.setInputPoints(insidePoints)
    tri.triangulate()

    # apply filter to extract boundary cell faces
    pdata = tri.delny
    surf = vtk.vtkGeometryFilter()
    if vtk.VTK_MAJOR_VERSION >= 6:
      surf.SetInputData(pdata)
    else:
      surf.SetInput(pdata)
    surf.Update()

    # get the boundary cells
    # copy all the surface meshes into a single connectivity array
    numCells = surf.GetNumberOfCells()
    cellArr = numpy.zeros( (numCells, 3), numpy.int )
    for i in range(numCells):
      cell = surf.GetCell(i)
      ptIds = cell.GetPointIds()
      npts = ptIds.GetNumberOfIds()
      pInds = [0 for j in range(npts)]
      for j in range(npts):
        pInds[j] = int(ptIds.GetId(j))
      cellArr[i, :] = pInds

    # single surface in the case of a composite object
    self.surfaceMeshes = [cellArr]

  def computeSurfaceMeshes2(self, maxTriArea):
    """
    Compute the surface meshes
    @param maxTriArea maximum triangle area
    """

    # get the bounds
    loBound, hiBound = self._getBounds(maxTriArea)

    # get the optimal number of cells in each direction
    nx, ny, nz = self._getUniformGridResolution(loBound, hiBound, maxTriArea)
    nx1, ny1, nz1 = nx + 1, ny + 1, nz + 1

    x = loBound[0] + (hiBound[0] - loBound[0])*numpy.array([i for i in range(nx1)])/nx
    y = loBound[1] + (hiBound[1] - loBound[1])*numpy.array([j for j in range(ny1)])/ny
    z = loBound[2] + (hiBound[2] - loBound[2])*numpy.array([i for k in range(nz1)])/nz
    xxx = numpy.outer(numpy.ones((nz1, ny1)), x)
    yyy = numpy.outer(numpy.outer(numpy.ones((nz1,)), y), numpy.ones((nx1,)))
    zzz = numpy.outer(z, numpy.ones((ny1, nx1)))
    verts = numpy.zeros( (nz1*ny1*nx1, 3), numpy.float64 )
    verts[:, 0] = xxx.flat
    verts[:, 1] = yyy.flat
    verts[:, 2] = zzz.flat

    insideVals = self.evaluate(verts)

    # build the VTK contour pipeline
    xyz = vtk.vtkDoubleArray()
    pts = vtk.vtkPoints()
    data = vtk.vtkDoubleArray()
    grd = vtk.vtkStructuredGrid()
    cont = vtk.vtkContourFilter()

    numPoints = nx1*ny1*nz1
    xyz.SetNumberOfTuples(numPoints)
    xyz.SetNumberOfComponents(3)
    xyz.SetVoidArray(verts, numPoints*3, 1)

    pts.SetData(xyz)

    data.SetNumberOfTuples(numPoints)
    data.SetNumberOfComponents(1)
    data.SetVoidArray(insideVals, numPoints*3, 1)

    grd.SetDimensions(nz1, ny1, nx1)
    grd.SetPoints(pts)
    grd.GetPointData().SetScalars(data)

    if vtk.VTK_MAJOR_VERSION >= 6:
      cont.SetInputData(grd)
    else:
      cont.SetInput(grd)
    cont.SetValue(0, 0.0) # or should it be 0.5?
    cont.Update()

    surfPolyData = cont.GetOutput()

    # gather the boundary points
    surfPoints = surfPolyData.GetPoints()
    numSurfPoints = surfPoints.GetNumberOfPoints()
    self.points = numpy.zeros( (numSurfPoints, 3), numpy.float64 )
    for i in range(numSurfPoints):
      self.points[i, :] = surfPoints.GetPoint(i)

    # gather the boundary triangles
    numSurfCells = surfPolyData.GetNumberOfCells()
    cellArr = numpy.zeros( (numSurfCells, 3), numpy.int )
    for i in range(numSurfCells):
      cell = surfPolyData.GetCell(i)
      ptIds = cell.GetPointIds()
      npts = ptIds.GetNumberOfIds()
      pInds = [0 for j in range(npts)]
      for j in range(npts):
        pInds[j] = int(ptIds.GetId(j))
      cellArr[i, :] = pInds

    # single surface mesh in this case
    self.surfaceMeshes = [cellArr]

  def _getBounds(self, maxTriArea):
    """
    Compute the min/max corner points
    @param maxTriArea may need to compute the individual shape surface meshes and use maxTriArea
                      to do so
    """
    loBound = numpy.array([float('inf')] * 3)
    hiBound = numpy.array([-float('inf')] * 3)
    for shp in self.argShapes:
      points = shp.getPoints()
      if len(points) == 0:
        shp.computeSurfaceMeshes(maxTriArea=maxTriArea)
        points = shp.getPoints()
      loBound[0] = min(loBound[0], points[:, 0].min())
      loBound[1] = min(loBound[1], points[:, 1].min())
      loBound[2] = min(loBound[2], points[:, 2].min())
      hiBound[0] = max(hiBound[0], points[:, 0].min())
      hiBound[1] = max(hiBound[1], points[:, 1].min())
      hiBound[2] = max(hiBound[2], points[:, 2].min())
    return loBound, hiBound

  def _getUniformGridResolution(self, loBound, hiBound, maxTriArea):
    """
    Get the uniform grid's resolution
    @param loBound low corner of the grid
    @param hiBound high corner of the grid
    @param maxTriArea maximum surface triangle area
    """
    sr3 = numpy.sqrt(3.)
    nx = max(1, int(sr3*(hiBound[0] - loBound[0])**2/maxTriArea + 0.5))
    ny = max(1, int(sr3*(hiBound[1] - loBound[1])**2/maxTriArea + 0.5))
    nz = max(1, int(sr3*(hiBound[2] - loBound[2])**2/maxTriArea + 0.5))
    return nx, ny, nz

################################################################################
def test():

  from icqPrimitiveShape import PrimitiveShape
  from numpy import sin, cos, pi

  # create first sphere
  radius1 = 0.5
  origin1 = numpy.array([0.1, 0.2, 0.3])
  surfaceFunctions1 = [(lambda u,v: radius1*sin(pi*u)*cos(2*pi*v) + origin1[0],
                       lambda u,v: radius1*sin(pi*u)*sin(2*pi*v) + origin1[1],
                       lambda u,v: radius1*cos(pi*u) + origin1[2])]
  def evalFunction1(pts):
    xNorm = pts[:, 0] - origin1[0]
    yNorm = pts[:, 1] - origin1[1]
    zNorm = pts[:, 2] - origin1[2]
    return radius1*radius1 - xNorm**2 - yNorm**2 - zNorm**2 > 0

  s1 = PrimitiveShape()
  s1.setSurfaceFunctions(surfaceFunctions1)
  s1.setEvaluateFunction(evalFunction1)
  s1.computeSurfaceMeshes(maxTriArea=0.1)

  # create second sphere
  radius2 = 0.5
  origin2 = numpy.array([0.4, 0.5, 0.6])
  surfaceFunctions2 = [(lambda u,v: radius2*sin(pi*u)*cos(2*pi*v) + origin2[0],
                       lambda u,v: radius2*sin(pi*u)*sin(2*pi*v) + origin2[1],
                       lambda u,v: radius2*cos(pi*u) + origin2[2])]
  def evalFunction2(pts):
    xNorm = pts[:, 0] - origin2[0]
    yNorm = pts[:, 1] - origin2[1]
    zNorm = pts[:, 2] - origin2[2]
    return radius2*radius2 - xNorm**2 - yNorm**2 - zNorm**2 > 0

  # create another sphere
  s2 = PrimitiveShape()
  s2.setSurfaceFunctions(surfaceFunctions2)
  s2.setEvaluateFunction(evalFunction2)
  s2.computeSurfaceMeshes(maxTriArea=0.1)

  # assemble
  cs = CompositeShape()
  cs.assemble('$0 + $1', (s1, s2))
  cs.computeSurfaceMeshes(maxTriArea=0.1)
  cs.save('testCompositeShape.vtk')

if __name__ == '__main__': test()
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Standard packages
import sys
import argparse

# Third-party packages
from toil.job import Job

# Package methods
from ddb import configuration
from ddb_ngsflow import gatk
from ddb_ngsflow import annotation
from ddb_ngsflow import pipeline
from ddb_ngsflow.align import bwa
from ddb_ngsflow.variation import variation
from ddb_ngsflow.variation import haplotypecaller

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('-s', '--samples_file', help="Input configuration file for samples")
    parser.add_argument('-c', '--configuration', help="Configuration file for various settings")
    Job.Runner.addToilOptions(parser)
    args = parser.parse_args()
    args.logLevel = "INFO"

    sys.stdout.write("Parsing configuration data\n")
    config = configuration.configure_runtime(args.configuration)

    sys.stdout.write("Parsing sample data\n")
    samples = configuration.configure_samples(args.samples_file, config)

    # Workflow Graph definition. The following workflow definition should create a valid Directed Acyclic Graph (DAG)
    root_job = Job.wrapJobFn(pipeline.spawn_batch_jobs, cores=1)

    # Per sample jobs
    # for sample in samples:
    #     # Alignment and Refinement Stages
    #     align_job = Job.wrapJobFn(bwa.run_bwa_mem, config, sample, samples,
    #                               cores=int(config['bwa']['num_cores']),
    #                               memory="{}G".format(config['bwa']['max_mem']))
    #
    #     add_job = Job.wrapJobFn(gatk.add_or_replace_readgroups, config, sample, align_job.rv(),
    #                             cores=1,
    #                             memory="{}G".format(config['picard-add']['max_mem']))
    #
    #     dedup_job = Job.wrapJobFn(gatk.mark_duplicates, config, sample, add_job.rv(),
    #                               cores=int(config['picard-dedup']['num_cores']),
    #                               memory="{}G".format(config['picard-dedup']['max_mem']))
    #
    #     creator_job = Job.wrapJobFn(gatk.realign_target_creator, config, sample, dedup_job.rv(),
    #                                 cores=int(config['gatk-realign']['num_cores']),
    #                                 memory="{}G".format(config['gatk-realign']['max_mem']))
    #
    #     realign_job = Job.wrapJobFn(gatk.realign_indels, config, sample, add_job.rv(), creator_job.rv(),
    #                                 cores=1,
    #                                 memory="{}G".format(config['gatk-realign']['max_mem']))
    #
    #     recal_job = Job.wrapJobFn(gatk.recalibrator, config, sample, realign_job.rv(),
    #                               cores=int(config['gatk-recal']['num_cores']),
    #                               memory="{}G".format(config['gatk-recal']['max_mem']))
    #     # Variant Calling
    #     haplotypecaller_job = Job.wrapJobFn(haplotypecaller.haplotypecaller_single, config, sample, samples,
    #                                         recal_job.rv(),
    #                                         cores=1,
    #                                         memory="{}G".format(config['gatk-haplotypecaller']['max_mem']))
    #
    #     # Create workflow from created jobs
    #     root_job.addChild(align_job)
    #     align_job.addChild(add_job)
    #     add_job.addChild(dedup_job)
    #     dedup_job.addChild(creator_job)
    #     creator_job.addChild(realign_job)
    #     realign_job.addChild(recal_job)
    #     recal_job.addChild(haplotypecaller_job)
    #
    # # Need to filter for on target only results somewhere as well
    joint_call_job = Job.wrapJobFn(haplotypecaller.joint_variant_calling, config, config['project'], samples,
                                   cores=int(config['gatk-jointgenotyper']),
                                   memory="{}G".format(config['gatk-jointgenotyper']['max_mem']))

    normalization_job = Job.wrapJobFn(variation.vt_normalization, config, config['project'], "hc",
                                      "{}.haplotypecaller.vcf".format(config['project']),
                                      cores=1,
                                      memory="{}G".format(config['gatk']['max_mem']))

    sample_inputs = list()
    for sample in samples:
        sample = "{}.recalibrated.sorted.bam".format(sample)
        sample_inputs.append(sample)
    sample_bam_string = " -I ".join(sample_inputs)

    gatk_annotate_job = Job.wrapJobFn(gatk.annotate_vcf, config, config['project'], normalization_job.rv(), sample_bam_string,
                                      cores=int(config['gatk-annotate']['num_cores']),
                                      memory="{}G".format(config['gatk-annotate']['max_mem']))

    gatk_filter_job = Job.wrapJobFn(gatk.filter_variants, config, config['project'], gatk_annotate_job.rv(),
                                    cores=1,
                                    memory="{}G".format(config['gatk-filter']['max_mem']))

    snpeff_job = Job.wrapJobFn(annotation.snpeff, config, config['project'], gatk_filter_job.rv(),
                               cores=int(config['snpeff']['num_cores']),
                               memory="{}G".format(config['snpeff']['max_mem']))

    gemini_job = Job.wrapJobFn(annotation.gemini, config, config['project'], gatk_filter_job.rv(),
                               cores=int(config['snpeff']['num_cores']),
                               memory="{}G".format(config['snpeff']['max_mem']))

    # root_job.addFollowOn(joint_call_job)
    # joint_call_job.addChild(normalization_job)
    root_job.addChild(normalization_job)
    normalization_job.addChild(gatk_annotate_job)
    gatk_annotate_job.addChild(gatk_filter_job)
    gatk_filter_job.addChild(snpeff_job)
    snpeff_job.addChild(gemini_job)

    # Start workflow execution
    Job.Runner.startToil(root_job, args)
<commit_msg>Restore and fix workflow<commit_after>#!/usr/bin/env python

# Standard packages
import sys
import argparse

# Third-party packages
from toil.job import Job

# Package methods
from ddb import configuration
from ddb_ngsflow import gatk
from ddb_ngsflow import annotation
from ddb_ngsflow import pipeline
from ddb_ngsflow.align import bwa
from ddb_ngsflow.variation import variation
from ddb_ngsflow.variation import haplotypecaller

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('-s', '--samples_file', help="Input configuration file for samples")
    parser.add_argument('-c', '--configuration', help="Configuration file for various settings")
    Job.Runner.addToilOptions(parser)
    args = parser.parse_args()
    args.logLevel = "INFO"

    sys.stdout.write("Parsing configuration data\n")
    config = configuration.configure_runtime(args.configuration)

    sys.stdout.write("Parsing sample data\n")
    samples = configuration.configure_samples(args.samples_file, config)

    # Workflow Graph definition. The following workflow definition should create a valid Directed Acyclic Graph (DAG)
    root_job = Job.wrapJobFn(pipeline.spawn_batch_jobs, cores=1)

    # Per sample jobs
    for sample in samples:
        # Alignment and Refinement Stages
        align_job = Job.wrapJobFn(bwa.run_bwa_mem, config, sample, samples,
                                  cores=int(config['bwa']['num_cores']),
                                  memory="{}G".format(config['bwa']['max_mem']))

        add_job = Job.wrapJobFn(gatk.add_or_replace_readgroups, config, sample, align_job.rv(),
                                cores=1,
                                memory="{}G".format(config['picard-add']['max_mem']))

        dedup_job = Job.wrapJobFn(gatk.mark_duplicates, config, sample, add_job.rv(),
                                  cores=int(config['picard-dedup']['num_cores']),
                                  memory="{}G".format(config['picard-dedup']['max_mem']))

        creator_job = Job.wrapJobFn(gatk.realign_target_creator, config, sample, dedup_job.rv(),
                                    cores=int(config['gatk-realign']['num_cores']),
                                    memory="{}G".format(config['gatk-realign']['max_mem']))

        realign_job = Job.wrapJobFn(gatk.realign_indels, config, sample, add_job.rv(), creator_job.rv(),
                                    cores=1,
                                    memory="{}G".format(config['gatk-realign']['max_mem']))

        recal_job = Job.wrapJobFn(gatk.recalibrator, config, sample, realign_job.rv(),
                                  cores=int(config['gatk-recal']['num_cores']),
                                  memory="{}G".format(config['gatk-recal']['max_mem']))
        # Variant Calling
        haplotypecaller_job = Job.wrapJobFn(haplotypecaller.haplotypecaller_single, config, sample, samples,
                                            recal_job.rv(),
                                            cores=1,
                                            memory="{}G".format(config['gatk-haplotypecaller']['max_mem']))

        # Create workflow from created jobs
        root_job.addChild(align_job)
        align_job.addChild(add_job)
        add_job.addChild(dedup_job)
        dedup_job.addChild(creator_job)
        creator_job.addChild(realign_job)
        realign_job.addChild(recal_job)
        recal_job.addChild(haplotypecaller_job)

    # Need to filter for on target only results somewhere as well
    joint_call_job = Job.wrapJobFn(haplotypecaller.joint_variant_calling, config, config['project'], samples,
                                   cores=int(config['gatk-jointgenotyper']['num_cores']),
                                   memory="{}G".format(config['gatk-jointgenotyper']['max_mem']))

    normalization_job = Job.wrapJobFn(variation.vt_normalization, config, config['project'], "hc",
                                      "{}.haplotypecaller.vcf".format(config['project']),
                                      cores=1,
                                      memory="{}G".format(config['gatk']['max_mem']))

    sample_inputs = list()
    for sample in samples:
        sample = "{}.recalibrated.sorted.bam".format(sample)
        sample_inputs.append(sample)
    sample_bam_string = " -I ".join(sample_inputs)

    gatk_annotate_job = Job.wrapJobFn(gatk.annotate_vcf, config, config['project'], normalization_job.rv(), sample_bam_string,
                                      cores=int(config['gatk-annotate']['num_cores']),
                                      memory="{}G".format(config['gatk-annotate']['max_mem']))

    gatk_filter_job = Job.wrapJobFn(gatk.filter_variants, config, config['project'], gatk_annotate_job.rv(),
                                    cores=1,
                                    memory="{}G".format(config['gatk-filter']['max_mem']))

    snpeff_job = Job.wrapJobFn(annotation.snpeff, config, config['project'], gatk_filter_job.rv(),
                               cores=int(config['snpeff']['num_cores']),
                               memory="{}G".format(config['snpeff']['max_mem']))

    root_job.addFollowOn(joint_call_job)
    joint_call_job.addChild(normalization_job)
    normalization_job.addChild(gatk_annotate_job)
    gatk_annotate_job.addChild(gatk_filter_job)
    gatk_filter_job.addChild(snpeff_job)

    # Start workflow execution
    Job.Runner.startToil(root_job, args)
<|endoftext|>


<commit_before>#
# SFA XML-RPC and SOAP interfaces
#
### $Id: api.py 17793 2010-04-26 21:40:57Z tmack $
### $URL: https://svn.planet-lab.org/svn/sfa/trunk/sfa/plc/api.py $
#

import sys
import os
import traceback
import string
import xmlrpclib

from sfa.util.sfalogging import sfa_logger
from sfa.trust.auth import Auth
from sfa.util.config import *
from sfa.util.faults import *
from sfa.trust.rights import *
from sfa.trust.credential import *
from sfa.trust.certificate import *
from sfa.util.namespace import *
from sfa.util.api import *
from sfa.util.nodemanager import NodeManager
from collections import defaultdict

class RecordInfo():

    shell = None
    auth = None

    # pl records
    sites = {}
    slices = {}
    persons = {}
    nodes = {}
    keys = {}

    # sfa records
    sfa_authorities = {}
    sfa_slices = {}
    sfa_users = {}
    sfa_nodes = {}

    records = []

    def __init__(self, api, records):
        self.api = api
        self.shell = api.plshell
        self.auth = api.plauth

        site_ids = []
        slice_ids = []
        person_ids = []
        node_ids = []

        # put records into groups based on types
        for record in records:
            pointer = record['pointer']
            if record['type'] == 'authority':
                self.sfa_authorities[pointer] = record
                self.records.append(record)
                site_ids.append(record['pointer'])
            elif record['type'] == 'slice':
                self.sfa_slices[pointer] = record
                self.records.append(record)
                slice_ids.append(record['pointer'])
            elif record['type'] == 'user':
                self.sfa_users[pointer] = record
                self.records.append(record)
                person_ids.append(record['pointer'])
            elif record['type'] == 'node':
                self.sfa_nodes[pointer] = record
                self.records.append(record)
                node_ids.append(record['pointer'])

        # get pl info for these records
        self.update_pl_sites(site_ids)
        self.update_pl_slices(slice_ids)
        self.update_pl_persons(person_ids)
        self.update_pl_nodes(node_ids)

        site_ids = []
        slice_ids = []
        person_ids = []
        node_ids = []
        # now get pl records for all ids associated with
        # these records
        for record in records:
            if 'site_id' in record:
                site_ids.append(record['site_id'])
            if 'site_ids' in records:
                site_ids.extend(record['site_ids'])
            if 'person_ids' in record:
                person_ids.extend(record['person_ids'])
            if 'slice_ids' in record:
                slice_ids.extend(record['slice_ids'])
            if 'node_ids' in record:
                node_ids.extend(record['node_ids'])

        # get pl info for these records
        self.update_pl_sites(site_ids)
        self.update_pl_slices(slice_ids)
        self.update_pl_persons(person_ids)
        self.update_pl_nodes(node_ids)

        # convert pl ids to hrns
        self.update_hrns()

        # update sfa info
        self.update_sfa_info(person_ids)

    def update_pl_sites(self, site_ids):
        """
        Update site records with PL info
        """
        if not site_ids:
            return
        sites = self.shell.GetSites(self.auth, site_ids)
        for site in sites:
            site_id = site['site_id']
            self.sites[site_id] = site
            if site_id in self.sfa_authorities:
                self.sfa_authorities[site_id].update(site)

    def update_pl_slices(self, slice_ids):
        """
        Update slice records with PL info
        """
        if not slice_ids:
            return
        slices = self.shell.GetSlices(self.auth, slice_ids)
        for slice in slices:
            slice_id = slice['slice_id']
            self.slices[slice_id] = slice
            if slice_id in self.sfa_slices:
                self.sfa_slices[slice_id].update(slice)

    def update_pl_persons(self, person_ids):
        """
        Update person records with PL info
        """
        key_ids = []
        if not person_ids:
            return
        persons = self.shell.GetPersons(self.auth, person_ids)
        for person in persons:
            person_id = person['person_id']
            self.persons[person_id] = person
            key_ids.extend(person['key_ids'])
            if person_id in self.sfa_users:
                self.sfa_users[person_id].update(person)
        self.update_pl_keys(key_ids)

    def update_pl_keys(self, key_ids):
        """
        Update user records with PL public key info
        """
        if not key_ids:
            return
        keys = self.shell.GetKeys(self.auth, key_ids)
        for key in keys:
            person_id = key['person_id']
            self.keys[key['key_id']] = key
            if person_id in self.sfa_users:
                person = self.sfa_users[person_id]
                if not 'keys' in person:
                    person['keys'] = [key['key']]
                else:
                    person['keys'].append(key['key'])

    def update_pl_nodes(self, node_ids):
        """
        Update node records with PL info
        """
        if not node_ids:
            return
        nodes = self.shell.GetNodes(self.auth, node_ids)
        for node in nodes:
            node_id = node['node_id']
            self.nodes[node['node_id']] = node
            if node_id in self.sfa_nodes:
                self.sfa_nodes[node_id].update(node)

    def update_hrns(self):
        """
        Convert pl ids to hrns
        """
        for record in self.records:
            # get all necessary data
            type = record['type']
            pointer = record['pointer']
            auth_hrn = self.api.hrn
            login_base = ''
            if pointer == -1:
                continue

            if 'site_id' in record:
                site = self.sites[record['site_id']]
                login_base = site['login_base']
                record['site'] = ".".join([auth_hrn, login_base])
            if 'person_ids' in record:
                emails = [self.persons[person_id]['email'] for person_id in record['person_ids'] \
                          if person_id in self.persons]
                usernames = [email.split('@')[0] for email in emails]
                person_hrns = [".".join([auth_hrn, login_base, username]) for username in usernames]
                record['persons'] = person_hrns
            if 'slice_ids' in record:
                slicenames = [self.slices[slice_id]['name'] for slice_id in record['slice_ids'] \
                              if slice_id in self.slices]
                slice_hrns = [slicename_to_hrn(auth_hrn, slicename) for slicename in slicenames]
                record['slices'] = slice_hrns
            if 'node_ids' in record:
                hostnames = [self.nodes[node_id]['hostname'] for node_id in record['node_ids'] \
                             if node_id in self.nodes]
                node_hrns = [hostname_to_hrn(auth_hrn, login_base, hostname) for hostname in hostnames]
                record['nodes'] = node_hrns
            if 'site_ids' in record:
                login_bases = [self.sites[site_id]['login_base'] for site_id in record['site_ids'] \
                               if site_id in self.sites]
                site_hrns = [".".join([auth_hrn, lbase]) for lbase in login_bases]
                record['sites'] = site_hrns

    def update_sfa_info(self, person_ids):
        from sfa.util.table import SfaTable
        table = SfaTable()
        persons = table.find({'type': 'user', 'pointer': person_ids})
        # create a hrns keyed on the sfa record's pointer.
        # Its possible for  multiple records to have the same pointer so
        # the dict's value will be a list of hrns.
        person_dict = defaultdict(list)
        for person in persons:
            person_dict[person['pointer']].append(person['hrn'])

        def startswith(prefix, values):
            return [value for value in values if value.startswith(prefix)]

        for record in self.records:
            authority = record['authority']
            if record['pointer'] == -1:
                continue

            if record['type'] == 'slice':
                # all slice users are researchers
                record['PI'] = []
                record['researchers'] = []
                for person_id in record['person_ids']:
                    record['researchers'].extend(person_dict[person_id])
                # also add the pis at the slice's site
                site = self.sites[record['site_id']]
                for person_id in site['person_ids']:
                    person = self.persons[person_id]
                    if 'pi' in person['roles']:
                        # PLCAPI doesn't support per site roles
                        # (a pi has the pi role at every site he belongs to).
                        # We shouldnt allow this in SFA
                        record['PI'].extend(startswith(authority, person_dict[person_id]))

            elif record['type'] == 'authority':
                record['PI'] = []
                record['operator'] = []
                record['owner'] = []
                for person_id in record['person_ids']:
                    person = self.persons[person_id]
                    if 'pi' in person['roles']:
                        # only get PI's at this site
                        record['PI'].extend(startswith(record['hrn'], person_dict[person_id]))
                    if 'tech' in person['roles']:
                        # only get PI's at this site
                        record['operator'].extend(startswith(record['hrn'], person_dict[person_id]))
                    if 'admin' in person['roles']:
                        record['owner'].extend(startswith(record['hrn'], person_dict[person_id]))

            elif record['type'] == 'node':
                record['dns'] = record['hostname']

            elif record['type'] == 'user':
                record['email'] = record['email']

    def get_records(self):
        return self.records

class SfaAPI(BaseAPI):

    # flat list of method names
    import sfa.methods
    methods = sfa.methods.all

    def __init__(self, config = "/etc/sfa/sfa_config.py", encoding = "utf-8",
                 methods='sfa.methods', peer_cert = None, interface = None,
                key_file = None, cert_file = None, cache = None):
        BaseAPI.__init__(self, config=config, encoding=encoding, methods=methods, \
                         peer_cert=peer_cert, interface=interface, key_file=key_file, \
                         cert_file=cert_file, cache=cache)

        self.encoding = encoding

        from sfa.util.table import SfaTable
        self.SfaTable = SfaTable
        # Better just be documenting the API
        if config is None:
            return

        # Load configuration
        self.config = Config(config)
        self.auth = Auth(peer_cert)
        self.interface = interface
        self.key_file = key_file
        self.key = Keypair(filename=self.key_file)
        self.cert_file = cert_file
        self.cert = Certificate(filename=self.cert_file)
        self.credential = None
        # Initialize the PLC shell only if SFA wraps a myPLC
        rspec_type = self.config.get_aggregate_type()
        if (rspec_type == 'pl' or rspec_type == 'vini'):
            self.plshell = self.getPLCShell()
            self.plshell_version = "4.3"

        self.hrn = self.config.SFA_INTERFACE_HRN
        self.time_format = "%Y-%m-%d %H:%M:%S"
        self.logger=sfa_logger

    def getPLCShell(self):
        self.plauth = {'Username': self.config.SFA_PLC_USER,
                       'AuthMethod': 'password',
                       'AuthString': self.config.SFA_PLC_PASSWORD}

        self.plshell_type = 'xmlrpc'
        # connect via xmlrpc
        url = self.config.SFA_PLC_URL
        shell = xmlrpclib.Server(url, verbose = 0, allow_none = True)
        return shell

    def getCredential(self):
        if self.interface in ['registry']:
            return self.getCredentialFromLocalRegistry()
        else:
            return self.getCredentialFromRegistry()

    def getCredentialFromRegistry(self):
        """
        Get our credential from a remote registry
        """
        type = 'authority'
        path = self.config.SFA_DATA_DIR
        filename = ".".join([self.interface, self.hrn, type, "cred"])
        cred_filename = path + os.sep + filename
        try:
            credential = Credential(filename = cred_filename)
            return credential.save_to_string(save_parents=True)
        except IOError:
            from sfa.server.registry import Registries
            registries = Registries(self)
            registry = registries[self.hrn]
            cert_string=self.cert.save_to_string(save_parents=True)
            # get self credential
            self_cred = registry.GetSelfCredential(cert_string, self.hrn, type)
            # get credential
            cred = registry.GetCredential(self_cred, type, self.hrn)

            # save cred to file
            Credential(string=cred).save_to_file(cred_filename, save_parents=True)
            return cred

    def getCredentialFromLocalRegistry(self):
        """
        Get our current credential directly from the local registry.
        """

        hrn = self.hrn
        auth_hrn = self.auth.get_authority(hrn)

        # is this a root or sub authority
        if not auth_hrn or hrn == self.config.SFA_INTERFACE_HRN:
            auth_hrn = hrn
        auth_info = self.auth.get_auth_info(auth_hrn)
        table = self.SfaTable()
        records = table.findObjects(hrn)
        if not records:
            raise RecordNotFound
        record = records[0]
        type = record['type']
        object_gid = record.get_gid_object()
        new_cred = Credential(subject = object_gid.get_subject())
        new_cred.set_gid_caller(object_gid)
        new_cred.set_gid_object(object_gid)
        new_cred.set_issuer(key=auth_info.get_pkey_object(), subject=auth_hrn)
        new_cred.set_pubkey(object_gid.get_pubkey())
        r1 = determine_rights(type, hrn)
        new_cred.set_privileges(r1)

        auth_kind = "authority,ma,sa"

        new_cred.set_parent(self.auth.hierarchy.get_auth_cred(auth_hrn, kind=auth_kind))

        new_cred.encode()
        new_cred.sign()

        return new_cred.save_to_string(save_parents=True)

    def loadCredential (self):
        """
        Attempt to load credential from file if it exists. If it doesnt get
        credential from registry.
        """

        # see if this file exists
        # XX This is really the aggregate's credential. Using this is easier than getting
        # the registry's credential from iteslf (ssl errors).
        ma_cred_filename = self.config.SFA_DATA_DIR + os.sep + self.interface + self.hrn + ".ma.cred"
        try:
            self.credential = Credential(filename = ma_cred_filename)
        except IOError:
            self.credential = self.getCredentialFromRegistry()

    ##
    # Convert SFA fields to PLC fields for use when registering up updating
    # registry record in the PLC database
    #
    # @param type type of record (user, slice, ...)
    # @param hrn human readable name
    # @param sfa_fields dictionary of SFA fields
    # @param pl_fields dictionary of PLC fields (output)

    def sfa_fields_to_pl_fields(self, type, hrn, record):

        def convert_ints(tmpdict, int_fields):
            for field in int_fields:
                if field in tmpdict:
                    tmpdict[field] = int(tmpdict[field])

        pl_record = {}
        #for field in record:
        #    pl_record[field] = record[field]

        if type == "slice":
            if not "instantiation" in pl_record:
                pl_record["instantiation"] = "plc-instantiated"
            pl_record["name"] = hrn_to_pl_slicename(hrn)
	    if "url" in record:
               pl_record["url"] = record["url"]
	    if "description" in record:
	        pl_record["description"] = record["description"]
	    if "expires" in record:
	        pl_record["expires"] = int(record["expires"])

        elif type == "node":
            if not "hostname" in pl_record:
                if not "hostname" in record:
                    raise MissingSfaInfo("hostname")
                pl_record["hostname"] = record["hostname"]
            if not "model" in pl_record:
                pl_record["model"] = "geni"

        elif type == "authority":
            pl_record["login_base"] = hrn_to_pl_login_base(hrn)

            if not "name" in pl_record:
                pl_record["name"] = hrn

            if not "abbreviated_name" in pl_record:
                pl_record["abbreviated_name"] = hrn

            if not "enabled" in pl_record:
                pl_record["enabled"] = True

            if not "is_public" in pl_record:
                pl_record["is_public"] = True

        return pl_record

    def fill_record_info(self, records):
        """
        Given a SFA record, fill in the PLC specific and SFA specific
        fields in the record.
        """
        if not isinstance(records, list):
            records = [records]

        record_info = RecordInfo(self, records)
        return record_info.get_records()

    def update_membership_list(self, oldRecord, record, listName, addFunc, delFunc):
        # get a list of the HRNs tht are members of the old and new records
        if oldRecord:
            oldList = oldRecord.get(listName, [])
        else:
            oldList = []
        newList = record.get(listName, [])

        # if the lists are the same, then we don't have to update anything
        if (oldList == newList):
            return

        # build a list of the new person ids, by looking up each person to get
        # their pointer
        newIdList = []
        table = self.SfaTable()
        records = table.find({'type': 'user', 'hrn': newList})
        for rec in records:
            newIdList.append(rec['pointer'])

        # build a list of the old person ids from the person_ids field
        if oldRecord:
            oldIdList = oldRecord.get("person_ids", [])
            containerId = oldRecord.get_pointer()
        else:
            # if oldRecord==None, then we are doing a Register, instead of an
            # update.
            oldIdList = []
            containerId = record.get_pointer()

    # add people who are in the new list, but not the oldList
        for personId in newIdList:
            if not (personId in oldIdList):
                addFunc(self.plauth, personId, containerId)

        # remove people who are in the old list, but not the new list
        for personId in oldIdList:
            if not (personId in newIdList):
                delFunc(self.plauth, personId, containerId)

    def update_membership(self, oldRecord, record):
        if record.type == "slice":
            self.update_membership_list(oldRecord, record, 'researcher',
                                        self.plshell.AddPersonToSlice,
                                        self.plshell.DeletePersonFromSlice)
        elif record.type == "authority":
            # xxx TODO
            pass

class ComponentAPI(BaseAPI):

    def __init__(self, config = "/etc/sfa/sfa_config.py", encoding = "utf-8", methods='sfa.methods',
                 peer_cert = None, interface = None, key_file = None, cert_file = None):

        BaseAPI.__init__(self, config=config, encoding=encoding, methods=methods, peer_cert=peer_cert,
                         interface=interface, key_file=key_file, cert_file=cert_file)
        self.encoding = encoding

        # Better just be documenting the API
        if config is None:
            return

        self.nodemanager = NodeManager(self.config)

    def sliver_exists(self):
        sliver_dict = self.nodemanager.GetXIDs()
        if slicename in sliver_dict.keys():
            return True
        else:
            return False
<commit_msg>removing api-dev<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python -*- coding: utf-8 -*-

from __future__ import unicode_literals

import io, sys
from itertools import chain
reload(sys); sys.setdefaultencoding("utf-8")

from threading import Thread
from Queue import Queue

import lucene
from java.io import File
# For Indexing.
from org.apache.lucene.analysis.standard import StandardAnalyzer
from org.apache.lucene.document import Document, Field, FieldType
from org.apache.lucene.index import FieldInfo, IndexWriter, IndexWriterConfig
from org.apache.lucene.store import FSDirectory
from org.apache.lucene.util import Version
# For Querying.
from org.apache.lucene.index import DirectoryReader
from org.apache.lucene.queryparser.classic import QueryParser
from org.apache.lucene.search import IndexSearcher

from nltk.corpus import wordnet as wn
from nltk import sent_tokenize, word_tokenize

from texeval import TexEval2015
from luluwiki import retrieve_wiki

texeval_corpus = TexEval2015()

wiki_index = '/home/alvas/engwiki/english-wiki'
lucene.initVM()

def filter_doc(term, doc):
    for i in sent_tokenize(doc):
        for j in i.split('\n'):
            if term.lower() in j.lower():
                yield j

def isa_doc(term, doc):
    for i in sent_tokenize(doc):
        if term+" is a" in i or "is a "+term in i or "is an "+term in i:
            yield i

def build_corpus(sbc):
    # Hack for parallelizing queries, uses one index per domain.
    directory = FSDirectory.open(File(wiki_index+'-'+sbc))
    searcher = IndexSearcher(DirectoryReader.open(directory))
    analyzer = StandardAnalyzer(Version.LUCENE_CURRENT)
    fout = io.open('WIKI_'+sbc, 'w', encoding='utf8')
    for termid, term in texeval_corpus.terms('test', sbc):
        docs = []
        for curl, doc in retrieve_wiki(term, searcher, analyzer):
            for fdoc in filter_doc(term, doc):
                docs.append(fdoc)

        print termid, term, len(docs)
        fout.write("{}\t{}\n".format(termid, term).decode('utf8'))
        for i in docs:

            fout.write(i.decode('utf8')+ '\n')
        fout.write('\n'.decode('utf8'))
    return sbc

def wrapper(func, arg, queue):
    """" Wrapper class for multi-threaded functions """
    queue.put(func(arg))

sbcs = texeval_corpus.test_subcorpora

q1, q2 = Queue(), Queue()
q3, q4 =  Queue(), Queue()
q5 = Queue()
q6 = Queue()
Thread(target=wrapper, args=(build_corpus, sbcs[0], q1)).start()
Thread(target=wrapper, args=(build_corpus, sbcs[1], q2)).start()
Thread(target=wrapper, args=(build_corpus, sbcs[2], q3)).start()
Thread(target=wrapper, args=(build_corpus, sbcs[3], q4)).start()
Thread(target=wrapper, args=(build_corpus, sbcs[4], q5)).start()
Thread(target=wrapper, args=(build_corpus, sbcs[5], q6)).start()

q1.get(); q2.get(); q3.get(); q4.get(); q5.get() ; q6.get()           <commit_msg>remove threading, doing it manually<commit_after>#!/usr/bin/env python -*- coding: utf-8 -*-

from __future__ import unicode_literals

import io, sys
from itertools import chain
reload(sys); sys.setdefaultencoding("utf-8")

from threading import Thread
from Queue import Queue

import lucene
from java.io import File
# For Indexing.
from org.apache.lucene.analysis.standard import StandardAnalyzer
from org.apache.lucene.document import Document, Field, FieldType
from org.apache.lucene.index import FieldInfo, IndexWriter, IndexWriterConfig
from org.apache.lucene.store import FSDirectory
from org.apache.lucene.util import Version
# For Querying.
from org.apache.lucene.index import DirectoryReader
from org.apache.lucene.queryparser.classic import QueryParser
from org.apache.lucene.search import IndexSearcher

from nltk.corpus import wordnet as wn
from nltk import sent_tokenize, word_tokenize

from texeval import TexEval2015
from luluwiki import retrieve_wiki

texeval_corpus = TexEval2015()

wiki_index = '/home/alvas/engwiki/english-wiki'
lucene.initVM()

def filter_doc(term, doc):
    for i in sent_tokenize(doc):
        for j in i.split('\n'):
            if term.lower() in j.lower():
                yield j

def isa_doc(term, doc):
    for i in sent_tokenize(doc):
        if term+" is a" in i or "is a "+term in i or "is an "+term in i:
            yield i

def build_corpus(sbc):
    fout = io.open('WIKI_'+sbc, 'w', encoding='utf8')
    for termid, term in texeval_corpus.terms('test', sbc):
        docs = []
        for curl, doc in retrieve_wiki(term, searcher, analyzer):
            for fdoc in filter_doc(term, doc):
                docs.append(fdoc)

        print termid, term, len(docs)
        fout.write("{}\t{}\n".format(termid, term).decode('utf8'))
        for i in docs:

            fout.write(i.decode('utf8')+ '\n')
        fout.write('\n'.decode('utf8'))
    return sbc

def wrapper(func, arg, queue):
    """" Wrapper class for multi-threaded functions """
    queue.put(func(arg))

sbcs = texeval_corpus.test_subcorpora

# Hack for parallelizing queries, uses one index per domain.
directory = FSDirectory.open(File(wiki_index+'-'+sbc))
searcher = IndexSearcher(DirectoryReader.open(directory))
analyzer = StandardAnalyzer(Version.LUCENE_CURRENT)

build_corpus(sbcs[0])<|endoftext|>


<commit_before>""" This is an example local_settings.py file meant for local development."""

PROJECT_PATH = '/path/to/django/root/'
STATIC_URL = '/static'
STATICFILES_DIRS = (
    PROJECT_PATH + '/static',
)

DEBUG = True

FINIAL_LOCAL_DIR_PREFIX = '/overrides' # This is the directory prefix from your PROJECT_PATH
FINIAL_MEDIA_URL_PREFIX = FINIAL_STATIC_URL_PREFIX = 'https://s3.amazonaws.com/com.finial.media'

if DEBUG:
    FINIAL_MEDIA_URL_PREFIX = FINIAL_STATIC_URL_PREFIX = STATIC_URL
    # Need to adjust url path to match dir path for local dev.
    STATICFILES_FINDERS = (
        'finial.finders.FinialFileSystemFinder',
        'django.contrib.staticfiles.finders.FileSystemFinder',
        'django.contrib.staticfiles.finders.AppDirectoriesFinder',
    )

# Here is where we map override names to URLconf module paths:
FINIAL_URL_OVERRIDES = {
    'test_or': 'airship.finial_override_patterns'
}
<commit_msg>updates to example_settings.py<commit_after>""" This is an example local_settings.py file meant for local development."""

PROJECT_PATH = '/path/to/django/root/'
STATIC_URL = '/static'
STATICFILES_DIRS = (
    PROJECT_PATH + '/static',
)

DEBUG = True

FINIAL_LOCAL_DIR_PREFIX = '/overrides' # This is the directory prefix from your PROJECT_PATH
FINIAL_MEDIA_URL_PREFIX = FINIAL_STATIC_URL_PREFIX = 'https://s3.amazonaws.com/com.finial.media'

if DEBUG:
    FINIAL_MEDIA_URL_PREFIX = FINIAL_STATIC_URL_PREFIX = STATIC_URL
    # Need to adjust url path to match dir path for local dev.
    STATICFILES_FINDERS = (
        'finial.finders.FinialFileSystemFinder',
        'django.contrib.staticfiles.finders.FileSystemFinder',
        'django.contrib.staticfiles.finders.AppDirectoriesFinder',
    )

# Here is where we map override names to URLconf module paths:
FINIAL_URL_OVERRIDES = {
    'test_or': 'my_project.finial_override_patterns'
}
<|endoftext|>


<commit_before>import xmlrpclib

from sfa.util.sfalogging import logger

class PlShell:
    """
    A simple xmlrpc shell to a myplc instance
    This class can receive all PLCAPI calls to the underlying testbed
    For safety this is limited to a set of hard-coded calls
    """

    direct_calls = ['AddNode', 'AddPerson', 'AddPersonKey', 'AddPersonToSite',
                    'AddPersonToSlice', 'AddRoleToPerson', 'AddSite', 'AddSiteTag', 'AddSlice',
                    'AddSliceTag', 'AddSliceToNodes', 'BindObjectToPeer', 'DeleteKey',
                    'DeleteNode', 'DeletePerson', 'DeletePersonFromSlice', 'DeleteSite',
                    'DeleteSlice', 'DeleteSliceFromNodes', 'DeleteSliceTag', 'GetInitScripts',
                    'GetInterfaces', 'GetKeys', 'GetNodeTags', 'GetPeers',
                    'GetPersons', 'GetSlices', 'GetSliceTags', 'GetTagTypes',
                    'UnBindObjectFromPeer', 'UpdateNode', 'UpdatePerson', 'UpdateSite',
                    'UpdateSlice', 'UpdateSliceTag',
                    # also used as-is in importer
                    'GetSites','GetNodes',
                    ]
    # support for other names - this is experimental
    alias_calls = { 'get_authorities':'GetSites',
                    'get_nodes':'GetNodes',
                    }

    # use the 'capability' auth mechanism for higher performance when the PLC db is local
    def __init__ ( self, config ) :
        url = config.SFA_PLC_URL
        # try to figure if the url is local
        hostname=urlparse(url).hostname
        is_local=False
        if hostname == 'localhost': is_local=True
        # otherwise compare IP addresses
        url_ip=socket.gethostbyname(hostname)
        local_ip=socket.gethostbyname(socket.gethostname())
        if url_ip==local_ip: is_local=True

        if is_local:
            try:
                # too bad this is not installed properly
                plcapi_path="/usr/share/plc_api"
                if plcapi_path not in sys.path: sys.path.append(plcapi_path)
                import PLC.Shell
                plc_direct_access=True
            except:
                plc_direct_access=False
        if is_local and plc_direct_access:
            logger.debug('plshell access - capability')
            self.plauth = { 'AuthMethod': 'capability',
                            'Username':   config.SFA_PLC_USER,
                            'AuthString': config.SFA_PLC_PASSWORD,
                            }
            self.proxy = PLC.Shell.Shell ()

        else:
            logger.debug('plshell access - xmlrpc')
            self.plauth = { 'AuthMethod': 'password',
                            'Username':   config.SFA_PLC_USER,
                            'AuthString': config.SFA_PLC_PASSWORD,
                            }
            self.proxy = xmlrpclib.Server(url, verbose = 0, allow_none = True)

    def __getattr__(self, name):
        def func(*args, **kwds):
            actual_name=None
            if name in PlShell.direct_calls: actual_name=name
            if name in PlShell.alias_calls: actual_name=PlShell.alias_calls[name]
            if not actual_name:
                raise Exception, "Illegal method call %s for PL driver"%(name)
            result=getattr(self.proxy, actual_name)(self.plauth, *args, **kwds)
            logger.info('%s (%s) returned ... %s'%(name,actual_name,result))
            return result
        return func
<commit_msg>oops this was intended to be in previous commit as well<commit_after>import sys
import xmlrpclib
import socket
from urlparse import urlparse

from sfa.util.sfalogging import logger

class PlShell:
    """
    A simple xmlrpc shell to a myplc instance
    This class can receive all PLCAPI calls to the underlying testbed
    For safety this is limited to a set of hard-coded calls
    """

    direct_calls = ['AddNode', 'AddPerson', 'AddPersonKey', 'AddPersonToSite',
                    'AddPersonToSlice', 'AddRoleToPerson', 'AddSite', 'AddSiteTag', 'AddSlice',
                    'AddSliceTag', 'AddSliceToNodes', 'BindObjectToPeer', 'DeleteKey',
                    'DeleteNode', 'DeletePerson', 'DeletePersonFromSlice', 'DeleteSite',
                    'DeleteSlice', 'DeleteSliceFromNodes', 'DeleteSliceTag', 'GetInitScripts',
                    'GetInterfaces', 'GetKeys', 'GetNodeTags', 'GetPeers',
                    'GetPersons', 'GetSlices', 'GetSliceTags', 'GetTagTypes',
                    'UnBindObjectFromPeer', 'UpdateNode', 'UpdatePerson', 'UpdateSite',
                    'UpdateSlice', 'UpdateSliceTag',
                    # also used as-is in importer
                    'GetSites','GetNodes',
                    ]
    # support for other names - this is experimental
    alias_calls = { 'get_authorities':'GetSites',
                    'get_nodes':'GetNodes',
                    }

    # use the 'capability' auth mechanism for higher performance when the PLC db is local
    def __init__ ( self, config ) :
        url = config.SFA_PLC_URL
        # try to figure if the url is local
        hostname=urlparse(url).hostname
        is_local=False
        if hostname == 'localhost': is_local=True
        # otherwise compare IP addresses
        url_ip=socket.gethostbyname(hostname)
        local_ip=socket.gethostbyname(socket.gethostname())
        if url_ip==local_ip: is_local=True

        if is_local:
            try:
                # too bad this is not installed properly
                plcapi_path="/usr/share/plc_api"
                if plcapi_path not in sys.path: sys.path.append(plcapi_path)
                import PLC.Shell
                plc_direct_access=True
            except:
                plc_direct_access=False
        if is_local and plc_direct_access:
            logger.debug('plshell access - capability')
            self.plauth = { 'AuthMethod': 'capability',
                            'Username':   config.SFA_PLC_USER,
                            'AuthString': config.SFA_PLC_PASSWORD,
                            }
            self.proxy = PLC.Shell.Shell ()

        else:
            logger.debug('plshell access - xmlrpc')
            self.plauth = { 'AuthMethod': 'password',
                            'Username':   config.SFA_PLC_USER,
                            'AuthString': config.SFA_PLC_PASSWORD,
                            }
            self.proxy = xmlrpclib.Server(url, verbose = 0, allow_none = True)

    def __getattr__(self, name):
        def func(*args, **kwds):
            actual_name=None
            if name in PlShell.direct_calls: actual_name=name
            if name in PlShell.alias_calls: actual_name=PlShell.alias_calls[name]
            if not actual_name:
                raise Exception, "Illegal method call %s for PL driver"%(name)
            result=getattr(self.proxy, actual_name)(self.plauth, *args, **kwds)
            logger.debug('%s (%s) returned ... %s'%(name,actual_name,result))
            return result
        return func
<|endoftext|>