<commit_before>#! /usr/bin/python
# -*- coding: utf-8 -*-

import tensorflow as tf

from tensorlayer.layers.core import Layer
from tensorlayer.layers.core import LayersConfig

from tensorlayer.layers.utils import compute_alpha
from tensorlayer.layers.utils import ternary_operation

from tensorlayer import logging

from tensorlayer.decorators import deprecated_alias

__all__ = ['TernaryConv2d']

class TernaryConv2d(Layer):
    """
    The :class:`TernaryConv2d` class is a 2D binary CNN layer, which weights are either -1 or 1 or 0 while inference.

    Note that, the bias vector would not be tenarized.

    Parameters
    ----------
    prev_layer : :class:`Layer`
        Previous layer.
    n_filter : int
        The number of filters.
    filter_size : tuple of int
        The filter size (height, width).
    strides : tuple of int
        The sliding window strides of corresponding input dimensions.
        It must be in the same order as the ``shape`` parameter.
    act : activation function
        The activation function of this layer.
    padding : str
        The padding algorithm type: "SAME" or "VALID".
    use_gemm : boolean
        If True, use gemm instead of ``tf.matmul`` for inference. (TODO).
    W_init : initializer
        The initializer for the the weight matrix.
    b_init : initializer or None
        The initializer for the the bias vector. If None, skip biases.
    W_init_args : dictionary
        The arguments for the weight matrix initializer.
    b_init_args : dictionary
        The arguments for the bias vector initializer.
    use_cudnn_on_gpu : bool
        Default is False.
    data_format : str
        "NHWC" or "NCHW", default is "NHWC".
    name : str
        A unique layer name.

    Examples
    ---------
    >>> import tensorflow as tf
    >>> import tensorlayer as tl
    >>> x = tf.placeholder(tf.float32, [None, 256, 256, 3])
    >>> net = tl.layers.InputLayer(x, name='input')
    >>> net = tl.layers.TernaryConv2d(net, 32, (5, 5), (1, 1), padding='SAME', name='bcnn1')
    >>> net = tl.layers.MaxPool2d(net, (2, 2), (2, 2), padding='SAME', name='pool1')
    >>> net = tl.layers.BatchNormLayer(net, act=tl.act.htanh, is_train=True, name='bn1')
    ...
    >>> net = tl.layers.SignLayer(net)
    >>> net = tl.layers.TernaryConv2d(net, 64, (5, 5), (1, 1), padding='SAME', name='bcnn2')
    >>> net = tl.layers.MaxPool2d(net, (2, 2), (2, 2), padding='SAME', name='pool2')
    >>> net = tl.layers.BatchNormLayer(net, act=tl.act.htanh, is_train=True, name='bn2')

    """

    @deprecated_alias(layer='prev_layer', end_support_version=1.9)  # TODO remove this line for the 1.9 release
    def __init__(
            self,
            prev_layer,
            n_filter=32,
            filter_size=(3, 3),
            strides=(1, 1),
            act=None,
            padding='SAME',
            use_gemm=False,
            W_init=tf.truncated_normal_initializer(stddev=0.02),
            b_init=tf.constant_initializer(value=0.0),
            W_init_args=None,
            b_init_args=None,
            use_cudnn_on_gpu=None,
            data_format=None,
            # act=None,
            # shape=(5, 5, 1, 100),
            # strides=(1, 1, 1, 1),
            # padding='SAME',
            # W_init=tf.truncated_normal_initializer(stddev=0.02),
            # b_init=tf.constant_initializer(value=0.0),
            # W_init_args=None,
            # b_init_args=None,
            # use_cudnn_on_gpu=None,
            # data_format=None,
            name='ternary_cnn2d',
    ):
        super(TernaryConv2d, self
             ).__init__(prev_layer=prev_layer, act=act, W_init_args=W_init_args, b_init_args=b_init_args, name=name)

        logging.info(
            "TernaryConv2d %s: n_filter: %d filter_size: %s strides: %s pad: %s act: %s" % (
                self.name, n_filter, str(filter_size), str(strides), padding,
                self.act.__name__ if self.act is not None else 'No Activation'
            )
        )

        if len(strides) != 2:
            raise ValueError("len(strides) should be 2.")

        if use_gemm:
            raise Exception("TODO. The current version use tf.matmul for inferencing.")

        try:
            pre_channel = int(prev_layer.outputs.get_shape()[-1])
        except Exception:  # if pre_channel is ?, it happens when using Spatial Transformer Net
            pre_channel = 1
            logging.warning("unknow input channels, set to 1")

        shape = (filter_size[0], filter_size[1], pre_channel, n_filter)
        strides = (1, strides[0], strides[1], 1)

        with tf.variable_scope(name):

            W = tf.get_variable(
                name='W_conv2d', shape=shape, initializer=W_init, dtype=LayersConfig.tf_dtype, **self.W_init_args
            )

            alpha = compute_alpha(W)

            W = ternary_operation(W)
            W = tf.multiply(alpha, W)

            self.outputs = tf.nn.conv2d(
                self.inputs, W, strides=strides, padding=padding, use_cudnn_on_gpu=use_cudnn_on_gpu,
                data_format=data_format
            )

            if b_init:

                b = tf.get_variable(
                    name='b_conv2d', shape=(shape[-1]), initializer=b_init, dtype=LayersConfig.tf_dtype,
                    **self.b_init_args
                )

                self.outputs = tf.nn.bias_add(self.outputs, b, name='bias_add')

            self.outputs = self._apply_activation(self.outputs)

        self._add_layers(self.outputs)

        if b_init:
            self._add_params([W, b])
        else:
            self._add_params(W)
<commit_msg>refactor TernaryConv2d<commit_after>#! /usr/bin/python
# -*- coding: utf-8 -*-

import tensorflow as tf

from tensorlayer.layers.core import Layer
from tensorlayer.layers.core import LayersConfig

from tensorlayer.layers.utils import compute_alpha
from tensorlayer.layers.utils import ternary_operation

from tensorlayer import logging

from tensorlayer.decorators import deprecated_alias

__all__ = ['TernaryConv2d']

class TernaryConv2d(Layer):
    """
    The :class:`TernaryConv2d` class is a 2D binary CNN layer, which weights are either -1 or 1 or 0 while inference.

    Note that, the bias vector would not be tenarized.

    Parameters
    ----------
    n_filter : int
        The number of filters.
    filter_size : tuple of int
        The filter size (height, width).
    strides : tuple of int
        The sliding window strides of corresponding input dimensions.
        It must be in the same order as the ``shape`` parameter.
    act : activation function
        The activation function of this layer.
    padding : str
        The padding algorithm type: "SAME" or "VALID".
    data_format : str
        "NHWC" or "NCHW", default is "NHWC".
    use_gemm : boolean
        If True, use gemm instead of ``tf.matmul`` for inference. (TODO).
    W_init : initializer
        The initializer for the the weight matrix.
    b_init : initializer or None
        The initializer for the the bias vector. If None, skip biases.
    W_init_args : dictionary
        The arguments for the weight matrix initializer.
    b_init_args : dictionary
        The arguments for the bias vector initializer.
    use_cudnn_on_gpu : bool
        Default is False.
    name : None or str
        A unique layer name.

    Examples
    ---------
    >>> import tensorflow as tf
    >>> import tensorlayer as tl
    >>> x = tf.placeholder(tf.float32, [None, 256, 256, 3])
    >>> net = tl.layers.Input(x, name='input')
    >>> net = tl.layers.TernaryConv2d(net, 32, (5, 5), (1, 1), padding='SAME', name='bcnn1')
    >>> net = tl.layers.MaxPool2d(net, (2, 2), (2, 2), padding='SAME', name='pool1')
    >>> net = tl.layers.BatchNorm(net, act=tl.act.htanh, is_train=True, name='bn1')
    ...
    >>> net = tl.layers.Sign(net)
    >>> net = tl.layers.TernaryConv2d(net, 64, (5, 5), (1, 1), padding='SAME', name='bcnn2')
    >>> net = tl.layers.MaxPool2d(net, (2, 2), (2, 2), padding='SAME', name='pool2')
    >>> net = tl.layers.BatchNorm(net, act=tl.act.htanh, is_train=True, name='bn2')

    """

    def __init__(
            self,
            n_filter=32,
            filter_size=(3, 3),
            strides=(1, 1),
            act=None,
            padding='SAME',
            data_format=None,
            use_gemm=False,
            W_init=tf.truncated_normal_initializer(stddev=0.02),
            b_init=tf.constant_initializer(value=0.0),
            W_init_args=None,
            b_init_args=None,
            use_cudnn_on_gpu=None,
            name=None, #'ternary_cnn2d',
    ):
        # super(TernaryConv2d, self
        #      ).__init__(prev_layer=prev_layer, act=act, W_init_args=W_init_args, b_init_args=b_init_args, name=name)
        super().__init__(name)
        self.n_filter = n_filter
        self.filter_size = filter_size
        self.strides = strides
        self.act = act
        self.padding = padding
        self.data_format = data_format
        self.use_gemm = use_gemm
        self.W_init = W_init
        self.b_init = b_init
        self.W_init_args = W_init_args
        self.b_init_args = b_init_args
        self.use_cudnn_on_gpu = use_cudnn_on_gpu

        logging.info(
            "TernaryConv2d %s: n_filter: %d filter_size: %s strides: %s pad: %s act: %s" % (
                self.name, n_filter, str(filter_size), str(strides), padding,
                self.act.__name__ if self.act is not None else 'No Activation'
            )
        )

    def build(self, inputs):
        if len(self.strides) != 2:
            raise ValueError("len(strides) should be 2.")

        if self.use_gemm:
            raise Exception("TODO. The current version use tf.matmul for inferencing.")

        try:
            self.pre_channel = int(inputs.get_shape()[-1])
        except Exception:  # if pre_channel is ?, it happens when using Spatial Transformer Net
            sefl.pre_channel = 1
            logging.warning("unknow input channels, set to 1")

        self.shape = (self.filter_size[0], self.filter_size[1], self.pre_channel, self.n_filter)
        self.strides = (1, self.strides[0], self.strides[1], 1)

        self.W = tf.get_variable(
                name=self.name+'\kernel', shape=self.shape, initializer=self.W_init, dtype=LayersConfig.tf_dtype, **self.W_init_args
            )
        if self.b_init:
            self.b = tf.get_variable(
                name=self.name+'\bias', shape=(self.shape[-1]), initializer=self.b_init, dtype=LayersConfig.tf_dtype,
                **self.b_init_args
            )
            self.add_weights([self.W, self.b])
        else:
            self.add_weights(self.W)
    def forward(self, inputs):

        alpha = compute_alpha(self.W)

        W_ = ternary_operation(self.W)
        W_ = tf.multiply(alpha, W_)

        outputs = tf.nn.conv2d(
            inputs, W_, strides=self.strides, padding=self.padding, use_cudnn_on_gpu=self.use_cudnn_on_gpu,
            data_format=self.data_format
        )

        if self.b_init:
            outputs = tf.nn.bias_add(outputs, self.b, name='bias_add')

        if self.act:
            outputs = self.act(outputs)

        return outputs
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# $Id$
#
# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Sun OS specific tests.  These are implicitly run by test_psutil.py."""

import unittest

class SunOSSpecificTestCase(unittest.TestCase):
    pass
<commit_msg>(sunos) provide specific test case for swap memory<commit_after>#!/usr/bin/env python
#
# $Id$
#
# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Sun OS specific tests.  These are implicitly run by test_psutil.py."""

import psutil
from test_psutil import *

class SunOSSpecificTestCase(unittest.TestCase):

    def test_swap_memory(self):
        out = sh('swap -l -k')
        lines = out.strip().split('\n')[1:]
        if not lines:
            raise ValueError('no swap device(s) configured')
        total = free = 0
        for line in lines:
            line = line.split()
            t, f = line[-2:]
            t = t.replace('K', '')
            f = f.replace('K', '')
            total += int(int(t) * 1024)
            free += int(int(f) * 1024)
        used = total - free

        psutil_swap = psutil.swap_memory()
        self.assertEqual(psutil_swap.total, total)
        self.assertEqual(psutil_swap.used, used)
        self.assertEqual(psutil_swap.free, free)

def test_main():
    test_suite = unittest.TestSuite()
    test_suite.addTest(unittest.makeSuite(SunOSSpecificTestCase))
    result = unittest.TextTestRunner(verbosity=2).run(test_suite)
    return result.wasSuccessful()

if __name__ == '__main__':
    if not test_main():
        sys.exit(1)
<|endoftext|>


<commit_before>import os
import logging
from collections import OrderedDict
from substance.logs import *
from substance.monads import *
from substance.config import Config
from substance.driver import Driver
from substance.constants import (EngineStates)
from substance.exceptions import (FileDoesNotExist)

from vbox import (vboxManager)
import network
import machine
from exceptions import *

from netaddr import (IPAddress, IPNetwork)

logger = logging.getLogger(__name__)

class VirtualBoxDriver(Driver):
  '''
  Substance VirtualBox driver class. Interface to virtual box manager.
  '''

  def __init__(self, core):
    super(self.__class__, self).__init__(core)
    self.config = Config(os.path.join(self.core.getBasePath(), "virtualbox.yml"))

  def assertSetup(self):
    return self.assertConfig().then(self.assertNetworking)

  # -- Configuration

  def getDefaultConfig(self):
    defaults = OrderedDict()
    defaults['network'] = "172.21.21.0/24"
    defaults['interface'] = None
    return defaults

  def makeDefaultConfig(self):
    self.logAdapter.info("Generating default virtualbox config in %s" % self.config.getConfigFile())
    defaults = self.getDefaultConfig()
    for kkk, vvv in defaults.iteritems():
      self.config.set(kkk, vvv)
    return self.config.saveConfig()

  def assertConfig(self):
    return self.config.loadConfigFile() \
      .catchError(FileDoesNotExist, lambda err: self.makeDefaultConfig())

  #---- Networking setup

  def getNetworkInterface(self):
    return self.config.get('interface', None)

  def readNetworkConfig(self):
    netrange = IPNetwork(self.config.get('network'))
    netconfig = {
      'gateway': netrange[1].format(),
      'netmask': netrange.netmask.format(),
      'lowerIP': netrange[2].format(),
      'upperIP': netrange[-1].format()
    }

    return OK(netconfig)

  def assertNetworking(self):
    interface = self.getNetworkInterface()
    netconfig = self.readNetworkConfig()

    if interface:
      hoif = network.readHostOnlyInterface(interface) \
        .catchError(VirtualBoxObjectNotFound, lambda err: OK(None))
      dhcp = network.readDHCP(interface) \
        .catchError(VirtualBoxObjectNotFound, lambda err: OK(None))

      return Try.sequence((netconfig, hoif, dhcp)) \
        .bind(self.assertNetworkConfiguration)
    else:
      return netconfig.bind(self.provisionNetworking)

  def assertNetworkConfiguration(self, netparts):

    (netconfig, hoif, dhcp) = netparts

    if not hoif:
      return self.provisionNetworking(netconfig)
    elif hoif.v4ip.format() != netconfig['gateway']:
      self.logAdapter.warning("VirtualBox interface \"%s\" is not properly configured. Creating a new host-only network." % hoif.name)
      return self.provisionNetworking(netconfig)
    elif dhcp is None:
      self.logAdapter.warning("VirtualBox interface \"%s\" does not have DHCP enabled. Re-Establishing now." % hoif.name)
      return self.provisionDHCP(hoif.name, netconfig)

    return OK(hoif)

  def provisionNetworking(self, netconfig):
    self.logAdapter.info("Provisioning VirtualBox networking for substance")
    ifm = network.addHostOnlyInterface()
    if ifm.isFail():
      return ifm
    iface = ifm.getOK()

    return network.configureHostOnlyInterface(iface, ip=netconfig['gateway'], netmask=netconfig['netmask']) \
      .then(defer(self.provisionDHCP, interface=iface, netconfig=netconfig)) \
      .then(defer(self.saveInterface, iface=iface)) \
      .then(defer(network.readHostOnlyInterface, name=iface))

  def provisionDHCP(self, interface, netconfig):
    self.logAdapter.info("Provisioning DHCP service for host only interface")
    network.removeDHCP(interface).catch(lambda x: OK(interface)) \
      .bind(defer(network.addDHCP, **netconfig))

  def saveInterface(self, iface):
    self.config.set('interface', iface)
    return self.config.saveConfig()

  #---- Machine API

  def importMachine(self, name, ovfFile, engineProfile=None):
    return self.assertSetup() \
      .then(defer(machine.inspectOVF, ovfFile)) \
      .bind(defer(machine.makeImportParams, name=name, engineProfile=engineProfile)) \
      .bind(defer(machine.importOVF, ovfFile=ovfFile, name=name))

  def startMachine(self, uuid):
    '''
    Start the machine by driver identifier.
    '''
    state = machine.readMachineState(uuid)
    if state.isFail():
      return state

    if state.getOK() in [machine.MachineStates.PAUSED]:
      return machine.resume(uuid)
    else:
      return machine.start(uuid)

  def readMachineWaitForNetwork(self, uuid, timeout=5000):
    return machine.readWaitGuestProperty(uuid, "/VirtualBox/GuestInfo/Net", timeout)

  def readMachineNetworkInfo(self, uuid):
    def format(res):
      info = OrderedDict()
      info['sshPort'] = res[0].hostPort if res[0] else None
      info['sshIP'] = '127.0.0.1'
      info['privateIP'] = res[1]
      info['publicIP'] = res[2]
      return info

    return Try.sequence([
      network.readPortForward(uuid, name="substance-ssh"),
      machine.readGuestProperty(uuid, "/VirtualBox/GuestInfo/Net/0/V4/IP"),
      machine.readGuestProperty(uuid, "/VirtualBox/GuestInfo/Net/1/V4/IP")
    ]).map(format)

  def configureMachine(self, uuid, engine):

    def engineFolderToVBoxFolder(acc, x):
      if x.mode == 'sharedfolder':
        acc.append(machine.SharedFolder(name=x.name, hostPath=x.hostPath))
      return acc

    folders = reduce(engineFolderToVBoxFolder, engine.getEngineFolders(), [])
    desiredPort = engine.getSSHPort()

    return self.assertSetup() \
      .then(defer(self.resolvePortConflict, uuid=uuid, desiredPort=desiredPort)) \
      .then(defer(self.configureMachineAdapters, uuid=uuid)) \
      .then(defer(self.configureMachineFolders, uuid=uuid, folders=folders)) \
      .then(defer(self.configureMachineProfile, uuid=uuid, engine=engine))

  def configureMachineProfile(self, uuid, engine):
    self.logAdapter.info("Configure machine profile")
    profile = engine.getProfile()
    return machine.configureProfile(uuid, profile.cpus, profile.memory)

  def configureMachineFolders(self, folders, uuid):
    self.logAdapter.info("Configure machine shared folders")
    return machine.clearSharedFolders(uuid) \
      .then(defer(machine.addSharedFolders, folders=folders, uuid=uuid))

  def resolvePortConflict(self, uuid, desiredPort):
    self.logAdapter.info("Checking for port conflicts")
    return network.readAllPortForwards(ignoreUUIDs=[uuid]) \
      .bind(defer(self.determinePort, desiredPort=desiredPort)) \
      .bind(defer(self.configurePort, uuid=uuid))

  def determinePort(self, usedPorts, desiredPort):
    basePort = 4500
    self.logAdapter.debug("Base port: %s" % basePort)
    self.logAdapter.debug("Desired port: %s" % desiredPort)
    unavailable = map(lambda x: x.hostPort, usedPorts)
    port = desiredPort if desiredPort >= basePort else basePort
    while port in unavailable or port < basePort:
      self.logAdapter.debug("Port %s is in use." % port)
      port += 1
    self.logAdapter.info("Determined SSH port as %s" % port)
    return OK(port)

  def configurePort(self, port, uuid):
    self.logAdapter.debug("Configure substance-ssh port on port %s" % port)
    pf = network.PortForward("substance-ssh", 1, "tcp", None, port, None, 22)
    return network.removePortForwards([pf], uuid) \
      .catch(lambda err: OK(None)) \
      .then(defer(network.addPortForwards, [pf], uuid))

  def configureMachineAdapters(self, uuid):
    interface = self.getNetworkInterface()
    adapters = [
      machine.AdapterSettings(machine.AdapterTypes.NAT, 'default', None, False),
      machine.AdapterSettings(machine.AdapterTypes.HOSTONLY, interface, None, False),
      machine.AdapterSettings(machine.AdapterTypes.NONE, None, None, False),
      machine.AdapterSettings(machine.AdapterTypes.NONE, None, None, False)
    ]
    return Try.sequence([machine.configureAdapter(uuid, i+1, x) for i, x in enumerate(adapters)])

  def suspendMachine(self, uuid):
    '''
    Suspend the machine.
    '''
    return machine.suspend(uuid)

  def haltMachine(self, uuid):
    '''
    Halt the machine.
    '''
    return machine.halt(uuid)

  def terminateMachine(self, uuid):
    '''
    Terminate the machine forcefully.
    '''
    return machine.terminate(uuid)

  def deleteMachine(self, uuid):
    '''
    Delete the machine by driver identifier.
    '''
    return machine.delete(uuid)

  def exportMachine(self, uuid):
    #XXX To be implemented
    pass

  def getMachines(self):
    '''
    Retrieve the list of machines and their driver identifiers.
    '''
    return machine.readMachines()

  # -- Parse results from Virtual Box

  def getMachineID(self, name):
    '''
    Retrieve the driver specific machine ID for a machine name.
    '''
    return machine.findMachineID(name)

  def exists(self, uuid):
    '''
    Check in the driver that the specified identifier exists.
    '''
    return machine.readMachineExists(uuid)

  def isRunning(self, uuid):
    if self.getMachineState(uuid) is EngineStates.RUNNING:
      return True

  def isStopped(self, uuid):
    if self.getMachineState(uuid) is not EngineStates.RUNNING:
      return True

  def isSuspended(self, uuid):
    if self.getMachineState(uuid) is EngineStates.SUSPENDED:
      return True

  def getMachineState(self, uuid):
    '''
    Retrieve the Substance machine state for this driver id
    '''
    return machine.readMachineState(uuid) \
      .bind(self.machineStateToEngineState)

  def machineStateToEngineState(self, vboxState):
    '''
    Resolve a vbox machine state to a substance engine state.
    '''

    mapping = {
      machine.MachineStates.POWEROFF: EngineStates.STOPPED,
      machine.MachineStates.SAVED: EngineStates.SUSPENDED,
      machine.MachineStates.PAUSED: EngineStates.SUSPENDED,
      machine.MachineStates.ABORTED: EngineStates.STOPPED,
      machine.MachineStates.STUCK: EngineStates.STOPPED,
      machine.MachineStates.RESTORING: EngineStates.STOPPED,
      machine.MachineStates.SNAPSHOTTING: EngineStates.STOPPED,
      machine.MachineStates.SETTING_UP: EngineStates.STOPPED,
      machine.MachineStates.ONLINE_SNAPSHOTTING: EngineStates.STOPPED,
      machine.MachineStates.RESTORING_SNAPSHOT: EngineStates.STOPPED,
      machine.MachineStates.DELETING_SNAPSHOT: EngineStates.STOPPED,
      machine.MachineStates.LIVE_SNAPSHOTTING: EngineStates.RUNNING,
      machine.MachineStates.RUNNING: EngineStates.RUNNING,
      machine.MachineStates.STARTING: EngineStates.RUNNING,
      machine.MachineStates.STOPPING : EngineStates.RUNNING,
      machine.MachineStates.SAVING: EngineStates.RUNNING,
      machine.MachineStates.UNKNOWN: EngineStates.UNKNOWN,
      machine.MachineStates.INACCESSIBLE: EngineStates.INEXISTENT,
      machine.MachineStates.INEXISTENT: EngineStates.INEXISTENT
    }
    state = mapping.get(vboxState, EngineStates.UNKNOWN)
    ddebug("Machine state: %s : %s", vboxState, state)
    return OK(state)

<commit_msg>Fix to DHCP range under latest VBOX.<commit_after>import os
import logging
from collections import OrderedDict
from substance.logs import *
from substance.monads import *
from substance.config import Config
from substance.driver import Driver
from substance.constants import (EngineStates)
from substance.exceptions import (FileDoesNotExist)

from vbox import (vboxManager)
import network
import machine
from exceptions import *

from netaddr import (IPAddress, IPNetwork)

logger = logging.getLogger(__name__)

class VirtualBoxDriver(Driver):
  '''
  Substance VirtualBox driver class. Interface to virtual box manager.
  '''

  def __init__(self, core):
    super(self.__class__, self).__init__(core)
    self.config = Config(os.path.join(self.core.getBasePath(), "virtualbox.yml"))

  def assertSetup(self):
    return self.assertConfig().then(self.assertNetworking)

  # -- Configuration

  def getDefaultConfig(self):
    defaults = OrderedDict()
    defaults['network'] = "172.21.21.0/24"
    defaults['interface'] = None
    return defaults

  def makeDefaultConfig(self):
    self.logAdapter.info("Generating default virtualbox config in %s" % self.config.getConfigFile())
    defaults = self.getDefaultConfig()
    for kkk, vvv in defaults.iteritems():
      self.config.set(kkk, vvv)
    return self.config.saveConfig()

  def assertConfig(self):
    return self.config.loadConfigFile() \
      .catchError(FileDoesNotExist, lambda err: self.makeDefaultConfig())

  #---- Networking setup

  def getNetworkInterface(self):
    return self.config.get('interface', None)

  def readNetworkConfig(self):
    netrange = IPNetwork(self.config.get('network'))
    netconfig = {
      'gateway': netrange[1].format(),
      'netmask': netrange.netmask.format(),
      'lowerIP': netrange[2].format(),
      'upperIP': netrange[-2].format()
    }

    return OK(netconfig)

  def assertNetworking(self):
    interface = self.getNetworkInterface()
    netconfig = self.readNetworkConfig()

    if interface:
      hoif = network.readHostOnlyInterface(interface) \
        .catchError(VirtualBoxObjectNotFound, lambda err: OK(None))
      dhcp = network.readDHCP(interface) \
        .catchError(VirtualBoxObjectNotFound, lambda err: OK(None))

      return Try.sequence((netconfig, hoif, dhcp)) \
        .bind(self.assertNetworkConfiguration)
    else:
      return netconfig.bind(self.provisionNetworking)

  def assertNetworkConfiguration(self, netparts):

    (netconfig, hoif, dhcp) = netparts

    if not hoif:
      return self.provisionNetworking(netconfig)
    elif hoif.v4ip.format() != netconfig['gateway']:
      self.logAdapter.warning("VirtualBox interface \"%s\" is not properly configured. Creating a new host-only network." % hoif.name)
      return self.provisionNetworking(netconfig)
    elif dhcp is None:
      self.logAdapter.warning("VirtualBox interface \"%s\" does not have DHCP enabled. Re-Establishing now." % hoif.name)
      return self.provisionDHCP(hoif.name, netconfig)

    return OK(hoif)

  def provisionNetworking(self, netconfig):
    self.logAdapter.info("Provisioning VirtualBox networking for substance")
    ifm = network.addHostOnlyInterface()
    if ifm.isFail():
      return ifm
    iface = ifm.getOK()

    return network.configureHostOnlyInterface(iface, ip=netconfig['gateway'], netmask=netconfig['netmask']) \
      .then(defer(self.provisionDHCP, interface=iface, netconfig=netconfig)) \
      .then(defer(self.saveInterface, iface=iface)) \
      .then(defer(network.readHostOnlyInterface, name=iface))

  def provisionDHCP(self, interface, netconfig):
    self.logAdapter.info("Provisioning DHCP service for host only interface")
    network.removeDHCP(interface).catch(lambda x: OK(interface)) \
      .bind(defer(network.addDHCP, **netconfig))

  def saveInterface(self, iface):
    self.config.set('interface', iface)
    return self.config.saveConfig()

  #---- Machine API

  def importMachine(self, name, ovfFile, engineProfile=None):
    return self.assertSetup() \
      .then(defer(machine.inspectOVF, ovfFile)) \
      .bind(defer(machine.makeImportParams, name=name, engineProfile=engineProfile)) \
      .bind(defer(machine.importOVF, ovfFile=ovfFile, name=name))

  def startMachine(self, uuid):
    '''
    Start the machine by driver identifier.
    '''
    state = machine.readMachineState(uuid)
    if state.isFail():
      return state

    if state.getOK() in [machine.MachineStates.PAUSED]:
      return machine.resume(uuid)
    else:
      return machine.start(uuid)

  def readMachineWaitForNetwork(self, uuid, timeout=5000):
    return machine.readWaitGuestProperty(uuid, "/VirtualBox/GuestInfo/Net", timeout)

  def readMachineNetworkInfo(self, uuid):
    def format(res):
      info = OrderedDict()
      info['sshPort'] = res[0].hostPort if res[0] else None
      info['sshIP'] = '127.0.0.1'
      info['privateIP'] = res[1]
      info['publicIP'] = res[2]
      return info

    return Try.sequence([
      network.readPortForward(uuid, name="substance-ssh"),
      machine.readGuestProperty(uuid, "/VirtualBox/GuestInfo/Net/0/V4/IP"),
      machine.readGuestProperty(uuid, "/VirtualBox/GuestInfo/Net/1/V4/IP")
    ]).map(format)

  def configureMachine(self, uuid, engine):

    def engineFolderToVBoxFolder(acc, x):
      if x.mode == 'sharedfolder':
        acc.append(machine.SharedFolder(name=x.name, hostPath=x.hostPath))
      return acc

    folders = reduce(engineFolderToVBoxFolder, engine.getEngineFolders(), [])
    desiredPort = engine.getSSHPort()

    return self.assertSetup() \
      .then(defer(self.resolvePortConflict, uuid=uuid, desiredPort=desiredPort)) \
      .then(defer(self.configureMachineAdapters, uuid=uuid)) \
      .then(defer(self.configureMachineFolders, uuid=uuid, folders=folders)) \
      .then(defer(self.configureMachineProfile, uuid=uuid, engine=engine))

  def configureMachineProfile(self, uuid, engine):
    self.logAdapter.info("Configure machine profile")
    profile = engine.getProfile()
    return machine.configureProfile(uuid, profile.cpus, profile.memory)

  def configureMachineFolders(self, folders, uuid):
    self.logAdapter.info("Configure machine shared folders")
    return machine.clearSharedFolders(uuid) \
      .then(defer(machine.addSharedFolders, folders=folders, uuid=uuid))

  def resolvePortConflict(self, uuid, desiredPort):
    self.logAdapter.info("Checking for port conflicts")
    return network.readAllPortForwards(ignoreUUIDs=[uuid]) \
      .bind(defer(self.determinePort, desiredPort=desiredPort)) \
      .bind(defer(self.configurePort, uuid=uuid))

  def determinePort(self, usedPorts, desiredPort):
    basePort = 4500
    self.logAdapter.debug("Base port: %s" % basePort)
    self.logAdapter.debug("Desired port: %s" % desiredPort)
    unavailable = map(lambda x: x.hostPort, usedPorts)
    port = desiredPort if desiredPort >= basePort else basePort
    while port in unavailable or port < basePort:
      self.logAdapter.debug("Port %s is in use." % port)
      port += 1
    self.logAdapter.info("Determined SSH port as %s" % port)
    return OK(port)

  def configurePort(self, port, uuid):
    self.logAdapter.debug("Configure substance-ssh port on port %s" % port)
    pf = network.PortForward("substance-ssh", 1, "tcp", None, port, None, 22)
    return network.removePortForwards([pf], uuid) \
      .catch(lambda err: OK(None)) \
      .then(defer(network.addPortForwards, [pf], uuid))

  def configureMachineAdapters(self, uuid):
    interface = self.getNetworkInterface()
    adapters = [
      machine.AdapterSettings(machine.AdapterTypes.NAT, 'default', None, False),
      machine.AdapterSettings(machine.AdapterTypes.HOSTONLY, interface, None, False),
      machine.AdapterSettings(machine.AdapterTypes.NONE, None, None, False),
      machine.AdapterSettings(machine.AdapterTypes.NONE, None, None, False)
    ]
    return Try.sequence([machine.configureAdapter(uuid, i+1, x) for i, x in enumerate(adapters)])

  def suspendMachine(self, uuid):
    '''
    Suspend the machine.
    '''
    return machine.suspend(uuid)

  def haltMachine(self, uuid):
    '''
    Halt the machine.
    '''
    return machine.halt(uuid)

  def terminateMachine(self, uuid):
    '''
    Terminate the machine forcefully.
    '''
    return machine.terminate(uuid)

  def deleteMachine(self, uuid):
    '''
    Delete the machine by driver identifier.
    '''
    return machine.delete(uuid)

  def exportMachine(self, uuid):
    #XXX To be implemented
    pass

  def getMachines(self):
    '''
    Retrieve the list of machines and their driver identifiers.
    '''
    return machine.readMachines()

  # -- Parse results from Virtual Box

  def getMachineID(self, name):
    '''
    Retrieve the driver specific machine ID for a machine name.
    '''
    return machine.findMachineID(name)

  def exists(self, uuid):
    '''
    Check in the driver that the specified identifier exists.
    '''
    return machine.readMachineExists(uuid)

  def isRunning(self, uuid):
    if self.getMachineState(uuid) is EngineStates.RUNNING:
      return True

  def isStopped(self, uuid):
    if self.getMachineState(uuid) is not EngineStates.RUNNING:
      return True

  def isSuspended(self, uuid):
    if self.getMachineState(uuid) is EngineStates.SUSPENDED:
      return True

  def getMachineState(self, uuid):
    '''
    Retrieve the Substance machine state for this driver id
    '''
    return machine.readMachineState(uuid) \
      .bind(self.machineStateToEngineState)

  def machineStateToEngineState(self, vboxState):
    '''
    Resolve a vbox machine state to a substance engine state.
    '''

    mapping = {
      machine.MachineStates.POWEROFF: EngineStates.STOPPED,
      machine.MachineStates.SAVED: EngineStates.SUSPENDED,
      machine.MachineStates.PAUSED: EngineStates.SUSPENDED,
      machine.MachineStates.ABORTED: EngineStates.STOPPED,
      machine.MachineStates.STUCK: EngineStates.STOPPED,
      machine.MachineStates.RESTORING: EngineStates.STOPPED,
      machine.MachineStates.SNAPSHOTTING: EngineStates.STOPPED,
      machine.MachineStates.SETTING_UP: EngineStates.STOPPED,
      machine.MachineStates.ONLINE_SNAPSHOTTING: EngineStates.STOPPED,
      machine.MachineStates.RESTORING_SNAPSHOT: EngineStates.STOPPED,
      machine.MachineStates.DELETING_SNAPSHOT: EngineStates.STOPPED,
      machine.MachineStates.LIVE_SNAPSHOTTING: EngineStates.RUNNING,
      machine.MachineStates.RUNNING: EngineStates.RUNNING,
      machine.MachineStates.STARTING: EngineStates.RUNNING,
      machine.MachineStates.STOPPING : EngineStates.RUNNING,
      machine.MachineStates.SAVING: EngineStates.RUNNING,
      machine.MachineStates.UNKNOWN: EngineStates.UNKNOWN,
      machine.MachineStates.INACCESSIBLE: EngineStates.INEXISTENT,
      machine.MachineStates.INEXISTENT: EngineStates.INEXISTENT
    }
    state = mapping.get(vboxState, EngineStates.UNKNOWN)
    ddebug("Machine state: %s : %s", vboxState, state)
    return OK(state)

<|endoftext|>


<commit_before># coding=utf-8
from phi import math
from phi import struct
from phi.physics.domain import Domain
from phi.physics.field import CenteredGrid
from phi.physics.material import Material
from phi.struct.functions import mappable

class PoissonSolver(object):
    """
    Base class for Poisson solvers
    """

    def __init__(self, name, supported_devices, supports_guess, supports_loop_counter, supports_continuous_masks):
        """Assign details such as name, supported device (CPU/GPU), etc."""
        self.name = name
        self.supported_devices = supported_devices
        self.supports_guess = supports_guess
        self.supports_loop_counter = supports_loop_counter
        self.supports_continuous_masks = supports_continuous_masks

    def solve(self, field, domain, guess, enable_backprop):
        """
        Solves the Poisson equation Δp = d for p for all active fluid cells where active cells are given by the active_mask.
        p is expected to fulfill (Δp-d) ≤ accuracy for every active cell.

        :param enable_backprop: Whether automatic differentiation should be enabled. If False, the gradient of this operation is undefined.
        :param field: scalar input field to the solve, e.g. the divergence of the velocity channel, ∇·v
        :param domain: DomainState object specifying boundary conditions and active/fluid masks. The domain must be equal for all examples (batch dimension equal to 1).
        :param guess: (Optional) Pressure channel which can be used as an initial state for the solver
        :return: pressure tensor (same shape as divergence tensor), number of iterations (integer, 1D integer tensor or None if unknown)
        """
        raise NotImplementedError(self.__class__)

    def __repr__(self):
        """representation = name"""
        return self.name

    def __and__(self, other):
        if isinstance(self, _PoissonSolverChain):
            return _PoissonSolverChain(self.solvers + (other,))
        if isinstance(other, _PoissonSolverChain):
            return _PoissonSolverChain((self,) + other.solvers)
        else:
            return _PoissonSolverChain([self, other])

PressureSolver = PoissonSolver

@struct.definition()
class PoissonDomain(struct.Struct):

    def __init__(self, domain, valid_state=(), active=None, accessible=None, **kwargs):
        struct.Struct.__init__(self, **struct.kwargs(locals(), ignore='valid_state'))
        self._valid_state = valid_state

    @struct.constant()
    def domain(self, domain):
        assert isinstance(domain, Domain), domain
        return domain

    @struct.constant(dependencies='domain')
    def active(self, active):
        extrapolation = _active_extrapolation(Material.extrapolation_mode(self.domain.boundaries))
        if active is not None:
            assert isinstance(active, CenteredGrid)
            assert active.rank == self.domain.rank
            assert active.component_count == 1
            if active.extrapolation != extrapolation:
                active = active.copied_with(extrapolation=extrapolation)
            return active
        else:
            return self.domain.centered_grid(1, extrapolation=extrapolation)

    @struct.constant(dependencies='domain')
    def accessible(self, accessible):
        if accessible is not None:
            assert isinstance(accessible, CenteredGrid)
            assert accessible.rank == self.domain.rank
            assert accessible.component_count == 1
            return accessible
        else:
            return self.domain.centered_grid(1, extrapolation=Material.extrapolation_mode(self.domain.boundaries))

    @property
    def rank(self):
        return self.domain.rank

    def active_tensor(self, extend=0):
        """
        Scalar channel encoding active cells as ones and inactive (open/obstacle) as zero.
        Active cells are those for which physical constants_dict such as pressure or velocity are calculated.

        :param extend: Extend the grid in all directions beyond the grid size specified by the domain
        """
        return self.active.padded([[extend, extend]] * self.rank).data

    def accessible_tensor(self, extend=0):
        """
        Scalar channel encoding cells that are accessible, i.e. not solid, as ones and obstacles as zero.

        :param extend: Extend the grid in all directions beyond the grid size specified by the domain
        """
        pad_values = struct.map(lambda solid: int(not solid), Material.solid(self.domain.boundaries))
        if isinstance(pad_values, (list, tuple)):
            pad_values = [0] + list(pad_values) + [0]
        result = math.pad(self.accessible.data, [[0,0]] + [[extend, extend]] * self.rank + [[0,0]], constant_values=pad_values)
        return result

    def with_hard_boundary_conditions(self, velocity):
        masked = velocity * self._frictionless_velocity_mask(velocity)
        return masked  # TODO add surface velocity

    def _frictionless_velocity_mask(self, velocity):
        tensors = []
        for axis in range(velocity.rank):
            upper = self.accessible.padded([[0, 1] if ax == axis else [0, 0] for ax in range(self.rank)])
            lower = self.accessible.padded([[1, 0] if ax == axis else [0, 0] for ax in range(self.rank)])
            tensors.append(math.minimum(upper.data, lower.data))
        return velocity.with_data(tensors)

FluidDomain = PoissonDomain

@mappable()
def _active_extrapolation(boundaries):
    return 'periodic' if boundaries == 'periodic' else 'constant'

def poisson_solve(input_field, poisson_domain, solver=None, guess=None, gradient='implicit'):
    """
    Solves the Poisson equation Δp = input_field for p.

    :param gradient: one of ('implicit', 'autodiff', 'inverse')
        If 'autodiff', use the built-in autodiff for backpropagation.
        The intermediate results of each loop iteration will be permanently stored if backpropagation is used.
        If 'implicit', computes a forward pressure solve in reverse accumulation backpropagation.
        This requires less memory but is only accurate if the solution is fully converged.
    :param input_field: CenteredGrid
    :param poisson_domain: PoissonDomain instance
    :param solver: PoissonSolver to use, None for default
    :param guess: CenteredGrid with same size and resolution as input_field
    :return: p as CenteredGrid, iteration count as int or None if not available
    :rtype: CenteredGrid, int
    """
    assert isinstance(input_field, CenteredGrid)
    if guess is not None:
        assert isinstance(guess, CenteredGrid)
        assert guess.compatible(input_field)
        guess = guess.data
    if isinstance(poisson_domain, Domain):
        poisson_domain = PoissonDomain(poisson_domain)
    if solver is None:
        solver = _choose_solver(input_field.resolution, math.choose_backend([input_field.data, poisson_domain.active.data, poisson_domain.accessible.data]))
    if not struct.any(Material.open(poisson_domain.domain.boundaries)):  # has no open boundary
        input_field = input_field - math.mean(input_field.data, axis=tuple(range(1, 1 + input_field.rank)), keepdims=True)  # Subtract mean divergence

    assert gradient in ('autodiff', 'implicit', 'inverse')
    if gradient == 'autodiff':
        pressure, iteration = solver.solve(input_field.data, poisson_domain, guess, enable_backprop=True)
    else:
        if gradient == 'implicit':
            def poisson_gradient(_op, grad):
                return poisson_solve(CenteredGrid.sample(grad, poisson_domain.domain), poisson_domain, solver, None, gradient=gradient)[0].data
        else:  # gradient = 'inverse'
            def poisson_gradient(_op, grad):
                return CenteredGrid.sample(grad, poisson_domain.domain).laplace(physical_units=False).data
        pressure, iteration = math.with_custom_gradient(solver.solve, [input_field.data, poisson_domain, guess, False], poisson_gradient, input_index=0, output_index=0, name_base='poisson_solve')

    pressure = CenteredGrid(pressure, input_field.box, extrapolation=input_field.extrapolation, name='pressure')
    return pressure, iteration

class _PoissonSolverChain(PoissonSolver):

    def __init__(self, solvers):
        PoissonSolver.__init__(self, 'chain%s' % solvers, supported_devices=(), supports_guess=solvers[0].supports_guess, supports_loop_counter=solvers[-1].supports_loop_counter, supports_continuous_masks=False)
        self.solvers = tuple(solvers)
        for solver in solvers[1:]:
            assert isinstance(solver, PoissonSolver)
            assert solver.supports_guess

    def solve(self, field, domain, guess):
        iterations = None
        for solver in self.solvers:
            guess, iterations = solver.solve(field, domain, guess)
        return guess, iterations

def _choose_solver(resolution, backend):
    use_fourier = math.max(resolution) > 64
    if backend.precision == 64:
        from .fourier import FourierSolver
        from .geom import GeometricCG
        return FourierSolver() & GeometricCG(accuracy=1e-8) if use_fourier else GeometricCG(accuracy=1e-8)
    elif backend.precision == 32 and backend.matches_name('SciPy'):
        from .sparse import SparseSciPy
        return SparseSciPy()
    elif backend.precision == 32:
        from .fourier import FourierSolver
        from .sparse import SparseCG
        return FourierSolver() & SparseCG(accuracy=1e-5) if use_fourier else SparseCG(accuracy=1e-5)
    else:  # lower precision
        from .geom import GeometricCG
        return GeometricCG(accuracy=1e-2)
<commit_msg>Bug fix: default accessible mask extrapolation<commit_after># coding=utf-8
from phi import math
from phi import struct
from phi.physics.domain import Domain
from phi.physics.field import CenteredGrid
from phi.physics.material import Material
from phi.struct.functions import mappable

class PoissonSolver(object):
    """
    Base class for Poisson solvers
    """

    def __init__(self, name, supported_devices, supports_guess, supports_loop_counter, supports_continuous_masks):
        """Assign details such as name, supported device (CPU/GPU), etc."""
        self.name = name
        self.supported_devices = supported_devices
        self.supports_guess = supports_guess
        self.supports_loop_counter = supports_loop_counter
        self.supports_continuous_masks = supports_continuous_masks

    def solve(self, field, domain, guess, enable_backprop):
        """
        Solves the Poisson equation Δp = d for p for all active fluid cells where active cells are given by the active_mask.
        p is expected to fulfill (Δp-d) ≤ accuracy for every active cell.

        :param enable_backprop: Whether automatic differentiation should be enabled. If False, the gradient of this operation is undefined.
        :param field: scalar input field to the solve, e.g. the divergence of the velocity channel, ∇·v
        :param domain: DomainState object specifying boundary conditions and active/fluid masks. The domain must be equal for all examples (batch dimension equal to 1).
        :param guess: (Optional) Pressure channel which can be used as an initial state for the solver
        :return: pressure tensor (same shape as divergence tensor), number of iterations (integer, 1D integer tensor or None if unknown)
        """
        raise NotImplementedError(self.__class__)

    def __repr__(self):
        """representation = name"""
        return self.name

    def __and__(self, other):
        if isinstance(self, _PoissonSolverChain):
            return _PoissonSolverChain(self.solvers + (other,))
        if isinstance(other, _PoissonSolverChain):
            return _PoissonSolverChain((self,) + other.solvers)
        else:
            return _PoissonSolverChain([self, other])

PressureSolver = PoissonSolver

@struct.definition()
class PoissonDomain(struct.Struct):

    def __init__(self, domain, valid_state=(), active=None, accessible=None, **kwargs):
        struct.Struct.__init__(self, **struct.kwargs(locals(), ignore='valid_state'))
        self._valid_state = valid_state

    @struct.constant()
    def domain(self, domain):
        assert isinstance(domain, Domain), domain
        return domain

    @struct.constant(dependencies='domain')
    def active(self, active):
        extrapolation = _active_extrapolation(Material.extrapolation_mode(self.domain.boundaries))
        if active is not None:
            assert isinstance(active, CenteredGrid)
            assert active.rank == self.domain.rank
            assert active.component_count == 1
            if active.extrapolation != extrapolation:
                active = active.copied_with(extrapolation=extrapolation)
            return active
        else:
            return self.domain.centered_grid(1, extrapolation=extrapolation)

    @struct.constant(dependencies='domain')
    def accessible(self, accessible):
        if accessible is not None:
            assert isinstance(accessible, CenteredGrid)
            assert accessible.rank == self.domain.rank
            assert accessible.component_count == 1
            return accessible
        else:
            return self.domain.centered_grid(1, extrapolation=Material.accessible_extrapolation_mode(self.domain.boundaries))

    @property
    def rank(self):
        return self.domain.rank

    def active_tensor(self, extend=0):
        """
        Scalar channel encoding active cells as ones and inactive (open/obstacle) as zero.
        Active cells are those for which physical constants_dict such as pressure or velocity are calculated.

        :param extend: Extend the grid in all directions beyond the grid size specified by the domain
        """
        return self.active.padded([[extend, extend]] * self.rank).data

    def accessible_tensor(self, extend=0):
        """
        Scalar channel encoding cells that are accessible, i.e. not solid, as ones and obstacles as zero.

        :param extend: Extend the grid in all directions beyond the grid size specified by the domain
        """
        pad_values = struct.map(lambda solid: int(not solid), Material.solid(self.domain.boundaries))
        if isinstance(pad_values, (list, tuple)):
            pad_values = [0] + list(pad_values) + [0]
        result = math.pad(self.accessible.data, [[0,0]] + [[extend, extend]] * self.rank + [[0,0]], constant_values=pad_values)
        return result

    def with_hard_boundary_conditions(self, velocity):
        masked = velocity * self._frictionless_velocity_mask(velocity)
        return masked  # TODO add surface velocity

    def _frictionless_velocity_mask(self, velocity):
        tensors = []
        for axis in range(velocity.rank):
            upper = self.accessible.padded([[0, 1] if ax == axis else [0, 0] for ax in range(self.rank)])
            lower = self.accessible.padded([[1, 0] if ax == axis else [0, 0] for ax in range(self.rank)])
            tensors.append(math.minimum(upper.data, lower.data))
        return velocity.with_data(tensors)

FluidDomain = PoissonDomain

@mappable()
def _active_extrapolation(boundaries):
    return 'periodic' if boundaries == 'periodic' else 'constant'

def poisson_solve(input_field, poisson_domain, solver=None, guess=None, gradient='implicit'):
    """
    Solves the Poisson equation Δp = input_field for p.

    :param gradient: one of ('implicit', 'autodiff', 'inverse')
        If 'autodiff', use the built-in autodiff for backpropagation.
        The intermediate results of each loop iteration will be permanently stored if backpropagation is used.
        If 'implicit', computes a forward pressure solve in reverse accumulation backpropagation.
        This requires less memory but is only accurate if the solution is fully converged.
    :param input_field: CenteredGrid
    :param poisson_domain: PoissonDomain instance
    :param solver: PoissonSolver to use, None for default
    :param guess: CenteredGrid with same size and resolution as input_field
    :return: p as CenteredGrid, iteration count as int or None if not available
    :rtype: CenteredGrid, int
    """
    assert isinstance(input_field, CenteredGrid)
    if guess is not None:
        assert isinstance(guess, CenteredGrid)
        assert guess.compatible(input_field)
        guess = guess.data
    if isinstance(poisson_domain, Domain):
        poisson_domain = PoissonDomain(poisson_domain)
    if solver is None:
        solver = _choose_solver(input_field.resolution, math.choose_backend([input_field.data, poisson_domain.active.data, poisson_domain.accessible.data]))
    if not struct.any(Material.open(poisson_domain.domain.boundaries)):  # has no open boundary
        input_field = input_field - math.mean(input_field.data, axis=tuple(range(1, 1 + input_field.rank)), keepdims=True)  # Subtract mean divergence

    assert gradient in ('autodiff', 'implicit', 'inverse')
    if gradient == 'autodiff':
        pressure, iteration = solver.solve(input_field.data, poisson_domain, guess, enable_backprop=True)
    else:
        if gradient == 'implicit':
            def poisson_gradient(_op, grad):
                return poisson_solve(CenteredGrid.sample(grad, poisson_domain.domain), poisson_domain, solver, None, gradient=gradient)[0].data
        else:  # gradient = 'inverse'
            def poisson_gradient(_op, grad):
                return CenteredGrid.sample(grad, poisson_domain.domain).laplace(physical_units=False).data
        pressure, iteration = math.with_custom_gradient(solver.solve, [input_field.data, poisson_domain, guess, False], poisson_gradient, input_index=0, output_index=0, name_base='poisson_solve')

    pressure = CenteredGrid(pressure, input_field.box, extrapolation=input_field.extrapolation, name='pressure')
    return pressure, iteration

class _PoissonSolverChain(PoissonSolver):

    def __init__(self, solvers):
        PoissonSolver.__init__(self, 'chain%s' % solvers, supported_devices=(), supports_guess=solvers[0].supports_guess, supports_loop_counter=solvers[-1].supports_loop_counter, supports_continuous_masks=False)
        self.solvers = tuple(solvers)
        for solver in solvers[1:]:
            assert isinstance(solver, PoissonSolver)
            assert solver.supports_guess

    def solve(self, field, domain, guess):
        iterations = None
        for solver in self.solvers:
            guess, iterations = solver.solve(field, domain, guess)
        return guess, iterations

def _choose_solver(resolution, backend):
    use_fourier = math.max(resolution) > 64
    if backend.precision == 64:
        from .fourier import FourierSolver
        from .geom import GeometricCG
        return FourierSolver() & GeometricCG(accuracy=1e-8) if use_fourier else GeometricCG(accuracy=1e-8)
    elif backend.precision == 32 and backend.matches_name('SciPy'):
        from .sparse import SparseSciPy
        return SparseSciPy()
    elif backend.precision == 32:
        from .fourier import FourierSolver
        from .sparse import SparseCG
        return FourierSolver() & SparseCG(accuracy=1e-5) if use_fourier else SparseCG(accuracy=1e-5)
    else:  # lower precision
        from .geom import GeometricCG
        return GeometricCG(accuracy=1e-2)
<|endoftext|>


<commit_before>#
# Author    : Manuel Bernal Llinares
# Project   : trackhub-creator
# Timestamp : 07-09-2017 11:24
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
This pipeline creates a trackhub for a PRIDE project, based on the information provided via a JSON formatted file, as it
can be seen on this sample:
{
  "trackHubName" : "PXD000625",
  "trackHubShortLabel" : "PXD000625 - Hepatoc...",
  "trackHubLongLabel" : "Experimental design For the label-free ...",
  "trackHubType" : "PROTEOMICS",
  "trackHubEmail" : "pride-support@ebi.ac.uk",
  "trackHubInternalAbsolutePath" : "...",
  "trackMaps" : [ {
    "trackName" : "PXD000625_10090_Original",
    "trackShortLabel" : "PXD000625 - Mus musc...",
    "trackLongLabel" : "Experimental design For the label-free proteome analysis 17 mice were used composed of 5 ...",
    "trackSpecie" : "10090",
    "pogoFile" : "..."
  } ]
}
"""

import os
import time
# App imports
import config_manager
import ensembl.service
import toolbox.general as general_toolbox
from pipelines.template_pipeline import PogoBasedPipelineDirector, DirectorConfigurationManager

# Globals
__configuration_file = None
__pipeline_arguments = None
__pipeline_director = None

# Pipeline properties access
def set_configuration_file(config_file):
    global __configuration_file
    if __configuration_file is None:
        __configuration_file = config_file
    return __configuration_file

def set_pipeline_arguments(pipeline_arguments):
    global __pipeline_arguments
    if __pipeline_arguments is None:
        __pipeline_arguments = pipeline_arguments
    return __pipeline_arguments

def get_pipeline_director():
    global __pipeline_director
    if __pipeline_director is None:
        __pipeline_director = TrackhubCreatorForProject(config_manager.read_config_from_file(__configuration_file),
                                                        __configuration_file,
                                                        __pipeline_arguments)
    return __pipeline_director

class ConfigManager(DirectorConfigurationManager):
    # Command Line Arguments for this pipeline look like
    #   # This is a JSON formatted file that contains all the relevant information needed for processing the project
    #   # data and create its trackhub
    #   project_data_file=project_data.json

    # Command Line Argument keys
    _CONFIG_COMMAND_LINE_ARGUMENT_KEY_PROJECT_DATA_FILE = 'project_data_file'

    def __init__(self, configuration_object, configuration_file, pipeline_arguments):
        super(ConfigManager, self).__init__(configuration_object, configuration_file, pipeline_arguments)
        # Lazy Process command line arguments
        self.__pipeline_arguments_object = None
        self.__running_mode = None

    def get_project_data_file_path(self):
        return self._get_value_for_pipeline_argument_key(self._CONFIG_COMMAND_LINE_ARGUMENT_KEY_PROJECT_DATA_FILE)

# Models for dealing with the data file that describes the project
class ProjectTrackDescriptor:
    # Project Data File keys relative to every TrackMap object
    _PROJECT_DATA_FILE_KEY_TRACK_NAME = 'trackName'
    _PROJECT_DATA_FILE_KEY_TRACK_SHORT_LABEL = 'trackShortLabel'
    _PROJECT_DATA_FILE_KEY_TRACK_LONG_LABEL = 'trackLongLabel'
    _PROJECT_DATA_FILE_KEY_TRACK_SPECIES = 'trackSpecie'
    _PROJECT_DATA_FILE_KEY_TRACK_POGO_FILE_PATH = 'pogoFile'

    def __init__(self, project_track_descriptor_object):
        self.__project_track_descriptor_object = project_track_descriptor_object

    def _get_value_for_key(self, key, default=""):
        if self.__project_track_descriptor_object and (key in self.__project_track_descriptor_object):
            return self.__project_track_descriptor_object[key]
        return default

    def get_track_name(self):
        return self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACK_NAME)

    def get_track_short_label(self):
        return self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACK_SHORT_LABEL)

    def get_track_long_label(self):
        return self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACK_LONG_LABEL)

    def get_track_species(self):
        return self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACK_SPECIES)

    def get_track_file_path_pogo(self):
        return self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACK_POGO_FILE_PATH)

class ProjectTrackhubDescriptor:
    # Project Data File keys
    _PROJECT_DATA_FILE_KEY_TRACKHUB_NAME = 'trackHubName'
    _PROJECT_DATA_FILE_KEY_TRACKHUB_SHORT_LABEL = 'trackHubShortLabel'
    _PROJECT_DATA_FILE_KEY_TRACKHUB_LONG_LABEL = 'trackHubLongLabel'
    _PROJECT_DATA_FILE_KEY_TRACKHUB_HUB_TYPE = 'trackHubType'
    _PROJECT_DATA_FILE_KEY_TRACKHUB_EMAIL = 'trackHubEmail'
    _PROJECT_DATA_FILE_KEY_TRACKHUB_INTERNAL_ABSOLUTE_PATH = 'trackHubInternalAbsolutePath'
    _PROJECT_DATA_FILE_KEY_TRACKHUB_SECTION_TRACKMAPS = 'trackMaps'

    def __init__(self, project_data_file_path):
        self.__project_data_file_path = project_data_file_path
        self.__project_data_object = None
        self.__project_tracks_descriptors = None

    def _get_project_data_object(self):
        if not self.__project_data_object:
            self.__project_data_object = general_toolbox.read_json(self.__project_data_file_path)
        return self.__project_data_object

    def _get_value_for_key(self, key, default=""):
        # TODO - I should start thinking about refactoring this out
        if key in self._get_project_data_object():
            return self._get_project_data_object()[key]
        return default

    def get_trackhub_name(self):
        return self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACKHUB_NAME,
                                       os.path.basename(self.__project_data_file_path))

    def get_trackhub_short_label(self):
        return self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACKHUB_SHORT_LABEL,
                                       "--- NO SHORT LABEL HAS BEEN DEFINED FOR THIS TRACKHUB ---")

    def get_trackhub_long_label(self):
        return self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACKHUB_LONG_LABEL,
                                       "--- NO LONG LABEL HAS BEEN DEFINED FOR THIS TRACKHUB ---")

    def get_trackhub_hub_type(self):
        return self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACKHUB_HUB_TYPE,
                                       "PROTEOMICS")

    def get_trackhub_email(self):
        return self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACKHUB_EMAIL,
                                       "pride-support@ebi.ac.uk")

    def get_trackhub_destination_path(self):
        return self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACKHUB_INTERNAL_ABSOLUTE_PATH)

    def get_trackhub_project_defined_tracks(self):
        if not self.__project_tracks_descriptors:
            # Default value is an empty list of tracks
            self.__project_tracks_descriptors = []
            data_file_project_track_description_objects = \
                self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACKHUB_SECTION_TRACKMAPS)
            if data_file_project_track_description_objects:
                self.__project_tracks_descriptors = \
                    [ProjectTrackDescriptor(data_file_project_track_description_object)
                     for data_file_project_track_description_object
                     in data_file_project_track_description_objects]
        return self.__project_tracks_descriptors

class PipelineResult:
    # TODO
    _VALUE_STATUS_SUCCESS = 'success'
    _VALUE_STATUS_ERROR = 'error'

    def __init__(self):
        self.status = self._VALUE_STATUS_SUCCESS
        self.error_messages = []
        self.success_messages = []
        self.hub_descriptor_file_path = ""
        # Absolute file path to the folder that represents the running session of the pipeline
        self.file_path_pipeline_session = ""
        # Absolute file path to the log files that belong to the running session of the pipeline
        self.file_path_log_files = []

    def set_status_error(self):
        self.status = self._VALUE_STATUS_ERROR

    def add_error_message(self, error_message):
        # This is the report on the final result from running the pipeline
        self.set_status_error()
        self.error_messages.append(error_message)

    def add_success_message(self, success_message):
        self.success_messages.append(success_message)

    def add_log_files(self, log_files):
        self.file_path_log_files.extend(log_files)

class TrackhubCreatorForProject(PogoBasedPipelineDirector):
    # TODO
    def __init__(self, configuration_object, configuration_file, pipeline_arguments):
        runner_id = "{}-{}".format(__name__, time.time())
        super(TrackhubCreatorForProject, self).__init__(runner_id)
        self.__config_manager = ConfigManager(configuration_object, configuration_file, pipeline_arguments)
        self.__project_trackhub_descriptor = None
        # Only the valid project tracks will be processed for being included in the trackhub
        self.__valid_project_tracks = None
        # Pipeline result object
        self.__pipeline_result_object = PipelineResult()

    def __get_valid_project_tracks(self):
        """
        This helper creates a list of valid trackhub tracks from the given project, i.e. tracks that meet this cirteria:
            - Its taxonomy ID is available on Ensembl
        :return: a list of valid trackhub tracks for the given project
        """
        if not self.__valid_project_tracks:
            self.__valid_project_tracks = []
            ensembl_service = ensembl.service.get_service()
            for project_track_descriptor in self.__project_trackhub_descriptor.get_trackhub_project_defined_tracks():
                if ensembl_service.get_species_data_service().get_species_entry_for_taxonomy_id(
                        project_track_descriptor.get_track_species()):
                    self.__valid_project_tracks.append(project_track_descriptor)
        return self.__valid_project_tracks

    def _before(self):
        # Set Pipeline Session working directory
        self.__pipeline_result_object.file_path_pipeline_session = \
            config_manager.get_app_config_manager().get_session_working_dir()
        # Add this pipeline session log files to the final report
        self.__pipeline_result_object.add_log_files(config_manager.get_app_config_manager().get_session_log_files())
        if self.__config_manager.get_project_data_file_path():
            self._get_logger().info("Reading Project Trackhub Descriptor from file at '{}'"
                                    .format(self.__config_manager.get_project_data_file_path()))
            self.__project_trackhub_descriptor = \
                ProjectTrackhubDescriptor(self.__config_manager.get_project_data_file_path())
            # Check that the destination folder exists
            if not os.path.isdir(self.__project_trackhub_descriptor.get_trackhub_destination_path()):
                error_message = "Trackhub destination path NOT VALID, '{}'"\
                    .format(self.__project_trackhub_descriptor.get_trackhub_destination_path())
                self.__pipeline_result_object.add_error_message(error_message)
                self._get_logger().error(error_message)
                return False
            # Check valid project tracks
            if not self.__get_valid_project_tracks():
                # It makes no sense to go ahead if this project has no valid tracks
                error_message = "Project Trackhub contains NO VALID TRACKS"
                self._get_logger().error(error_message)
                self.__pipeline_result_object.add_error_message(error_message)
                return False
            return True
        error_message = "INVALID / MISSING Project Trackhub Descriptor file, '{}'"\
            .format(self.__config_manager.get_project_data_file_path())
        self.__pipeline_result_object.add_error_message(error_message)
        self._get_logger().error(error_message)
        return False

    def _after(self):
        # TODO - Export to file the pipeline result object
        pass

if __name__ == '__main__':
    print("ERROR: This script is part of a pipeline collection and it is not meant to be run in stand alone mode")
<commit_msg>refactoring<commit_after>#
# Author    : Manuel Bernal Llinares
# Project   : trackhub-creator
# Timestamp : 07-09-2017 11:24
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
This pipeline creates a trackhub for a PRIDE project, based on the information provided via a JSON formatted file, as it
can be seen on this sample:
{
  "trackHubName" : "PXD000625",
  "trackHubShortLabel" : "PXD000625 - Hepatoc...",
  "trackHubLongLabel" : "Experimental design For the label-free ...",
  "trackHubType" : "PROTEOMICS",
  "trackHubEmail" : "pride-support@ebi.ac.uk",
  "trackHubInternalAbsolutePath" : "...",
  "trackMaps" : [ {
    "trackName" : "PXD000625_10090_Original",
    "trackShortLabel" : "PXD000625 - Mus musc...",
    "trackLongLabel" : "Experimental design For the label-free proteome analysis 17 mice were used composed of 5 ...",
    "trackSpecie" : "10090",
    "pogoFile" : "..."
  } ]
}
"""

import os
import time
# App imports
import config_manager
import ensembl.service
import toolbox.general as general_toolbox
from pipelines.template_pipeline import PogoBasedPipelineDirector, DirectorConfigurationManager

# Globals
__configuration_file = None
__pipeline_arguments = None
__pipeline_director = None

# Pipeline properties access
def set_configuration_file(config_file):
    global __configuration_file
    if __configuration_file is None:
        __configuration_file = config_file
    return __configuration_file

def set_pipeline_arguments(pipeline_arguments):
    global __pipeline_arguments
    if __pipeline_arguments is None:
        __pipeline_arguments = pipeline_arguments
    return __pipeline_arguments

def get_pipeline_director():
    global __pipeline_director
    if __pipeline_director is None:
        __pipeline_director = TrackhubCreatorForProject(config_manager.read_config_from_file(__configuration_file),
                                                        __configuration_file,
                                                        __pipeline_arguments)
    return __pipeline_director

class ConfigManager(DirectorConfigurationManager):
    # Command Line Arguments for this pipeline look like
    #   # This is a JSON formatted file that contains all the relevant information needed for processing the project
    #   # data and create its trackhub
    #   project_data_file=project_data.json

    # Command Line Argument keys
    _CONFIG_COMMAND_LINE_ARGUMENT_KEY_PROJECT_DATA_FILE = 'project_data_file'

    def __init__(self, configuration_object, configuration_file, pipeline_arguments):
        super(ConfigManager, self).__init__(configuration_object, configuration_file, pipeline_arguments)
        # Lazy Process command line arguments
        self.__pipeline_arguments_object = None
        self.__running_mode = None

    def get_project_data_file_path(self):
        return self._get_value_for_pipeline_argument_key(self._CONFIG_COMMAND_LINE_ARGUMENT_KEY_PROJECT_DATA_FILE)

# Models for dealing with the data file that describes the project
class ProjectTrackDescriptor:
    # Project Data File keys relative to every TrackMap object
    _PROJECT_DATA_FILE_KEY_TRACK_NAME = 'trackName'
    _PROJECT_DATA_FILE_KEY_TRACK_SHORT_LABEL = 'trackShortLabel'
    _PROJECT_DATA_FILE_KEY_TRACK_LONG_LABEL = 'trackLongLabel'
    _PROJECT_DATA_FILE_KEY_TRACK_SPECIES = 'trackSpecie'
    _PROJECT_DATA_FILE_KEY_TRACK_POGO_FILE_PATH = 'pogoFile'

    def __init__(self, project_track_descriptor_object):
        self.__project_track_descriptor_object = project_track_descriptor_object

    def _get_value_for_key(self, key, default=""):
        if self.__project_track_descriptor_object and (key in self.__project_track_descriptor_object):
            return self.__project_track_descriptor_object[key]
        return default

    def get_track_name(self):
        return self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACK_NAME)

    def get_track_short_label(self):
        return self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACK_SHORT_LABEL)

    def get_track_long_label(self):
        return self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACK_LONG_LABEL)

    def get_track_species(self):
        return self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACK_SPECIES)

    def get_track_file_path_pogo(self):
        return self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACK_POGO_FILE_PATH)

class ProjectTrackhubDescriptor:
    # Project Data File keys
    _PROJECT_DATA_FILE_KEY_TRACKHUB_NAME = 'trackHubName'
    _PROJECT_DATA_FILE_KEY_TRACKHUB_SHORT_LABEL = 'trackHubShortLabel'
    _PROJECT_DATA_FILE_KEY_TRACKHUB_LONG_LABEL = 'trackHubLongLabel'
    _PROJECT_DATA_FILE_KEY_TRACKHUB_HUB_TYPE = 'trackHubType'
    _PROJECT_DATA_FILE_KEY_TRACKHUB_EMAIL = 'trackHubEmail'
    _PROJECT_DATA_FILE_KEY_TRACKHUB_INTERNAL_ABSOLUTE_PATH = 'trackHubInternalAbsolutePath'
    _PROJECT_DATA_FILE_KEY_TRACKHUB_SECTION_TRACKMAPS = 'trackMaps'

    def __init__(self, project_data_file_path):
        self.__project_data_file_path = project_data_file_path
        self.__project_data_object = None
        self.__project_tracks_descriptors = None

    def _get_project_data_object(self):
        if not self.__project_data_object:
            self.__project_data_object = general_toolbox.read_json(self.__project_data_file_path)
        return self.__project_data_object

    def _get_value_for_key(self, key, default=""):
        # TODO - I should start thinking about refactoring this out
        if key in self._get_project_data_object():
            return self._get_project_data_object()[key]
        return default

    def get_trackhub_name(self):
        return self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACKHUB_NAME,
                                       os.path.basename(self.__project_data_file_path))

    def get_trackhub_short_label(self):
        return self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACKHUB_SHORT_LABEL,
                                       "--- NO SHORT LABEL HAS BEEN DEFINED FOR THIS TRACKHUB ---")

    def get_trackhub_long_label(self):
        return self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACKHUB_LONG_LABEL,
                                       "--- NO LONG LABEL HAS BEEN DEFINED FOR THIS TRACKHUB ---")

    def get_trackhub_hub_type(self):
        return self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACKHUB_HUB_TYPE,
                                       "PROTEOMICS")

    def get_trackhub_email(self):
        return self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACKHUB_EMAIL,
                                       "pride-support@ebi.ac.uk")

    def get_trackhub_destination_path(self):
        return self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACKHUB_INTERNAL_ABSOLUTE_PATH)

    def get_trackhub_project_defined_tracks(self):
        if not self.__project_tracks_descriptors:
            # Default value is an empty list of tracks
            self.__project_tracks_descriptors = []
            data_file_project_track_description_objects = \
                self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACKHUB_SECTION_TRACKMAPS)
            if data_file_project_track_description_objects:
                self.__project_tracks_descriptors = \
                    [ProjectTrackDescriptor(data_file_project_track_description_object)
                     for data_file_project_track_description_object
                     in data_file_project_track_description_objects]
        return self.__project_tracks_descriptors

class PipelineResult:
    # TODO
    _VALUE_STATUS_SUCCESS = 'success'
    _VALUE_STATUS_ERROR = 'error'

    def __init__(self):
        self.status = self._VALUE_STATUS_SUCCESS
        self.error_messages = []
        self.success_messages = []
        self.hub_descriptor_file_path = ""
        # Absolute file path to the folder that represents the running session of the pipeline
        self.file_path_pipeline_session = ""
        # Absolute file path to the log files that belong to the running session of the pipeline
        self.file_path_log_files = []

    def set_status_error(self):
        self.status = self._VALUE_STATUS_ERROR

    def add_error_message(self, error_message):
        # This is the report on the final result from running the pipeline
        self.set_status_error()
        self.error_messages.append(error_message)

    def add_success_message(self, success_message):
        self.success_messages.append(success_message)

    def add_log_files(self, log_files):
        self.file_path_log_files.extend(log_files)

class TrackhubCreatorForProject(PogoBasedPipelineDirector):
    # TODO
    def __init__(self, configuration_object, configuration_file, pipeline_arguments):
        runner_id = "{}-{}".format(__name__, time.time())
        super(TrackhubCreatorForProject, self).__init__(runner_id)
        self.__config_manager = ConfigManager(configuration_object, configuration_file, pipeline_arguments)
        self.__project_trackhub_descriptor = None
        # Only the valid project tracks will be processed for being included in the trackhub
        self.__valid_project_tracks = None
        # Pipeline result object
        self.__pipeline_result_object = PipelineResult()

    def __get_valid_project_tracks(self):
        """
        This helper creates a list of valid trackhub tracks from the given project, i.e. tracks that meet this cirteria:
            - Its taxonomy ID is available on Ensembl
        :return: a list of valid trackhub tracks for the given project
        """
        if not self.__valid_project_tracks:
            self.__valid_project_tracks = []
            ensembl_service = ensembl.service.get_service()
            for project_track_descriptor in self.__project_trackhub_descriptor.get_trackhub_project_defined_tracks():
                if ensembl_service.get_species_data_service().get_species_entry_for_taxonomy_id(
                        project_track_descriptor.get_track_species()):
                    self.__valid_project_tracks.append(project_track_descriptor)
        return self.__valid_project_tracks

    def _before(self):
        # Set Pipeline Session working directory
        self.__pipeline_result_object.file_path_pipeline_session = \
            config_manager.get_app_config_manager().get_session_working_dir()
        # Add this pipeline session log files to the final report
        self.__pipeline_result_object.add_log_files(config_manager.get_app_config_manager().get_session_log_files())
        if self.__config_manager.get_project_data_file_path():
            self._get_logger().info("Reading Project Trackhub Descriptor from file at '{}'"
                                    .format(self.__config_manager.get_project_data_file_path()))
            self.__project_trackhub_descriptor = \
                ProjectTrackhubDescriptor(self.__config_manager.get_project_data_file_path())
            # Check that the destination folder exists
            if not os.path.isdir(self.__project_trackhub_descriptor.get_trackhub_destination_path()):
                error_message = "Trackhub destination path NOT VALID, '{}'"\
                    .format(self.__project_trackhub_descriptor.get_trackhub_destination_path())
                self._get_logger().error(error_message)
                self.__pipeline_result_object.add_error_message(error_message)
                return False
            # Check valid project tracks
            if not self.__get_valid_project_tracks():
                # It makes no sense to go ahead if this project has no valid tracks
                error_message = "Project Trackhub contains NO VALID TRACKS"
                self._get_logger().error(error_message)
                self.__pipeline_result_object.add_error_message(error_message)
                return False
            return True
        error_message = "INVALID / MISSING Project Trackhub Descriptor file, '{}'"\
            .format(self.__config_manager.get_project_data_file_path())
        self._get_logger().error(error_message)
        self.__pipeline_result_object.add_error_message(error_message)
        return False

    def _after(self):
        # TODO - Export to file the pipeline result object
        pass

if __name__ == '__main__':
    print("ERROR: This script is part of a pipeline collection and it is not meant to be run in stand alone mode")
<|endoftext|>


<commit_before>#
# Author    : Manuel Bernal Llinares
# Project   : trackhub-creator
# Timestamp : 07-09-2017 11:24
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
This pipeline creates a trackhub for a PRIDE project, based on the information provided via a JSON formatted file, as it
can be seen on this sample:
{
  "trackHubName" : "PXD000625",
  "trackHubShortLabel" : "PXD000625 - Hepatoc...",
  "trackHubLongLabel" : "Experimental design For the label-free ...",
  "trackHubType" : "PROTEOMICS",
  "trackHubEmail" : "pride-support@ebi.ac.uk",
  "trackHubInternalAbsolutePath" : "/nfs/pride/prod/archive/2014/05/PXD000625/internal/trackHub",
  "trackMaps" : [ {
    "trackName" : "PXD000625_10090_Original",
    "trackShortLabel" : "PXD000625 - Mus musc...",
    "trackLongLabel" : "Experimental design For the label-free proteome analysis 17 mice were used composed of 5 ...",
    "trackSpecie" : "10090",
    "pogoFile" : "/nfs/pride/prod/archive/2014/05/PXD000625/internal/PXD000625-10090.pogo"
  } ]
}
"""

# Globals
__configuration_file = None
__pipeline_arguments = None
__pipeline_director = None

# Pipeline properties access
def set_configuration_file(config_file):
    # TODO
    pass

def set_pipeline_arguments(pipeline_arguments):
    # TODO
    pass

def get_pipeline_director():
    # TODO
    pass

if __name__ == '__main__':
    print("ERROR: This script is part of a pipeline collection and it is not meant to be run in stand alone mode")
<commit_msg>set_configuration_file, implementation<commit_after>#
# Author    : Manuel Bernal Llinares
# Project   : trackhub-creator
# Timestamp : 07-09-2017 11:24
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
This pipeline creates a trackhub for a PRIDE project, based on the information provided via a JSON formatted file, as it
can be seen on this sample:
{
  "trackHubName" : "PXD000625",
  "trackHubShortLabel" : "PXD000625 - Hepatoc...",
  "trackHubLongLabel" : "Experimental design For the label-free ...",
  "trackHubType" : "PROTEOMICS",
  "trackHubEmail" : "pride-support@ebi.ac.uk",
  "trackHubInternalAbsolutePath" : "/nfs/pride/prod/archive/2014/05/PXD000625/internal/trackHub",
  "trackMaps" : [ {
    "trackName" : "PXD000625_10090_Original",
    "trackShortLabel" : "PXD000625 - Mus musc...",
    "trackLongLabel" : "Experimental design For the label-free proteome analysis 17 mice were used composed of 5 ...",
    "trackSpecie" : "10090",
    "pogoFile" : "/nfs/pride/prod/archive/2014/05/PXD000625/internal/PXD000625-10090.pogo"
  } ]
}
"""

# Globals
__configuration_file = None
__pipeline_arguments = None
__pipeline_director = None

# Pipeline properties access
def set_configuration_file(config_file):
    global __configuration_file
    if __configuration_file is None:
        __configuration_file = config_file
    return __configuration_file

def set_pipeline_arguments(pipeline_arguments):
    # TODO
    pass

def get_pipeline_director():
    # TODO
    pass

if __name__ == '__main__':
    print("ERROR: This script is part of a pipeline collection and it is not meant to be run in stand alone mode")
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    Author: Romain Deheele. Copyright Camptocamp SA
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
import re
import time

from openerp import addons
from openerp.report import report_sxw
from openerp.osv import orm

from openerp.tools import mod10r
from openerp.tools.translate import _
from openerp.osv import orm
from ..invoice import AccountInvoice

class L10nCHReportWebkitHtmlMulti(report_sxw.rml_parse):
    """Report that output single BVR from invoice.
    This report is deprectated and will be merged
    with multi payment term BVR report when porting to V8"""

    def __init__(self, cr, uid, name, context):
        super(L10nCHReportWebkitHtmlMulti, self).__init__(cr, uid, name,
                                                          context=context)
        self.localcontext.update({
            'time': time,
            'cr': cr,
            'uid': uid,
            'user': self.pool.get("res.users").browse(cr, uid, uid),
            'mod10r': mod10r,
            '_space': AccountInvoice._space,
            '_get_ref': self._get_ref,
            'comma_me': self.comma_me,
            'police_absolute_path': self.police_absolute_path,
            'bvr_absolute_path': self.bvr_absolute_path,
            'headheight': self.headheight,
            'amount': self.amount,
        })

    _compile_get_ref = re.compile('[^0-9]')
    _compile_comma_me = re.compile("^(-?\d+)(\d{3})")
    _compile_check_bvr = re.compile('[0-9][0-9]-[0-9]{3,6}-[0-9]')
    _compile_check_bvr_add_num = re.compile('[0-9]*$')

    def set_context(self, objects, data, ids, report_type=None):
        ml_ids = self.get_obj_reference(ids)
        self._check(ml_ids)
        objects = self.pool['account.move.line'].browse(self.cr, self.uid,
                                                           ml_ids)

        return super(L10nCHReportWebkitHtmlMulti, self).set_context(
            objects,
            data,
            ids,
            report_type=report_type
        )

    def get_obj_reference(self, ids, context=None):
        cursor, uid = self.cr, self.uid
        move_line_obj = self.pool.get('account.move.line')
        account_obj = self.pool.get('account.account')
        invoice_obj = self.pool.get('account.invoice')
        inv = invoice_obj.browse(cursor, uid, ids[0], context=context)
        tier_account_ids = account_obj.search(
            cursor, uid,
            [('type', 'in', ['receivable', 'payable'])],
            context=context)
        move_line_ids = move_line_obj.search(
            cursor, uid,
            [('move_id', '=', inv.move_id.id),
             ('account_id', 'in', tier_account_ids)],
            context=context)
        return move_line_ids

    def amount(self, move, rtype=None):
        ml_model = self.pool['account.move.line']
        return ml_model._get_bvr_amount(self.cr, self.uid, move, rtype=rtype)

    def _get_ref(self, move):
        """Get BVR reference using move related to invoice"""
        model = self.pool['account.move.line']
        return model.get_bvr_ref(self.cr, self.uid, move.id)

    def police_absolute_path(self, inner_path):
        """Will get the ocrb police absolute path"""
        path = addons.get_module_resource('l10n_ch_payment_slip', 'report', inner_path)
        return path

    def bvr_absolute_path(self):
        """Will get the ocrb police absolute path"""
        path = addons.get_module_resource('l10n_ch_payment_slip', 'report', 'bvr1.jpg')
        return path

    def headheight(self):
        report_id = self.pool['ir.actions.report.xml'].search(self.cr, self.uid,
                                                              [('name', '=', 'BVR invoice')])[0]
        report = self.pool['ir.actions.report.xml'].browse(self.cr, self.uid, report_id)
        return report.webkit_header.margin_top

    def comma_me(self, amount):
        """Fast swiss number formatting"""
        if isinstance(amount, float):
            amount = str('%.2f' % amount)
        else:
            amount = str(amount)
        orig = amount
        new = self._compile_comma_me.sub("\g<1>'\g<2>", amount)
        if orig == new:
            return new
        else:
            return self.comma_me(new)

    def _check_invoice(self, invoice_ids):
        """Check if the invoice is ready to be printed"""
        if not invoice_ids:
            invoice_ids = []
        cursor = self.cr
        invoice_obj = self.pool['account.invoice']
        ids = invoice_ids
        for invoice in invoice_obj.browse(cursor, self.uid, ids):
            invoice_name = "%s %s" % (invoice.name, invoice.number)
            if not invoice.number:
                raise orm.except_orm(
                    _('UserError'),
                    _('Your invoice should be validated to generate a BVR reference.')
                )
            if not invoice.partner_bank_id:
                raise orm.except_orm(
                    _('UserError'),
                    _('No bank specified on invoice:\n%s' % (invoice_name))
                )
            if not self._compile_check_bvr.match(
                    invoice.partner_bank_id.get_account_number() or ''):
                raise orm.except_orm(
                    _('UserError'),
                    _(('Your bank BVR number should be of the form 0X-XXX-X! '
                       'Please check your company '
                       'information for the invoice:\n%s') % (invoice_name))
                )
            adherent_num = invoice.partner_bank_id.bvr_adherent_num
            if invoice.partner_bank_id.bvr_adherent_num \
                and not self._compile_check_bvr_add_num.match(adherent_num):
                raise orm.except_orm(
                    _('UserError'),
                    _(('Your bank BVR adherent number must contain only '
                       'digits!\nPlease check your company '
                       'information for the invoice:\n%s') % (invoice_name))
                )
        return ''

    def _check(self, move_ids):
        cursor = self.cr
        move_line_obj = self.pool['account.move.line']
        if not move_ids:
            raise orm.except_orm(
                _('UserError'),
                _('Your invoice should be validated to generate BVR reference.')
            )
        invoice_id = move_line_obj.read(cursor, self.uid, move_ids[0],
                                        ['invoice'])['invoice'][0]
        if invoice_id:
            self._check_invoice([invoice_id])

report_sxw.report_sxw('report.invoice_bvr_webkit_multi',
                      'account.invoice',
                      'l10n_ch_payment_slip/report/multi_bvr.mako',
                      parser=L10nCHReportWebkitHtmlMulti)
<commit_msg>remove import of orm imported twice<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    Author: Romain Deheele. Copyright Camptocamp SA
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
import re
import time

from openerp import addons
from openerp.report import report_sxw
from openerp.osv import orm

from openerp.tools import mod10r
from openerp.tools.translate import _
from ..invoice import AccountInvoice

class L10nCHReportWebkitHtmlMulti(report_sxw.rml_parse):
    """Report that output single BVR from invoice.
    This report is deprectated and will be merged
    with multi payment term BVR report when porting to V8"""

    def __init__(self, cr, uid, name, context):
        super(L10nCHReportWebkitHtmlMulti, self).__init__(cr, uid, name,
                                                          context=context)
        self.localcontext.update({
            'time': time,
            'cr': cr,
            'uid': uid,
            'user': self.pool.get("res.users").browse(cr, uid, uid),
            'mod10r': mod10r,
            '_space': AccountInvoice._space,
            '_get_ref': self._get_ref,
            'comma_me': self.comma_me,
            'police_absolute_path': self.police_absolute_path,
            'bvr_absolute_path': self.bvr_absolute_path,
            'headheight': self.headheight,
            'amount': self.amount,
        })

    _compile_get_ref = re.compile('[^0-9]')
    _compile_comma_me = re.compile("^(-?\d+)(\d{3})")
    _compile_check_bvr = re.compile('[0-9][0-9]-[0-9]{3,6}-[0-9]')
    _compile_check_bvr_add_num = re.compile('[0-9]*$')

    def set_context(self, objects, data, ids, report_type=None):
        ml_ids = self.get_obj_reference(ids)
        self._check(ml_ids)
        objects = self.pool['account.move.line'].browse(self.cr, self.uid,
                                                           ml_ids)

        return super(L10nCHReportWebkitHtmlMulti, self).set_context(
            objects,
            data,
            ids,
            report_type=report_type
        )

    def get_obj_reference(self, ids, context=None):
        cursor, uid = self.cr, self.uid
        move_line_obj = self.pool.get('account.move.line')
        account_obj = self.pool.get('account.account')
        invoice_obj = self.pool.get('account.invoice')
        inv = invoice_obj.browse(cursor, uid, ids[0], context=context)
        tier_account_ids = account_obj.search(
            cursor, uid,
            [('type', 'in', ['receivable', 'payable'])],
            context=context)
        move_line_ids = move_line_obj.search(
            cursor, uid,
            [('move_id', '=', inv.move_id.id),
             ('account_id', 'in', tier_account_ids)],
            context=context)
        return move_line_ids

    def amount(self, move, rtype=None):
        ml_model = self.pool['account.move.line']
        return ml_model._get_bvr_amount(self.cr, self.uid, move, rtype=rtype)

    def _get_ref(self, move):
        """Get BVR reference using move related to invoice"""
        model = self.pool['account.move.line']
        return model.get_bvr_ref(self.cr, self.uid, move.id)

    def police_absolute_path(self, inner_path):
        """Will get the ocrb police absolute path"""
        path = addons.get_module_resource('l10n_ch_payment_slip', 'report', inner_path)
        return path

    def bvr_absolute_path(self):
        """Will get the ocrb police absolute path"""
        path = addons.get_module_resource('l10n_ch_payment_slip', 'report', 'bvr1.jpg')
        return path

    def headheight(self):
        report_id = self.pool['ir.actions.report.xml'].search(self.cr, self.uid,
                                                              [('name', '=', 'BVR invoice')])[0]
        report = self.pool['ir.actions.report.xml'].browse(self.cr, self.uid, report_id)
        return report.webkit_header.margin_top

    def comma_me(self, amount):
        """Fast swiss number formatting"""
        if isinstance(amount, float):
            amount = str('%.2f' % amount)
        else:
            amount = str(amount)
        orig = amount
        new = self._compile_comma_me.sub("\g<1>'\g<2>", amount)
        if orig == new:
            return new
        else:
            return self.comma_me(new)

    def _check_invoice(self, invoice_ids):
        """Check if the invoice is ready to be printed"""
        if not invoice_ids:
            invoice_ids = []
        cursor = self.cr
        invoice_obj = self.pool['account.invoice']
        ids = invoice_ids
        for invoice in invoice_obj.browse(cursor, self.uid, ids):
            invoice_name = "%s %s" % (invoice.name, invoice.number)
            if not invoice.number:
                raise orm.except_orm(
                    _('UserError'),
                    _('Your invoice should be validated to generate a BVR reference.')
                )
            if not invoice.partner_bank_id:
                raise orm.except_orm(
                    _('UserError'),
                    _('No bank specified on invoice:\n%s' % (invoice_name))
                )
            if not self._compile_check_bvr.match(
                    invoice.partner_bank_id.get_account_number() or ''):
                raise orm.except_orm(
                    _('UserError'),
                    _(('Your bank BVR number should be of the form 0X-XXX-X! '
                       'Please check your company '
                       'information for the invoice:\n%s') % (invoice_name))
                )
            adherent_num = invoice.partner_bank_id.bvr_adherent_num
            if invoice.partner_bank_id.bvr_adherent_num \
                and not self._compile_check_bvr_add_num.match(adherent_num):
                raise orm.except_orm(
                    _('UserError'),
                    _(('Your bank BVR adherent number must contain only '
                       'digits!\nPlease check your company '
                       'information for the invoice:\n%s') % (invoice_name))
                )
        return ''

    def _check(self, move_ids):
        cursor = self.cr
        move_line_obj = self.pool['account.move.line']
        if not move_ids:
            raise orm.except_orm(
                _('UserError'),
                _('Your invoice should be validated to generate BVR reference.')
            )
        invoice_id = move_line_obj.read(cursor, self.uid, move_ids[0],
                                        ['invoice'])['invoice'][0]
        if invoice_id:
            self._check_invoice([invoice_id])

report_sxw.report_sxw('report.invoice_bvr_webkit_multi',
                      'account.invoice',
                      'l10n_ch_payment_slip/report/multi_bvr.mako',
                      parser=L10nCHReportWebkitHtmlMulti)
<|endoftext|>