<commit_before># Copyright (c) 2016 Ultimaker B.V.
# Uranium is released under the terms of the AGPLv3 or higher.

from PyQt5.QtCore import QObject, QVariant, pyqtProperty, pyqtSlot, pyqtSignal

from UM.Logger import Logger

import UM.Settings

##  This class provides the value and change notifications for the properties of a single setting
#
#   Since setting values and other properties are provided by a stack, we need some way to
#   query the stack from QML to provide us with those values. This class takes care of that.
#
#   This class provides the property values through QObject dynamic properties so that they
#   are available from QML.
class SettingPropertyProvider(QObject):
    def __init__(self, parent = None, *args, **kwargs):
        super().__init__(parent = parent, *args, **kwargs)

        self._stack_id = ""
        self._stack = None
        self._key = ""
        self._relations = set()
        self._watched_properties = []
        self._property_values = {}
        self._store_index = 0
        self._value_used = None
        self._stack_levels = []

    ##  Set the containerStackId property.
    def setContainerStackId(self, stack_id):
        if stack_id == self._stack_id:
            return # No change.

        self._stack_id = stack_id

        if self._stack:
            self._stack.propertyChanged.disconnect(self._onPropertyChanged)
            self._stack.containersChanged.disconnect(self._update)

        if self._stack_id:
            if self._stack_id == "global":
                self._stack = UM.Application.getInstance().getGlobalContainerStack()
            else:
                stacks = UM.Settings.ContainerRegistry.getInstance().findContainerStacks(id = self._stack_id)
                if stacks:
                    self._stack = stacks[0]

            if self._stack:
                self._stack.propertyChanged.connect(self._onPropertyChanged)
                self._stack.containersChanged.connect(self._update)
        else:
            self._stack = None

        self._update()
        self.containerStackIdChanged.emit()

    ##  Emitted when the containerStackId property changes.
    containerStackIdChanged = pyqtSignal()
    ##  The ID of the container stack we should query for property values.
    @pyqtProperty(str, fset = setContainerStackId, notify = containerStackIdChanged)
    def containerStackId(self):
        return self._stack_id

    ##  Set the watchedProperties property.
    def setWatchedProperties(self, properties):
        if properties != self._watched_properties:
            self._watched_properties = properties
            self._update()
            self.watchedPropertiesChanged.emit()

    ##  Emitted when the watchedProperties property changes.
    watchedPropertiesChanged = pyqtSignal()
    ##  A list of property names that should be watched for changes.
    @pyqtProperty("QVariantList", fset = setWatchedProperties, notify = watchedPropertiesChanged)
    def watchedProperties(self):
        return self._watched_properties

    ##  Set the key property.
    def setKey(self, key):
        if key != self._key:
            self._key = key
            self._update()
            self.keyChanged.emit()

    ##  Emitted when the key property changes.
    keyChanged = pyqtSignal()
    ##  The key of the setting that we should provide property values for.
    @pyqtProperty(str, fset = setKey, notify = keyChanged)
    def key(self):
        return self._key

    propertiesChanged = pyqtSignal()
    @pyqtProperty("QVariantMap", notify = propertiesChanged)
    def properties(self):
        return self._property_values

    def setStoreIndex(self, index):
        if index != self._store_index:
            self._store_index = index
            self.indexChanged.emit()

    storeIndexChanged = pyqtSignal()
    @pyqtProperty(int, fset = setStoreIndex, notify = storeIndexChanged)
    def storeIndex(self):
        return self._store_index

    stackLevelChanged = pyqtSignal()

    ##  At what levels in the stack does the value(s) for this setting occur?
    @pyqtProperty("QVariantList", notify = stackLevelChanged)
    def stackLevels(self):
        if not self._stack:
            return -1
        return self._stack_levels

    def _updateStackLevels(self):
        levels = []
        # Start looking at the stack this provider is attached to.
        current_stack = self._stack
        index = 0
        while current_stack:
            for container in current_stack.getContainers():
                try:
                    if container.getProperty(self._key, "value") is not None:
                        levels.append(index)
                except AttributeError:
                    pass
                index += 1
            # If there is a next stack, check that one as well.
            current_stack = current_stack.getNextStack()
        if levels != self._stack_levels:
            self._stack_levels = levels
            self.stackLevelChanged.emit()

    ##  Set the value of a property.
    #
    #   \param stack_index At which level in the stack should this property be set?
    #   \param property_name The name of the property to set.
    #   \param property_value The value of the property to set.
    @pyqtSlot(str, "QVariant")
    def setPropertyValue(self, property_name, property_value):
        if not self._stack or not self._key:
            return

        if property_name not in self._watched_properties:
            Logger.log("w", "Tried to set a property that is not being watched")
            return

        if self._property_values[property_name] == property_value:
            return

        container = self._stack.getContainer(self._store_index)
        if isinstance(container, UM.Settings.DefinitionContainer):
            return

        container.setProperty(self._key, property_name, property_value, self._stack)

    ##  Manually request the value of a property.
    #   The most notable difference with the properties is that you have more control over at what point in the stack
    #   you want the setting to be retrieved (instead of always taking the top one)
    #   \param property_name The name of the property to get the value from.
    #   \param stack_level the index of the container to get the value from.
    @pyqtSlot(str, int, result = "QVariant")
    def getPropertyValue(self, property_name, stack_level):
        try:
            # Because we continue to count if there are multiple linked stacks, we need to check what stack is targeted
            current_stack = self._stack
            while current_stack:
                num_containers = len(current_stack.getContainers())
                if num_containers < stack_level:
                    stack_level -= num_containers
                    current_stack = self._stack.getNextStack()
                else:
                    break  # Found the right stack

            if not current_stack:
                Logger.log("w", "Could not find the right stack for setting %s at stack level %d while trying to get property %s", self._key, stack_level, property_name)
                return None

            value = current_stack.getContainers()[stack_level].getProperty(self._key, property_name)
        except IndexError:  # Requested stack level does not exist
            Logger.log("w", "Tried to get property of type %s from %s but it did not exist on requested index %d", property_name, self._key, stack_level)
            return None
        return value

    @pyqtSlot(int)
    def removeFromContainer(self, index):
        current_stack = self._stack
        while current_stack:
            num_containers = len(current_stack.getContainers())
            if num_containers < index:
                index -= num_containers
                current_stack = self._stack.getNextStack()
            else:
                break  # Found the right stack

        if not current_stack:
            Logger.log("w", "Unable to remove instance from container because the right stack at stack level %d could not be found", stack_level)
            return

        container = current_stack.getContainer(index)
        if not container or not isinstance(container, UM.Settings.InstanceContainer):
            Logger.log("w", "Unable to remove instance from container as it was either not found or not an instance container")
            return

        container.removeInstance(self._key)

    isValueUsedChanged = pyqtSignal()
    @pyqtProperty(bool, notify = isValueUsedChanged)
    def isValueUsed(self):
        if self._value_used is not None:
            return self._value_used

        definition = self._stack.getSettingDefinition(self._key)
        if not definition:
            return False

        relation_count = 0
        value_used_count = 0
        for key in self._relations:
            # If the setting is not a descendant of this setting, ignore it.
            if not definition.isDescendant(key):
                continue

            relation_count += 1

            if self._stack.getProperty(key, "state") != UM.Settings.InstanceState.User:
                value_used_count += 1

        self._value_used = relation_count == 0 or (relation_count > 0 and value_used_count != 0)
        return self._value_used

    # protected:

    def _onPropertyChanged(self, key, property_name):
        if key != self._key:
            if key in self._relations:
                self._value_used = None
                self.isValueUsedChanged.emit()

            return

        if property_name not in self._watched_properties:
            return

        value = self._getPropertyValue(property_name)

        if self._property_values[property_name] != value:
            self._property_values[property_name] = value
            self.propertiesChanged.emit()
        self._updateStackLevels()

    def _update(self, container = None):
        if not self._stack or not self._watched_properties or not self._key:
            return
        self._updateStackLevels()
        relations = self._stack.getProperty(self._key, "relations")
        if relations:  # If the setting doesn't have the property releations, None is returned
            for relation in filter(lambda r: r.type == UM.Settings.SettingRelation.RelationType.RequiredByTarget and r.role == "value", relations):
                self._relations.add(relation.target.key)

        new_properties = {}
        for property_name in self._watched_properties:
            new_properties[property_name] = self._getPropertyValue(property_name)

        if new_properties != self._property_values:
            self._property_values = new_properties
            self.propertiesChanged.emit()

    def _getPropertyValue(self, property_name):
        property_value = self._stack.getProperty(self._key, property_name)
        if isinstance(property_value, UM.Settings.SettingFunction):
            property_value = property_value(self._stack)

        #if property_value is None:

        if property_name == "value":
            property_value = UM.Settings.SettingDefinition.settingValueToString(self._stack.getProperty(self._key, "type"), property_value)

        return str(property_value)
<commit_msg>Fix small error in SettingPropertyProvider when setting the storeIndex<commit_after># Copyright (c) 2016 Ultimaker B.V.
# Uranium is released under the terms of the AGPLv3 or higher.

from PyQt5.QtCore import QObject, QVariant, pyqtProperty, pyqtSlot, pyqtSignal

from UM.Logger import Logger

import UM.Settings

##  This class provides the value and change notifications for the properties of a single setting
#
#   Since setting values and other properties are provided by a stack, we need some way to
#   query the stack from QML to provide us with those values. This class takes care of that.
#
#   This class provides the property values through QObject dynamic properties so that they
#   are available from QML.
class SettingPropertyProvider(QObject):
    def __init__(self, parent = None, *args, **kwargs):
        super().__init__(parent = parent, *args, **kwargs)

        self._stack_id = ""
        self._stack = None
        self._key = ""
        self._relations = set()
        self._watched_properties = []
        self._property_values = {}
        self._store_index = 0
        self._value_used = None
        self._stack_levels = []

    ##  Set the containerStackId property.
    def setContainerStackId(self, stack_id):
        if stack_id == self._stack_id:
            return # No change.

        self._stack_id = stack_id

        if self._stack:
            self._stack.propertyChanged.disconnect(self._onPropertyChanged)
            self._stack.containersChanged.disconnect(self._update)

        if self._stack_id:
            if self._stack_id == "global":
                self._stack = UM.Application.getInstance().getGlobalContainerStack()
            else:
                stacks = UM.Settings.ContainerRegistry.getInstance().findContainerStacks(id = self._stack_id)
                if stacks:
                    self._stack = stacks[0]

            if self._stack:
                self._stack.propertyChanged.connect(self._onPropertyChanged)
                self._stack.containersChanged.connect(self._update)
        else:
            self._stack = None

        self._update()
        self.containerStackIdChanged.emit()

    ##  Emitted when the containerStackId property changes.
    containerStackIdChanged = pyqtSignal()
    ##  The ID of the container stack we should query for property values.
    @pyqtProperty(str, fset = setContainerStackId, notify = containerStackIdChanged)
    def containerStackId(self):
        return self._stack_id

    ##  Set the watchedProperties property.
    def setWatchedProperties(self, properties):
        if properties != self._watched_properties:
            self._watched_properties = properties
            self._update()
            self.watchedPropertiesChanged.emit()

    ##  Emitted when the watchedProperties property changes.
    watchedPropertiesChanged = pyqtSignal()
    ##  A list of property names that should be watched for changes.
    @pyqtProperty("QVariantList", fset = setWatchedProperties, notify = watchedPropertiesChanged)
    def watchedProperties(self):
        return self._watched_properties

    ##  Set the key property.
    def setKey(self, key):
        if key != self._key:
            self._key = key
            self._update()
            self.keyChanged.emit()

    ##  Emitted when the key property changes.
    keyChanged = pyqtSignal()
    ##  The key of the setting that we should provide property values for.
    @pyqtProperty(str, fset = setKey, notify = keyChanged)
    def key(self):
        return self._key

    propertiesChanged = pyqtSignal()
    @pyqtProperty("QVariantMap", notify = propertiesChanged)
    def properties(self):
        return self._property_values

    def setStoreIndex(self, index):
        if index != self._store_index:
            self._store_index = index
            self.storeIndexChanged.emit()

    storeIndexChanged = pyqtSignal()
    @pyqtProperty(int, fset = setStoreIndex, notify = storeIndexChanged)
    def storeIndex(self):
        return self._store_index

    stackLevelChanged = pyqtSignal()

    ##  At what levels in the stack does the value(s) for this setting occur?
    @pyqtProperty("QVariantList", notify = stackLevelChanged)
    def stackLevels(self):
        if not self._stack:
            return -1
        return self._stack_levels

    def _updateStackLevels(self):
        levels = []
        # Start looking at the stack this provider is attached to.
        current_stack = self._stack
        index = 0
        while current_stack:
            for container in current_stack.getContainers():
                try:
                    if container.getProperty(self._key, "value") is not None:
                        levels.append(index)
                except AttributeError:
                    pass
                index += 1
            # If there is a next stack, check that one as well.
            current_stack = current_stack.getNextStack()
        if levels != self._stack_levels:
            self._stack_levels = levels
            self.stackLevelChanged.emit()

    ##  Set the value of a property.
    #
    #   \param stack_index At which level in the stack should this property be set?
    #   \param property_name The name of the property to set.
    #   \param property_value The value of the property to set.
    @pyqtSlot(str, "QVariant")
    def setPropertyValue(self, property_name, property_value):
        if not self._stack or not self._key:
            return

        if property_name not in self._watched_properties:
            Logger.log("w", "Tried to set a property that is not being watched")
            return

        if self._property_values[property_name] == property_value:
            return

        container = self._stack.getContainer(self._store_index)
        if isinstance(container, UM.Settings.DefinitionContainer):
            return

        container.setProperty(self._key, property_name, property_value, self._stack)

    ##  Manually request the value of a property.
    #   The most notable difference with the properties is that you have more control over at what point in the stack
    #   you want the setting to be retrieved (instead of always taking the top one)
    #   \param property_name The name of the property to get the value from.
    #   \param stack_level the index of the container to get the value from.
    @pyqtSlot(str, int, result = "QVariant")
    def getPropertyValue(self, property_name, stack_level):
        try:
            # Because we continue to count if there are multiple linked stacks, we need to check what stack is targeted
            current_stack = self._stack
            while current_stack:
                num_containers = len(current_stack.getContainers())
                if num_containers < stack_level:
                    stack_level -= num_containers
                    current_stack = self._stack.getNextStack()
                else:
                    break  # Found the right stack

            if not current_stack:
                Logger.log("w", "Could not find the right stack for setting %s at stack level %d while trying to get property %s", self._key, stack_level, property_name)
                return None

            value = current_stack.getContainers()[stack_level].getProperty(self._key, property_name)
        except IndexError:  # Requested stack level does not exist
            Logger.log("w", "Tried to get property of type %s from %s but it did not exist on requested index %d", property_name, self._key, stack_level)
            return None
        return value

    @pyqtSlot(int)
    def removeFromContainer(self, index):
        current_stack = self._stack
        while current_stack:
            num_containers = len(current_stack.getContainers())
            if num_containers < index:
                index -= num_containers
                current_stack = self._stack.getNextStack()
            else:
                break  # Found the right stack

        if not current_stack:
            Logger.log("w", "Unable to remove instance from container because the right stack at stack level %d could not be found", stack_level)
            return

        container = current_stack.getContainer(index)
        if not container or not isinstance(container, UM.Settings.InstanceContainer):
            Logger.log("w", "Unable to remove instance from container as it was either not found or not an instance container")
            return

        container.removeInstance(self._key)

    isValueUsedChanged = pyqtSignal()
    @pyqtProperty(bool, notify = isValueUsedChanged)
    def isValueUsed(self):
        if self._value_used is not None:
            return self._value_used

        definition = self._stack.getSettingDefinition(self._key)
        if not definition:
            return False

        relation_count = 0
        value_used_count = 0
        for key in self._relations:
            # If the setting is not a descendant of this setting, ignore it.
            if not definition.isDescendant(key):
                continue

            relation_count += 1

            if self._stack.getProperty(key, "state") != UM.Settings.InstanceState.User:
                value_used_count += 1

        self._value_used = relation_count == 0 or (relation_count > 0 and value_used_count != 0)
        return self._value_used

    # protected:

    def _onPropertyChanged(self, key, property_name):
        if key != self._key:
            if key in self._relations:
                self._value_used = None
                self.isValueUsedChanged.emit()

            return

        if property_name not in self._watched_properties:
            return

        value = self._getPropertyValue(property_name)

        if self._property_values[property_name] != value:
            self._property_values[property_name] = value
            self.propertiesChanged.emit()
        self._updateStackLevels()

    def _update(self, container = None):
        if not self._stack or not self._watched_properties or not self._key:
            return
        self._updateStackLevels()
        relations = self._stack.getProperty(self._key, "relations")
        if relations:  # If the setting doesn't have the property releations, None is returned
            for relation in filter(lambda r: r.type == UM.Settings.SettingRelation.RelationType.RequiredByTarget and r.role == "value", relations):
                self._relations.add(relation.target.key)

        new_properties = {}
        for property_name in self._watched_properties:
            new_properties[property_name] = self._getPropertyValue(property_name)

        if new_properties != self._property_values:
            self._property_values = new_properties
            self.propertiesChanged.emit()

    def _getPropertyValue(self, property_name):
        property_value = self._stack.getProperty(self._key, property_name)
        if isinstance(property_value, UM.Settings.SettingFunction):
            property_value = property_value(self._stack)

        #if property_value is None:

        if property_name == "value":
            property_value = UM.Settings.SettingDefinition.settingValueToString(self._stack.getProperty(self._key, "type"), property_value)

        return str(property_value)
<|endoftext|>


<commit_before>from os.path import dirname, basename, abspath
from textwrap import dedent

from .config import Configuration
from .utils import Finder, UwsgiRunner

if False:  # pragma: nocover
    from .config import Section

TYPE_UPSTART = 'upstart'
TYPE_SYSTEMD = 'systemd'

TYPES = [
    TYPE_UPSTART,
    TYPE_SYSTEMD,
]

def get_tpl_systemd(conf):
    """

    Some Systemd hints:

        * uwsgiconf sysinit > my.service
        * sudo cp my.service /etc/systemd/system/

        * sudo sh -c "systemctl daemon-reload; systemctl start my.service"

        * journalctl -fu my.service

    :param Section conf: Section object.
    :rtype: str|unicode

    """
    tpl = '''
        # Place into:   /etc/systemd/system/{project}.service
        # Setup:        sudo sh -c "systemctl daemon-reload; systemctl start {project}.service"
        # Start:        systemctl start {project}.service
        # Stop:         systemctl stop {project}.service
        # Restart:      systemctl restart {project}.service
        # Status:       systemctl status {project}.service
        # Journal:      journalctl -fu {project}.service

        [Unit]
        Description={project} uWSGI Service
        After=syslog.target

        [Service]
        Environment="PATH=%(path)s"
        ExecStartPre=-/bin/install -d -m 0755 -o %(user)s -g %(group)s %(runtime_dir)s
        ExecStart={command}
        Restart=on-failure
        KillSignal=SIGTERM
        Type=notify
        StandardError=syslog
        NotifyAccess=all

        [Install]
        WantedBy=multi-user.target
    '''
    # We do not use 'RuntimeDirectory' systemd directive since we need to chown.

    uid, gid = conf.main_process.get_owner()

    tpl = tpl % {
        'runtime_dir': conf.replace_placeholders('{project_runtime_dir}'),
        'path': UwsgiRunner.get_env_path(),
        'user':  uid,
        'group': gid,
    }

    return tpl

def get_tpl_upstart(conf):
    """

    :param Section conf: Section object.
    :rtype: str|unicode

    """
    tpl = '''
        # Place into: /etc/init/{project}.conf
        # Verify:     initctl check-config {project}
        # Start:      initctl start {project}
        # Stop:       initctl stop {project}
        # Restart:    initctl restart {project}

        description "{project} uWSGI Service"
        start on runlevel [2345]
        stop on runlevel [06]

        respawn

        env PATH=%(path)s
        pre-start exec -/bin/install -d -m 0755 -o %(user)s -g %(group)s %(runtime_dir)s

        exec {command}
    '''

    # todo preExec dirs

    uid, gid = conf.main_process.get_owner()

    tpl = tpl % {
        'path': UwsgiRunner.get_env_path(),
        'runtime_dir': conf.replace_placeholders('{project_runtime_dir}'),
        'user': uid,
        'group': gid,
    }

    return tpl

TEMPLATES = {
    TYPE_SYSTEMD: get_tpl_systemd,
    TYPE_UPSTART: get_tpl_upstart,
}

def get_config(systype, conf, conf_path, runner=None, project_name=None):
    """Returns init system configuration file contents.

    :param str|unicode systype: System type alias, e.g. systemd, upstart
    :param Section|Configuration conf: Configuration/Section object.
    :param str|unicode conf_path: File path to a configuration file or a command producing such a configuration.
    :param str|unicode runner: Runner command to execute conf_path. Defaults to ``uwsgiconf`` runner.
    :param str|unicode project_name: Project name to override.
    :rtype: str|unicode

    """
    runner = runner or ('%s run' % Finder.uwsgiconf())
    conf_path = abspath(conf_path)

    if isinstance(conf, Configuration):
        conf = conf.sections[0]  # todo Maybe something more intelligent.

    tpl = dedent(TEMPLATES.get(systype)(conf=conf))

    formatted = tpl.strip().format(
        project=project_name or conf.project_name or basename(dirname(conf_path)),
        command='%s %s' % (runner, conf_path),
    )

    return formatted
<commit_msg>Removed stale todo.<commit_after>from os.path import dirname, basename, abspath
from textwrap import dedent

from .config import Configuration
from .utils import Finder, UwsgiRunner

if False:  # pragma: nocover
    from .config import Section

TYPE_UPSTART = 'upstart'
TYPE_SYSTEMD = 'systemd'

TYPES = [
    TYPE_UPSTART,
    TYPE_SYSTEMD,
]

def get_tpl_systemd(conf):
    """

    Some Systemd hints:

        * uwsgiconf sysinit > my.service
        * sudo cp my.service /etc/systemd/system/

        * sudo sh -c "systemctl daemon-reload; systemctl start my.service"

        * journalctl -fu my.service

    :param Section conf: Section object.
    :rtype: str|unicode

    """
    tpl = '''
        # Place into:   /etc/systemd/system/{project}.service
        # Setup:        sudo sh -c "systemctl daemon-reload; systemctl start {project}.service"
        # Start:        systemctl start {project}.service
        # Stop:         systemctl stop {project}.service
        # Restart:      systemctl restart {project}.service
        # Status:       systemctl status {project}.service
        # Journal:      journalctl -fu {project}.service

        [Unit]
        Description={project} uWSGI Service
        After=syslog.target

        [Service]
        Environment="PATH=%(path)s"
        ExecStartPre=-/bin/install -d -m 0755 -o %(user)s -g %(group)s %(runtime_dir)s
        ExecStart={command}
        Restart=on-failure
        KillSignal=SIGTERM
        Type=notify
        StandardError=syslog
        NotifyAccess=all

        [Install]
        WantedBy=multi-user.target
    '''
    # We do not use 'RuntimeDirectory' systemd directive since we need to chown.

    uid, gid = conf.main_process.get_owner()

    tpl = tpl % {
        'runtime_dir': conf.replace_placeholders('{project_runtime_dir}'),
        'path': UwsgiRunner.get_env_path(),
        'user':  uid,
        'group': gid,
    }

    return tpl

def get_tpl_upstart(conf):
    """

    :param Section conf: Section object.
    :rtype: str|unicode

    """
    tpl = '''
        # Place into: /etc/init/{project}.conf
        # Verify:     initctl check-config {project}
        # Start:      initctl start {project}
        # Stop:       initctl stop {project}
        # Restart:    initctl restart {project}

        description "{project} uWSGI Service"
        start on runlevel [2345]
        stop on runlevel [06]

        respawn

        env PATH=%(path)s
        pre-start exec -/bin/install -d -m 0755 -o %(user)s -g %(group)s %(runtime_dir)s

        exec {command}
    '''

    uid, gid = conf.main_process.get_owner()

    tpl = tpl % {
        'path': UwsgiRunner.get_env_path(),
        'runtime_dir': conf.replace_placeholders('{project_runtime_dir}'),
        'user': uid,
        'group': gid,
    }

    return tpl

TEMPLATES = {
    TYPE_SYSTEMD: get_tpl_systemd,
    TYPE_UPSTART: get_tpl_upstart,
}

def get_config(systype, conf, conf_path, runner=None, project_name=None):
    """Returns init system configuration file contents.

    :param str|unicode systype: System type alias, e.g. systemd, upstart
    :param Section|Configuration conf: Configuration/Section object.
    :param str|unicode conf_path: File path to a configuration file or a command producing such a configuration.
    :param str|unicode runner: Runner command to execute conf_path. Defaults to ``uwsgiconf`` runner.
    :param str|unicode project_name: Project name to override.
    :rtype: str|unicode

    """
    runner = runner or ('%s run' % Finder.uwsgiconf())
    conf_path = abspath(conf_path)

    if isinstance(conf, Configuration):
        conf = conf.sections[0]  # todo Maybe something more intelligent.

    tpl = dedent(TEMPLATES.get(systype)(conf=conf))

    formatted = tpl.strip().format(
        project=project_name or conf.project_name or basename(dirname(conf_path)),
        command='%s %s' % (runner, conf_path),
    )

    return formatted
<|endoftext|>


<commit_before>import json
import oauth2
import optparse
import urllib
import urllib2

parser = optparse.OptionParser()
parser.add_option('-c', '--consumer_key', dest='consumer_key', help='OAuth consumer key (REQUIRED)')
parser.add_option('-s', '--consumer_secret', dest='consumer_secret', help='OAuth consumer secret (REQUIRED)')
parser.add_option('-t', '--token', dest='token', help='OAuth token (REQUIRED)')
parser.add_option('-e', '--token_secret', dest='token_secret', help='OAuth token secret (REQUIRED)')
parser.add_option('-a', '--api_host', dest='api_host', help='API Host', default='api.yelp.com')
parser.add_option('-q', '--term', dest='term', help='Search term')

parser.add_option('-l', '--location', dest='location', help='Location (address)')
parser.add_option('-b', '--bounds', dest='bounds', help='Bounds (sw_latitude,sw_longitude|ne_latitude,ne_longitude)')
parser.add_option('-p', '--point', dest='point', help='Latitude,longitude')
# Not sure if current location hints are currently working
#parser.add_option('-i', '--current_location', dest='current_location', help='Current location latitude,longitude for location disambiguation')

parser.add_option('-o', '--offset', dest='offset', help='Offset (starting position)')
parser.add_option('-r', '--limit', dest='limit', help='Limit (number of results to return)')
parser.add_option('-u', '--cc', dest='cc', help='Country code')
parser.add_option('-n', '--lang', dest='lang', help='Language code')

(options, args) = parser.parse_args()

# Required options
if not options.consumer_key:
	parser.error('--consumer_key required')
if not options.consumer_secret:
	parser.error('--consumer_secret required')
if not options.token:
	parser.error('--token required')
if not options.token_secret:
	parser.error('--token_secret required')

if not options.location and not options.bounds and not options.point:
	parser.error('--location, --bounds, or --point required')

# Setup URL params from options
url_params = {}
if options.term:
	url_params['term'] = options.term
if options.location:
	url_params['location'] = options.location
if options.bounds:
	url_params['bounds'] = options.bounds
if options.point:
	url_params['ll'] = options.point
if options.offset:
	url_params['offset'] = options.offset
if options.limit:
	url_params['limit'] = options.limit
if options.cc:
	url_params['cc'] = options.cc
if options.lang:
	url_params['lang'] = options.lang
if options.current_location:
	url_params['cll'] = options.current_location

# Unsigned URL
url = 'http://%s%s?%s' % (options.api_host, '/v2/search', urllib.urlencode(url_params))
print 'URL: %s' % (url,)

# Sign the URL
consumer = oauth2.Consumer(options.consumer_key, options.consumer_secret)
oauth_request = oauth2.Request('GET', url, {})
oauth_request.update({'oauth_nonce': oauth2.generate_nonce(),
                      'oauth_timestamp': oauth2.generate_timestamp(),
                      'oauth_token': options.token,
                      'oauth_consumer_key': options.consumer_key})

token = oauth2.Token(options.token, options.token_secret)
oauth_request.sign_request(oauth2.SignatureMethod_HMAC_SHA1(), consumer, token)
signed_url = oauth_request.to_url()
print 'Signed URL: %s\n' % (signed_url,)

# Connect
conn = urllib2.urlopen(signed_url, None)
try:
  response = json.loads(conn.read())
finally:
  conn.close()

print json.dumps(response, sort_keys=True, indent=2)

<commit_msg>Fixing option<commit_after>import json
import oauth2
import optparse
import urllib
import urllib2

parser = optparse.OptionParser()
parser.add_option('-c', '--consumer_key', dest='consumer_key', help='OAuth consumer key (REQUIRED)')
parser.add_option('-s', '--consumer_secret', dest='consumer_secret', help='OAuth consumer secret (REQUIRED)')
parser.add_option('-t', '--token', dest='token', help='OAuth token (REQUIRED)')
parser.add_option('-e', '--token_secret', dest='token_secret', help='OAuth token secret (REQUIRED)')
parser.add_option('-a', '--api_host', dest='api_host', help='API Host', default='api.yelp.com')
parser.add_option('-q', '--term', dest='term', help='Search term')

parser.add_option('-l', '--location', dest='location', help='Location (address)')
parser.add_option('-b', '--bounds', dest='bounds', help='Bounds (sw_latitude,sw_longitude|ne_latitude,ne_longitude)')
parser.add_option('-p', '--point', dest='point', help='Latitude,longitude')
# Not sure if current location hints are currently working
parser.add_option('-i', '--current_location', dest='current_location', help='Current location latitude,longitude for location disambiguation')

parser.add_option('-o', '--offset', dest='offset', help='Offset (starting position)')
parser.add_option('-r', '--limit', dest='limit', help='Limit (number of results to return)')
parser.add_option('-u', '--cc', dest='cc', help='Country code')
parser.add_option('-n', '--lang', dest='lang', help='Language code')

(options, args) = parser.parse_args()

# Required options
if not options.consumer_key:
	parser.error('--consumer_key required')
if not options.consumer_secret:
	parser.error('--consumer_secret required')
if not options.token:
	parser.error('--token required')
if not options.token_secret:
	parser.error('--token_secret required')

if not options.location and not options.bounds and not options.point:
	parser.error('--location, --bounds, or --point required')

# Setup URL params from options
url_params = {}
if options.term:
	url_params['term'] = options.term
if options.location:
	url_params['location'] = options.location
if options.bounds:
	url_params['bounds'] = options.bounds
if options.point:
	url_params['ll'] = options.point
if options.offset:
	url_params['offset'] = options.offset
if options.limit:
	url_params['limit'] = options.limit
if options.cc:
	url_params['cc'] = options.cc
if options.lang:
	url_params['lang'] = options.lang
if options.current_location:
	url_params['cll'] = options.current_location

# Unsigned URL
url = 'http://%s%s?%s' % (options.api_host, '/v2/search', urllib.urlencode(url_params))
print 'URL: %s' % (url,)

# Sign the URL
consumer = oauth2.Consumer(options.consumer_key, options.consumer_secret)
oauth_request = oauth2.Request('GET', url, {})
oauth_request.update({'oauth_nonce': oauth2.generate_nonce(),
                      'oauth_timestamp': oauth2.generate_timestamp(),
                      'oauth_token': options.token,
                      'oauth_consumer_key': options.consumer_key})

token = oauth2.Token(options.token, options.token_secret)
oauth_request.sign_request(oauth2.SignatureMethod_HMAC_SHA1(), consumer, token)
signed_url = oauth_request.to_url()
print 'Signed URL: %s\n' % (signed_url,)

# Connect
conn = urllib2.urlopen(signed_url, None)
try:
  response = json.loads(conn.read())
finally:
  conn.close()

print json.dumps(response, sort_keys=True, indent=2)

<|endoftext|>


<commit_before>"""A setuptools based setup module.

See:
https://packaging.python.org/en/latest/distributing.html
https://github.com/pypa/sampleproject
"""

import sys

# To use a consistent encoding
from codecs import open
from os import path

# Always prefer setuptools over distutils
from setuptools import find_packages, setup

if "bdist_wheel" in sys.argv:
    raise RuntimeError("This setup.py does not support wheels")

here = path.abspath(path.dirname(__file__))

# Get the long description from the relevant file
with open(path.join(here, "README.md"), encoding="utf-8") as f:
    long_description = f.read()

setup(
    name="ctf-shell-manager",
    # Versions should comply with PEP440.  For a discussion on single-sourcing
    # the version across setup.py and the project code, see
    # https://packaging.python.org/en/latest/single_source_version.html
    version="1.2.1",
    description="deploy and package hacksport problems",
    long_description=long_description,
    # The project's main homepage.
    url="https://github.com/picoCTF/picoCTF",
    # Author details
    author="Christopher Ganas",
    author_email="cganas@forallsecure.com",
    # Choose your license
    license="MIT",
    # See https://pypi.python.org/pypi?%3Aaction=list_classifiers
    classifiers=[
        # How mature is this project? Common values are
        #   3 - Alpha
        #   4 - Beta
        #   5 - Production/Stable
        "Development Status :: 3 - Alpha",
        # Indicate who your project is intended for
        "Intended Audience :: Developers",
        "Topic :: Software Development :: Build Tools",
        # Specify the Python versions you support here. In particular, ensure
        # that you indicate whether you support Python 2, Python 3 or both.
        "Programming Language :: Python :: 3.7",
    ],
    # What does your project relate to?
    keywords="ctf hacksports",
    # You can just specify the packages manually here if your project is
    # simple. Or you can use find_packages().
    packages=find_packages(exclude=["contrib", "docs", "tests*"]),
    # List run-time dependencies here.  These will be installed by pip when
    # your project is installed. For an analysis of "install_requires" vs pip's
    # requirements files see:
    # https://packaging.python.org/en/latest/requirements.html
    install_requires=[
        "coloredlogs==10.0",
        "Flask==1.1.1",
        "Jinja2==2.10.1",
        "openssh-wrapper==0.4",
        "psutil==5.6.3",
        "pytest==3.6.1",
        "spur==0.3.21",
        "voluptuous==0.11.7",
        "Werkzeug==0.15.5",
    ],
    extras_require={"dev": ["black", "flake8", "pydocstyle"]},
    # To provide executable scripts, use entry points in preference to the
    # "scripts" keyword. Entry points provide cross-platform support and allow
    # pip to create the appropriate form of executable for the target platform.
    entry_points={"console_scripts": ["shell_manager=shell_manager.run:main"]},
)
<commit_msg>Bump psutil from 5.6.3 to 5.6.6 in /picoCTF-shell<commit_after>"""A setuptools based setup module.

See:
https://packaging.python.org/en/latest/distributing.html
https://github.com/pypa/sampleproject
"""

import sys

# To use a consistent encoding
from codecs import open
from os import path

# Always prefer setuptools over distutils
from setuptools import find_packages, setup

if "bdist_wheel" in sys.argv:
    raise RuntimeError("This setup.py does not support wheels")

here = path.abspath(path.dirname(__file__))

# Get the long description from the relevant file
with open(path.join(here, "README.md"), encoding="utf-8") as f:
    long_description = f.read()

setup(
    name="ctf-shell-manager",
    # Versions should comply with PEP440.  For a discussion on single-sourcing
    # the version across setup.py and the project code, see
    # https://packaging.python.org/en/latest/single_source_version.html
    version="1.2.1",
    description="deploy and package hacksport problems",
    long_description=long_description,
    # The project's main homepage.
    url="https://github.com/picoCTF/picoCTF",
    # Author details
    author="Christopher Ganas",
    author_email="cganas@forallsecure.com",
    # Choose your license
    license="MIT",
    # See https://pypi.python.org/pypi?%3Aaction=list_classifiers
    classifiers=[
        # How mature is this project? Common values are
        #   3 - Alpha
        #   4 - Beta
        #   5 - Production/Stable
        "Development Status :: 3 - Alpha",
        # Indicate who your project is intended for
        "Intended Audience :: Developers",
        "Topic :: Software Development :: Build Tools",
        # Specify the Python versions you support here. In particular, ensure
        # that you indicate whether you support Python 2, Python 3 or both.
        "Programming Language :: Python :: 3.7",
    ],
    # What does your project relate to?
    keywords="ctf hacksports",
    # You can just specify the packages manually here if your project is
    # simple. Or you can use find_packages().
    packages=find_packages(exclude=["contrib", "docs", "tests*"]),
    # List run-time dependencies here.  These will be installed by pip when
    # your project is installed. For an analysis of "install_requires" vs pip's
    # requirements files see:
    # https://packaging.python.org/en/latest/requirements.html
    install_requires=[
        "coloredlogs==10.0",
        "Flask==1.1.1",
        "Jinja2==2.10.1",
        "openssh-wrapper==0.4",
        "psutil==5.6.6",
        "pytest==3.6.1",
        "spur==0.3.21",
        "voluptuous==0.11.7",
        "Werkzeug==0.15.5",
    ],
    extras_require={"dev": ["black", "flake8", "pydocstyle"]},
    # To provide executable scripts, use entry points in preference to the
    # "scripts" keyword. Entry points provide cross-platform support and allow
    # pip to create the appropriate form of executable for the target platform.
    entry_points={"console_scripts": ["shell_manager=shell_manager.run:main"]},
)
<|endoftext|>


<commit_before>from pikos.recorders.list_recorder import ListRecorder
from pikos.recorders.text_file_recorder import TextFileRecorder
from pikos.recorders.csv_file_recorder import CSVFileRecorder
from pikos.recorders.csv_recorder import CSVRecorder
from pikos.recorders.text_stream_recorder import TextStreamRecorder
from pikos.recorders.zeromq_recorder import ZeroMQRecorder
<commit_msg>remove zero mq recorder from api since it is optional<commit_after>from pikos.recorders.list_recorder import ListRecorder
from pikos.recorders.text_file_recorder import TextFileRecorder
from pikos.recorders.csv_file_recorder import CSVFileRecorder
from pikos.recorders.csv_recorder import CSVRecorder
from pikos.recorders.text_stream_recorder import TextStreamRecorder
<|endoftext|>


<commit_before>
# This section is used by the variant calling pipeline.py to specify reference data files.
#
# Note that if you have downloaded the pipeline the directory names below are examples
# only and you will need to edit them to suit your needs.
#
# Required variables:
#  - fastq_dirs: a list of directories where the raw input data is found. Currently this
#      data is expected to be paired-end gzipped fastq and to follow a specific naming
#      convention (see below).
#  - fastq_symlink_dir: symlinks to all raw fastq files will be written to this directory
#      and used by the rest of the pipeline. These symlinks have standardised names and
#      are a useful flattened summary of all known input data.
#  - output_dir: the directory used by the pipeline for output and intermediate files.
#      A directory structure will be created under this directory by pipeline.py.
#
# Input data naming convention:
# The input fastq files must follow a naming convention so that the pipeline can determine
# the metadata fields. This convention in the default script is to use the regex
#     ([a-zA-Z0-9-.]+)_([^_/]+)_[CAGTN]+_L([0-9]+)_R(1|2).fastq.gz
# This corresponds to metadata fields
#    SAMPLE_RUN_TAG_LANE_READPAIR.fastq.gz
# where
#    SAMPLE is a unique identifier for the sample sequenced
#    RUN is a unique identifier for the experiment (e.g. run or flowcell ID)
#    TAG is the barcode sequence used for multiplexing (NA if none)
#    LANE is the flowcell lane identifier, written like L001
#    READPAIR identifies whether the file contains forward or reverse reads, R1 or R2
#
# For example: Sample395_C0WK7ACXX_ACTTGA_L007_R1.fastq.gz
#
# This file naming convention follows that returned by many sequencing centres for
# Illumina data.
#
working_files = {
    'fastq_dirs': [
         '/vlsci/VR0244/shared/meg/EXOME/FASTQ'
    ],
    'fastq_symlink_dir': '/vlsci/VR0244/shared/test_ruffus_wgs/fastq_symlinks'
    'output_dir': '/vlsci/VR0244/shared/test_ruffus_wgs/output',
}

# This section is used by the variant calling pipeline.py to specify reference data files.
#
# Note that if you have downloaded the pipeline the filenames below are examples only and
# you will need to get the relevant reference files for your data. Exactly which files
# you need depend on your data. At time of writing reference data can be obtained from:
#  - Reference genome: many sources depending on data. For our human data we used the
#     1000 genomes version of the b37 (hg19) genome build, found at
#     ftp://ftp-trace.ncbi.nih.gov/1000genomes/ftp/technical/reference/human_g1k_v37.fasta.gz
#     Note that your genome must use the same chromosome naming convention as any other
#     reference files (such as dbSNP); if you use hg19 (chr1,chr2) instead of b37 (1,2)
#     you may need to convert the files suggested below.
#  - dbSNP variants: dbSNP is at http://www.ncbi.nlm.nih.gov/projects/SNP/
#     A useful release summary is at http://www.ncbi.nlm.nih.gov/projects/SNP/snp_summary.cgi
#     We used human variants which were obtained in VCF format from
#     ftp://ftp.ncbi.nlm.nih.gov/snp/organisms/human_9606/VCF/00-All.vcf.gz
#  - known indels for local realignment: these follow the Broad recommendations for the
#     GATK tool suite and come from the GATK resource bundle. See
#     http://gatkforums.broadinstitute.org/discussion/1213/what-s-in-the-resource-bundle-and-how-can-i-get-it
#
# Expected variables (if you use the relevant pipeline steps):
#  - fasta_reference: the reference genome fasta. Should be in the same location as the
#      .fai files produced by samtools faidx.
#      TODO: do this indexing as part of the pipeline and check for index files.
#  - bwa_reference: the reference genome fasta. Should be in the same location as the
#      index files produced by bwa index.
#      TODO: do this indexing as part of the pipeline and check for index files.
#  - dbsnp: the dbSNP variants file in VCF format, for annotating variants and for
#       GATK base quality recalibration.
#  - indels_realign_goldstandard and
#  - indels_realign_1000G: files of known indels for use in GATK local realignment.
#       Currently the Broad Institute recommends using these two files (see above).
ref_files = {
    'fasta_reference': '/vlsci/VR0002/shared/Reference_Files/Indexed_Ref_Genomes/bwa_Indexed/human_g1k_v37.fasta',
    'bwa_reference': '/vlsci/VR0002/shared/Reference_Files/Indexed_Ref_Genomes/bwa_Indexed/human_g1k_v37.fasta',

    'dbsnp': '/vlsci/VR0002/shared/Reference_Files/SNP_db/dbSNP137.vcf',

    'indels_realign_goldstandard': '/vlsci/VR0002/shared/Reference_Files/Indels_for_realignment/Mills_and_1000G_gold_standard.indels.b37.vcf',
    'indels_realign_1000G': '/vlsci/VR0002/shared/Reference_Files/Indels_for_realignment/1000G_phase1.indels.b37.vcf'
}

# pipeline should hold configuration options for Rubra and for the pipeline.
# This section is required for every Rubra pipeline,
# but restrict_samples and allowed_samples are specific to the variant-calling pipeline.
#
# Rubra variables:
#  - logDir: the directory where batch queue scripts, stdout and sterr dumps are stored.
#  - logFile: the file used to log all jobs that are run.
#  - style: the default style, one of 'flowchart', 'print', 'run', 'touchfiles'. Can be
#      overridden by specifying --style on the command line.
#  - procs: the number of python processes to run simultaneously. This determines the
#      maximum parallelism of the pipeline. For distributed jobs it also constrains the
#      maximum total jobs submitted to the queue at any one time.
#  - verbosity: one of 0 (quiet), 1 (normal), 2 (chatty). Can be overridden by specifying
#      --verbose on the command line.
#  - end: the desired tasks to be run. Rubra will also run all tasks which are dependencies
#      of these tasks. Can be overridden by specifying --end on the command line.
#  - force: tasks which will be forced to run, regardless of timestamps. Can be overridden
#      by supplying --force on the command line.
#  - rebuild: one of 'fromstart','fromend'. Whether to calculate which dependencies will
#      be rerun by working back from an end task to the latest up-to-date task, or forward
#      from the earliest out-of-date task. 'fromstart' is the most conservative and
#      commonly used as it brings all intermediate tasks up to date.
#
# Variant-calling pipeline variables:  (TODO: move to a separate section)
#  - restrict_samples: whether to restrict input files to those specified by allowd_samples
#  - allowed_samples: sample names that will be run of restrict_samples is True
pipeline = {
    'logDir': 'log_varcall_test',
    'logFile': 'pipeline.log',
    'style': 'print',
    'procs': 30,
    'verbose': 1,
    'end': ['earlyDepthOfCoverage', 'finalDepthOfCoverage',
            'fastqc',
            'igvcountMergedBams', 'countDedupedBam', 'countRunBam', 'countMergedBam',
            'getEnsemblAnnotations',
            'collateReadCounts',
            'vcfIndexSNPs', 'vcfIndexIndels'
            ],
    'force': [],
    'rebuild' : "fromstart",

    'restrict_samples': True,
    'allowed_samples': ['mw4241']
}
<commit_msg>Documentation typo fix.<commit_after>
# This section is used by the variant calling pipeline.py to specify input data and
# working directories.
#
# Note that if you have downloaded the pipeline the directory names below are examples
# only and you will need to edit them to suit your needs.
#
# Required variables:
#  - fastq_dirs: a list of directories where the raw input data is found. Currently this
#      data is expected to be paired-end gzipped fastq and to follow a specific naming
#      convention (see below).
#  - fastq_symlink_dir: symlinks to all raw fastq files will be written to this directory
#      and used by the rest of the pipeline. These symlinks have standardised names and
#      are a useful flattened summary of all known input data.
#  - output_dir: the directory used by the pipeline for output and intermediate files.
#      A directory structure will be created under this directory by pipeline.py.
#
# Input data naming convention:
# The input fastq files must follow a naming convention so that the pipeline can determine
# the metadata fields. This convention in the default script is to use the regex
#     ([a-zA-Z0-9-.]+)_([^_/]+)_[CAGTN]+_L([0-9]+)_R(1|2).fastq.gz
# This corresponds to metadata fields
#    SAMPLE_RUN_TAG_LANE_READPAIR.fastq.gz
# where
#    SAMPLE is a unique identifier for the sample sequenced
#    RUN is a unique identifier for the experiment (e.g. run or flowcell ID)
#    TAG is the barcode sequence used for multiplexing (NA if none)
#    LANE is the flowcell lane identifier, written like L001
#    READPAIR identifies whether the file contains forward or reverse reads, R1 or R2
#
# For example: Sample395_C0WK7ACXX_ACTTGA_L007_R1.fastq.gz
#
# This file naming convention follows that returned by many sequencing centres for
# Illumina data.
#
working_files = {
    'fastq_dirs': [
         '/vlsci/VR0244/shared/meg/EXOME/FASTQ'
    ],
    'fastq_symlink_dir': '/vlsci/VR0244/shared/test_ruffus_wgs/fastq_symlinks'
    'output_dir': '/vlsci/VR0244/shared/test_ruffus_wgs/output',
}

# This section is used by the variant calling pipeline.py to specify reference data files.
#
# Note that if you have downloaded the pipeline the filenames below are examples only and
# you will need to get the relevant reference files for your data. Exactly which files
# you need depend on your data. At time of writing reference data can be obtained from:
#  - Reference genome: many sources depending on data. For our human data we used the
#     1000 genomes version of the b37 (hg19) genome build, found at
#     ftp://ftp-trace.ncbi.nih.gov/1000genomes/ftp/technical/reference/human_g1k_v37.fasta.gz
#     Note that your genome must use the same chromosome naming convention as any other
#     reference files (such as dbSNP); if you use hg19 (chr1,chr2) instead of b37 (1,2)
#     you may need to convert the files suggested below.
#  - dbSNP variants: dbSNP is at http://www.ncbi.nlm.nih.gov/projects/SNP/
#     A useful release summary is at http://www.ncbi.nlm.nih.gov/projects/SNP/snp_summary.cgi
#     We used human variants which were obtained in VCF format from
#     ftp://ftp.ncbi.nlm.nih.gov/snp/organisms/human_9606/VCF/00-All.vcf.gz
#  - known indels for local realignment: these follow the Broad recommendations for the
#     GATK tool suite and come from the GATK resource bundle. See
#     http://gatkforums.broadinstitute.org/discussion/1213/what-s-in-the-resource-bundle-and-how-can-i-get-it
#
# Expected variables (if you use the relevant pipeline steps):
#  - fasta_reference: the reference genome fasta. Should be in the same location as the
#      .fai files produced by samtools faidx.
#      TODO: do this indexing as part of the pipeline and check for index files.
#  - bwa_reference: the reference genome fasta. Should be in the same location as the
#      index files produced by bwa index.
#      TODO: do this indexing as part of the pipeline and check for index files.
#  - dbsnp: the dbSNP variants file in VCF format, for annotating variants and for
#       GATK base quality recalibration.
#  - indels_realign_goldstandard and
#  - indels_realign_1000G: files of known indels for use in GATK local realignment.
#       Currently the Broad Institute recommends using these two files (see above).
ref_files = {
    'fasta_reference': '/vlsci/VR0002/shared/Reference_Files/Indexed_Ref_Genomes/bwa_Indexed/human_g1k_v37.fasta',
    'bwa_reference': '/vlsci/VR0002/shared/Reference_Files/Indexed_Ref_Genomes/bwa_Indexed/human_g1k_v37.fasta',

    'dbsnp': '/vlsci/VR0002/shared/Reference_Files/SNP_db/dbSNP137.vcf',

    'indels_realign_goldstandard': '/vlsci/VR0002/shared/Reference_Files/Indels_for_realignment/Mills_and_1000G_gold_standard.indels.b37.vcf',
    'indels_realign_1000G': '/vlsci/VR0002/shared/Reference_Files/Indels_for_realignment/1000G_phase1.indels.b37.vcf'
}

# pipeline should hold configuration options for Rubra and for the pipeline.
# This section is required for every Rubra pipeline,
# but restrict_samples and allowed_samples are specific to the variant-calling pipeline.
#
# Rubra variables:
#  - logDir: the directory where batch queue scripts, stdout and sterr dumps are stored.
#  - logFile: the file used to log all jobs that are run.
#  - style: the default style, one of 'flowchart', 'print', 'run', 'touchfiles'. Can be
#      overridden by specifying --style on the command line.
#  - procs: the number of python processes to run simultaneously. This determines the
#      maximum parallelism of the pipeline. For distributed jobs it also constrains the
#      maximum total jobs submitted to the queue at any one time.
#  - verbosity: one of 0 (quiet), 1 (normal), 2 (chatty). Can be overridden by specifying
#      --verbose on the command line.
#  - end: the desired tasks to be run. Rubra will also run all tasks which are dependencies
#      of these tasks. Can be overridden by specifying --end on the command line.
#  - force: tasks which will be forced to run, regardless of timestamps. Can be overridden
#      by supplying --force on the command line.
#  - rebuild: one of 'fromstart','fromend'. Whether to calculate which dependencies will
#      be rerun by working back from an end task to the latest up-to-date task, or forward
#      from the earliest out-of-date task. 'fromstart' is the most conservative and
#      commonly used as it brings all intermediate tasks up to date.
#
# Variant-calling pipeline variables:  (TODO: move to a separate section)
#  - restrict_samples: whether to restrict input files to those specified by allowd_samples
#  - allowed_samples: sample names that will be run of restrict_samples is True
pipeline = {
    'logDir': 'log_varcall_test',
    'logFile': 'pipeline.log',
    'style': 'print',
    'procs': 30,
    'verbose': 1,
    'end': ['earlyDepthOfCoverage', 'finalDepthOfCoverage',
            'fastqc',
            'igvcountMergedBams', 'countDedupedBam', 'countRunBam', 'countMergedBam',
            'getEnsemblAnnotations',
            'collateReadCounts',
            'vcfIndexSNPs', 'vcfIndexIndels'
            ],
    'force': [],
    'rebuild' : "fromstart",

    'restrict_samples': True,
    'allowed_samples': ['mw4241']
}
<|endoftext|>


<commit_before>'''
Created on Aug 6, 2015

@author: theo
'''
from django.core.management.base import BaseCommand
from django.contrib.auth.models import User

from optparse import make_option
from django.contrib.gis.geos import Point
from django.utils import timezone
from django.conf import settings
import os,pytz,datetime
import json,logging

from acacia.data.models import ProjectLocatie
from iom import util
from iom.akvo import FlowAPI, as_timestamp
from iom.models import AkvoFlow, CartoDb, Meetpunt, Waarnemer, Alias
from iom.exif import Exif

logger = logging.getLogger(__name__)

def maak_naam(parameter,diep):
    if diep and not diep.endswith('iep'):
        diep = None
    return parameter + '_' + diep if diep else parameter

def get_or_create_waarnemer(akvoname):
    try:
        # is deze alias al geregistreerd?
        alias = Alias.objects.get(alias=akvoname)
        waarnemer = alias.waarnemer
        logger.debug('Waarnemer {name} gevonden met alias {alias}'.format(name=unicode(waarnemer),alias=alias))
    except Alias.DoesNotExist:
        # Bestaat er al een waarnemer met deze naam?
        words = akvoname.split()
        if len(words) > 1:
            voornaam = words[0]
            initialen = voornaam[0]
            achternaam = words[-1]
            if len(words) > 2:
                tussenvoegsel = ' '.join(words[1:-1])
            else:
                tussenvoegsel = ''
            waarnemer, created = Waarnemer.objects.get_or_create(initialen=initialen, voornaam=voornaam, tussenvoegsel=tussenvoegsel, achternaam=achternaam)
        else:
            waarnemer, created = Waarnemer.objects.get_or_create(achternaam=akvoname)
        if created:
            logger.info('Waarnemer {} aangemaakt'.format(unicode(waarnemer)))
        # alias toevoegen aan waarnemer
        alias = waarnemer.alias_set.create(alias=akvoname)
        logger.info('alias {alias} toegevoegd aan waarnemer {name}'.format(alias=unicode(alias),name=unicode(waarnemer)))
    return waarnemer

def download_photo(url):
    # copy photo to local storage and rotate if necessary
    try:
        filename = os.path.basename(url)
        Exif.copyImage(url, os.path.join(settings.PHOTO_DIR,filename))
        return os.path.join(settings.PHOTO_URL,filename)
    except:
        return url

def importAkvoRegistration(api,akvo,projectlocatie,user):
    surveyId = akvo.regform
    meetpunten=set()
    num_meetpunten = 0

    beginDate=as_timestamp(akvo.last_update)
    instances = api.get_registration_instances(surveyId,beginDate=beginDate).items()
#    instances = api.get_registration_instances(surveyId).items()
    for key,instance in instances:
        identifier=instance['surveyedLocaleIdentifier']
        displayName = instance['surveyedLocaleDisplayName']
        submitter = instance['submitterName']
        device = instance['deviceIdentifier']
        date=instance['collectionDate']
        date=datetime.datetime.utcfromtimestamp(date/1000.0).replace(tzinfo=pytz.utc)
        answers = api.get_answers(instance['keyId'])
        akvowaarnemer = api.get_answer(answers,questionText='Waarnemer')
        meetid = api.get_answer(answers,questionText='Meetpunt ID')
        foto = api.get_answer(answers,questionText='Maak een foto van het meetgebied')
        geoloc = api.get_answer(answers,questionText='Geolocatie')
        omschrijving = api.get_answer(answers,questionText='Meetpunt omschrijving')
        num_meetpunten += 1
        try:
            lat,lon,elev,code = geoloc.split('|')
            location = Point(float(lon),float(lat),srid=4326)
            location.transform(28992)
        except:
            logger.error('Probleem met coordinaten {loc}. waarnemer = {waar}, meetpunt = {mp}'.format(loc=geoloc, waar = akvowaarnemer or submitter, mp=meetid))
            continue

        akvoname = akvowaarnemer or submitter
        waarnemer = get_or_create_waarnemer(akvoname)

        # change reference to photo from smartphone storage to amazon storage and download to this server
        if foto:
            foto = download_photo(os.path.join(akvo.storage,os.path.basename(foto)))

        if meetid:
            # Gebuik waarnemer naam + meetid
            meetName = '{name} - {id}'. format(name=akvoname, id=meetid)
        else:
            meetName = displayName
        try:
            meetpunt, created = waarnemer.meetpunt_set.get_or_create(identifier=identifier, defaults={
                'name': meetName,
                'projectlocatie': projectlocatie,
                'location': location,
                'displayname': displayName,
                'device': device,
                'photo_url': foto,
                'description': omschrijving})
        except Exception as e:
            # acacia.data.models.meetlocatie probably exists
            try:
                meetpunt = Meetpunt.objects.get(name=meetName, projectlocatie=projectlocatie)
                meetpunt.identifier = identifier
                meetpunt.displayname = displayName
                meetpunt.device = device
                meetpunt.identifier = identifier
                meetpunt.photo_url = foto
                meetpunt.save()
                meetpunten.add(meetpunt)
                created = True
            except:
                logger.exception('Probleem bij toevoegen meetpunt {mname} aan waarnemer {wname}'.format(mname=meetName, wname=unicode(waarnemer)))
                continue

        if created:
            logger.info('Meetpunt {id} aangemaakt voor waarnemer {name}'.format(id=meetName,name=unicode(waarnemer)))
            meetpunten.add(meetpunt)

        if device != 'IMPORTER':
            ec = api.get_answer(answers,questionText='Meet EC waarde - ECOND')
            diep = api.get_answer(answers,questionText='Diep of ondiep gemeten?')
            waarneming_naam = maak_naam('EC',diep)

            try:
                waarneming, created = meetpunt.waarneming_set.get_or_create(naam=waarneming_naam, waarnemer=waarnemer, datum=date,
                                                  defaults = {'waarde': ec, 'device': device, 'opmerking': '', 'foto_url': foto, 'eenheid': 'uS/cm'})
            except Exception as e:
                logger.exception('Probleem bij toevoegen waarneming {wname} aan meetpunt {mname}'.format(wname=waarneming_naam, mname=unicode(meetpunt)))
                continue

            if created:
                logger.debug('{id}, {date}, EC={ec}'.format(id=waarneming.naam, date=waarneming.datum, ec=waarneming.waarde))
                meetpunten.add(meetpunt)

            elif int(waarneming.waarde) != int(ec):
                waarneming.waarde = ec
                waarneming.save()
                meetpunten.add(meetpunt)

    logger.info('Aantal meetpunten: {aantal}, nieuwe meetpunten: {new}'.format(aantal=num_meetpunten, new=len(meetpunten)))

    return meetpunten

def importAkvoMonitoring(api,akvo):
    meetpunten = set()
    num_waarnemingen = 0
    num_replaced = 0

    beginDate = as_timestamp(akvo.last_update)
    for surveyId in [f.strip() for f in akvo.monforms.split(',')]:
        survey = api.get_survey(surveyId)
        instances,meta = api.get_survey_instances(surveyId=surveyId,beginDate=beginDate)
        while instances:
            for instance in instances:
                submitter = instance['submitterName']
                waarnemer = get_or_create_waarnemer(submitter)

                #find related registration form (meetpunt)
                localeId = instance['surveyedLocaleIdentifier']
                try:
                    meetpunt = Meetpunt.objects.get(identifier=localeId)
                except Meetpunt.DoesNotExist:
                    logger.error('Meetpunt {locale} niet gevonden voor {submitter}'.format(locale=localeId, submitter=submitter))
                    continue

                device = instance['deviceIdentifier']
                date=instance['collectionDate']
                date=datetime.datetime.utcfromtimestamp(date/1000.0).replace(tzinfo=pytz.utc)

                answers = api.get_answers(instance['keyId'])
                ec=api.get_answer(answers,questionText='EC waarde - ECOND')
                foto=api.get_answer(answers,questionID='Maak een foto van het meetgebied')
                diep=api.get_answer(answers,questionText='Diep of ondiep')
                waarneming_naam = maak_naam('EC',diep)

                if foto:
                    foto = download_photo(os.path.join(akvo.storage,os.path.basename(foto)))

                if foto and not meetpunt.photo_url:
                    # update meetpunt along the way..
                    meetpunt.photo_url = foto
                    meetpunt.save(update_fields=['photo_url'])

                waarneming, created = meetpunt.waarneming_set.get_or_create(naam=waarneming_naam, waarnemer=waarnemer, datum=date,
                                              defaults = {'waarde': ec, 'device': device, 'opmerking': '', 'foto_url': foto, 'eenheid': 'uS/cm'})
                if created:
                    logger.info('{locale}={mp}, {id}({date})={ec}'.format(locale=localeId, mp=unicode(meetpunt), id=waarneming.naam, date=waarneming.datum, ec=waarneming.waarde))
                    num_waarnemingen += 1
                    meetpunten.add(meetpunt)
                elif int(waarneming.waarde) != int(ec):
                    waarneming.waarde = ec
                    waarneming.save()
                    logger.info('{locale}={mp}, {id}({date})={ec}'.format(locale=localeId, mp=unicode(meetpunt), id=waarneming.naam, date=waarneming.datum, ec=waarneming.waarde))
                    num_replaced += 1
                    meetpunten.add(meetpunt)
            instances,meta = api.get_survey_instances(surveyId=surveyId, beginDate=beginDate, since=meta['since'])
    logger.info('Aantal nieuwe metingen: {meet}, bijgewerkt: {repl}'.format(meet=num_waarnemingen,repl=num_replaced))
    return meetpunten

class Command(BaseCommand):
    args = ''
    help = 'Importeer metingen vanuit akvo flow'
    option_list = BaseCommand.option_list + (
            make_option('--akvo',
                action='store',
                dest = 'akvo',
                default = 1,
                help = 'id van Akvoflow configuratie'),
            make_option('--cartodb',
                action='store',
                dest = 'cartodb',
                default = 1,
                help = 'id van Cartodb configuratie'),
            make_option('--project',
                action='store',
                dest = 'proj',
                default = 1,
                help = 'id van project locatie'),
            make_option('--user',
                action='store',
                dest = 'user',
                default = 'akvo',
                help = 'user name'),
        )

    def handle(self, *args, **options):

        akvo = AkvoFlow.objects.get(pk=options.get('akvo'))
        api = FlowAPI(instance=akvo.instance, key=akvo.key, secret=akvo.secret)
        cartodb = CartoDb.objects.get(pk=options.get('cartodb'))

        project = ProjectLocatie.objects.get(pk=options.get('proj'))
        user = User.objects.get(username=options.get('user'))

        try:
            logger.debug('Meetpuntgegevens ophalen')
            mp = importAkvoRegistration(api, akvo, projectlocatie=project,user=user)
            logger.debug('Waarnemingen ophalen')
            mp.update(importAkvoMonitoring(api, akvo))

            if mp:
                logger.debug('Grafieken aanpassen')
                util.updateSeries(mp, user)
                logger.debug('Cartodb actualiseren')
                util.updateCartodb(cartodb, mp)
                logger.debug('Triggers evalueren')
                util.processTriggers(mp)

            akvo.last_update = timezone.now()
            akvo.save()
        except Exception as e:
            logger.exception('Probleem met verwerken nieuwe EC metingen: %s',e)
        finally:
            pass
<commit_msg>update from one day ago (allow for missed updates)<commit_after>'''
Created on Aug 6, 2015

@author: theo
'''
from django.core.management.base import BaseCommand
from django.contrib.auth.models import User

from optparse import make_option
from django.contrib.gis.geos import Point
from django.utils import timezone
from django.conf import settings
import os,pytz,datetime
import json,logging

from acacia.data.models import ProjectLocatie
from iom import util
from iom.akvo import FlowAPI, as_timestamp
from iom.models import AkvoFlow, CartoDb, Meetpunt, Waarnemer, Alias
from iom.exif import Exif

logger = logging.getLogger(__name__)

def maak_naam(parameter,diep):
    if diep and not diep.endswith('iep'):
        diep = None
    return parameter + '_' + diep if diep else parameter

def get_or_create_waarnemer(akvoname):
    try:
        # is deze alias al geregistreerd?
        alias = Alias.objects.get(alias=akvoname)
        waarnemer = alias.waarnemer
        logger.debug('Waarnemer {name} gevonden met alias {alias}'.format(name=unicode(waarnemer),alias=alias))
    except Alias.DoesNotExist:
        # Bestaat er al een waarnemer met deze naam?
        words = akvoname.split()
        if len(words) > 1:
            voornaam = words[0]
            initialen = voornaam[0]
            achternaam = words[-1]
            if len(words) > 2:
                tussenvoegsel = ' '.join(words[1:-1])
            else:
                tussenvoegsel = ''
            waarnemer, created = Waarnemer.objects.get_or_create(initialen=initialen, voornaam=voornaam, tussenvoegsel=tussenvoegsel, achternaam=achternaam)
        else:
            waarnemer, created = Waarnemer.objects.get_or_create(achternaam=akvoname)
        if created:
            logger.info('Waarnemer {} aangemaakt'.format(unicode(waarnemer)))
        # alias toevoegen aan waarnemer
        alias = waarnemer.alias_set.create(alias=akvoname)
        logger.info('alias {alias} toegevoegd aan waarnemer {name}'.format(alias=unicode(alias),name=unicode(waarnemer)))
    return waarnemer

def download_photo(url):
    # copy photo to local storage and rotate if necessary
    try:
        filename = os.path.basename(url)
        Exif.copyImage(url, os.path.join(settings.PHOTO_DIR,filename))
        return os.path.join(settings.PHOTO_URL,filename)
    except:
        return url

def importAkvoRegistration(api,akvo,projectlocatie,user):
    surveyId = akvo.regform
    meetpunten=set()
    num_meetpunten = 0
    update = akvo.last_update + datetime.timedelta(days=-1)
    beginDate=as_timestamp(update)
    instances = api.get_registration_instances(surveyId,beginDate=beginDate).items()
#    instances = api.get_registration_instances(surveyId).items()
    for key,instance in instances:
        identifier=instance['surveyedLocaleIdentifier']
        displayName = instance['surveyedLocaleDisplayName']
        submitter = instance['submitterName']
        device = instance['deviceIdentifier']
        date=instance['collectionDate']
        date=datetime.datetime.utcfromtimestamp(date/1000.0).replace(tzinfo=pytz.utc)
        answers = api.get_answers(instance['keyId'])
        akvowaarnemer = api.get_answer(answers,questionText='Waarnemer')
        meetid = api.get_answer(answers,questionText='Meetpunt ID')
        foto = api.get_answer(answers,questionText='Maak een foto van het meetgebied')
        geoloc = api.get_answer(answers,questionText='Geolocatie')
        omschrijving = api.get_answer(answers,questionText='Meetpunt omschrijving')
        num_meetpunten += 1
        try:
            lat,lon,elev,code = geoloc.split('|')
            location = Point(float(lon),float(lat),srid=4326)
            location.transform(28992)
        except:
            logger.error('Probleem met coordinaten {loc}. waarnemer = {waar}, meetpunt = {mp}'.format(loc=geoloc, waar = akvowaarnemer or submitter, mp=meetid))
            continue

        akvoname = akvowaarnemer or submitter
        waarnemer = get_or_create_waarnemer(akvoname)

        # change reference to photo from smartphone storage to amazon storage and download to this server
        if foto:
            foto = download_photo(os.path.join(akvo.storage,os.path.basename(foto)))

        if meetid:
            # Gebuik waarnemer naam + meetid
            meetName = '{name} - {id}'. format(name=akvoname, id=meetid)
        else:
            meetName = displayName
        try:
            meetpunt, created = waarnemer.meetpunt_set.get_or_create(identifier=identifier, defaults={
                'name': meetName,
                'projectlocatie': projectlocatie,
                'location': location,
                'displayname': displayName,
                'device': device,
                'photo_url': foto,
                'description': omschrijving})
        except Exception as e:
            # acacia.data.models.meetlocatie probably exists
            try:
                meetpunt = Meetpunt.objects.get(name=meetName, projectlocatie=projectlocatie)
                meetpunt.identifier = identifier
                meetpunt.displayname = displayName
                meetpunt.device = device
                meetpunt.identifier = identifier
                meetpunt.photo_url = foto
                meetpunt.save()
                meetpunten.add(meetpunt)
                created = True
            except:
                logger.exception('Probleem bij toevoegen meetpunt {mname} aan waarnemer {wname}'.format(mname=meetName, wname=unicode(waarnemer)))
                continue

        if created:
            logger.info('Meetpunt {id} aangemaakt voor waarnemer {name}'.format(id=meetName,name=unicode(waarnemer)))
            meetpunten.add(meetpunt)

        if device != 'IMPORTER':
            ec = api.get_answer(answers,questionText='Meet EC waarde - ECOND')
            diep = api.get_answer(answers,questionText='Diep of ondiep gemeten?')
            waarneming_naam = maak_naam('EC',diep)

            try:
                waarneming, created = meetpunt.waarneming_set.get_or_create(naam=waarneming_naam, waarnemer=waarnemer, datum=date,
                                                  defaults = {'waarde': ec, 'device': device, 'opmerking': '', 'foto_url': foto, 'eenheid': 'uS/cm'})
            except Exception as e:
                logger.exception('Probleem bij toevoegen waarneming {wname} aan meetpunt {mname}'.format(wname=waarneming_naam, mname=unicode(meetpunt)))
                continue

            if created:
                logger.debug('{id}, {date}, EC={ec}'.format(id=waarneming.naam, date=waarneming.datum, ec=waarneming.waarde))
                meetpunten.add(meetpunt)

            elif int(waarneming.waarde) != int(ec):
                waarneming.waarde = ec
                waarneming.save()
                meetpunten.add(meetpunt)

    logger.info('Aantal meetpunten: {aantal}, nieuwe meetpunten: {new}'.format(aantal=num_meetpunten, new=len(meetpunten)))

    return meetpunten

def importAkvoMonitoring(api,akvo):
    meetpunten = set()
    num_waarnemingen = 0
    num_replaced = 0

    update = akvo.last_update + datetime.timedelta(days=-1)
    beginDate=as_timestamp(update)
    for surveyId in [f.strip() for f in akvo.monforms.split(',')]:
        survey = api.get_survey(surveyId)
        instances,meta = api.get_survey_instances(surveyId=surveyId,beginDate=beginDate)
        while instances:
            for instance in instances:
                submitter = instance['submitterName']
                waarnemer = get_or_create_waarnemer(submitter)

                #find related registration form (meetpunt)
                localeId = instance['surveyedLocaleIdentifier']
                try:
                    meetpunt = Meetpunt.objects.get(identifier=localeId)
                except Meetpunt.DoesNotExist:
                    logger.error('Meetpunt {locale} niet gevonden voor {submitter}'.format(locale=localeId, submitter=submitter))
                    continue

                device = instance['deviceIdentifier']
                date=instance['collectionDate']
                date=datetime.datetime.utcfromtimestamp(date/1000.0).replace(tzinfo=pytz.utc)

                answers = api.get_answers(instance['keyId'])
                ec=api.get_answer(answers,questionText='EC waarde - ECOND')
                foto=api.get_answer(answers,questionID='Maak een foto van het meetgebied')
                diep=api.get_answer(answers,questionText='Diep of ondiep')
                waarneming_naam = maak_naam('EC',diep)

                if foto:
                    foto = download_photo(os.path.join(akvo.storage,os.path.basename(foto)))

                if foto and not meetpunt.photo_url:
                    # update meetpunt along the way..
                    meetpunt.photo_url = foto
                    meetpunt.save(update_fields=['photo_url'])

                waarneming, created = meetpunt.waarneming_set.get_or_create(naam=waarneming_naam, waarnemer=waarnemer, datum=date,
                                              defaults = {'waarde': ec, 'device': device, 'opmerking': '', 'foto_url': foto, 'eenheid': 'uS/cm'})
                if created:
                    logger.info('{locale}={mp}, {id}({date})={ec}'.format(locale=localeId, mp=unicode(meetpunt), id=waarneming.naam, date=waarneming.datum, ec=waarneming.waarde))
                    num_waarnemingen += 1
                    meetpunten.add(meetpunt)
                elif int(waarneming.waarde) != int(ec):
                    waarneming.waarde = ec
                    waarneming.save()
                    logger.info('{locale}={mp}, {id}({date})={ec}'.format(locale=localeId, mp=unicode(meetpunt), id=waarneming.naam, date=waarneming.datum, ec=waarneming.waarde))
                    num_replaced += 1
                    meetpunten.add(meetpunt)
            instances,meta = api.get_survey_instances(surveyId=surveyId, beginDate=beginDate, since=meta['since'])
    logger.info('Aantal nieuwe metingen: {meet}, bijgewerkt: {repl}'.format(meet=num_waarnemingen,repl=num_replaced))
    return meetpunten

class Command(BaseCommand):
    args = ''
    help = 'Importeer metingen vanuit akvo flow'
    option_list = BaseCommand.option_list + (
            make_option('--akvo',
                action='store',
                dest = 'akvo',
                default = 1,
                help = 'id van Akvoflow configuratie'),
            make_option('--cartodb',
                action='store',
                dest = 'cartodb',
                default = 1,
                help = 'id van Cartodb configuratie'),
            make_option('--project',
                action='store',
                dest = 'proj',
                default = 1,
                help = 'id van project locatie'),
            make_option('--user',
                action='store',
                dest = 'user',
                default = 'akvo',
                help = 'user name'),
        )

    def handle(self, *args, **options):

        akvo = AkvoFlow.objects.get(pk=options.get('akvo'))
        api = FlowAPI(instance=akvo.instance, key=akvo.key, secret=akvo.secret)
        cartodb = CartoDb.objects.get(pk=options.get('cartodb'))

        project = ProjectLocatie.objects.get(pk=options.get('proj'))
        user = User.objects.get(username=options.get('user'))

        try:
            logger.debug('Meetpuntgegevens ophalen')
            mp = importAkvoRegistration(api, akvo, projectlocatie=project,user=user)
            logger.debug('Waarnemingen ophalen')
            mp.update(importAkvoMonitoring(api, akvo))

            if mp:
                logger.debug('Grafieken aanpassen')
                util.updateSeries(mp, user)
                logger.debug('Cartodb actualiseren')
                util.updateCartodb(cartodb, mp)
                logger.debug('Triggers evalueren')
                util.processTriggers(mp)

            akvo.last_update = timezone.now()
            akvo.save()
        except Exception as e:
            logger.exception('Probleem met verwerken nieuwe EC metingen: %s',e)
        finally:
            pass
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Copyright (c) 2012 NTT DOCOMO, INC.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import os
import time

import re
import socket
import stat

from ironic.common import exception
from ironic.common import utils
from ironic.openstack.common import excutils
from ironic.openstack.common import log as logging
from ironic.openstack.common import processutils

LOG = logging.getLogger(__name__)

# All functions are called from deploy() directly or indirectly.
# They are split for stub-out.

def discovery(portal_address, portal_port):
    """Do iSCSI discovery on portal."""
    utils.execute('iscsiadm',
                  '-m', 'discovery',
                  '-t', 'st',
                  '-p', '%s:%s' % (portal_address, portal_port),
                  run_as_root=True,
                  check_exit_code=[0])

def login_iscsi(portal_address, portal_port, target_iqn):
    """Login to an iSCSI target."""
    utils.execute('iscsiadm',
                  '-m', 'node',
                  '-p', '%s:%s' % (portal_address, portal_port),
                  '-T', target_iqn,
                  '--login',
                  run_as_root=True,
                  check_exit_code=[0])
    # Ensure the login complete
    time.sleep(3)

def logout_iscsi(portal_address, portal_port, target_iqn):
    """Logout from an iSCSI target."""
    utils.execute('iscsiadm',
                  '-m', 'node',
                  '-p', '%s:%s' % (portal_address, portal_port),
                  '-T', target_iqn,
                  '--logout',
                  run_as_root=True,
                  check_exit_code=[0])

def delete_iscsi(portal_address, portal_port, target_iqn):
    """Delete the iSCSI target."""
    utils.execute('iscsiadm',
                  '-m', 'node',
                  '-p', '%s:%s' % (portal_address, portal_port),
                  '-T', target_iqn,
                  '-o', 'delete',
                  run_as_root=True,
                  check_exit_code=[0])

def make_partitions(dev, root_mb, swap_mb):
    """Create partitions for root and swap on a disk device."""
    # Lead in with 1MB to allow room for the partition table itself, otherwise
    # the way sfdisk adjusts doesn't shift the partition up to compensate, and
    # we lose the space.
    # http://bazaar.launchpad.net/~ubuntu-branches/ubuntu/raring/util-linux/
    # raring/view/head:/fdisk/sfdisk.c#L1940
    stdin_command = ('1,%d,83;\n,%d,82;\n0,0;\n0,0;\n' % (root_mb, swap_mb))
    utils.execute('sfdisk', '-uM', dev, process_input=stdin_command,
            run_as_root=True,
            attempts=3,
            check_exit_code=[0])
    # avoid "device is busy"
    time.sleep(3)

def is_block_device(dev):
    """Check whether a device is block or not."""
    s = os.stat(dev)
    return stat.S_ISBLK(s.st_mode)

def dd(src, dst):
    """Execute dd from src to dst."""
    utils.execute('dd',
                  'if=%s' % src,
                  'of=%s' % dst,
                  'bs=1M',
                  'oflag=direct',
                  run_as_root=True,
                  check_exit_code=[0])

def mkswap(dev, label='swap1'):
    """Execute mkswap on a device."""
    utils.execute('mkswap',
                  '-L', label,
                  dev,
                  run_as_root=True,
                  check_exit_code=[0])

def block_uuid(dev):
    """Get UUID of a block device."""
    out, _ = utils.execute('blkid', '-s', 'UUID', '-o', 'value', dev,
                           run_as_root=True,
                           check_exit_code=[0])
    return out.strip()

def switch_pxe_config(path, root_uuid):
    """Switch a pxe config from deployment mode to service mode."""
    with open(path) as f:
        lines = f.readlines()
    root = 'UUID=%s' % root_uuid
    rre = re.compile(r'\{\{ ROOT \}\}')
    dre = re.compile('^default .*$')
    with open(path, 'w') as f:
        for line in lines:
            line = rre.sub(root, line)
            line = dre.sub('default boot', line)
            f.write(line)

def notify(address, port):
    """Notify a node that it becomes ready to reboot."""
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    try:
        s.connect((address, port))
        s.send('done')
    finally:
        s.close()

def get_dev(address, port, iqn, lun):
    """Returns a device path for given parameters."""
    dev = "/dev/disk/by-path/ip-%s:%s-iscsi-%s-lun-%s" \
            % (address, port, iqn, lun)
    return dev

def get_image_mb(image_path):
    """Get size of an image in Megabyte."""
    mb = 1024 * 1024
    image_byte = os.path.getsize(image_path)
    # round up size to MB
    image_mb = int((image_byte + mb - 1) / mb)
    return image_mb

def work_on_disk(dev, root_mb, swap_mb, image_path):
    """Creates partitions and write an image to the root partition."""
    root_part = "%s-part1" % dev
    swap_part = "%s-part2" % dev

    if not is_block_device(dev):
        raise exception.InstanceDeployFailure(_("Parent device '%s' not found")
                                              % dev)
    make_partitions(dev, root_mb, swap_mb)

    if not is_block_device(root_part):
        raise exception.InstanceDeployFailure(_("Root device '%s' not found")
                                              % root_part)
    if not is_block_device(swap_part):
        raise exception.InstanceDeployFailure(_("Swap device '%s' not found")
                                              % swap_part)
    dd(image_path, root_part)
    mkswap(swap_part)

    try:
        root_uuid = block_uuid(root_part)
    except processutils.ProcessExecutionError:
        with excutils.save_and_reraise_exception():
            LOG.error(_("Failed to detect root device UUID."))
    return root_uuid

def deploy(address, port, iqn, lun, image_path, pxe_config_path,
           root_mb, swap_mb):
    """All-in-one function to deploy a node."""
    dev = get_dev(address, port, iqn, lun)
    image_mb = get_image_mb(image_path)
    if image_mb > root_mb:
        root_mb = image_mb
    discovery(address, port)
    login_iscsi(address, port, iqn)
    try:
        root_uuid = work_on_disk(dev, root_mb, swap_mb, image_path)
    except processutils.ProcessExecutionError as err:
        with excutils.save_and_reraise_exception():
            LOG.error(_("Deploy to address %s failed.") % address)
            LOG.error(_("Command: %s") % err.cmd)
            LOG.error(_("StdOut: %r") % err.stdout)
            LOG.error(_("StdErr: %r") % err.stderr)
    except exception.InstanceDeployFailure as e:
        with excutils.save_and_reraise_exception():
            LOG.error(_("Deploy to address %s failed.") % address)
            LOG.error(e)
    finally:
        logout_iscsi(address, port, iqn)
        delete_iscsi(address, port, iqn)
    switch_pxe_config(pxe_config_path, root_uuid)
    # Ensure the node started netcat on the port after POST the request.
    time.sleep(3)
    notify(address, 10000)
<commit_msg>Removed duplicated function to create a swap fs<commit_after>#!/usr/bin/env python

# Copyright (c) 2012 NTT DOCOMO, INC.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import os
import time

import re
import socket
import stat

from ironic.common import exception
from ironic.common import utils
from ironic.openstack.common import excutils
from ironic.openstack.common import log as logging
from ironic.openstack.common import processutils

LOG = logging.getLogger(__name__)

# All functions are called from deploy() directly or indirectly.
# They are split for stub-out.

def discovery(portal_address, portal_port):
    """Do iSCSI discovery on portal."""
    utils.execute('iscsiadm',
                  '-m', 'discovery',
                  '-t', 'st',
                  '-p', '%s:%s' % (portal_address, portal_port),
                  run_as_root=True,
                  check_exit_code=[0])

def login_iscsi(portal_address, portal_port, target_iqn):
    """Login to an iSCSI target."""
    utils.execute('iscsiadm',
                  '-m', 'node',
                  '-p', '%s:%s' % (portal_address, portal_port),
                  '-T', target_iqn,
                  '--login',
                  run_as_root=True,
                  check_exit_code=[0])
    # Ensure the login complete
    time.sleep(3)

def logout_iscsi(portal_address, portal_port, target_iqn):
    """Logout from an iSCSI target."""
    utils.execute('iscsiadm',
                  '-m', 'node',
                  '-p', '%s:%s' % (portal_address, portal_port),
                  '-T', target_iqn,
                  '--logout',
                  run_as_root=True,
                  check_exit_code=[0])

def delete_iscsi(portal_address, portal_port, target_iqn):
    """Delete the iSCSI target."""
    utils.execute('iscsiadm',
                  '-m', 'node',
                  '-p', '%s:%s' % (portal_address, portal_port),
                  '-T', target_iqn,
                  '-o', 'delete',
                  run_as_root=True,
                  check_exit_code=[0])

def make_partitions(dev, root_mb, swap_mb):
    """Create partitions for root and swap on a disk device."""
    # Lead in with 1MB to allow room for the partition table itself, otherwise
    # the way sfdisk adjusts doesn't shift the partition up to compensate, and
    # we lose the space.
    # http://bazaar.launchpad.net/~ubuntu-branches/ubuntu/raring/util-linux/
    # raring/view/head:/fdisk/sfdisk.c#L1940
    stdin_command = ('1,%d,83;\n,%d,82;\n0,0;\n0,0;\n' % (root_mb, swap_mb))
    utils.execute('sfdisk', '-uM', dev, process_input=stdin_command,
            run_as_root=True,
            attempts=3,
            check_exit_code=[0])
    # avoid "device is busy"
    time.sleep(3)

def is_block_device(dev):
    """Check whether a device is block or not."""
    s = os.stat(dev)
    return stat.S_ISBLK(s.st_mode)

def dd(src, dst):
    """Execute dd from src to dst."""
    utils.execute('dd',
                  'if=%s' % src,
                  'of=%s' % dst,
                  'bs=1M',
                  'oflag=direct',
                  run_as_root=True,
                  check_exit_code=[0])

def mkswap(dev, label='swap1'):
    """Execute mkswap on a device."""
    utils.mkfs('swap', dev, label)

def block_uuid(dev):
    """Get UUID of a block device."""
    out, _ = utils.execute('blkid', '-s', 'UUID', '-o', 'value', dev,
                           run_as_root=True,
                           check_exit_code=[0])
    return out.strip()

def switch_pxe_config(path, root_uuid):
    """Switch a pxe config from deployment mode to service mode."""
    with open(path) as f:
        lines = f.readlines()
    root = 'UUID=%s' % root_uuid
    rre = re.compile(r'\{\{ ROOT \}\}')
    dre = re.compile('^default .*$')
    with open(path, 'w') as f:
        for line in lines:
            line = rre.sub(root, line)
            line = dre.sub('default boot', line)
            f.write(line)

def notify(address, port):
    """Notify a node that it becomes ready to reboot."""
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    try:
        s.connect((address, port))
        s.send('done')
    finally:
        s.close()

def get_dev(address, port, iqn, lun):
    """Returns a device path for given parameters."""
    dev = "/dev/disk/by-path/ip-%s:%s-iscsi-%s-lun-%s" \
            % (address, port, iqn, lun)
    return dev

def get_image_mb(image_path):
    """Get size of an image in Megabyte."""
    mb = 1024 * 1024
    image_byte = os.path.getsize(image_path)
    # round up size to MB
    image_mb = int((image_byte + mb - 1) / mb)
    return image_mb

def work_on_disk(dev, root_mb, swap_mb, image_path):
    """Creates partitions and write an image to the root partition."""
    root_part = "%s-part1" % dev
    swap_part = "%s-part2" % dev

    if not is_block_device(dev):
        raise exception.InstanceDeployFailure(_("Parent device '%s' not found")
                                              % dev)
    make_partitions(dev, root_mb, swap_mb)

    if not is_block_device(root_part):
        raise exception.InstanceDeployFailure(_("Root device '%s' not found")
                                              % root_part)
    if not is_block_device(swap_part):
        raise exception.InstanceDeployFailure(_("Swap device '%s' not found")
                                              % swap_part)
    dd(image_path, root_part)
    mkswap(swap_part)

    try:
        root_uuid = block_uuid(root_part)
    except processutils.ProcessExecutionError:
        with excutils.save_and_reraise_exception():
            LOG.error(_("Failed to detect root device UUID."))
    return root_uuid

def deploy(address, port, iqn, lun, image_path, pxe_config_path,
           root_mb, swap_mb):
    """All-in-one function to deploy a node."""
    dev = get_dev(address, port, iqn, lun)
    image_mb = get_image_mb(image_path)
    if image_mb > root_mb:
        root_mb = image_mb
    discovery(address, port)
    login_iscsi(address, port, iqn)
    try:
        root_uuid = work_on_disk(dev, root_mb, swap_mb, image_path)
    except processutils.ProcessExecutionError as err:
        with excutils.save_and_reraise_exception():
            LOG.error(_("Deploy to address %s failed.") % address)
            LOG.error(_("Command: %s") % err.cmd)
            LOG.error(_("StdOut: %r") % err.stdout)
            LOG.error(_("StdErr: %r") % err.stderr)
    except exception.InstanceDeployFailure as e:
        with excutils.save_and_reraise_exception():
            LOG.error(_("Deploy to address %s failed.") % address)
            LOG.error(e)
    finally:
        logout_iscsi(address, port, iqn)
        delete_iscsi(address, port, iqn)
    switch_pxe_config(pxe_config_path, root_uuid)
    # Ensure the node started netcat on the port after POST the request.
    time.sleep(3)
    notify(address, 10000)
<|endoftext|>