<commit_before># Copyright (c) 2016 Ultimaker B.V.
# Cura is released under the terms of the AGPLv3 or higher.

import os.path
import urllib

from PyQt5.QtCore import QObject, pyqtSlot, pyqtProperty, pyqtSignal, QUrl
from PyQt5.QtWidgets import QMessageBox

import UM.PluginRegistry
import UM.Settings
import UM.SaveFile
import UM.Platform
import UM.MimeTypeDatabase
import UM.Logger

from UM.MimeTypeDatabase import MimeTypeNotFoundError

from UM.i18n import i18nCatalog
catalog = i18nCatalog("cura")

##  Manager class that contains common actions to deal with containers in Cura.
#
#   This is primarily intended as a class to be able to perform certain actions
#   from within QML. We want to be able to trigger things like removing a container
#   when a certain action happens. This can be done through this class.
class ContainerManager(QObject):
    def __init__(self, parent = None):
        super().__init__(parent)

        self._registry = UM.Settings.ContainerRegistry.getInstance()
        self._container_name_filters = {}

    ##  Create a duplicate of the specified container
    #
    #   This will create and add a duplicate of the container corresponding
    #   to the container ID.
    #
    #   \param container_id \type{str} The ID of the container to duplicate.
    #
    #   \return The ID of the new container, or an empty string if duplication failed.
    @pyqtSlot(str, result = str)
    def duplicateContainer(self, container_id):
        containers = self._registry.findContainers(None, id = container_id)
        if not containers:
            UM.Logger.log("w", "Could duplicate container %s because it was not found.", container_id)
            return ""

        container = containers[0]

        new_container = None
        new_name = self._registry.uniqueName(container.getName())
        # Only InstanceContainer has a duplicate method at the moment.
        # So fall back to serialize/deserialize when no duplicate method exists.
        if hasattr(container, "duplicate"):
            new_container = container.duplicate(new_name)
        else:
            new_container = container.__class__(new_name)
            new_container.deserialize(container.serialize())
            new_container.setName(new_name)

        if new_container:
            self._registry.addContainer(new_container)

        return new_container.getId()

    ##  Change the name of a specified container to a new name.
    #
    #   \param container_id \type{str} The ID of the container to change the name of.
    #   \param new_id \type{str} The new ID of the container.
    #   \param new_name \type{str} The new name of the specified container.
    #
    #   \return True if successful, False if not.
    @pyqtSlot(str, str, str, result = bool)
    def renameContainer(self, container_id, new_id, new_name):
        containers = self._registry.findContainers(None, id = container_id)
        if not containers:
            UM.Logger.log("w", "Could rename container %s because it was not found.", container_id)
            return False

        container = containers[0]
        # First, remove the container from the registry. This will clean up any files related to the container.
        self._registry.removeContainer(container)

        # Ensure we have a unique name for the container
        new_name = self._registry.uniqueName(new_name)

        # Then, update the name and ID of the container
        container.setName(new_name)
        container._id = new_id # TODO: Find a nicer way to set a new, unique ID

        # Finally, re-add the container so it will be properly serialized again.
        self._registry.addContainer(container)

        return True

    ##  Remove the specified container.
    #
    #   \param container_id \type{str} The ID of the container to remove.
    #
    #   \return True if the container was successfully removed, False if not.
    @pyqtSlot(str, result = bool)
    def removeContainer(self, container_id):
        containers = self._registry.findContainers(None, id = container_id)
        if not containers:
            UM.Logger.log("w", "Could remove container %s because it was not found.", container_id)
            return False

        self._registry.removeContainer(containers[0].getId())

        return True

    ##  Merge a container with another.
    #
    #   This will try to merge one container into the other, by going through the container
    #   and setting the right properties on the other container.
    #
    #   \param merge_into_id \type{str} The ID of the container to merge into.
    #   \param merge_id \type{str} The ID of the container to merge.
    #
    #   \return True if successfully merged, False if not.
    @pyqtSlot(str, result = bool)
    def mergeContainers(self, merge_into_id, merge_id):
        containers = self._registry.findContainers(None, id = merge_into_id)
        if not containers:
            UM.Logger.log("w", "Could merge into container %s because it was not found.", merge_into_id)
            return False

        merge_into = containers[0]

        containers = self._registry.findContainers(None, id = merge_id)
        if not containers:
            UM.Logger.log("w", "Could not merge container %s because it was not found", merge_id)
            return False

        merge = containers[0]

        if type(merge) != type(merge_into):
            UM.Logger.log("w", "Cannot merge two containers of different types")
            return False

        for key in merge.getAllKeys():
            merge_into.setProperty(key, "value", merge.getProperty(key, "value"))

        return True

    ##  Clear the contents of a container.
    #
    #   \param container_id \type{str} The ID of the container to clear.
    #
    #   \return True if successful, False if not.
    @pyqtSlot(str, result = bool)
    def clearContainer(self, container_id):
        containers = self._registry.findContainers(None, id = container_id)
        if not containers:
            UM.Logger.log("w", "Could clear container %s because it was not found.", container_id)
            return False

        if containers[0].isReadOnly():
            UM.Logger.log("w", "Cannot clear read-only container %s", container_id)
            return False

        containers[0].clear()

        return True

    ##  Set a metadata entry of the specified container.
    #
    #   This will set the specified entry of the container's metadata to the specified
    #   value. Note that entries containing dictionaries can have their entries changed
    #   by using "/" as a separator. For example, to change an entry "foo" in a
    #   dictionary entry "bar", you can specify "bar/foo" as entry name.
    #
    #   \param container_id \type{str} The ID of the container to change.
    #   \param entry_name \type{str} The name of the metadata entry to change.
    #   \param entry_value The new value of the entry.
    #
    #   \return True if successful, False if not.
    @pyqtSlot(str, str, str, result = bool)
    def setContainerMetaDataEntry(self, container_id, entry_name, entry_value):
        containers = UM.Settings.ContainerRegistry.getInstance().findContainers(None, id = container_id)
        if not containers:
            UM.Logger.log("w", "Could set metadata of container %s because it was not found.", container_id)
            return False

        container = containers[0]

        if container.isReadOnly():
            UM.Logger.log("w", "Cannot set metadata of read-only container %s.", container_id)
            return False

        entries = entry_name.split("/")
        entry_name = entries.pop()

        if entries:
            root_name = entries.pop(0)
            root = container.getMetaDataEntry(root_name)

            item = root
            for entry in entries:
                item = item.get(entries.pop(0), { })

            item[entry_name] = entry_value

            entry_name = root_name
            entry_value = root

        container.setMetaDataEntry(entry_name, entry_value)

        return True

    ##  Find instance containers matching certain criteria.
    #
    #   This effectively forwards to ContainerRegistry::findInstanceContainers.
    #
    #   \param criteria A dict of key - value pairs to search for.
    #
    #   \return A list of container IDs that match the given criteria.
    @pyqtSlot("QVariantMap", result = "QVariantList")
    def findInstanceContainers(self, criteria):
        result = []
        for entry in self._registry.findInstanceContainers(**criteria):
            result.append(entry.getId())

        return result

    ##  Get a list of string that can be used as name filters for a Qt File Dialog
    #
    #   This will go through the list of available container types and generate a list of strings
    #   out of that. The strings are formatted as "description (*.extension)" and can be directly
    #   passed to a nameFilters property of a Qt File Dialog.
    #
    #   \param type_name Which types of containers to list. These types correspond to the "type"
    #                    key of the plugin metadata.
    #
    #   \return A string list with name filters.
    @pyqtSlot(str, result = "QStringList")
    def getContainerNameFilters(self, type_name):
        if not self._container_name_filters:
            self._updateContainerNameFilters()

        filters = []
        for filter_string, entry in self._container_name_filters.items():
            if not type_name or entry["type"] == type_name:
                filters.append(filter_string)

        filters.append("All Files (*)")
        return filters

    ##  Export a container to a file
    #
    #   \param container_id The ID of the container to export
    #   \param file_type The type of file to save as. Should be in the form of "description (*.extension, *.ext)"
    #   \param file_url The URL where to save the file.
    #
    #   \return A dictionary containing a key "status" with a status code and a key "message" with a message
    #           explaining the status.
    #           The status code can be one of "error", "cancelled", "success"
    @pyqtSlot(str, str, QUrl, result = "QVariantMap")
    def exportContainer(self, container_id, file_type, file_url):
        if not container_id or not file_type or not file_url:
            return { "status": "error", "message": "Invalid arguments"}

        if isinstance(file_url, QUrl):
            file_url = file_url.toLocalFile()

        if not file_url:
            return { "status": "error", "message": "Invalid path"}

        mime_type = None
        if not file_type in self._container_name_filters:
            try:
                mime_type = UM.MimeTypeDatabase.getMimeTypeForFile(file_url)
            except MimeTypeNotFoundError:
                return { "status": "error", "message": "Unknown File Type" }
        else:
            mime_type = self._container_name_filters[file_type]["mime"]

        containers = UM.Settings.ContainerRegistry.getInstance().findContainers(None, id = container_id)
        if not containers:
            return { "status": "error", "message": "Container not found"}
        container = containers[0]

        if UM.Platform.isOSX() and "." in file_url:
            file_url = file_url[:file_url.rfind(".")]

        for suffix in mime_type.suffixes:
            if file_url.endswith(suffix):
                break
        else:
            file_url += "." + mime_type.preferredSuffix

        if not UM.Platform.isWindows():
            if os.path.exists(file_url):
                result = QMessageBox.question(None, catalog.i18nc("@title:window", "File Already Exists"),
                                              catalog.i18nc("@label", "The file <filename>{0}</filename> already exists. Are you sure you want to overwrite it?").format(file_url))
                if result == QMessageBox.No:
                    return { "status": "cancelled", "message": "User cancelled"}

        try:
            contents = container.serialize()
        except NotImplementedError:
            return { "status": "error", "message": "Unable to serialize container"}

        with UM.SaveFile(file_url, "w") as f:
            f.write(contents)

        return { "status": "success", "message": "Succesfully exported container", "path": file_url}

    ##  Imports a profile from a file
    #
    #   \param file_url A URL that points to the file to import.
    #
    #   \return \type{Dict} dict with a 'status' key containing the string 'success' or 'error', and a 'message' key
    #       containing a message for the user
    @pyqtSlot(QUrl, result = "QVariantMap")
    def importContainer(self, file_url):
        if not file_url:
            return { "status": "error", "message": "Invalid path"}

        if isinstance(file_url, QUrl):
            file_url = file_url.toLocalFile()

        if not file_url or not os.path.exists(file_url):
            return { "status": "error", "message": "Invalid path" }

        try:
            mime_type = UM.MimeTypeDatabase.getMimeTypeForFile(file_url)
        except MimeTypeNotFoundError:
            return { "status": "error", "message": "Could not determine mime type of file" }

        container_type = UM.Settings.ContainerRegistry.getContainerForMimeType(mime_type)
        if not container_type:
            return { "status": "error", "message": "Could not find a container to handle the specified file."}

        container_id = urllib.parse.unquote_plus(mime_type.stripExtension(os.path.basename(file_url)))
        container_id = UM.Settings.ContainerRegistry.getInstance().uniqueName(container_id)

        container = container_type(container_id)

        try:
            with open(file_url, "rt") as f:
                container.deserialize(f.read())
        except PermissionError:
            return { "status": "error", "message": "Permission denied when trying to read the file"}

        container.setName(container_id)

        UM.Settings.ContainerRegistry.getInstance().addContainer(container)

        return { "status": "success", "message": "Successfully imported container {0}".format(container.getName()) }

    def _updateContainerNameFilters(self):
        self._container_name_filters = {}
        for plugin_id, container_type in UM.Settings.ContainerRegistry.getContainerTypes():
            # Ignore default container types since those are not plugins
            if container_type in (UM.Settings.InstanceContainer, UM.Settings.ContainerStack, UM.Settings.DefinitionContainer):
                continue

            serialize_type = ""
            try:
                plugin_metadata = UM.PluginRegistry.getInstance().getMetaData(plugin_id)
                if plugin_metadata:
                    serialize_type = plugin_metadata["settings_container"]["type"]
                else:
                    continue
            except KeyError as e:
                continue

            mime_type = UM.Settings.ContainerRegistry.getMimeTypeForContainer(container_type)

            entry = {
                "type": serialize_type,
                "mime": mime_type,
                "container": container_type
            }

            suffix = mime_type.preferredSuffix
            if UM.Platform.isOSX() and "." in suffix:
                # OSX's File dialog is stupid and does not allow selecting files with a . in its name
                suffix = suffix[suffix.index(".") + 1:]

            suffix_list = "*." + suffix
            for suffix in mime_type.suffixes:
                if suffix == mime_type.preferredSuffix:
                    continue

                if UM.Platform.isOSX() and "." in suffix:
                    # OSX's File dialog is stupid and does not allow selecting files with a . in its name
                    suffix = suffix[suffix.index("."):]

                suffix_list += ", *." + suffix

            name_filter = "{0} ({1})".format(mime_type.comment, suffix_list)
            self._container_name_filters[name_filter] = entry

    # Factory function, used by QML
    @staticmethod
    def createContainerManager(engine, js_engine):
        return ContainerManager()
<commit_msg>Split ContainerManager::mergeContainer into a checking bit and a function that does the actual work<commit_after># Copyright (c) 2016 Ultimaker B.V.
# Cura is released under the terms of the AGPLv3 or higher.

import os.path
import urllib

from PyQt5.QtCore import QObject, pyqtSlot, pyqtProperty, pyqtSignal, QUrl
from PyQt5.QtWidgets import QMessageBox

import UM.PluginRegistry
import UM.Settings
import UM.SaveFile
import UM.Platform
import UM.MimeTypeDatabase
import UM.Logger

from UM.MimeTypeDatabase import MimeTypeNotFoundError

from UM.i18n import i18nCatalog
catalog = i18nCatalog("cura")

##  Manager class that contains common actions to deal with containers in Cura.
#
#   This is primarily intended as a class to be able to perform certain actions
#   from within QML. We want to be able to trigger things like removing a container
#   when a certain action happens. This can be done through this class.
class ContainerManager(QObject):
    def __init__(self, parent = None):
        super().__init__(parent)

        self._registry = UM.Settings.ContainerRegistry.getInstance()
        self._container_name_filters = {}

    ##  Create a duplicate of the specified container
    #
    #   This will create and add a duplicate of the container corresponding
    #   to the container ID.
    #
    #   \param container_id \type{str} The ID of the container to duplicate.
    #
    #   \return The ID of the new container, or an empty string if duplication failed.
    @pyqtSlot(str, result = str)
    def duplicateContainer(self, container_id):
        containers = self._registry.findContainers(None, id = container_id)
        if not containers:
            UM.Logger.log("w", "Could duplicate container %s because it was not found.", container_id)
            return ""

        container = containers[0]

        new_container = None
        new_name = self._registry.uniqueName(container.getName())
        # Only InstanceContainer has a duplicate method at the moment.
        # So fall back to serialize/deserialize when no duplicate method exists.
        if hasattr(container, "duplicate"):
            new_container = container.duplicate(new_name)
        else:
            new_container = container.__class__(new_name)
            new_container.deserialize(container.serialize())
            new_container.setName(new_name)

        if new_container:
            self._registry.addContainer(new_container)

        return new_container.getId()

    ##  Change the name of a specified container to a new name.
    #
    #   \param container_id \type{str} The ID of the container to change the name of.
    #   \param new_id \type{str} The new ID of the container.
    #   \param new_name \type{str} The new name of the specified container.
    #
    #   \return True if successful, False if not.
    @pyqtSlot(str, str, str, result = bool)
    def renameContainer(self, container_id, new_id, new_name):
        containers = self._registry.findContainers(None, id = container_id)
        if not containers:
            UM.Logger.log("w", "Could rename container %s because it was not found.", container_id)
            return False

        container = containers[0]
        # First, remove the container from the registry. This will clean up any files related to the container.
        self._registry.removeContainer(container)

        # Ensure we have a unique name for the container
        new_name = self._registry.uniqueName(new_name)

        # Then, update the name and ID of the container
        container.setName(new_name)
        container._id = new_id # TODO: Find a nicer way to set a new, unique ID

        # Finally, re-add the container so it will be properly serialized again.
        self._registry.addContainer(container)

        return True

    ##  Remove the specified container.
    #
    #   \param container_id \type{str} The ID of the container to remove.
    #
    #   \return True if the container was successfully removed, False if not.
    @pyqtSlot(str, result = bool)
    def removeContainer(self, container_id):
        containers = self._registry.findContainers(None, id = container_id)
        if not containers:
            UM.Logger.log("w", "Could remove container %s because it was not found.", container_id)
            return False

        self._registry.removeContainer(containers[0].getId())

        return True

    ##  Merge a container with another.
    #
    #   This will try to merge one container into the other, by going through the container
    #   and setting the right properties on the other container.
    #
    #   \param merge_into_id \type{str} The ID of the container to merge into.
    #   \param merge_id \type{str} The ID of the container to merge.
    #
    #   \return True if successfully merged, False if not.
    @pyqtSlot(str, result = bool)
    def mergeContainers(self, merge_into_id, merge_id):
        containers = self._registry.findContainers(None, id = merge_into_id)
        if not containers:
            UM.Logger.log("w", "Could merge into container %s because it was not found.", merge_into_id)
            return False

        merge_into = containers[0]

        containers = self._registry.findContainers(None, id = merge_id)
        if not containers:
            UM.Logger.log("w", "Could not merge container %s because it was not found", merge_id)
            return False

        merge = containers[0]

        if not isinstance(merge, type(merge_into)):
            UM.Logger.log("w", "Cannot merge two containers of different types")
            return False

        self._performMerge(merge_into, merge)

        return True

    ##  Clear the contents of a container.
    #
    #   \param container_id \type{str} The ID of the container to clear.
    #
    #   \return True if successful, False if not.
    @pyqtSlot(str, result = bool)
    def clearContainer(self, container_id):
        containers = self._registry.findContainers(None, id = container_id)
        if not containers:
            UM.Logger.log("w", "Could clear container %s because it was not found.", container_id)
            return False

        if containers[0].isReadOnly():
            UM.Logger.log("w", "Cannot clear read-only container %s", container_id)
            return False

        containers[0].clear()

        return True

    ##  Set a metadata entry of the specified container.
    #
    #   This will set the specified entry of the container's metadata to the specified
    #   value. Note that entries containing dictionaries can have their entries changed
    #   by using "/" as a separator. For example, to change an entry "foo" in a
    #   dictionary entry "bar", you can specify "bar/foo" as entry name.
    #
    #   \param container_id \type{str} The ID of the container to change.
    #   \param entry_name \type{str} The name of the metadata entry to change.
    #   \param entry_value The new value of the entry.
    #
    #   \return True if successful, False if not.
    @pyqtSlot(str, str, str, result = bool)
    def setContainerMetaDataEntry(self, container_id, entry_name, entry_value):
        containers = UM.Settings.ContainerRegistry.getInstance().findContainers(None, id = container_id)
        if not containers:
            UM.Logger.log("w", "Could set metadata of container %s because it was not found.", container_id)
            return False

        container = containers[0]

        if container.isReadOnly():
            UM.Logger.log("w", "Cannot set metadata of read-only container %s.", container_id)
            return False

        entries = entry_name.split("/")
        entry_name = entries.pop()

        if entries:
            root_name = entries.pop(0)
            root = container.getMetaDataEntry(root_name)

            item = root
            for entry in entries:
                item = item.get(entries.pop(0), { })

            item[entry_name] = entry_value

            entry_name = root_name
            entry_value = root

        container.setMetaDataEntry(entry_name, entry_value)

        return True

    ##  Find instance containers matching certain criteria.
    #
    #   This effectively forwards to ContainerRegistry::findInstanceContainers.
    #
    #   \param criteria A dict of key - value pairs to search for.
    #
    #   \return A list of container IDs that match the given criteria.
    @pyqtSlot("QVariantMap", result = "QVariantList")
    def findInstanceContainers(self, criteria):
        result = []
        for entry in self._registry.findInstanceContainers(**criteria):
            result.append(entry.getId())

        return result

    ##  Get a list of string that can be used as name filters for a Qt File Dialog
    #
    #   This will go through the list of available container types and generate a list of strings
    #   out of that. The strings are formatted as "description (*.extension)" and can be directly
    #   passed to a nameFilters property of a Qt File Dialog.
    #
    #   \param type_name Which types of containers to list. These types correspond to the "type"
    #                    key of the plugin metadata.
    #
    #   \return A string list with name filters.
    @pyqtSlot(str, result = "QStringList")
    def getContainerNameFilters(self, type_name):
        if not self._container_name_filters:
            self._updateContainerNameFilters()

        filters = []
        for filter_string, entry in self._container_name_filters.items():
            if not type_name or entry["type"] == type_name:
                filters.append(filter_string)

        filters.append("All Files (*)")
        return filters

    ##  Export a container to a file
    #
    #   \param container_id The ID of the container to export
    #   \param file_type The type of file to save as. Should be in the form of "description (*.extension, *.ext)"
    #   \param file_url The URL where to save the file.
    #
    #   \return A dictionary containing a key "status" with a status code and a key "message" with a message
    #           explaining the status.
    #           The status code can be one of "error", "cancelled", "success"
    @pyqtSlot(str, str, QUrl, result = "QVariantMap")
    def exportContainer(self, container_id, file_type, file_url):
        if not container_id or not file_type or not file_url:
            return { "status": "error", "message": "Invalid arguments"}

        if isinstance(file_url, QUrl):
            file_url = file_url.toLocalFile()

        if not file_url:
            return { "status": "error", "message": "Invalid path"}

        mime_type = None
        if not file_type in self._container_name_filters:
            try:
                mime_type = UM.MimeTypeDatabase.getMimeTypeForFile(file_url)
            except MimeTypeNotFoundError:
                return { "status": "error", "message": "Unknown File Type" }
        else:
            mime_type = self._container_name_filters[file_type]["mime"]

        containers = UM.Settings.ContainerRegistry.getInstance().findContainers(None, id = container_id)
        if not containers:
            return { "status": "error", "message": "Container not found"}
        container = containers[0]

        if UM.Platform.isOSX() and "." in file_url:
            file_url = file_url[:file_url.rfind(".")]

        for suffix in mime_type.suffixes:
            if file_url.endswith(suffix):
                break
        else:
            file_url += "." + mime_type.preferredSuffix

        if not UM.Platform.isWindows():
            if os.path.exists(file_url):
                result = QMessageBox.question(None, catalog.i18nc("@title:window", "File Already Exists"),
                                              catalog.i18nc("@label", "The file <filename>{0}</filename> already exists. Are you sure you want to overwrite it?").format(file_url))
                if result == QMessageBox.No:
                    return { "status": "cancelled", "message": "User cancelled"}

        try:
            contents = container.serialize()
        except NotImplementedError:
            return { "status": "error", "message": "Unable to serialize container"}

        with UM.SaveFile(file_url, "w") as f:
            f.write(contents)

        return { "status": "success", "message": "Succesfully exported container", "path": file_url}

    ##  Imports a profile from a file
    #
    #   \param file_url A URL that points to the file to import.
    #
    #   \return \type{Dict} dict with a 'status' key containing the string 'success' or 'error', and a 'message' key
    #       containing a message for the user
    @pyqtSlot(QUrl, result = "QVariantMap")
    def importContainer(self, file_url):
        if not file_url:
            return { "status": "error", "message": "Invalid path"}

        if isinstance(file_url, QUrl):
            file_url = file_url.toLocalFile()

        if not file_url or not os.path.exists(file_url):
            return { "status": "error", "message": "Invalid path" }

        try:
            mime_type = UM.MimeTypeDatabase.getMimeTypeForFile(file_url)
        except MimeTypeNotFoundError:
            return { "status": "error", "message": "Could not determine mime type of file" }

        container_type = UM.Settings.ContainerRegistry.getContainerForMimeType(mime_type)
        if not container_type:
            return { "status": "error", "message": "Could not find a container to handle the specified file."}

        container_id = urllib.parse.unquote_plus(mime_type.stripExtension(os.path.basename(file_url)))
        container_id = UM.Settings.ContainerRegistry.getInstance().uniqueName(container_id)

        container = container_type(container_id)

        try:
            with open(file_url, "rt") as f:
                container.deserialize(f.read())
        except PermissionError:
            return { "status": "error", "message": "Permission denied when trying to read the file"}

        container.setName(container_id)

        UM.Settings.ContainerRegistry.getInstance().addContainer(container)

        return { "status": "success", "message": "Successfully imported container {0}".format(container.getName()) }

    # Factory function, used by QML
    @staticmethod
    def createContainerManager(engine, js_engine):
        return ContainerManager()

    def _performMerge(self, merge_into, merge):
        assert isinstance(merge, type(merge_into))

        if merge == merge_into:
            return

        for key in merge.getAllKeys():
            merge_into.setProperty(key, "value", merge.getProperty(key, "value"))

        merge.clear()

    def _updateContainerNameFilters(self):
        self._container_name_filters = {}
        for plugin_id, container_type in UM.Settings.ContainerRegistry.getContainerTypes():
            # Ignore default container types since those are not plugins
            if container_type in (UM.Settings.InstanceContainer, UM.Settings.ContainerStack, UM.Settings.DefinitionContainer):
                continue

            serialize_type = ""
            try:
                plugin_metadata = UM.PluginRegistry.getInstance().getMetaData(plugin_id)
                if plugin_metadata:
                    serialize_type = plugin_metadata["settings_container"]["type"]
                else:
                    continue
            except KeyError as e:
                continue

            mime_type = UM.Settings.ContainerRegistry.getMimeTypeForContainer(container_type)

            entry = {
                "type": serialize_type,
                "mime": mime_type,
                "container": container_type
            }

            suffix = mime_type.preferredSuffix
            if UM.Platform.isOSX() and "." in suffix:
                # OSX's File dialog is stupid and does not allow selecting files with a . in its name
                suffix = suffix[suffix.index(".") + 1:]

            suffix_list = "*." + suffix
            for suffix in mime_type.suffixes:
                if suffix == mime_type.preferredSuffix:
                    continue

                if UM.Platform.isOSX() and "." in suffix:
                    # OSX's File dialog is stupid and does not allow selecting files with a . in its name
                    suffix = suffix[suffix.index("."):]

                suffix_list += ", *." + suffix

            name_filter = "{0} ({1})".format(mime_type.comment, suffix_list)
            self._container_name_filters[name_filter] = entry
<|endoftext|>


<commit_before>#!/usr/bin/env python
#-- Content-Encoding: UTF-8 --
"""
Forker heart beat.

Sends heart beat signals to the monitors to signal the forker presence.

:author: Thomas Calmant
"""

# Documentation strings format
__docformat__ = "restructuredtext en"

# ------------------------------------------------------------------------------

from pelix.ipopo.decorators import ComponentFactory, Instantiate, \
    Requires, Validate, Invalidate

# ------------------------------------------------------------------------------

import logging
_logger = logging.getLogger(__name__)

import socket
import struct
import sys
import threading

# ------------------------------------------------------------------------------

# Heart beat packet type
PACKET_TYPE_HEARTBEAT = 1

# PSEM2M property: Isolate ID
PROPERTY_ISOLATE_ID = 'psem2m.isolate.id'

# PSEM2M property: Isolate Node
PROPERTY_ISOLATE_NODE = 'psem2m.isolate.node'

# ------------------------------------------------------------------------------

if sys.version_info[0] == 3:
    # Python 3
    def _to_bytes(data, encoding="UTF-8"):
        """
        Converts the given string to a bytes array
        """
        if type(data) is bytes:
            # Nothing to do
            return data

        return data.encode(encoding)

else:
    # Python 2
    def _to_bytes(data, encoding="UTF-8"):
        """
        Converts the given string to a bytes array
        """
        if type(data) is str:
            # Nothing to do
            return data

        return data.encode(encoding)

# ------------------------------------------------------------------------------

import os

if os.name == "nt":
    # Windows Specific code
    def pton(family, address):
        """
        Calls inet_pton

        :param family: Socket family
        :param address: A string address
        :return: The binary form of the given address
        """
        if family == socket.AF_INET:
            return socket.inet_aton(address)

        elif family == socket.AF_INET6:
            # Do it using WinSocks
            import ctypes
            winsock = ctypes.windll.ws2_32

            # Prepare structure
            class sockaddr_in6(ctypes.Structure):
                _fields_ = [("sin6_family", ctypes.c_short),
                            ("sin6_port", ctypes.c_ushort),
                            ("sin6_flowinfo", ctypes.c_ulong),
                            ("sin6_addr", ctypes.c_ubyte * 16),
                            ("sin6_scope_id", ctypes.c_ulong)
                            ]

            # Prepare pointers
            addr_ptr = ctypes.c_char_p(_to_bytes(address))

            out_address = sockaddr_in6()
            size = len(sockaddr_in6)
            size_ptr = ctypes.pointer(size)

            # Second call
            winsock.WSAStringToAddressA(addr_ptr, family, 0,
                                        out_address, size_ptr)

            # Convert the array...
            bin_addr = 0
            for part in out_address.sin6_addr:
                bin_addr = bin_addr * 16 + part

            return bin_addr

        else:
            raise ValueError("Unhandled socket family: %s", family)

else:
    # Other systems
    def pton(family, address):
        """
        Calls inet_pton

        :param family: Socket family
        :param address: A string address
        :return: The binary form of the given address
        """
        return socket.inet_pton(family, address)

def make_mreq(family, address):
    """
    Makes a mreq structure object for the given address and socket family.

    :param family: A socket family (AF_INET or AF_INET6)
    :param address: A multicast address
    :raise ValueError: Invalid family or address
    """
    if not address:
        raise ValueError("Empty address")

    # Convert the address to a binary form
    group_bin = pton(family, address)

    if family == socket.AF_INET:
        # IPv4
        # struct ip_mreq
        # {
        #     struct in_addr imr_multiaddr; /* IP multicast address of group */
        #     struct in_addr imr_interface; /* local IP address of interface */
        # };
        # "=I" : Native order, standard size unsigned int
        return group_bin + struct.pack("=I", socket.INADDR_ANY)

    elif family == socket.AF_INET6:
        # IPv6
        # struct ipv6_mreq {
        #    struct in6_addr ipv6mr_multiaddr;
        #    unsigned int    ipv6mr_interface;
        # };
        # "@I" : Native order, native size unsigned int
        return group_bin + struct.pack("@I", 0)

    raise ValueError("Unknown family %s", family)

def create_multicast_socket(address, port):
    """
    Creates a multicast socket according to the given address and port.
    Handles both IPv4 and IPv6 addresses.

    :raise ValueError: Invalid address or port
    """
    # Get the information about a datagram (UDP) socket, of any family
    try:
        addr_info = socket.getaddrinfo(address, port, socket.AF_UNSPEC,
                                       socket.SOCK_DGRAM)
    except socket.gaierror:
        raise ValueError("Error retrieving address informations (%s, %s)" \
                         % (address, port))

    if len(addr_info) > 1:
        _logger.debug("More than one address information found. "
                      "Using the first one.")

    # Get the first entry : (family, socktype, proto, canonname, sockaddr)
    addr_info = addr_info[0]

    # Only accept IPv4/v6 addresses
    if addr_info[0] not in (socket.AF_INET, socket.AF_INET6):
        # Unhandled address family
        raise ValueError("Unhandled socket family : %d" % (addr_info[0]))

    # Prepare the socket
    sock = socket.socket(addr_info[0], socket.SOCK_DGRAM)

    # Reuse address
    sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    if hasattr(socket, 'SO_REUSEPORT'):
        # Special for MacOS
        sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)

    # Bind the socket
    if sock.family == socket.AF_INET:
        # IPv4 binding
        sock.bind(('0.0.0.0', port))

    else:
        # IPv6 Binding
        sock.bind(('::', port))

    # Prepare the mreq structure to join the group
    # addrinfo[4] = (addr,port)
    mreq = make_mreq(sock.family, addr_info[4][0])

    # Join the group
    if sock.family == socket.AF_INET:
        # IPv4
        sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq)

        # Allow multicast packets to get back on this host
        sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_LOOP, 1)

    else:
        # IPv6
        sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_JOIN_GROUP, mreq)

        # Allow multicast packets to get back on this host
        sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_LOOP, 1)

    return (sock, addr_info[4][0])

def close_multicast_socket(sock, address):
    """
    Cleans up the given multicast socket.
    Unregisters it of the multicast group.

    Parameters should be the result of create_multicast_socket

    :param sock: A multicast socket
    :param address: The multicast address used by the socket
    """
    if not sock:
        return

    if address:
        # Prepare the mreq structure to join the group
        mreq = make_mreq(sock.family, address)

        # Quit group
        if sock.family == socket.AF_INET:
            # IPv4
            sock.setsockopt(socket.IPPROTO_IP, socket.IP_DROP_MEMBERSHIP, mreq)

        elif sock.family == socket.AF_INET6:
            # IPv6
            sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_LEAVE_GROUP, mreq)

    # Close the socket
    sock.close()

# ------------------------------------------------------------------------------

def make_heartbeat(port, application_id, isolate_id, node_id):
    """
    Prepares the heart beat UDP packet

    Format : Little endian
    * Packet type (1 byte)
    * Signals port (2 bytes)
    * Application ID length (2 bytes)
    * application_id (variable, UTF-8)
    * Isolate ID length (2 bytes)
    * Isolate ID (variable, UTF-8)
    * Node ID length (2 bytes)
    * Node ID (variable, UTF-8)

    :param port: The Signals access port
    :param application_id: The ID of the current application
    :param isolate_id: The ID of this isolate
    :param node_id: The host node ID
    :return: The heart beat packet content (byte array)
    """
    # Type and port...
    packet = struct.pack("<BH", PACKET_TYPE_HEARTBEAT, port)

    for string in (application_id, isolate_id, node_id):
        # Strings...
        string_bytes = _to_bytes(string)
        packet += struct.pack("<H", len(string_bytes))
        packet += string_bytes

    return packet

# ------------------------------------------------------------------------------

@ComponentFactory("psem2m-forker-heart-factory")
@Instantiate("psem2m-forker-heart")
@Requires("_config", "org.psem2m.isolates.services.conf.ISvcConfig")
# Just to have the same life cycle than the forker...
@Requires("_forker", "org.psem2m.isolates.services.forker.IForker")
# To retrieve access information
@Requires("_http", "HttpService")
@Requires("_receiver", "org.psem2m.signals.ISignalReceiver")
@Requires("_sender", "org.psem2m.signals.ISignalBroadcaster")
class Heart(object):
    """
    The heart beat sender
    """
    def __init__(self):
        """
        Constructor
        """
        self._config = None
        self._forker = None
        self._http = None
        self._receiver = None
        self._sender = None

        # Bundle context
        self._context = None

        # Socket
        self._socket = None
        self._target = None

        # Thread control
        self._event = None
        self._thread = None

    def _run(self):
        """
        Heart beat sender
        """
        # Prepare the packet
        app = self._config.get_application()
        if app:
            appId = app.get_application_id()

        else:
            appId = "<unknown-app>"

        beat = make_heartbeat(self._http.get_port(),
                              appId,
                              self._context.get_property(PROPERTY_ISOLATE_ID),
                              self._context.get_property(PROPERTY_ISOLATE_NODE))

        _logger.debug("HEARTBEAT: appId='%s', id='%s', node='%s'",
                      appId,
                      self._context.get_property(PROPERTY_ISOLATE_ID),
                      self._context.get_property(PROPERTY_ISOLATE_NODE))

        while not self._event.is_set():
            # Send the heart beat using the multicast socket
            self._socket.sendto(beat, 0, self._target)

            # Wait 3 seconds before next loop
            self._event.wait(3)

    @Invalidate
    def invalidate(self, context):
        """
        Component invalidated
        """
        # Get out of the waiting condition
        self._event.set()

        # Unregister to the signals
        # self._receiver.unregister_listener(SIGNAL_MATCH_ALL, self)

        # Wait for the thread to stop
        self._thread.join(.5)

        # Close the socket
        close_multicast_socket(self._socket, self._target[0])

        # Clean up
        self._context = None
        self._event = None
        self._socket = None
        self._target = None
        self._thread = None

    @Validate
    def validate(self, context):
        """
        Component validated
        """
        self._context = context

        # Get the multicast configuration
        group = self._config.get_application().get_multicast_group()
        port = self._config.get_application().get_multicast_port()

        # Create the socket
        self._socket, address = create_multicast_socket(group, port)

        # Store group information
        self._target = (address, port)

        # Prepare the thread controls
        self._event = threading.Event()

        # Register to signals
        # self._receiver.register_listener(SIGNAL_MATCH_ALL, self)

        # Start the heart
        self._thread = threading.Thread(target=self._run, name="HeartBeart")
        self._thread.start()
<commit_msg>Suppression d'une trace dans HeartBeat<commit_after>#!/usr/bin/env python
#-- Content-Encoding: UTF-8 --
"""
Forker heart beat.

Sends heart beat signals to the monitors to signal the forker presence.

:author: Thomas Calmant
"""

# Documentation strings format
__docformat__ = "restructuredtext en"

# ------------------------------------------------------------------------------

from pelix.ipopo.decorators import ComponentFactory, Instantiate, \
    Requires, Validate, Invalidate

# ------------------------------------------------------------------------------

import logging
_logger = logging.getLogger(__name__)

import socket
import struct
import sys
import threading

# ------------------------------------------------------------------------------

# Heart beat packet type
PACKET_TYPE_HEARTBEAT = 1

# PSEM2M property: Isolate ID
PROPERTY_ISOLATE_ID = 'psem2m.isolate.id'

# PSEM2M property: Isolate Node
PROPERTY_ISOLATE_NODE = 'psem2m.isolate.node'

# ------------------------------------------------------------------------------

if sys.version_info[0] == 3:
    # Python 3
    def _to_bytes(data, encoding="UTF-8"):
        """
        Converts the given string to a bytes array
        """
        if type(data) is bytes:
            # Nothing to do
            return data

        return data.encode(encoding)

else:
    # Python 2
    def _to_bytes(data, encoding="UTF-8"):
        """
        Converts the given string to a bytes array
        """
        if type(data) is str:
            # Nothing to do
            return data

        return data.encode(encoding)

# ------------------------------------------------------------------------------

import os

if os.name == "nt":
    # Windows Specific code
    def pton(family, address):
        """
        Calls inet_pton

        :param family: Socket family
        :param address: A string address
        :return: The binary form of the given address
        """
        if family == socket.AF_INET:
            return socket.inet_aton(address)

        elif family == socket.AF_INET6:
            # Do it using WinSocks
            import ctypes
            winsock = ctypes.windll.ws2_32

            # Prepare structure
            class sockaddr_in6(ctypes.Structure):
                _fields_ = [("sin6_family", ctypes.c_short),
                            ("sin6_port", ctypes.c_ushort),
                            ("sin6_flowinfo", ctypes.c_ulong),
                            ("sin6_addr", ctypes.c_ubyte * 16),
                            ("sin6_scope_id", ctypes.c_ulong)
                            ]

            # Prepare pointers
            addr_ptr = ctypes.c_char_p(_to_bytes(address))

            out_address = sockaddr_in6()
            size = len(sockaddr_in6)
            size_ptr = ctypes.pointer(size)

            # Second call
            winsock.WSAStringToAddressA(addr_ptr, family, 0,
                                        out_address, size_ptr)

            # Convert the array...
            bin_addr = 0
            for part in out_address.sin6_addr:
                bin_addr = bin_addr * 16 + part

            return bin_addr

        else:
            raise ValueError("Unhandled socket family: %s", family)

else:
    # Other systems
    def pton(family, address):
        """
        Calls inet_pton

        :param family: Socket family
        :param address: A string address
        :return: The binary form of the given address
        """
        return socket.inet_pton(family, address)

def make_mreq(family, address):
    """
    Makes a mreq structure object for the given address and socket family.

    :param family: A socket family (AF_INET or AF_INET6)
    :param address: A multicast address
    :raise ValueError: Invalid family or address
    """
    if not address:
        raise ValueError("Empty address")

    # Convert the address to a binary form
    group_bin = pton(family, address)

    if family == socket.AF_INET:
        # IPv4
        # struct ip_mreq
        # {
        #     struct in_addr imr_multiaddr; /* IP multicast address of group */
        #     struct in_addr imr_interface; /* local IP address of interface */
        # };
        # "=I" : Native order, standard size unsigned int
        return group_bin + struct.pack("=I", socket.INADDR_ANY)

    elif family == socket.AF_INET6:
        # IPv6
        # struct ipv6_mreq {
        #    struct in6_addr ipv6mr_multiaddr;
        #    unsigned int    ipv6mr_interface;
        # };
        # "@I" : Native order, native size unsigned int
        return group_bin + struct.pack("@I", 0)

    raise ValueError("Unknown family %s", family)

def create_multicast_socket(address, port):
    """
    Creates a multicast socket according to the given address and port.
    Handles both IPv4 and IPv6 addresses.

    :raise ValueError: Invalid address or port
    """
    # Get the information about a datagram (UDP) socket, of any family
    try:
        addr_info = socket.getaddrinfo(address, port, socket.AF_UNSPEC,
                                       socket.SOCK_DGRAM)
    except socket.gaierror:
        raise ValueError("Error retrieving address informations (%s, %s)" \
                         % (address, port))

    if len(addr_info) > 1:
        _logger.debug("More than one address information found. "
                      "Using the first one.")

    # Get the first entry : (family, socktype, proto, canonname, sockaddr)
    addr_info = addr_info[0]

    # Only accept IPv4/v6 addresses
    if addr_info[0] not in (socket.AF_INET, socket.AF_INET6):
        # Unhandled address family
        raise ValueError("Unhandled socket family : %d" % (addr_info[0]))

    # Prepare the socket
    sock = socket.socket(addr_info[0], socket.SOCK_DGRAM)

    # Reuse address
    sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    if hasattr(socket, 'SO_REUSEPORT'):
        # Special for MacOS
        sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)

    # Bind the socket
    if sock.family == socket.AF_INET:
        # IPv4 binding
        sock.bind(('0.0.0.0', port))

    else:
        # IPv6 Binding
        sock.bind(('::', port))

    # Prepare the mreq structure to join the group
    # addrinfo[4] = (addr,port)
    mreq = make_mreq(sock.family, addr_info[4][0])

    # Join the group
    if sock.family == socket.AF_INET:
        # IPv4
        sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq)

        # Allow multicast packets to get back on this host
        sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_LOOP, 1)

    else:
        # IPv6
        sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_JOIN_GROUP, mreq)

        # Allow multicast packets to get back on this host
        sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_LOOP, 1)

    return (sock, addr_info[4][0])

def close_multicast_socket(sock, address):
    """
    Cleans up the given multicast socket.
    Unregisters it of the multicast group.

    Parameters should be the result of create_multicast_socket

    :param sock: A multicast socket
    :param address: The multicast address used by the socket
    """
    if not sock:
        return

    if address:
        # Prepare the mreq structure to join the group
        mreq = make_mreq(sock.family, address)

        # Quit group
        if sock.family == socket.AF_INET:
            # IPv4
            sock.setsockopt(socket.IPPROTO_IP, socket.IP_DROP_MEMBERSHIP, mreq)

        elif sock.family == socket.AF_INET6:
            # IPv6
            sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_LEAVE_GROUP, mreq)

    # Close the socket
    sock.close()

# ------------------------------------------------------------------------------

def make_heartbeat(port, application_id, isolate_id, node_id):
    """
    Prepares the heart beat UDP packet

    Format : Little endian
    * Packet type (1 byte)
    * Signals port (2 bytes)
    * Application ID length (2 bytes)
    * application_id (variable, UTF-8)
    * Isolate ID length (2 bytes)
    * Isolate ID (variable, UTF-8)
    * Node ID length (2 bytes)
    * Node ID (variable, UTF-8)

    :param port: The Signals access port
    :param application_id: The ID of the current application
    :param isolate_id: The ID of this isolate
    :param node_id: The host node ID
    :return: The heart beat packet content (byte array)
    """
    # Type and port...
    packet = struct.pack("<BH", PACKET_TYPE_HEARTBEAT, port)

    for string in (application_id, isolate_id, node_id):
        # Strings...
        string_bytes = _to_bytes(string)
        packet += struct.pack("<H", len(string_bytes))
        packet += string_bytes

    return packet

# ------------------------------------------------------------------------------

@ComponentFactory("psem2m-forker-heart-factory")
@Instantiate("psem2m-forker-heart")
@Requires("_config", "org.psem2m.isolates.services.conf.ISvcConfig")
# Just to have the same life cycle than the forker...
@Requires("_forker", "org.psem2m.isolates.services.forker.IForker")
# To retrieve access information
@Requires("_http", "HttpService")
@Requires("_receiver", "org.psem2m.signals.ISignalReceiver")
@Requires("_sender", "org.psem2m.signals.ISignalBroadcaster")
class Heart(object):
    """
    The heart beat sender
    """
    def __init__(self):
        """
        Constructor
        """
        self._config = None
        self._forker = None
        self._http = None
        self._receiver = None
        self._sender = None

        # Bundle context
        self._context = None

        # Socket
        self._socket = None
        self._target = None

        # Thread control
        self._event = None
        self._thread = None

    def _run(self):
        """
        Heart beat sender
        """
        # Prepare the packet
        app = self._config.get_application()
        if app:
            appId = app.get_application_id()

        else:
            appId = "<unknown-app>"

        beat = make_heartbeat(self._http.get_port(),
                              appId,
                              self._context.get_property(PROPERTY_ISOLATE_ID),
                              self._context.get_property(PROPERTY_ISOLATE_NODE))

        while not self._event.is_set():
            # Send the heart beat using the multicast socket
            self._socket.sendto(beat, 0, self._target)

            # Wait 3 seconds before next loop
            self._event.wait(3)

    @Invalidate
    def invalidate(self, context):
        """
        Component invalidated
        """
        # Get out of the waiting condition
        self._event.set()

        # Unregister to the signals
        # self._receiver.unregister_listener(SIGNAL_MATCH_ALL, self)

        # Wait for the thread to stop
        self._thread.join(.5)

        # Close the socket
        close_multicast_socket(self._socket, self._target[0])

        # Clean up
        self._context = None
        self._event = None
        self._socket = None
        self._target = None
        self._thread = None

    @Validate
    def validate(self, context):
        """
        Component validated
        """
        self._context = context

        # Get the multicast configuration
        group = self._config.get_application().get_multicast_group()
        port = self._config.get_application().get_multicast_port()

        # Create the socket
        self._socket, address = create_multicast_socket(group, port)

        # Store group information
        self._target = (address, port)

        # Prepare the thread controls
        self._event = threading.Event()

        # Register to signals
        # self._receiver.register_listener(SIGNAL_MATCH_ALL, self)

        # Start the heart
        self._thread = threading.Thread(target=self._run, name="HeartBeart")
        self._thread.start()
<|endoftext|>


<commit_before>"""Scrapy settings"""
from os.path import join, dirname

EXTENSIONS = {
    'scrapy.contrib.logstats.LogStats': None,
    'scrapy.webservice.WebService': None,
    'scrapy.telnet.TelnetConsole': None,
    'scrapy.contrib.throttle.AutoThrottle': None
}

LOG_LEVEL = 'DEBUG'

# location of slybot projects - assumes a subdir per project
DATA_DIR = join(dirname(dirname(__file__)), 'data')
SPEC_DATA_DIR = join(DATA_DIR, 'projects')

SPEC_FACTORY = {
	'PROJECT_SPEC': 'slyd.projectspec.ProjectSpec',
	'PROJECT_MANAGER': 'slyd.projectspec.ProjectsManager',
	'PARAMS': {
		'location': SPEC_DATA_DIR,
	},
	'CAPABILITIES': {
		'version_control': False,
		'create_projects': True,
		'rename_projects': True,
		'deploy_projects': False,
	}
}

# recommended for development - use scrapy to cache http responses
# add them to local_settings.py
# HTTPCACHE_ENABLED = True
# HTTPCACHE_DIR = join(DATA_DIR, 'cache')

try:
    from local_settings import *
except ImportError:
    pass
<commit_msg>Fix incorrect path in settings<commit_after>"""Scrapy settings"""
from os.path import join, dirname

EXTENSIONS = {
    'scrapy.contrib.logstats.LogStats': None,
    'scrapy.webservice.WebService': None,
    'scrapy.telnet.TelnetConsole': None,
    'scrapy.contrib.throttle.AutoThrottle': None
}

LOG_LEVEL = 'DEBUG'

# location of slybot projects - assumes a subdir per project
DATA_DIR = join(dirname(dirname(__file__)), 'data')
SPEC_DATA_DIR = join(DATA_DIR, 'projects')

SPEC_FACTORY = {
    'PROJECT_SPEC': 'slyd.projectspec.ProjectSpec',
    'PROJECT_MANAGER': 'slyd.projects.ProjectsManager',
    'PARAMS': {
        'location': SPEC_DATA_DIR,
    },
    'CAPABILITIES': {
        'version_control': False,
        'create_projects': True,
        'rename_projects': True,
        'deploy_projects': False,
    }
}

# recommended for development - use scrapy to cache http responses
# add them to local_settings.py
# HTTPCACHE_ENABLED = True
# HTTPCACHE_DIR = join(DATA_DIR, 'cache')

try:
    from local_settings import *
except ImportError:
    pass
<|endoftext|>


<commit_before>import os
import shutil
from unittest import mock

import pytest

from dvc.path_info import PathInfo
from dvc.repo.tree import RepoTree

def test_exists(tmp_dir, dvc):
    tmp_dir.gen("foo", "foo")
    dvc.add("foo")
    (tmp_dir / "foo").unlink()

    tree = RepoTree(dvc)
    assert tree.exists("foo")

def test_open(tmp_dir, dvc):
    tmp_dir.gen("foo", "foo")
    dvc.add("foo")
    (tmp_dir / "foo").unlink()

    tree = RepoTree(dvc)
    with dvc.state:
        with tree.open("foo", "r") as fobj:
            assert fobj.read() == "foo"

def test_open_dirty_hash(tmp_dir, dvc):
    tmp_dir.dvc_gen("file", "file")
    (tmp_dir / "file").write_text("something")

    tree = RepoTree(dvc)
    with tree.open("file", "r") as fobj:
        assert fobj.read() == "something"

def test_open_dirty_no_hash(tmp_dir, dvc):
    tmp_dir.gen("file", "file")
    (tmp_dir / "file.dvc").write_text("outs:\n- path: file\n")

    tree = RepoTree(dvc)
    with tree.open("file", "r") as fobj:
        assert fobj.read() == "file"

def test_open_in_history(tmp_dir, scm, dvc):
    tmp_dir.gen("foo", "foo")
    dvc.add("foo")
    dvc.scm.add(["foo.dvc", ".gitignore"])
    dvc.scm.commit("foo")

    tmp_dir.gen("foo", "foofoo")
    dvc.add("foo")
    dvc.scm.add(["foo.dvc", ".gitignore"])
    dvc.scm.commit("foofoo")

    for rev in dvc.brancher(revs=["HEAD~1"]):
        if rev == "workspace":
            continue

        tree = RepoTree(dvc)
        with tree.open("foo", "r") as fobj:
            assert fobj.read() == "foo"

def test_isdir_isfile(tmp_dir, dvc):
    tmp_dir.gen({"datafile": "data", "datadir": {"foo": "foo", "bar": "bar"}})

    tree = RepoTree(dvc)
    assert tree.isdir("datadir")
    assert not tree.isfile("datadir")
    assert not tree.isdvc("datadir")
    assert not tree.isdir("datafile")
    assert tree.isfile("datafile")
    assert not tree.isdvc("datafile")

    dvc.add(["datadir", "datafile"])
    shutil.rmtree(tmp_dir / "datadir")
    (tmp_dir / "datafile").unlink()

    assert tree.isdir("datadir")
    assert not tree.isfile("datadir")
    assert tree.isdvc("datadir")
    assert not tree.isdir("datafile")
    assert tree.isfile("datafile")
    assert tree.isdvc("datafile")

def test_isdir_mixed(tmp_dir, dvc):
    tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}})

    dvc.add(str(tmp_dir / "dir" / "foo"))

    tree = RepoTree(dvc)
    assert tree.isdir("dir")
    assert not tree.isfile("dir")

@pytest.mark.parametrize(
    "dvcfiles,extra_expected",
    [
        (False, []),
        (
            True,
            [
                PathInfo("dir") / "subdir1" / "foo1.dvc",
                PathInfo("dir") / "subdir1" / "bar1.dvc",
                PathInfo("dir") / "subdir2" / "foo2.dvc",
            ],
        ),
    ],
)
def test_walk(tmp_dir, dvc, dvcfiles, extra_expected):
    tmp_dir.gen(
        {
            "dir": {
                "subdir1": {"foo1": "foo1", "bar1": "bar1"},
                "subdir2": {"foo2": "foo2"},
            }
        }
    )
    dvc.add(str(tmp_dir / "dir"), recursive=True)
    tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}})
    tree = RepoTree(dvc)

    expected = [
        PathInfo("dir") / "subdir1",
        PathInfo("dir") / "subdir2",
        PathInfo("dir") / "subdir1" / "foo1",
        PathInfo("dir") / "subdir1" / "bar1",
        PathInfo("dir") / "subdir2" / "foo2",
        PathInfo("dir") / "foo",
        PathInfo("dir") / "bar",
    ]

    actual = []
    for root, dirs, files in tree.walk("dir", dvcfiles=dvcfiles):
        for entry in dirs + files:
            actual.append(os.path.join(root, entry))

    expected = [str(path) for path in expected + extra_expected]
    assert set(actual) == set(expected)
    assert len(actual) == len(expected)

def test_walk_onerror(tmp_dir, dvc):
    def onerror(exc):
        raise exc

    tmp_dir.dvc_gen("foo", "foo")
    tree = RepoTree(dvc)

    # path does not exist
    for _ in tree.walk("dir"):
        pass
    with pytest.raises(OSError):
        for _ in tree.walk("dir", onerror=onerror):
            pass

    # path is not a directory
    for _ in tree.walk("foo"):
        pass
    with pytest.raises(OSError):
        for _ in tree.walk("foo", onerror=onerror):
            pass

def test_isdvc(tmp_dir, dvc):
    tmp_dir.gen({"foo": "foo", "bar": "bar", "dir": {"baz": "baz"}})
    dvc.add("foo")
    dvc.add("dir")
    tree = RepoTree(dvc)
    assert tree.isdvc("foo")
    assert not tree.isdvc("bar")
    assert tree.isdvc("dir")
    assert not tree.isdvc("dir/baz")
    assert tree.isdvc("dir/baz", recursive=True, strict=False)

def make_subrepo(dir_, scm, config=None):
    dir_.mkdir(parents=True)
    with dir_.chdir():
        dir_.scm = scm
        dir_.init(dvc=True, subdir=True)
        if config:
            dir_.add_remote(config=config)

def test_subrepos(tmp_dir, scm, dvc):
    tmp_dir.scm_gen(
        {"dir": {"repo.txt": "file to confuse RepoTree"}},
        commit="dir/repo.txt",
    )

    subrepo1 = tmp_dir / "dir" / "repo"
    subrepo2 = tmp_dir / "dir" / "repo2"

    for repo in [subrepo1, subrepo2]:
        make_subrepo(repo, scm)

    subrepo1.dvc_gen({"foo": "foo", "dir1": {"bar": "bar"}}, commit="FOO")
    subrepo2.dvc_gen(
        {"lorem": "lorem", "dir2": {"ipsum": "ipsum"}}, commit="BAR"
    )

    dvc.tree._reset()
    tree = RepoTree(dvc, subrepos=True, fetch=True)

    def assert_tree_belongs_to_repo(ret_val):
        method = tree._get_repo

        def f(*args, **kwargs):
            r = method(*args, **kwargs)
            assert r.root_dir == ret_val.root_dir
            return r

        return f

    with mock.patch.object(
        tree,
        "_get_repo",
        side_effect=assert_tree_belongs_to_repo(subrepo1.dvc),
    ):
        assert tree.exists(subrepo1 / "foo") is True
        assert tree.exists(subrepo1 / "bar") is False

        assert tree.isfile(subrepo1 / "foo") is True
        assert tree.isfile(subrepo1 / "foo") is True
        assert tree.isfile(subrepo1 / "dir1" / "bar") is True
        assert tree.isfile(subrepo1 / "dir1") is False

        assert tree.isdir(subrepo1 / "dir1") is True
        assert tree.isdir(subrepo1 / "dir1" / "bar") is False
        assert tree.isdvc(subrepo1 / "foo") is True

    with mock.patch.object(
        tree,
        "_get_repo",
        side_effect=assert_tree_belongs_to_repo(subrepo2.dvc),
    ):
        assert tree.exists(subrepo2 / "lorem") is True
        assert tree.exists(subrepo2 / "ipsum") is False

        assert tree.isfile(subrepo2 / "lorem") is True
        assert tree.isfile(subrepo2 / "lorem") is True
        assert tree.isfile(subrepo2 / "dir2" / "ipsum") is True
        assert tree.isfile(subrepo2 / "dir2") is False

        assert tree.isdir(subrepo2 / "dir2") is True
        assert tree.isdir(subrepo2 / "dir2" / "ipsum") is False
        assert tree.isdvc(subrepo2 / "lorem") is True

@pytest.mark.parametrize(
    "dvcfiles,extra_expected",
    [
        (False, []),
        (
            True,
            [
                PathInfo("dir") / "repo" / "foo.dvc",
                PathInfo("dir") / "repo" / "dir1.dvc",
                PathInfo("dir") / "repo2" / "lorem.dvc",
                PathInfo("dir") / "repo2" / "dir2.dvc",
            ],
        ),
    ],
)
def test_subrepo_walk(tmp_dir, scm, dvc, dvcfiles, extra_expected):
    tmp_dir.scm_gen(
        {"dir": {"repo.txt": "file to confuse RepoTree"}},
        commit="dir/repo.txt",
    )

    subrepo1 = tmp_dir / "dir" / "repo"
    subrepo2 = tmp_dir / "dir" / "repo2"

    subdirs = [subrepo1, subrepo2]
    for dir_ in subdirs:
        make_subrepo(dir_, scm)

    subrepo1.dvc_gen({"foo": "foo", "dir1": {"bar": "bar"}}, commit="FOO")
    subrepo2.dvc_gen(
        {"lorem": "lorem", "dir2": {"ipsum": "ipsum"}}, commit="BAR"
    )

    # using tree that does not have dvcignore
    dvc.tree._reset()
    tree = RepoTree(dvc, subrepos=True, fetch=True)
    expected = [
        PathInfo("dir") / "repo",
        PathInfo("dir") / "repo.txt",
        PathInfo("dir") / "repo2",
        PathInfo("dir") / "repo" / ".gitignore",
        PathInfo("dir") / "repo" / "foo",
        PathInfo("dir") / "repo" / "dir1",
        PathInfo("dir") / "repo" / "dir1" / "bar",
        PathInfo("dir") / "repo2" / ".gitignore",
        PathInfo("dir") / "repo2" / "lorem",
        PathInfo("dir") / "repo2" / "dir2",
        PathInfo("dir") / "repo2" / "dir2" / "ipsum",
    ]

    actual = []
    for root, dirs, files in tree.walk("dir", dvcfiles=dvcfiles):
        for entry in dirs + files:
            actual.append(os.path.join(root, entry))

    expected = [str(path) for path in expected + extra_expected]
    assert set(actual) == set(expected)
    assert len(actual) == len(expected)

def test_repo_tree_no_subrepos(tmp_dir, dvc, scm):
    tmp_dir.scm_gen(
        {"dir": {"repo.txt": "file to confuse RepoTree"}},
        commit="dir/repo.txt",
    )
    tmp_dir.dvc_gen({"lorem": "lorem"}, commit="add foo")

    subrepo = tmp_dir / "dir" / "repo"
    make_subrepo(subrepo, scm)
    subrepo.dvc_gen({"foo": "foo", "dir1": {"bar": "bar"}}, commit="FOO")
    subrepo.scm_gen({"ipsum": "ipsum"}, commit="BAR")

    # using tree that does not have dvcignore
    dvc.tree._reset()
    tree = RepoTree(dvc, subrepos=False, fetch=True)
    expected = [
        tmp_dir / ".gitignore",
        tmp_dir / "lorem",
        tmp_dir / "lorem.dvc",
        tmp_dir / "dir",
        tmp_dir / "dir" / "repo.txt",
    ]

    actual = []
    for root, dirs, files in tree.walk(tmp_dir, dvcfiles=True):
        for entry in dirs + files:
            actual.append(os.path.normpath(os.path.join(root, entry)))

    expected = [str(path) for path in expected]
    assert set(actual) == set(expected)
    assert len(actual) == len(expected)

    assert tree.isfile(tmp_dir / "lorem") is True
    assert tree.isfile(tmp_dir / "dir" / "repo" / "foo") is False
    assert tree.isdir(tmp_dir / "dir" / "repo") is False
    assert tree.isdir(tmp_dir / "dir") is True

    assert tree.isdvc(tmp_dir / "lorem") is True
    assert tree.isdvc(tmp_dir / "dir" / "repo" / "dir1") is False

    assert tree.exists(tmp_dir / "dir" / "repo.txt") is True
    assert tree.exists(tmp_dir / "repo" / "ipsum") is False
<commit_msg>tests: add .dvcignore on the tests expectations (#4396)<commit_after>import os
import shutil
from unittest import mock

import pytest

from dvc.path_info import PathInfo
from dvc.repo.tree import RepoTree

def test_exists(tmp_dir, dvc):
    tmp_dir.gen("foo", "foo")
    dvc.add("foo")
    (tmp_dir / "foo").unlink()

    tree = RepoTree(dvc)
    assert tree.exists("foo")

def test_open(tmp_dir, dvc):
    tmp_dir.gen("foo", "foo")
    dvc.add("foo")
    (tmp_dir / "foo").unlink()

    tree = RepoTree(dvc)
    with dvc.state:
        with tree.open("foo", "r") as fobj:
            assert fobj.read() == "foo"

def test_open_dirty_hash(tmp_dir, dvc):
    tmp_dir.dvc_gen("file", "file")
    (tmp_dir / "file").write_text("something")

    tree = RepoTree(dvc)
    with tree.open("file", "r") as fobj:
        assert fobj.read() == "something"

def test_open_dirty_no_hash(tmp_dir, dvc):
    tmp_dir.gen("file", "file")
    (tmp_dir / "file.dvc").write_text("outs:\n- path: file\n")

    tree = RepoTree(dvc)
    with tree.open("file", "r") as fobj:
        assert fobj.read() == "file"

def test_open_in_history(tmp_dir, scm, dvc):
    tmp_dir.gen("foo", "foo")
    dvc.add("foo")
    dvc.scm.add(["foo.dvc", ".gitignore"])
    dvc.scm.commit("foo")

    tmp_dir.gen("foo", "foofoo")
    dvc.add("foo")
    dvc.scm.add(["foo.dvc", ".gitignore"])
    dvc.scm.commit("foofoo")

    for rev in dvc.brancher(revs=["HEAD~1"]):
        if rev == "workspace":
            continue

        tree = RepoTree(dvc)
        with tree.open("foo", "r") as fobj:
            assert fobj.read() == "foo"

def test_isdir_isfile(tmp_dir, dvc):
    tmp_dir.gen({"datafile": "data", "datadir": {"foo": "foo", "bar": "bar"}})

    tree = RepoTree(dvc)
    assert tree.isdir("datadir")
    assert not tree.isfile("datadir")
    assert not tree.isdvc("datadir")
    assert not tree.isdir("datafile")
    assert tree.isfile("datafile")
    assert not tree.isdvc("datafile")

    dvc.add(["datadir", "datafile"])
    shutil.rmtree(tmp_dir / "datadir")
    (tmp_dir / "datafile").unlink()

    assert tree.isdir("datadir")
    assert not tree.isfile("datadir")
    assert tree.isdvc("datadir")
    assert not tree.isdir("datafile")
    assert tree.isfile("datafile")
    assert tree.isdvc("datafile")

def test_isdir_mixed(tmp_dir, dvc):
    tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}})

    dvc.add(str(tmp_dir / "dir" / "foo"))

    tree = RepoTree(dvc)
    assert tree.isdir("dir")
    assert not tree.isfile("dir")

@pytest.mark.parametrize(
    "dvcfiles,extra_expected",
    [
        (False, []),
        (
            True,
            [
                PathInfo("dir") / "subdir1" / "foo1.dvc",
                PathInfo("dir") / "subdir1" / "bar1.dvc",
                PathInfo("dir") / "subdir2" / "foo2.dvc",
            ],
        ),
    ],
)
def test_walk(tmp_dir, dvc, dvcfiles, extra_expected):
    tmp_dir.gen(
        {
            "dir": {
                "subdir1": {"foo1": "foo1", "bar1": "bar1"},
                "subdir2": {"foo2": "foo2"},
            }
        }
    )
    dvc.add(str(tmp_dir / "dir"), recursive=True)
    tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}})
    tree = RepoTree(dvc)

    expected = [
        PathInfo("dir") / "subdir1",
        PathInfo("dir") / "subdir2",
        PathInfo("dir") / "subdir1" / "foo1",
        PathInfo("dir") / "subdir1" / "bar1",
        PathInfo("dir") / "subdir2" / "foo2",
        PathInfo("dir") / "foo",
        PathInfo("dir") / "bar",
    ]

    actual = []
    for root, dirs, files in tree.walk("dir", dvcfiles=dvcfiles):
        for entry in dirs + files:
            actual.append(os.path.join(root, entry))

    expected = [str(path) for path in expected + extra_expected]
    assert set(actual) == set(expected)
    assert len(actual) == len(expected)

def test_walk_onerror(tmp_dir, dvc):
    def onerror(exc):
        raise exc

    tmp_dir.dvc_gen("foo", "foo")
    tree = RepoTree(dvc)

    # path does not exist
    for _ in tree.walk("dir"):
        pass
    with pytest.raises(OSError):
        for _ in tree.walk("dir", onerror=onerror):
            pass

    # path is not a directory
    for _ in tree.walk("foo"):
        pass
    with pytest.raises(OSError):
        for _ in tree.walk("foo", onerror=onerror):
            pass

def test_isdvc(tmp_dir, dvc):
    tmp_dir.gen({"foo": "foo", "bar": "bar", "dir": {"baz": "baz"}})
    dvc.add("foo")
    dvc.add("dir")
    tree = RepoTree(dvc)
    assert tree.isdvc("foo")
    assert not tree.isdvc("bar")
    assert tree.isdvc("dir")
    assert not tree.isdvc("dir/baz")
    assert tree.isdvc("dir/baz", recursive=True, strict=False)

def make_subrepo(dir_, scm, config=None):
    dir_.mkdir(parents=True)
    with dir_.chdir():
        dir_.scm = scm
        dir_.init(dvc=True, subdir=True)
        if config:
            dir_.add_remote(config=config)

def test_subrepos(tmp_dir, scm, dvc):
    tmp_dir.scm_gen(
        {"dir": {"repo.txt": "file to confuse RepoTree"}},
        commit="dir/repo.txt",
    )

    subrepo1 = tmp_dir / "dir" / "repo"
    subrepo2 = tmp_dir / "dir" / "repo2"

    for repo in [subrepo1, subrepo2]:
        make_subrepo(repo, scm)

    subrepo1.dvc_gen({"foo": "foo", "dir1": {"bar": "bar"}}, commit="FOO")
    subrepo2.dvc_gen(
        {"lorem": "lorem", "dir2": {"ipsum": "ipsum"}}, commit="BAR"
    )

    dvc.tree._reset()
    tree = RepoTree(dvc, subrepos=True, fetch=True)

    def assert_tree_belongs_to_repo(ret_val):
        method = tree._get_repo

        def f(*args, **kwargs):
            r = method(*args, **kwargs)
            assert r.root_dir == ret_val.root_dir
            return r

        return f

    with mock.patch.object(
        tree,
        "_get_repo",
        side_effect=assert_tree_belongs_to_repo(subrepo1.dvc),
    ):
        assert tree.exists(subrepo1 / "foo") is True
        assert tree.exists(subrepo1 / "bar") is False

        assert tree.isfile(subrepo1 / "foo") is True
        assert tree.isfile(subrepo1 / "dir1" / "bar") is True
        assert tree.isfile(subrepo1 / "dir1") is False

        assert tree.isdir(subrepo1 / "dir1") is True
        assert tree.isdir(subrepo1 / "dir1" / "bar") is False
        assert tree.isdvc(subrepo1 / "foo") is True

    with mock.patch.object(
        tree,
        "_get_repo",
        side_effect=assert_tree_belongs_to_repo(subrepo2.dvc),
    ):
        assert tree.exists(subrepo2 / "lorem") is True
        assert tree.exists(subrepo2 / "ipsum") is False

        assert tree.isfile(subrepo2 / "lorem") is True
        assert tree.isfile(subrepo2 / "dir2" / "ipsum") is True
        assert tree.isfile(subrepo2 / "dir2") is False

        assert tree.isdir(subrepo2 / "dir2") is True
        assert tree.isdir(subrepo2 / "dir2" / "ipsum") is False
        assert tree.isdvc(subrepo2 / "lorem") is True

@pytest.mark.parametrize(
    "dvcfiles,extra_expected",
    [
        (False, []),
        (
            True,
            [
                PathInfo("dir") / "repo" / "foo.dvc",
                PathInfo("dir") / "repo" / "dir1.dvc",
                PathInfo("dir") / "repo2" / "lorem.dvc",
                PathInfo("dir") / "repo2" / "dir2.dvc",
            ],
        ),
    ],
)
def test_subrepo_walk(tmp_dir, scm, dvc, dvcfiles, extra_expected):
    tmp_dir.scm_gen(
        {"dir": {"repo.txt": "file to confuse RepoTree"}},
        commit="dir/repo.txt",
    )

    subrepo1 = tmp_dir / "dir" / "repo"
    subrepo2 = tmp_dir / "dir" / "repo2"

    subdirs = [subrepo1, subrepo2]
    for dir_ in subdirs:
        make_subrepo(dir_, scm)

    subrepo1.dvc_gen({"foo": "foo", "dir1": {"bar": "bar"}}, commit="FOO")
    subrepo2.dvc_gen(
        {"lorem": "lorem", "dir2": {"ipsum": "ipsum"}}, commit="BAR"
    )

    # using tree that does not have dvcignore
    dvc.tree._reset()
    tree = RepoTree(dvc, subrepos=True, fetch=True)
    expected = [
        PathInfo("dir") / "repo",
        PathInfo("dir") / "repo.txt",
        PathInfo("dir") / "repo2",
        PathInfo("dir") / "repo" / ".dvcignore",
        PathInfo("dir") / "repo" / ".gitignore",
        PathInfo("dir") / "repo" / "foo",
        PathInfo("dir") / "repo" / "dir1",
        PathInfo("dir") / "repo" / "dir1" / "bar",
        PathInfo("dir") / "repo2" / ".dvcignore",
        PathInfo("dir") / "repo2" / ".gitignore",
        PathInfo("dir") / "repo2" / "lorem",
        PathInfo("dir") / "repo2" / "dir2",
        PathInfo("dir") / "repo2" / "dir2" / "ipsum",
    ]

    actual = []
    for root, dirs, files in tree.walk("dir", dvcfiles=dvcfiles):
        for entry in dirs + files:
            actual.append(os.path.join(root, entry))

    expected = [str(path) for path in expected + extra_expected]
    assert set(actual) == set(expected)
    assert len(actual) == len(expected)

def test_repo_tree_no_subrepos(tmp_dir, dvc, scm):
    tmp_dir.scm_gen(
        {"dir": {"repo.txt": "file to confuse RepoTree"}},
        commit="dir/repo.txt",
    )
    tmp_dir.dvc_gen({"lorem": "lorem"}, commit="add foo")

    subrepo = tmp_dir / "dir" / "repo"
    make_subrepo(subrepo, scm)
    subrepo.dvc_gen({"foo": "foo", "dir1": {"bar": "bar"}}, commit="FOO")
    subrepo.scm_gen({"ipsum": "ipsum"}, commit="BAR")

    # using tree that does not have dvcignore
    dvc.tree._reset()
    tree = RepoTree(dvc, subrepos=False, fetch=True)
    expected = [
        tmp_dir / ".dvcignore",
        tmp_dir / ".gitignore",
        tmp_dir / "lorem",
        tmp_dir / "lorem.dvc",
        tmp_dir / "dir",
        tmp_dir / "dir" / "repo.txt",
    ]

    actual = []
    for root, dirs, files in tree.walk(tmp_dir, dvcfiles=True):
        for entry in dirs + files:
            actual.append(os.path.normpath(os.path.join(root, entry)))

    expected = [str(path) for path in expected]
    assert set(actual) == set(expected)
    assert len(actual) == len(expected)

    assert tree.isfile(tmp_dir / "lorem") is True
    assert tree.isfile(tmp_dir / "dir" / "repo" / "foo") is False
    assert tree.isdir(tmp_dir / "dir" / "repo") is False
    assert tree.isdir(tmp_dir / "dir") is True

    assert tree.isdvc(tmp_dir / "lorem") is True
    assert tree.isdvc(tmp_dir / "dir" / "repo" / "dir1") is False

    assert tree.exists(tmp_dir / "dir" / "repo.txt") is True
    assert tree.exists(tmp_dir / "repo" / "ipsum") is False
<|endoftext|>


<commit_before>def test_ProxyToVim_instantiation():
    from vimpdb.proxy import ProxyToVim
    to_vim = ProxyToVim()
    assert isinstance(to_vim, ProxyToVim)

def test_ProxyToVim_setupRemote():
    from vimpdb.testing import ProxyToVimForTests
    to_vim = ProxyToVimForTests()
    to_vim.setState(to_vim.IS_REMOTE_SETUP_IS_FALSE)
    to_vim.setupRemote()
    lines = to_vim.logged().splitlines()
    assert len(lines) == 4
    assert lines[1] == "expr: exists('*PDB_init')"
    assert lines[2] == "return: '0'"
    assert lines[3].startswith('send: <C-\\><C-N>:source ')
    assert lines[3].endswith('vimpdb/vimpdb.vim<CR>')

def test_ProxyToVim_setupRemote_does_nothing():
    from vimpdb.testing import ProxyToVimForTests
    to_vim = ProxyToVimForTests()
    to_vim.setState(to_vim.IS_REMOTE_SETUP_IS_TRUE)
    to_vim.setupRemote()
    assert (to_vim.logged() ==
"""
expr: exists('*PDB_init')
return: '1'
""")

def test_ProxyToVim_isRemoteSetup():
    from vimpdb.testing import ProxyToVimForTests
    to_vim = ProxyToVimForTests()
    to_vim.isRemoteSetup()
    assert (to_vim.logged() ==
"""
expr: exists('*PDB_init')
return: None
""")

def test_ProxyFromVim_instantiation():
    from vimpdb.proxy import ProxyFromVim
    from_vim = ProxyFromVim()
    assert isinstance(from_vim, ProxyFromVim)
<commit_msg>test showFeedback<commit_after>def test_ProxyToVim_instantiation():
    from vimpdb.proxy import ProxyToVim
    to_vim = ProxyToVim()
    assert isinstance(to_vim, ProxyToVim)

def test_ProxyToVim_setupRemote():
    from vimpdb.testing import ProxyToVimForTests
    to_vim = ProxyToVimForTests()
    to_vim.setState(to_vim.IS_REMOTE_SETUP_IS_FALSE)
    to_vim.setupRemote()
    lines = to_vim.logged().splitlines()
    assert len(lines) == 4
    assert lines[1] == "expr: exists('*PDB_init')"
    assert lines[2] == "return: '0'"
    assert lines[3].startswith('send: <C-\\><C-N>:source ')
    assert lines[3].endswith('vimpdb/vimpdb.vim<CR>')

def test_ProxyToVim_setupRemote_does_nothing():
    from vimpdb.testing import ProxyToVimForTests
    to_vim = ProxyToVimForTests()
    to_vim.setState(to_vim.IS_REMOTE_SETUP_IS_TRUE)
    to_vim.setupRemote()
    assert (to_vim.logged() ==
"""
expr: exists('*PDB_init')
return: '1'
""")

def test_ProxyToVim_isRemoteSetup():
    from vimpdb.testing import ProxyToVimForTests
    to_vim = ProxyToVimForTests()
    to_vim.isRemoteSetup()
    assert (to_vim.logged() ==
"""
expr: exists('*PDB_init')
return: None
""")

def test_ProxyToVim_showFeedback_empty():
    from vimpdb.testing import ProxyToVimForTests
    to_vim = ProxyToVimForTests()
    to_vim.setState(to_vim.IS_REMOTE_SETUP_IS_TRUE)
    to_vim.showFeedback('')
    assert (to_vim.logged() ==
"""
""")

def test_ProxyToVim_showFeedback_content():
    from vimpdb.testing import ProxyToVimForTests
    to_vim = ProxyToVimForTests()
    to_vim.setState(to_vim.IS_REMOTE_SETUP_IS_TRUE)
    to_vim.showFeedback('first\nsecond')
    assert (to_vim.logged() ==
"""
expr: exists('*PDB_init')
return: '1'
send: :call PDB_show_feedback(['first', 'second'])<CR>
""")

def test_ProxyFromVim_instantiation():
    from vimpdb.proxy import ProxyFromVim
    from_vim = ProxyFromVim()
    assert isinstance(from_vim, ProxyFromVim)
<|endoftext|>


<commit_before># This file is part of BenchExec, a framework for reliable benchmarking:
# https://github.com/sosy-lab/benchexec
#
# SPDX-FileCopyrightText: 2007-2020 Dirk Beyer <https://www.sosy-lab.org>
# SPDX-FileCopyrightText: 2016-2019 Marek Chalupa
#
# SPDX-License-Identifier: Apache-2.0

import benchexec.result as result
from benchexec.tools.template import ToolNotFoundException

from .symbiotic4 import Tool as OldSymbiotic

class Tool(OldSymbiotic):
    """
    Symbiotic tool info object
    """

    REQUIRED_PATHS_4_0_1 = ["bin", "include", "lib", "lib32", "llvm-3.8.1"]

    REQUIRED_PATHS_5_0_0 = ["bin", "include", "lib", "lib32", "llvm-3.9.1"]

    REQUIRED_PATHS_6_0_0 = ["bin", "include", "properties", "lib", "llvm-4.0.1"]

    REQUIRED_PATHS_7_0_0 = ["bin", "include", "properties", "lib", "llvm-8.0.1"]

    def executable(self, tool_locator):
        """
        Find the path to the executable file that will get executed.
        This method always needs to be overridden,
        and most implementations will look similar to this one.
        The path returned should be relative to the current directory.
        """
        try:
            executable = tool_locator.find_executable("symbiotic", subdir="bin")
        except ToolNotFoundException:
            # this may be the old version of Symbiotic
            executable = OldSymbiotic.executable(self, tool_locator)

        self._version = self.version(executable)
        return executable

    def program_files(self, executable):
        if self._version_newer_than("7.0.0"):
            paths = self.REQUIRED_PATHS_7_0_0
        elif self._version_newer_than("6.0.0"):
            paths = self.REQUIRED_PATHS_6_0_0
        elif self._version_newer_than("5.0.0"):
            paths = self.REQUIRED_PATHS_5_0_0
        elif self._version_newer_than("4.0.1"):
            paths = self.REQUIRED_PATHS_4_0_1
        else:
            paths = OldSymbiotic.REQUIRED_PATHS

        return [executable] + self._program_files_from_executable(
            executable, paths, parent_dir=True
        )

    def _version_newer_than(self, vers):
        """
        Determine whether the version is greater than some given version
        """
        vers_num = self._version[: self._version.index("-")]
        if not vers_num[0].isdigit():
            # this is the old version which is "older" than any given version
            return False

        v1 = list(map(int, vers_num.split(".")))
        v2 = list(map(int, vers.split(".")))
        assert len(v1) == 3
        assert len(v2) == 3

        if v1[0] > v2[0]:
            return True
        elif v1[0] == v2[0]:
            if v1[1] == v2[1]:
                return v1[2] >= v2[2]
            elif v1[1] > v2[1]:
                return True

        return False

    def _getPhase(self, output):
        lastphase = "before-instr"
        for line in output:
            if line.startswith("INFO: Starting instrumentation"):
                lastphase = "instrumentation"
            elif line.startswith("INFO: Instrumentation time"):
                lastphase = "instr-finished"
            elif line.startswith("INFO: Starting slicing"):
                lastphase = "slicing"
            elif line.startswith("INFO: Total slicing time"):
                lastphase = "slicing-finished"
            elif line.startswith("INFO: Starting verification"):
                lastphase = "verification"
            elif line.startswith("INFO: Verification time"):
                lastphase = "verification-finished"
            elif line.startswith("INFO: Replaying error path"):
                lastphase = "cex-confirmation"
            elif line.startswith("INFO: Replaying error path time"):
                lastphase = "cex-confirmation-finished"

        return lastphase

    def determine_result(self, run):
        if not run.output:
            return "{0}(no output)".format(result.RESULT_ERROR)

        if self._version_newer_than("4.0.1"):
            for line in run.output:
                line = line.strip()
                if line == "RESULT: true":
                    return result.RESULT_TRUE_PROP
                elif line == "RESULT: unknown":
                    return result.RESULT_UNKNOWN
                elif line == "RESULT: done":
                    return result.RESULT_DONE
                elif line.startswith("RESULT: false(valid-deref)"):
                    return result.RESULT_FALSE_DEREF
                elif line.startswith("RESULT: false(valid-free)"):
                    return result.RESULT_FALSE_FREE
                elif line.startswith("RESULT: false(valid-memtrack)"):
                    return result.RESULT_FALSE_MEMTRACK
                elif line.startswith("RESULT: false(valid-memcleanup)"):
                    return result.RESULT_FALSE_MEMCLEANUP
                elif line.startswith("RESULT: false(no-overflow)"):
                    return result.RESULT_FALSE_OVERFLOW
                elif line.startswith("RESULT: false(termination)"):
                    return result.RESULT_FALSE_TERMINATION
                elif line.startswith("RESULT: false"):
                    return result.RESULT_FALSE_REACH
        else:
            # old version of Symbiotic
            return OldSymbiotic.determine_result(self, run)

        if run.was_timeout:
            return self._getPhase(run.output)  # generates TIMEOUT(phase)
        elif run.exit_code.signal:
            return "KILLED (signal {0}, {1})".format(
                run.exit_code.signal, self._getPhase(run.output)
            )
        elif run.exit_code.value != 0:
            return "{0}(returned {1}, {2})".format(
                result.RESULT_ERROR, run.exit_code.value, self._getPhase(run.output)
            )

        return "{0}(unknown, {1})".format(
            result.RESULT_ERROR, self._getPhase(run.output)
        )
<commit_msg>symbiotic: fix program_files() method<commit_after># This file is part of BenchExec, a framework for reliable benchmarking:
# https://github.com/sosy-lab/benchexec
#
# SPDX-FileCopyrightText: 2007-2020 Dirk Beyer <https://www.sosy-lab.org>
# SPDX-FileCopyrightText: 2016-2020 Marek Chalupa
#
# SPDX-License-Identifier: Apache-2.0

import benchexec.result as result
from benchexec.tools.template import ToolNotFoundException

from .symbiotic4 import Tool as OldSymbiotic

class Tool(OldSymbiotic):
    """
    Symbiotic tool info object
    """

    REQUIRED_PATHS_4_0_1 = ["bin", "include", "lib", "lib32", "llvm-3.8.1"]

    REQUIRED_PATHS_5_0_0 = ["bin", "include", "lib", "lib32", "llvm-3.9.1"]

    REQUIRED_PATHS_NEW = ["bin", "include", "properties", "lib", "llvm-*"]

    def executable(self, tool_locator):
        """
        Find the path to the executable file that will get executed.
        This method always needs to be overridden,
        and most implementations will look similar to this one.
        The path returned should be relative to the current directory.
        """
        try:
            executable = tool_locator.find_executable("symbiotic", subdir="bin")
        except ToolNotFoundException:
            # this may be the old version of Symbiotic
            executable = OldSymbiotic.executable(self, tool_locator)

        self._version = self.version(executable)
        return executable

    def program_files(self, executable):
        if self._version_newer_than("6.0.0"):
            paths = self.REQUIRED_PATHS_NEW
        elif self._version_newer_than("5.0.0"):
            paths = self.REQUIRED_PATHS_5_0_0
        elif self._version_newer_than("4.0.1"):
            paths = self.REQUIRED_PATHS_4_0_1
        else:
            paths = OldSymbiotic.REQUIRED_PATHS

        return [executable] + self._program_files_from_executable(
            executable, paths, parent_dir=True
        )

    def _version_newer_than(self, vers):
        """
        Determine whether the version is greater than some given version
        """
        vers_num = self._version[: self._version.index("-")]
        if not vers_num[0].isdigit():
            # this is the old version which is "older" than any given version
            return False

        v1 = list(map(int, vers_num.split(".")))
        v2 = list(map(int, vers.split(".")))
        assert len(v1) == 3
        assert len(v2) == 3

        if v1[0] > v2[0]:
            return True
        elif v1[0] == v2[0]:
            if v1[1] == v2[1]:
                return v1[2] >= v2[2]
            elif v1[1] > v2[1]:
                return True

        return False

    def _getPhase(self, output):
        lastphase = "before-instr"
        for line in output:
            if line.startswith("INFO: Starting instrumentation"):
                lastphase = "instrumentation"
            elif line.startswith("INFO: Instrumentation time"):
                lastphase = "instr-finished"
            elif line.startswith("INFO: Starting slicing"):
                lastphase = "slicing"
            elif line.startswith("INFO: Total slicing time"):
                lastphase = "slicing-finished"
            elif line.startswith("INFO: Starting verification"):
                lastphase = "verification"
            elif line.startswith("INFO: Verification time"):
                lastphase = "verification-finished"
            elif line.startswith("INFO: Replaying error path"):
                lastphase = "cex-confirmation"
            elif line.startswith("INFO: Replaying error path time"):
                lastphase = "cex-confirmation-finished"

        return lastphase

    def determine_result(self, run):
        if not run.output:
            return "{0}(no output)".format(result.RESULT_ERROR)

        if self._version_newer_than("4.0.1"):
            for line in run.output:
                line = line.strip()
                if line == "RESULT: true":
                    return result.RESULT_TRUE_PROP
                elif line == "RESULT: unknown":
                    return result.RESULT_UNKNOWN
                elif line == "RESULT: done":
                    return result.RESULT_DONE
                elif line.startswith("RESULT: false(valid-deref)"):
                    return result.RESULT_FALSE_DEREF
                elif line.startswith("RESULT: false(valid-free)"):
                    return result.RESULT_FALSE_FREE
                elif line.startswith("RESULT: false(valid-memtrack)"):
                    return result.RESULT_FALSE_MEMTRACK
                elif line.startswith("RESULT: false(valid-memcleanup)"):
                    return result.RESULT_FALSE_MEMCLEANUP
                elif line.startswith("RESULT: false(no-overflow)"):
                    return result.RESULT_FALSE_OVERFLOW
                elif line.startswith("RESULT: false(termination)"):
                    return result.RESULT_FALSE_TERMINATION
                elif line.startswith("RESULT: false"):
                    return result.RESULT_FALSE_REACH
        else:
            # old version of Symbiotic
            return OldSymbiotic.determine_result(self, run)

        if run.was_timeout:
            return self._getPhase(run.output)  # generates TIMEOUT(phase)
        elif run.exit_code.signal:
            return "KILLED (signal {0}, {1})".format(
                run.exit_code.signal, self._getPhase(run.output)
            )
        elif run.exit_code.value != 0:
            return "{0}(returned {1}, {2})".format(
                result.RESULT_ERROR, run.exit_code.value, self._getPhase(run.output)
            )

        return "{0}(unknown, {1})".format(
            result.RESULT_ERROR, self._getPhase(run.output)
        )
<|endoftext|>