<commit_before>#
# Author    : Manuel Bernal Llinares
# Project   : trackhub-creator
# Timestamp : 07-09-2017 11:24
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
This pipeline creates a trackhub for a PRIDE project, based on the information provided via a JSON formatted file, as it
can be seen on this sample:
{
  "trackHubName" : "PXD000625",
  "trackHubShortLabel" : "PXD000625 - Hepatoc...",
  "trackHubLongLabel" : "Experimental design For the label-free ...",
  "trackHubType" : "PROTEOMICS",
  "trackHubEmail" : "pride-support@ebi.ac.uk",
  "trackHubInternalAbsolutePath" : "...",
  "trackMaps" : [ {
    "trackName" : "PXD000625_10090_Original",
    "trackShortLabel" : "PXD000625 - Mus musc...",
    "trackLongLabel" : "Experimental design For the label-free proteome analysis 17 mice were used composed of 5 ...",
    "trackSpecie" : "10090",
    "pogoFile" : "..."
  } ]
}
"""

import os
import time
# App imports
import config_manager
import ensembl.service
import toolbox.general as general_toolbox
from pipelines.template_pipeline import Director, DirectorConfigurationManager

# Globals
__configuration_file = None
__pipeline_arguments = None
__pipeline_director = None

# Pipeline properties access
def set_configuration_file(config_file):
    global __configuration_file
    if __configuration_file is None:
        __configuration_file = config_file
    return __configuration_file

def set_pipeline_arguments(pipeline_arguments):
    global __pipeline_arguments
    if __pipeline_arguments is None:
        __pipeline_arguments = pipeline_arguments
    return __pipeline_arguments

def get_pipeline_director():
    global __pipeline_director
    if __pipeline_director is None:
        __pipeline_director = TrackhubCreatorForProject(config_manager.read_config_from_file(__configuration_file),
                                                        __configuration_file,
                                                        __pipeline_arguments)
    return __pipeline_director

class ConfigManager(DirectorConfigurationManager):
    # Command Line Arguments for this pipeline look like
    #   # This is a JSON formatted file that contains all the relevant information needed for processing the project
    #   # data and create its trackhub
    #   project_data_file=project_data.json

    # Command Line Argument keys
    _CONFIG_COMMAND_LINE_ARGUMENT_KEY_PROJECT_DATA_FILE = 'project_data_file'

    def __init__(self, configuration_object, configuration_file, pipeline_arguments):
        super(ConfigManager, self).__init__(configuration_object, configuration_file, pipeline_arguments)
        # Lazy Process command line arguments
        self.__pipeline_arguments_object = None
        self.__running_mode = None

    def get_project_data_file_path(self):
        return self._get_value_for_pipeline_argument_key(self._CONFIG_COMMAND_LINE_ARGUMENT_KEY_PROJECT_DATA_FILE)

# Models for dealing with the data file that describes the project
class ProjectTrackDescriptor:
    # Project Data File keys relative to every TrackMap object
    _PROJECT_DATA_FILE_KEY_TRACK_NAME = 'trackName'
    _PROJECT_DATA_FILE_KEY_TRACK_SHORT_LABEL = 'trackShortLabel'
    _PROJECT_DATA_FILE_KEY_TRACK_LONG_LABEL = 'trackLongLabel'
    _PROJECT_DATA_FILE_KEY_TRACK_SPECIES = 'trackSpecie'
    _PROJECT_DATA_FILE_KEY_TRACK_POGO_FILE_PATH = 'pogoFile'

    def __init__(self, project_track_descriptor_object):
        self.__project_track_descriptor_object = project_track_descriptor_object

    def _get_value_for_key(self, key, default=""):
        if self.__project_track_descriptor_object and (key in self.__project_track_descriptor_object):
            return self.__project_track_descriptor_object[key]
        return default

    def get_track_name(self):
        return self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACK_NAME)

    def get_track_short_label(self):
        return self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACK_SHORT_LABEL)

    def get_track_long_label(self):
        return self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACK_LONG_LABEL)

    def get_track_species(self):
        return self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACK_SPECIES)

    def get_track_file_path_pogo(self):
        return self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACK_POGO_FILE_PATH)

class ProjectTrackhubDescriptor:
    # Project Data File keys
    _PROJECT_DATA_FILE_KEY_TRACKHUB_NAME = 'trackHubName'
    _PROJECT_DATA_FILE_KEY_TRACKHUB_SHORT_LABEL = 'trackHubShortLabel'
    _PROJECT_DATA_FILE_KEY_TRACKHUB_LONG_LABEL = 'trackHubLongLabel'
    _PROJECT_DATA_FILE_KEY_TRACKHUB_HUB_TYPE = 'trackHubType'
    _PROJECT_DATA_FILE_KEY_TRACKHUB_EMAIL = 'trackHubEmail'
    _PROJECT_DATA_FILE_KEY_TRACKHUB_INTERNAL_ABSOLUTE_PATH = 'trackHubInternalAbsolutePath'
    _PROJECT_DATA_FILE_KEY_TRACKHUB_SECTION_TRACKMAPS = 'trackMaps'

    def __init__(self, project_data_file_path):
        self.__project_data_file_path = project_data_file_path
        self.__project_data_object = None
        self.__project_tracks_descriptors = None

    def _get_project_data_object(self):
        if not self.__project_data_object:
            self.__project_data_object = general_toolbox.read_json(self.__project_data_file_path)
        return self.__project_data_object

    def _get_value_for_key(self, key, default=""):
        # TODO - I should start thinking about refactoring this out
        if key in self._get_project_data_object():
            return self._get_project_data_object()[key]
        return default

    def get_trackhub_name(self):
        return self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACKHUB_NAME,
                                       os.path.basename(self.__project_data_file_path))

    def get_trackhub_short_label(self):
        return self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACKHUB_SHORT_LABEL,
                                       "--- NO SHORT LABEL HAS BEEN DEFINED FOR THIS TRACKHUB ---")

    def get_trackhub_long_label(self):
        return self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACKHUB_LONG_LABEL,
                                       "--- NO LONG LABEL HAS BEEN DEFINED FOR THIS TRACKHUB ---")

    def get_trackhub_hub_type(self):
        return self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACKHUB_HUB_TYPE,
                                       "PROTEOMICS")

    def get_trackhub_email(self):
        return self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACKHUB_EMAIL,
                                       "pride-support@ebi.ac.uk")

    def get_trackhub_destination_path(self):
        return self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACKHUB_INTERNAL_ABSOLUTE_PATH)

    def get_trackhub_project_defined_tracks(self):
        if not self.__project_tracks_descriptors:
            # Default value is an empty list of tracks
            self.__project_tracks_descriptors = []
            data_file_project_track_description_objects = \
                self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACKHUB_SECTION_TRACKMAPS)
            if data_file_project_track_description_objects:
                self.__project_tracks_descriptors = \
                    [ProjectTrackDescriptor(data_file_project_track_description_object)
                     for data_file_project_track_description_object
                     in data_file_project_track_description_objects]
        return self.__project_tracks_descriptors

class TrackhubCreatorForProject(Director):
    # TODO
    def __init__(self, configuration_object, configuration_file, pipeline_arguments):
        runner_id = "{}-{}".format(__name__, time.time())
        super(TrackhubCreatorForProject, self).__init__(runner_id)
        self.__config_manager = ConfigManager(configuration_object, configuration_file, pipeline_arguments)
        self.__project_trackhub_descriptor = None
        # Only the valid project tracks will be processed for being included in the trackhub
        self.__valid_project_tracks = []

    def __get_valid_project_tracks(self):
        pass

    def _before(self):
        if self.__config_manager.get_project_data_file_path():
            self._get_logger().info("Reading Project Trackhub Descriptor from file at '{}'"
                                    .format(self.__config_manager.get_project_data_file_path()))
            self.__project_trackhub_descriptor = \
                ProjectTrackhubDescriptor(self.__config_manager.get_project_data_file_path())
            # Check that the destination folder exists
            if not os.path.isdir(self.__project_trackhub_descriptor.get_trackhub_destination_path()):
                self._get_logger().error("Trackhub destination path NOT VALID, '{}'"
                                         .format(self.__project_trackhub_descriptor.get_trackhub_destination_path()))
                return False
            return True
        self._get_logger().error("INVALID / MISSING Project Trackhub Descriptor file, '{}'"
                                 .format(self.__config_manager.get_project_data_file_path()))
        return False

if __name__ == '__main__':
    print("ERROR: This script is part of a pipeline collection and it is not meant to be run in stand alone mode")
<commit_msg>refactor inheritance for TrackhubCreatorForProject<commit_after>#
# Author    : Manuel Bernal Llinares
# Project   : trackhub-creator
# Timestamp : 07-09-2017 11:24
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
This pipeline creates a trackhub for a PRIDE project, based on the information provided via a JSON formatted file, as it
can be seen on this sample:
{
  "trackHubName" : "PXD000625",
  "trackHubShortLabel" : "PXD000625 - Hepatoc...",
  "trackHubLongLabel" : "Experimental design For the label-free ...",
  "trackHubType" : "PROTEOMICS",
  "trackHubEmail" : "pride-support@ebi.ac.uk",
  "trackHubInternalAbsolutePath" : "...",
  "trackMaps" : [ {
    "trackName" : "PXD000625_10090_Original",
    "trackShortLabel" : "PXD000625 - Mus musc...",
    "trackLongLabel" : "Experimental design For the label-free proteome analysis 17 mice were used composed of 5 ...",
    "trackSpecie" : "10090",
    "pogoFile" : "..."
  } ]
}
"""

import os
import time
# App imports
import config_manager
import ensembl.service
import toolbox.general as general_toolbox
from pipelines.template_pipeline import PogoBasedPipelineDirector, DirectorConfigurationManager

# Globals
__configuration_file = None
__pipeline_arguments = None
__pipeline_director = None

# Pipeline properties access
def set_configuration_file(config_file):
    global __configuration_file
    if __configuration_file is None:
        __configuration_file = config_file
    return __configuration_file

def set_pipeline_arguments(pipeline_arguments):
    global __pipeline_arguments
    if __pipeline_arguments is None:
        __pipeline_arguments = pipeline_arguments
    return __pipeline_arguments

def get_pipeline_director():
    global __pipeline_director
    if __pipeline_director is None:
        __pipeline_director = TrackhubCreatorForProject(config_manager.read_config_from_file(__configuration_file),
                                                        __configuration_file,
                                                        __pipeline_arguments)
    return __pipeline_director

class ConfigManager(DirectorConfigurationManager):
    # Command Line Arguments for this pipeline look like
    #   # This is a JSON formatted file that contains all the relevant information needed for processing the project
    #   # data and create its trackhub
    #   project_data_file=project_data.json

    # Command Line Argument keys
    _CONFIG_COMMAND_LINE_ARGUMENT_KEY_PROJECT_DATA_FILE = 'project_data_file'

    def __init__(self, configuration_object, configuration_file, pipeline_arguments):
        super(ConfigManager, self).__init__(configuration_object, configuration_file, pipeline_arguments)
        # Lazy Process command line arguments
        self.__pipeline_arguments_object = None
        self.__running_mode = None

    def get_project_data_file_path(self):
        return self._get_value_for_pipeline_argument_key(self._CONFIG_COMMAND_LINE_ARGUMENT_KEY_PROJECT_DATA_FILE)

# Models for dealing with the data file that describes the project
class ProjectTrackDescriptor:
    # Project Data File keys relative to every TrackMap object
    _PROJECT_DATA_FILE_KEY_TRACK_NAME = 'trackName'
    _PROJECT_DATA_FILE_KEY_TRACK_SHORT_LABEL = 'trackShortLabel'
    _PROJECT_DATA_FILE_KEY_TRACK_LONG_LABEL = 'trackLongLabel'
    _PROJECT_DATA_FILE_KEY_TRACK_SPECIES = 'trackSpecie'
    _PROJECT_DATA_FILE_KEY_TRACK_POGO_FILE_PATH = 'pogoFile'

    def __init__(self, project_track_descriptor_object):
        self.__project_track_descriptor_object = project_track_descriptor_object

    def _get_value_for_key(self, key, default=""):
        if self.__project_track_descriptor_object and (key in self.__project_track_descriptor_object):
            return self.__project_track_descriptor_object[key]
        return default

    def get_track_name(self):
        return self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACK_NAME)

    def get_track_short_label(self):
        return self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACK_SHORT_LABEL)

    def get_track_long_label(self):
        return self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACK_LONG_LABEL)

    def get_track_species(self):
        return self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACK_SPECIES)

    def get_track_file_path_pogo(self):
        return self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACK_POGO_FILE_PATH)

class ProjectTrackhubDescriptor:
    # Project Data File keys
    _PROJECT_DATA_FILE_KEY_TRACKHUB_NAME = 'trackHubName'
    _PROJECT_DATA_FILE_KEY_TRACKHUB_SHORT_LABEL = 'trackHubShortLabel'
    _PROJECT_DATA_FILE_KEY_TRACKHUB_LONG_LABEL = 'trackHubLongLabel'
    _PROJECT_DATA_FILE_KEY_TRACKHUB_HUB_TYPE = 'trackHubType'
    _PROJECT_DATA_FILE_KEY_TRACKHUB_EMAIL = 'trackHubEmail'
    _PROJECT_DATA_FILE_KEY_TRACKHUB_INTERNAL_ABSOLUTE_PATH = 'trackHubInternalAbsolutePath'
    _PROJECT_DATA_FILE_KEY_TRACKHUB_SECTION_TRACKMAPS = 'trackMaps'

    def __init__(self, project_data_file_path):
        self.__project_data_file_path = project_data_file_path
        self.__project_data_object = None
        self.__project_tracks_descriptors = None

    def _get_project_data_object(self):
        if not self.__project_data_object:
            self.__project_data_object = general_toolbox.read_json(self.__project_data_file_path)
        return self.__project_data_object

    def _get_value_for_key(self, key, default=""):
        # TODO - I should start thinking about refactoring this out
        if key in self._get_project_data_object():
            return self._get_project_data_object()[key]
        return default

    def get_trackhub_name(self):
        return self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACKHUB_NAME,
                                       os.path.basename(self.__project_data_file_path))

    def get_trackhub_short_label(self):
        return self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACKHUB_SHORT_LABEL,
                                       "--- NO SHORT LABEL HAS BEEN DEFINED FOR THIS TRACKHUB ---")

    def get_trackhub_long_label(self):
        return self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACKHUB_LONG_LABEL,
                                       "--- NO LONG LABEL HAS BEEN DEFINED FOR THIS TRACKHUB ---")

    def get_trackhub_hub_type(self):
        return self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACKHUB_HUB_TYPE,
                                       "PROTEOMICS")

    def get_trackhub_email(self):
        return self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACKHUB_EMAIL,
                                       "pride-support@ebi.ac.uk")

    def get_trackhub_destination_path(self):
        return self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACKHUB_INTERNAL_ABSOLUTE_PATH)

    def get_trackhub_project_defined_tracks(self):
        if not self.__project_tracks_descriptors:
            # Default value is an empty list of tracks
            self.__project_tracks_descriptors = []
            data_file_project_track_description_objects = \
                self._get_value_for_key(self._PROJECT_DATA_FILE_KEY_TRACKHUB_SECTION_TRACKMAPS)
            if data_file_project_track_description_objects:
                self.__project_tracks_descriptors = \
                    [ProjectTrackDescriptor(data_file_project_track_description_object)
                     for data_file_project_track_description_object
                     in data_file_project_track_description_objects]
        return self.__project_tracks_descriptors

class TrackhubCreatorForProject(PogoBasedPipelineDirector):
    # TODO
    def __init__(self, configuration_object, configuration_file, pipeline_arguments):
        runner_id = "{}-{}".format(__name__, time.time())
        super(TrackhubCreatorForProject, self).__init__(runner_id)
        self.__config_manager = ConfigManager(configuration_object, configuration_file, pipeline_arguments)
        self.__project_trackhub_descriptor = None
        # Only the valid project tracks will be processed for being included in the trackhub
        self.__valid_project_tracks = []

    def __get_valid_project_tracks(self):
        pass

    def _before(self):
        if self.__config_manager.get_project_data_file_path():
            self._get_logger().info("Reading Project Trackhub Descriptor from file at '{}'"
                                    .format(self.__config_manager.get_project_data_file_path()))
            self.__project_trackhub_descriptor = \
                ProjectTrackhubDescriptor(self.__config_manager.get_project_data_file_path())
            # Check that the destination folder exists
            if not os.path.isdir(self.__project_trackhub_descriptor.get_trackhub_destination_path()):
                self._get_logger().error("Trackhub destination path NOT VALID, '{}'"
                                         .format(self.__project_trackhub_descriptor.get_trackhub_destination_path()))
                return False
            return True
        self._get_logger().error("INVALID / MISSING Project Trackhub Descriptor file, '{}'"
                                 .format(self.__config_manager.get_project_data_file_path()))
        return False

if __name__ == '__main__':
    print("ERROR: This script is part of a pipeline collection and it is not meant to be run in stand alone mode")
<|endoftext|>


<commit_before>from functools import wraps

from django.utils import timezone
from rest_framework import exceptions, status, viewsets

from core import exceptions as core_exceptions
from core.models import IdentityMembership
from core.models.status_type import StatusType

def unresolved_requests_only(fn):
    """
    Only allow an unresolved request to be processed.
    """
    @wraps(fn)
    def wrapper(self, request, *args, **kwargs):
        pending_request = self.get_object()
        if (hasattr(pending_request, "is_closed")
            and pending_request.is_closed()):
            message = (
                "Method '%s' not allowed: "
                "the request has already been resolved."
                % self.request.method
            )
            raise exceptions.MethodNotAllowed(self.request.method,
                                              detail=message)
        else:
            return fn(self, request, *args, **kwargs)
    return wrapper

class BaseRequestViewSet(viewsets.ModelViewSet):
    """
    Base class ViewSet to handle requests
    """
    admin_serializer_class = None

    model = None

    def get_queryset(self):
        """
        Return users requests or all the requests if the user is an admin.
        """
        assert self.model is not None, (
            "%s should include a `model` attribute."
            % self.__class__.__name__
        )
        if self.request.user.is_staff:
            return self.model.objects.all()
        return self.model.objects.filter(created_by=self.request.user)

    def get_serializer_class(self):
        """
        Return the `serializer_class` or `admin_serializer_class`
        given the users privileges.
        """
        assert self.admin_serializer_class is not None, (
            "%s should include an `admin_serializer_class` attribute."
            % self.__class__.__name__
        )
        if self.request.user.is_staff:
            return self.admin_serializer_class
        return self.serializer_class

    def perform_create(self, serializer):
        # NOTE: An identity could possible have multiple memberships
        # It may be better to directly take membership rather than an identity
        identity_id = serializer.initial_data("identity")
        status, _ = StatusType.objects.get_or_create(name="pending")
        try:
            membership = IdentityMembership.objects.get(identity=identity_id)
            instance  = serializer.save(
                membership=membership,
                status=status,
                created_by=self.request.user
            )
            self.submit_action(instance)
        except core_exceptions.ProviderLimitExceeded:
            message = "Only one active request is allowed per provider."
            raise exceptions.MethodNotAllowed('create', detail=message)
        except core_exceptions.InvalidMembership:
            message = (
                "The user '%s' is not a valid member."
                % self.request.user.username
            )
            raise exceptions.ParseError(detail=message)
        except IdentityMembership.DoesNotExist:
            message = (
                "The identity '%s' does not have a membership"
                % identity_id
            )
            raise exceptions.ParseError(detail=message)

    @unresolved_requests_only
    def destroy(self, request, *args, **kwargs):
        instance = self.get_object()
        self.perform_destroy(instance)
        return Response(status=status.HTTP_204_NO_CONTENT)

    def perform_update(self, serializer):
        """
        Updates the request and performs any update actions.
        """
        # NOTE: An identity could possible have multiple memberships
        # It may be better to directly take membership rather than an identity
        identity_id = serializer.initial_data.get('identity').get("id", None)
        try:
            membership = IdentityMembership.objects.get(identity=identity_id)
            instance = serializer.save(end_date=timezone.now(),
                                       membership=membership)
        except core_exceptions.ProviderLimitExceeded:
            message = "Only one active request is allowed per provider."
            raise exceptions.MethodNotAllowed('create', detail=message)
        except core_exceptions.InvalidMembership:
            message = (
                "The user '%s' is not a valid member."
                % self.request.user.username
            )
            raise exceptions.ParseError(detail=message)
        except IdentityMembership.DoesNotExist:
            message = (
                "The identity '%s' does not have a membership"
                % identity_id
            )
            raise exceptions.ParseError(detail=message)

        if instance.is_approved():
            self.approve_action(instance)

        if instance.is_denied():
            self.deny_action(instance)

    @unresolved_requests_only
    def update(self, request, *args, **kwargs):
        """
        Update the request for the specific identifier
        """
        return super(BaseRequestViewSet, self).update(request, *args, **kwargs)

    def approve_action(self, instance):
        """
        Perform the approved action for the request
        """

    def deny_action(self, instance):
        """
        Perform the denied action for the request
        """

    def submit_action(self, instance):
        """
        Perform the submit action for a new request
        """
<commit_msg>change variable name for consistency<commit_after>from functools import wraps

from django.utils import timezone
from rest_framework import exceptions, status, viewsets

from core import exceptions as core_exceptions
from core.models import IdentityMembership
from core.models.status_type import StatusType

def unresolved_requests_only(fn):
    """
    Only allow an unresolved request to be processed.
    """
    @wraps(fn)
    def wrapper(self, request, *args, **kwargs):
        instance = self.get_object()
        if (hasattr(instance, "is_closed")
            and instance.is_closed()):
            message = (
                "Method '%s' not allowed: "
                "the request has already been resolved."
                % self.request.method
            )
            raise exceptions.MethodNotAllowed(self.request.method,
                                              detail=message)
        else:
            return fn(self, request, *args, **kwargs)
    return wrapper

class BaseRequestViewSet(viewsets.ModelViewSet):
    """
    Base class ViewSet to handle requests
    """
    admin_serializer_class = None

    model = None

    def get_queryset(self):
        """
        Return users requests or all the requests if the user is an admin.
        """
        assert self.model is not None, (
            "%s should include a `model` attribute."
            % self.__class__.__name__
        )
        if self.request.user.is_staff:
            return self.model.objects.all()
        return self.model.objects.filter(created_by=self.request.user)

    def get_serializer_class(self):
        """
        Return the `serializer_class` or `admin_serializer_class`
        given the users privileges.
        """
        assert self.admin_serializer_class is not None, (
            "%s should include an `admin_serializer_class` attribute."
            % self.__class__.__name__
        )
        if self.request.user.is_staff:
            return self.admin_serializer_class
        return self.serializer_class

    def perform_create(self, serializer):
        # NOTE: An identity could possible have multiple memberships
        # It may be better to directly take membership rather than an identity
        identity_id = serializer.initial_data("identity")
        status, _ = StatusType.objects.get_or_create(name="pending")
        try:
            membership = IdentityMembership.objects.get(identity=identity_id)
            instance  = serializer.save(
                membership=membership,
                status=status,
                created_by=self.request.user
            )
            self.submit_action(instance)
        except core_exceptions.ProviderLimitExceeded:
            message = "Only one active request is allowed per provider."
            raise exceptions.MethodNotAllowed('create', detail=message)
        except core_exceptions.InvalidMembership:
            message = (
                "The user '%s' is not a valid member."
                % self.request.user.username
            )
            raise exceptions.ParseError(detail=message)
        except IdentityMembership.DoesNotExist:
            message = (
                "The identity '%s' does not have a membership"
                % identity_id
            )
            raise exceptions.ParseError(detail=message)

    @unresolved_requests_only
    def destroy(self, request, *args, **kwargs):
        instance = self.get_object()
        self.perform_destroy(instance)
        return Response(status=status.HTTP_204_NO_CONTENT)

    def perform_update(self, serializer):
        """
        Updates the request and performs any update actions.
        """
        # NOTE: An identity could possible have multiple memberships
        # It may be better to directly take membership rather than an identity
        identity_id = serializer.initial_data.get('identity').get("id", None)
        try:
            membership = IdentityMembership.objects.get(identity=identity_id)
            instance = serializer.save(end_date=timezone.now(),
                                       membership=membership)
        except core_exceptions.ProviderLimitExceeded:
            message = "Only one active request is allowed per provider."
            raise exceptions.MethodNotAllowed('create', detail=message)
        except core_exceptions.InvalidMembership:
            message = (
                "The user '%s' is not a valid member."
                % self.request.user.username
            )
            raise exceptions.ParseError(detail=message)
        except IdentityMembership.DoesNotExist:
            message = (
                "The identity '%s' does not have a membership"
                % identity_id
            )
            raise exceptions.ParseError(detail=message)

        if instance.is_approved():
            self.approve_action(instance)

        if instance.is_denied():
            self.deny_action(instance)

    @unresolved_requests_only
    def update(self, request, *args, **kwargs):
        """
        Update the request for the specific identifier
        """
        return super(BaseRequestViewSet, self).update(request, *args, **kwargs)

    def approve_action(self, instance):
        """
        Perform the approved action for the request
        """

    def deny_action(self, instance):
        """
        Perform the denied action for the request
        """

    def submit_action(self, instance):
        """
        Perform the submit action for a new request
        """
<|endoftext|>


<commit_before>"""
Read the best hit, get the taxa of that hit and report the number of [kingdom/phyla/genera/species] vs the number of proteins
"""

import os
import sys
import argparse
import taxon
import re

ffdir = 'flatfiles' # the location of our genbank flatfiles
blastpf = 'phage_genes.nr.blastp'

genome = {} # hash with protein IDs as key and genome as value
numprots = {} # hash with genome ID as key and number of proteins as value

taxondir = "/home2/db/taxonomy/current/"
sys.stderr.write("Reading taxonomy\n")
taxa = taxon.read_nodes(directory=taxondir)
names, blastname = taxon.read_names(directory=taxondir)
sys.stderr.write("Read taxonomy\n")

# first read how many proteins there are per genome
# and create a list of protein ids->genomes.
# Note also throw a fatal error if IDs duplicated because I forgot to check this earlier!
for f in os.listdir(ffdir):
    with open(os.path.join(ffdir, f), 'r') as fin:
        for l in fin:
            p=l.strip().split("\t")
            if p[5] in genome:
                sys.stderr.write("FATAL AND BUGGER: {} was found in {} and {}\n".format(p[5], genome[p[0]], p[0]))
            genome[p[5]] = p[0]
            numprots[p[0]] = numprots.get(p[0], 0) + 1

prottaxa = {} # hash of protein taxa
want = ['superkingdom', 'phylum', 'genus', 'species']
with open(blastpf, 'r') as fin:
    for l in fin:
        p = l.strip().split("\t")
        if p[0] in prottaxa:
            continue
        for tid in p[14].split(";"):
            level = {}

            while tid != '0' and tid != '1' and tid in taxa and taxa[tid].parent != '1':
                if taxa[tid].rank in want:
                    level[taxa[tid].rank] = names[tid].name
                tid = taxa[tid].parent
            # find the first entry that has all the taxonomic levels we want
            keep = True
            for w in want:
                if w not in level:
                    keep = False
            if keep:
                prottaxa[p[0]] = level
                break

hostlocation={}
with open('phage_host_location.txt', 'r') as fin:
    for l in fin:
        p=l.strip().split("\t")
        hostlocation[p[0]] = [p[2], p[3]]

hitcount = {}
genometaxa = {}
for p in prottaxa:
    if genome[p] not in genometaxa:
        genometaxa[genome[p]] = {}
        hitcount[genome[p]] = 0
        for w in want:
            genometaxa[genome[p]][w] = set()
    for w in prottaxa[p]:
        genometaxa[genome[p]][w].add(prottaxa[p][w])
    hitcount[genome[p]] += 1

for g in genometaxa:
    sys.stdout.write("{}\t{}\t{}\t".format(g, numprots[g], hitcount[g]))
    sys.stdout.write("\t".join(hostlocation[g]))
    for w in want:
        sys.stdout.write("\t{}".format(len(genometaxa[g][w])))
    sys.stdout.write("\n")

<commit_msg>Number of genomes<commit_after>"""
Read the best hit, get the taxa of that hit and report the number of [kingdom/phyla/genera/species] vs the number of proteins
"""

import os
import sys
import argparse
import taxon
import re

ffdir = 'flatfiles' # the location of our genbank flatfiles
blastpf = 'phage_genes.nr.blastp'

genome = {} # hash with protein IDs as key and genome as value
numprots = {} # hash with genome ID as key and number of proteins as value

taxondir = "/home2/db/taxonomy/current/"
sys.stderr.write("Reading taxonomy\n")
taxa = taxon.read_nodes(directory=taxondir)
names, blastname = taxon.read_names(directory=taxondir)
sys.stderr.write("Read taxonomy\n")

# first read how many proteins there are per genome
# and create a list of protein ids->genomes.
# Note also throw a fatal error if IDs duplicated because I forgot to check this earlier!
for f in os.listdir(ffdir):
    with open(os.path.join(ffdir, f), 'r') as fin:
        for l in fin:
            p=l.strip().split("\t")
            if p[5] in genome:
                sys.stderr.write("FATAL AND BUGGER: {} was found in {} and {}\n".format(p[5], genome[p[0]], p[0]))
            genome[p[5]] = p[0]
            numprots[p[0]] = numprots.get(p[0], 0) + 1

prottaxa = {} # hash of protein taxa
want = ['superkingdom', 'phylum', 'genus', 'species']
with open(blastpf, 'r') as fin:
    for l in fin:
        p = l.strip().split("\t")
        if p[0] in prottaxa:
            continue
        for tid in p[14].split(";"):
            level = {}

            while tid != '0' and tid != '1' and tid in taxa and taxa[tid].parent != '1':
                if taxa[tid].rank in want:
                    level[taxa[tid].rank] = names[tid].name
                tid = taxa[tid].parent
            # find the first entry that has all the taxonomic levels we want
            keep = True
            for w in want:
                if w not in level:
                    keep = False
            if keep:
                prottaxa[p[0]] = level
                break

hostlocation={}
with open('phage_host_location.txt', 'r') as fin:
    for l in fin:
        p=l.strip().split("\t")
        hostlocation[p[0]] = [p[2], p[3]]

hitcount = {}
genometaxa = {}
for p in prottaxa:
    if genome[p] not in genometaxa:
        genometaxa[genome[p]] = {}
        hitcount[genome[p]] = 0
        for w in want:
            genometaxa[genome[p]][w] = set()
    for w in prottaxa[p]:
        genometaxa[genome[p]][w].add(prottaxa[p][w])
    hitcount[genome[p]] += 1

for g in genometaxa:
    sys.stdout.write("{}\t{}\t{}\t".format(g, numprots[g], hitcount[g]))
    if g in hostlocation:
        sys.stdout.write("\t".join(hostlocation[g]))
    else:
        sys.stdout.write("-\t-\t")
    for w in want:
        sys.stdout.write("\t{}".format(len(genometaxa[g][w])))
    sys.stdout.write("\n")

<|endoftext|>


<commit_before>"""Run the same 'model' but with different ds types
and different osi types.
Ensure that:
LagrangianTIDS + MoreauJeanOSI,
LagrangianLinearDiagonalDS + MoreauJeanOSI,
LagrangianLinearDiagonalDS + MoreauJeanBilbaoOSI
"""

import siconos.kernel as sk
import numpy as np

def test_lagrangian_and_osis():
    """Tests osi/lagrangian combinations
    """
    # --- The dynamical systems ---
    # dimension
    ndof = 3
    # initial conditons
    q0 = np.zeros(ndof, dtype=np.float64)
    v0 = np.zeros_like(q0)
    q0[0] = 1.
    # mass, stiffness and damping matrices
    # (diagonal values only)
    mass_diag = np.ones(ndof, dtype=np.float64)
    k_diag = np.zeros_like(mass_diag)
    c_diag = np.zeros_like(mass_diag)
    sigma = 0.2
    omega2 = 1.2
    k_diag[...] = omega2
    c_diag[...] = sigma

    ds_list = {}
    # - LagrangianLinearTIDS -
    ds_list['LTIDS+MJ'] = sk.LagrangianLinearTIDS(q0, v0, np.diag(mass_diag),
                                                  np.diag(k_diag),
                                                  np.diag(c_diag))
    # - LagrangianLinearDiagonalDS -
    ds_list['LLDDS+MJ'] = sk.LagrangianLinearDiagonalDS(q0, v0, k_diag,
                                                        c_diag,
                                                        mass_diag)
    ds_list['LLDDS+MJB'] = sk.LagrangianLinearDiagonalDS(q0, v0, k_diag,
                                                         c_diag,
                                                         mass_diag)
    # no mass (i.e. implicitely equal to Id)
    ds_list['LLDDS+MJB2'] = sk.LagrangianLinearDiagonalDS(q0, v0,
                                                          k_diag, c_diag)

    nb_ds = len(ds_list)
    for ds in ds_list.values():
        ds.computeForces(0., q0, v0)

    assert np.allclose(ds_list['LTIDS+MJ'].forces(),
                       ds_list['LTIDS+MJ'].forces())

    # --- Interactions ---
    cor = 0.9

    nslaw = sk.NewtonImpactNSL(cor)
    hmat = np.zeros((1, ndof), dtype=np.float64)
    hmat[0, 0] = 1.
    relation = sk.LagrangianLinearTIR(hmat)

    interactions = []
    for k in range(nb_ds):
        interactions.append(sk.Interaction(nslaw, relation))

    # --- Model ---
    tinit = 0.
    tend = 3.
    model = sk.Model(tinit, tend)

    # - insert ds into the model and link them with their interaction -
    nsds = model.nonSmoothDynamicalSystem()
    ninter = 0
    for ds in ds_list.values():
        nsds.insertDynamicalSystem(ds)
        nsds.link(interactions[ninter], ds)
        ninter += 1

    # --- Simulation ---

    # - (1) OneStepIntegrators --
    theta = 0.5000001
    standard = sk.MoreauJeanOSI(theta)
    bilbao = sk.MoreauJeanBilbaoOSI()
    #-- (2) Time discretisation --
    time_step = 1e-4
    td = sk.TimeDiscretisation(tinit, time_step)
    # -- (3) one step non smooth problem
    lcp = sk.LCP()
    # -- (4) Simulation setup with (1) (2) (3)
    simu = sk.TimeStepping(td, standard, lcp)
    # extra osi must be explicitely inserted into simu and linked to ds
    simu.prepareIntegratorForDS(bilbao, ds_list['LLDDS+MJB'], model, tinit)
    simu.prepareIntegratorForDS(bilbao, ds_list['LLDDS+MJB2'], model, tinit)

    # link simu and model, initialize
    model.setSimulation(simu)
    model.initialize()

    positions = []
    velocities = []
    for ds in ds_list.values():
        positions.append(ds.q())
        velocities.append(ds.velocity())

    # number of time steps and buffer used to save results
    nb_steps = int((tend - tinit) / time_step) + 1
    data = np.zeros((nb_steps, 1 + 2 * len(ds_list)), dtype=np.float64)
    data[0, 0] = tinit

    for k in range(nb_ds):
        data[0, k * 2 + 1] = positions[k][0]
        data[0, k * 2 + 2] = velocities[k][0]

    current_iter = 1
    # --- Time loop ---
    while simu.hasNextEvent():
        simu.computeOneStep()
        data[current_iter, 0] = simu.nextTime()
        for k in range(nb_ds):
            data[current_iter, k * 2 + 1] = positions[k][0]
            data[current_iter, k * 2 + 2] = velocities[k][0]
        simu.nextStep()
        current_iter += 1

    # -- check results
    ref_pos = data[:, 1]
    ref_vel = data[:, 2]
    for k in range(1, nb_ds):
        assert np.allclose(ref_pos, data[:, k * 2 + 1], atol=time_step)
        assert np.allclose(ref_vel, data[:, k * 2 + 2], atol=time_step)
<commit_msg>[kernel] fix OSI for each DS in test_lagrangiands_osi<commit_after>"""Run the same 'model' but with different ds types
and different osi types.
Ensure that:
LagrangianTIDS + MoreauJeanOSI,
LagrangianLinearDiagonalDS + MoreauJeanOSI,
LagrangianLinearDiagonalDS + MoreauJeanBilbaoOSI
"""

import siconos.kernel as sk
import numpy as np

def test_lagrangian_and_osis():
    """Tests osi/lagrangian combinations
    """
    # --- The dynamical systems ---
    # dimension
    ndof = 3
    # initial conditons
    q0 = np.zeros(ndof, dtype=np.float64)
    v0 = np.zeros_like(q0)
    q0[0] = 1.
    # mass, stiffness and damping matrices
    # (diagonal values only)
    mass_diag = np.ones(ndof, dtype=np.float64)
    k_diag = np.zeros_like(mass_diag)
    c_diag = np.zeros_like(mass_diag)
    sigma = 0.2
    omega2 = 1.2
    k_diag[...] = omega2
    c_diag[...] = sigma

    ds_list = {}
    # - LagrangianLinearTIDS -
    ds_list['LTIDS+MJ'] = sk.LagrangianLinearTIDS(q0, v0, np.diag(mass_diag),
                                                  np.diag(k_diag),
                                                  np.diag(c_diag))
    # - LagrangianLinearDiagonalDS -
    ds_list['LLDDS+MJ'] = sk.LagrangianLinearDiagonalDS(q0, v0, k_diag,
                                                        c_diag,
                                                        mass_diag)
    ds_list['LLDDS+MJB'] = sk.LagrangianLinearDiagonalDS(q0, v0, k_diag,
                                                         c_diag,
                                                         mass_diag)
    # no mass (i.e. implicitely equal to Id)
    ds_list['LLDDS+MJB2'] = sk.LagrangianLinearDiagonalDS(q0, v0,
                                                          k_diag, c_diag)

    nb_ds = len(ds_list)
    for ds in ds_list.values():
        ds.computeForces(0., q0, v0)

    assert np.allclose(ds_list['LTIDS+MJ'].forces(),
                       ds_list['LTIDS+MJ'].forces())

    # --- Interactions ---
    cor = 0.9

    nslaw = sk.NewtonImpactNSL(cor)
    hmat = np.zeros((1, ndof), dtype=np.float64)
    hmat[0, 0] = 1.
    relation = sk.LagrangianLinearTIR(hmat)

    interactions = []
    for k in range(nb_ds):
        interactions.append(sk.Interaction(nslaw, relation))

    # --- Model ---
    tinit = 0.
    tend = 3.
    model = sk.Model(tinit, tend)

    # - insert ds into the model and link them with their interaction -
    nsds = model.nonSmoothDynamicalSystem()
    ninter = 0
    for ds in ds_list.values():
        nsds.insertDynamicalSystem(ds)
        nsds.link(interactions[ninter], ds)
        ninter += 1

    # --- Simulation ---

    # - (1) OneStepIntegrators --
    theta = 0.5000001
    standard = sk.MoreauJeanOSI(theta)
    bilbao = sk.MoreauJeanBilbaoOSI()
    #-- (2) Time discretisation --
    time_step = 1e-4
    td = sk.TimeDiscretisation(tinit, time_step)
    # -- (3) one step non smooth problem
    lcp = sk.LCP()
    # -- (4) Simulation setup with (1) (2) (3)
    simu = sk.TimeStepping(td, standard, lcp)

    # extra osi must be explicitely inserted into simu and linked to ds
    simu.prepareIntegratorForDS(standard, ds_list['LTIDS+MJ'], model, tinit)
    simu.prepareIntegratorForDS(standard, ds_list['LLDDS+MJ'], model, tinit)

    simu.prepareIntegratorForDS(bilbao, ds_list['LLDDS+MJB'], model, tinit)
    simu.prepareIntegratorForDS(bilbao, ds_list['LLDDS+MJB2'], model, tinit)

    # link simu and model, initialize
    model.setSimulation(simu)
    model.initialize()

    positions = []
    velocities = []
    for ds in ds_list.values():
        positions.append(ds.q())
        velocities.append(ds.velocity())

    # number of time steps and buffer used to save results
    nb_steps = int((tend - tinit) / time_step) + 1
    data = np.zeros((nb_steps, 1 + 2 * len(ds_list)), dtype=np.float64)
    data[0, 0] = tinit

    for k in range(nb_ds):
        data[0, k * 2 + 1] = positions[k][0]
        data[0, k * 2 + 2] = velocities[k][0]

    current_iter = 1
    # --- Time loop ---
    while simu.hasNextEvent():
        simu.computeOneStep()
        data[current_iter, 0] = simu.nextTime()
        for k in range(nb_ds):
            data[current_iter, k * 2 + 1] = positions[k][0]
            data[current_iter, k * 2 + 2] = velocities[k][0]
        simu.nextStep()
        current_iter += 1

    # -- check results
    ref_pos = data[:, 1]
    ref_vel = data[:, 2]
    for k in range(1, nb_ds):
        assert np.allclose(ref_pos, data[:, k * 2 + 1], atol=time_step)
        assert np.allclose(ref_vel, data[:, k * 2 + 2], atol=time_step)
<|endoftext|>


<commit_before>from copy import copy, deepcopy
import re

class DictList(list):
    """A combined dict and list that feels like a list, but has
    the speed benefits of a dict.  This may be eventually
    replaced by collections.OrderedDict.

    This was written to address the performance issues associated
    with searching, accessing, or iterating over a list in python
    that resulted in notable performance decays with COBRA for
    python.

    """
    def __init__(self, *args, **kwargs):
        list.__init__(self, *args, **kwargs)
        self._dict = {}
        self._object_dict = {}
        self._generate_index()

    def _check(self, id):
        """make sure duplicate id's are not added.
        This function is called before adding in elements.

        """
        if id in self._dict:
            raise ValueError, "id %s is already present in list" % str(id)

    def _generate_index(self):
        """rebuild the _dict index

        """
        new_dict = {}
        new_object_dict = {}
        for k, v in enumerate(self):
            new_dict[v.id] = k
            new_object_dict[v.id] = v
        self._dict = new_dict
        self._object_dict = new_object_dict

    def get_by_id(self, id):
        """return the element with a matching id

        """
        return self._object_dict[id]

    def list_attr(self, attribute):
        """return a list of the given attribute for every object

        """
        return [getattr(i, attribute) for i in self]

    def query(self, search_function, attribute="id"):
        """query the list

        search_function: this will be used to select which objects to return
        This can be:
            - a string, in which case any object.attribute containing
              the string will be returned
            - a compiled regular expression
            - a boolean function which takes one argument and returns True
              for desired values

        attribute: the attribute to be searched for (default is 'id').
                   If this is None, the object itself is used.

        returns: a list of objects which match the query
        """
        if attribute == None:
            select_attribute = lambda x : x
        else:
            select_attribute = lambda x: getattr(the_object, attribute)

        # if the search_function is a regular expression
        match_list = DictList()
        if isinstance(search_function, str):
            search_function = re.compile(search_function)
        if hasattr(search_function, "findall"):
            for the_object in self:
                if search_function.findall(select_attribute(the_object)) != []:
                    match_list.append(the_object)
        else:
            for the_object in self:
                if search_function(select_attribute(the_object)):
                    match_list.append(the_object)
        return match_list

    # overriding default list functions with new ones
    def __setitem__(self, i, y):
        the_id = y.id
        self._check(the_id)
        super(DictList, self).__setitem__(i, y)
        self._dict[the_id] = i
        self._object_dict[the_id] = y

    def _replace_on_id(self, new_object):
        """Allows one to replace an object by one with
        the same id.

        """
        the_id = new_object.id
        the_index = self._dict[the_id]
        super(DictList, self).__setitem__(the_index, new_object)
        self._object_dict[the_id] = new_object

    def append(self, object):
        the_id = object.id
        self._check(the_id)
        self._dict[the_id] = len(self)
        super(DictList, self).append(object)
        self._object_dict[the_id] = object

    def union(self, iterable):
        """adds elements with id's not already in the model"""
        _dict = self._dict
        append = self.append
        for i in iterable:
            if i.id not in _dict:
                append(i)

    def extend(self, iterable):
        append = self.append
        for i in iterable:
            append(i)

    def __add__(self, other, should_deepcopy=True):
        """
        other: an DictList

        should_deepcopy: Boolean.  Allow for shallow copying, however,
        this can cause problems if one doesn't know that the
        items are referenceable from different id

        """
        if should_deepcopy:
            sum = deepcopy(self) # should this be deepcopy or shallow?
        else:
            sum = self
        sum.extend(other)
        sum._generate_index()
        return sum

    def __iadd__(self, other):
        self.extend(other)
        return self

    def index(self, id):
        """
        id: A string or a :class:`~cobra.core.Object`
        """
        # because values are unique, start and stop are not relevant
        try:
            return self._dict[id]
        except:
            i = self._dict[id.id]
            if self[i] is not id:
                raise Exception(
                    "Another object with the identical id (%s) found" % id.id)
            return i
        raise Exception("%s not found" % str(i))

    def __contains__(self, object):
        """DictList.__contains__(object) <==> object in DictList
        object can either be the object to search for itself, or
        simply the id

        """
        if hasattr(object, "id"):
            the_id = object.id
        # allow to check with the object itself in addition to the id
        else:
            the_id = object
        return self._dict.has_key(the_id)

    def __copy__(self):
        self._dict.clear()
        self._object_dict.clear()
        the_copy = copy(super(DictList, self))
        self._generate_index()
        the_copy._generate_index()
        return the_copy

    def __deepcopy__(self, *args, **kwargs):
        self._dict.clear()
        self._object_dict.clear()
        the_copy = deepcopy(super(DictList, self), *args, **kwargs)
        self._generate_index()
        the_copy._generate_index()
        return the_copy

    # these functions are slower because they rebuild the _dict every time
    # TODO: speed up
    def insert(self, index, object):
        self._check(object.id)
        super(DictList, self).insert(index, object)
        self._generate_index()

    def pop(self, *args, **kwargs):
        value = super(DictList, self).pop(*args, **kwargs)
        self._generate_index()
        return value

    def remove(self, x):
        # Each item is unique in the list which allows this
        # It is much faster to do a dict lookup than n string comparisons
        super(DictList, self).pop(self.index(x))
        self._generate_index()

    def reverse(self, *args, **kwargs):
        super(DictList, self).reverse(*args, **kwargs)
        self._generate_index()

    def sort(self, *args, **kwargs):
        super(DictList, self).sort(*args, **kwargs)
        self._generate_index()

    def __setslice__(self, *args, **kwargs):
        super(DictList, self).__setslice__(*args, **kwargs)
        self._generate_index()

    def __delslice__(self, *args, **kwargs):
        super(DictList, self).__delslice__(*args, **kwargs)
        self._generate_index()

    def __delitem__(self, *args, **kwargs):
        super(DictList, self).__delitem__(*args, **kwargs)
        self._generate_index()

    def __getattr__(self, attr):
        try:
            return super(DictList, self).__getattribute__(attr)
        except:
            try:
                func = super(DictList, self).__getattribute__("get_by_id")
                return func(attr)
            except:
                raise AttributeError("DictList has no attribute or entry %s" % \
                    (attr))

    def __dir__(self):
        attributes = self.__class__.__dict__.keys()
        attributes.extend(self._dict.keys())
        return attributes

<commit_msg>DictList.query uses list comprehensions<commit_after>from copy import copy, deepcopy
import re

class DictList(list):
    """A combined dict and list that feels like a list, but has
    the speed benefits of a dict.  This may be eventually
    replaced by collections.OrderedDict.

    This was written to address the performance issues associated
    with searching, accessing, or iterating over a list in python
    that resulted in notable performance decays with COBRA for
    python.

    """
    def __init__(self, *args, **kwargs):
        list.__init__(self, *args, **kwargs)
        self._dict = {}
        self._object_dict = {}
        self._generate_index()

    def _check(self, id):
        """make sure duplicate id's are not added.
        This function is called before adding in elements.

        """
        if id in self._dict:
            raise ValueError, "id %s is already present in list" % str(id)

    def _generate_index(self):
        """rebuild the _dict index

        """
        _dict = self._dict
        object_dict = self._object_dict
        _dict.clear()
        object_dict.clear()
        for k, v in enumerate(self):
            _dict[v.id] = k
            object_dict[v.id] = v

    def get_by_id(self, id):
        """return the element with a matching id

        """
        return self._object_dict[id]

    def list_attr(self, attribute):
        """return a list of the given attribute for every object

        """
        return [getattr(i, attribute) for i in self]

    def query(self, search_function, attribute="id"):
        """query the list

        search_function: this will be used to select which objects to return
        This can be:
            - a string, in which case any object.attribute containing
              the string will be returned
            - a compiled regular expression
            - a boolean function which takes one argument and returns True
              for desired values

        attribute: the attribute to be searched for (default is 'id').
                   If this is None, the object itself is used.

        returns: a list of objects which match the query
        """
        if attribute == None:
            select_attribute = lambda x : x
        else:
            select_attribute = lambda x: getattr(the_object, attribute)

        # if the search_function is a regular expression
        if isinstance(search_function, str):
            search_function = re.compile(search_function)
        if hasattr(search_function, "findall"):
            matches = [i for i in self
                if search_function.findall(select_attribute(the_object)) != []]
        else:
            matches = [i for i in self
                if search_function(select_attribute(the_object))]
        return DictList(matches)

    # overriding default list functions with new ones
    def __setitem__(self, i, y):
        the_id = y.id
        self._check(the_id)
        super(DictList, self).__setitem__(i, y)
        self._dict[the_id] = i
        self._object_dict[the_id] = y

    def _replace_on_id(self, new_object):
        """Allows one to replace an object by one with
        the same id.

        """
        the_id = new_object.id
        the_index = self._dict[the_id]
        super(DictList, self).__setitem__(the_index, new_object)
        self._object_dict[the_id] = new_object

    def append(self, object):
        the_id = object.id
        self._check(the_id)
        self._dict[the_id] = len(self)
        super(DictList, self).append(object)
        self._object_dict[the_id] = object

    def union(self, iterable):
        """adds elements with id's not already in the model"""
        _dict = self._dict
        append = self.append
        for i in iterable:
            if i.id not in _dict:
                append(i)

    def extend(self, iterable):
        append = self.append
        for i in iterable:
            append(i)

    def __add__(self, other, should_deepcopy=True):
        """
        other: an DictList

        should_deepcopy: Boolean.  Allow for shallow copying, however,
        this can cause problems if one doesn't know that the
        items are referenceable from different id

        """
        if should_deepcopy:
            sum = deepcopy(self) # should this be deepcopy or shallow?
        else:
            sum = self
        sum.extend(other)
        sum._generate_index()
        return sum

    def __iadd__(self, other):
        self.extend(other)
        return self

    def index(self, id):
        """
        id: A string or a :class:`~cobra.core.Object`
        """
        # because values are unique, start and stop are not relevant
        try:
            return self._dict[id]
        except:
            i = self._dict[id.id]
            if self[i] is not id:
                raise Exception(
                    "Another object with the identical id (%s) found" % id.id)
            return i
        raise Exception("%s not found" % str(i))

    def __contains__(self, object):
        """DictList.__contains__(object) <==> object in DictList
        object can either be the object to search for itself, or
        simply the id

        """
        if hasattr(object, "id"):
            the_id = object.id
        # allow to check with the object itself in addition to the id
        else:
            the_id = object
        return self._dict.has_key(the_id)

    def __copy__(self):
        self._dict.clear()
        self._object_dict.clear()
        the_copy = copy(super(DictList, self))
        self._generate_index()
        the_copy._generate_index()
        return the_copy

    def __deepcopy__(self, *args, **kwargs):
        self._dict.clear()
        self._object_dict.clear()
        the_copy = deepcopy(super(DictList, self), *args, **kwargs)
        self._generate_index()
        the_copy._generate_index()
        return the_copy

    # these functions are slower because they rebuild the _dict every time
    # TODO: speed up
    def insert(self, index, object):
        self._check(object.id)
        super(DictList, self).insert(index, object)
        self._generate_index()

    def pop(self, *args, **kwargs):
        value = super(DictList, self).pop(*args, **kwargs)
        self._generate_index()
        return value

    def remove(self, x):
        # Each item is unique in the list which allows this
        # It is much faster to do a dict lookup than n string comparisons
        super(DictList, self).pop(self.index(x))
        self._generate_index()

    def reverse(self, *args, **kwargs):
        super(DictList, self).reverse(*args, **kwargs)
        self._generate_index()

    def sort(self, *args, **kwargs):
        super(DictList, self).sort(*args, **kwargs)
        self._generate_index()

    def __setslice__(self, *args, **kwargs):
        super(DictList, self).__setslice__(*args, **kwargs)
        self._generate_index()

    def __delslice__(self, *args, **kwargs):
        super(DictList, self).__delslice__(*args, **kwargs)
        self._generate_index()

    def __delitem__(self, *args, **kwargs):
        super(DictList, self).__delitem__(*args, **kwargs)
        self._generate_index()

    def __getattr__(self, attr):
        # makes items attributes as well
        try:
            return super(DictList, self).__getattribute__(attr)
        except:
            try:
                func = super(DictList, self).__getattribute__("get_by_id")
                return func(attr)
            except:
                raise AttributeError("DictList has no attribute or entry %s" % \
                    (attr))

    def __dir__(self):
        # override this to allow tab complete of items by their id
        attributes = self.__class__.__dict__.keys()
        attributes.extend(self._dict.keys())
        return attributes
<|endoftext|>


<commit_before>from django.contrib.auth.models import User
from django.db import models

# Create your models here.

class Report(models.Model):
    report_author = models.ForeignKey(User, related_name='report_author')
    report_title = models.CharField(max_length=50)
    report_content = models.TextField(max_length=999)
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)

class Score(models.Model):
    report = models.ForeignKey(Report, related_name='score')
    score_author = models.ForeignKey(User, related_name='score_author')
    score = models.IntegerField()
    evaluate_point = models.TextField(max_length=30)
    comment = models.TextField(max_length=999, blank=True)
    scored_at = models.DateTimeField(auto_now=True)
<commit_msg>modify model<commit_after>from django.contrib.auth.models import User
from django.core.validators import MaxValueValidator, MinValueValidator
from django.db import models

# Create your models here.

class Report(models.Model):
    report_author = models.ForeignKey(User, related_name='report_author')
    report_title = models.CharField(max_length=50)
    report_content = models.TextField(max_length=999)
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)

class Score(models.Model):
    report = models.ForeignKey(Report, related_name='score')
    score_author = models.ForeignKey(User, related_name='score_author')
    score = models.IntegerField(default=1, validators=[MaxValueValidator(5), MinValueValidator(1)])
    evaluate_point = models.TextField(max_length=30)
    comment = models.TextField(max_length=999, blank=True)
    scored_at = models.DateTimeField(auto_now=True)
<|endoftext|>


<commit_before>from gevent import sleep

from que.tasks import cq, get_task_logger
from que.mgmt import MgmtCallbackTask
from que.utils import user_id_from_task_id
from que.exceptions import TaskException
from gui.models import User
from vms.models import Vm, Snapshot, SnapshotDefine, BackupDefine, Node
from vms.signals import vm_created, vm_deployed, vm_notcreated, vm_json_active_changed, vm_node_changed
from api.email import sendmail
from api.utils.request import get_dummy_request
from api.task.tasks import task_log_cb_success, task_log_cb_error
from api.task.utils import callback
from api.vm.base.utils import (is_vm_missing, vm_update_ipaddress_usage, vm_delete_snapshots_of_removed_disks,
                                vm_reset, vm_update)
from api.vm.status.tasks import vm_status_changed, vm_status_one
from api.vm.snapshot.vm_define_snapshot import SnapshotDefineView
from api.vm.backup.vm_define_backup import BackupDefineView

__all__ = ('vm_create_cb', 'vm_update_cb', 'vm_delete_cb', 'vm_deploy_cb')

logger = get_task_logger(__name__)

VMS_VM_DEPLOY_TOOLONG = 600
VMS_VM_DEPLOY_TOOLONG_MAX_CYCLES = 1

def _vm_create_cb_failed(result, task_id, vm, inform_user=True):
    """
    Callback helper for failed task - revert status and inform user.
    """
    # revert status
    if result['meta']['apiview']['recreate']:
        if result['message'].find('Successfully deleted') == 0:
            Snapshot.objects.filter(vm=vm).delete()
            SnapshotDefine.objects.filter(vm=vm).delete()
            BackupDefine.objects.filter(vm=vm).delete()
            vm.save_metadata('installed', False, save=True, update_fields=('enc_json', 'changed'))
            revert_state = Vm.NOTCREATED
        else:
            revert_state = Vm.STOPPED
    else:
        revert_state = Vm.NOTCREATED
    # save reverted status and inform user
    if inform_user:
        if vm.status != revert_state:
            vm_status_changed(task_id, vm, revert_state, save_state=True)
    else:
        vm.status = revert_state
        vm.save_status()

# noinspection PyUnusedLocal
def _vm_update_cb_done(result, task_id, vm):
    """
    Callback helper for failed task - revert status.
    """
    vm.revert_notready()

# noinspection PyUnusedLocal
def _vm_delete_cb_failed(result, task_id, vm):
    """
    Callback helper for failed task - revert status.
    """
    vm.revert_notready()

def _vm_delete_cb_succeeded(task_id, vm):
    """
    Helper function used by vm_delete_cb to run proper vm methods in order to delete the VM.
    """
    vm.status = Vm.NOTCREATED
    vm.json_active = {}
    # Let's update resource counters, because they depend on Vm.is_deployed() status and json_active
    vm.save(update_node_resources=True, update_storage_resources=True)
    vm_update_ipaddress_usage(vm)
    Snapshot.objects.filter(vm=vm).delete()
    SnapshotDefine.objects.filter(vm=vm).delete()
    BackupDefine.objects.filter(vm=vm).delete()
    vm_notcreated.send(task_id, vm=vm)  # Signal!

def _vm_error(task_id, vm):
    """
    Helper function for vm_create_cb -> sets VM to ERROR state and runs real status detection.
    """
    vm.status = vm.ERROR
    vm.save_status()
    vm_status_one(task_id, vm)

@cq.task(name='api.vm.base.tasks.vm_create_cb', base=MgmtCallbackTask, bind=True)
@callback()
def vm_create_cb(result, task_id, vm_uuid=None):
    """
    A callback function for api.vm.base.views.vm_manage.
    """
    vm = Vm.objects.select_related('dc').get(uuid=vm_uuid)
    msg = result.get('message', '')

    if result['returncode'] == 0 and msg.find('Successfully created') >= 0:
        json = result.pop('json', None)

        try:  # save json from smartos
            json_active = vm.json.load(json)
            vm.json_active = json_active
            vm.json = json_active
            if result['meta']['apiview']['recreate']:
                Snapshot.objects.filter(vm=vm).delete()
                SnapshotDefine.objects.filter(vm=vm).delete()
                BackupDefine.objects.filter(vm=vm).delete()
                vm.save_metadata('installed', False, save=False)

        except Exception as e:
            logger.error('Could not parse json output from POST vm_manage(%s). Error: %s', vm_uuid, e)
            _vm_error(task_id, vm)
            logger.exception(e)
            raise TaskException(result, 'Could not parse json output')

        else:
            # save all
            vm.save(update_node_resources=True, update_storage_resources=True)
            vm_update_ipaddress_usage(vm)
            # vm_json_active_changed.send(task_id, vm=vm)  # Signal! -> not needed because vm_deployed is called below
            vm_created.send(task_id, vm=vm)  # Signal!

            if msg.find('Successfully started') < 0:  # VM was created, but could not be started
                logger.error('VM %s was created, but could not be started! Error: %s', vm_uuid, msg)
                _vm_error(task_id, vm)
                raise TaskException(result, 'Initial start failed (%s)' % msg)

            sendmail(vm.owner, 'vm/base/vm_create_subject.txt', 'vm/base/vm_create_email.txt',
                     extra_context={'vm': vm}, user_i18n=True, dc=vm.dc, fail_silently=True)

    else:
        logger.error('Found nonzero returncode in result from POST vm_manage(%s). Error: %s', vm_uuid, msg)
        # Revert status and inform user
        _vm_create_cb_failed(result, task_id, vm)

        if result['meta']['apiview']['recreate'] and msg.find('Successfully deleted') >= 0:
            _vm_error(task_id, vm)  # Something went terribly wrong

        # and FAIL this task
        raise TaskException(result, 'Got bad return code (%s). Error: %s' % (result['returncode'], msg))

    # So far so good. Now wait for deploy_over in vm_status_event_cb
    logger.info('VM %s is waiting for deploy_over...', vm_uuid)
    timer = 0
    repeat = 0

    while not vm.has_deploy_finished():
        if timer > VMS_VM_DEPLOY_TOOLONG:  # 10 minutes is too long
            if repeat == VMS_VM_DEPLOY_TOOLONG_MAX_CYCLES:  # 20 minutes is really too long
                logger.error('VM %s deploy process has timed out!', vm_uuid)
                _vm_error(task_id, vm)
                result['message'] = 'VM %s deploy has timed out' % vm.hostname
                task_log_cb_error(result, task_id, vm=vm, **result['meta'])
                return result

            repeat += 1
            timer = 0
            logger.error('VM %s takes too long to deploy. Sending force stop/start', vm_uuid)
            # noinspection PyUnusedLocal
            tid, err = vm_reset(vm)

        sleep(3.0)
        timer += 3

    logger.info('VM %s is completely deployed!', vm_uuid)
    internal_metadata = vm.json.get('internal_metadata', {}).copy()  # save internal_metadata for email
    vm = Vm.objects.select_related('dc', 'template').get(pk=vm.pk)  # Reload vm
    vm_deployed.send(task_id, vm=vm)  # Signal!
    sendmail(vm.owner, 'vm/base/vm_deploy_subject.txt', 'vm/base/vm_deploy_email.txt', fail_silently=True,
             extra_context={'vm': vm, 'internal_metadata': internal_metadata}, user_i18n=True, dc=vm.dc)

    try:
        result['message'] = '\n'.join(result['message'].strip().split('\n')[:-1])  # Remove "started" stuff
    except Exception as e:
        logger.exception(e)

    task_log_cb_success(result, task_id, vm=vm, **result['meta'])

    try:
        if vm.template:  # Try to create snapshot/backup definitions defined by template
            vm_define_snapshot, vm_define_backup = vm.template.vm_define_snapshot, vm.template.vm_define_backup

            if vm_define_snapshot or vm_define_backup:
                user = User.objects.get(id=user_id_from_task_id(task_id))
                request = get_dummy_request(vm.dc, method='POST', user=user)
                SnapshotDefineView.create_from_template(request, vm, vm_define_snapshot, log=logger)
                BackupDefineView.create_from_template(request, vm, vm_define_backup, log=logger)
    except Exception as e:
        logger.exception(e)

    return result

@cq.task(name='api.vm.base.tasks.vm_update_cb', base=MgmtCallbackTask, bind=True)
@callback()
def vm_update_cb(result, task_id, vm_uuid=None, new_node_uuid=None):
    """
    A callback function for api.vm.base.views.vm_manage.
    """
    vm = Vm.objects.select_related('dc').get(uuid=vm_uuid)
    _vm_update_cb_done(result, task_id, vm)
    msg = result.get('message', '')
    force = result['meta']['apiview']['force']

    if result['returncode'] == 0 and (force or msg.find('Successfully updated') >= 0):
        json = result.pop('json', None)

        try:  # save json from smartos
            json_active = vm.json.load(json)
        except Exception as e:
            logger.exception(e)
            logger.error('Could not parse json output from PUT vm_manage(%s). Error: %s', vm_uuid, e)
            raise TaskException(result, 'Could not parse json output')

        vm_delete_snapshots_of_removed_disks(vm)  # Do this before updating json and json_active
        vm.json = json_active
        update_fields = ['enc_json', 'enc_json_active', 'changed']
        ignored_changed_vm_attrs = (
            'set_customer_metadata',
            'remove_customer_metadata',
            'boot_timestamp',
            'autoboot',
            'vnc_port',
        )

        if new_node_uuid:
            update_dict = vm.json_update()

            for i in ignored_changed_vm_attrs:
                update_dict.pop(i, None)

            if update_dict:
                raise TaskException(result, 'VM definition on compute node differs from definition in DB in '
                                    'following attributes: %s' % ','.join(update_dict.keys()))
            update_fields.append('node_id')

        vm.json_active = json_active

        if new_node_uuid:
            node = Node.objects.get(uuid=new_node_uuid)
            vm.set_node(node)

        vm.save(update_node_resources=True, update_storage_resources=True, update_fields=update_fields)
        vm_update_ipaddress_usage(vm)
        vm_json_active_changed.send(task_id, vm=vm)  # Signal!

        if new_node_uuid:
            vm_node_changed.send(task_id, vm=vm, force_update=True)  # Signal!
            result['message'] = 'Node association successfully changed on VM %s' % vm.hostname
            if vm.json_changed():
                vm_update(vm)

    else:
        logger.error('Found nonzero returncode in result from PUT vm_manage(%s). Error: %s', vm_uuid, msg)
        raise TaskException(result, 'Got bad return code (%s). Error: %s' % (result['returncode'], msg))

    task_log_cb_success(result, task_id, vm=vm, **result['meta'])
    return result

@cq.task(name='api.vm.base.tasks.vm_delete_cb', base=MgmtCallbackTask, bind=True)
@callback()
def vm_delete_cb(result, task_id, vm_uuid=None):
    """
    A callback function for api.vm.base.views.vm_manage.
    """
    vm = Vm.objects.select_related('dc').get(uuid=vm_uuid)
    msg = result.get('message', '')

    if result['returncode'] == 0 and msg.find('Successfully deleted') == 0:
        _vm_delete_cb_succeeded(task_id, vm)
    else:
        logger.error('Found nonzero returncode in result from DELETE vm_manage(%s). Error: %s', vm_uuid, msg)

        if is_vm_missing(vm, msg):
            logger.critical('VM %s has vanished from compute node!', vm_uuid)
            _vm_delete_cb_succeeded(task_id, vm)
        else:
            _vm_delete_cb_failed(result, task_id, vm)

        raise TaskException(result, 'Got bad return code (%s). Error: %s' % (result['returncode'], msg))

    task_log_cb_success(result, task_id, vm=vm, **result['meta'])
    return result

# noinspection PyUnusedLocal
@cq.task(name='api.vm.base.tasks.vm_deploy_cb', base=MgmtCallbackTask, bind=True)
@callback()
def vm_deploy_cb(result, task_id, vm_uuid=None):
    """
    A callback function for api.vm.base.views.vm_deploy.
    Do not task_log/raise anything here - it is used as a internal task.
    """
    if result['returncode'] == 0:
        vm = Vm.objects.get(uuid=vm_uuid)
        json = result.pop('json', None)
        try:
            # save json from smartos (again)
            json_active = vm.json.load(json)
            vm.json_active = json_active
            vm.json = json_active  # We will loose root_pw here
            vm.save(update_fields=('enc_json', 'enc_json_active', 'changed'))
            # vm_json_active_changed.send(task_id, vm=vm)  # Signal! -> not needed (already informed in vm_create_cb)
        except Exception as e:
            logger.exception(e)
            logger.error('Could not parse json output from vm_deploy(%s). Error: %s', vm_uuid, e)
    else:
        logger.error('Found nonzero returncode in result from vm_deploy(%s). Error: %s',
                     vm_uuid, result.get('message', ''))

    return result
<commit_msg>Added create_timestamp into list of unchanged VM attributes<commit_after>from gevent import sleep

from que.tasks import cq, get_task_logger
from que.mgmt import MgmtCallbackTask
from que.utils import user_id_from_task_id
from que.exceptions import TaskException
from gui.models import User
from vms.models import Vm, Snapshot, SnapshotDefine, BackupDefine, Node
from vms.signals import vm_created, vm_deployed, vm_notcreated, vm_json_active_changed, vm_node_changed
from api.email import sendmail
from api.utils.request import get_dummy_request
from api.task.tasks import task_log_cb_success, task_log_cb_error
from api.task.utils import callback
from api.vm.base.utils import (is_vm_missing, vm_update_ipaddress_usage, vm_delete_snapshots_of_removed_disks,
                                vm_reset, vm_update)
from api.vm.status.tasks import vm_status_changed, vm_status_one
from api.vm.snapshot.vm_define_snapshot import SnapshotDefineView
from api.vm.backup.vm_define_backup import BackupDefineView

__all__ = ('vm_create_cb', 'vm_update_cb', 'vm_delete_cb', 'vm_deploy_cb')

logger = get_task_logger(__name__)

VMS_VM_DEPLOY_TOOLONG = 600
VMS_VM_DEPLOY_TOOLONG_MAX_CYCLES = 1

def _vm_create_cb_failed(result, task_id, vm, inform_user=True):
    """
    Callback helper for failed task - revert status and inform user.
    """
    # revert status
    if result['meta']['apiview']['recreate']:
        if result['message'].find('Successfully deleted') == 0:
            Snapshot.objects.filter(vm=vm).delete()
            SnapshotDefine.objects.filter(vm=vm).delete()
            BackupDefine.objects.filter(vm=vm).delete()
            vm.save_metadata('installed', False, save=True, update_fields=('enc_json', 'changed'))
            revert_state = Vm.NOTCREATED
        else:
            revert_state = Vm.STOPPED
    else:
        revert_state = Vm.NOTCREATED
    # save reverted status and inform user
    if inform_user:
        if vm.status != revert_state:
            vm_status_changed(task_id, vm, revert_state, save_state=True)
    else:
        vm.status = revert_state
        vm.save_status()

# noinspection PyUnusedLocal
def _vm_update_cb_done(result, task_id, vm):
    """
    Callback helper for failed task - revert status.
    """
    vm.revert_notready()

# noinspection PyUnusedLocal
def _vm_delete_cb_failed(result, task_id, vm):
    """
    Callback helper for failed task - revert status.
    """
    vm.revert_notready()

def _vm_delete_cb_succeeded(task_id, vm):
    """
    Helper function used by vm_delete_cb to run proper vm methods in order to delete the VM.
    """
    vm.status = Vm.NOTCREATED
    vm.json_active = {}
    # Let's update resource counters, because they depend on Vm.is_deployed() status and json_active
    vm.save(update_node_resources=True, update_storage_resources=True)
    vm_update_ipaddress_usage(vm)
    Snapshot.objects.filter(vm=vm).delete()
    SnapshotDefine.objects.filter(vm=vm).delete()
    BackupDefine.objects.filter(vm=vm).delete()
    vm_notcreated.send(task_id, vm=vm)  # Signal!

def _vm_error(task_id, vm):
    """
    Helper function for vm_create_cb -> sets VM to ERROR state and runs real status detection.
    """
    vm.status = vm.ERROR
    vm.save_status()
    vm_status_one(task_id, vm)

@cq.task(name='api.vm.base.tasks.vm_create_cb', base=MgmtCallbackTask, bind=True)
@callback()
def vm_create_cb(result, task_id, vm_uuid=None):
    """
    A callback function for api.vm.base.views.vm_manage.
    """
    vm = Vm.objects.select_related('dc').get(uuid=vm_uuid)
    msg = result.get('message', '')

    if result['returncode'] == 0 and msg.find('Successfully created') >= 0:
        json = result.pop('json', None)

        try:  # save json from smartos
            json_active = vm.json.load(json)
            vm.json_active = json_active
            vm.json = json_active
            if result['meta']['apiview']['recreate']:
                Snapshot.objects.filter(vm=vm).delete()
                SnapshotDefine.objects.filter(vm=vm).delete()
                BackupDefine.objects.filter(vm=vm).delete()
                vm.save_metadata('installed', False, save=False)

        except Exception as e:
            logger.error('Could not parse json output from POST vm_manage(%s). Error: %s', vm_uuid, e)
            _vm_error(task_id, vm)
            logger.exception(e)
            raise TaskException(result, 'Could not parse json output')

        else:
            # save all
            vm.save(update_node_resources=True, update_storage_resources=True)
            vm_update_ipaddress_usage(vm)
            # vm_json_active_changed.send(task_id, vm=vm)  # Signal! -> not needed because vm_deployed is called below
            vm_created.send(task_id, vm=vm)  # Signal!

            if msg.find('Successfully started') < 0:  # VM was created, but could not be started
                logger.error('VM %s was created, but could not be started! Error: %s', vm_uuid, msg)
                _vm_error(task_id, vm)
                raise TaskException(result, 'Initial start failed (%s)' % msg)

            sendmail(vm.owner, 'vm/base/vm_create_subject.txt', 'vm/base/vm_create_email.txt',
                     extra_context={'vm': vm}, user_i18n=True, dc=vm.dc, fail_silently=True)

    else:
        logger.error('Found nonzero returncode in result from POST vm_manage(%s). Error: %s', vm_uuid, msg)
        # Revert status and inform user
        _vm_create_cb_failed(result, task_id, vm)

        if result['meta']['apiview']['recreate'] and msg.find('Successfully deleted') >= 0:
            _vm_error(task_id, vm)  # Something went terribly wrong

        # and FAIL this task
        raise TaskException(result, 'Got bad return code (%s). Error: %s' % (result['returncode'], msg))

    # So far so good. Now wait for deploy_over in vm_status_event_cb
    logger.info('VM %s is waiting for deploy_over...', vm_uuid)
    timer = 0
    repeat = 0

    while not vm.has_deploy_finished():
        if timer > VMS_VM_DEPLOY_TOOLONG:  # 10 minutes is too long
            if repeat == VMS_VM_DEPLOY_TOOLONG_MAX_CYCLES:  # 20 minutes is really too long
                logger.error('VM %s deploy process has timed out!', vm_uuid)
                _vm_error(task_id, vm)
                result['message'] = 'VM %s deploy has timed out' % vm.hostname
                task_log_cb_error(result, task_id, vm=vm, **result['meta'])
                return result

            repeat += 1
            timer = 0
            logger.error('VM %s takes too long to deploy. Sending force stop/start', vm_uuid)
            # noinspection PyUnusedLocal
            tid, err = vm_reset(vm)

        sleep(3.0)
        timer += 3

    logger.info('VM %s is completely deployed!', vm_uuid)
    internal_metadata = vm.json.get('internal_metadata', {}).copy()  # save internal_metadata for email
    vm = Vm.objects.select_related('dc', 'template').get(pk=vm.pk)  # Reload vm
    vm_deployed.send(task_id, vm=vm)  # Signal!
    sendmail(vm.owner, 'vm/base/vm_deploy_subject.txt', 'vm/base/vm_deploy_email.txt', fail_silently=True,
             extra_context={'vm': vm, 'internal_metadata': internal_metadata}, user_i18n=True, dc=vm.dc)

    try:
        result['message'] = '\n'.join(result['message'].strip().split('\n')[:-1])  # Remove "started" stuff
    except Exception as e:
        logger.exception(e)

    task_log_cb_success(result, task_id, vm=vm, **result['meta'])

    try:
        if vm.template:  # Try to create snapshot/backup definitions defined by template
            vm_define_snapshot, vm_define_backup = vm.template.vm_define_snapshot, vm.template.vm_define_backup

            if vm_define_snapshot or vm_define_backup:
                user = User.objects.get(id=user_id_from_task_id(task_id))
                request = get_dummy_request(vm.dc, method='POST', user=user)
                SnapshotDefineView.create_from_template(request, vm, vm_define_snapshot, log=logger)
                BackupDefineView.create_from_template(request, vm, vm_define_backup, log=logger)
    except Exception as e:
        logger.exception(e)

    return result

@cq.task(name='api.vm.base.tasks.vm_update_cb', base=MgmtCallbackTask, bind=True)
@callback()
def vm_update_cb(result, task_id, vm_uuid=None, new_node_uuid=None):
    """
    A callback function for api.vm.base.views.vm_manage.
    """
    vm = Vm.objects.select_related('dc').get(uuid=vm_uuid)
    _vm_update_cb_done(result, task_id, vm)
    msg = result.get('message', '')
    force = result['meta']['apiview']['force']

    if result['returncode'] == 0 and (force or msg.find('Successfully updated') >= 0):
        json = result.pop('json', None)

        try:  # save json from smartos
            json_active = vm.json.load(json)
        except Exception as e:
            logger.exception(e)
            logger.error('Could not parse json output from PUT vm_manage(%s). Error: %s', vm_uuid, e)
            raise TaskException(result, 'Could not parse json output')

        vm_delete_snapshots_of_removed_disks(vm)  # Do this before updating json and json_active
        vm.json = json_active
        update_fields = ['enc_json', 'enc_json_active', 'changed']
        ignored_changed_vm_attrs = (
            'set_customer_metadata',
            'remove_customer_metadata',
            'create_timestamp',
            'boot_timestamp',
            'autoboot',
            'vnc_port',
        )

        if new_node_uuid:
            update_dict = vm.json_update()

            for i in ignored_changed_vm_attrs:
                update_dict.pop(i, None)

            if update_dict:
                raise TaskException(result, 'VM definition on compute node differs from definition in DB in '
                                    'following attributes: %s' % ','.join(update_dict.keys()))
            update_fields.append('node_id')

        vm.json_active = json_active

        if new_node_uuid:
            node = Node.objects.get(uuid=new_node_uuid)
            vm.set_node(node)

        vm.save(update_node_resources=True, update_storage_resources=True, update_fields=update_fields)
        vm_update_ipaddress_usage(vm)
        vm_json_active_changed.send(task_id, vm=vm)  # Signal!

        if new_node_uuid:
            vm_node_changed.send(task_id, vm=vm, force_update=True)  # Signal!
            result['message'] = 'Node association successfully changed on VM %s' % vm.hostname
            if vm.json_changed():
                vm_update(vm)

    else:
        logger.error('Found nonzero returncode in result from PUT vm_manage(%s). Error: %s', vm_uuid, msg)
        raise TaskException(result, 'Got bad return code (%s). Error: %s' % (result['returncode'], msg))

    task_log_cb_success(result, task_id, vm=vm, **result['meta'])
    return result

@cq.task(name='api.vm.base.tasks.vm_delete_cb', base=MgmtCallbackTask, bind=True)
@callback()
def vm_delete_cb(result, task_id, vm_uuid=None):
    """
    A callback function for api.vm.base.views.vm_manage.
    """
    vm = Vm.objects.select_related('dc').get(uuid=vm_uuid)
    msg = result.get('message', '')

    if result['returncode'] == 0 and msg.find('Successfully deleted') == 0:
        _vm_delete_cb_succeeded(task_id, vm)
    else:
        logger.error('Found nonzero returncode in result from DELETE vm_manage(%s). Error: %s', vm_uuid, msg)

        if is_vm_missing(vm, msg):
            logger.critical('VM %s has vanished from compute node!', vm_uuid)
            _vm_delete_cb_succeeded(task_id, vm)
        else:
            _vm_delete_cb_failed(result, task_id, vm)

        raise TaskException(result, 'Got bad return code (%s). Error: %s' % (result['returncode'], msg))

    task_log_cb_success(result, task_id, vm=vm, **result['meta'])
    return result

# noinspection PyUnusedLocal
@cq.task(name='api.vm.base.tasks.vm_deploy_cb', base=MgmtCallbackTask, bind=True)
@callback()
def vm_deploy_cb(result, task_id, vm_uuid=None):
    """
    A callback function for api.vm.base.views.vm_deploy.
    Do not task_log/raise anything here - it is used as a internal task.
    """
    if result['returncode'] == 0:
        vm = Vm.objects.get(uuid=vm_uuid)
        json = result.pop('json', None)
        try:
            # save json from smartos (again)
            json_active = vm.json.load(json)
            vm.json_active = json_active
            vm.json = json_active  # We will loose root_pw here
            vm.save(update_fields=('enc_json', 'enc_json_active', 'changed'))
            # vm_json_active_changed.send(task_id, vm=vm)  # Signal! -> not needed (already informed in vm_create_cb)
        except Exception as e:
            logger.exception(e)
            logger.error('Could not parse json output from vm_deploy(%s). Error: %s', vm_uuid, e)
    else:
        logger.error('Found nonzero returncode in result from vm_deploy(%s). Error: %s',
                     vm_uuid, result.get('message', ''))

    return result
<|endoftext|>


<commit_before>import serial
import time

class lockinController():
### Initialization ###
    def __init__(self, serial_port='/dev/ttyUSB2', terminator='\n'):
        # For linux, theself.serial port will be something like the default address.
        # However, for windows it will be something like 'COM6'.
        self.address=serial_port
        self.baudrate=19200
        self.terminator=terminator
        self.ser=serial.Serial(self.address, baudrate=self.baudrate,timeout=1,rtscts=True)
        #self.ser.close()
        self.setup_rs232_output()

    def setup_rs232_output(self):
        #ser=serial.Serial(self.address, baudrate=self.baudrate)
        #self.ser.open()
        try:
           self.ser.write('OUTX 0'+self.terminator)
        except Exception as e:
            print e
            raise e
        finally:
            pass
#            self.ser.close()

### Basic send and receive methods ###

    def send(self,msg):
       #self.ser=serial.Serial(self.address, baudrate=self.baudrate)
#        self.ser.open()
        time.sleep(.1)
        try:
            self.ser.write(msg+self.terminator)
        except Exception as e:
            print e
            raise e
        finally:
            pass
#            self.ser.close()

    def send_and_receive(self,msg):
        #self.ser=serial.Serial(self.address, baudrate=self.baudrate, timeout=2)
#        self.ser.open()
        time.sleep(.1)
        # This delay is necessary... for some reason. The system behaves very poorly otherwise.
        # Perhaps theself.serial port takes some time to initialize?
        try:
            self.ser.write(msg+self.terminator)
            return self.read_until_terminator()
        except Exception as e:
            print e
            raise e
        finally:
            pass
#            self.ser.close()

    def read_until_terminator(self):
        message=''
        new_char=None
        while new_char!='\r':
            new_char= self.ser.read(1)
            if new_char=='':
                # This meansself.ser has timed out. We don't want an unending loop if the terminator has somehow been lost.
                print 'Serial port timed out while reading.'
                break
            message+=new_char
        return message

### Stub methods ###

    def get_data(self):
        data_string = self.get_snap()
        data_string=data_string.strip('\r')
        data_list=data_string.split(',')
        try:
            x = float(data_list[0])
            y = float(data_list[1])
            r = float(data_list[2])
            theta = float(data_list[3])
            return x,y,r,theta
        except Exception:
            return 0,0,0,0
    def get_snap(self):
        return self.send_and_receive('SNAP? 1,2,3,4')

    def get_idn(self):
        return self.send_and_receive('*IDN?')
<commit_msg>improve lockin interface<commit_after>import serial
import time
import numpy as np

sensitivities = ((np.array([1,2,5])[None,:])*((10.**np.arange(-9,1))[:,None])).flatten()[1:-2]
time_constants = ((np.array([1,3])[None,:])*((10.**np.arange(-5,5))[:,None])).flatten()

class lockinController():
### Initialization ###
    def __init__(self, serial_port='/dev/ttyUSB2', terminator='\n'):
        # For linux, theself.serial port will be something like the default address.
        # However, for windows it will be something like 'COM6'.
        self.address=serial_port
        self.baudrate=19200
        self.terminator=terminator
        self.ser=serial.Serial(self.address, baudrate=self.baudrate,timeout=1,rtscts=True)
        #self.ser.close()
        self.setup_rs232_output()

    def setup_rs232_output(self):
        #ser=serial.Serial(self.address, baudrate=self.baudrate)
        #self.ser.open()
        try:
           self.ser.write('OUTX 0'+self.terminator)
        except Exception as e:
            print e
            raise e
        finally:
            pass
#            self.ser.close()

### Basic send and receive methods ###

    def send(self,msg):
       #self.ser=serial.Serial(self.address, baudrate=self.baudrate)
#        self.ser.open()
        time.sleep(.1)
        try:
            self.ser.write(msg+self.terminator)
        except Exception as e:
            print e
            raise e
        finally:
            pass
#            self.ser.close()

    def send_and_receive(self,msg):
        #self.ser=serial.Serial(self.address, baudrate=self.baudrate, timeout=2)
#        self.ser.open()
        time.sleep(.1)
        # This delay is necessary... for some reason. The system behaves very poorly otherwise.
        # Perhaps theself.serial port takes some time to initialize?
        try:
            self.ser.write(msg+self.terminator)
            return self.read_until_terminator()
        except Exception as e:
            print e
            raise e
        finally:
            pass
#            self.ser.close()

    def read_until_terminator(self):
        message=''
        new_char=None
        while new_char!='\r':
            new_char= self.ser.read(1)
            if new_char=='':
                # This meansself.ser has timed out. We don't want an unending loop if the terminator has somehow been lost.
                print 'Serial port timed out while reading.'
                break
            message+=new_char
        return message

### Stub methods ###

    def get_data(self):
        data_string = self.get_snap()
        data_string=data_string.strip('\r')
        data_list=data_string.split(',')
        try:
            x = float(data_list[0])
            y = float(data_list[1])
            r = float(data_list[2])
            theta = float(data_list[3])
            return x,y,r,theta
        except Exception:
            return 0,0,0,0
    def get_snap(self):
        return self.send_and_receive('SNAP? 1,2,3,4')

    def get_idn(self):
        return self.send_and_receive('*IDN?')
<|endoftext|>