<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
WSGI middleware for OpenStack Compute API.
"""

import nova.api.openstack
from nova.api.openstack.compute import consoles
from nova.api.openstack.compute import extensions
from nova.api.openstack.compute import flavors
from nova.api.openstack.compute import image_metadata
from nova.api.openstack.compute import images
from nova.api.openstack.compute import ips
from nova.api.openstack.compute import limits
from nova.api.openstack.compute import server_metadata
from nova.api.openstack.compute import servers
from nova.api.openstack.compute import versions
from nova.openstack.common import cfg
from nova.openstack.common import log as logging

LOG = logging.getLogger(__name__)

allow_instance_snapshots_opt = cfg.BoolOpt('allow_instance_snapshots',
        default=True,
        help='Permit instance snapshot operations.')

CONF = cfg.CONF
CONF.register_opt(allow_instance_snapshots_opt)

class APIRouter(nova.api.openstack.APIRouter):
    """
    Routes requests on the OpenStack API to the appropriate controller
    and method.
    """
    ExtensionManager = extensions.ExtensionManager

    def _setup_routes(self, mapper, ext_mgr, init_only):
        if init_only is None or 'versions' in init_only:
            self.resources['versions'] = versions.create_resource()
            mapper.connect("versions", "/",
                        controller=self.resources['versions'],
                        action='show')

        mapper.redirect("", "/")

        if init_only is None or 'consoles' in init_only:
            self.resources['consoles'] = consoles.create_resource()
            mapper.resource("console", "consoles",
                        controller=self.resources['consoles'],
                        parent_resource=dict(member_name='server',
                        collection_name='servers'))

        if init_only is None or 'consoles' in init_only or \
                'servers' in init_only or ips in init_only:
            self.resources['servers'] = servers.create_resource(ext_mgr)
            mapper.resource("server", "servers",
                            controller=self.resources['servers'],
                            collection={'detail': 'GET'},
                            member={'action': 'POST'})

        if init_only is None or 'ips' in init_only:
            self.resources['ips'] = ips.create_resource()
            mapper.resource("ip", "ips", controller=self.resources['ips'],
                            parent_resource=dict(member_name='server',
                                                 collection_name='servers'))

        if init_only is None or 'images' in init_only:
            self.resources['images'] = images.create_resource()
            mapper.resource("image", "images",
                            controller=self.resources['images'],
                            collection={'detail': 'GET'})

        if init_only is None or 'limits' in init_only:
            self.resources['limits'] = limits.create_resource()
            mapper.resource("limit", "limits",
                            controller=self.resources['limits'])

        if init_only is None or 'flavors' in init_only:
            self.resources['flavors'] = flavors.create_resource()
            mapper.resource("flavor", "flavors",
                            controller=self.resources['flavors'],
                            collection={'detail': 'GET'},
                            member={'action': 'POST'})

        if init_only is None or 'image_metadata' in init_only:
            self.resources['image_metadata'] = image_metadata.create_resource()
            image_metadata_controller = self.resources['image_metadata']

            mapper.resource("image_meta", "metadata",
                            controller=image_metadata_controller,
                            parent_resource=dict(member_name='image',
                            collection_name='images'))

            mapper.connect("metadata",
                           "/{project_id}/images/{image_id}/metadata",
                           controller=image_metadata_controller,
                           action='update_all',
                           conditions={"method": ['PUT']})

        if init_only is None or 'server_metadata' in init_only:
            self.resources['server_metadata'] = \
                server_metadata.create_resource()
            server_metadata_controller = self.resources['server_metadata']

            mapper.resource("server_meta", "metadata",
                            controller=server_metadata_controller,
                            parent_resource=dict(member_name='server',
                            collection_name='servers'))

            mapper.connect("metadata",
                           "/{project_id}/servers/{server_id}/metadata",
                           controller=server_metadata_controller,
                           action='update_all',
                           conditions={"method": ['PUT']})
<commit_msg>Add conditions in compute APIRouter<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
WSGI middleware for OpenStack Compute API.
"""

import nova.api.openstack
from nova.api.openstack.compute import consoles
from nova.api.openstack.compute import extensions
from nova.api.openstack.compute import flavors
from nova.api.openstack.compute import image_metadata
from nova.api.openstack.compute import images
from nova.api.openstack.compute import ips
from nova.api.openstack.compute import limits
from nova.api.openstack.compute import server_metadata
from nova.api.openstack.compute import servers
from nova.api.openstack.compute import versions
from nova.openstack.common import cfg
from nova.openstack.common import log as logging

LOG = logging.getLogger(__name__)

allow_instance_snapshots_opt = cfg.BoolOpt('allow_instance_snapshots',
        default=True,
        help='Permit instance snapshot operations.')

CONF = cfg.CONF
CONF.register_opt(allow_instance_snapshots_opt)

class APIRouter(nova.api.openstack.APIRouter):
    """
    Routes requests on the OpenStack API to the appropriate controller
    and method.
    """
    ExtensionManager = extensions.ExtensionManager

    def _setup_routes(self, mapper, ext_mgr, init_only):
        if init_only is None or 'versions' in init_only:
            self.resources['versions'] = versions.create_resource()
            mapper.connect("versions", "/",
                        controller=self.resources['versions'],
                        action='show',
                        conditions={"method": ['GET']})

        mapper.redirect("", "/")

        if init_only is None or 'consoles' in init_only:
            self.resources['consoles'] = consoles.create_resource()
            mapper.resource("console", "consoles",
                        controller=self.resources['consoles'],
                        parent_resource=dict(member_name='server',
                        collection_name='servers'))

        if init_only is None or 'consoles' in init_only or \
                'servers' in init_only or ips in init_only:
            self.resources['servers'] = servers.create_resource(ext_mgr)
            mapper.resource("server", "servers",
                            controller=self.resources['servers'],
                            collection={'detail': 'GET'},
                            member={'action': 'POST'})

        if init_only is None or 'ips' in init_only:
            self.resources['ips'] = ips.create_resource()
            mapper.resource("ip", "ips", controller=self.resources['ips'],
                            parent_resource=dict(member_name='server',
                                                 collection_name='servers'))

        if init_only is None or 'images' in init_only:
            self.resources['images'] = images.create_resource()
            mapper.resource("image", "images",
                            controller=self.resources['images'],
                            collection={'detail': 'GET'})

        if init_only is None or 'limits' in init_only:
            self.resources['limits'] = limits.create_resource()
            mapper.resource("limit", "limits",
                            controller=self.resources['limits'])

        if init_only is None or 'flavors' in init_only:
            self.resources['flavors'] = flavors.create_resource()
            mapper.resource("flavor", "flavors",
                            controller=self.resources['flavors'],
                            collection={'detail': 'GET'},
                            member={'action': 'POST'})

        if init_only is None or 'image_metadata' in init_only:
            self.resources['image_metadata'] = image_metadata.create_resource()
            image_metadata_controller = self.resources['image_metadata']

            mapper.resource("image_meta", "metadata",
                            controller=image_metadata_controller,
                            parent_resource=dict(member_name='image',
                            collection_name='images'))

            mapper.connect("metadata",
                           "/{project_id}/images/{image_id}/metadata",
                           controller=image_metadata_controller,
                           action='update_all',
                           conditions={"method": ['PUT']})

        if init_only is None or 'server_metadata' in init_only:
            self.resources['server_metadata'] = \
                server_metadata.create_resource()
            server_metadata_controller = self.resources['server_metadata']

            mapper.resource("server_meta", "metadata",
                            controller=server_metadata_controller,
                            parent_resource=dict(member_name='server',
                            collection_name='servers'))

            mapper.connect("metadata",
                           "/{project_id}/servers/{server_id}/metadata",
                           controller=server_metadata_controller,
                           action='update_all',
                           conditions={"method": ['PUT']})
<|endoftext|>


<commit_before># Copyright (c) 2011 Openstack, LLC.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
The Zone Aware Scheduler is a base class Scheduler for creating instances
across zones. There are two expansion points to this class for:
1. Assigning Weights to hosts for requested instances
2. Filtering Hosts based on required instance capabilities
"""

import operator
import json

import M2Crypto
import novaclient

from nova import crypto
from nova import db
from nova import exception
from nova import flags
from nova import log as logging
from nova import rpc

from nova.scheduler import api
from nova.scheduler import driver

FLAGS = flags.FLAGS
LOG = logging.getLogger('nova.scheduler.zone_aware_scheduler')

class InvalidBlob(exception.NovaException):
    message = _("Ill-formed or incorrectly routed 'blob' data sent "
                "to instance create request.")

class ZoneAwareScheduler(driver.Scheduler):
    """Base class for creating Zone Aware Schedulers."""

    def _call_zone_method(self, context, method, specs):
        """Call novaclient zone method. Broken out for testing."""
        return api.call_zone_method(context, method, specs=specs)

    def _provision_resource_locally(self, context, item, instance_id, kwargs):
        """Create the requested resource in this Zone."""
        host = item['hostname']
        kwargs['instance_id'] = instance_id
        rpc.cast(context,
                 db.queue_get_for(context, "compute", host),
                 {"method": "run_instance",
                  "args": kwargs})
        LOG.debug(_("Provisioning locally via compute node %(host)s")
                            % locals())

    def _decrypt_blob(self, blob):
        """Returns the decrypted blob or None if invalid. Broken out
        for testing."""
        decryptor = crypto.decryptor(FLAGS.build_plan_encryption_key)
        try:
            json_entry = decryptor(blob)
            return json.dumps(entry)
        except M2Crypto.EVP.EVPError:
            pass
        return None

    def _ask_child_zone_to_create_instance(self, context, zone_info,
                                           request_spec, kwargs):
        """Once we have determined that the request should go to one
        of our children, we need to fabricate a new POST /servers/
        call with the same parameters that were passed into us.

        Note that we have to reverse engineer from our args to get back the
        image, flavor, ipgroup, etc. since the original call could have
        come in from EC2 (which doesn't use these things)."""

        instance_type = request_spec['instance_type']
        instance_properties = request_spec['instance_properties']

        name = instance_properties['display_name']
        image_ref = instance_properties['image_ref']
        meta = instance_properties['metadata']
        flavor_id = instance_type['flavorid']
        reservation_id = instance_properties['reservation_id']

        files = kwargs['injected_files']
        ipgroup = None  # Not supported in OS API ... yet

        child_zone = zone_info['child_zone']
        child_blob = zone_info['child_blob']
        zone = db.zone_get(context, child_zone)
        url = zone.api_url
        LOG.debug(_("Forwarding instance create call to child zone %(url)s"
                    ". ReservationID=%(reservation_id)s")
                    % locals())
        nova = None
        try:
            nova = novaclient.OpenStack(zone.username, zone.password, None,
                                        url)
            nova.authenticate()
        except novaclient.exceptions.BadRequest, e:
            raise exception.NotAuthorized(_("Bad credentials attempting "
                            "to talk to zone at %(url)s.") % locals())

        nova.servers.create(name, image_ref, flavor_id, ipgroup, meta, files,
                            child_blob, reservation_id=reservation_id)

    def _provision_resource_from_blob(self, context, item, instance_id,
                                          request_spec, kwargs):
        """Create the requested resource locally or in a child zone
           based on what is stored in the zone blob info.

           Attempt to decrypt the blob to see if this request is:
           1. valid, and
           2. intended for this zone or a child zone.

           Note: If we have "blob" that means the request was passed
           into us from a parent zone. If we have "child_blob" that
           means we gathered the info from one of our children.
           It's possible that, when we decrypt the 'blob' field, it
           contains "child_blob" data. In which case we forward the
           request."""

        host_info = None
        if "blob" in item:
            # Request was passed in from above. Is it for us?
            host_info = self._decrypt_blob(item['blob'])
        elif "child_blob" in item:
            # Our immediate child zone provided this info ...
            host_info = item

        if not host_info:
            raise InvalidBlob()

        # Valid data ... is it for us?
        if 'child_zone' in host_info and 'child_blob' in host_info:
            self._ask_child_zone_to_create_instance(context, host_info,
                                                    request_spec, kwargs)
        else:
            self._provision_resource_locally(context, host_info,
                                             instance_id, kwargs)

    def _provision_resource(self, context, item, instance_id, request_spec,
                           kwargs):
        """Create the requested resource in this Zone or a child zone."""
        if "hostname" in item:
            self._provision_resource_locally(context, item, instance_id,
                            kwargs)
            return

        self._provision_resource_from_blob(context, item, instance_id,
                                               request_spec, kwargs)

    def schedule_run_instance(self, context, instance_id, request_spec,
                              *args, **kwargs):
        """This method is called from nova.compute.api to provision
        an instance. However we need to look at the parameters being
        passed in to see if this is a request to:
        1. Create a Build Plan and then provision, or
        2. Use the Build Plan information in the request parameters
           to simply create the instance (either in this zone or
           a child zone).
        """

        # TODO(sandy): We'll have to look for richer specs at some point.

        blob = request_spec.get('blob')
        if blob:
            self._provision_resource(context, request_spec, instance_id,
                                    request_spec, kwargs)
            return None

        # Create build plan and provision ...
        build_plan = self.select(context, request_spec)
        if not build_plan:
            raise driver.NoValidHost(_('No hosts were available'))

        for item in build_plan:
            self._provision_resource(context, item, instance_id, request_spec,
                                    kwargs)

        # Returning None short-circuits the routing to Compute (since
        # we've already done it here)
        return None

    def select(self, context, request_spec, *args, **kwargs):
        """Select returns a list of weights and zone/host information
        corresponding to the best hosts to service the request. Any
        child zone information has been encrypted so as not to reveal
        anything about the children.
        """
        return self._schedule(context, "compute", request_spec,
                              *args, **kwargs)

    # TODO(sandy): We're only focused on compute instances right now,
    # so we don't implement the default "schedule()" method required
    # of Schedulers.
    def schedule(self, context, topic, request_spec, *args, **kwargs):
        """The schedule() contract requires we return the one
        best-suited host for this request.
        """
        raise driver.NoValidHost(_('No hosts were available'))

    def _schedule(self, context, topic, request_spec, *args, **kwargs):
        """Returns a list of hosts that meet the required specs,
        ordered by their fitness.
        """

        if topic != "compute":
            raise NotImplemented(_("Zone Aware Scheduler only understands "
                                   "Compute nodes (for now)"))

        #TODO(sandy): how to infer this from OS API params?
        num_instances = 1

        # Filter local hosts based on requirements ...
        host_list = self.filter_hosts(num_instances, request_spec)

        # TODO(sirp): weigh_hosts should also be a function of 'topic' or
        # resources, so that we can apply different objective functions to it

        # then weigh the selected hosts.
        # weighted = [{weight=weight, name=hostname}, ...]
        weighted = self.weigh_hosts(num_instances, request_spec, host_list)

        # Next, tack on the best weights from the child zones ...
        json_spec = json.dumps(request_spec)
        child_results = self._call_zone_method(context, "select",
                specs=json_spec)
        for child_zone, result in child_results:
            for weighting in result:
                # Remember the child_zone so we can get back to
                # it later if needed. This implicitly builds a zone
                # path structure.
                host_dict = {"weight": weighting["weight"],
                             "child_zone": child_zone,
                             "child_blob": weighting["blob"]}
                weighted.append(host_dict)

        weighted.sort(key=operator.itemgetter('weight'))
        return weighted

    def filter_hosts(self, num, request_spec):
        """Derived classes must override this method and return
           a list of hosts in [(hostname, capability_dict)] format.
        """
        # NOTE(sirp): The default logic is the equivalent to AllHostsFilter
        service_states = self.zone_manager.service_states
        return [(host, services)
                for host, services in service_states.iteritems()]

    def weigh_hosts(self, num, request_spec, hosts):
        """Derived classes may override this to provide more sophisticated
        scheduling objectives
        """
        # NOTE(sirp): The default logic is the same as the NoopCostFunction
        return [dict(weight=1, hostname=host) for host, caps in hosts]
<commit_msg>don't provision to all child zones<commit_after># Copyright (c) 2011 Openstack, LLC.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
The Zone Aware Scheduler is a base class Scheduler for creating instances
across zones. There are two expansion points to this class for:
1. Assigning Weights to hosts for requested instances
2. Filtering Hosts based on required instance capabilities
"""

import operator
import json

import M2Crypto
import novaclient

from nova import crypto
from nova import db
from nova import exception
from nova import flags
from nova import log as logging
from nova import rpc

from nova.scheduler import api
from nova.scheduler import driver

FLAGS = flags.FLAGS
LOG = logging.getLogger('nova.scheduler.zone_aware_scheduler')

class InvalidBlob(exception.NovaException):
    message = _("Ill-formed or incorrectly routed 'blob' data sent "
                "to instance create request.")

class ZoneAwareScheduler(driver.Scheduler):
    """Base class for creating Zone Aware Schedulers."""

    def _call_zone_method(self, context, method, specs):
        """Call novaclient zone method. Broken out for testing."""
        return api.call_zone_method(context, method, specs=specs)

    def _provision_resource_locally(self, context, item, instance_id, kwargs):
        """Create the requested resource in this Zone."""
        host = item['hostname']
        kwargs['instance_id'] = instance_id
        rpc.cast(context,
                 db.queue_get_for(context, "compute", host),
                 {"method": "run_instance",
                  "args": kwargs})
        LOG.debug(_("Provisioning locally via compute node %(host)s")
                            % locals())

    def _decrypt_blob(self, blob):
        """Returns the decrypted blob or None if invalid. Broken out
        for testing."""
        decryptor = crypto.decryptor(FLAGS.build_plan_encryption_key)
        try:
            json_entry = decryptor(blob)
            return json.dumps(entry)
        except M2Crypto.EVP.EVPError:
            pass
        return None

    def _ask_child_zone_to_create_instance(self, context, zone_info,
                                           request_spec, kwargs):
        """Once we have determined that the request should go to one
        of our children, we need to fabricate a new POST /servers/
        call with the same parameters that were passed into us.

        Note that we have to reverse engineer from our args to get back the
        image, flavor, ipgroup, etc. since the original call could have
        come in from EC2 (which doesn't use these things)."""

        instance_type = request_spec['instance_type']
        instance_properties = request_spec['instance_properties']

        name = instance_properties['display_name']
        image_ref = instance_properties['image_ref']
        meta = instance_properties['metadata']
        flavor_id = instance_type['flavorid']
        reservation_id = instance_properties['reservation_id']

        files = kwargs['injected_files']
        ipgroup = None  # Not supported in OS API ... yet

        child_zone = zone_info['child_zone']
        child_blob = zone_info['child_blob']
        zone = db.zone_get(context, child_zone)
        url = zone.api_url
        LOG.debug(_("Forwarding instance create call to child zone %(url)s"
                    ". ReservationID=%(reservation_id)s")
                    % locals())
        nova = None
        try:
            nova = novaclient.OpenStack(zone.username, zone.password, None,
                                        url)
            nova.authenticate()
        except novaclient.exceptions.BadRequest, e:
            raise exception.NotAuthorized(_("Bad credentials attempting "
                            "to talk to zone at %(url)s.") % locals())

        nova.servers.create(name, image_ref, flavor_id, ipgroup, meta, files,
                            child_blob, reservation_id=reservation_id)

    def _provision_resource_from_blob(self, context, item, instance_id,
                                          request_spec, kwargs):
        """Create the requested resource locally or in a child zone
           based on what is stored in the zone blob info.

           Attempt to decrypt the blob to see if this request is:
           1. valid, and
           2. intended for this zone or a child zone.

           Note: If we have "blob" that means the request was passed
           into us from a parent zone. If we have "child_blob" that
           means we gathered the info from one of our children.
           It's possible that, when we decrypt the 'blob' field, it
           contains "child_blob" data. In which case we forward the
           request."""

        host_info = None
        if "blob" in item:
            # Request was passed in from above. Is it for us?
            host_info = self._decrypt_blob(item['blob'])
        elif "child_blob" in item:
            # Our immediate child zone provided this info ...
            host_info = item

        if not host_info:
            raise InvalidBlob()

        # Valid data ... is it for us?
        if 'child_zone' in host_info and 'child_blob' in host_info:
            self._ask_child_zone_to_create_instance(context, host_info,
                                                    request_spec, kwargs)
        else:
            self._provision_resource_locally(context, host_info,
                                             instance_id, kwargs)

    def _provision_resource(self, context, item, instance_id, request_spec,
                           kwargs):
        """Create the requested resource in this Zone or a child zone."""
        if "hostname" in item:
            self._provision_resource_locally(context, item, instance_id,
                            kwargs)
            return

        self._provision_resource_from_blob(context, item, instance_id,
                                               request_spec, kwargs)

    def schedule_run_instance(self, context, instance_id, request_spec,
                              *args, **kwargs):
        """This method is called from nova.compute.api to provision
        an instance. However we need to look at the parameters being
        passed in to see if this is a request to:
        1. Create a Build Plan and then provision, or
        2. Use the Build Plan information in the request parameters
           to simply create the instance (either in this zone or
           a child zone).
        """

        # TODO(sandy): We'll have to look for richer specs at some point.

        blob = request_spec.get('blob')
        if blob:
            self._provision_resource(context, request_spec, instance_id,
                                    request_spec, kwargs)
            return None

        # Create build plan and provision ...
        build_plan = self.select(context, request_spec)
        if not build_plan:
            raise driver.NoValidHost(_('No hosts were available'))

        for num in xrange(request_spec['num_instances']):
            if not build_plan:
                break

            item = build_plan.pop(0)
            self._provision_resource(context, item, instance_id, request_spec,
                                    kwargs)

        # Returning None short-circuits the routing to Compute (since
        # we've already done it here)
        return None

    def select(self, context, request_spec, *args, **kwargs):
        """Select returns a list of weights and zone/host information
        corresponding to the best hosts to service the request. Any
        child zone information has been encrypted so as not to reveal
        anything about the children.
        """
        return self._schedule(context, "compute", request_spec,
                              *args, **kwargs)

    # TODO(sandy): We're only focused on compute instances right now,
    # so we don't implement the default "schedule()" method required
    # of Schedulers.
    def schedule(self, context, topic, request_spec, *args, **kwargs):
        """The schedule() contract requires we return the one
        best-suited host for this request.
        """
        raise driver.NoValidHost(_('No hosts were available'))

    def _schedule(self, context, topic, request_spec, *args, **kwargs):
        """Returns a list of hosts that meet the required specs,
        ordered by their fitness.
        """

        if topic != "compute":
            raise NotImplemented(_("Zone Aware Scheduler only understands "
                                   "Compute nodes (for now)"))

        #TODO(sandy): how to infer this from OS API params?
        num_instances = 1

        # Filter local hosts based on requirements ...
        host_list = self.filter_hosts(num_instances, request_spec)

        # TODO(sirp): weigh_hosts should also be a function of 'topic' or
        # resources, so that we can apply different objective functions to it

        # then weigh the selected hosts.
        # weighted = [{weight=weight, name=hostname}, ...]
        weighted = self.weigh_hosts(num_instances, request_spec, host_list)

        # Next, tack on the best weights from the child zones ...
        json_spec = json.dumps(request_spec)
        child_results = self._call_zone_method(context, "select",
                specs=json_spec)
        for child_zone, result in child_results:
            for weighting in result:
                # Remember the child_zone so we can get back to
                # it later if needed. This implicitly builds a zone
                # path structure.
                host_dict = {"weight": weighting["weight"],
                             "child_zone": child_zone,
                             "child_blob": weighting["blob"]}
                weighted.append(host_dict)

        weighted.sort(key=operator.itemgetter('weight'))
        return weighted

    def filter_hosts(self, num, request_spec):
        """Derived classes must override this method and return
           a list of hosts in [(hostname, capability_dict)] format.
        """
        # NOTE(sirp): The default logic is the equivalent to AllHostsFilter
        service_states = self.zone_manager.service_states
        return [(host, services)
                for host, services in service_states.iteritems()]

    def weigh_hosts(self, num, request_spec, hosts):
        """Derived classes may override this to provide more sophisticated
        scheduling objectives
        """
        # NOTE(sirp): The default logic is the same as the NoopCostFunction
        return [dict(weight=1, hostname=host) for host, caps in hosts]
<|endoftext|>


<commit_before># Copyright (c) 2011 Openstack, LLC.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
The Zone Aware Scheduler is a base class Scheduler for creating instances
across zones. There are two expansion points to this class for:
1. Assigning Weights to hosts for requested instances
2. Filtering Hosts based on required instance capabilities
"""

import operator
import json

import M2Crypto
import novaclient

from nova import crypto
from nova import db
from nova import exception
from nova import flags
from nova import log as logging
from nova import rpc

from nova.compute import api as compute_api
from nova.scheduler import api
from nova.scheduler import driver

FLAGS = flags.FLAGS
LOG = logging.getLogger('nova.scheduler.zone_aware_scheduler')

class InvalidBlob(exception.NovaException):
    message = _("Ill-formed or incorrectly routed 'blob' data sent "
                "to instance create request.")

class ZoneAwareScheduler(driver.Scheduler):
    """Base class for creating Zone Aware Schedulers."""

    def _call_zone_method(self, context, method, specs, zones):
        """Call novaclient zone method. Broken out for testing."""
        return api.call_zone_method(context, method, specs=specs, zones=zones)

    def _provision_resource_locally(self, context, build_plan_item,
                                    request_spec, kwargs):
        """Create the requested resource in this Zone."""
        host = build_plan_item['hostname']
        base_options = request_spec['instance_properties']

        # TODO(sandy): I guess someone needs to add block_device_mapping
        # support at some point? Also, OS API has no concept of security
        # groups.
        instance = compute_api.API().create_db_entry_for_new_instance(context,
            base_options, None, [])

        instance_id = instance['id']
        kwargs['instance_id'] = instance_id

        rpc.cast(context,
                 db.queue_get_for(context, "compute", host),
                 {"method": "run_instance",
                  "args": kwargs})
        LOG.debug(_("Provisioning locally via compute node %(host)s")
                            % locals())

    def _decrypt_blob(self, blob):
        """Returns the decrypted blob or None if invalid. Broken out
        for testing."""
        decryptor = crypto.decryptor(FLAGS.build_plan_encryption_key)
        try:
            json_entry = decryptor(blob)
            return json.dumps(json_entry)
        except M2Crypto.EVP.EVPError:
            pass
        return None

    def _ask_child_zone_to_create_instance(self, context, zone_info,
                                           request_spec, kwargs):
        """Once we have determined that the request should go to one
        of our children, we need to fabricate a new POST /servers/
        call with the same parameters that were passed into us.

        Note that we have to reverse engineer from our args to get back the
        image, flavor, ipgroup, etc. since the original call could have
        come in from EC2 (which doesn't use these things)."""

        instance_type = request_spec['instance_type']
        instance_properties = request_spec['instance_properties']

        name = instance_properties['display_name']
        image_ref = instance_properties['image_ref']
        meta = instance_properties['metadata']
        flavor_id = instance_type['flavorid']
        reservation_id = instance_properties['reservation_id']

        files = kwargs['injected_files']
        ipgroup = None  # Not supported in OS API ... yet

        child_zone = zone_info['child_zone']
        child_blob = zone_info['child_blob']
        zone = db.zone_get(context, child_zone)
        url = zone.api_url
        LOG.debug(_("Forwarding instance create call to child zone %(url)s"
                    ". ReservationID=%(reservation_id)s")
                    % locals())
        nova = None
        try:
            nova = novaclient.OpenStack(zone.username, zone.password, None,
                                        url)
            nova.authenticate()
        except novaclient.exceptions.BadRequest, e:
            raise exception.NotAuthorized(_("Bad credentials attempting "
                            "to talk to zone at %(url)s.") % locals())

        nova.servers.create(name, image_ref, flavor_id, ipgroup, meta, files,
                            child_blob, reservation_id=reservation_id)

    def _provision_resource_from_blob(self, context, build_plan_item,
                                      instance_id, request_spec, kwargs):
        """Create the requested resource locally or in a child zone
           based on what is stored in the zone blob info.

           Attempt to decrypt the blob to see if this request is:
           1. valid, and
           2. intended for this zone or a child zone.

           Note: If we have "blob" that means the request was passed
           into us from a parent zone. If we have "child_blob" that
           means we gathered the info from one of our children.
           It's possible that, when we decrypt the 'blob' field, it
           contains "child_blob" data. In which case we forward the
           request."""

        host_info = None
        if "blob" in build_plan_item:
            # Request was passed in from above. Is it for us?
            host_info = self._decrypt_blob(build_plan_item['blob'])
        elif "child_blob" in build_plan_item:
            # Our immediate child zone provided this info ...
            host_info = build_plan_item

        if not host_info:
            raise InvalidBlob()

        # Valid data ... is it for us?
        if 'child_zone' in host_info and 'child_blob' in host_info:
            self._ask_child_zone_to_create_instance(context, host_info,
                                                    request_spec, kwargs)
        else:
            self._provision_resource_locally(context, host_info, request_spec,
                                             kwargs)

    def _provision_resource(self, context, build_plan_item, instance_id,
                            request_spec, kwargs):
        """Create the requested resource in this Zone or a child zone."""
        if "hostname" in build_plan_item:
            self._provision_resource_locally(context, build_plan_item,
                                             request_spec, kwargs)
            return

        self._provision_resource_from_blob(context, build_plan_item,
                                           instance_id, request_spec, kwargs)

    def _adjust_child_weights(self, child_results, zones):
        """Apply the Scale and Offset values from the Zone definition
        to adjust the weights returned from the child zones. Alters
        child_results in place.
        """
        for zone_id, result in child_results:
            if not result:
                continue

            assert isinstance(zone_id, int)

            for zone_rec in zones:
                if zone_rec['id'] != zone_id:
                    continue

                for item in result:
                    try:
                        offset = zone_rec['weight_offset']
                        scale = zone_rec['weight_scale']
                        raw_weight = item['weight']
                        cooked_weight = offset + scale * raw_weight
                        item['weight'] = cooked_weight
                        item['raw_weight'] = raw_weight
                    except KeyError:
                        LOG.exception(_("Bad child zone scaling values "
                                        "for Zone: %(zone_id)s") % locals())

    def schedule_run_instance(self, context, instance_id, request_spec,
                              *args, **kwargs):
        """This method is called from nova.compute.api to provision
        an instance. However we need to look at the parameters being
        passed in to see if this is a request to:
        1. Create a Build Plan and then provision, or
        2. Use the Build Plan information in the request parameters
           to simply create the instance (either in this zone or
           a child zone).
        """

        # TODO(sandy): We'll have to look for richer specs at some point.

        blob = request_spec.get('blob')
        if blob:
            self._provision_resource(context, request_spec, instance_id,
                                    request_spec, kwargs)
            return None

        num_instances = request_spec.get('num_instances', 1)
        LOG.debug(_("Attempting to build %(num_instances)d instance(s)") %
                locals())

        # Create build plan and provision ...
        build_plan = self.select(context, request_spec)
        if not build_plan:
            raise driver.NoValidHost(_('No hosts were available'))

        for num in xrange(num_instances):
            if not build_plan:
                break

            build_plan_item = build_plan.pop(0)
            self._provision_resource(context, build_plan_item, instance_id,
                                     request_spec, kwargs)

        # Returning None short-circuits the routing to Compute (since
        # we've already done it here)
        return None

    def select(self, context, request_spec, *args, **kwargs):
        """Select returns a list of weights and zone/host information
        corresponding to the best hosts to service the request. Any
        child zone information has been encrypted so as not to reveal
        anything about the children.
        """
        return self._schedule(context, "compute", request_spec,
                              *args, **kwargs)

    # TODO(sandy): We're only focused on compute instances right now,
    # so we don't implement the default "schedule()" method required
    # of Schedulers.
    def schedule(self, context, topic, request_spec, *args, **kwargs):
        """The schedule() contract requires we return the one
        best-suited host for this request.
        """
        raise driver.NoValidHost(_('No hosts were available'))

    def _schedule(self, context, topic, request_spec, *args, **kwargs):
        """Returns a list of hosts that meet the required specs,
        ordered by their fitness.
        """

        if topic != "compute":
            raise NotImplementedError(_("Zone Aware Scheduler only understands"
                                   " Compute nodes (for now)"))

        num_instances = request_spec.get('num_instances', 1)
        instance_type = request_spec['instance_type']

        weighted = []
        host_list = None

        for i in xrange(num_instances):
            # Filter local hosts based on requirements ...
            #
            # The first pass through here will pass 'None' as the
            # host_list.. which tells the filter to build the full
            # list of hosts.
            # On a 2nd pass, the filter can modify the host_list with
            # any updates it needs to make based on resources that
            # may have been consumed from a previous build..
            host_list = self.filter_hosts(topic, request_spec, host_list)
            if not host_list:
                LOG.warn(_("Filter returned no hosts after processing "
                        "%(i)d of %(num_instances)d instances") % locals())
                break

            # then weigh the selected hosts.
            # weighted = [{weight=weight, hostname=hostname,
            #              capabilities=capabs}, ...]
            weights = self.weigh_hosts(topic, request_spec, host_list)
            weights.sort(key=operator.itemgetter('weight'))
            best_weight = weights[0]
            weighted.append(best_weight)
            self.consume_resources(topic, best_weight['capabilities'],
                    instance_type)

        # Next, tack on the best weights from the child zones ...
        json_spec = json.dumps(request_spec)
        all_zones = db.zone_get_all(context)
        child_results = self._call_zone_method(context, "select",
                specs=json_spec, zones=all_zones)
        self._adjust_child_weights(child_results, all_zones)
        for child_zone, result in child_results:
            for weighting in result:
                # Remember the child_zone so we can get back to
                # it later if needed. This implicitly builds a zone
                # path structure.
                host_dict = {"weight": weighting["weight"],
                             "child_zone": child_zone,
                             "child_blob": weighting["blob"]}
                weighted.append(host_dict)

        weighted.sort(key=operator.itemgetter('weight'))
        return weighted

    def compute_filter(self, hostname, capabilities, request_spec):
        """Return whether or not we can schedule to this compute node.
        Derived classes should override this and return True if the host
        is acceptable for scheduling.
        """
        instance_type = request_spec['instance_type']
        requested_mem = instance_type['memory_mb'] * 1024 * 1024
        return capabilities['host_memory_free'] >= requested_mem

    def filter_hosts(self, topic, request_spec, host_list=None):
        """Return a list of hosts which are acceptable for scheduling.
        Return value should be a list of (hostname, capability_dict)s.
        Derived classes may override this, but may find the
        '<topic>_filter' function more appropriate.
        """

        def _default_filter(self, hostname, capabilities, request_spec):
            """Default filter function if there's no <topic>_filter"""
            # NOTE(sirp): The default logic is the equivalent to
            # AllHostsFilter
            return True

        filter_func = getattr(self, '%s_filter' % topic, _default_filter)

        if host_list is None:
            first_run = True
            host_list = self.zone_manager.service_states.iteritems()
        else:
            first_run = False

        filtered_hosts = []
        for host, services in host_list:
            if first_run:
                if topic not in services:
                    continue
                services = services[topic]
            if filter_func(host, services, request_spec):
                filtered_hosts.append((host, services))
        return filtered_hosts

    def weigh_hosts(self, topic, request_spec, hosts):
        """Derived classes may override this to provide more sophisticated
        scheduling objectives
        """
        # NOTE(sirp): The default logic is the same as the NoopCostFunction
        return [dict(weight=1, hostname=hostname, capabilities=capabilities)
                for hostname, capabilities in hosts]

    def compute_consume(self, capabilities, instance_type):
        """Consume compute resources for selected host"""

        requested_mem = max(instance_type['memory_mb'], 0) * 1024 * 1024
        capabilities['host_memory_free'] -= requested_mem

    def consume_resources(self, topic, capabilities, instance_type):
        """Consume resources for a specific host.  'host' is a tuple
        of the hostname and the services"""

        consume_func = getattr(self, '%s_consume' % topic, None)
        if not consume_func:
            return
        consume_func(capabilities, instance_type)
<commit_msg>align multi-line string<commit_after># Copyright (c) 2011 Openstack, LLC.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
The Zone Aware Scheduler is a base class Scheduler for creating instances
across zones. There are two expansion points to this class for:
1. Assigning Weights to hosts for requested instances
2. Filtering Hosts based on required instance capabilities
"""

import operator
import json

import M2Crypto
import novaclient

from nova import crypto
from nova import db
from nova import exception
from nova import flags
from nova import log as logging
from nova import rpc

from nova.compute import api as compute_api
from nova.scheduler import api
from nova.scheduler import driver

FLAGS = flags.FLAGS
LOG = logging.getLogger('nova.scheduler.zone_aware_scheduler')

class InvalidBlob(exception.NovaException):
    message = _("Ill-formed or incorrectly routed 'blob' data sent "
                "to instance create request.")

class ZoneAwareScheduler(driver.Scheduler):
    """Base class for creating Zone Aware Schedulers."""

    def _call_zone_method(self, context, method, specs, zones):
        """Call novaclient zone method. Broken out for testing."""
        return api.call_zone_method(context, method, specs=specs, zones=zones)

    def _provision_resource_locally(self, context, build_plan_item,
                                    request_spec, kwargs):
        """Create the requested resource in this Zone."""
        host = build_plan_item['hostname']
        base_options = request_spec['instance_properties']

        # TODO(sandy): I guess someone needs to add block_device_mapping
        # support at some point? Also, OS API has no concept of security
        # groups.
        instance = compute_api.API().create_db_entry_for_new_instance(context,
            base_options, None, [])

        instance_id = instance['id']
        kwargs['instance_id'] = instance_id

        rpc.cast(context,
                 db.queue_get_for(context, "compute", host),
                 {"method": "run_instance",
                  "args": kwargs})
        LOG.debug(_("Provisioning locally via compute node %(host)s")
                            % locals())

    def _decrypt_blob(self, blob):
        """Returns the decrypted blob or None if invalid. Broken out
        for testing."""
        decryptor = crypto.decryptor(FLAGS.build_plan_encryption_key)
        try:
            json_entry = decryptor(blob)
            return json.dumps(json_entry)
        except M2Crypto.EVP.EVPError:
            pass
        return None

    def _ask_child_zone_to_create_instance(self, context, zone_info,
                                           request_spec, kwargs):
        """Once we have determined that the request should go to one
        of our children, we need to fabricate a new POST /servers/
        call with the same parameters that were passed into us.

        Note that we have to reverse engineer from our args to get back the
        image, flavor, ipgroup, etc. since the original call could have
        come in from EC2 (which doesn't use these things)."""

        instance_type = request_spec['instance_type']
        instance_properties = request_spec['instance_properties']

        name = instance_properties['display_name']
        image_ref = instance_properties['image_ref']
        meta = instance_properties['metadata']
        flavor_id = instance_type['flavorid']
        reservation_id = instance_properties['reservation_id']

        files = kwargs['injected_files']
        ipgroup = None  # Not supported in OS API ... yet

        child_zone = zone_info['child_zone']
        child_blob = zone_info['child_blob']
        zone = db.zone_get(context, child_zone)
        url = zone.api_url
        LOG.debug(_("Forwarding instance create call to child zone %(url)s"
                    ". ReservationID=%(reservation_id)s")
                    % locals())
        nova = None
        try:
            nova = novaclient.OpenStack(zone.username, zone.password, None,
                                        url)
            nova.authenticate()
        except novaclient.exceptions.BadRequest, e:
            raise exception.NotAuthorized(_("Bad credentials attempting "
                            "to talk to zone at %(url)s.") % locals())

        nova.servers.create(name, image_ref, flavor_id, ipgroup, meta, files,
                            child_blob, reservation_id=reservation_id)

    def _provision_resource_from_blob(self, context, build_plan_item,
                                      instance_id, request_spec, kwargs):
        """Create the requested resource locally or in a child zone
           based on what is stored in the zone blob info.

           Attempt to decrypt the blob to see if this request is:
           1. valid, and
           2. intended for this zone or a child zone.

           Note: If we have "blob" that means the request was passed
           into us from a parent zone. If we have "child_blob" that
           means we gathered the info from one of our children.
           It's possible that, when we decrypt the 'blob' field, it
           contains "child_blob" data. In which case we forward the
           request."""

        host_info = None
        if "blob" in build_plan_item:
            # Request was passed in from above. Is it for us?
            host_info = self._decrypt_blob(build_plan_item['blob'])
        elif "child_blob" in build_plan_item:
            # Our immediate child zone provided this info ...
            host_info = build_plan_item

        if not host_info:
            raise InvalidBlob()

        # Valid data ... is it for us?
        if 'child_zone' in host_info and 'child_blob' in host_info:
            self._ask_child_zone_to_create_instance(context, host_info,
                                                    request_spec, kwargs)
        else:
            self._provision_resource_locally(context, host_info, request_spec,
                                             kwargs)

    def _provision_resource(self, context, build_plan_item, instance_id,
                            request_spec, kwargs):
        """Create the requested resource in this Zone or a child zone."""
        if "hostname" in build_plan_item:
            self._provision_resource_locally(context, build_plan_item,
                                             request_spec, kwargs)
            return

        self._provision_resource_from_blob(context, build_plan_item,
                                           instance_id, request_spec, kwargs)

    def _adjust_child_weights(self, child_results, zones):
        """Apply the Scale and Offset values from the Zone definition
        to adjust the weights returned from the child zones. Alters
        child_results in place.
        """
        for zone_id, result in child_results:
            if not result:
                continue

            assert isinstance(zone_id, int)

            for zone_rec in zones:
                if zone_rec['id'] != zone_id:
                    continue

                for item in result:
                    try:
                        offset = zone_rec['weight_offset']
                        scale = zone_rec['weight_scale']
                        raw_weight = item['weight']
                        cooked_weight = offset + scale * raw_weight
                        item['weight'] = cooked_weight
                        item['raw_weight'] = raw_weight
                    except KeyError:
                        LOG.exception(_("Bad child zone scaling values "
                                        "for Zone: %(zone_id)s") % locals())

    def schedule_run_instance(self, context, instance_id, request_spec,
                              *args, **kwargs):
        """This method is called from nova.compute.api to provision
        an instance. However we need to look at the parameters being
        passed in to see if this is a request to:
        1. Create a Build Plan and then provision, or
        2. Use the Build Plan information in the request parameters
           to simply create the instance (either in this zone or
           a child zone).
        """

        # TODO(sandy): We'll have to look for richer specs at some point.

        blob = request_spec.get('blob')
        if blob:
            self._provision_resource(context, request_spec, instance_id,
                                    request_spec, kwargs)
            return None

        num_instances = request_spec.get('num_instances', 1)
        LOG.debug(_("Attempting to build %(num_instances)d instance(s)") %
                locals())

        # Create build plan and provision ...
        build_plan = self.select(context, request_spec)
        if not build_plan:
            raise driver.NoValidHost(_('No hosts were available'))

        for num in xrange(num_instances):
            if not build_plan:
                break

            build_plan_item = build_plan.pop(0)
            self._provision_resource(context, build_plan_item, instance_id,
                                     request_spec, kwargs)

        # Returning None short-circuits the routing to Compute (since
        # we've already done it here)
        return None

    def select(self, context, request_spec, *args, **kwargs):
        """Select returns a list of weights and zone/host information
        corresponding to the best hosts to service the request. Any
        child zone information has been encrypted so as not to reveal
        anything about the children.
        """
        return self._schedule(context, "compute", request_spec,
                              *args, **kwargs)

    # TODO(sandy): We're only focused on compute instances right now,
    # so we don't implement the default "schedule()" method required
    # of Schedulers.
    def schedule(self, context, topic, request_spec, *args, **kwargs):
        """The schedule() contract requires we return the one
        best-suited host for this request.
        """
        raise driver.NoValidHost(_('No hosts were available'))

    def _schedule(self, context, topic, request_spec, *args, **kwargs):
        """Returns a list of hosts that meet the required specs,
        ordered by their fitness.
        """

        if topic != "compute":
            raise NotImplementedError(_("Zone Aware Scheduler only understands"
                                        " Compute nodes (for now)"))

        num_instances = request_spec.get('num_instances', 1)
        instance_type = request_spec['instance_type']

        weighted = []
        host_list = None

        for i in xrange(num_instances):
            # Filter local hosts based on requirements ...
            #
            # The first pass through here will pass 'None' as the
            # host_list.. which tells the filter to build the full
            # list of hosts.
            # On a 2nd pass, the filter can modify the host_list with
            # any updates it needs to make based on resources that
            # may have been consumed from a previous build..
            host_list = self.filter_hosts(topic, request_spec, host_list)
            if not host_list:
                LOG.warn(_("Filter returned no hosts after processing "
                        "%(i)d of %(num_instances)d instances") % locals())
                break

            # then weigh the selected hosts.
            # weighted = [{weight=weight, hostname=hostname,
            #              capabilities=capabs}, ...]
            weights = self.weigh_hosts(topic, request_spec, host_list)
            weights.sort(key=operator.itemgetter('weight'))
            best_weight = weights[0]
            weighted.append(best_weight)
            self.consume_resources(topic, best_weight['capabilities'],
                    instance_type)

        # Next, tack on the best weights from the child zones ...
        json_spec = json.dumps(request_spec)
        all_zones = db.zone_get_all(context)
        child_results = self._call_zone_method(context, "select",
                specs=json_spec, zones=all_zones)
        self._adjust_child_weights(child_results, all_zones)
        for child_zone, result in child_results:
            for weighting in result:
                # Remember the child_zone so we can get back to
                # it later if needed. This implicitly builds a zone
                # path structure.
                host_dict = {"weight": weighting["weight"],
                             "child_zone": child_zone,
                             "child_blob": weighting["blob"]}
                weighted.append(host_dict)

        weighted.sort(key=operator.itemgetter('weight'))
        return weighted

    def compute_filter(self, hostname, capabilities, request_spec):
        """Return whether or not we can schedule to this compute node.
        Derived classes should override this and return True if the host
        is acceptable for scheduling.
        """
        instance_type = request_spec['instance_type']
        requested_mem = instance_type['memory_mb'] * 1024 * 1024
        return capabilities['host_memory_free'] >= requested_mem

    def filter_hosts(self, topic, request_spec, host_list=None):
        """Return a list of hosts which are acceptable for scheduling.
        Return value should be a list of (hostname, capability_dict)s.
        Derived classes may override this, but may find the
        '<topic>_filter' function more appropriate.
        """

        def _default_filter(self, hostname, capabilities, request_spec):
            """Default filter function if there's no <topic>_filter"""
            # NOTE(sirp): The default logic is the equivalent to
            # AllHostsFilter
            return True

        filter_func = getattr(self, '%s_filter' % topic, _default_filter)

        if host_list is None:
            first_run = True
            host_list = self.zone_manager.service_states.iteritems()
        else:
            first_run = False

        filtered_hosts = []
        for host, services in host_list:
            if first_run:
                if topic not in services:
                    continue
                services = services[topic]
            if filter_func(host, services, request_spec):
                filtered_hosts.append((host, services))
        return filtered_hosts

    def weigh_hosts(self, topic, request_spec, hosts):
        """Derived classes may override this to provide more sophisticated
        scheduling objectives
        """
        # NOTE(sirp): The default logic is the same as the NoopCostFunction
        return [dict(weight=1, hostname=hostname, capabilities=capabilities)
                for hostname, capabilities in hosts]

    def compute_consume(self, capabilities, instance_type):
        """Consume compute resources for selected host"""

        requested_mem = max(instance_type['memory_mb'], 0) * 1024 * 1024
        capabilities['host_memory_free'] -= requested_mem

    def consume_resources(self, topic, capabilities, instance_type):
        """Consume resources for a specific host.  'host' is a tuple
        of the hostname and the services"""

        consume_func = getattr(self, '%s_consume' % topic, None)
        if not consume_func:
            return
        consume_func(capabilities, instance_type)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import six

from collections import OrderedDict
from functools import wraps

if six.PY3:
    from inspect import signature
else:
    from funcsigs import signature

from six import string_types

class GuardError(Exception):
    pass

def _params(f, argumets, keyword_argumets, params):
    sig = signature(f)
    bind = sig.bind(*argumets, **keyword_argumets)
    if six.PY3:
        bind.apply_defaults()

    return OrderedDict(
        filter(
            lambda item: item[0] in params,
            bind.arguments.items()
        )
    )

def guard(params, guardian, error_class=GuardError, message=''):
    '''
    A guard function - check parameters
    with guardian function on decorated function

    :param tuple or string params: guarded function parameter/s
    :param function guardian: verifying the conditions for the selected parameter
    :param Exception error_class: raised class when guardian return false
    :param string message: error message
    '''
    params = [params] if isinstance(params, string_types) else params

    def guard_decorate(f):
        @wraps(f)
        def _guard_decorate(*args, **kwargs):
            if guardian(**_params(f, args, kwargs, params)):
                return f(*args, **kwargs)
            else:
                raise error_class(message)
        return _guard_decorate
    return guard_decorate
<commit_msg>Fixed applying default values (avaliable after python 3.4)<commit_after># -*- coding: utf-8 -*-
import sys

import six

from collections import OrderedDict
from functools import wraps

if six.PY3:  # pragma: no branch
    from inspect import signature
else:
    from funcsigs import signature

from six import string_types

class GuardError(Exception):
    pass

def _params(f, argumets, keyword_argumets, params):
    sig = signature(f)
    bind = sig.bind(*argumets, **keyword_argumets)
    if sys.version_info >= (3, 5):
        bind.apply_defaults()

    return OrderedDict(
        filter(
            lambda item: item[0] in params,
            bind.arguments.items()
        )
    )

def guard(params, guardian, error_class=GuardError, message=''):
    '''
    A guard function - check parameters
    with guardian function on decorated function

    :param tuple or string params: guarded function parameter/s
    :param function guardian: verifying the conditions for the selected parameter
    :param Exception error_class: raised class when guardian return false
    :param string message: error message
    '''
    params = [params] if isinstance(params, string_types) else params

    def guard_decorate(f):
        @wraps(f)
        def _guard_decorate(*args, **kwargs):
            if guardian(**_params(f, args, kwargs, params)):
                return f(*args, **kwargs)
            else:
                raise error_class(message)
        return _guard_decorate
    return guard_decorate
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

class Trie:

    @staticmethod
    def find(trie, key, depth=0):
        """
        Args:
            <dict> trie
            <str> key
            <int> depth
        Return:
            <float> value
        """
        if depth == len(key):
            return trie.get('v')
        else:
            node = trie.get(key[depth])
            if node:
                return Trie.find(node, key, depth + 1)

    @staticmethod
    def find_partial(trie, key, depth=0):
        """
        Args:
            <dict> trie
            <str> key
            <int> depth
        Return:
            <dict> found_trie
        """
        while depth < len(key):
            trie = trie.get(key[depth])
            if not trie:
                return
            depth += 1
        return trie

    @staticmethod
    def insert(trie, key, val, depth=0):
        """
        Args:
            <dict> trie
            <str> key
            <float> val
            <int> depth
        Return:
            <dict> trie
        """
        if depth < len(key):
            trie[key[depth]] = Trie.insert(trie.get(key[depth], {}), key, val,
                                           depth + 1)
        else:
            trie['v'] = val
        return trie

    @staticmethod
    def inner_prod(trie1, trie2):
        """
        Args:
            <dict> trie1
            <dict> trie2
        Return:
            <float> inner_product
        """
        res = 0.
        for key in trie1:
            if key == "v":
                res += trie1['v'] * trie2.get('v', 0)
            elif key in trie2:
                res += Trie.inner_prod(trie1[key], trie2[key])
        return res

    @staticmethod
    def add_coef(trie1, trie2, coef, _def=0.0):
        """calc trie1 + trie2 * coef
        Args:
            <dict> trie1
            <dict> trie2
            <float> coef
            <float> _def: default value
        Return:
            <dict> trie1
        """
        for key in trie2:
            if key == "v":
                trie1['v'] = trie1.get('v', _def) + trie2['v'] * coef
            else:
                trie1[key] = Trie.add_coef(trie1.get(key, {}), trie2[key],
                                           coef, _def)
        return trie1

    @staticmethod
    def mult(trie1, trie2):
        """calc trie1 * trie 2 (element wise multiplication)
        Args:
            <dict> trie1
            <dict> trie2
        Return:
            <dict> trie1
        """
        for key in trie2:
            if key in trie1:
                if key == "v":
                    trie1['v'] *= trie2['v']
                else:
                    trie1[key] = Trie.mult(trie1[key], trie2[key])
        return trie1

    @staticmethod
    def toString(trie, path=[]):
        """
        Args:
            <dict> trie
            <list> path
        Return:
            <str> res
        """
        res = ""
        for key in trie:
            if key == "v":
                res += "%s\t%s\n" % (" ".join(path), trie['v'])
            else:
                res += Trie.toString(trie[key], path + [key])
        return res

    @staticmethod
    def each(trie, callback, path=[], *args):
        """calls the callback function (with path, value and more)
        for each pair of [key, value] in this trie
        Args:
            <dict> trie1
            <function> callback
            <list> path
        """
        for key in trie:
            if key == "v":
                callback(path, trie['v'], *args)
            else:
                Trie.each(trie[key], callback, path + [key], *args)
<commit_msg>Improve Trie<commit_after># -*- coding: utf-8 -*-

class Trie:

    @staticmethod
    def find(trie, key, depth=0):
        """
        Args:
            <dict> trie
            <str> key
            <int> depth
        Return:
            <float> value
        """
        if depth == len(key):
            return trie.get("v")
        elif key[depth] in trie:
            return Trie.find(trie[key[depth]], key, depth + 1)

    @staticmethod
    def find_partial(trie, key, depth=0):
        """
        Args:
            <dict> trie
            <str> key
            <int> depth
        Return:
            <dict> found_trie
        """
        while depth < len(key):
            if key[depth] not in trie:
                return
            trie = trie[key[depth]]
            depth += 1
        return trie

    @staticmethod
    def insert(trie, key, val, depth=0, key_length=0):
        """
        Args:
            <dict> trie
            <str> key
            <float> val
            <int> depth
            <int> length
        Return:
            <dict> trie
        """
        if key_length == 0:
            key_length = len(key)
        if depth < key_length:
            trie[key[depth]] = Trie.insert(trie.get(key[depth], {}), key, val,
                                           depth + 1, key_length)
        else:
            trie['v'] = val
        return trie

    @staticmethod
    def inner_prod(trie1, trie2):
        """
        Args:
            <dict> trie1
            <dict> trie2
        Return:
            <float> inner_product
        """
        res = 0.
        for key in trie1:
            if key in trie2:
                if key == "v":
                    res += trie1["v"] * trie2["v"]
                else:
                    res += Trie.inner_prod(trie1[key], trie2[key])
        return res

    @staticmethod
    def add_coef(trie1, trie2, coef, _def=0.0):
        """calc trie1 + trie2 * coef
        Args:
            <dict> trie1
            <dict> trie2
            <float> coef
            <float> _def: default value
        Return:
            <dict> trie1
        """
        for key in trie2:
            if key == "v":
                trie1['v'] = trie1.get('v', _def) + trie2['v'] * coef
            else:
                trie1[key] = Trie.add_coef(trie1.get(key, {}), trie2[key],
                                           coef, _def)
        return trie1

    @staticmethod
    def mult(trie1, trie2):
        """calc trie1 * trie 2 (element wise multiplication)
        Args:
            <dict> trie1
            <dict> trie2
        Return:
            <dict> trie1
        """
        for key in trie2:
            if key in trie1:
                if key == "v":
                    trie1['v'] *= trie2['v']
                else:
                    trie1[key] = Trie.mult(trie1[key], trie2[key])
        return trie1

    @staticmethod
    def toString(trie, path=[]):
        """
        Args:
            <dict> trie
            <list> path
        Return:
            <str> res
        """
        res = ""
        for key in trie:
            if key == "v":
                res += "%s\t%s\n" % (" ".join(path), trie['v'])
            else:
                res += Trie.toString(trie[key], path + [key])
        return res

    @staticmethod
    def each(trie, callback, path=[], *args):
        """calls the callback function (with path, value and more)
        for each pair of [key, value] in this trie
        Args:
            <dict> trie1
            <function> callback
            <list> path
        """
        for key in trie:
            if key == "v":
                callback(path, trie['v'], *args)
            else:
                Trie.each(trie[key], callback, path + [key], *args)
<|endoftext|>


<commit_before>"""Sphinx ReadTheDocs theme.

From https://github.com/ryan-roemer/sphinx-bootstrap-theme.

"""
import os

VERSION = (0, 1, 5)

__version__ = ".".join(str(v) for v in VERSION)
__version_full__ = __version__

def get_html_theme_path():
    """Return list of HTML theme paths."""
    cur_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
    return cur_dir
<commit_msg>Delete __init__.py<commit_after><|endoftext|>


<commit_before>import numpy as np
import pandas as pd
import multiprocessing
from multiprocessing import Pool

from utils.preprocessing import one_hot_encoding
from utils.preprocessing import get_weekday

def process_user_session(user):
    user_session = sessions.loc[sessions['user_id'] == user]
    # Get the user session
    user_session_data = pd.Series()

    # Length of the session
    user_session_data['session_lenght'] = len(user_session)
    user_session_data['id'] = user

    suffix = '_secs_elapsed'

    for column in ['action', 'action_type', 'action_detail', 'device_type']:
        column_data = user_session.groupby(column).secs_elapsed.sum()
        column_data.rename(lambda x: x + suffix, inplace=True)
        user_session_data = user_session_data.append(column_data)

    # Get the most used device
    user_session_data['most_used_device'] = user_session['device_type'].max()

    return user_session_data.groupby(level=0).sum()

def process_user_secs_elapsed(user):
    user_secs = sessions.loc[sessions['user_id'] == user, 'secs_elapsed']
    user_processed_secs = pd.Series()
    user_processed_secs['id'] = user

    user_processed_secs['secs_elapsed_sum'] = user_secs.sum()
    user_processed_secs['secs_elapsed_mean'] = user_secs.mean()
    user_processed_secs['secs_elapsed_min'] = user_secs.min()
    user_processed_secs['secs_elapsed_max'] = user_secs.max()
    user_processed_secs['secs_elapsed_quantile_1'] = user_secs.quantile(0.25)
    user_processed_secs['secs_elapsed_quantile_3'] = user_secs.quantile(0.75)
    user_processed_secs['secs_elapsed_median'] = user_secs.median()
    user_processed_secs['secs_elapsed_std'] = user_secs.std()
    user_processed_secs['secs_elapsed_var'] = user_secs.var()
    user_processed_secs['secs_elapsed_skew'] = user_secs.skew()

    return user_processed_secs

raw_data_path = '../data/raw/'
processed_data_path = '../data/processed/'

global sessions

# Load raw data
train_users = pd.read_csv(raw_data_path + 'train_users.csv')
test_users = pd.read_csv(raw_data_path + 'test_users.csv')
sessions = pd.read_csv(raw_data_path + 'sessions.csv')

# Join users
users = pd.concat((train_users, test_users), axis=0, ignore_index=True)

# Drop date_first_booking column (empty since competition's restart)
users = users.drop('date_first_booking', axis=1)

# Replace NaNs
users['gender'].replace('-unknown-', np.nan, inplace=True)
users['language'].replace('-unknown-', np.nan, inplace=True)
sessions.replace('-unknown-', np.nan, inplace=True)

# Remove weird age values
users.loc[users['age'] > 100, 'age'] = np.nan
users.loc[users['age'] < 14, 'age'] = np.nan

# Change type to date
users['date_account_created'] = pd.to_datetime(users['date_account_created'])
users['date_first_active'] = pd.to_datetime(users['timestamp_first_active'],
                                            format='%Y%m%d%H%M%S')

users['weekday_account_created'] = users[
    'date_account_created'].apply(get_weekday)
users['weekday_first_active'] = users['date_first_active'].apply(get_weekday)

# Split dates into day, month, year
year_account_created = pd.DatetimeIndex(users['date_account_created']).year
users['year_account_created'] = year_account_created
month_account_created = pd.DatetimeIndex(users['date_account_created']).month
users['month_account_created'] = month_account_created
day_account_created = pd.DatetimeIndex(users['date_account_created']).day
users['day_account_created'] = day_account_created
year_first_active = pd.DatetimeIndex(users['date_first_active']).year
users['year_first_active'] = year_first_active
month_first_active = pd.DatetimeIndex(users['date_first_active']).month
users['month_first_active'] = month_first_active
day_first_active = pd.DatetimeIndex(users['date_first_active']).day
users['day_first_active'] = day_first_active

p = Pool(multiprocessing.cpu_count())
processed_sessions = p.map(process_user_session, sessions['user_id'].unique())
user_sessions = pd.DataFrame(processed_sessions).set_index('id')

# Joint the processed data with each user
users = users.set_index('id')
users = pd.concat([users, user_sessions], axis=1)

# TODO: Classify by dispositive

# Get the count of general session information
user_sessions = sessions.groupby('user_id').count()
user_sessions.rename(columns=lambda x: x + '_count', inplace=True)
users = pd.concat([users, user_sessions], axis=1)

p = Pool(multiprocessing.cpu_count())
processed_secs_elapsed = p.map(
    process_user_secs_elapsed, sessions['user_id'].unique())
processed_secs_elapsed = pd.DataFrame(processed_secs_elapsed).set_index('id')

users = pd.concat([users, processed_secs_elapsed], axis=1)

train_users = train_users.set_index('id')
test_users = test_users.set_index('id')

users.index.name = 'id'
processed_train_users = users.loc[train_users.index]
processed_test_users = users.loc[test_users.index]
processed_test_users.drop(['country_destination'], inplace=True, axis=1)

processed_train_users.to_csv(processed_data_path + 'processed_train_users.csv')
processed_test_users.to_csv(processed_data_path + 'processed_test_users.csv')

drop_list = [
    'date_account_created',
    'date_first_active',
    'timestamp_first_active'
]

# Drop columns
users = users.drop(drop_list, axis=1)

# TODO: Try with StandardScaler
# from sklearn.preprocessing import StandardScaler
# scaler = StandardScaler()
# scaler.fit_transform(users)

# Encode categorical features
categorical_features = [
    'gender', 'signup_method', 'signup_flow', 'language', 'affiliate_channel',
    'affiliate_provider', 'first_affiliate_tracked', 'signup_app',
    'first_device_type', 'first_browser', 'most_used_device'
]

users = one_hot_encoding(users, categorical_features)

processed_train_users = users.loc[train_users.index]
processed_test_users = users.loc[test_users.index]
processed_test_users.drop('country_destination', inplace=True, axis=1)

processed_train_users.to_csv(processed_data_path + 'encoded_train_users.csv')
processed_test_users.to_csv(processed_data_path + 'encoded_test_users.csv')
<commit_msg>Update preprocessing script<commit_after>import numpy as np
import pandas as pd
import multiprocessing
from multiprocessing import Pool

from utils.preprocessing import one_hot_encoding
from utils.preprocessing import get_weekday

def process_user_actions(user):
    user_session = sessions.loc[sessions['user_id'] == user]
    # Get the user session
    user_session_data = pd.Series()

    # Length of the session
    user_session_data['session_lenght'] = len(user_session)
    user_session_data['id'] = user

    for column in ['action', 'action_type', 'action_detail', 'device_type']:
        column_data = user_session[column].value_counts()
        column_data.index = column_data.index + '_count'
        user_session_data = user_session_data.append(column_data)

    # Get the most used device
    user_session_data['most_used_device'] = user_session['device_type'].max()

    return user_session_data.groupby(user_session_data.index).sum()

def process_user_secs_elapsed(user):
    user_secs = sessions.loc[sessions['user_id'] == user, 'secs_elapsed']
    user_processed_secs = pd.Series()
    user_processed_secs['id'] = user

    user_processed_secs['secs_elapsed_sum'] = user_secs.sum()
    user_processed_secs['secs_elapsed_mean'] = user_secs.mean()
    user_processed_secs['secs_elapsed_min'] = user_secs.min()
    user_processed_secs['secs_elapsed_max'] = user_secs.max()
    user_processed_secs['secs_elapsed_quantile_1'] = user_secs.quantile(0.25)
    user_processed_secs['secs_elapsed_quantile_3'] = user_secs.quantile(0.75)
    user_processed_secs['secs_elapsed_median'] = user_secs.median()
    user_processed_secs['secs_elapsed_std'] = user_secs.std()
    user_processed_secs['secs_elapsed_var'] = user_secs.var()
    user_processed_secs['secs_elapsed_skew'] = user_secs.skew()

    return user_processed_secs

raw_data_path = '../data/raw/'
processed_data_path = '../data/processed/'

# Load raw data
train_users = pd.read_csv(raw_data_path + 'train_users.csv')
test_users = pd.read_csv(raw_data_path + 'test_users.csv')
sessions = pd.read_csv(raw_data_path + 'sessions.csv', nrows=2000000)

# Join users
users = pd.concat((train_users, test_users), axis=0, ignore_index=True)
users = users.set_index('id')

# Drop date_first_booking column (empty since competition's restart)
users = users.drop('date_first_booking', axis=1)

# Replace NaNs
users['gender'].replace('-unknown-', np.nan, inplace=True)
users['language'].replace('-unknown-', np.nan, inplace=True)
sessions.replace('-unknown-', np.nan, inplace=True)

# Remove weird age values
users.loc[users['age'] > 100, 'age'] = np.nan
users.loc[users['age'] < 14, 'age'] = np.nan

# Change type to date
users['date_account_created'] = pd.to_datetime(users['date_account_created'])
users['date_first_active'] = pd.to_datetime(users['timestamp_first_active'],
                                            format='%Y%m%d%H%M%S')

users['weekday_account_created'] = users[
    'date_account_created'].apply(get_weekday)
users['weekday_first_active'] = users['date_first_active'].apply(get_weekday)

# Split dates into day, month, year
year_account_created = pd.DatetimeIndex(users['date_account_created']).year
users['year_account_created'] = year_account_created
month_account_created = pd.DatetimeIndex(users['date_account_created']).month
users['month_account_created'] = month_account_created
day_account_created = pd.DatetimeIndex(users['date_account_created']).day
users['day_account_created'] = day_account_created
year_first_active = pd.DatetimeIndex(users['date_first_active']).year
users['year_first_active'] = year_first_active
month_first_active = pd.DatetimeIndex(users['date_first_active']).month
users['month_first_active'] = month_first_active
day_first_active = pd.DatetimeIndex(users['date_first_active']).day
users['day_first_active'] = day_first_active

p = Pool(multiprocessing.cpu_count())
result = p.map(process_user_actions, sessions['user_id'].unique())
result = pd.DataFrame(result).set_index('id')
users = pd.concat([users, result], axis=1)

print 'user_actions'

# TODO: Classify by dispositive

# Get the count of general session information
result = sessions.groupby('user_id').count()
result.rename(columns=lambda x: x + '_count', inplace=True)
users = pd.concat([users, result], axis=1)

p = Pool(multiprocessing.cpu_count())
result = p.map(process_user_secs_elapsed, sessions['user_id'].unique())
result = pd.DataFrame(result).set_index('id')
users = pd.concat([users, result], axis=1)

print 'seconds_info'

train_users = train_users.set_index('id')
test_users = test_users.set_index('id')

users.index.name = 'id'
processed_train_users = users.loc[train_users.index]
processed_test_users = users.loc[test_users.index]
processed_test_users.drop(['country_destination'], inplace=True, axis=1)

suffix = 'count_processed_'
processed_train_users.to_csv(processed_data_path + suffix + 'train_users.csv')
processed_test_users.to_csv(processed_data_path + suffix + 'test_users.csv')

# drop_list = [
#     'date_account_created',
#     'date_first_active',
#     'timestamp_first_active'
# ]
#
# # Drop columns
# users.drop(drop_list, axis=1, inplace=True)
#
# # TODO: Try with StandardScaler
# # from sklearn.preprocessing import StandardScaler
# # scaler = StandardScaler()
# # scaler.fit_transform(users)
#
# tr_i = train_users.index
# te_i = test_users.index
#
# train_users = 0
# test_users = 0
# sessions = 0
# result = 0
# processed_train_users = 0
# processed_test_users = 0
#
# import gc
# gc.collect()
#
# # Encode categorical features
# categorical_features = [
#     'gender', 'signup_method', 'signup_flow', 'language', 'affiliate_channel',
#     'affiliate_provider', 'first_affiliate_tracked', 'signup_app',
#     'first_device_type', 'first_browser', 'most_used_device'
# ]
#
# users = one_hot_encoding(users, categorical_features)
#
# processed_train_users = users.loc[tr_i]
# processed_test_users = users.loc[te_i]
# processed_test_users.drop('country_destination', inplace=True, axis=1)
#
# suffix = 'ohe_count_processed_'
# processed_train_users.to_csv(processed_data_path + suffix + 'train_users.csv')
# processed_test_users.to_csv(processed_data_path + suffix + 'test_users.csv')
<|endoftext|>