<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# Copyright 2010 OpenStack LLC.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""Utility methods for working with WSGI servers."""

import errno
import os
import socket
import ssl
import sys
import time

import eventlet
import eventlet.wsgi
import greenlet
from oslo.config import cfg
from paste import deploy
import routes.middleware
import webob.dec
import webob.exc

from manila import exception

from manila.openstack.common import log as logging
from manila import utils

socket_opts = [
    cfg.IntOpt('backlog',
               default=4096,
               help="Number of backlog requests to configure the socket with"),
    cfg.IntOpt('tcp_keepidle',
               default=600,
               help="Sets the value of TCP_KEEPIDLE in seconds for each "
                    "server socket. Not supported on OS X."),
    cfg.StrOpt('ssl_ca_file',
               default=None,
               help="CA certificate file to use to verify "
                    "connecting clients"),
    cfg.StrOpt('ssl_cert_file',
               default=None,
               help="Certificate file to use when starting "
                    "the server securely"),
    cfg.StrOpt('ssl_key_file',
               default=None,
               help="Private key file to use when starting "
                    "the server securely"),
]

CONF = cfg.CONF
CONF.register_opts(socket_opts)

CONF = cfg.CONF
LOG = logging.getLogger(__name__)

class Server(object):
    """Server class to manage a WSGI server, serving a WSGI application."""

    default_pool_size = 1000

    def __init__(self, name, app, host=None, port=None, pool_size=None,
                 protocol=eventlet.wsgi.HttpProtocol):
        """Initialize, but do not start, a WSGI server.

        :param name: Pretty name for logging.
        :param app: The WSGI application to serve.
        :param host: IP address to serve the application.
        :param port: Port number to server the application.
        :param pool_size: Maximum number of eventlets to spawn concurrently.
        :returns: None

        """
        self.name = name
        self.app = app
        self._host = host or "0.0.0.0"
        self._port = port or 0
        self._server = None
        self._socket = None
        self._protocol = protocol
        self._pool = eventlet.GreenPool(pool_size or self.default_pool_size)
        self._logger = logging.getLogger("eventlet.wsgi.server")
        self._wsgi_logger = logging.WritableLogger(self._logger)

    def _get_socket(self, host, port, backlog):
        bind_addr = (host, port)
        # TODO(dims): eventlet's green dns/socket module does not actually
        # support IPv6 in getaddrinfo(). We need to get around this in the
        # future or monitor upstream for a fix
        try:
            info = socket.getaddrinfo(bind_addr[0],
                                      bind_addr[1],
                                      socket.AF_UNSPEC,
                                      socket.SOCK_STREAM)[0]
            family = info[0]
            bind_addr = info[-1]
        except Exception:
            family = socket.AF_INET

        cert_file = CONF.ssl_cert_file
        key_file = CONF.ssl_key_file
        ca_file = CONF.ssl_ca_file
        use_ssl = cert_file or key_file

        if cert_file and not os.path.exists(cert_file):
            raise RuntimeError(_("Unable to find cert_file : %s") % cert_file)

        if ca_file and not os.path.exists(ca_file):
            raise RuntimeError(_("Unable to find ca_file : %s") % ca_file)

        if key_file and not os.path.exists(key_file):
            raise RuntimeError(_("Unable to find key_file : %s") % key_file)

        if use_ssl and (not cert_file or not key_file):
            raise RuntimeError(_("When running server in SSL mode, you must "
                                 "specify both a cert_file and key_file "
                                 "option value in your configuration file"))

        def wrap_ssl(sock):
            ssl_kwargs = {
                'server_side': True,
                'certfile': cert_file,
                'keyfile': key_file,
                'cert_reqs': ssl.CERT_NONE,
            }

            if CONF.ssl_ca_file:
                ssl_kwargs['ca_certs'] = ca_file
                ssl_kwargs['cert_reqs'] = ssl.CERT_REQUIRED

            return ssl.wrap_socket(sock, **ssl_kwargs)

        sock = None
        retry_until = time.time() + 30
        while not sock and time.time() < retry_until:
            try:
                sock = eventlet.listen(bind_addr,
                                       backlog=backlog,
                                       family=family)
                if use_ssl:
                    sock = wrap_ssl(sock)

            except socket.error, err:
                if err.args[0] != errno.EADDRINUSE:
                    raise
                eventlet.sleep(0.1)
        if not sock:
            raise RuntimeError(_("Could not bind to %(host)s:%(port)s "
                               "after trying for 30 seconds") %
                               {'host': host, 'port': port})
        sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        # sockets can hang around forever without keepalive
        sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)

        # This option isn't available in the OS X version of eventlet
        if hasattr(socket, 'TCP_KEEPIDLE'):
            sock.setsockopt(socket.IPPROTO_TCP,
                            socket.TCP_KEEPIDLE,
                            CONF.tcp_keepidle)

        return sock

    def _start(self):
        """Run the blocking eventlet WSGI server.

        :returns: None

        """
        eventlet.wsgi.server(self._socket,
                             self.app,
                             protocol=self._protocol,
                             custom_pool=self._pool,
                             log=self._wsgi_logger)

    def start(self, backlog=128):
        """Start serving a WSGI application.

        :param backlog: Maximum number of queued connections.
        :returns: None
        :raises: manila.exception.InvalidInput

        """
        if backlog < 1:
            raise exception.InvalidInput(
                reason='The backlog must be more than 1')

        self._socket = self._get_socket(self._host,
                                        self._port,
                                        backlog=backlog)
        self._server = eventlet.spawn(self._start)
        (self._host, self._port) = self._socket.getsockname()[0:2]
        LOG.info(_("Started %(name)s on %(_host)s:%(_port)s") % self.__dict__)

    @property
    def host(self):
        return self._host

    @property
    def port(self):
        return self._port

    def stop(self):
        """Stop this server.

        This is not a very nice action, as currently the method by which a
        server is stopped is by killing its eventlet.

        :returns: None

        """
        LOG.info(_("Stopping WSGI server."))
        self._server.kill()

    def wait(self):
        """Block, until the server has stopped.

        Waits on the server's eventlet to finish, then returns.

        :returns: None

        """
        try:
            self._server.wait()
        except greenlet.GreenletExit:
            LOG.info(_("WSGI server has stopped."))

class Request(webob.Request):
    pass

class Application(object):
    """Base WSGI application wrapper. Subclasses need to implement __call__."""

    @classmethod
    def factory(cls, global_config, **local_config):
        """Used for paste app factories in paste.deploy config files.

        Any local configuration (that is, values under the [app:APPNAME]
        section of the paste config) will be passed into the `__init__` method
        as kwargs.

        A hypothetical configuration would look like:

            [app:wadl]
            latest_version = 1.3
            paste.app_factory = manila.api.fancy_api:Wadl.factory

        which would result in a call to the `Wadl` class as

            import manila.api.fancy_api
            fancy_api.Wadl(latest_version='1.3')

        You could of course re-implement the `factory` method in subclasses,
        but using the kwarg passing it shouldn't be necessary.

        """
        return cls(**local_config)

    def __call__(self, environ, start_response):
        r"""Subclasses will probably want to implement __call__ like this:

        @webob.dec.wsgify(RequestClass=Request)
        def __call__(self, req):
          # Any of the following objects work as responses:

          # Option 1: simple string
          res = 'message\n'

          # Option 2: a nicely formatted HTTP exception page
          res = exc.HTTPForbidden(detail='Nice try')

          # Option 3: a webob Response object (in case you need to play with
          # headers, or you want to be treated like an iterable, or or or)
          res = Response();
          res.app_iter = open('somefile')

          # Option 4: any wsgi app to be run next
          res = self.application

          # Option 5: you can get a Response object for a wsgi app, too, to
          # play with headers etc
          res = req.get_response(self.application)

          # You can then just return your response...
          return res
          # ... or set req.response and return None.
          req.response = res

        See the end of http://pythonpaste.org/webob/modules/dec.html
        for more info.

        """
        raise NotImplementedError(_('You must implement __call__'))

class Middleware(Application):
    """Base WSGI middleware.

    These classes require an application to be
    initialized that will be called next.  By default the middleware will
    simply call its wrapped app, or you can override __call__ to customize its
    behavior.

    """

    @classmethod
    def factory(cls, global_config, **local_config):
        """Used for paste app factories in paste.deploy config files.

        Any local configuration (that is, values under the [filter:APPNAME]
        section of the paste config) will be passed into the `__init__` method
        as kwargs.

        A hypothetical configuration would look like:

            [filter:analytics]
            redis_host = 127.0.0.1
            paste.filter_factory = manila.api.analytics:Analytics.factory

        which would result in a call to the `Analytics` class as

            import manila.api.analytics
            analytics.Analytics(app_from_paste, redis_host='127.0.0.1')

        You could of course re-implement the `factory` method in subclasses,
        but using the kwarg passing it shouldn't be necessary.

        """
        def _factory(app):
            return cls(app, **local_config)
        return _factory

    def __init__(self, application):
        self.application = application

    def process_request(self, req):
        """Called on each request.

        If this returns None, the next application down the stack will be
        executed. If it returns a response then that response will be returned
        and execution will stop here.

        """
        return None

    def process_response(self, response):
        """Do whatever you'd like to the response."""
        return response

    @webob.dec.wsgify(RequestClass=Request)
    def __call__(self, req):
        response = self.process_request(req)
        if response:
            return response
        response = req.get_response(self.application)
        return self.process_response(response)

class Debug(Middleware):
    """Helper class for debugging a WSGI application.

    Can be inserted into any WSGI application chain to get information
    about the request and response.

    """

    @webob.dec.wsgify(RequestClass=Request)
    def __call__(self, req):
        print ('*' * 40) + ' REQUEST ENVIRON'
        for key, value in req.environ.items():
            print key, '=', value
        print
        resp = req.get_response(self.application)

        print ('*' * 40) + ' RESPONSE HEADERS'
        for (key, value) in resp.headers.iteritems():
            print key, '=', value
        print

        resp.app_iter = self.print_generator(resp.app_iter)

        return resp

    @staticmethod
    def print_generator(app_iter):
        """Iterator that prints the contents of a wrapper string."""
        print ('*' * 40) + ' BODY'
        for part in app_iter:
            sys.stdout.write(part)
            sys.stdout.flush()
            yield part
        print

class Router(object):
    """WSGI middleware that maps incoming requests to WSGI apps."""

    def __init__(self, mapper):
        """Create a router for the given routes.Mapper.

        Each route in `mapper` must specify a 'controller', which is a
        WSGI app to call.  You'll probably want to specify an 'action' as
        well and have your controller be an object that can route
        the request to the action-specific method.

        Examples:
          mapper = routes.Mapper()
          sc = ServerController()

          # Explicit mapping of one route to a controller+action
          mapper.connect(None, '/svrlist', controller=sc, action='list')

          # Actions are all implicitly defined
          mapper.resource('server', 'servers', controller=sc)

          # Pointing to an arbitrary WSGI app.  You can specify the
          # {path_info:.*} parameter so the target app can be handed just that
          # section of the URL.
          mapper.connect(None, '/v1.0/{path_info:.*}', controller=BlogApp())

        """
        self.map = mapper
        self._router = routes.middleware.RoutesMiddleware(self._dispatch,
                                                          self.map)

    @webob.dec.wsgify(RequestClass=Request)
    def __call__(self, req):
        """Route the incoming request to a controller based on self.map.

        If no match, return a 404.

        """
        return self._router

    @staticmethod
    @webob.dec.wsgify(RequestClass=Request)
    def _dispatch(req):
        """Dispatch the request to the appropriate controller.

        Called by self._router after matching the incoming request to a route
        and putting the information into req.environ.  Either returns 404
        or the routed WSGI app's response.

        """
        match = req.environ['wsgiorg.routing_args'][1]
        if not match:
            return webob.exc.HTTPNotFound()
        app = match['controller']
        return app

class Loader(object):
    """Used to load WSGI applications from paste configurations."""

    def __init__(self, config_path=None):
        """Initialize the loader, and attempt to find the config.

        :param config_path: Full or relative path to the paste config.
        :returns: None

        """
        config_path = config_path or CONF.api_paste_config
        self.config_path = utils.find_config(config_path)

    def load_app(self, name):
        """Return the paste URLMap wrapped WSGI application.

        :param name: Name of the application to load.
        :returns: Paste URLMap object wrapping the requested application.
        :raises: `manila.exception.PasteAppNotFound`

        """
        try:
            return deploy.loadapp("config:%s" % self.config_path, name=name)
        except LookupError as err:
            LOG.error(err)
            raise exception.PasteAppNotFound(name=name, path=self.config_path)
<commit_msg>Raise max header size to accommodate large tokens<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# Copyright 2010 OpenStack LLC.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""Utility methods for working with WSGI servers."""

import errno
import os
import socket
import ssl
import sys
import time

import eventlet
import eventlet.wsgi
import greenlet
from oslo.config import cfg
from paste import deploy
import routes.middleware
import webob.dec
import webob.exc

from manila import exception

from manila.openstack.common import log as logging
from manila import utils

socket_opts = [
    cfg.IntOpt('backlog',
               default=4096,
               help="Number of backlog requests to configure the socket with"),
    cfg.IntOpt('tcp_keepidle',
               default=600,
               help="Sets the value of TCP_KEEPIDLE in seconds for each "
                    "server socket. Not supported on OS X."),
    cfg.StrOpt('ssl_ca_file',
               default=None,
               help="CA certificate file to use to verify "
                    "connecting clients"),
    cfg.StrOpt('ssl_cert_file',
               default=None,
               help="Certificate file to use when starting "
                    "the server securely"),
    cfg.StrOpt('ssl_key_file',
               default=None,
               help="Private key file to use when starting "
                    "the server securely"),
]

eventlet_opts = [
    cfg.IntOpt('max_header_line',
               default=16384,
               help="Maximum line size of message headers to be accepted. "
                    "max_header_line may need to be increased when using "
                    "large tokens (typically those generated by the "
                    "Keystone v3 API with big service catalogs)."),
]

CONF = cfg.CONF
CONF.register_opts(socket_opts)
CONF.register_opts(eventlet_opts)

CONF = cfg.CONF
LOG = logging.getLogger(__name__)

class Server(object):
    """Server class to manage a WSGI server, serving a WSGI application."""

    default_pool_size = 1000

    def __init__(self, name, app, host=None, port=None, pool_size=None,
                 protocol=eventlet.wsgi.HttpProtocol):
        """Initialize, but do not start, a WSGI server.

        :param name: Pretty name for logging.
        :param app: The WSGI application to serve.
        :param host: IP address to serve the application.
        :param port: Port number to server the application.
        :param pool_size: Maximum number of eventlets to spawn concurrently.
        :returns: None

        """
        eventlet.wsgi.MAX_HEADER_LINE = CONF.max_header_line
        self.name = name
        self.app = app
        self._host = host or "0.0.0.0"
        self._port = port or 0
        self._server = None
        self._socket = None
        self._protocol = protocol
        self._pool = eventlet.GreenPool(pool_size or self.default_pool_size)
        self._logger = logging.getLogger("eventlet.wsgi.server")
        self._wsgi_logger = logging.WritableLogger(self._logger)

    def _get_socket(self, host, port, backlog):
        bind_addr = (host, port)
        # TODO(dims): eventlet's green dns/socket module does not actually
        # support IPv6 in getaddrinfo(). We need to get around this in the
        # future or monitor upstream for a fix
        try:
            info = socket.getaddrinfo(bind_addr[0],
                                      bind_addr[1],
                                      socket.AF_UNSPEC,
                                      socket.SOCK_STREAM)[0]
            family = info[0]
            bind_addr = info[-1]
        except Exception:
            family = socket.AF_INET

        cert_file = CONF.ssl_cert_file
        key_file = CONF.ssl_key_file
        ca_file = CONF.ssl_ca_file
        use_ssl = cert_file or key_file

        if cert_file and not os.path.exists(cert_file):
            raise RuntimeError(_("Unable to find cert_file : %s") % cert_file)

        if ca_file and not os.path.exists(ca_file):
            raise RuntimeError(_("Unable to find ca_file : %s") % ca_file)

        if key_file and not os.path.exists(key_file):
            raise RuntimeError(_("Unable to find key_file : %s") % key_file)

        if use_ssl and (not cert_file or not key_file):
            raise RuntimeError(_("When running server in SSL mode, you must "
                                 "specify both a cert_file and key_file "
                                 "option value in your configuration file"))

        def wrap_ssl(sock):
            ssl_kwargs = {
                'server_side': True,
                'certfile': cert_file,
                'keyfile': key_file,
                'cert_reqs': ssl.CERT_NONE,
            }

            if CONF.ssl_ca_file:
                ssl_kwargs['ca_certs'] = ca_file
                ssl_kwargs['cert_reqs'] = ssl.CERT_REQUIRED

            return ssl.wrap_socket(sock, **ssl_kwargs)

        sock = None
        retry_until = time.time() + 30
        while not sock and time.time() < retry_until:
            try:
                sock = eventlet.listen(bind_addr,
                                       backlog=backlog,
                                       family=family)
                if use_ssl:
                    sock = wrap_ssl(sock)

            except socket.error, err:
                if err.args[0] != errno.EADDRINUSE:
                    raise
                eventlet.sleep(0.1)
        if not sock:
            raise RuntimeError(_("Could not bind to %(host)s:%(port)s "
                               "after trying for 30 seconds") %
                               {'host': host, 'port': port})
        sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        # sockets can hang around forever without keepalive
        sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)

        # This option isn't available in the OS X version of eventlet
        if hasattr(socket, 'TCP_KEEPIDLE'):
            sock.setsockopt(socket.IPPROTO_TCP,
                            socket.TCP_KEEPIDLE,
                            CONF.tcp_keepidle)

        return sock

    def _start(self):
        """Run the blocking eventlet WSGI server.

        :returns: None

        """
        eventlet.wsgi.server(self._socket,
                             self.app,
                             protocol=self._protocol,
                             custom_pool=self._pool,
                             log=self._wsgi_logger)

    def start(self, backlog=128):
        """Start serving a WSGI application.

        :param backlog: Maximum number of queued connections.
        :returns: None
        :raises: manila.exception.InvalidInput

        """
        if backlog < 1:
            raise exception.InvalidInput(
                reason='The backlog must be more than 1')

        self._socket = self._get_socket(self._host,
                                        self._port,
                                        backlog=backlog)
        self._server = eventlet.spawn(self._start)
        (self._host, self._port) = self._socket.getsockname()[0:2]
        LOG.info(_("Started %(name)s on %(_host)s:%(_port)s") % self.__dict__)

    @property
    def host(self):
        return self._host

    @property
    def port(self):
        return self._port

    def stop(self):
        """Stop this server.

        This is not a very nice action, as currently the method by which a
        server is stopped is by killing its eventlet.

        :returns: None

        """
        LOG.info(_("Stopping WSGI server."))
        self._server.kill()

    def wait(self):
        """Block, until the server has stopped.

        Waits on the server's eventlet to finish, then returns.

        :returns: None

        """
        try:
            self._server.wait()
        except greenlet.GreenletExit:
            LOG.info(_("WSGI server has stopped."))

class Request(webob.Request):
    pass

class Application(object):
    """Base WSGI application wrapper. Subclasses need to implement __call__."""

    @classmethod
    def factory(cls, global_config, **local_config):
        """Used for paste app factories in paste.deploy config files.

        Any local configuration (that is, values under the [app:APPNAME]
        section of the paste config) will be passed into the `__init__` method
        as kwargs.

        A hypothetical configuration would look like:

            [app:wadl]
            latest_version = 1.3
            paste.app_factory = manila.api.fancy_api:Wadl.factory

        which would result in a call to the `Wadl` class as

            import manila.api.fancy_api
            fancy_api.Wadl(latest_version='1.3')

        You could of course re-implement the `factory` method in subclasses,
        but using the kwarg passing it shouldn't be necessary.

        """
        return cls(**local_config)

    def __call__(self, environ, start_response):
        r"""Subclasses will probably want to implement __call__ like this:

        @webob.dec.wsgify(RequestClass=Request)
        def __call__(self, req):
          # Any of the following objects work as responses:

          # Option 1: simple string
          res = 'message\n'

          # Option 2: a nicely formatted HTTP exception page
          res = exc.HTTPForbidden(detail='Nice try')

          # Option 3: a webob Response object (in case you need to play with
          # headers, or you want to be treated like an iterable, or or or)
          res = Response();
          res.app_iter = open('somefile')

          # Option 4: any wsgi app to be run next
          res = self.application

          # Option 5: you can get a Response object for a wsgi app, too, to
          # play with headers etc
          res = req.get_response(self.application)

          # You can then just return your response...
          return res
          # ... or set req.response and return None.
          req.response = res

        See the end of http://pythonpaste.org/webob/modules/dec.html
        for more info.

        """
        raise NotImplementedError(_('You must implement __call__'))

class Middleware(Application):
    """Base WSGI middleware.

    These classes require an application to be
    initialized that will be called next.  By default the middleware will
    simply call its wrapped app, or you can override __call__ to customize its
    behavior.

    """

    @classmethod
    def factory(cls, global_config, **local_config):
        """Used for paste app factories in paste.deploy config files.

        Any local configuration (that is, values under the [filter:APPNAME]
        section of the paste config) will be passed into the `__init__` method
        as kwargs.

        A hypothetical configuration would look like:

            [filter:analytics]
            redis_host = 127.0.0.1
            paste.filter_factory = manila.api.analytics:Analytics.factory

        which would result in a call to the `Analytics` class as

            import manila.api.analytics
            analytics.Analytics(app_from_paste, redis_host='127.0.0.1')

        You could of course re-implement the `factory` method in subclasses,
        but using the kwarg passing it shouldn't be necessary.

        """
        def _factory(app):
            return cls(app, **local_config)
        return _factory

    def __init__(self, application):
        self.application = application

    def process_request(self, req):
        """Called on each request.

        If this returns None, the next application down the stack will be
        executed. If it returns a response then that response will be returned
        and execution will stop here.

        """
        return None

    def process_response(self, response):
        """Do whatever you'd like to the response."""
        return response

    @webob.dec.wsgify(RequestClass=Request)
    def __call__(self, req):
        response = self.process_request(req)
        if response:
            return response
        response = req.get_response(self.application)
        return self.process_response(response)

class Debug(Middleware):
    """Helper class for debugging a WSGI application.

    Can be inserted into any WSGI application chain to get information
    about the request and response.

    """

    @webob.dec.wsgify(RequestClass=Request)
    def __call__(self, req):
        print ('*' * 40) + ' REQUEST ENVIRON'
        for key, value in req.environ.items():
            print key, '=', value
        print
        resp = req.get_response(self.application)

        print ('*' * 40) + ' RESPONSE HEADERS'
        for (key, value) in resp.headers.iteritems():
            print key, '=', value
        print

        resp.app_iter = self.print_generator(resp.app_iter)

        return resp

    @staticmethod
    def print_generator(app_iter):
        """Iterator that prints the contents of a wrapper string."""
        print ('*' * 40) + ' BODY'
        for part in app_iter:
            sys.stdout.write(part)
            sys.stdout.flush()
            yield part
        print

class Router(object):
    """WSGI middleware that maps incoming requests to WSGI apps."""

    def __init__(self, mapper):
        """Create a router for the given routes.Mapper.

        Each route in `mapper` must specify a 'controller', which is a
        WSGI app to call.  You'll probably want to specify an 'action' as
        well and have your controller be an object that can route
        the request to the action-specific method.

        Examples:
          mapper = routes.Mapper()
          sc = ServerController()

          # Explicit mapping of one route to a controller+action
          mapper.connect(None, '/svrlist', controller=sc, action='list')

          # Actions are all implicitly defined
          mapper.resource('server', 'servers', controller=sc)

          # Pointing to an arbitrary WSGI app.  You can specify the
          # {path_info:.*} parameter so the target app can be handed just that
          # section of the URL.
          mapper.connect(None, '/v1.0/{path_info:.*}', controller=BlogApp())

        """
        self.map = mapper
        self._router = routes.middleware.RoutesMiddleware(self._dispatch,
                                                          self.map)

    @webob.dec.wsgify(RequestClass=Request)
    def __call__(self, req):
        """Route the incoming request to a controller based on self.map.

        If no match, return a 404.

        """
        return self._router

    @staticmethod
    @webob.dec.wsgify(RequestClass=Request)
    def _dispatch(req):
        """Dispatch the request to the appropriate controller.

        Called by self._router after matching the incoming request to a route
        and putting the information into req.environ.  Either returns 404
        or the routed WSGI app's response.

        """
        match = req.environ['wsgiorg.routing_args'][1]
        if not match:
            return webob.exc.HTTPNotFound()
        app = match['controller']
        return app

class Loader(object):
    """Used to load WSGI applications from paste configurations."""

    def __init__(self, config_path=None):
        """Initialize the loader, and attempt to find the config.

        :param config_path: Full or relative path to the paste config.
        :returns: None

        """
        config_path = config_path or CONF.api_paste_config
        self.config_path = utils.find_config(config_path)

    def load_app(self, name):
        """Return the paste URLMap wrapped WSGI application.

        :param name: Name of the application to load.
        :returns: Paste URLMap object wrapping the requested application.
        :raises: `manila.exception.PasteAppNotFound`

        """
        try:
            return deploy.loadapp("config:%s" % self.config_path, name=name)
        except LookupError as err:
            LOG.error(err)
            raise exception.PasteAppNotFound(name=name, path=self.config_path)
<|endoftext|>


<commit_before>"""RestSession class for creating 'connections' to the Cisco Spark APIs."""

import urlparse
import requests
from .exceptions import ciscosparkapiException, SparkApiError
from copy import deepcopy
from datetime import datetime
from ciscosparkapi.helperfunc import sparkISO8601, utf8

# Default api.ciscospark.com base URL
DEFAULT_API_URL = 'https://api.ciscospark.com/v1/'

# Cisco Spark cloud Expected Response Codes (HTTP Response Codes)
ERC = {
    'GET': 200,
    'POST': 200,
    'PUT': 200,
    'DELETE': 204
}

# Spark API responds with:
# Response Code [429] - The number of allowed requests is exceeded.
# Please try your request later
_API_THROTTLE_STATUS_CODE = 429

# if API throttling occurs, how much time to wait by default?
# given in the n-th Fibonacci number (9th = 34s)
_DEFAULT_BACKOFF = 9

def _validate_base_url(base_url):
    parsed_url = urlparse.urlparse(base_url)
    if parsed_url.scheme and parsed_url.netloc:
        return parsed_url.geturl()
    else:
        error_message = "base_url must contain a valid scheme (protocol " \
                        "specifier) and network location (hostname)"
        raise ciscosparkapiException(error_message)

def _process_args(what, apiattr, args):
    """ make sure the arguments have the proper type / encoding
        also move args in the apiattr into a separate dict
        which will be passed as the 'json' parameter to the request
    """

    data = dict()
    for k, v in args.items():
        if isinstance(v, basestring):
            args[k] = utf8(v)
        elif isinstance(v, datetime):
            args[k] = sparkISO8601(v)
        else:
            pass
        if k in apiattr:
            v = args.pop(k)
            data[k] = v

    if len(data) > 0:
        if what in ['POST', 'PUT']:
            args['json'] = data
        elif what in ['GET', 'DELETE']:
            args['params'] = data
        else:
            raise ValueError("Unexpected method %r" % what)
    return args

def _extract_and_parse_json(response):
    return response.json()

def _fib():
    """a Fibonacci generator"""
    a, b = 0, 1
    while True:
        yield a
        a, b = b, a + b

def fib(n):
    """give the n-th Fibonacci number"""
    f = _fib()
    counter = 0
    for i in f:
        if counter < n:
            counter += 1
        else:
            break
    return i

class RestSession(object):

    def __init__(self, access_token, base_url=DEFAULT_API_URL, timeout=None):
        super(RestSession, self).__init__()
        self._base_url = _validate_base_url(base_url)
        self._access_token = access_token
        self._req_session = requests.session()
        self._timeout = None
        self._last_response = None
        self._ratelimit_callback = None
        self._ratelimit_step = _DEFAULT_BACKOFF
        self.update_headers({'Authorization': 'Bearer ' + access_token,
                             'Content-type': 'application/json;charset=utf-8'})
        self.timeout = timeout

    def _req_wrapper(self, method, url, erc, apiattr, **kwargs):
        """ this wraps the actual request. If it gets throttled (429)
            then there's multiple options:
            - no callback defined: it just errors out with ciscosparkapiException
            - if the callback is defined: calls callback with suggested cool down time

            if the callback returns True then the request is attempted again
            if the callback returns False then a ciscosparkapiException is raised

            There's two ways to back off:
            - server sends 'Retry-After' header, we can use that time
            - if no Retry-After is sent, we apply a Fibonacci sequence and increase
              the wait. The class remembers the last 'step' and for every successful
              response the step is decreased until it reaches the _DEFAULT_BACKOFF step

            The 'sleep time' is reported to the callback (if configured).
        """

        # make the response code a list if it's just an int
        if isinstance(erc, int):
            ercList = list((erc,))
        elif: isinstance(erc, list):
            pass
        else
            raise TypeError('unexpected response code type <%r>' % erc)

        # if 429 (API throttling) is in list, remove it
        if _API_THROTTLE_STATUS_CODE in ercList:
            ercList.remove(_API_THROTTLE_STATUS_CODE)

        # ensure proper encoding and parameter handling
        kwargs = _process_args(method, apiattr, kwargs)

        done = False
        while not done:
            done = True
            r = self._req_session.request(method, url, **kwargs)
            # was rate limiting in effect?
            if r.status_code == _API_THROTTLE_STATUS_CODE:
                # does the server respond with a rate-limit header?
                retry_after = int(r.headers.get('Retry-After', 0))
                if retry_after > 0:
                    sleep_time = retry_after
                    self._ratelimit_step = _DEFAULT_BACKOFF
                else:
                    sleep_time = fib(self._ratelimit_step)
                # has a callback been configured?
                # if yes, call it and see if we should try again
                if self._ratelimit_callback is not None:
                    done = not self._ratelimit_callback(sleep_time)
                    self._ratelimit_step += 1
            else:
                # no throttling: reduce the backoff step, if above threshold
                if self._ratelimit_step > _DEFAULT_BACKOFF:
                    self._ratelimit_step -= 1

        # check response code
        self._last_response = deepcopy(r)
        if not r.status_code in ercList:
            raise SparkApiError(r.status_code,
                                request=r.request,
                                response=r)
        return r

    @property
    def ratelimit_callback(self):
        """ get the API throttling callback"""
        return self._ratelimit_callback

    @ratelimit_callback.setter
    def ratelimit_callback(self, fn):
        """ set the API throttling callback"""
        self._ratelimit_callback = fn

    @property
    def last_response(self):
        """ retrieve the last response from the API"""
        return self._last_response

    @property
    def base_url(self):
        return self._base_url

    @property
    def access_token(self):
        return self._access_token

    @property
    def headers(self):
        return self._req_session.headers.copy()

    def update_headers(self, headers):
        assert isinstance(headers, dict)
        self._req_session.headers.update(headers)

    @property
    def timeout(self):
        return self._timeout

    @timeout.setter
    def timeout(self, value):
        assert value is None or value > 0
        self._timeout = value

    def urljoin(self, suffix_url):
        return urlparse.urljoin(self.base_url, suffix_url)

    def get_pages(self, url, apiattr, **kwargs):
        response = self._process('GET', url, apiattr, **kwargs)
        while True:
            # Process response - Yield page's JSON data
            yield _extract_and_parse_json(response)
            # Get next page
            if response.links.get('next'):
                next_url = response.links.get('next').get('url')
                # API request - get next page
                response = self._process('GET', next_url, apiattr, **kwargs)
            else:
                raise StopIteration

    def get_items(self, url, apiattr, **kwargs):
        # Get iterator for pages of JSON data
        pages = self.get_pages(url, apiattr, **kwargs)
        # Process pages
        for json_page in pages:
            # Process each page of JSON data yielding the individual JSON
            # objects contained within the top level 'items' array
            assert isinstance(json_page, dict)
            # sometimes there's an empty list returned
            # I guess this should not happen server side, but if there's
            # a lengthy list then the last page can have zero elements
            # and the 2nd to last page still has a 'next' link header
            items = json_page.get(u'items', None)
            if items is not None:
                for item in items:
                    yield item
            else:
                error_message = "'items' object not found in JSON data: %r" \
                                % json_page
                raise ciscosparkapiException(error_message)

    def _process(self, what, url, apiattr, **kwargs):
        # Process args
        assert isinstance(url, basestring)
        assert isinstance(apiattr, list)
        abs_url = self.urljoin(url)
        erc = kwargs.pop('erc', ERC[what])
        return self._req_wrapper(what, abs_url, erc, apiattr, **kwargs)

    def get(self, url, apiattr, **kwargs):
        return _extract_and_parse_json(self._process('GET', url, apiattr, **kwargs))

    def post(self, url, apiattr, **kwargs):
        return _extract_and_parse_json(self._process('POST', url, apiattr, **kwargs))

    def put(self, url, apiattr, **kwargs):
        return _extract_and_parse_json(self._process('PUT', url, apiattr, **kwargs))

    def delete(self, url, apiattr, **kwargs):
        return _extract_and_parse_json(self._process('DELETE', url, apiattr, **kwargs))
<commit_msg>syntax errors. grr<commit_after>"""RestSession class for creating 'connections' to the Cisco Spark APIs."""

import urlparse
import requests
from .exceptions import ciscosparkapiException, SparkApiError
from copy import deepcopy
from datetime import datetime
from ciscosparkapi.helperfunc import sparkISO8601, utf8

# Default api.ciscospark.com base URL
DEFAULT_API_URL = 'https://api.ciscospark.com/v1/'

# Cisco Spark cloud Expected Response Codes (HTTP Response Codes)
ERC = {
    'GET': 200,
    'POST': 200,
    'PUT': 200,
    'DELETE': 204
}

# Spark API responds with:
# Response Code [429] - The number of allowed requests is exceeded.
# Please try your request later
_API_THROTTLE_STATUS_CODE = 429

# if API throttling occurs, how much time to wait by default?
# given in the n-th Fibonacci number (9th = 34s)
_DEFAULT_BACKOFF = 9

def _validate_base_url(base_url):
    parsed_url = urlparse.urlparse(base_url)
    if parsed_url.scheme and parsed_url.netloc:
        return parsed_url.geturl()
    else:
        error_message = "base_url must contain a valid scheme (protocol " \
                        "specifier) and network location (hostname)"
        raise ciscosparkapiException(error_message)

def _process_args(what, apiattr, args):
    """ make sure the arguments have the proper type / encoding
        also move args in the apiattr into a separate dict
        which will be passed as the 'json' parameter to the request
    """

    data = dict()
    for k, v in args.items():
        if isinstance(v, basestring):
            args[k] = utf8(v)
        elif isinstance(v, datetime):
            args[k] = sparkISO8601(v)
        else:
            pass
        if k in apiattr:
            v = args.pop(k)
            data[k] = v

    if len(data) > 0:
        if what in ['POST', 'PUT']:
            args['json'] = data
        elif what in ['GET', 'DELETE']:
            args['params'] = data
        else:
            raise ValueError("Unexpected method %r" % what)
    return args

def _extract_and_parse_json(response):
    return response.json()

def _fib():
    """a Fibonacci generator"""
    a, b = 0, 1
    while True:
        yield a
        a, b = b, a + b

def fib(n):
    """give the n-th Fibonacci number"""
    f = _fib()
    counter = 0
    for i in f:
        if counter < n:
            counter += 1
        else:
            break
    return i

class RestSession(object):

    def __init__(self, access_token, base_url=DEFAULT_API_URL, timeout=None):
        super(RestSession, self).__init__()
        self._base_url = _validate_base_url(base_url)
        self._access_token = access_token
        self._req_session = requests.session()
        self._timeout = None
        self._last_response = None
        self._ratelimit_callback = None
        self._ratelimit_step = _DEFAULT_BACKOFF
        self.update_headers({'Authorization': 'Bearer ' + access_token,
                             'Content-type': 'application/json;charset=utf-8'})
        self.timeout = timeout

    def _req_wrapper(self, method, url, erc, apiattr, **kwargs):
        """ this wraps the actual request. If it gets throttled (429)
            then there's multiple options:
            - no callback defined: it just errors out with ciscosparkapiException
            - if the callback is defined: calls callback with suggested cool down time

            if the callback returns True then the request is attempted again
            if the callback returns False then a ciscosparkapiException is raised

            There's two ways to back off:
            - server sends 'Retry-After' header, we can use that time
            - if no Retry-After is sent, we apply a Fibonacci sequence and increase
              the wait. The class remembers the last 'step' and for every successful
              response the step is decreased until it reaches the _DEFAULT_BACKOFF step

            The 'sleep time' is reported to the callback (if configured).
        """

        # make the response code a list if it's just an int
        if isinstance(erc, int):
            ercList = list((erc,))
        elif isinstance(erc, list):
            pass
        else:
            raise TypeError('unexpected response code type <%r>' % erc)

        # if 429 (API throttling) is in list, remove it
        if _API_THROTTLE_STATUS_CODE in ercList:
            ercList.remove(_API_THROTTLE_STATUS_CODE)

        # ensure proper encoding and parameter handling
        kwargs = _process_args(method, apiattr, kwargs)

        done = False
        while not done:
            done = True
            r = self._req_session.request(method, url, **kwargs)
            # was rate limiting in effect?
            if r.status_code == _API_THROTTLE_STATUS_CODE:
                # does the server respond with a rate-limit header?
                retry_after = int(r.headers.get('Retry-After', 0))
                if retry_after > 0:
                    sleep_time = retry_after
                    self._ratelimit_step = _DEFAULT_BACKOFF
                else:
                    sleep_time = fib(self._ratelimit_step)
                # has a callback been configured?
                # if yes, call it and see if we should try again
                if self._ratelimit_callback is not None:
                    done = not self._ratelimit_callback(sleep_time)
                    self._ratelimit_step += 1
            else:
                # no throttling: reduce the backoff step, if above threshold
                if self._ratelimit_step > _DEFAULT_BACKOFF:
                    self._ratelimit_step -= 1

        # check response code
        self._last_response = deepcopy(r)
        if not r.status_code in ercList:
            raise SparkApiError(r.status_code,
                                request=r.request,
                                response=r)
        return r

    @property
    def ratelimit_callback(self):
        """ get the API throttling callback"""
        return self._ratelimit_callback

    @ratelimit_callback.setter
    def ratelimit_callback(self, fn):
        """ set the API throttling callback"""
        self._ratelimit_callback = fn

    @property
    def last_response(self):
        """ retrieve the last response from the API"""
        return self._last_response

    @property
    def base_url(self):
        return self._base_url

    @property
    def access_token(self):
        return self._access_token

    @property
    def headers(self):
        return self._req_session.headers.copy()

    def update_headers(self, headers):
        assert isinstance(headers, dict)
        self._req_session.headers.update(headers)

    @property
    def timeout(self):
        return self._timeout

    @timeout.setter
    def timeout(self, value):
        assert value is None or value > 0
        self._timeout = value

    def urljoin(self, suffix_url):
        return urlparse.urljoin(self.base_url, suffix_url)

    def get_pages(self, url, apiattr, **kwargs):
        response = self._process('GET', url, apiattr, **kwargs)
        while True:
            # Process response - Yield page's JSON data
            yield _extract_and_parse_json(response)
            # Get next page
            if response.links.get('next'):
                next_url = response.links.get('next').get('url')
                # API request - get next page
                response = self._process('GET', next_url, apiattr, **kwargs)
            else:
                raise StopIteration

    def get_items(self, url, apiattr, **kwargs):
        # Get iterator for pages of JSON data
        pages = self.get_pages(url, apiattr, **kwargs)
        # Process pages
        for json_page in pages:
            # Process each page of JSON data yielding the individual JSON
            # objects contained within the top level 'items' array
            assert isinstance(json_page, dict)
            # sometimes there's an empty list returned
            # I guess this should not happen server side, but if there's
            # a lengthy list then the last page can have zero elements
            # and the 2nd to last page still has a 'next' link header
            items = json_page.get(u'items', None)
            if items is not None:
                for item in items:
                    yield item
            else:
                error_message = "'items' object not found in JSON data: %r" \
                                % json_page
                raise ciscosparkapiException(error_message)

    def _process(self, what, url, apiattr, **kwargs):
        # Process args
        assert isinstance(url, basestring)
        assert isinstance(apiattr, list)
        abs_url = self.urljoin(url)
        erc = kwargs.pop('erc', ERC[what])
        return self._req_wrapper(what, abs_url, erc, apiattr, **kwargs)

    def get(self, url, apiattr, **kwargs):
        return _extract_and_parse_json(self._process('GET', url, apiattr, **kwargs))

    def post(self, url, apiattr, **kwargs):
        return _extract_and_parse_json(self._process('POST', url, apiattr, **kwargs))

    def put(self, url, apiattr, **kwargs):
        return _extract_and_parse_json(self._process('PUT', url, apiattr, **kwargs))

    def delete(self, url, apiattr, **kwargs):
        return _extract_and_parse_json(self._process('DELETE', url, apiattr, **kwargs))
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
Cobbler external inventory script
=================================

Ansible has a feature where instead of reading from /etc/ansible/hosts
as a text file, it can query external programs to obtain the list
of hosts, groups the hosts are in, and even variables to assign to each host.

To use this, copy this file over /etc/ansible/hosts and chmod +x the file.
This, more or less, allows you to keep one central database containing
info about all of your managed instances.

This script is an example of sourcing that data from Cobbler
(http://cobbler.github.com).  With cobbler each --mgmt-class in cobbler
will correspond to a group in Ansible, and --ks-meta variables will be
passed down for use in templates or even in argument lines.

NOTE: The cobbler system names will not be used.  Make sure a
cobbler --dns-name is set for each cobbler system.   If a system
appears with two DNS names we do not add it twice because we don't want
ansible talking to it twice.  The first one found will be used. If no
--dns-name is set the system will NOT be visible to ansible.  We do
not add cobbler system names because there is no requirement in cobbler
that those correspond to addresses.

See http://ansible.github.com/api.html for more info

Tested with Cobbler 2.0.11.

Changelog:
    - 2013-09-01 pgehres: Refactored implementation to make use of caching and to
        limit the number of connections to external cobbler server for performance.
        Added use of cobbler.ini file to configure settings. Tested with Cobbler 2.4.0
"""

# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible,
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

######################################################################

import argparse
import ConfigParser
import os
import re
from time import time
import xmlrpclib

try:
    import json
except ImportError:
    import simplejson as json

# NOTE -- this file assumes Ansible is being accessed FROM the cobbler
# server, so it does not attempt to login with a username and password.
# this will be addressed in a future version of this script.

class CobblerInventory(object):

    def __init__(self):
        """ Main execution path """
        self.conn = None

        self.inventory = dict()  # A list of groups and the hosts in that group
        self.cache = dict()  # Details about hosts in the inventory

        # Read settings and parse CLI arguments
        self.read_settings()
        self.parse_cli_args()

        # Cache
        if self.args.refresh_cache:
            self.update_cache()
        elif not self.is_cache_valid():
            self.update_cache()
        else:
            self.load_inventory_from_cache()
            self.load_cache_from_cache()

        data_to_print = ""

        # Data to print
        if self.args.host:
            data_to_print = self.get_host_info()

        elif self.args.list:
            # Display list of instances for inventory
            data_to_print = self.json_format_dict(self.inventory, True)

        else:  # default action with no options
            data_to_print = self.json_format_dict(self.inventory, True)

        print data_to_print

    def _connect(self):
        if not self.conn:
            self.conn = xmlrpclib.Server(self.cobbler_host, allow_none=True)

    def is_cache_valid(self):
        """ Determines if the cache files have expired, or if it is still valid """

        if os.path.isfile(self.cache_path_cache):
            mod_time = os.path.getmtime(self.cache_path_cache)
            current_time = time()
            if (mod_time + self.cache_max_age) > current_time:
                if os.path.isfile(self.cache_path_inventory):
                    return True

        return False

    def read_settings(self):
        """ Reads the settings from the cobbler.ini file """

        config = ConfigParser.SafeConfigParser()
        config.read(os.path.dirname(os.path.realpath(__file__)) + '/cobbler.ini')

        self.cobbler_host = config.get('cobbler', 'host')

        # Cache related
        cache_path = config.get('cobbler', 'cache_path')
        self.cache_path_cache = cache_path + "/ansible-cobbler.cache"
        self.cache_path_inventory = cache_path + "/ansible-cobbler.index"
        self.cache_max_age = config.getint('cobbler', 'cache_max_age')

    def parse_cli_args(self):
        """ Command line argument processing """

        parser = argparse.ArgumentParser(description='Produce an Ansible Inventory file based on Cobbler')
        parser.add_argument('--list', action='store_true', default=True, help='List instances (default: True)')
        parser.add_argument('--host', action='store', help='Get all the variables about a specific instance')
        parser.add_argument('--refresh-cache', action='store_true', default=False,
                            help='Force refresh of cache by making API requests to cobbler (default: False - use cache files)')
        self.args = parser.parse_args()

    def update_cache(self):
        """ Make calls to cobbler and save the output in a cache """

        self._connect()
        self.groups = dict()
        self.hosts = dict()

        data = self.conn.get_systems()

        for host in data:
            # Get the FQDN for the host and add it to the right groups
            dns_name = None
            ksmeta = None
            interfaces = host['interfaces']
            for (iname, ivalue) in interfaces.iteritems():
                if ivalue['management']:
                    this_dns_name = ivalue.get('dns_name', None)
                    if this_dns_name is not None and this_dns_name is not "":
                        dns_name = this_dns_name

            if dns_name is None:
                continue

            status = host['status']
            profile = host['profile']
            classes = host['mgmt_classes']

            if status not in self.inventory:
                self.inventory[status] = []
            self.inventory[status].append(dns_name)

            if profile not in self.inventory:
                self.inventory[profile] = []
            self.inventory[profile].append(dns_name)

            for cls in classes:
                if cls not in self.inventory:
                    self.inventory[cls] = []
                self.inventory[cls].append(dns_name)

            # Since we already have all of the data for the host, update the host details as well

            # The old way was ksmeta only -- provide backwards compatibility

            self.cache[dns_name] = dict()
            if "ks_meta" in host:
                for key, value in host["ks_meta"].iteritems():
                    self.cache[dns_name][key] = value

        self.write_to_cache(self.cache, self.cache_path_cache)
        self.write_to_cache(self.inventory, self.cache_path_inventory)

    def get_host_info(self):
        """ Get variables about a specific host """

        if not self.cache or len(self.cache) == 0:
            # Need to load index from cache
            self.load_cache_from_cache()

        if not self.args.host in self.cache:
            # try updating the cache
            self.update_cache()

            if not self.args.host in self.cache:
                # host might not exist anymore
                return self.json_format_dict({}, True)

        return self.json_format_dict(self.cache[self.args.host], True)

    def push(self, my_dict, key, element):
        """ Pushed an element onto an array that may not have been defined in the dict """

        if key in my_dict:
            my_dict[key].append(element)
        else:
            my_dict[key] = [element]

    def load_inventory_from_cache(self):
        """ Reads the index from the cache file sets self.index """

        cache = open(self.cache_path_inventory, 'r')
        json_inventory = cache.read()
        self.inventory = json.loads(json_inventory)

    def load_cache_from_cache(self):
        """ Reads the cache from the cache file sets self.cache """

        cache = open(self.cache_path_cache, 'r')
        json_cache = cache.read()
        self.cache = json.loads(json_cache)

    def write_to_cache(self, data, filename):
        """ Writes data in JSON format to a file """

        json_data = self.json_format_dict(data, True)
        cache = open(filename, 'w')
        cache.write(json_data)
        cache.close()

    def to_safe(self, word):
        """ Converts 'bad' characters in a string to underscores so they can be used as Ansible groups """

        return re.sub("[^A-Za-z0-9\-]", "_", word)

    def json_format_dict(self, data, pretty=False):
        """ Converts a dict to a JSON object and dumps it as a formatted string """

        if pretty:
            return json.dumps(data, sort_keys=True, indent=2)
        else:
            return json.dumps(data)

CobblerInventory()
<commit_msg>support for _meta retrieval<commit_after>#!/usr/bin/env python

"""
Cobbler external inventory script
=================================

Ansible has a feature where instead of reading from /etc/ansible/hosts
as a text file, it can query external programs to obtain the list
of hosts, groups the hosts are in, and even variables to assign to each host.

To use this, copy this file over /etc/ansible/hosts and chmod +x the file.
This, more or less, allows you to keep one central database containing
info about all of your managed instances.

This script is an example of sourcing that data from Cobbler
(http://cobbler.github.com).  With cobbler each --mgmt-class in cobbler
will correspond to a group in Ansible, and --ks-meta variables will be
passed down for use in templates or even in argument lines.

NOTE: The cobbler system names will not be used.  Make sure a
cobbler --dns-name is set for each cobbler system.   If a system
appears with two DNS names we do not add it twice because we don't want
ansible talking to it twice.  The first one found will be used. If no
--dns-name is set the system will NOT be visible to ansible.  We do
not add cobbler system names because there is no requirement in cobbler
that those correspond to addresses.

See http://ansible.github.com/api.html for more info

Tested with Cobbler 2.0.11.

Changelog:
    - 2015-06-21 dmccue: Heavily modified to support run-once _meta retrieval, results in
         higher performance at ansible startup.  Groups are determined by owner rather than
         default mgmt_classes.  DNS name determined from hostname.

    - 2013-09-01 pgehres: Refactored implementation to make use of caching and to
        limit the number of connections to external cobbler server for performance.
        Added use of cobbler.ini file to configure settings. Tested with Cobbler 2.4.0

"""

# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible,
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

######################################################################

import argparse
import ConfigParser
import os
import re
from time import time
import xmlrpclib

try:
    import json
except ImportError:
    import simplejson as json

# NOTE -- this file assumes Ansible is being accessed FROM the cobbler
# server, so it does not attempt to login with a username and password.
# this will be addressed in a future version of this script.

class CobblerInventory(object):

    def __init__(self):

        """ Main execution path """
        self.conn = None

        self.inventory = dict()  # A list of groups and the hosts in that group
        self.cache = dict()  # Details about hosts in the inventory

        # Read settings and parse CLI arguments
        self.read_settings()
        self.parse_cli_args()

        # Cache
        if self.args.refresh_cache:
            self.update_cache()
        elif not self.is_cache_valid():
            self.update_cache()
        else:
            self.load_inventory_from_cache()
            self.load_cache_from_cache()

        data_to_print = ""

        # Data to print
        #if self.args.host:
        #    data_to_print = self.get_host_info()

        self.inventory['_meta'] = { 'hostvars': {} }
        for hostname in self.cache:
            self.inventory['_meta']['hostvars'][hostname] = {'cobbler': self.cache[hostname] }

        data_to_print = self.json_format_dict(self.inventory, True)

        print data_to_print

    def _connect(self):
        if not self.conn:
            self.conn = xmlrpclib.Server(self.cobbler_host, allow_none=True)

    def is_cache_valid(self):
        """ Determines if the cache files have expired, or if it is still valid """

        if os.path.isfile(self.cache_path_cache):
            mod_time = os.path.getmtime(self.cache_path_cache)
            current_time = time()
            if (mod_time + self.cache_max_age) > current_time:
                if os.path.isfile(self.cache_path_inventory):
                    return True

        return False

    def read_settings(self):
        """ Reads the settings from the cobbler.ini file """

        config = ConfigParser.SafeConfigParser()
        config.read(os.path.dirname(os.path.realpath(__file__)) + '/cobbler.ini')

        self.cobbler_host = config.get('cobbler', 'host')

        # Cache related
        cache_path = config.get('cobbler', 'cache_path')
        self.cache_path_cache = cache_path + "/ansible-cobbler.cache"
        self.cache_path_inventory = cache_path + "/ansible-cobbler.index"
        self.cache_max_age = config.getint('cobbler', 'cache_max_age')

    def parse_cli_args(self):
        """ Command line argument processing """

        parser = argparse.ArgumentParser(description='Produce an Ansible Inventory file based on Cobbler')
        parser.add_argument('--list', action='store_true', default=True, help='List instances (default: True)')
        #parser.add_argument('--host', action='store', help='Get all the variables about a specific instance')
        parser.add_argument('--refresh-cache', action='store_true', default=False,
                            help='Force refresh of cache by making API requests to cobbler (default: False - use cache files)')
        self.args = parser.parse_args()

    def update_cache(self):
        """ Make calls to cobbler and save the output in a cache """

        self._connect()
        self.groups = dict()
        self.hosts = dict()

        data = self.conn.get_systems()

        for host in data:
            # Get the FQDN for the host and add it to the right groups
            dns_name = host['hostname'] #None
            ksmeta = None
            interfaces = host['interfaces']
            #for (iname, ivalue) in interfaces.iteritems():
            #    if ivalue['management']:
            #        this_dns_name = ivalue.get('dns_name', None)
            #        #this_dns_name = ivalue.get('ip_address', None)
            #        if this_dns_name is not None and this_dns_name is not "":
            #            dns_name = this_dns_name

            if dns_name is None:
                continue

            status = host['status']
            profile = host['profile']
            classes = host['owners'] #host['mgmt_classes']

            if status not in self.inventory:
                self.inventory[status] = []
            self.inventory[status].append(dns_name)

            if profile not in self.inventory:
                self.inventory[profile] = []
            self.inventory[profile].append(dns_name)

            for cls in classes:
                if cls not in self.inventory:
                    self.inventory[cls] = []
                self.inventory[cls].append(dns_name)

            # Since we already have all of the data for the host, update the host details as well

            # The old way was ksmeta only -- provide backwards compatibility

            self.cache[dns_name] = host #dict()   # sub dict with host to output json
            if "ks_meta" in host:
                for key, value in host["ks_meta"].iteritems():
                    self.cache[dns_name][key] = value

        self.write_to_cache(self.cache, self.cache_path_cache)
        self.write_to_cache(self.inventory, self.cache_path_inventory)

    def get_host_info(self):
        """ Get variables about a specific host """

        if not self.cache or len(self.cache) == 0:
            # Need to load index from cache
            self.load_cache_from_cache()

        if not self.args.host in self.cache:
            # try updating the cache
            self.update_cache()

            if not self.args.host in self.cache:
                # host might not exist anymore
                return self.json_format_dict({}, True)

        return self.json_format_dict(self.cache[self.args.host], True)

    def push(self, my_dict, key, element):
        """ Pushed an element onto an array that may not have been defined in the dict """

        if key in my_dict:
            my_dict[key].append(element)
        else:
            my_dict[key] = [element]

    def load_inventory_from_cache(self):
        """ Reads the index from the cache file sets self.index """

        cache = open(self.cache_path_inventory, 'r')
        json_inventory = cache.read()
        self.inventory = json.loads(json_inventory)

    def load_cache_from_cache(self):
        """ Reads the cache from the cache file sets self.cache """

        cache = open(self.cache_path_cache, 'r')
        json_cache = cache.read()
        self.cache = json.loads(json_cache)

    def write_to_cache(self, data, filename):
        """ Writes data in JSON format to a file """
        if data: print "DEBUG: data = " + str(data)
        json_data = self.json_format_dict(data, True)
        cache = open(filename, 'w')
        cache.write(json_data)
        cache.close()

    def to_safe(self, word):
        """ Converts 'bad' characters in a string to underscores so they can be used as Ansible groups """

        return re.sub("[^A-Za-z0-9\-]", "_", word)

    def json_format_dict(self, data, pretty=False):
        """ Converts a dict to a JSON object and dumps it as a formatted string """

        if pretty:
            return json.dumps(data, sort_keys=True, indent=2)
        else:
            return json.dumps(data)

CobblerInventory()
<|endoftext|>


<commit_before>from citrination_client.views.client import DataViewsClient

from citrination_client.models import ModelsClient
from citrination_client.search import SearchClient
from citrination_client.data import DataClient
from citrination_client.util.credentials import get_preferred_credentials

"""
Generates a lambda method in a closure such that the
client and method_name are constant (avoides reassignment
in the assignment loop in the client class)
"""

def _generate_lambda_proxy_method(client, method_name):
    subclient_m = getattr(client, method_name)
    return lambda *args, **kw: subclient_m(*args, **kw)

class CitrinationClient(object):
    """
    The top level of the client hierarchy. Instantiating this class handles
    authentication information (api_key and site) and provides access to instances of each of the sub-clients, for more specific actions.

    Instantiation requires authentication information, but that can be provided
    via direct parameterization, environment variables, or a .citrination credentials file. See the tutorial on client Initialization for more information.
    """

    def __init__(self, api_key=None, site=None, suppress_warnings=False, proxies=None):
        """
        Constructor.

        :param api_key: Your API key for Citrination
        :type api_key: str
        :param site: The domain name of your Citrination deployment
            (the default is https://citrination.com)
        :type site: str
        :param suppress_warnings: A flag allowing you to suppress warning
            statements guarding against misuse printed to stdout.
        :type suppress_warnings: bool
        :param proxies: proxies to use when making HTTP requests. E.g.,
                proxies = {
                  'http': 'http://10.10.1.10:3128',
                  'https': 'http://10.10.1.10:1080',
                }
        :type proxies: dict(string, string)
        """
        api_key, site = get_preferred_credentials(api_key, site)
        self.models = ModelsClient(api_key, site, suppress_warnings=suppress_warnings, proxies=proxies)
        self.search = SearchClient(api_key, site, suppress_warnings=suppress_warnings, proxies=proxies)
        self.data = DataClient(api_key, site, suppress_warnings=suppress_warnings, proxies=proxies)
        self.data_views = DataViewsClient(api_key, site, suppress_warnings=suppress_warnings, proxies=proxies)

        # clients = [self.models, self.search, self.data, self.data_views]

        # for client in clients:
        #     client_methods = [a for a in dir(client) if not a.startswith('_')]
        #     for method in client_methods:
        #         setattr(self, method, _generate_lambda_proxy_method(client, method))

    def __repr__(self):
        return "['models', 'search', 'data', 'data_views']"

<commit_msg>Remove legacy code<commit_after>from citrination_client.views.client import DataViewsClient

from citrination_client.models import ModelsClient
from citrination_client.search import SearchClient
from citrination_client.data import DataClient
from citrination_client.util.credentials import get_preferred_credentials

class CitrinationClient(object):
    """
    The top level of the client hierarchy. Instantiating this class handles
    authentication information (api_key and site) and provides access to instances of each of the sub-clients, for more specific actions.

    Instantiation requires authentication information, but that can be provided
    via direct parameterization, environment variables, or a .citrination credentials file. See the tutorial on client Initialization for more information.
    """

    def __init__(self, api_key=None, site=None, suppress_warnings=False, proxies=None):
        """
        Constructor.

        :param api_key: Your API key for Citrination
        :type api_key: str
        :param site: The domain name of your Citrination deployment
            (the default is https://citrination.com)
        :type site: str
        :param suppress_warnings: A flag allowing you to suppress warning
            statements guarding against misuse printed to stdout.
        :type suppress_warnings: bool
        :param proxies: proxies to use when making HTTP requests. E.g.,
                proxies = {
                  'http': 'http://10.10.1.10:3128',
                  'https': 'http://10.10.1.10:1080',
                }
        :type proxies: dict(string, string)
        """
        api_key, site = get_preferred_credentials(api_key, site)
        self.models = ModelsClient(api_key, site, suppress_warnings=suppress_warnings, proxies=proxies)
        self.search = SearchClient(api_key, site, suppress_warnings=suppress_warnings, proxies=proxies)
        self.data = DataClient(api_key, site, suppress_warnings=suppress_warnings, proxies=proxies)
        self.data_views = DataViewsClient(api_key, site, suppress_warnings=suppress_warnings, proxies=proxies)

    def __repr__(self):
        return "['models', 'search', 'data', 'data_views']"
<|endoftext|>