<commit_before>"""Base Command class, and related routines"""
from __future__ import absolute_import

import logging
import logging.config
import optparse
import os
import sys
import warnings

from pip import cmdoptions
from pip.baseparser import ConfigOptionParser, UpdatingDefaultsHelpFormatter
from pip.compat import WINDOWS
from pip.download import PipSession
from pip.exceptions import (
    BadCommand, CommandError, InstallationError, PreviousBuildDirError,
    UninstallationError
)
from pip.index import PackageFinder
from pip.locations import running_under_virtualenv
from pip.req import InstallRequirement, parse_requirements
from pip.resolve import Resolver
from pip.status_codes import (
    ERROR, PREVIOUS_BUILD_DIR_ERROR, SUCCESS, UNKNOWN_ERROR,
    VIRTUALENV_NOT_FOUND
)
from pip.utils import deprecation, get_prog, normalize_path
from pip.utils.logging import IndentingFormatter
from pip.utils.outdated import pip_version_check

__all__ = ['Command']

logger = logging.getLogger(__name__)

class Command(object):
    name = None
    usage = None
    hidden = False
    ignore_require_venv = False
    log_streams = ("ext://sys.stdout", "ext://sys.stderr")

    def __init__(self, isolated=False):
        parser_kw = {
            'usage': self.usage,
            'prog': '%s %s' % (get_prog(), self.name),
            'formatter': UpdatingDefaultsHelpFormatter(),
            'add_help_option': False,
            'name': self.name,
            'description': self.__doc__,
            'isolated': isolated,
        }

        self.parser = ConfigOptionParser(**parser_kw)

        # Commands should add options to this option group
        optgroup_name = '%s Options' % self.name.capitalize()
        self.cmd_opts = optparse.OptionGroup(self.parser, optgroup_name)

        # Add the general options
        gen_opts = cmdoptions.make_option_group(
            cmdoptions.general_group,
            self.parser,
        )
        self.parser.add_option_group(gen_opts)

    def _build_session(self, options, retries=None, timeout=None):
        session = PipSession(
            cache=(
                normalize_path(os.path.join(options.cache_dir, "http"))
                if options.cache_dir else None
            ),
            retries=retries if retries is not None else options.retries,
            insecure_hosts=options.trusted_hosts,
        )

        # Handle custom ca-bundles from the user
        if options.cert:
            session.verify = options.cert

        # Handle SSL client certificate
        if options.client_cert:
            session.cert = options.client_cert

        # Handle timeouts
        if options.timeout or timeout:
            session.timeout = (
                timeout if timeout is not None else options.timeout
            )

        # Handle configured proxies
        if options.proxy:
            session.proxies = {
                "http": options.proxy,
                "https": options.proxy,
            }

        # Determine if we can prompt the user for authentication or not
        session.auth.prompting = not options.no_input

        return session

    def parse_args(self, args):
        # factored out for testability
        return self.parser.parse_args(args)

    def main(self, args):
        options, args = self.parse_args(args)

        verbosity = options.verbose - options.quiet
        if verbosity >= 1:
            level = "DEBUG"
        elif verbosity == -1:
            level = "WARNING"
        elif verbosity == -2:
            level = "ERROR"
        elif verbosity <= -3:
            level = "CRITICAL"
        else:
            level = "INFO"

        # The root logger should match the "console" level *unless* we
        # specified "--log" to send debug logs to a file.
        root_level = level
        if options.log:
            root_level = "DEBUG"

        logging.config.dictConfig({
            "version": 1,
            "disable_existing_loggers": False,
            "filters": {
                "exclude_warnings": {
                    "()": "pip.utils.logging.MaxLevelFilter",
                    "level": logging.WARNING,
                },
            },
            "formatters": {
                "indent": {
                    "()": IndentingFormatter,
                    "format": "%(message)s",
                },
            },
            "handlers": {
                "console": {
                    "level": level,
                    "class": "pip.utils.logging.ColorizedStreamHandler",
                    "stream": self.log_streams[0],
                    "filters": ["exclude_warnings"],
                    "formatter": "indent",
                },
                "console_errors": {
                    "level": "WARNING",
                    "class": "pip.utils.logging.ColorizedStreamHandler",
                    "stream": self.log_streams[1],
                    "formatter": "indent",
                },
                "user_log": {
                    "level": "DEBUG",
                    "class": "pip.utils.logging.BetterRotatingFileHandler",
                    "filename": options.log or "/dev/null",
                    "delay": True,
                    "formatter": "indent",
                },
            },
            "root": {
                "level": root_level,
                "handlers": list(filter(None, [
                    "console",
                    "console_errors",
                    "user_log" if options.log else None,
                ])),
            },
            # Disable any logging besides WARNING unless we have DEBUG level
            # logging enabled. These use both pip._vendor and the bare names
            # for the case where someone unbundles our libraries.
            "loggers": dict(
                (
                    name,
                    {
                        "level": (
                            "WARNING"
                            if level in ["INFO", "ERROR"]
                            else "DEBUG"
                        ),
                    },
                )
                for name in ["pip._vendor", "distlib", "requests", "urllib3"]
            ),
        })

        if sys.version_info[:2] == (3, 3):
            warnings.warn(
                "Python 3.3 supported has been deprecated and support for it "
                "will be dropped in the future. Please upgrade your Python.",
                deprecation.RemovedInPip11Warning,
            )

        # TODO: try to get these passing down from the command?
        #      without resorting to os.environ to hold these.

        if options.no_input:
            os.environ['PIP_NO_INPUT'] = '1'

        if options.exists_action:
            os.environ['PIP_EXISTS_ACTION'] = ' '.join(options.exists_action)

        if options.require_venv and not self.ignore_require_venv:
            # If a venv is required check if it can really be found
            if not running_under_virtualenv():
                logger.critical(
                    'Could not find an activated virtualenv (required).'
                )
                sys.exit(VIRTUALENV_NOT_FOUND)

        original_root_handlers = set(logging.root.handlers)

        try:
            status = self.run(options, args)
            # FIXME: all commands should return an exit status
            # and when it is done, isinstance is not needed anymore
            if isinstance(status, int):
                return status
        except PreviousBuildDirError as exc:
            logger.critical(str(exc))
            logger.debug('Exception information:', exc_info=True)

            return PREVIOUS_BUILD_DIR_ERROR
        except (InstallationError, UninstallationError, BadCommand) as exc:
            logger.critical(str(exc))
            logger.debug('Exception information:', exc_info=True)

            return ERROR
        except CommandError as exc:
            logger.critical('ERROR: %s', exc)
            logger.debug('Exception information:', exc_info=True)

            return ERROR
        except KeyboardInterrupt:
            logger.critical('Operation cancelled by user')
            logger.debug('Exception information:', exc_info=True)

            return ERROR
        except:
            logger.critical('Exception:', exc_info=True)

            return UNKNOWN_ERROR
        finally:
            # Check if we're using the latest version of pip available
            if (not options.disable_pip_version_check and not
                    getattr(options, "no_index", False)):
                with self._build_session(
                        options,
                        retries=0,
                        timeout=min(5, options.timeout)) as session:
                    pip_version_check(session, options)
            # Avoid leaking loggers
            for handler in set(logging.root.handlers) - original_root_handlers:
                # this method benefit from the Logger class internal lock
                logging.root.removeHandler(handler)

        return SUCCESS

class RequirementCommand(Command):

    @staticmethod
    def populate_requirement_set(requirement_set, args, options, finder,
                                 session, name, wheel_cache):
        """
        Marshal cmd line args into a requirement set.
        """
        for filename in options.constraints:
            for req in parse_requirements(
                    filename,
                    constraint=True, finder=finder, options=options,
                    session=session, wheel_cache=wheel_cache):
                requirement_set.add_requirement(req)

        for req in args:
            requirement_set.add_requirement(
                InstallRequirement.from_line(
                    req, None, isolated=options.isolated_mode,
                    wheel_cache=wheel_cache
                )
            )

        for req in options.editables:
            requirement_set.add_requirement(
                InstallRequirement.from_editable(
                    req,
                    isolated=options.isolated_mode,
                    wheel_cache=wheel_cache
                )
            )

        for filename in options.requirements:
            for req in parse_requirements(
                    filename,
                    finder=finder, options=options, session=session,
                    wheel_cache=wheel_cache):
                requirement_set.add_requirement(req)
        # If --require-hashes was a line in a requirements file, tell
        # RequirementSet about it:
        requirement_set.require_hashes = options.require_hashes

        if not (args or options.editables or options.requirements):
            opts = {'name': name}
            if options.find_links:
                raise CommandError(
                    'You must give at least one requirement to %(name)s '
                    '(maybe you meant "pip %(name)s %(links)s"?)' %
                    dict(opts, links=' '.join(options.find_links)))
            else:
                raise CommandError(
                    'You must give at least one requirement to %(name)s '
                    '(see "pip help %(name)s")' % opts)

        # On Windows, any operation modifying pip should be run as:
        #     python -m pip ...
        # See https://github.com/pypa/pip/issues/1299 for more discussion
        should_show_use_python_msg = (
            WINDOWS and
            requirement_set.has_requirement('pip') and
            "pip" in os.path.basename(sys.argv[0])
        )
        if should_show_use_python_msg:
            new_command = [
                sys.executable, "-m", "pip"
            ] + sys.argv[1:]
            raise CommandError(
                'To modify pip, please run the following command:\n{}'
                .format(" ".join(new_command))
            )

    def _build_package_finder(self, options, session,
                              platform=None, python_versions=None,
                              abi=None, implementation=None):
        """
        Create a package finder appropriate to this requirement command.
        """
        index_urls = [options.index_url] + options.extra_index_urls
        if options.no_index:
            logger.debug('Ignoring indexes: %s', ','.join(index_urls))
            index_urls = []

        return PackageFinder(
            find_links=options.find_links,
            format_control=options.format_control,
            index_urls=index_urls,
            trusted_hosts=options.trusted_hosts,
            allow_all_prereleases=options.pre,
            process_dependency_links=options.process_dependency_links,
            session=session,
            platform=platform,
            versions=python_versions,
            abi=abi,
            implementation=implementation,
        )

    def _build_resolver(self, options, finder):
        strategy = options.upgrade_strategy
        if not options.upgrade:
            strategy = "not-allowed"

        return Resolver(
            upgrade_strategy=strategy,
            finder=finder
        )
<commit_msg>I forgot that download doesn't have an upgrade_strategy.<commit_after>"""Base Command class, and related routines"""
from __future__ import absolute_import

import logging
import logging.config
import optparse
import os
import sys
import warnings

from pip import cmdoptions
from pip.baseparser import ConfigOptionParser, UpdatingDefaultsHelpFormatter
from pip.compat import WINDOWS
from pip.download import PipSession
from pip.exceptions import (
    BadCommand, CommandError, InstallationError, PreviousBuildDirError,
    UninstallationError
)
from pip.index import PackageFinder
from pip.locations import running_under_virtualenv
from pip.req import InstallRequirement, parse_requirements
from pip.resolve import Resolver
from pip.status_codes import (
    ERROR, PREVIOUS_BUILD_DIR_ERROR, SUCCESS, UNKNOWN_ERROR,
    VIRTUALENV_NOT_FOUND
)
from pip.utils import deprecation, get_prog, normalize_path
from pip.utils.logging import IndentingFormatter
from pip.utils.outdated import pip_version_check

__all__ = ['Command']

logger = logging.getLogger(__name__)

class Command(object):
    name = None
    usage = None
    hidden = False
    ignore_require_venv = False
    log_streams = ("ext://sys.stdout", "ext://sys.stderr")

    def __init__(self, isolated=False):
        parser_kw = {
            'usage': self.usage,
            'prog': '%s %s' % (get_prog(), self.name),
            'formatter': UpdatingDefaultsHelpFormatter(),
            'add_help_option': False,
            'name': self.name,
            'description': self.__doc__,
            'isolated': isolated,
        }

        self.parser = ConfigOptionParser(**parser_kw)

        # Commands should add options to this option group
        optgroup_name = '%s Options' % self.name.capitalize()
        self.cmd_opts = optparse.OptionGroup(self.parser, optgroup_name)

        # Add the general options
        gen_opts = cmdoptions.make_option_group(
            cmdoptions.general_group,
            self.parser,
        )
        self.parser.add_option_group(gen_opts)

    def _build_session(self, options, retries=None, timeout=None):
        session = PipSession(
            cache=(
                normalize_path(os.path.join(options.cache_dir, "http"))
                if options.cache_dir else None
            ),
            retries=retries if retries is not None else options.retries,
            insecure_hosts=options.trusted_hosts,
        )

        # Handle custom ca-bundles from the user
        if options.cert:
            session.verify = options.cert

        # Handle SSL client certificate
        if options.client_cert:
            session.cert = options.client_cert

        # Handle timeouts
        if options.timeout or timeout:
            session.timeout = (
                timeout if timeout is not None else options.timeout
            )

        # Handle configured proxies
        if options.proxy:
            session.proxies = {
                "http": options.proxy,
                "https": options.proxy,
            }

        # Determine if we can prompt the user for authentication or not
        session.auth.prompting = not options.no_input

        return session

    def parse_args(self, args):
        # factored out for testability
        return self.parser.parse_args(args)

    def main(self, args):
        options, args = self.parse_args(args)

        verbosity = options.verbose - options.quiet
        if verbosity >= 1:
            level = "DEBUG"
        elif verbosity == -1:
            level = "WARNING"
        elif verbosity == -2:
            level = "ERROR"
        elif verbosity <= -3:
            level = "CRITICAL"
        else:
            level = "INFO"

        # The root logger should match the "console" level *unless* we
        # specified "--log" to send debug logs to a file.
        root_level = level
        if options.log:
            root_level = "DEBUG"

        logging.config.dictConfig({
            "version": 1,
            "disable_existing_loggers": False,
            "filters": {
                "exclude_warnings": {
                    "()": "pip.utils.logging.MaxLevelFilter",
                    "level": logging.WARNING,
                },
            },
            "formatters": {
                "indent": {
                    "()": IndentingFormatter,
                    "format": "%(message)s",
                },
            },
            "handlers": {
                "console": {
                    "level": level,
                    "class": "pip.utils.logging.ColorizedStreamHandler",
                    "stream": self.log_streams[0],
                    "filters": ["exclude_warnings"],
                    "formatter": "indent",
                },
                "console_errors": {
                    "level": "WARNING",
                    "class": "pip.utils.logging.ColorizedStreamHandler",
                    "stream": self.log_streams[1],
                    "formatter": "indent",
                },
                "user_log": {
                    "level": "DEBUG",
                    "class": "pip.utils.logging.BetterRotatingFileHandler",
                    "filename": options.log or "/dev/null",
                    "delay": True,
                    "formatter": "indent",
                },
            },
            "root": {
                "level": root_level,
                "handlers": list(filter(None, [
                    "console",
                    "console_errors",
                    "user_log" if options.log else None,
                ])),
            },
            # Disable any logging besides WARNING unless we have DEBUG level
            # logging enabled. These use both pip._vendor and the bare names
            # for the case where someone unbundles our libraries.
            "loggers": dict(
                (
                    name,
                    {
                        "level": (
                            "WARNING"
                            if level in ["INFO", "ERROR"]
                            else "DEBUG"
                        ),
                    },
                )
                for name in ["pip._vendor", "distlib", "requests", "urllib3"]
            ),
        })

        if sys.version_info[:2] == (3, 3):
            warnings.warn(
                "Python 3.3 supported has been deprecated and support for it "
                "will be dropped in the future. Please upgrade your Python.",
                deprecation.RemovedInPip11Warning,
            )

        # TODO: try to get these passing down from the command?
        #      without resorting to os.environ to hold these.

        if options.no_input:
            os.environ['PIP_NO_INPUT'] = '1'

        if options.exists_action:
            os.environ['PIP_EXISTS_ACTION'] = ' '.join(options.exists_action)

        if options.require_venv and not self.ignore_require_venv:
            # If a venv is required check if it can really be found
            if not running_under_virtualenv():
                logger.critical(
                    'Could not find an activated virtualenv (required).'
                )
                sys.exit(VIRTUALENV_NOT_FOUND)

        original_root_handlers = set(logging.root.handlers)

        try:
            status = self.run(options, args)
            # FIXME: all commands should return an exit status
            # and when it is done, isinstance is not needed anymore
            if isinstance(status, int):
                return status
        except PreviousBuildDirError as exc:
            logger.critical(str(exc))
            logger.debug('Exception information:', exc_info=True)

            return PREVIOUS_BUILD_DIR_ERROR
        except (InstallationError, UninstallationError, BadCommand) as exc:
            logger.critical(str(exc))
            logger.debug('Exception information:', exc_info=True)

            return ERROR
        except CommandError as exc:
            logger.critical('ERROR: %s', exc)
            logger.debug('Exception information:', exc_info=True)

            return ERROR
        except KeyboardInterrupt:
            logger.critical('Operation cancelled by user')
            logger.debug('Exception information:', exc_info=True)

            return ERROR
        except:
            logger.critical('Exception:', exc_info=True)

            return UNKNOWN_ERROR
        finally:
            # Check if we're using the latest version of pip available
            if (not options.disable_pip_version_check and not
                    getattr(options, "no_index", False)):
                with self._build_session(
                        options,
                        retries=0,
                        timeout=min(5, options.timeout)) as session:
                    pip_version_check(session, options)
            # Avoid leaking loggers
            for handler in set(logging.root.handlers) - original_root_handlers:
                # this method benefit from the Logger class internal lock
                logging.root.removeHandler(handler)

        return SUCCESS

class RequirementCommand(Command):

    @staticmethod
    def populate_requirement_set(requirement_set, args, options, finder,
                                 session, name, wheel_cache):
        """
        Marshal cmd line args into a requirement set.
        """
        for filename in options.constraints:
            for req in parse_requirements(
                    filename,
                    constraint=True, finder=finder, options=options,
                    session=session, wheel_cache=wheel_cache):
                requirement_set.add_requirement(req)

        for req in args:
            requirement_set.add_requirement(
                InstallRequirement.from_line(
                    req, None, isolated=options.isolated_mode,
                    wheel_cache=wheel_cache
                )
            )

        for req in options.editables:
            requirement_set.add_requirement(
                InstallRequirement.from_editable(
                    req,
                    isolated=options.isolated_mode,
                    wheel_cache=wheel_cache
                )
            )

        for filename in options.requirements:
            for req in parse_requirements(
                    filename,
                    finder=finder, options=options, session=session,
                    wheel_cache=wheel_cache):
                requirement_set.add_requirement(req)
        # If --require-hashes was a line in a requirements file, tell
        # RequirementSet about it:
        requirement_set.require_hashes = options.require_hashes

        if not (args or options.editables or options.requirements):
            opts = {'name': name}
            if options.find_links:
                raise CommandError(
                    'You must give at least one requirement to %(name)s '
                    '(maybe you meant "pip %(name)s %(links)s"?)' %
                    dict(opts, links=' '.join(options.find_links)))
            else:
                raise CommandError(
                    'You must give at least one requirement to %(name)s '
                    '(see "pip help %(name)s")' % opts)

        # On Windows, any operation modifying pip should be run as:
        #     python -m pip ...
        # See https://github.com/pypa/pip/issues/1299 for more discussion
        should_show_use_python_msg = (
            WINDOWS and
            requirement_set.has_requirement('pip') and
            "pip" in os.path.basename(sys.argv[0])
        )
        if should_show_use_python_msg:
            new_command = [
                sys.executable, "-m", "pip"
            ] + sys.argv[1:]
            raise CommandError(
                'To modify pip, please run the following command:\n{}'
                .format(" ".join(new_command))
            )

    def _build_package_finder(self, options, session,
                              platform=None, python_versions=None,
                              abi=None, implementation=None):
        """
        Create a package finder appropriate to this requirement command.
        """
        index_urls = [options.index_url] + options.extra_index_urls
        if options.no_index:
            logger.debug('Ignoring indexes: %s', ','.join(index_urls))
            index_urls = []

        return PackageFinder(
            find_links=options.find_links,
            format_control=options.format_control,
            index_urls=index_urls,
            trusted_hosts=options.trusted_hosts,
            allow_all_prereleases=options.pre,
            process_dependency_links=options.process_dependency_links,
            session=session,
            platform=platform,
            versions=python_versions,
            abi=abi,
            implementation=implementation,
        )

    def _build_resolver(self, options, finder):
        strategy = getattr(options, "upgrade_strategy", "not-allowed")
        if not options.upgrade:
            strategy = "not-allowed"

        return Resolver(
            upgrade_strategy=strategy,
            finder=finder
        )
<|endoftext|>


<commit_before>import sys, inspect

from django.http import (HttpResponse, Http404, HttpResponseNotAllowed,
    HttpResponseForbidden, HttpResponseServerError)
from django.views.debug import ExceptionReporter
from django.views.decorators.vary import vary_on_headers
from django.conf import settings
from django.core.mail import send_mail, EmailMessage
from django.db.models.query import QuerySet

from emitters import Emitter
from handler import typemapper
from doc import HandlerMethod
from authentication import NoAuthentication
from utils import coerce_put_post, FormValidationError, HttpStatusCode
from utils import rc, format_error, translate_mime, MimerDataException

class Resource(object):
    """
    Resource. Create one for your URL mappings, just
    like you would with Django. Takes one argument,
    the handler. The second argument is optional, and
    is an authentication handler. If not specified,
    `NoAuthentication` will be used by default.
    """
    callmap = { 'GET': 'read', 'POST': 'create',
                'PUT': 'update', 'DELETE': 'delete' }

    def __init__(self, handler, authentication=None):
        if not callable(handler):
            raise AttributeError, "Handler not callable."

        self.handler = handler()

        if not authentication:
            self.authentication = NoAuthentication()
        else:
            self.authentication = authentication

        # Erroring
        self.email_errors = getattr(settings, 'PISTON_EMAIL_ERRORS', True)
        self.display_errors = getattr(settings, 'PISTON_DISPLAY_ERRORS', True)
        self.stream = getattr(settings, 'PISTON_STREAM_OUTPUT', False)

    def determine_emitter(self, request, *args, **kwargs):
        """
        Function for determening which emitter to use
        for output. It lives here so you can easily subclass
        `Resource` in order to change how emission is detected.

        You could also check for the `Accept` HTTP header here,
        since that pretty much makes sense. Refer to `Mimer` for
        that as well.
        """
        em = kwargs.pop('emitter_format', None)

        if not em:
            em = request.GET.get('format', 'json')

        return em

    @vary_on_headers('Authorization')
    def __call__(self, request, *args, **kwargs):
        """
        NB: Sends a `Vary` header so we don't cache requests
        that are different (OAuth stuff in `Authorization` header.)
        """
        rm = request.method.upper()

        # Django's internal mechanism doesn't pick up
        # PUT request, so we trick it a little here.
        if rm == "PUT":
            coerce_put_post(request)

        if not self.authentication.is_authenticated(request):
            if hasattr(self.handler, 'anonymous') and \
                callable(self.handler.anonymous) and \
                rm in self.handler.anonymous.allowed_methods:

                handler = self.handler.anonymous()
                anonymous = True
            else:
                return self.authentication.challenge()
        else:
            handler = self.handler
            anonymous = handler.is_anonymous

        # Translate nested datastructs into `request.data` here.
        if rm in ('POST', 'PUT'):
            try:
                translate_mime(request)
            except MimerDataException:
                return rc.BAD_REQUEST

        if not rm in handler.allowed_methods:
            return HttpResponseNotAllowed(handler.allowed_methods)

        meth = getattr(handler, self.callmap.get(rm), None)

        if not meth:
            raise Http404

        # Support emitter both through (?P<emitter_format>) and ?format=emitter.
        em_format = self.determine_emitter(request, *args, **kwargs)

        kwargs.pop('emitter_format', None)

        # Clean up the request object a bit, since we might
        # very well have `oauth_`-headers in there, and we
        # don't want to pass these along to the handler.
        request = self.cleanup_request(request)

        try:
            result = meth(request, *args, **kwargs)
        except FormValidationError, e:
            # TODO: Use rc.BAD_REQUEST here
            return HttpResponse("Bad Request: %s" % e.form.errors, status=400)
        except TypeError, e:
            result = rc.BAD_REQUEST
            hm = HandlerMethod(meth)
            sig = hm.get_signature()

            msg = 'Method signature does not match.\n\n'

            if sig:
                msg += 'Signature should be: %s' % sig
            else:
                msg += 'Resource does not expect any parameters.'

            if self.display_errors:
                msg += '\n\nException was: %s' % str(e)

            result.content = format_error(msg)
        except HttpStatusCode, e:
            #result = e ## why is this being passed on and not just dealt with now?
            return e.response
        except Exception, e:
            """
            On errors (like code errors), we'd like to be able to
            give crash reports to both admins and also the calling
            user. There's two setting parameters for this:

            Parameters::
             - `PISTON_EMAIL_ERRORS`: Will send a Django formatted
               error email to people in `settings.ADMINS`.
             - `PISTON_DISPLAY_ERRORS`: Will return a simple traceback
               to the caller, so he can tell you what error they got.

            If `PISTON_DISPLAY_ERRORS` is not enabled, the caller will
            receive a basic "500 Internal Server Error" message.
            """
            exc_type, exc_value, tb = sys.exc_info()
            rep = ExceptionReporter(request, exc_type, exc_value, tb.tb_next)
            if self.email_errors:
                self.email_exception(rep)
            if self.display_errors:
                return HttpResponseServerError(
                    format_error('\n'.join(rep.format_exception())))
            else:
                raise

        emitter, ct = Emitter.get(em_format)
        fields = handler.fields
        if hasattr(handler, 'list_fields') and (
                isinstance(result, list) or isinstance(result, QuerySet)):
            fields = handler.list_fields

        srl = emitter(result, typemapper, handler, fields, anonymous)

        try:
            """
            Decide whether or not we want a generator here,
            or we just want to buffer up the entire result
            before sending it to the client. Won't matter for
            smaller datasets, but larger will have an impact.
            """
            if self.stream: stream = srl.stream_render(request)
            else: stream = srl.render(request)

            resp = HttpResponse(stream, mimetype=ct)

            resp.streaming = self.stream

            return resp
        except HttpStatusCode, e:
            return e.response

    @staticmethod
    def cleanup_request(request):
        """
        Removes `oauth_` keys from various dicts on the
        request object, and returns the sanitized version.
        """
        for method_type in ('GET', 'PUT', 'POST', 'DELETE'):
            block = getattr(request, method_type, { })

            if True in [ k.startswith("oauth_") for k in block.keys() ]:
                sanitized = block.copy()

                for k in sanitized.keys():
                    if k.startswith("oauth_"):
                        sanitized.pop(k)

                setattr(request, method_type, sanitized)

        return request

    # --

    def email_exception(self, reporter):
        subject = "Piston crash report"
        html = reporter.get_traceback_html()

        message = EmailMessage(settings.EMAIL_SUBJECT_PREFIX+subject,
                                html, settings.SERVER_EMAIL,
                                [ admin[1] for admin in settings.ADMINS ])

        message.content_subtype = 'html'
        message.send(fail_silently=True)
<commit_msg>possible to specify anonymous handlers as strings to reference them ahead of time<commit_after>import sys, inspect

from django.http import (HttpResponse, Http404, HttpResponseNotAllowed,
    HttpResponseForbidden, HttpResponseServerError)
from django.views.debug import ExceptionReporter
from django.views.decorators.vary import vary_on_headers
from django.conf import settings
from django.core.mail import send_mail, EmailMessage
from django.db.models.query import QuerySet

from emitters import Emitter
from handler import typemapper
from doc import HandlerMethod
from authentication import NoAuthentication
from utils import coerce_put_post, FormValidationError, HttpStatusCode
from utils import rc, format_error, translate_mime, MimerDataException

class Resource(object):
    """
    Resource. Create one for your URL mappings, just
    like you would with Django. Takes one argument,
    the handler. The second argument is optional, and
    is an authentication handler. If not specified,
    `NoAuthentication` will be used by default.
    """
    callmap = { 'GET': 'read', 'POST': 'create',
                'PUT': 'update', 'DELETE': 'delete' }

    def __init__(self, handler, authentication=None):
        if not callable(handler):
            raise AttributeError, "Handler not callable."

        self.handler = handler()

        if not authentication:
            self.authentication = NoAuthentication()
        else:
            self.authentication = authentication

        # Erroring
        self.email_errors = getattr(settings, 'PISTON_EMAIL_ERRORS', True)
        self.display_errors = getattr(settings, 'PISTON_DISPLAY_ERRORS', True)
        self.stream = getattr(settings, 'PISTON_STREAM_OUTPUT', False)

    def determine_emitter(self, request, *args, **kwargs):
        """
        Function for determening which emitter to use
        for output. It lives here so you can easily subclass
        `Resource` in order to change how emission is detected.

        You could also check for the `Accept` HTTP header here,
        since that pretty much makes sense. Refer to `Mimer` for
        that as well.
        """
        em = kwargs.pop('emitter_format', None)

        if not em:
            em = request.GET.get('format', 'json')

        return em

    @property
    def anonymous(self):
        """
        Gets the anonymous handler. Also tries to grab a class
        if the `anonymous` value is a string, so that we can define
        anonymous handlers that aren't defined yet (like, when
        you're subclassing your basehandler into an anonymous one.)
        """
        if hasattr(self.handler, 'anonymous'):
            anon = self.handler.anonymous

            if callable(anon):
                return anon

            for klass in typemapper.keys():
                if anon == klass.__name__:
                    return klass

        return None

    @vary_on_headers('Authorization')
    def __call__(self, request, *args, **kwargs):
        """
        NB: Sends a `Vary` header so we don't cache requests
        that are different (OAuth stuff in `Authorization` header.)
        """
        rm = request.method.upper()

        # Django's internal mechanism doesn't pick up
        # PUT request, so we trick it a little here.
        if rm == "PUT":
            coerce_put_post(request)

        if not self.authentication.is_authenticated(request):
            if self.anonymous and \
                rm in self.anonymous.allowed_methods:

                handler = self.anonymous()
                anonymous = True
            else:
                return self.authentication.challenge()
        else:
            handler = self.handler
            anonymous = handler.is_anonymous

        # Translate nested datastructs into `request.data` here.
        if rm in ('POST', 'PUT'):
            try:
                translate_mime(request)
            except MimerDataException:
                return rc.BAD_REQUEST

        if not rm in handler.allowed_methods:
            return HttpResponseNotAllowed(handler.allowed_methods)

        meth = getattr(handler, self.callmap.get(rm), None)

        if not meth:
            raise Http404

        # Support emitter both through (?P<emitter_format>) and ?format=emitter.
        em_format = self.determine_emitter(request, *args, **kwargs)

        kwargs.pop('emitter_format', None)

        # Clean up the request object a bit, since we might
        # very well have `oauth_`-headers in there, and we
        # don't want to pass these along to the handler.
        request = self.cleanup_request(request)

        try:
            result = meth(request, *args, **kwargs)
        except FormValidationError, e:
            # TODO: Use rc.BAD_REQUEST here
            return HttpResponse("Bad Request: %s" % e.form.errors, status=400)
        except TypeError, e:
            result = rc.BAD_REQUEST
            hm = HandlerMethod(meth)
            sig = hm.get_signature()

            msg = 'Method signature does not match.\n\n'

            if sig:
                msg += 'Signature should be: %s' % sig
            else:
                msg += 'Resource does not expect any parameters.'

            if self.display_errors:
                msg += '\n\nException was: %s' % str(e)

            result.content = format_error(msg)
        except HttpStatusCode, e:
            #result = e ## why is this being passed on and not just dealt with now?
            return e.response
        except Exception, e:
            """
            On errors (like code errors), we'd like to be able to
            give crash reports to both admins and also the calling
            user. There's two setting parameters for this:

            Parameters::
             - `PISTON_EMAIL_ERRORS`: Will send a Django formatted
               error email to people in `settings.ADMINS`.
             - `PISTON_DISPLAY_ERRORS`: Will return a simple traceback
               to the caller, so he can tell you what error they got.

            If `PISTON_DISPLAY_ERRORS` is not enabled, the caller will
            receive a basic "500 Internal Server Error" message.
            """
            exc_type, exc_value, tb = sys.exc_info()
            rep = ExceptionReporter(request, exc_type, exc_value, tb.tb_next)
            if self.email_errors:
                self.email_exception(rep)
            if self.display_errors:
                return HttpResponseServerError(
                    format_error('\n'.join(rep.format_exception())))
            else:
                raise

        emitter, ct = Emitter.get(em_format)
        fields = handler.fields
        if hasattr(handler, 'list_fields') and (
                isinstance(result, list) or isinstance(result, QuerySet)):
            fields = handler.list_fields

        srl = emitter(result, typemapper, handler, fields, anonymous)

        try:
            """
            Decide whether or not we want a generator here,
            or we just want to buffer up the entire result
            before sending it to the client. Won't matter for
            smaller datasets, but larger will have an impact.
            """
            if self.stream: stream = srl.stream_render(request)
            else: stream = srl.render(request)

            resp = HttpResponse(stream, mimetype=ct)

            resp.streaming = self.stream

            return resp
        except HttpStatusCode, e:
            return e.response

    @staticmethod
    def cleanup_request(request):
        """
        Removes `oauth_` keys from various dicts on the
        request object, and returns the sanitized version.
        """
        for method_type in ('GET', 'PUT', 'POST', 'DELETE'):
            block = getattr(request, method_type, { })

            if True in [ k.startswith("oauth_") for k in block.keys() ]:
                sanitized = block.copy()

                for k in sanitized.keys():
                    if k.startswith("oauth_"):
                        sanitized.pop(k)

                setattr(request, method_type, sanitized)

        return request

    # --

    def email_exception(self, reporter):
        subject = "Piston crash report"
        html = reporter.get_traceback_html()

        message = EmailMessage(settings.EMAIL_SUBJECT_PREFIX+subject,
                                html, settings.SERVER_EMAIL,
                                [ admin[1] for admin in settings.ADMINS ])

        message.content_subtype = 'html'
        message.send(fail_silently=True)
<|endoftext|>


<commit_before>from .store import Store
from .utils import request_json
from .urls import Urls, COUNTRY_USA

class Address(object):
    def __init__(self, street, city, region='', zip='', country=COUNTRY_USA, *args):
        self.street = street.strip()
        self.city = city.strip()
        self.region = region.strip()
        self.zip = str(zip).strip(),
        self.urls = Urls(country)
        self.country = country

    @property
    def data(self):
        return {'Street': self.street, 'City': self.city,
                'Region': self.region, 'PostalCode': self.zip}

    @property
    def line1(self):
        return '{Street}'.format(**self.data)

    @property
    def line2(self):
        return '{City}, {Region}, {PostalCode}'.format(**self.data)

    def nearby_stores(self, service='Delivery'):
        data = request_json(self.urls.find_url(), line1=self.line1, line2=self.line2, type=service)
        return [Store(x, self.country) for x in data['Stores']
                if x['IsOnlineNow'] and x['ServiceIsOpen'][service]]

    def closest_store(self, service='Delivery'):
        stores = self.nearby_stores(service=service)
        if not stores:
            raise Exception('No local stores are currently open')
        return stores[0]
<commit_msg>update address with zip fix<commit_after>from .store import Store
from .utils import request_json
from .urls import Urls, COUNTRY_USA

class Address(object):
    def __init__(self, street, city, region='', zip='', country=COUNTRY_USA, *args):
        self.street = street.strip()
        self.city = city.strip()
        self.region = region.strip()
        self.zip = str(zip).strip()
        self.urls = Urls(country)
        self.country = country

    @property
    def data(self):
        return {'Street': self.street, 'City': self.city,
                'Region': self.region, 'PostalCode': self.zip}

    @property
    def line1(self):
        return '{Street}'.format(**self.data)

    @property
    def line2(self):
        return '{City}, {Region}, {PostalCode}'.format(**self.data)

    def nearby_stores(self, service='Delivery'):
        data = request_json(self.urls.find_url(), line1=self.line1, line2=self.line2, type=service)
        return [Store(x, self.country) for x in data['Stores']
                if x['IsOnlineNow'] and x['ServiceIsOpen'][service]]

    def closest_store(self, service='Delivery'):
        stores = self.nearby_stores(service=service)
        if not stores:
            raise Exception('No local stores are currently open')
        return stores[0]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
VERSION = (0, 2, 0, "f") # following PEP 386

def get_version():
    version = "%s.%s" % (VERSION[0], VERSION[1])
    if VERSION[2]:
        version = "%s.%s" % (version, VERSION[2])
    if VERSION[3] != "f":
        version = "%s%s%s" % (version, VERSION[3], VERSION[4])
    return version

__version__ = get_version()
<commit_msg>Bumping version to 0.2.1.<commit_after># -*- coding: utf-8 -*-
VERSION = (0, 2, 1, "f") # following PEP 386

def get_version():
    version = "%s.%s" % (VERSION[0], VERSION[1])
    if VERSION[2]:
        version = "%s.%s" % (version, VERSION[2])
    if VERSION[3] != "f":
        version = "%s%s%s" % (version, VERSION[3], VERSION[4])
    return version

__version__ = get_version()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright 2011 Google Inc.
# Copyright 2013 Patrick von Reth <vonreth@kde.org>
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import sys

def platforms():
    return ['linux', 'darwin', 'freebsd', 'openbsd', 'solaris', 'sunos5',
            'mingw', 'msvc']

class Platform( object ):
    def __init__( self, platform):
        self._platform = platform
        if not self._platform is None:
            return
        self._platform = sys.platform
        if self._platform.startswith('linux'):
            self._platform = 'linux'
        elif self._platform.startswith('freebsd'):
            self._platform = 'freebsd'
        elif self._platform.startswith('openbsd'):
            self._platform = 'openbsd'
        elif self._platform.startswith('solaris'):
            self._platform = 'solaris'
        elif self._platform.startswith('mingw'):
            self._platform = 'mingw'
        elif self._platform.startswith('win'):
            self._platform = 'msvc'

    def platform(self):
        return self._platform

    def is_linux(self):
        return self._platform == 'linux'

    def is_mingw(self):
        return self._platform == 'mingw'

    def is_msvc(self):
        return self._platform == 'msvc'

    def is_windows(self):
        return self.is_mingw() or self.is_msvc()

    def is_solaris(self):
        return self._platform == 'solaris'

    def is_freebsd(self):
        return self._platform == 'freebsd'

    def is_openbsd(self):
        return self._platform == 'openbsd'

    def is_sunos5(self):
        return self._platform == 'sunos5'
<commit_msg>Fixed the build on Debian/kfreebsd<commit_after>#!/usr/bin/env python
# Copyright 2011 Google Inc.
# Copyright 2013 Patrick von Reth <vonreth@kde.org>
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import sys

def platforms():
    return ['linux', 'darwin', 'freebsd', 'openbsd', 'solaris', 'sunos5',
            'mingw', 'msvc', 'gnukfreebsd8']

class Platform( object ):
    def __init__( self, platform):
        self._platform = platform
        if not self._platform is None:
            return
        self._platform = sys.platform
        if self._platform.startswith('linux'):
            self._platform = 'linux'
        elif self._platform.startswith('freebsd'):
            self._platform = 'freebsd'
        elif self._platform.startswith('gnukfreebsd8'):
            self._platform = 'freebsd'
        elif self._platform.startswith('openbsd'):
            self._platform = 'openbsd'
        elif self._platform.startswith('solaris'):
            self._platform = 'solaris'
        elif self._platform.startswith('mingw'):
            self._platform = 'mingw'
        elif self._platform.startswith('win'):
            self._platform = 'msvc'

    def platform(self):
        return self._platform

    def is_linux(self):
        return self._platform == 'linux'

    def is_mingw(self):
        return self._platform == 'mingw'

    def is_msvc(self):
        return self._platform == 'msvc'

    def is_windows(self):
        return self.is_mingw() or self.is_msvc()

    def is_solaris(self):
        return self._platform == 'solaris'

    def is_freebsd(self):
        return self._platform == 'freebsd'

    def is_openbsd(self):
        return self._platform == 'openbsd'

    def is_sunos5(self):
        return self._platform == 'sunos5'
<|endoftext|>


<commit_before>from abc import ABCMeta, abstractmethod
import logging
import os
import shutil
import subprocess
import tempfile
import threading
import unittest
from xml.etree import ElementTree
import time

from jobTree.batchSystems.abstractBatchSystem import AbstractBatchSystem
from jobTree.batchSystems.mesos import MesosBatchSystem
from jobTree.batchSystems.singleMachine import SingleMachineBatchSystem
from jobTree.batchSystems.abstractBatchSystem import InsufficientSystemResources
from jobTree.test import JobTreeTest

log = logging.getLogger(__name__)

class hidden:
    """
    Hide abstract base class from unittest's test case loader

    http://stackoverflow.com/questions/1323455/python-unit-test-with-base-and-sub-class#answer-25695512
    """

    class AbstractBatchSystemTest(JobTreeTest):
        """
        A base test case with generic tests that every batch system should pass
        """
        __metaclass__ = ABCMeta

        @abstractmethod
        def createBatchSystem(self):
            """
            :rtype: AbstractBatchSystem
            """
            raise NotImplementedError

        def setUp(self):
            super(hidden.AbstractBatchSystemTest, self).setUp()
            self.batchSystem = self.createBatchSystem()
            self.tempDir = tempfile.mkdtemp()

        config = None
        tempDir = None

        @classmethod
        def setUpClass(cls):
            cls.config = cls._createDummyConfig()
            cls.tempDir = tempfile.mkdtemp()

        ###################
        # Begin Tests   #
        ###################

        def testIssueJob(self):
            test_path = os.path.join(self.tempDir, 'test.txt')
            jobCommand = 'touch {}'.format(test_path)
            self.batchSystem.issueJob(jobCommand, memory=10, cpu=1)
            self.wait_for_jobs(wait_for_completion=True)
            self.assertTrue(os.path.exists(test_path))

        def testCheckResourceRequest(self):
            self.assertRaises(InsufficientSystemResources, self.batchSystem.checkResourceRequest, memory=1000, cpu=200)
            self.assertRaises(InsufficientSystemResources, self.batchSystem.checkResourceRequest, memory=5, cpu=200)
            self.assertRaises(InsufficientSystemResources, self.batchSystem.checkResourceRequest, memory=100, cpu=1)
            self.assertRaises(AssertionError, self.batchSystem.checkResourceRequest, memory=None, cpu=1)
            self.assertRaises(AssertionError, self.batchSystem.checkResourceRequest, memory=10, cpu=None)
            self.batchSystem.checkResourceRequest(memory=10, cpu=1)

        def testGetIssuedJobIDs(self):
            self.batchSystem.issueJob('sleep 1', memory=10, cpu=1)
            self.batchSystem.issueJob('sleep 1', memory=10, cpu=1)
            self.assertEqual({0, 1}, set( self.batchSystem.getIssuedJobIDs()))

        def testGetRunningJobIDs(self):
            self.batchSystem.issueJob('sleep 100', memory=10, cpu=1)
            self.batchSystem.issueJob('sleep 100', memory=10, cpu=1)
            self.wait_for_jobs()
            self.assertEqual({0, 1}, set( self.batchSystem.getRunningJobIDs().keys()))

        def testKillJobs(self):
            jobCommand = 'sleep 100'
            self.batchSystem.issueJob(jobCommand, memory=10, cpu=1)
            self.wait_for_jobs()
            self.assertEqual([0], self.batchSystem.getRunningJobIDs().keys())
            self.batchSystem.killJobs([0])
            self.assertEqual({}, self.batchSystem.getRunningJobIDs())
            # Make sure that killJob doesn't hang on unknown job IDs
            self.batchSystem.killJobs([0])

        def testGetUpdatedJob(self):
            delay = 1
            num_jobs = 2
            jobCommand = 'sleep %i' % delay
            for i in range( num_jobs ):
                self.batchSystem.issueJob(jobCommand, memory=10, cpu=1)
            jobs = set((i,0) for i in range(num_jobs))
            self.wait_for_jobs(wait_for_completion=True)
            for i in range( num_jobs ):
                jobs.remove(self.batchSystem.getUpdatedJob(delay * 2))
            self.assertFalse( jobs )

        def testGetRescueJobFrequency(self):
            self.assertTrue(self.batchSystem.getRescueJobFrequency() > 0)

        #################
        #   End Tests   #
        #################

        @staticmethod
        def _createDummyConfig():
            config = ElementTree.Element("config")
            config.attrib["log_level"] = 'DEBUG'
            config.attrib["job_store"] = '.'
            config.attrib["parasol_command"] = None
            config.attrib["try_count"] = str(2)
            config.attrib["max_job_duration"] = str(1)
            config.attrib["batch_system"] = None
            config.attrib["job_time"] = str(1)
            config.attrib["max_log_file_size"] = str(1)
            config.attrib["default_memory"] = str(1)
            config.attrib["default_cpu"] = str(1)
            config.attrib["max_cpus"] = str(1)
            config.attrib["max_memory"] = str(1)
            config.attrib["max_threads"] = str(1)
            return config

        def wait_for_jobs(self, wait_for_completion=False):
            while not self.batchSystem.getIssuedJobIDs():
                pass
            while not self.batchSystem.getRunningJobIDs():
                time.sleep(0.1)
            if wait_for_completion:
                while self.batchSystem.getRunningJobIDs():
                    time.sleep(0.1)
                    # pass does not work here, which is why I'm using time.sleep(0.1):
                    # for jobID,data in self.runningJobMap.iteritems():
                    # RuntimeError: dictionary changed size during iteration

        @classmethod
        def tearDownClass(cls):
            shutil.rmtree(cls.tempDir)

class MesosBatchSystemTest(hidden.AbstractBatchSystemTest):
    """
    Tests against the Mesos batch system
    """

    def createBatchSystem(self):
        shutil.rmtree('/tmp/mesos/', ignore_errors=True)
        # Launch Mesos Master and Slave if running MesosBatchSystemTest
        self.master = self.MesosMasterThread()
        self.master.start()
        self.slave = self.MesosSlaveThread()
        self.slave.start()
        while self.master.popen is None or self.slave.popen is None:
            log.info("Waiting for master and slave processes")
            time.sleep(1)
        return MesosBatchSystem(config=self.config, maxCpus=2, maxMemory=20, badExecutor=False)

    def setUp(self):
        super(MesosBatchSystemTest, self).setUp()

    def tearDown(self):
        super(MesosBatchSystemTest, self).tearDown()
        self.slave.popen.kill()
        self.slave.join()
        self.master.popen.kill()
        self.master.join()
        self.batchSystem.shutDown()

    class MesosThread(threading.Thread):
        __metaclass__ = ABCMeta

        # Lock is used because subprocess is NOT thread safe: http://tinyurl.com/pkp5pgq
        lock = threading.Lock()

        def __init__(self):
            threading.Thread.__init__(self)
            self.popen = None

        @abstractmethod
        def mesosCommand(self):
            raise NotImplementedError

        def run(self):
            with self.lock:
                self.popen = subprocess.Popen(self.mesosCommand())
            self.popen.wait()
            log.info('Exiting %s', self.__class__.__name__)

    class MesosMasterThread(MesosThread):
        def mesosCommand(self):
            return ['/usr/local/sbin/mesos-master', '--registry=in_memory', '--ip=127.0.0.1']

    class MesosSlaveThread(MesosThread):
        def mesosCommand(self):
            return ['/usr/local/sbin/mesos-slave', '--ip=127.0.0.1', '--master=127.0.0.1:5050']

# FIXME: the single machine backend does not support crucial methods necessary for this test

class SingleMachineBatchSystemTest(hidden.AbstractBatchSystemTest):
    def createBatchSystem(self):
        return SingleMachineBatchSystem(config=self.config, maxCpus=2, maxMemory=20)

    def testGetIssuedJobIDs(self):
        # TODO: Fix SingleMachineBatchSystem to support this call
        pass

    def testGetRunningJobIDs(self):
        # TODO: Fix SingleMachineBatchSystem to support this call
        pass

    def testGetUpdatedJob(self):
        # TODO: Fix SingleMachineBatchSystem to support this call
        pass

    def testIssueJob(self):
        # TODO: Fix SingleMachineBatchSystem to support this call
        pass

    def testKillJobs(self):
        # TODO: Fix SingleMachineBatchSystem to support this call
        pass

<commit_msg>Avoid absolute paths to mesos-master and -slave in tests<commit_after>from abc import ABCMeta, abstractmethod
import logging
import os
import shutil
import subprocess
import tempfile
import threading
import unittest
from xml.etree import ElementTree
import time

from jobTree.batchSystems.abstractBatchSystem import AbstractBatchSystem
from jobTree.batchSystems.mesos import MesosBatchSystem
from jobTree.batchSystems.singleMachine import SingleMachineBatchSystem
from jobTree.batchSystems.abstractBatchSystem import InsufficientSystemResources
from jobTree.test import JobTreeTest

log = logging.getLogger(__name__)

class hidden:
    """
    Hide abstract base class from unittest's test case loader

    http://stackoverflow.com/questions/1323455/python-unit-test-with-base-and-sub-class#answer-25695512
    """

    class AbstractBatchSystemTest(JobTreeTest):
        """
        A base test case with generic tests that every batch system should pass
        """
        __metaclass__ = ABCMeta

        @abstractmethod
        def createBatchSystem(self):
            """
            :rtype: AbstractBatchSystem
            """
            raise NotImplementedError

        def setUp(self):
            super(hidden.AbstractBatchSystemTest, self).setUp()
            self.batchSystem = self.createBatchSystem()
            self.tempDir = tempfile.mkdtemp()

        config = None
        tempDir = None

        @classmethod
        def setUpClass(cls):
            cls.config = cls._createDummyConfig()
            cls.tempDir = tempfile.mkdtemp()

        ###################
        # Begin Tests   #
        ###################

        def testIssueJob(self):
            test_path = os.path.join(self.tempDir, 'test.txt')
            jobCommand = 'touch {}'.format(test_path)
            self.batchSystem.issueJob(jobCommand, memory=10, cpu=1)
            self.wait_for_jobs(wait_for_completion=True)
            self.assertTrue(os.path.exists(test_path))

        def testCheckResourceRequest(self):
            self.assertRaises(InsufficientSystemResources, self.batchSystem.checkResourceRequest, memory=1000, cpu=200)
            self.assertRaises(InsufficientSystemResources, self.batchSystem.checkResourceRequest, memory=5, cpu=200)
            self.assertRaises(InsufficientSystemResources, self.batchSystem.checkResourceRequest, memory=100, cpu=1)
            self.assertRaises(AssertionError, self.batchSystem.checkResourceRequest, memory=None, cpu=1)
            self.assertRaises(AssertionError, self.batchSystem.checkResourceRequest, memory=10, cpu=None)
            self.batchSystem.checkResourceRequest(memory=10, cpu=1)

        def testGetIssuedJobIDs(self):
            self.batchSystem.issueJob('sleep 1', memory=10, cpu=1)
            self.batchSystem.issueJob('sleep 1', memory=10, cpu=1)
            self.assertEqual({0, 1}, set( self.batchSystem.getIssuedJobIDs()))

        def testGetRunningJobIDs(self):
            self.batchSystem.issueJob('sleep 100', memory=10, cpu=1)
            self.batchSystem.issueJob('sleep 100', memory=10, cpu=1)
            self.wait_for_jobs()
            self.assertEqual({0, 1}, set( self.batchSystem.getRunningJobIDs().keys()))

        def testKillJobs(self):
            jobCommand = 'sleep 100'
            self.batchSystem.issueJob(jobCommand, memory=10, cpu=1)
            self.wait_for_jobs()
            self.assertEqual([0], self.batchSystem.getRunningJobIDs().keys())
            self.batchSystem.killJobs([0])
            self.assertEqual({}, self.batchSystem.getRunningJobIDs())
            # Make sure that killJob doesn't hang on unknown job IDs
            self.batchSystem.killJobs([0])

        def testGetUpdatedJob(self):
            delay = 1
            num_jobs = 2
            jobCommand = 'sleep %i' % delay
            for i in range( num_jobs ):
                self.batchSystem.issueJob(jobCommand, memory=10, cpu=1)
            jobs = set((i,0) for i in range(num_jobs))
            self.wait_for_jobs(wait_for_completion=True)
            for i in range( num_jobs ):
                jobs.remove(self.batchSystem.getUpdatedJob(delay * 2))
            self.assertFalse( jobs )

        def testGetRescueJobFrequency(self):
            self.assertTrue(self.batchSystem.getRescueJobFrequency() > 0)

        #################
        #   End Tests   #
        #################

        @staticmethod
        def _createDummyConfig():
            config = ElementTree.Element("config")
            config.attrib["log_level"] = 'DEBUG'
            config.attrib["job_store"] = '.'
            config.attrib["parasol_command"] = None
            config.attrib["try_count"] = str(2)
            config.attrib["max_job_duration"] = str(1)
            config.attrib["batch_system"] = None
            config.attrib["job_time"] = str(1)
            config.attrib["max_log_file_size"] = str(1)
            config.attrib["default_memory"] = str(1)
            config.attrib["default_cpu"] = str(1)
            config.attrib["max_cpus"] = str(1)
            config.attrib["max_memory"] = str(1)
            config.attrib["max_threads"] = str(1)
            return config

        def wait_for_jobs(self, wait_for_completion=False):
            while not self.batchSystem.getIssuedJobIDs():
                pass
            while not self.batchSystem.getRunningJobIDs():
                time.sleep(0.1)
            if wait_for_completion:
                while self.batchSystem.getRunningJobIDs():
                    time.sleep(0.1)
                    # pass does not work here, which is why I'm using time.sleep(0.1):
                    # for jobID,data in self.runningJobMap.iteritems():
                    # RuntimeError: dictionary changed size during iteration

        @classmethod
        def tearDownClass(cls):
            shutil.rmtree(cls.tempDir)

class MesosBatchSystemTest(hidden.AbstractBatchSystemTest):
    """
    Tests against the Mesos batch system
    """

    def createBatchSystem(self):
        shutil.rmtree('/tmp/mesos/', ignore_errors=True)
        # Launch Mesos Master and Slave if running MesosBatchSystemTest
        self.master = self.MesosMasterThread()
        self.master.start()
        self.slave = self.MesosSlaveThread()
        self.slave.start()
        while self.master.popen is None or self.slave.popen is None:
            log.info("Waiting for master and slave processes")
            time.sleep(1)
        return MesosBatchSystem(config=self.config, maxCpus=2, maxMemory=20, badExecutor=False)

    def setUp(self):
        super(MesosBatchSystemTest, self).setUp()

    def tearDown(self):
        super(MesosBatchSystemTest, self).tearDown()
        self.slave.popen.kill()
        self.slave.join()
        self.master.popen.kill()
        self.master.join()
        self.batchSystem.shutDown()

    class MesosThread(threading.Thread):
        __metaclass__ = ABCMeta

        # Lock is used because subprocess is NOT thread safe: http://tinyurl.com/pkp5pgq
        lock = threading.Lock()

        def __init__(self):
            threading.Thread.__init__(self)
            self.popen = None

        @abstractmethod
        def mesosCommand(self):
            raise NotImplementedError

        def run(self):
            with self.lock:
                self.popen = subprocess.Popen(self.mesosCommand())
            self.popen.wait()
            log.info('Exiting %s', self.__class__.__name__)

    class MesosMasterThread(MesosThread):
        def mesosCommand(self):
            return ['mesos-master', '--registry=in_memory', '--ip=127.0.0.1']

    class MesosSlaveThread(MesosThread):
        def mesosCommand(self):
            return ['mesos-slave', '--ip=127.0.0.1', '--master=127.0.0.1:5050']

# FIXME: the single machine backend does not support crucial methods necessary for this test

class SingleMachineBatchSystemTest(hidden.AbstractBatchSystemTest):
    def createBatchSystem(self):
        return SingleMachineBatchSystem(config=self.config, maxCpus=2, maxMemory=20)

    def testGetIssuedJobIDs(self):
        # TODO: Fix SingleMachineBatchSystem to support this call
        pass

    def testGetRunningJobIDs(self):
        # TODO: Fix SingleMachineBatchSystem to support this call
        pass

    def testGetUpdatedJob(self):
        # TODO: Fix SingleMachineBatchSystem to support this call
        pass

    def testIssueJob(self):
        # TODO: Fix SingleMachineBatchSystem to support this call
        pass

    def testKillJobs(self):
        # TODO: Fix SingleMachineBatchSystem to support this call
        pass

<|endoftext|>


<commit_before>#!/usr/bin/python3
#
# This file is part of Plinth.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

import argparse
import django.conf
from django.contrib.messages import constants as message_constants
import django.core.management
import django.core.wsgi
import importlib
import logging
import os
import stat
import sys

import cherrypy
from cherrypy.process.plugins import Daemonizer

from plinth import cfg
from plinth import module_loader
from plinth import service

logger = logging.getLogger(__name__)

arguments = None

def parse_arguments():
    """Parse command line arguments"""
    parser = argparse.ArgumentParser(
        description='Plinth web interface for FreedomBox',
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument(
        '--pidfile', default=cfg.pidfile,
        help='specify a file in which the server may write its pid')
    # TODO: server_dir is actually a url prefix; use a better variable name
    parser.add_argument(
        '--server_dir', default=cfg.server_dir,
        help='web server path under which to serve')
    parser.add_argument(
        '--debug', action='store_true', default=cfg.debug,
        help='enable debugging and run server *insecurely*')
    parser.add_argument(
        '--no-daemon', action='store_true', default=cfg.no_daemon,
        help='do not start as a daemon')
    parser.add_argument(
        '--diagnose', action='store_true', default=False,
        help='run diagnostic tests and exit')

    global arguments
    arguments = parser.parse_args()
    cfg.pidfile = arguments.pidfile
    cfg.server_dir = arguments.server_dir
    cfg.debug = arguments.debug
    cfg.no_daemon = arguments.no_daemon

def setup_logging():
    """Setup logging framework"""
    # Don't propagate cherrypy log messages to root logger
    logging.getLogger('cherrypy').propagate = False

    cherrypy.log.error_file = cfg.status_log_file
    cherrypy.log.access_file = cfg.access_log_file
    if not cfg.no_daemon:
        cherrypy.log.screen = False

def setup_server():
    """Setup CherryPy server"""
    logger.info('Setting up CherryPy server')

    # Set the PID file path
    try:
        if cfg.pidfile:
            from cherrypy.process.plugins import PIDFile
            PIDFile(cherrypy.engine, cfg.pidfile).subscribe()
    except AttributeError:
        pass

    # Configure default server
    cherrypy.config.update(
        {'server.socket_host': cfg.host,
         'server.socket_port': cfg.port,
         'server.thread_pool': 10,
         # Avoid stating files once per second in production
         'engine.autoreload.on': cfg.debug,
        })

    application = django.core.wsgi.get_wsgi_application()
    cherrypy.tree.graft(application, cfg.server_dir)

    static_dir = os.path.join(cfg.file_root, 'static')
    config = {
        '/': {'tools.staticdir.root': static_dir,
              'tools.staticdir.on': True,
              'tools.staticdir.dir': '.'}}
    cherrypy.tree.mount(None, django.conf.settings.STATIC_URL, config)
    logger.debug('Serving static directory %s on %s', static_dir,
                 django.conf.settings.STATIC_URL)

    js_dir = '/usr/share/javascript'
    js_url = '/javascript'
    config = {
        '/': {'tools.staticdir.root': js_dir,
              'tools.staticdir.on': True,
              'tools.staticdir.dir': '.'}}
    cherrypy.tree.mount(None, js_url, config)
    logger.debug('Serving javascript directory %s on %s', js_dir, js_url)

    manual_dir = os.path.join(cfg.doc_dir, 'images')
    manual_url = '/'.join([cfg.server_dir, 'help/manual/images']) \
        .replace('//', '/')
    config = {
        '/': {'tools.staticdir.root': manual_dir,
              'tools.staticdir.on': True,
              'tools.staticdir.dir': '.'}}
    cherrypy.tree.mount(None, manual_url, config)
    logger.debug('Serving manual images %s on %s', manual_dir, manual_url)

    for module_import_path in module_loader.loaded_modules:
        module = importlib.import_module(module_import_path)
        module_name = module_import_path.split('.')[-1]
        module_path = os.path.dirname(module.__file__)
        static_dir = os.path.join(module_path, 'static')
        if not os.path.isdir(static_dir):
            continue

        config = {
            '/': {'tools.staticdir.root': static_dir,
                  'tools.staticdir.on': True,
                  'tools.staticdir.dir': '.'}}
        urlprefix = "%s%s" % (django.conf.settings.STATIC_URL, module_name)
        cherrypy.tree.mount(None, urlprefix, config)
        logger.debug('Serving static directory %s on %s', static_dir,
                     urlprefix)

    if not cfg.no_daemon:
        Daemonizer(cherrypy.engine).subscribe()

    cherrypy.engine.signal_handler.subscribe()

def configure_django():
    """Setup Django configuration in the absense of .settings file"""

    # In module_loader.py we reverse URLs for the menu before having a proper
    # request. In this case, get_script_prefix (used by reverse) returns the
    # wrong prefix. Set it here manually to have the correct prefix right away.
    django.core.urlresolvers.set_script_prefix(cfg.server_dir)

    logging_configuration = {
        'version': 1,
        'disable_existing_loggers': False,
        'formatters': {
            'default': {
                'format':
                '[%(asctime)s] %(name)-14s %(levelname)-8s %(message)s',
                }
            },
        'handlers': {
            'file': {
                'class': 'logging.FileHandler',
                'filename': cfg.status_log_file,
                'formatter': 'default'
                },
            'console': {
                'class': 'logging.StreamHandler',
                'formatter': 'default'
                }
            },
        'root': {
            'handlers': ['console', 'file'],
            'level': 'DEBUG' if cfg.debug else 'INFO'
            }
        }

    templates = [
        {
            'BACKEND': 'django.template.backends.django.DjangoTemplates',
            'APP_DIRS': True,
            'OPTIONS': {
                'context_processors': [
                    'django.contrib.auth.context_processors.auth',
                    'django.template.context_processors.debug',
                    'django.template.context_processors.i18n',
                    'django.template.context_processors.media',
                    'django.template.context_processors.request',
                    'django.template.context_processors.static',
                    'django.template.context_processors.tz',
                    'django.contrib.messages.context_processors.messages',
                    'plinth.context_processors.common',
                ],
            },
        },
    ]

    applications = ['bootstrapform',
                    'django.contrib.auth',
                    'django.contrib.contenttypes',
                    'django.contrib.messages',
                    'stronghold',
                    'plinth']
    applications += module_loader.get_modules_to_load()
    sessions_directory = os.path.join(cfg.data_dir, 'sessions')

    secure_proxy_ssl_header = None
    if cfg.secure_proxy_ssl_header:
        secure_proxy_ssl_header = (cfg.secure_proxy_ssl_header, 'https')

    # Read translated languages from the 'locale' directory
    languages = [('en', 'English')]
    locale_dir = os.path.join(os.path.dirname(__file__), 'locale')
    if os.path.isdir(locale_dir):
        translated_language_codes = next(os.walk(locale_dir))[1]
        all_languages = dict(django.conf.global_settings.LANGUAGES)
        for code in translated_language_codes:
            if code in all_languages:
                languages.append((code, all_languages[code]))
        languages = sorted(languages, key=lambda tup: tup[1])

    django.conf.settings.configure(
        ALLOWED_HOSTS=['*'],
        CACHES={'default':
                {'BACKEND': 'django.core.cache.backends.dummy.DummyCache'}},
        DATABASES={'default':
                   {'ENGINE': 'django.db.backends.sqlite3',
                    'NAME': cfg.store_file}},
        DEBUG=cfg.debug,
        INSTALLED_APPS=applications,
        LANGUAGES=languages,
        LOGGING=logging_configuration,
        LOGIN_URL='users:login',
        LOGIN_REDIRECT_URL='apps:index',
        LOGOUT_URL='users:logout',
        MESSAGE_TAGS={message_constants.ERROR: 'danger'},
        MIDDLEWARE_CLASSES=(
            'django.contrib.sessions.middleware.SessionMiddleware',
            'django.middleware.locale.LocaleMiddleware',
            'django.middleware.common.CommonMiddleware',
            'django.middleware.csrf.CsrfViewMiddleware',
            'django.contrib.auth.middleware.AuthenticationMiddleware',
            'django.contrib.messages.middleware.MessageMiddleware',
            'django.middleware.clickjacking.XFrameOptionsMiddleware',
            'stronghold.middleware.LoginRequiredMiddleware',
            'plinth.modules.first_boot.middleware.FirstBootMiddleware',
        ),
        ROOT_URLCONF='plinth.urls',
        SECURE_PROXY_SSL_HEADER=secure_proxy_ssl_header,
        SESSION_ENGINE='django.contrib.sessions.backends.file',
        SESSION_FILE_PATH=sessions_directory,
        STATIC_URL='/'.join([cfg.server_dir, 'static/']).replace('//', '/'),
        STRONGHOLD_PUBLIC_NAMED_URLS=('users:login', 'users:logout'),
        TEMPLATES=templates,
        USE_L10N=True,
        USE_X_FORWARDED_HOST=cfg.use_x_forwarded_host)
    django.setup()

    logger.info('Configured Django with applications - %s', applications)

    logger.info('Creating or adding new tables to data file')
    django.core.management.call_command('migrate', interactive=False)
    os.chmod(cfg.store_file, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP)

def run_diagnostics_and_exit():
    """Run diagostics on all modules and exit."""
    module = importlib.import_module('plinth.modules.diagnostics.diagnostics')
    error_code = 0
    try:
        module.run_on_all_modules()
    except Exception as exception:
        logger.exception('Error running diagnostics - %s', exception)
        error_code = 2

    for module, results in module.current_results['results'].items():
        for test, result_value in results:
            print('{result_value}: {module}: {test}'.format(
                result_value=result_value, test=test, module=module))
            if result_value != 'passed':
                error_code = 1

    sys.exit(error_code)

def main():
    """Intialize and start the application"""
    cfg.read()

    parse_arguments()

    setup_logging()

    service.init()

    configure_django()

    logger.info('Configuration loaded from file - %s', cfg.CONFIG_FILE)
    logger.info('Script prefix - %s', cfg.server_dir)

    module_loader.load_modules()

    if arguments.diagnose:
        run_diagnostics_and_exit()

    setup_server()

    cherrypy.engine.start()
    cherrypy.engine.block()

if __name__ == '__main__':
    main()
<commit_msg>Make migration work for older version upgrades<commit_after>#!/usr/bin/python3
#
# This file is part of Plinth.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

import argparse
import django.conf
from django.contrib.messages import constants as message_constants
import django.core.management
import django.core.wsgi
import importlib
import logging
import os
import stat
import sys

import cherrypy
from cherrypy.process.plugins import Daemonizer

from plinth import cfg
from plinth import module_loader
from plinth import service

logger = logging.getLogger(__name__)

arguments = None

def parse_arguments():
    """Parse command line arguments"""
    parser = argparse.ArgumentParser(
        description='Plinth web interface for FreedomBox',
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument(
        '--pidfile', default=cfg.pidfile,
        help='specify a file in which the server may write its pid')
    # TODO: server_dir is actually a url prefix; use a better variable name
    parser.add_argument(
        '--server_dir', default=cfg.server_dir,
        help='web server path under which to serve')
    parser.add_argument(
        '--debug', action='store_true', default=cfg.debug,
        help='enable debugging and run server *insecurely*')
    parser.add_argument(
        '--no-daemon', action='store_true', default=cfg.no_daemon,
        help='do not start as a daemon')
    parser.add_argument(
        '--diagnose', action='store_true', default=False,
        help='run diagnostic tests and exit')

    global arguments
    arguments = parser.parse_args()
    cfg.pidfile = arguments.pidfile
    cfg.server_dir = arguments.server_dir
    cfg.debug = arguments.debug
    cfg.no_daemon = arguments.no_daemon

def setup_logging():
    """Setup logging framework"""
    # Don't propagate cherrypy log messages to root logger
    logging.getLogger('cherrypy').propagate = False

    cherrypy.log.error_file = cfg.status_log_file
    cherrypy.log.access_file = cfg.access_log_file
    if not cfg.no_daemon:
        cherrypy.log.screen = False

def setup_server():
    """Setup CherryPy server"""
    logger.info('Setting up CherryPy server')

    # Set the PID file path
    try:
        if cfg.pidfile:
            from cherrypy.process.plugins import PIDFile
            PIDFile(cherrypy.engine, cfg.pidfile).subscribe()
    except AttributeError:
        pass

    # Configure default server
    cherrypy.config.update(
        {'server.socket_host': cfg.host,
         'server.socket_port': cfg.port,
         'server.thread_pool': 10,
         # Avoid stating files once per second in production
         'engine.autoreload.on': cfg.debug,
        })

    application = django.core.wsgi.get_wsgi_application()
    cherrypy.tree.graft(application, cfg.server_dir)

    static_dir = os.path.join(cfg.file_root, 'static')
    config = {
        '/': {'tools.staticdir.root': static_dir,
              'tools.staticdir.on': True,
              'tools.staticdir.dir': '.'}}
    cherrypy.tree.mount(None, django.conf.settings.STATIC_URL, config)
    logger.debug('Serving static directory %s on %s', static_dir,
                 django.conf.settings.STATIC_URL)

    js_dir = '/usr/share/javascript'
    js_url = '/javascript'
    config = {
        '/': {'tools.staticdir.root': js_dir,
              'tools.staticdir.on': True,
              'tools.staticdir.dir': '.'}}
    cherrypy.tree.mount(None, js_url, config)
    logger.debug('Serving javascript directory %s on %s', js_dir, js_url)

    manual_dir = os.path.join(cfg.doc_dir, 'images')
    manual_url = '/'.join([cfg.server_dir, 'help/manual/images']) \
        .replace('//', '/')
    config = {
        '/': {'tools.staticdir.root': manual_dir,
              'tools.staticdir.on': True,
              'tools.staticdir.dir': '.'}}
    cherrypy.tree.mount(None, manual_url, config)
    logger.debug('Serving manual images %s on %s', manual_dir, manual_url)

    for module_import_path in module_loader.loaded_modules:
        module = importlib.import_module(module_import_path)
        module_name = module_import_path.split('.')[-1]
        module_path = os.path.dirname(module.__file__)
        static_dir = os.path.join(module_path, 'static')
        if not os.path.isdir(static_dir):
            continue

        config = {
            '/': {'tools.staticdir.root': static_dir,
                  'tools.staticdir.on': True,
                  'tools.staticdir.dir': '.'}}
        urlprefix = "%s%s" % (django.conf.settings.STATIC_URL, module_name)
        cherrypy.tree.mount(None, urlprefix, config)
        logger.debug('Serving static directory %s on %s', static_dir,
                     urlprefix)

    if not cfg.no_daemon:
        Daemonizer(cherrypy.engine).subscribe()

    cherrypy.engine.signal_handler.subscribe()

def configure_django():
    """Setup Django configuration in the absense of .settings file"""

    # In module_loader.py we reverse URLs for the menu before having a proper
    # request. In this case, get_script_prefix (used by reverse) returns the
    # wrong prefix. Set it here manually to have the correct prefix right away.
    django.core.urlresolvers.set_script_prefix(cfg.server_dir)

    logging_configuration = {
        'version': 1,
        'disable_existing_loggers': False,
        'formatters': {
            'default': {
                'format':
                '[%(asctime)s] %(name)-14s %(levelname)-8s %(message)s',
                }
            },
        'handlers': {
            'file': {
                'class': 'logging.FileHandler',
                'filename': cfg.status_log_file,
                'formatter': 'default'
                },
            'console': {
                'class': 'logging.StreamHandler',
                'formatter': 'default'
                }
            },
        'root': {
            'handlers': ['console', 'file'],
            'level': 'DEBUG' if cfg.debug else 'INFO'
            }
        }

    templates = [
        {
            'BACKEND': 'django.template.backends.django.DjangoTemplates',
            'APP_DIRS': True,
            'OPTIONS': {
                'context_processors': [
                    'django.contrib.auth.context_processors.auth',
                    'django.template.context_processors.debug',
                    'django.template.context_processors.i18n',
                    'django.template.context_processors.media',
                    'django.template.context_processors.request',
                    'django.template.context_processors.static',
                    'django.template.context_processors.tz',
                    'django.contrib.messages.context_processors.messages',
                    'plinth.context_processors.common',
                ],
            },
        },
    ]

    applications = ['bootstrapform',
                    'django.contrib.auth',
                    'django.contrib.contenttypes',
                    'django.contrib.messages',
                    'stronghold',
                    'plinth']
    applications += module_loader.get_modules_to_load()
    sessions_directory = os.path.join(cfg.data_dir, 'sessions')

    secure_proxy_ssl_header = None
    if cfg.secure_proxy_ssl_header:
        secure_proxy_ssl_header = (cfg.secure_proxy_ssl_header, 'https')

    # Read translated languages from the 'locale' directory
    languages = [('en', 'English')]
    locale_dir = os.path.join(os.path.dirname(__file__), 'locale')
    if os.path.isdir(locale_dir):
        translated_language_codes = next(os.walk(locale_dir))[1]
        all_languages = dict(django.conf.global_settings.LANGUAGES)
        for code in translated_language_codes:
            if code in all_languages:
                languages.append((code, all_languages[code]))
        languages = sorted(languages, key=lambda tup: tup[1])

    django.conf.settings.configure(
        ALLOWED_HOSTS=['*'],
        CACHES={'default':
                {'BACKEND': 'django.core.cache.backends.dummy.DummyCache'}},
        DATABASES={'default':
                   {'ENGINE': 'django.db.backends.sqlite3',
                    'NAME': cfg.store_file}},
        DEBUG=cfg.debug,
        INSTALLED_APPS=applications,
        LANGUAGES=languages,
        LOGGING=logging_configuration,
        LOGIN_URL='users:login',
        LOGIN_REDIRECT_URL='apps:index',
        LOGOUT_URL='users:logout',
        MESSAGE_TAGS={message_constants.ERROR: 'danger'},
        MIDDLEWARE_CLASSES=(
            'django.contrib.sessions.middleware.SessionMiddleware',
            'django.middleware.locale.LocaleMiddleware',
            'django.middleware.common.CommonMiddleware',
            'django.middleware.csrf.CsrfViewMiddleware',
            'django.contrib.auth.middleware.AuthenticationMiddleware',
            'django.contrib.messages.middleware.MessageMiddleware',
            'django.middleware.clickjacking.XFrameOptionsMiddleware',
            'stronghold.middleware.LoginRequiredMiddleware',
            'plinth.modules.first_boot.middleware.FirstBootMiddleware',
        ),
        ROOT_URLCONF='plinth.urls',
        SECURE_PROXY_SSL_HEADER=secure_proxy_ssl_header,
        SESSION_ENGINE='django.contrib.sessions.backends.file',
        SESSION_FILE_PATH=sessions_directory,
        STATIC_URL='/'.join([cfg.server_dir, 'static/']).replace('//', '/'),
        STRONGHOLD_PUBLIC_NAMED_URLS=('users:login', 'users:logout'),
        TEMPLATES=templates,
        USE_L10N=True,
        USE_X_FORWARDED_HOST=cfg.use_x_forwarded_host)
    django.setup()

    logger.info('Configured Django with applications - %s', applications)

    logger.info('Creating or adding new tables to data file')
    django.core.management.call_command('migrate', '--fake-initial',
                                        interactive=False)
    os.chmod(cfg.store_file, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP)

def run_diagnostics_and_exit():
    """Run diagostics on all modules and exit."""
    module = importlib.import_module('plinth.modules.diagnostics.diagnostics')
    error_code = 0
    try:
        module.run_on_all_modules()
    except Exception as exception:
        logger.exception('Error running diagnostics - %s', exception)
        error_code = 2

    for module, results in module.current_results['results'].items():
        for test, result_value in results:
            print('{result_value}: {module}: {test}'.format(
                result_value=result_value, test=test, module=module))
            if result_value != 'passed':
                error_code = 1

    sys.exit(error_code)

def main():
    """Intialize and start the application"""
    cfg.read()

    parse_arguments()

    setup_logging()

    service.init()

    configure_django()

    logger.info('Configuration loaded from file - %s', cfg.CONFIG_FILE)
    logger.info('Script prefix - %s', cfg.server_dir)

    module_loader.load_modules()

    if arguments.diagnose:
        run_diagnostics_and_exit()

    setup_server()

    cherrypy.engine.start()
    cherrypy.engine.block()

if __name__ == '__main__':
    main()
<|endoftext|>