<commit_before>import hashlib
import itertools
import re
import sys
import types
from collections import defaultdict
from importlib import import_module

from django.conf import settings
from django.conf.urls import include, url
from django.core.exceptions import ValidationError
from django.db import models
from django.db.models import Q, signals
from django.urls import NoReverseMatch, reverse
from django.utils.translation import get_language, ugettext_lazy as _

from feincms3.utils import concrete_model, positional, validation_error

__all__ = (
    "AppsMixin",
    "apps_middleware",
    "apps_urlconf",
    "page_for_app_request",
    "reverse_any",
    "reverse_app",
    "reverse_fallback",
)

def reverse_any(viewnames, urlconf=None, args=None, kwargs=None, *fargs, **fkwargs):
    """
    Tries reversing a list of viewnames with the same arguments, and returns
    the first result where no ``NoReverseMatch`` exception is raised.

    Usage::

        url = reverse_any((
            'blog:article-detail',
            'articles:article-detail',
        ), kwargs={'slug': 'article-slug'})
    """

    for viewname in viewnames:
        try:
            return reverse(viewname, urlconf, args, kwargs, *fargs, **fkwargs)
        except NoReverseMatch:
            pass

    raise NoReverseMatch(
        "Reverse for any of '%s' with arguments '%s' and keyword arguments"
        " '%s' not found." % ("', '".join(viewnames), args or [], kwargs or {})
    )

def reverse_app(namespaces, viewname, *args, **kwargs):
    """
    Reverse app URLs, preferring the active language.

    ``reverse_app`` first generates a list of viewnames and passes them on
    to ``reverse_any``.

    Assuming that we're trying to reverse the URL of an article detail view,
    that the project is configured with german, english and french as available
    languages, french as active language and that the current article is a
    publication, the viewnames are:

    - ``apps-fr.publications.article-detail``
    - ``apps-fr.articles.article-detail``
    - ``apps-de.publications.article-detail``
    - ``apps-de.articles.article-detail``
    - ``apps-en.publications.article-detail``
    - ``apps-en.articles.article-detail``

    reverse_app tries harder returning an URL in the correct language than
    returning an URL for the correct instance namespace.

    Example::

        url = reverse_app(
            ("category-1", "blog"),
            "post-detail",
            kwargs={"year": 2016, "slug": "my-cat"},
        )
    """

    page_model = concrete_model(AppsMixin)
    current = get_language()
    language_codes_namespaces = []
    if current is not None:
        language_codes_namespaces.append(
            "%s-%s" % (page_model.LANGUAGE_CODES_NAMESPACE, current)
        )
    language_codes_namespaces.extend(
        "%s-%s" % (page_model.LANGUAGE_CODES_NAMESPACE, language[0])
        for language in settings.LANGUAGES
        if language[0] != current
    )
    viewnames = [
        ":".join(r)
        for r in itertools.product(
            language_codes_namespaces,
            (namespaces if isinstance(namespaces, (list, tuple)) else (namespaces,)),
            (viewname,),
        )
    ]
    return reverse_any(viewnames, *args, **kwargs)

def reverse_fallback(fallback, fn, *args, **kwargs):
    """
    Returns the result of ``fn(*args, **kwargs)``, or ``fallback`` if the
    former raises a ``NoReverseMatch`` exception. This is especially useful for
    reversing app URLs from outside the app and you do not want crashes if the
    app isn't available anywhere.

    The following two examples are equivalent, choose whichever you like best::

        reverse_fallback("/", lambda: reverse_app(
            ("articles",),
            "article-detail",
            kwargs={"slug": self.slug},
        ))

        reverse_fallback(
            "/",
            reverse_app
            ("articles",),
            "article-detail",
            kwargs={"slug": self.slug},
        )
    """
    try:
        return fn(*args, **kwargs)
    except NoReverseMatch:
        return fallback

@positional(0)
def apps_urlconf(apps=None):
    """apps_urlconf(*, apps=None)
    Generates a dynamic URLconf Python module including all applications in
    their assigned place and adding the ``urlpatterns`` from ``ROOT_URLCONF``
    at the end. Returns the value of ``ROOT_URLCONF`` directly if there are
    no active applications.

    Since Django uses an LRU cache for URL resolvers, we try hard to only
    generate a changed URLconf when application URLs actually change.

    The application URLconfs are put in nested namespaces:

    - The outer application namespace is ``apps`` by default. This value can be
      overridden by setting the ``LANGUAGE_CODES_NAMESPACE`` class attribute of
      the page class to a different value. The instance namespaces consist of
      the ``LANGUAGE_CODES_NAMESPACE`` value with a language added at the end.
      As long as you're always using ``reverse_app`` you do not have to know
      the specifics.
    - The inner namespace is the app namespace, where the application
      namespace is defined by the app itself (assign ``app_name`` in the
      same module as ``urlpatterns``) and the instance namespace is defined
      by the application name (from ``APPLICATIONS``).

    Modules stay around as long as the Python (most of the time WSGI) process
    lives. Unloading modules is tricky and probably not worth it since the
    URLconf modules shouldn't gobble up much memory.

    The set of applications can be overridden by passing a list of
    ``(path, application, app_instance_namespace, language_code)`` tuples.
    """

    page_model = concrete_model(AppsMixin)
    if apps is None:
        fields = ("path", "application", "app_instance_namespace", "language_code")
        apps = (
            page_model.objects.active()
            .with_tree_fields(False)
            .exclude(app_instance_namespace="")
            .values_list(*fields)
            .order_by(*fields)
        )

    if not apps:
        # No point wrapping ROOT_URLCONF if there are no additional URLs
        return settings.ROOT_URLCONF

    key = ",".join(itertools.chain.from_iterable(apps))
    module_name = "urlconf_%s" % hashlib.md5(key.encode("utf-8")).hexdigest()

    if module_name not in sys.modules:
        app_config = {app[0]: app[2] for app in page_model.APPLICATIONS if app[0]}

        m = types.ModuleType(str(module_name))  # Correct for Python 2 and 3

        mapping = defaultdict(list)
        for path, application, app_instance_namespace, language_code in apps:
            if application not in app_config:
                continue
            mapping[language_code].append(
                url(
                    r"^%s" % re.escape(path.lstrip("/")),
                    include(
                        app_config[application]["urlconf"],
                        namespace=app_instance_namespace,
                    ),
                )
            )

        m.urlpatterns = [
            url(
                r"",
                include(
                    (instances, page_model.LANGUAGE_CODES_NAMESPACE),
                    namespace="%s-%s"
                    % (page_model.LANGUAGE_CODES_NAMESPACE, language_code),
                ),
            )
            for language_code, instances in mapping.items()
        ]

        # Append patterns from ROOT_URLCONF instead of including them because
        # i18n_patterns only work in the root URLconf.
        m.urlpatterns += import_module(settings.ROOT_URLCONF).urlpatterns
        sys.modules[module_name] = m

    return module_name

@positional(1)
def page_for_app_request(request, queryset=None):
    """page_for_app_request(request, *, queryset=None)
    Returns the current page if we're inside an app. Should only be called
    while processing app views. Will pass along exceptions caused by
    non-existing or duplicated apps (this should never happen inside an app
    because :func:`~feincms3.apps.apps_urlconf` wouldn't have added the app
    in the first place if a matching page wouldn't exist, but still.)

    Example::

        def article_detail(request, slug):
            page = page_for_app_request(request)
            page.activate_language(request)
            instance = get_object_or_404(Article, slug=slug)
            return render(request, "articles/article_detail.html", {
                "article": article,
                "page": page,
            })

    It is possible to override the queryset used to fetch a page instance. The
    default implementation simply uses the first concrete subclass of
    :class:`~feincms3.apps.AppsMixin`.
    """

    page_model = concrete_model(AppsMixin)
    if queryset is None:
        queryset = page_model.objects.with_tree_fields()
    # Unguarded - if this fails, we shouldn't even be here.
    return queryset.get(
        language_code=request.resolver_match.namespaces[0][
            len(page_model.LANGUAGE_CODES_NAMESPACE) + 1 :
        ],
        app_instance_namespace=request.resolver_match.namespaces[1],
    )

def apps_middleware(get_response):
    """
    This middleware must be put in ``MIDDLEWARE``; it simply assigns
    the return value of :func:`~feincms3.apps.apps_urlconf` to
    ``request.urlconf``. This middleware should probably be one of the first
    since it has to run before any resolving happens.
    """

    def middleware(request):
        request.urlconf = apps_urlconf()
        return get_response(request)

    return middleware

# Alias, for old times
AppsMiddleware = apps_middleware

class AppsMixin(models.Model):
    """
    The page class should inherit this mixin. It adds an ``application`` field
    containing the selected application, and an ``app_instance_namespace``
    field which contains the instance namespace of the application. Most of
    the time these two fields will have the same value. This mixin also ensures
    that applications can only be activated on leaf nodes in the page tree.
    Note that currently the :class:`~feincms3.mixins.LanguageMixin` is a
    required dependency of :mod:`feincms3.apps`.

    ``APPLICATIONS`` contains a list of application configurations consisting
    of:

    - Application name (used as instance namespace)
    - User-visible name
    - Options dictionary

    Available options include:

    - ``urlconf``: The path to the URLconf module for the application. Besides
      the ``urlpatterns`` list the module should probably also specify a
      ``app_name``.
    - ``required_fields``: A list of page class fields which must be non-empty
      for the application to work. The values are checked in
      ``AppsMixin.clean_fields``.
    - ``app_instance_namespace``: A callable which receives the page instance
      as its only argument and returns a string suitable for use as an
      instance namespace.

    Usage::

        from django.utils.translation import ugettext_lazy as _
        from feincms3.apps import AppsMixin
        from feincms3.mixins import LanguageMixin
        from feincms3.pages import AbstractPage

        class Page(AppsMixin, LanguageMixin, AbstractPage):
            APPLICATIONS = [
                ("publications", _("publications"), {
                    "urlconf": "app.articles.urls",
                }),
                ("blog", _("blog"), {
                    "urlconf": "app.articles.urls",
                }),
                ("contact", _("contact form"), {
                    "urlconf": "app.forms.contact_urls",
                }),
                ("teams", _("teams"), {
                    "urlconf": "app.teams.urls",
                    "app_instance_namespace": lambda page: "%s-%s" % (
                        page.application,
                        page.team_id,
                    ),
                    "required_fields": ("team",),
                }),
            ]
    """

    #: Override this to set a different name for the outer namespace.
    LANGUAGE_CODES_NAMESPACE = "apps"

    application = models.CharField(
        _("application"),
        max_length=20,
        blank=True,
        choices=(("", ""),),  # Non-empty choices for get_*_display
    )
    app_instance_namespace = models.CharField(
        _("app instance namespace"), max_length=100, editable=False
    )

    class Meta:
        abstract = True

    def application_config(self):
        """
        Returns the selected application options dictionary, or ``None`` if
        no application is selected or the application does not exist anymore.
        """
        try:
            return {app[0]: app[2] for app in self.APPLICATIONS if app[0]}[
                self.application
            ]
        except KeyError:
            return None

    def save(self, *args, **kwargs):
        """
        Updates ``app_instance_namespace``.
        """
        app_config = self.application_config() or {}
        # If app_config is empty or None, this simply sets
        # app_instance_namespace to the empty string.
        setter = app_config.get(
            "app_instance_namespace", lambda instance: instance.application
        )
        self.app_instance_namespace = setter(self)

        super(AppsMixin, self).save(*args, **kwargs)

    save.alters_data = True

    def clean_fields(self, exclude=None):
        """
        Checks that application nodes do not have any descendants, and that
        required fields for the selected application (if any) are filled out,
        and that app instances with the same instance namespace and same
        language only exist once on a site.
        """
        exclude = [] if exclude is None else exclude
        super(AppsMixin, self).clean_fields(exclude)

        if self.parent and (
            self.parent.application
            or self.parent.ancestors().exclude(application="").exists()
        ):
            error = _("Apps may not have any descendants.")
            raise validation_error(
                _("Invalid parent: %s") % (error,), field="parent", exclude=exclude
            )

        if self.application and self.children.exists():
            raise validation_error(
                _("Apps may not have any descendants in the tree."),
                field="application",
                exclude=exclude,
            )

        app_config = self.application_config()
        if app_config and app_config.get("required_fields"):
            missing = [
                field
                for field in app_config["required_fields"]
                if not getattr(self, field)
            ]
            if missing:
                error = _("This field is required for the application %s.") % (
                    self.get_application_display(),
                )
                errors = {}
                for field in missing:
                    if field in exclude:
                        errors.setdefault("__all__", []).append(
                            "%s: %s" % (field, error)
                        )
                    else:
                        errors[field] = error

                raise ValidationError(errors)

        if app_config:
            app_instance_namespace = app_config.get(
                "app_instance_namespace", lambda instance: instance.application
            )(self)

            if self.__class__._default_manager.filter(
                Q(app_instance_namespace=app_instance_namespace),
                Q(language_code=self.language_code),
                ~Q(pk=self.pk),
            ).exists():
                fields = ["application"]
                fields.extend(app_config.get("required_fields", ()))
                raise ValidationError(
                    {field: _(_("This exact app already exists.")) for field in fields}
                )

    @staticmethod
    def fill_application_choices(sender, **kwargs):
        """
        Fills in the choices for ``application`` from the ``APPLICATIONS``
        class variable. This method is a receiver of Django's
        ``class_prepared`` signal.
        """
        if issubclass(sender, AppsMixin) and not sender._meta.abstract:
            sender._meta.get_field("application").choices = [
                app[:2] for app in sender.APPLICATIONS
            ]

signals.class_prepared.connect(AppsMixin.fill_application_choices)
<commit_msg>Make app validation fail even if there are no common fields in the form and model<commit_after>import hashlib
import itertools
import re
import sys
import types
from collections import defaultdict
from importlib import import_module

from django.conf import settings
from django.conf.urls import include, url
from django.core.exceptions import ValidationError
from django.db import models
from django.db.models import Q, signals
from django.urls import NoReverseMatch, reverse
from django.utils.translation import get_language, ugettext_lazy as _

from feincms3.utils import concrete_model, positional, validation_error

__all__ = (
    "AppsMixin",
    "apps_middleware",
    "apps_urlconf",
    "page_for_app_request",
    "reverse_any",
    "reverse_app",
    "reverse_fallback",
)

def reverse_any(viewnames, urlconf=None, args=None, kwargs=None, *fargs, **fkwargs):
    """
    Tries reversing a list of viewnames with the same arguments, and returns
    the first result where no ``NoReverseMatch`` exception is raised.

    Usage::

        url = reverse_any((
            'blog:article-detail',
            'articles:article-detail',
        ), kwargs={'slug': 'article-slug'})
    """

    for viewname in viewnames:
        try:
            return reverse(viewname, urlconf, args, kwargs, *fargs, **fkwargs)
        except NoReverseMatch:
            pass

    raise NoReverseMatch(
        "Reverse for any of '%s' with arguments '%s' and keyword arguments"
        " '%s' not found." % ("', '".join(viewnames), args or [], kwargs or {})
    )

def reverse_app(namespaces, viewname, *args, **kwargs):
    """
    Reverse app URLs, preferring the active language.

    ``reverse_app`` first generates a list of viewnames and passes them on
    to ``reverse_any``.

    Assuming that we're trying to reverse the URL of an article detail view,
    that the project is configured with german, english and french as available
    languages, french as active language and that the current article is a
    publication, the viewnames are:

    - ``apps-fr.publications.article-detail``
    - ``apps-fr.articles.article-detail``
    - ``apps-de.publications.article-detail``
    - ``apps-de.articles.article-detail``
    - ``apps-en.publications.article-detail``
    - ``apps-en.articles.article-detail``

    reverse_app tries harder returning an URL in the correct language than
    returning an URL for the correct instance namespace.

    Example::

        url = reverse_app(
            ("category-1", "blog"),
            "post-detail",
            kwargs={"year": 2016, "slug": "my-cat"},
        )
    """

    page_model = concrete_model(AppsMixin)
    current = get_language()
    language_codes_namespaces = []
    if current is not None:
        language_codes_namespaces.append(
            "%s-%s" % (page_model.LANGUAGE_CODES_NAMESPACE, current)
        )
    language_codes_namespaces.extend(
        "%s-%s" % (page_model.LANGUAGE_CODES_NAMESPACE, language[0])
        for language in settings.LANGUAGES
        if language[0] != current
    )
    viewnames = [
        ":".join(r)
        for r in itertools.product(
            language_codes_namespaces,
            (namespaces if isinstance(namespaces, (list, tuple)) else (namespaces,)),
            (viewname,),
        )
    ]
    return reverse_any(viewnames, *args, **kwargs)

def reverse_fallback(fallback, fn, *args, **kwargs):
    """
    Returns the result of ``fn(*args, **kwargs)``, or ``fallback`` if the
    former raises a ``NoReverseMatch`` exception. This is especially useful for
    reversing app URLs from outside the app and you do not want crashes if the
    app isn't available anywhere.

    The following two examples are equivalent, choose whichever you like best::

        reverse_fallback("/", lambda: reverse_app(
            ("articles",),
            "article-detail",
            kwargs={"slug": self.slug},
        ))

        reverse_fallback(
            "/",
            reverse_app
            ("articles",),
            "article-detail",
            kwargs={"slug": self.slug},
        )
    """
    try:
        return fn(*args, **kwargs)
    except NoReverseMatch:
        return fallback

@positional(0)
def apps_urlconf(apps=None):
    """apps_urlconf(*, apps=None)
    Generates a dynamic URLconf Python module including all applications in
    their assigned place and adding the ``urlpatterns`` from ``ROOT_URLCONF``
    at the end. Returns the value of ``ROOT_URLCONF`` directly if there are
    no active applications.

    Since Django uses an LRU cache for URL resolvers, we try hard to only
    generate a changed URLconf when application URLs actually change.

    The application URLconfs are put in nested namespaces:

    - The outer application namespace is ``apps`` by default. This value can be
      overridden by setting the ``LANGUAGE_CODES_NAMESPACE`` class attribute of
      the page class to a different value. The instance namespaces consist of
      the ``LANGUAGE_CODES_NAMESPACE`` value with a language added at the end.
      As long as you're always using ``reverse_app`` you do not have to know
      the specifics.
    - The inner namespace is the app namespace, where the application
      namespace is defined by the app itself (assign ``app_name`` in the
      same module as ``urlpatterns``) and the instance namespace is defined
      by the application name (from ``APPLICATIONS``).

    Modules stay around as long as the Python (most of the time WSGI) process
    lives. Unloading modules is tricky and probably not worth it since the
    URLconf modules shouldn't gobble up much memory.

    The set of applications can be overridden by passing a list of
    ``(path, application, app_instance_namespace, language_code)`` tuples.
    """

    page_model = concrete_model(AppsMixin)
    if apps is None:
        fields = ("path", "application", "app_instance_namespace", "language_code")
        apps = (
            page_model.objects.active()
            .with_tree_fields(False)
            .exclude(app_instance_namespace="")
            .values_list(*fields)
            .order_by(*fields)
        )

    if not apps:
        # No point wrapping ROOT_URLCONF if there are no additional URLs
        return settings.ROOT_URLCONF

    key = ",".join(itertools.chain.from_iterable(apps))
    module_name = "urlconf_%s" % hashlib.md5(key.encode("utf-8")).hexdigest()

    if module_name not in sys.modules:
        app_config = {app[0]: app[2] for app in page_model.APPLICATIONS if app[0]}

        m = types.ModuleType(str(module_name))  # Correct for Python 2 and 3

        mapping = defaultdict(list)
        for path, application, app_instance_namespace, language_code in apps:
            if application not in app_config:
                continue
            mapping[language_code].append(
                url(
                    r"^%s" % re.escape(path.lstrip("/")),
                    include(
                        app_config[application]["urlconf"],
                        namespace=app_instance_namespace,
                    ),
                )
            )

        m.urlpatterns = [
            url(
                r"",
                include(
                    (instances, page_model.LANGUAGE_CODES_NAMESPACE),
                    namespace="%s-%s"
                    % (page_model.LANGUAGE_CODES_NAMESPACE, language_code),
                ),
            )
            for language_code, instances in mapping.items()
        ]

        # Append patterns from ROOT_URLCONF instead of including them because
        # i18n_patterns only work in the root URLconf.
        m.urlpatterns += import_module(settings.ROOT_URLCONF).urlpatterns
        sys.modules[module_name] = m

    return module_name

@positional(1)
def page_for_app_request(request, queryset=None):
    """page_for_app_request(request, *, queryset=None)
    Returns the current page if we're inside an app. Should only be called
    while processing app views. Will pass along exceptions caused by
    non-existing or duplicated apps (this should never happen inside an app
    because :func:`~feincms3.apps.apps_urlconf` wouldn't have added the app
    in the first place if a matching page wouldn't exist, but still.)

    Example::

        def article_detail(request, slug):
            page = page_for_app_request(request)
            page.activate_language(request)
            instance = get_object_or_404(Article, slug=slug)
            return render(request, "articles/article_detail.html", {
                "article": article,
                "page": page,
            })

    It is possible to override the queryset used to fetch a page instance. The
    default implementation simply uses the first concrete subclass of
    :class:`~feincms3.apps.AppsMixin`.
    """

    page_model = concrete_model(AppsMixin)
    if queryset is None:
        queryset = page_model.objects.with_tree_fields()
    # Unguarded - if this fails, we shouldn't even be here.
    return queryset.get(
        language_code=request.resolver_match.namespaces[0][
            len(page_model.LANGUAGE_CODES_NAMESPACE) + 1 :
        ],
        app_instance_namespace=request.resolver_match.namespaces[1],
    )

def apps_middleware(get_response):
    """
    This middleware must be put in ``MIDDLEWARE``; it simply assigns
    the return value of :func:`~feincms3.apps.apps_urlconf` to
    ``request.urlconf``. This middleware should probably be one of the first
    since it has to run before any resolving happens.
    """

    def middleware(request):
        request.urlconf = apps_urlconf()
        return get_response(request)

    return middleware

# Alias, for old times
AppsMiddleware = apps_middleware

class AppsMixin(models.Model):
    """
    The page class should inherit this mixin. It adds an ``application`` field
    containing the selected application, and an ``app_instance_namespace``
    field which contains the instance namespace of the application. Most of
    the time these two fields will have the same value. This mixin also ensures
    that applications can only be activated on leaf nodes in the page tree.
    Note that currently the :class:`~feincms3.mixins.LanguageMixin` is a
    required dependency of :mod:`feincms3.apps`.

    ``APPLICATIONS`` contains a list of application configurations consisting
    of:

    - Application name (used as instance namespace)
    - User-visible name
    - Options dictionary

    Available options include:

    - ``urlconf``: The path to the URLconf module for the application. Besides
      the ``urlpatterns`` list the module should probably also specify a
      ``app_name``.
    - ``required_fields``: A list of page class fields which must be non-empty
      for the application to work. The values are checked in
      ``AppsMixin.clean_fields``.
    - ``app_instance_namespace``: A callable which receives the page instance
      as its only argument and returns a string suitable for use as an
      instance namespace.

    Usage::

        from django.utils.translation import ugettext_lazy as _
        from feincms3.apps import AppsMixin
        from feincms3.mixins import LanguageMixin
        from feincms3.pages import AbstractPage

        class Page(AppsMixin, LanguageMixin, AbstractPage):
            APPLICATIONS = [
                ("publications", _("publications"), {
                    "urlconf": "app.articles.urls",
                }),
                ("blog", _("blog"), {
                    "urlconf": "app.articles.urls",
                }),
                ("contact", _("contact form"), {
                    "urlconf": "app.forms.contact_urls",
                }),
                ("teams", _("teams"), {
                    "urlconf": "app.teams.urls",
                    "app_instance_namespace": lambda page: "%s-%s" % (
                        page.application,
                        page.team_id,
                    ),
                    "required_fields": ("team",),
                }),
            ]
    """

    #: Override this to set a different name for the outer namespace.
    LANGUAGE_CODES_NAMESPACE = "apps"

    application = models.CharField(
        _("application"),
        max_length=20,
        blank=True,
        choices=(("", ""),),  # Non-empty choices for get_*_display
    )
    app_instance_namespace = models.CharField(
        _("app instance namespace"), max_length=100, editable=False
    )

    class Meta:
        abstract = True

    def application_config(self):
        """
        Returns the selected application options dictionary, or ``None`` if
        no application is selected or the application does not exist anymore.
        """
        try:
            return {app[0]: app[2] for app in self.APPLICATIONS if app[0]}[
                self.application
            ]
        except KeyError:
            return None

    def save(self, *args, **kwargs):
        """
        Updates ``app_instance_namespace``.
        """
        app_config = self.application_config() or {}
        # If app_config is empty or None, this simply sets
        # app_instance_namespace to the empty string.
        setter = app_config.get(
            "app_instance_namespace", lambda instance: instance.application
        )
        self.app_instance_namespace = setter(self)

        super(AppsMixin, self).save(*args, **kwargs)

    save.alters_data = True

    def clean_fields(self, exclude=None):
        """
        Checks that application nodes do not have any descendants, and that
        required fields for the selected application (if any) are filled out,
        and that app instances with the same instance namespace and same
        language only exist once on a site.
        """
        exclude = [] if exclude is None else exclude
        super(AppsMixin, self).clean_fields(exclude)

        if self.parent and (
            self.parent.application
            or self.parent.ancestors().exclude(application="").exists()
        ):
            error = _("Apps may not have any descendants.")
            raise validation_error(
                _("Invalid parent: %s") % (error,), field="parent", exclude=exclude
            )

        if self.application and self.children.exists():
            raise validation_error(
                _("Apps may not have any descendants in the tree."),
                field="application",
                exclude=exclude,
            )

        app_config = self.application_config()
        if app_config and app_config.get("required_fields"):
            missing = [
                field
                for field in app_config["required_fields"]
                if not getattr(self, field)
            ]
            if missing:
                error = _("This field is required for the application %s.") % (
                    self.get_application_display(),
                )
                errors = {}
                for field in missing:
                    if field in exclude:
                        errors.setdefault("__all__", []).append(
                            "%s: %s" % (field, error)
                        )
                    else:
                        errors[field] = error

                raise ValidationError(errors)

        if app_config:
            app_instance_namespace = app_config.get(
                "app_instance_namespace", lambda instance: instance.application
            )(self)

            if self.__class__._default_manager.filter(
                Q(app_instance_namespace=app_instance_namespace),
                Q(language_code=self.language_code),
                ~Q(pk=self.pk),
            ).exists():
                fields = ["__all__", "application"]
                fields.extend(app_config.get("required_fields", ()))
                raise ValidationError(
                    {
                        field: _("This exact app already exists.")
                        for field in fields
                        if field not in exclude
                    }
                )

    @staticmethod
    def fill_application_choices(sender, **kwargs):
        """
        Fills in the choices for ``application`` from the ``APPLICATIONS``
        class variable. This method is a receiver of Django's
        ``class_prepared`` signal.
        """
        if issubclass(sender, AppsMixin) and not sender._meta.abstract:
            sender._meta.get_field("application").choices = [
                app[:2] for app in sender.APPLICATIONS
            ]

signals.class_prepared.connect(AppsMixin.fill_application_choices)
<|endoftext|>


<commit_before># Copyright (C) 2011, Google Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import copy
import logging

_log = logging.getLogger(__name__)

# Yes, it's a hypergraph.
# FIXME: Should this function live with the ports somewhere?
# Perhaps this should move onto PortFactory?
def _baseline_search_hypergraph(host, port_names):
    hypergraph = {}

    # FIXME: Should we get this constant from somewhere?
    fallback_path = ['LayoutTests']

    port_factory = host.port_factory
    for port_name in port_names:
        port = port_factory.get(port_name)
        webkit_base = port.webkit_base()
        search_path = port.baseline_search_path()
        if search_path:
            hypergraph[port_name] = [host.filesystem.relpath(path, webkit_base) for path in search_path] + fallback_path
    return hypergraph

# FIXME: Should this function be somewhere more general?
def _invert_dictionary(dictionary):
    inverted_dictionary = {}
    for key, value in dictionary.items():
        if inverted_dictionary.get(value):
            inverted_dictionary[value].append(key)
        else:
            inverted_dictionary[value] = [key]
    return inverted_dictionary

# FIXME: This class is massively more complicated than necessary now that we only need to support Chromium baselines.
class BaselineOptimizer(object):
    def __init__(self, host, port_names):
        self._host = host
        self._filesystem = self._host.filesystem
        self._scm = self._host.scm()
        self._hypergraph = _baseline_search_hypergraph(host, port_names)
        self._directories = reduce(set.union, map(set, self._hypergraph.values()))

    def read_results_by_directory(self, baseline_name):
        results_by_directory = {}
        for directory in self._directories:
            path = self._filesystem.join(self._scm.checkout_root, directory, baseline_name)
            if self._filesystem.exists(path):
                results_by_directory[directory] = self._filesystem.sha1(path)
        return results_by_directory

    def _results_by_port_name(self, results_by_directory):
        results_by_port_name = {}
        for port_name, search_path in self._hypergraph.items():
            for directory in search_path:
                if directory in results_by_directory:
                    results_by_port_name[port_name] = results_by_directory[directory]
                    break
        return results_by_port_name

    def _most_specific_common_directory(self, port_names):
        paths = [self._hypergraph[port_name] for port_name in port_names]
        common_directories = reduce(set.intersection, map(set, paths))

        def score(directory):
            return sum([path.index(directory) for path in paths])

        _, directory = sorted([(score(directory), directory) for directory in common_directories])[0]
        return directory

    def _filter_port_names_by_result(self, predicate, port_names_by_result):
        filtered_port_names_by_result = {}
        for result, port_names in port_names_by_result.items():
            filtered_port_names = filter(predicate, port_names)
            if filtered_port_names:
                filtered_port_names_by_result[result] = filtered_port_names
        return filtered_port_names_by_result

    def _place_results_in_most_specific_common_directory(self, port_names_by_result, results_by_directory):
        for result, port_names in port_names_by_result.items():
            directory = self._most_specific_common_directory(port_names)
            results_by_directory[directory] = result

    def _find_optimal_result_placement(self, baseline_name):
        results_by_directory = self.read_results_by_directory(baseline_name)
        results_by_port_name = self._results_by_port_name(results_by_directory)
        port_names_by_result = _invert_dictionary(results_by_port_name)

        new_results_by_directory = self._optimize_by_most_specific_common_directory(results_by_directory, results_by_port_name, port_names_by_result)
        if not new_results_by_directory:
            new_results_by_directory = self._optimize_by_pushing_results_up(results_by_directory, results_by_port_name, port_names_by_result)

        return results_by_directory, new_results_by_directory

    def _optimize_by_most_specific_common_directory(self, results_by_directory, results_by_port_name, port_names_by_result):
        new_results_by_directory = {}
        unsatisfied_port_names_by_result = port_names_by_result
        while unsatisfied_port_names_by_result:
            self._place_results_in_most_specific_common_directory(unsatisfied_port_names_by_result, new_results_by_directory)
            new_results_by_port_name = self._results_by_port_name(new_results_by_directory)

            def is_unsatisfied(port_name):
                return results_by_port_name[port_name] != new_results_by_port_name[port_name]

            new_unsatisfied_port_names_by_result = self._filter_port_names_by_result(is_unsatisfied, port_names_by_result)

            if len(new_unsatisfied_port_names_by_result.values()) >= len(unsatisfied_port_names_by_result.values()):
                return {}  # Frowns. We do not appear to be converging.
            unsatisfied_port_names_by_result = new_unsatisfied_port_names_by_result

        return new_results_by_directory

    def _optimize_by_pushing_results_up(self, results_by_directory, results_by_port_name, port_names_by_result):
        try:
            results_by_directory = results_by_directory
            best_so_far = results_by_directory
            while True:
                new_results_by_directory = copy.copy(best_so_far)
                for port_name in self._hypergraph.keys():
                    fallback_path = self._hypergraph[port_name]
                    current_index, current_directory = self._find_in_fallbackpath(fallback_path, results_by_port_name[port_name], best_so_far)
                    current_result = results_by_port_name[port_name]
                    for index in range(current_index + 1, len(fallback_path)):
                        new_directory = fallback_path[index]
                        if not new_directory in new_results_by_directory:
                            new_results_by_directory[new_directory] = current_result
                            if current_directory in new_results_by_directory:
                                del new_results_by_directory[current_directory]
                        elif new_results_by_directory[new_directory] == current_result:
                            if current_directory in new_results_by_directory:
                                del new_results_by_directory[current_directory]
                        else:
                            # The new_directory contains a different result, so stop trying to push results up.
                            break

                if len(new_results_by_directory) >= len(best_so_far):
                    # We've failed to improve, so give up.
                    break
                best_so_far = new_results_by_directory

            return best_so_far
        except KeyError as e:
            # FIXME: KeyErrors get raised if we're missing baselines. We should handle this better.
            return {}

    def _find_in_fallbackpath(self, fallback_path, current_result, results_by_directory):
        for index, directory in enumerate(fallback_path):
            if directory in results_by_directory and (results_by_directory[directory] == current_result):
                return index, directory
        assert False, "result %s not found in fallback_path %s, %s" % (current_result, fallback_path, results_by_directory)

    # FIXME: Inline this function into its callers.
    def _filtered_results_by_port_name(self, results_by_directory):
        return self._results_by_port_name(results_by_directory)

    def _platform(self, filename):
        platform_dir = 'LayoutTests' + self._filesystem.sep + 'platform' + self._filesystem.sep
        if filename.startswith(platform_dir):
            return filename.replace(platform_dir, '').split(self._filesystem.sep)[0]
        platform_dir = self._filesystem.join(self._scm.checkout_root, platform_dir)
        if filename.startswith(platform_dir):
            return filename.replace(platform_dir, '').split(self._filesystem.sep)[0]
        return '(generic)'

    def _move_baselines(self, baseline_name, results_by_directory, new_results_by_directory):
        data_for_result = {}
        for directory, result in results_by_directory.items():
            if not result in data_for_result:
                source = self._filesystem.join(self._scm.checkout_root, directory, baseline_name)
                data_for_result[result] = self._filesystem.read_binary_file(source)

        file_names = []
        for directory, result in results_by_directory.items():
            if new_results_by_directory.get(directory) != result:
                file_names.append(self._filesystem.join(self._scm.checkout_root, directory, baseline_name))
        if file_names:
            _log.debug("    Deleting:")
            for platform_dir in sorted(self._platform(filename) for filename in file_names):
                _log.debug("      " + platform_dir)
            self._scm.delete_list(file_names)
        else:
            _log.debug("    (Nothing to delete)")

        file_names = []
        for directory, result in new_results_by_directory.items():
            if results_by_directory.get(directory) != result:
                destination = self._filesystem.join(self._scm.checkout_root, directory, baseline_name)
                self._filesystem.maybe_make_directory(self._filesystem.split(destination)[0])
                self._filesystem.write_binary_file(destination, data_for_result[result])
                file_names.append(destination)
        if file_names:
            _log.debug("    Adding:")
            for platform_dir in sorted(self._platform(filename) for filename in file_names):
                _log.debug("      " + platform_dir)
            self._scm.add_list(file_names)
        else:
            _log.debug("    (Nothing to add)")

    def directories_by_result(self, baseline_name):
        results_by_directory = self.read_results_by_directory(baseline_name)
        return _invert_dictionary(results_by_directory)

    def write_by_directory(self, results_by_directory, writer, indent):
        for path in sorted(results_by_directory):
            writer("%s%s: %s" % (indent, self._platform(path), results_by_directory[path][0:6]))

    def optimize(self, baseline_name):
        basename = self._filesystem.basename(baseline_name)
        results_by_directory, new_results_by_directory = self._find_optimal_result_placement(baseline_name)
        self.new_results_by_directory = new_results_by_directory
        if new_results_by_directory == results_by_directory:
            if new_results_by_directory:
                _log.debug("  %s: (already optimal)" % basename)
                self.write_by_directory(results_by_directory, _log.debug, "    ")
            else:
                _log.debug("  %s: (no baselines found)" % basename)
            return True
        if self._filtered_results_by_port_name(results_by_directory) != self._filtered_results_by_port_name(new_results_by_directory):
            _log.warning("  %s: optimization failed" % basename)
            self.write_by_directory(results_by_directory, _log.warning, "      ")
            return False

        _log.debug("  %s:" % basename)
        _log.debug("    Before: ")
        self.write_by_directory(results_by_directory, _log.debug, "      ")
        _log.debug("    After: ")
        self.write_by_directory(new_results_by_directory, _log.debug, "      ")

        self._move_baselines(baseline_name, results_by_directory, new_results_by_directory)
        return True
<commit_msg>Minor cleanups to baselineoptimizer. Now that the hypergraph is just a regular old tree the code can be a little simpler. This is in preparation for trying to simplify the actual logic now that it's a tree.<commit_after># Copyright (C) 2011, Google Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import copy
import logging

_log = logging.getLogger(__name__)

# FIXME: Should this function live with the ports somewhere?
# Perhaps this should move onto PortFactory?
def _baseline_search_tree(host, port_names):
    tree = {}
    for port_name in port_names:
        port = host.port_factory.get(port_name)
        webkit_base = port.webkit_base()
        search_path = port.baseline_search_path()
        if search_path:
            tree[port_name] = [host.filesystem.relpath(path, webkit_base) for path in search_path] + ['LayoutTests']
    return tree

# FIXME: Should this function be somewhere more general?
def _invert_dictionary(dictionary):
    inverted_dictionary = {}
    for key, value in dictionary.items():
        if inverted_dictionary.get(value):
            inverted_dictionary[value].append(key)
        else:
            inverted_dictionary[value] = [key]
    return inverted_dictionary

# FIXME: This class is massively more complicated than necessary now that we only need to support Chromium baselines.
class BaselineOptimizer(object):
    def __init__(self, host, port_names):
        self._filesystem = host.filesystem
        self._scm = host.scm()
        self._tree = _baseline_search_tree(host, port_names)

    def read_results_by_directory(self, baseline_name):
        results_by_directory = {}
        directories = reduce(set.union, map(set, self._tree.values()))
        for directory in directories:
            path = self._filesystem.join(self._scm.checkout_root, directory, baseline_name)
            if self._filesystem.exists(path):
                results_by_directory[directory] = self._filesystem.sha1(path)
        return results_by_directory

    def _results_by_port_name(self, results_by_directory):
        results_by_port_name = {}
        for port_name, search_path in self._tree.items():
            for directory in search_path:
                if directory in results_by_directory:
                    results_by_port_name[port_name] = results_by_directory[directory]
                    break
        return results_by_port_name

    def _most_specific_common_directory(self, port_names):
        paths = [self._tree[port_name] for port_name in port_names]
        common_directories = reduce(set.intersection, map(set, paths))

        def score(directory):
            return sum([path.index(directory) for path in paths])

        _, directory = sorted([(score(directory), directory) for directory in common_directories])[0]
        return directory

    def _filter_port_names_by_result(self, predicate, port_names_by_result):
        filtered_port_names_by_result = {}
        for result, port_names in port_names_by_result.items():
            filtered_port_names = filter(predicate, port_names)
            if filtered_port_names:
                filtered_port_names_by_result[result] = filtered_port_names
        return filtered_port_names_by_result

    def _place_results_in_most_specific_common_directory(self, port_names_by_result, results_by_directory):
        for result, port_names in port_names_by_result.items():
            directory = self._most_specific_common_directory(port_names)
            results_by_directory[directory] = result

    def _find_optimal_result_placement(self, baseline_name):
        results_by_directory = self.read_results_by_directory(baseline_name)
        results_by_port_name = self._results_by_port_name(results_by_directory)
        port_names_by_result = _invert_dictionary(results_by_port_name)

        new_results_by_directory = self._optimize_by_most_specific_common_directory(results_by_directory, results_by_port_name, port_names_by_result)
        if not new_results_by_directory:
            new_results_by_directory = self._optimize_by_pushing_results_up(results_by_directory, results_by_port_name, port_names_by_result)

        return results_by_directory, new_results_by_directory

    def _optimize_by_most_specific_common_directory(self, results_by_directory, results_by_port_name, port_names_by_result):
        new_results_by_directory = {}
        unsatisfied_port_names_by_result = port_names_by_result
        while unsatisfied_port_names_by_result:
            self._place_results_in_most_specific_common_directory(unsatisfied_port_names_by_result, new_results_by_directory)
            new_results_by_port_name = self._results_by_port_name(new_results_by_directory)

            def is_unsatisfied(port_name):
                return results_by_port_name[port_name] != new_results_by_port_name[port_name]

            new_unsatisfied_port_names_by_result = self._filter_port_names_by_result(is_unsatisfied, port_names_by_result)

            if len(new_unsatisfied_port_names_by_result.values()) >= len(unsatisfied_port_names_by_result.values()):
                return {}  # Frowns. We do not appear to be converging.
            unsatisfied_port_names_by_result = new_unsatisfied_port_names_by_result

        return new_results_by_directory

    def _optimize_by_pushing_results_up(self, results_by_directory, results_by_port_name, port_names_by_result):
        try:
            results_by_directory = results_by_directory
            best_so_far = results_by_directory
            while True:
                new_results_by_directory = copy.copy(best_so_far)
                for port_name in self._tree.keys():
                    fallback_path = self._tree[port_name]
                    current_index, current_directory = self._find_in_fallbackpath(fallback_path, results_by_port_name[port_name], best_so_far)
                    current_result = results_by_port_name[port_name]
                    for index in range(current_index + 1, len(fallback_path)):
                        new_directory = fallback_path[index]
                        if not new_directory in new_results_by_directory:
                            new_results_by_directory[new_directory] = current_result
                            if current_directory in new_results_by_directory:
                                del new_results_by_directory[current_directory]
                        elif new_results_by_directory[new_directory] == current_result:
                            if current_directory in new_results_by_directory:
                                del new_results_by_directory[current_directory]
                        else:
                            # The new_directory contains a different result, so stop trying to push results up.
                            break

                if len(new_results_by_directory) >= len(best_so_far):
                    # We've failed to improve, so give up.
                    break
                best_so_far = new_results_by_directory

            return best_so_far
        except KeyError as e:
            # FIXME: KeyErrors get raised if we're missing baselines. We should handle this better.
            return {}

    def _find_in_fallbackpath(self, fallback_path, current_result, results_by_directory):
        for index, directory in enumerate(fallback_path):
            if directory in results_by_directory and (results_by_directory[directory] == current_result):
                return index, directory
        assert False, "result %s not found in fallback_path %s, %s" % (current_result, fallback_path, results_by_directory)

    def _platform(self, filename):
        platform_dir = 'LayoutTests' + self._filesystem.sep + 'platform' + self._filesystem.sep
        if filename.startswith(platform_dir):
            return filename.replace(platform_dir, '').split(self._filesystem.sep)[0]
        platform_dir = self._filesystem.join(self._scm.checkout_root, platform_dir)
        if filename.startswith(platform_dir):
            return filename.replace(platform_dir, '').split(self._filesystem.sep)[0]
        return '(generic)'

    def _move_baselines(self, baseline_name, results_by_directory, new_results_by_directory):
        data_for_result = {}
        for directory, result in results_by_directory.items():
            if not result in data_for_result:
                source = self._filesystem.join(self._scm.checkout_root, directory, baseline_name)
                data_for_result[result] = self._filesystem.read_binary_file(source)

        file_names = []
        for directory, result in results_by_directory.items():
            if new_results_by_directory.get(directory) != result:
                file_names.append(self._filesystem.join(self._scm.checkout_root, directory, baseline_name))
        if file_names:
            _log.debug("    Deleting:")
            for platform_dir in sorted(self._platform(filename) for filename in file_names):
                _log.debug("      " + platform_dir)
            self._scm.delete_list(file_names)
        else:
            _log.debug("    (Nothing to delete)")

        file_names = []
        for directory, result in new_results_by_directory.items():
            if results_by_directory.get(directory) != result:
                destination = self._filesystem.join(self._scm.checkout_root, directory, baseline_name)
                self._filesystem.maybe_make_directory(self._filesystem.split(destination)[0])
                self._filesystem.write_binary_file(destination, data_for_result[result])
                file_names.append(destination)
        if file_names:
            _log.debug("    Adding:")
            for platform_dir in sorted(self._platform(filename) for filename in file_names):
                _log.debug("      " + platform_dir)
            self._scm.add_list(file_names)
        else:
            _log.debug("    (Nothing to add)")

    def write_by_directory(self, results_by_directory, writer, indent):
        for path in sorted(results_by_directory):
            writer("%s%s: %s" % (indent, self._platform(path), results_by_directory[path][0:6]))

    def optimize(self, baseline_name):
        basename = self._filesystem.basename(baseline_name)
        results_by_directory, new_results_by_directory = self._find_optimal_result_placement(baseline_name)
        self.new_results_by_directory = new_results_by_directory
        if new_results_by_directory == results_by_directory:
            if new_results_by_directory:
                _log.debug("  %s: (already optimal)" % basename)
                self.write_by_directory(results_by_directory, _log.debug, "    ")
            else:
                _log.debug("  %s: (no baselines found)" % basename)
            return True
        if self._results_by_port_name(results_by_directory) != self._results_by_port_name(new_results_by_directory):
            _log.warning("  %s: optimization failed" % basename)
            self.write_by_directory(results_by_directory, _log.warning, "      ")
            return False

        _log.debug("  %s:" % basename)
        _log.debug("    Before: ")
        self.write_by_directory(results_by_directory, _log.debug, "      ")
        _log.debug("    After: ")
        self.write_by_directory(new_results_by_directory, _log.debug, "      ")

        self._move_baselines(baseline_name, results_by_directory, new_results_by_directory)
        return True
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    flask.logging
    ~~~~~~~~~~~~~

    Implements the logging support for Flask.

    :copyright: (c) 2010 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

from logging import getLogger, StreamHandler, Formatter, Logger, DEBUG

def create_logger(app):
    """Creates a logger for the given application.  This logger works
    similar to a regular Python logger but changes the effective logging
    level based on the application's debug flag.  Furthermore this
    function also removes all attached handlers in case there was a
    logger with the log name before.
    """

    class DebugLogger(Logger):
        def getEffectiveLevel(x):
            return DEBUG if app.debug else Logger.getEffectiveLevel(x)

    class DebugHandler(StreamHandler):
        def emit(x, record):
            StreamHandler.emit(x, record) if app.debug else None

    handler = DebugHandler()
    handler.setLevel(DEBUG)
    handler.setFormatter(Formatter(app.debug_log_format))
    logger = getLogger(app.logger_name)
    # just in case that was not a new logger, get rid of all the handlers
    # already attached to it.
    del logger.handlers[:]
    logger.__class__ = DebugLogger
    logger.addHandler(handler)
    return logger
<commit_msg>Extend the logging from the dynamically set logger class.  This fixes #234<commit_after># -*- coding: utf-8 -*-
"""
    flask.logging
    ~~~~~~~~~~~~~

    Implements the logging support for Flask.

    :copyright: (c) 2010 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

from logging import getLogger, StreamHandler, Formatter, getLoggerClass, DEBUG

def create_logger(app):
    """Creates a logger for the given application.  This logger works
    similar to a regular Python logger but changes the effective logging
    level based on the application's debug flag.  Furthermore this
    function also removes all attached handlers in case there was a
    logger with the log name before.
    """
    Logger = getLoggerClass()

    class DebugLogger(Logger):
        def getEffectiveLevel(x):
            return DEBUG if app.debug else Logger.getEffectiveLevel(x)

    class DebugHandler(StreamHandler):
        def emit(x, record):
            StreamHandler.emit(x, record) if app.debug else None

    handler = DebugHandler()
    handler.setLevel(DEBUG)
    handler.setFormatter(Formatter(app.debug_log_format))
    logger = getLogger(app.logger_name)
    # just in case that was not a new logger, get rid of all the handlers
    # already attached to it.
    del logger.handlers[:]
    logger.__class__ = DebugLogger
    logger.addHandler(handler)
    return logger
<|endoftext|>


<commit_before>#!/usr/bin/python3

import os
import shutil
import hashlib as hl
from datetime import date
from flask import render_template

# TODO find better newline
TEMPLATE_NEWLINE = 'FLASK_ARTICLE_NEWLINE'
CACHE_SEPERATOR = '\n\0\n'

class CacheHandler():
	'''Handler for caching files

	It uses 2 caches, the heap (limited) and the discspace.
	The last cached scripts are the ones in heap.
	'''
	def __init__(self, cache_limit=10, script_folder='scripts', cache_folder='.cache', debug=True, hash_alg='sha1'):
		'''Constructor

		Keyword arguments:
		cache_limit -- how many files should be in heap-cache (default 10)
		script_folder -- the directory for your scripts (default 'scripts')
		cache_folder -- the directory for cached files (default '.cache')
		debug -- specefies wether the handler will output debug information (default True)
		'''
		# TODO function docs
		# TODO test performance
		self.cache_limit = cache_limit
		self.script_folder = script_folder
		self.cache_folder = cache_folder
		self.debug = debug
		self.heap_cache = {}

		if hash_alg in hl.algorithms_available:
			self.hash_alg = hash_alg
		else:
			self.report('The hash algorithm "' + hash_alg + '" is not available'\
				' on this system! (Using SHA1 instead)')
			self.hash_alg = 'sha1'
		self.hash_size = hl.new(self.hash_alg).digest_size

		self.create_new_cache_dir()

	def report(self, data):
		if self.debug:
			print(data)

	def create_new_cache_dir(self):
		'''Deletes the cache dir and creates a new empty one'''
		self.report('Renewing cache dir')
		if os.path.exists(self.cache_folder) and os.path.isdir(self.cache_folder):
			shutil.rmtree(self.cache_folder)
		os.mkdir(self.cache_folder)

	def new_cache_entry(self, script):
		'''Creates a new cache entry in the cache dir for a file

		The entry saves a hash of the script and the parsed toc and content.
		'''
		name = script['Filename']
		self.report("Creating new cache entry for " + name)

		# Creating content
		file_data = open(self.script_folder + '/' + name, 'rb').read()
		h = hl.new(self.hash_alg, file_data).digest()
		disc_content = h
		for tag in script.keys():
			if type(script[tag]) == str:
				disc_content += CACHE_SEPERATOR.encode() \
								+ tag.encode() \
								+ b'\0' + script[tag].encode()

		# Writing cache entry to disc
		cache_path = self.cache_folder + '/' + name
		if os.path.exists(cache_path):
			os.remove(cache_path)
		cache_entry = open(cache_path, 'wb')
		cache_entry.write(disc_content)

		# Writing cache entry to heap
		if len(self.heap_cache) < self.cache_limit:
			pass
		elif name in self.heap_cache:
			pass
		else:
			# Delete the first entry in the cache to make room for the new one
			del self.heap_cache[list(self.heap_cache.keys())[0]]
		self.heap_cache[name] = (h, script)

	def check_cache_entry(self, name):
		'''Checks with the hash of a cache entry if a script has changed'''
		file_data = open(self.script_folder + '/' + name, 'rb').read()
		h = hl.new(self.hash_alg, file_data).digest()
		if name in self.heap_cache:
			# Check cache entry in heap
			cached_h = self.heap_cache[name][0]
		else:
			# Check cache entry on disc
			cache_path = self.cache_folder + '/' + name
			if not os.path.exists(cache_path):
				# The cache entry does not exist
				return False
			cached_h = open(cache_path, 'rb').read(self.hash_size)
		return h == cached_h

	def get_cached_content(self, name):
		if name in self.heap_cache:
			self.report('Loaded {} from heap cache'.format(name))
			return self.heap_cache[name][1]
		else:
			script = {}
			cache_entry = open(self.cache_folder + '/' + name, 'rb').read()
			cache_entry = cache_entry.split(CACHE_SEPERATOR.encode())
			for item in cache_entry:
				item = item.split(b'\0')
				tag = item[0].decode()
				value = item[1].decode()
				script[tag] = value
			self.__parse_date__(script)
			self.report('Loaded {} from disc cache'.format(name))
			return script

class ScriptLoader():
	'''Loader for the scripts'''
	def __init__(self, script_folder='scripts', cache_folder='.cache'):
		'''Constructor

		Keyword arguments:
		script_folder -- the directory for your scripts (default 'scripts')
		cache_folder -- the directory for cached files
		'''
		self.script_folder = script_folder
		self.cache_handler = CacheHandler(cache_folder = cache_folder)

	def render_article(self, script, template_file):
		'''Generates a 'render_template' function call which sets all the tags
		of your script. Evaluates that call and returns it.

		e.g.
		Your tags are "Author", "Title" and "Date".
		The resulting command will be:
		render_template(<filename>, Author = script["Author"], Title = script["Title"], Date = script["Date"])

		Keyword arguments:
		script -- parsed script
		template_file -- filename of your template file
		'''
		func_call = 'render_template("' + template_file + '"'
		for tag in script.keys():
			func_call += ', ' + tag + ' = script["' + tag + '"]'
		func_call += ')'
		return eval(func_call)

	def get_single_script(self, script_name):
		'''Opens single script and parses it so it can be used with a html template

		Keyword arguments:
		script_name -- script name
		'''
		return self.__get_script_info__(script_name)

	def __get_script_info__(self, script_name):
		'''Open a script and parse its content
		If the script is faulty, None is returned

		Keyword arguments:
		script_name -- script name
		'''
		script_info = {}
		script_info['Filename'] = script_name

		# The file couldn't be found
		script_path = os.getcwd() + '/' + self.script_folder
		if not script_name in os.listdir(script_path):
			return None

		# Checking cache entry
		if not self.cache_handler.check_cache_entry(script_name):
			# Parsing file and creating new cache entry
			data = open(script_path + '/' + script_name).read()
			data = data.split('\n')
			try:
				i = data.index('---End-Tags---')
			except:
				return None

			while '' in data:
				data.remove('')

			tags = data[:i-1]
			script_info['Content'] = TEMPLATE_NEWLINE.join(data[i:]).replace("\\\\", "
")

			for tag in tags:
				tag_name = tag[ tag.find('{')+1 : tag.find('}') ]
				tag_data = tag[ tag.rfind('{')+1 : tag.rfind('}') ]
				script_info[tag_name] = tag_data

			# Check script validity
			required_tags = ['Author','Title','Date']
			for rt in required_tags:
				if not rt in script_info.keys():
					return None
				# Make the title titlecased
				if rt == 'Title':
					if not script_info['Title'].startswith('_'):
						script_info['Title'] = script_info['Title'].title()
				# Create a date-object (later used for sorting)
				if rt == 'Date':
					self.__parse_date__(script_info)

			# Final formatting
			self.__parse_content__(script_info)

			# Caching
			self.cache_handler.new_cache_entry(script_info)
		else:
			# Loading parsed contents
			script_info = self.cache_handler.get_cached_content(script_name)

		return script_info

	def __parse_date__(self, script_info):
		d = script_info['Date'].split('/')
		d = date(day=int(d[0]), month=int(d[1]), year=int(d[2]))
		script_info['Date'] = d.strftime("%d. %h %Y")
		script_info['Date_object'] = d

	def __get_scripts__(self, sort=True, key='Date_object'):
		'''Get all the scripts from the script directory and optionally sort them

		Keyword arguments:
		sort -- if the scripts should be sorted by date (default True)
		key -- the key to sort after (default 'Date_object')
		'''
		script_infos = []
		files = os.listdir( os.getcwd() + '/' +  self.script_folder )
		for script in files:
			info = self.__get_script_info__(script)
			if info is not None:
				script_infos.append(info)
		if sort:
			script_infos.sort(reverse=True, key=lambda x : x[key])
		return script_infos

	def __parse_content__(self, script_info, numbered=True):
		'''Generates a table of contents and modifies an articles content

		Keyword arguments:
		content -- the content to parse
		numbered -- if the sections should be numbered (default True)
		'''
		script = script_info['Content'].split(TEMPLATE_NEWLINE)

		sections = []
		section_num = 0
		subsection_num = 1
		for i, line in enumerate(script):
			if line.startswith("**"):
				top_sec = sections.pop()
				num = float(str(section_num) + "." + str(subsection_num))
				if line.startswith("**_"):
					sec_title = line[3:].strip()
				else:
					sec_title = line[2:].strip().title()
				top_sec.append((sec_title, i, num))
				subsection_num += 1
				sections.append(top_sec)
			elif line.startswith("*"):
				subsection_num = 1
				section_num += 1
				if line.startswith("*_"):
					sec_title = line[2:].strip()
				else:
					sec_title = line[1:].strip().title()
				sections.append([(sec_title, i, section_num)])

		# Create html content
		content = ""
		for sec_list in sections:
			for sec in sec_list:
				if numbered:
					sec_name = str(sec[2]) + " - " + sec[0]
				else:
					sec_name = sec[0]
				if type(sec[2]) == int:
					content += "
" + sec_name + "
\n
\n"
				if type(sec[2]) == float:
					content += "
" + sec_name + "
\n
\n"
				index = sec[1] + 1
				if index >= len(script):
					break
				while not script[index].startswith("*"):
					content += script[index] + " "
					index += 1
					if index >= len(script):
						break
				content = content[:-1].replace("\\\\", "
")
				content += "\n
\n"

		# Create table of contents
		toc = "
\n"
		single = False
		for sec_list in sections:
			single = False
			if len(sec_list) == 1:
					single = True
			for sec in sec_list:
				if numbered:
					sec_name = str(sec[2]) + " - " + sec[0]
				else:
					sec_name = sec[0]
				toc += "
" + sec_name + "
\n"
				if sec_list.index(sec) == 0:
					if not single:
						toc += "
\n"
				if sec_list.index(sec) == len(sec_list)-1:
					if not single:
						toc += "
\n"
		toc += "
"
		script_info['TableOfContents'] = toc
		script_info['Content'] = content

	def get_article_list(self, format_string='{} - {}', sort=True, key='Date_object'):
		'''Get a list of html-links to all the articles

		Keyword arguments:
		format_string -- the string to use as a template for creating links to the articles (default '{} - {}')
		sort -- if the scripts should be sorted by date (default True)
		key -- the key to sort after (default 'Date_object')
		'''
		scripts = self.__get_scripts__(sort, key)
		article_html = '';
		for info in scripts:
			article_html += '' \
			+ format_string.format(info['Date'], info['Title']) + '\n'
		return article_html
		<commit_msg>Caching can now be disabled in the ScriptLoader<commit_after>#!/usr/bin/python3

import os
import shutil
import hashlib as hl
from datetime import date
from flask import render_template

# TODO find better newline
TEMPLATE_NEWLINE = 'FLASK_ARTICLE_NEWLINE'
CACHE_SEPERATOR = '\n\0\n'

class CacheHandler():
	'''Handler for caching files

	It uses 2 caches, the heap (limited) and the discspace.
	The last cached scripts are the ones in heap.
	'''
	def __init__(self, cache_limit=10, script_folder='scripts', cache_folder='.cache', debug=True, hash_alg='sha1'):
		'''Constructor

		Keyword arguments:
		cache_limit -- how many files should be in heap-cache (default 10)
		script_folder -- the directory for your scripts (default 'scripts')
		cache_folder -- the directory for cached files (default '.cache')
		debug -- specefies wether the handler will output debug information (default True)
		'''
		# TODO documentation
		# TODO test performance
		self.cache_limit = cache_limit
		self.script_folder = script_folder
		self.cache_folder = cache_folder
		self.debug = debug
		self.heap_cache = {}

		if hash_alg in hl.algorithms_available:
			self.hash_alg = hash_alg
		else:
			self.report('The hash algorithm "' + hash_alg + '" is not available'\
				' on this system! (Using SHA1 instead)')
			self.hash_alg = 'sha1'
		self.hash_size = hl.new(self.hash_alg).digest_size

		self.create_new_cache_dir()

	def report(self, data):
		if self.debug:
			print(data)

	def create_new_cache_dir(self):
		'''Deletes the cache dir and creates a new empty one'''
		self.report('Renewing cache dir')
		if os.path.exists(self.cache_folder) and os.path.isdir(self.cache_folder):
			shutil.rmtree(self.cache_folder)
		os.mkdir(self.cache_folder)

	def new_cache_entry(self, script):
		'''Creates a new cache entry in the cache dir for a file

		The entry saves a hash of the script and the parsed toc and content.
		'''
		name = script['Filename']
		self.report("Creating new cache entry for " + name)

		# Creating content
		file_data = open(self.script_folder + '/' + name, 'rb').read()
		h = hl.new(self.hash_alg, file_data).digest()
		disc_content = h
		for tag in script.keys():
			if type(script[tag]) == str:
				disc_content += CACHE_SEPERATOR.encode() \
								+ tag.encode() \
								+ b'\0' + script[tag].encode()

		# Writing cache entry to disc
		cache_path = self.cache_folder + '/' + name
		if os.path.exists(cache_path):
			os.remove(cache_path)
		cache_entry = open(cache_path, 'wb')
		cache_entry.write(disc_content)

		# Writing cache entry to heap
		if len(self.heap_cache) < self.cache_limit:
			pass
		elif name in self.heap_cache:
			pass
		else:
			# Delete the first entry in the cache to make room for the new one
			del self.heap_cache[list(self.heap_cache.keys())[0]]
		self.heap_cache[name] = (h, script)

	def check_cache_entry(self, name):
		'''Checks with the hash of a cache entry if a script has changed'''
		file_data = open(self.script_folder + '/' + name, 'rb').read()
		h = hl.new(self.hash_alg, file_data).digest()
		if name in self.heap_cache:
			# Check cache entry in heap
			cached_h = self.heap_cache[name][0]
		else:
			# Check cache entry on disc
			cache_path = self.cache_folder + '/' + name
			if not os.path.exists(cache_path):
				# The cache entry does not exist
				return False
			cached_h = open(cache_path, 'rb').read(self.hash_size)
		return h == cached_h

	def get_cached_content(self, name):
		if name in self.heap_cache:
			self.report('Loaded {} from heap cache'.format(name))
			return self.heap_cache[name][1]
		else:
			script = {}
			cache_entry = open(self.cache_folder + '/' + name, 'rb').read()
			cache_entry = cache_entry.split(CACHE_SEPERATOR.encode())
			for item in cache_entry:
				item = item.split(b'\0')
				tag = item[0].decode()
				value = item[1].decode()
				script[tag] = value
			self.__parse_date__(script)
			self.report('Loaded {} from disc cache'.format(name))
			return script

class ScriptLoader():
	'''Loader for the scripts'''
	def __init__(self, script_folder='scripts', cache_folder='.cache', caching=True):
		'''Constructor

		Keyword arguments:
		script_folder -- the directory for your scripts (default 'scripts')
		cache_folder -- the directory for cached files (default '.cache')
		caching -- specifies if caching is enabled (default True)
		'''
		self.script_folder = script_folder
		self.cache_handler = CacheHandler(cache_folder = cache_folder)
		if not caching:
			print("Caching disabled!")
		self.caching = caching

	def render_article(self, script, template_file):
		'''Generates a 'render_template' function call which sets all the tags
		of your script. Evaluates that call and returns it.

		e.g.
		Your tags are "Author", "Title" and "Date".
		The resulting command will be:
		render_template(<filename>, Author = script["Author"], Title = script["Title"], Date = script["Date"])

		Keyword arguments:
		script -- parsed script
		template_file -- filename of your template file
		'''
		func_call = 'render_template("' + template_file + '"'
		for tag in script.keys():
			func_call += ', ' + tag + ' = script["' + tag + '"]'
		func_call += ')'
		return eval(func_call)

	def get_single_script(self, script_name):
		'''Opens single script and parses it so it can be used with a html template

		Keyword arguments:
		script_name -- script name
		'''
		return self.__get_script_info__(script_name)

	def __get_script_info__(self, script_name):
		'''Open a script and parse its content
		If the script is faulty, None is returned

		Keyword arguments:
		script_name -- script name
		'''
		script_info = {}
		script_info['Filename'] = script_name

		# The file couldn't be found
		script_path = os.getcwd() + '/' + self.script_folder
		if not script_name in os.listdir(script_path):
			return None

		# Checking cache entry
		if not (self.cache_handler.check_cache_entry(script_name) and self.caching):
			# Parsing file and creating new cache entry
			data = open(script_path + '/' + script_name).read()
			data = data.split('\n')
			try:
				i = data.index('---End-Tags---')
			except:
				return None

			while '' in data:
				data.remove('')

			tags = data[:i-1]
			script_info['Content'] = TEMPLATE_NEWLINE.join(data[i:]).replace("\\\\", "
")

			for tag in tags:
				tag_name = tag[ tag.find('{')+1 : tag.find('}') ]
				tag_data = tag[ tag.rfind('{')+1 : tag.rfind('}') ]
				script_info[tag_name] = tag_data

			# Check script validity
			required_tags = ['Author','Title','Date']
			for rt in required_tags:
				if not rt in script_info.keys():
					return None
				# Make the title titlecased
				if rt == 'Title':
					if not script_info['Title'].startswith('_'):
						script_info['Title'] = script_info['Title'].title()
				# Create a date-object (later used for sorting)
				if rt == 'Date':
					self.__parse_date__(script_info)

			# Final formatting
			self.__parse_content__(script_info)

			# Caching
			if self.caching:
				self.cache_handler.new_cache_entry(script_info)
		else:
			# Loading parsed contents
			script_info = self.cache_handler.get_cached_content(script_name)

		return script_info

	def __parse_date__(self, script_info):
		d = script_info['Date'].split('/')
		d = date(day=int(d[0]), month=int(d[1]), year=int(d[2]))
		script_info['Date'] = d.strftime("%d. %h %Y")
		script_info['Date_object'] = d

	def __get_scripts__(self, sort=True, key='Date_object'):
		'''Get all the scripts from the script directory and optionally sort them

		Keyword arguments:
		sort -- if the scripts should be sorted by date (default True)
		key -- the key to sort after (default 'Date_object')
		'''
		script_infos = []
		files = os.listdir( os.getcwd() + '/' +  self.script_folder )
		for script in files:
			info = self.__get_script_info__(script)
			if info is not None:
				script_infos.append(info)
		if sort:
			script_infos.sort(reverse=True, key=lambda x : x[key])
		return script_infos

	def __parse_content__(self, script_info, numbered=True):
		'''Generates a table of contents and modifies an articles content

		Keyword arguments:
		content -- the content to parse
		numbered -- if the sections should be numbered (default True)
		'''
		script = script_info['Content'].split(TEMPLATE_NEWLINE)

		sections = []
		section_num = 0
		subsection_num = 1
		for i, line in enumerate(script):
			if line.startswith("**"):
				top_sec = sections.pop()
				num = float(str(section_num) + "." + str(subsection_num))
				if line.startswith("**_"):
					sec_title = line[3:].strip()
				else:
					sec_title = line[2:].strip().title()
				top_sec.append((sec_title, i, num))
				subsection_num += 1
				sections.append(top_sec)
			elif line.startswith("*"):
				subsection_num = 1
				section_num += 1
				if line.startswith("*_"):
					sec_title = line[2:].strip()
				else:
					sec_title = line[1:].strip().title()
				sections.append([(sec_title, i, section_num)])

		# Create html content
		content = ""
		for sec_list in sections:
			for sec in sec_list:
				if numbered:
					sec_name = str(sec[2]) + " - " + sec[0]
				else:
					sec_name = sec[0]
				if type(sec[2]) == int:
					content += "
" + sec_name + "
\n
\n"
				if type(sec[2]) == float:
					content += "
" + sec_name + "
\n
\n"
				index = sec[1] + 1
				if index >= len(script):
					break
				while not script[index].startswith("*"):
					content += script[index] + " "
					index += 1
					if index >= len(script):
						break
				content = content[:-1].replace("\\\\", "
")
				content += "\n
\n"

		# Create table of contents
		toc = "
\n"
		single = False
		for sec_list in sections:
			single = False
			if len(sec_list) == 1:
					single = True
			for sec in sec_list:
				if numbered:
					sec_name = str(sec[2]) + " - " + sec[0]
				else:
					sec_name = sec[0]
				toc += "
" + sec_name + "
\n"
				if sec_list.index(sec) == 0:
					if not single:
						toc += "
\n"
				if sec_list.index(sec) == len(sec_list)-1:
					if not single:
						toc += "
\n"
		toc += "
"
		script_info['TableOfContents'] = toc
		script_info['Content'] = content

	def get_article_list(self, format_string='{} - {}', sort=True, key='Date_object'):
		'''Get a list of html-links to all the articles

		Keyword arguments:
		format_string -- the string to use as a template for creating links to the articles (default '{} - {}')
		sort -- if the scripts should be sorted by date (default True)
		key -- the key to sort after (default 'Date_object')
		'''
		scripts = self.__get_scripts__(sort, key)
		article_html = '';
		for info in scripts:
			article_html += '' \
			+ format_string.format(info['Date'], info['Title']) + '\n'
		return article_html
		<|endoftext|>