<commit_before>import json
import random
import string

from twisted.application import internet, service
from twisted.internet import protocol, reactor, defer
from twisted.protocols import basic
from twisted.web import resource, server, static, http
from twisted.web.microdom import escape

from cyclone.web import RequestHandler, Application

from twisted.protocols import policies, basic
from twisted.web.http import Request

class SimpleHTTPChannel(basic.LineReceiver, policies.TimeoutMixin):
    """
    This is a simplified version of twisted.web.http.HTTPChannel to overcome
    header lowercase normalization. It does not actually implement the HTTP
    protocol, but only the subset of it that we need for testing.

    What this HTTP channel currently does is process the HTTP Request Line and
    the Request Headers and returns them in a JSON datastructure in the order
    we received them.

    The returned JSON dict looks like so:

    {
        'request_headers':
            [['User-Agent', 'IE6'], ['Content-Length', 200]]
        'request_line':
            'GET / HTTP/1.1'
    }
    """
    requestFactory = Request
    __first_line = 1
    __header = ''
    __content = None

    length = 0
    maxHeaders = 500
    requestLine = ''
    headers = []

    timeOut = 60 * 60 * 12

    def __init__(self):
        self.requests = []

    def connectionMade(self):
        self.setTimeout(self.timeOut)

    def lineReceived(self, line):
        if self.__first_line:
            self.requestLine = line
            self.__first_line = 0
        elif line == '':
            # We have reached the end of the headers.
            if self.__header:
                self.headerReceived(self.__header)
            self.__header = ''
            self.allHeadersReceived()
            self.setRawMode()
        elif line[0] in ' \t':
            # This is to support header field value folding over multiple lines
            # as specified by rfc2616.
            self.__header = self.__header+'\n'+line
        else:
            if self.__header:
                self.headerReceived(self.__header)
            self.__header = line

    def headerReceived(self, line):
        try:
            header, data = line.split(':', 1)
            self.headers.append((header, data.strip()))
        except:
            log.err("Got malformed HTTP Header request field")
            log.err("%s" % line)

    def allHeadersReceived(self):
        headers_dict = {}
        for k, v in self.headers:
            if k not in headers_dict:
                headers_dict[k] = []
            headers_dict[k].append(v)

        response = {'request_headers': self.headers,
            'request_line': self.requestLine,
            'headers_dict': headers_dict
        }
        json_response = json.dumps(response)
        self.transport.write('HTTP/1.1 200 OK\r\n\r\n')
        self.transport.write('%s' % json_response)
        self.transport.loseConnection()

class HTTPReturnJSONHeadersHelper(protocol.ServerFactory):
    protocol = SimpleHTTPChannel
    def buildProtocol(self, addr):
        p = self.protocol()
        p.headers = []
        return p

class HTTPTrapAll(RequestHandler):
    def _execute(self, transforms, *args, **kwargs):
        self._transforms = transforms
        defer.maybeDeferred(self.prepare).addCallbacks(
                    self._execute_handler,
                    lambda f: self._handle_request_exception(f.value),
                    callbackArgs=(args, kwargs))

    def _execute_handler(self, r, args, kwargs):
        if not self._finished:
            args = [self.decode_argument(arg) for arg in args]
            kwargs = dict((k, self.decode_argument(v, name=k))
                            for (k, v) in kwargs.iteritems())
            # This is where we do the patching
            # XXX this is somewhat hackish
            d = defer.maybeDeferred(self.all, *args, **kwargs)
            d.addCallbacks(self._execute_success, self._execute_failure)
            self.notifyFinish().addCallback(self.on_connection_close)

class HTTPRandomPage(HTTPTrapAll):
    """
    This generates a random page of arbitrary length and containing the string
    selected by the user.
    /<length>/<keyword>
    XXX this is currently disabled as it is not of use to any test.
    """
    isLeaf = True
    def _gen_random_string(self, length):
        return ''.join(random.choice(string.letters) for x in range(length))

    def genRandomPage(self, length=100, keyword=None):
        data = self._gen_random_string(length/2)
        if keyword:
            data += keyword
        data += self._gen_random_string(length - length/2)
        data += '\n'
        return data

    def all(self, length, keyword):
        length = 100
        if length > 100000:
            length = 100000
        return self.genRandomPage(length, keyword)

HTTPRandomPageHelper = Application([
    # XXX add regexps here
    (r"/(.*)/(.*)", HTTPRandomPage)
])

<commit_msg>remove mutable class-level variable that's always overridden by a fresh instance variable with the same name (at least within this code, and I'm going to assume that's how it's meant to be used)<commit_after>import json
import random
import string

from twisted.application import internet, service
from twisted.internet import protocol, reactor, defer
from twisted.protocols import basic
from twisted.web import resource, server, static, http
from twisted.web.microdom import escape

from cyclone.web import RequestHandler, Application

from twisted.protocols import policies, basic
from twisted.web.http import Request

class SimpleHTTPChannel(basic.LineReceiver, policies.TimeoutMixin):
    """
    This is a simplified version of twisted.web.http.HTTPChannel to overcome
    header lowercase normalization. It does not actually implement the HTTP
    protocol, but only the subset of it that we need for testing.

    What this HTTP channel currently does is process the HTTP Request Line and
    the Request Headers and returns them in a JSON datastructure in the order
    we received them.

    The returned JSON dict looks like so:

    {
        'request_headers':
            [['User-Agent', 'IE6'], ['Content-Length', 200]]
        'request_line':
            'GET / HTTP/1.1'
    }
    """
    requestFactory = Request
    __first_line = 1
    __header = ''
    __content = None

    length = 0
    maxHeaders = 500
    requestLine = ''

    timeOut = 60 * 60 * 12

    def __init__(self):
        self.headers = []
        self.requests = []

    def connectionMade(self):
        self.setTimeout(self.timeOut)

    def lineReceived(self, line):
        if self.__first_line:
            self.requestLine = line
            self.__first_line = 0
        elif line == '':
            # We have reached the end of the headers.
            if self.__header:
                self.headerReceived(self.__header)
            self.__header = ''
            self.allHeadersReceived()
            self.setRawMode()
        elif line[0] in ' \t':
            # This is to support header field value folding over multiple lines
            # as specified by rfc2616.
            self.__header = self.__header+'\n'+line
        else:
            if self.__header:
                self.headerReceived(self.__header)
            self.__header = line

    def headerReceived(self, line):
        try:
            header, data = line.split(':', 1)
            self.headers.append((header, data.strip()))
        except:
            log.err("Got malformed HTTP Header request field")
            log.err("%s" % line)

    def allHeadersReceived(self):
        headers_dict = {}
        for k, v in self.headers:
            if k not in headers_dict:
                headers_dict[k] = []
            headers_dict[k].append(v)

        response = {'request_headers': self.headers,
            'request_line': self.requestLine,
            'headers_dict': headers_dict
        }
        json_response = json.dumps(response)
        self.transport.write('HTTP/1.1 200 OK\r\n\r\n')
        self.transport.write('%s' % json_response)
        self.transport.loseConnection()

class HTTPReturnJSONHeadersHelper(protocol.ServerFactory):
    protocol = SimpleHTTPChannel
    def buildProtocol(self, addr):
        return self.protocol()

class HTTPTrapAll(RequestHandler):
    def _execute(self, transforms, *args, **kwargs):
        self._transforms = transforms
        defer.maybeDeferred(self.prepare).addCallbacks(
                    self._execute_handler,
                    lambda f: self._handle_request_exception(f.value),
                    callbackArgs=(args, kwargs))

    def _execute_handler(self, r, args, kwargs):
        if not self._finished:
            args = [self.decode_argument(arg) for arg in args]
            kwargs = dict((k, self.decode_argument(v, name=k))
                            for (k, v) in kwargs.iteritems())
            # This is where we do the patching
            # XXX this is somewhat hackish
            d = defer.maybeDeferred(self.all, *args, **kwargs)
            d.addCallbacks(self._execute_success, self._execute_failure)
            self.notifyFinish().addCallback(self.on_connection_close)

class HTTPRandomPage(HTTPTrapAll):
    """
    This generates a random page of arbitrary length and containing the string
    selected by the user.
    /<length>/<keyword>
    XXX this is currently disabled as it is not of use to any test.
    """
    isLeaf = True
    def _gen_random_string(self, length):
        return ''.join(random.choice(string.letters) for x in range(length))

    def genRandomPage(self, length=100, keyword=None):
        data = self._gen_random_string(length/2)
        if keyword:
            data += keyword
        data += self._gen_random_string(length - length/2)
        data += '\n'
        return data

    def all(self, length, keyword):
        length = 100
        if length > 100000:
            length = 100000
        return self.genRandomPage(length, keyword)

HTTPRandomPageHelper = Application([
    # XXX add regexps here
    (r"/(.*)/(.*)", HTTPRandomPage)
])

<|endoftext|>


<commit_before># -*- coding: UTF-8 -*-
# Copyright 2011-2017 Luc Saffre
# License: BSD (see file COPYING for details)

"""Default settings module for a :ref:`cosi` project. This is being
inherited by the other applications below :mod:`lino_cosi.projects`.

"""

from __future__ import unicode_literals

from lino.projects.std.settings import *

from django.utils.translation import ugettext_lazy as _

import lino_cosi

class Site(Site):
    """Base class for a :ref:`cosi` application.

    """

    verbose_name = "Lino Cosi"
    version = lino_cosi.SETUP_INFO['version']
    url = lino_cosi.SETUP_INFO['url']

    # demo_fixtures = 'std few_countries minimal_ledger euvatrates \
    # furniture demo demo_bookings payments demo2'.split()
    demo_fixtures = 'std few_countries minimal_ledger \
    furniture demo demo_bookings payments demo2'.split()

    # languages = 'en de fr'
    languages = 'en'

    user_types_module = 'lino_cosi.lib.cosi.user_types'
    custom_layouts_module = 'lino_cosi.lib.cosi.layouts'

    # default_build_method = 'wkhtmltopdf'

    # textfield_format = 'html'

    def get_installed_apps(self):
        yield super(Site, self).get_installed_apps()
        yield 'lino.modlib.gfks'
        # yield 'lino.modlib.system'
        yield 'lino.modlib.users'
        yield 'lino_xl.lib.countries'
        yield 'lino_cosi.lib.contacts'
        yield 'lino_xl.lib.sepa'
        #~ yield 'lino_xl.lib.households'
        yield 'lino_xl.lib.products'
        yield 'lino_xl.lib.accounts'

        yield 'lino_xl.lib.excerpts'

        # yield 'lino_xl.lib.outbox'
        # yield 'lino.modlib.uploads'
        yield 'lino_xl.lib.appypod'
        yield 'lino.modlib.export_excel'
        yield 'lino.modlib.tinymce'
        yield 'lino.modlib.wkhtmltopdf'

        # ledger must come before sales because its demo fixture
        # creates journals

        # yield 'lino.modlib.ledger'
        yield 'lino_xl.lib.vat'
        yield 'lino_xl.lib.finan'
        yield 'lino_xl.lib.sales'
        yield 'lino_xl.lib.invoicing'
        yield 'lino_xl.lib.bevat'
        #~ 'lino.modlib.journals',
        #~ 'lino_xl.lib.projects',
        #~ yield 'lino_xl.lib.blogs'
        #~ yield 'lino.modlib.tickets'
        #~ 'lino.modlib.links',
        #~ 'lino_xl.lib.thirds',
        #~ yield 'lino_xl.lib.postings'
        # yield 'lino_xl.lib.pages'
        # yield 'lino_cosi.lib.cosi'

    def setup_plugins(self):
        """
        Change the default value of certain plugin settings.

        """
        super(Site, self).setup_plugins()
        self.plugins.countries.configure(hide_region=True)
        self.plugins.ledger.configure(use_pcmn=True)
        self.plugins.countries.configure(country_code='BE')

    def setup_actions(self):
        super(Site, self).setup_actions()
        partners = self.modules.contacts
        from lino.core.merge import MergeAction
        for m in (partners.Person, partners.Organisation):
            m.define_action(merge_row=MergeAction(m))

    # def setup_layouts(self):
    #     super(Site, self).setup_layouts()

    #     self.models.system.SiteConfigs.set_detail_layout("""
    #     site_company next_partner_id:10
    #     default_build_method
    #     clients_account   sales_account
    #     suppliers_account purchases_account tax_offices_account
    #     """)

    #     self.models.accounts.Accounts.set_detail_layout("""
    #     ref:10 name id:5
    #     seqno group type clearable
    #     ledger.MovementsByAccount
    #     """)

# class DocsSite(Site):
#     """A special variant used to build the docs.
#     """
#     def get_installed_apps(self):
#         yield super(DocsSite, self).get_installed_apps()
#         yield 'lino_xl.lib.courses'  # needed for Sphinx autosummary
#         yield 'lino_xl.lib.vatless'

    # def get_apps_modifiers(self, **kw):
    #     kw = super(DocsSite, self).get_apps_modifiers(**kw)
    #     kw.update(sales='lino_cosi.lib.auto.sales')
    #     return kw
<commit_msg>http://luc.lino-framework.org/blog/2017/1209.html<commit_after># -*- coding: UTF-8 -*-
# Copyright 2011-2017 Luc Saffre
# License: BSD (see file COPYING for details)

"""Default settings module for a :ref:`cosi` project. This is being
inherited by the other applications below :mod:`lino_cosi.projects`.

"""

from __future__ import unicode_literals

from lino.projects.std.settings import *

from django.utils.translation import ugettext_lazy as _

import lino_cosi

class Site(Site):
    """Base class for a :ref:`cosi` application.

    """

    verbose_name = "Lino Cosi"
    version = lino_cosi.SETUP_INFO['version']
    url = lino_cosi.SETUP_INFO['url']

    # demo_fixtures = 'std few_countries minimal_ledger euvatrates \
    # furniture demo demo_bookings payments demo2'.split()
    demo_fixtures = 'std few_countries minimal_ledger \
    furniture demo demo_bookings payments demo2'.split()

    # languages = 'en de fr'
    languages = 'en'

    user_types_module = 'lino_cosi.lib.cosi.user_types'
    custom_layouts_module = 'lino_cosi.lib.cosi.layouts'

    # default_build_method = 'wkhtmltopdf'

    # textfield_format = 'html'

    def get_installed_apps(self):
        yield super(Site, self).get_installed_apps()
        yield 'lino.modlib.gfks'
        # yield 'lino.modlib.system'
        yield 'lino.modlib.users'
        yield 'lino_xl.lib.countries'
        yield 'lino_cosi.lib.contacts'
        yield 'lino_xl.lib.sepa'
        #~ yield 'lino_xl.lib.households'
        yield 'lino_xl.lib.products'
        yield 'lino_xl.lib.accounts'

        yield 'lino_xl.lib.excerpts'

        # yield 'lino_xl.lib.outbox'
        # yield 'lino.modlib.uploads'
        yield 'lino_xl.lib.appypod'
        yield 'lino.modlib.export_excel'
        yield 'lino.modlib.tinymce'
        yield 'lino.modlib.wkhtmltopdf'

        # ledger must come before sales because its demo fixture
        # creates journals

        # yield 'lino.modlib.ledger'
        yield 'lino_xl.lib.vat'
        yield 'lino_xl.lib.finan'
        yield 'lino_xl.lib.sales'
        yield 'lino_xl.lib.invoicing'
        yield 'lino_xl.lib.bevat'
        #~ 'lino.modlib.journals',
        #~ 'lino_xl.lib.projects',
        #~ yield 'lino_xl.lib.blogs'
        #~ yield 'lino.modlib.tickets'
        #~ 'lino.modlib.links',
        #~ 'lino_xl.lib.thirds',
        #~ yield 'lino_xl.lib.postings'
        # yield 'lino_xl.lib.pages'
        # yield 'lino_cosi.lib.cosi'

    def setup_plugins(self):
        """
        Change the default value of certain plugin settings.

        """
        super(Site, self).setup_plugins()
        self.plugins.countries.configure(hide_region=True)
        self.plugins.ledger.configure(use_pcmn=True)
        self.plugins.countries.configure(country_code='BE')

    # def setup_actions(self):
    #     super(Site, self).setup_actions()
    #     partners = self.modules.contacts
    #     from lino.core.merge import MergeAction
    #     for m in (partners.Person, partners.Organisation):
    #         m.define_action(merge_row=MergeAction(m))

    # def setup_layouts(self):
    #     super(Site, self).setup_layouts()

    #     self.models.system.SiteConfigs.set_detail_layout("""
    #     site_company next_partner_id:10
    #     default_build_method
    #     clients_account   sales_account
    #     suppliers_account purchases_account tax_offices_account
    #     """)

    #     self.models.accounts.Accounts.set_detail_layout("""
    #     ref:10 name id:5
    #     seqno group type clearable
    #     ledger.MovementsByAccount
    #     """)

# class DocsSite(Site):
#     """A special variant used to build the docs.
#     """
#     def get_installed_apps(self):
#         yield super(DocsSite, self).get_installed_apps()
#         yield 'lino_xl.lib.courses'  # needed for Sphinx autosummary
#         yield 'lino_xl.lib.vatless'

    # def get_apps_modifiers(self, **kw):
    #     kw = super(DocsSite, self).get_apps_modifiers(**kw)
    #     kw.update(sales='lino_cosi.lib.auto.sales')
    #     return kw
<|endoftext|>


<commit_before># -*- coding: UTF-8 -*-
# Copyright 2016-2017 Luc Saffre
# License: BSD (see file COPYING for details)

"""Database models specific for the Team variant of Lino Noi.

Defines a customized :class:`TicketDetail`.

"""
from __future__ import print_function
from lino_xl.lib.tickets.models import *
from lino.modlib.users.mixins import Assignable
from lino.api import _

class Ticket(Ticket, Assignable):
    class Meta(Ticket.Meta):
        # app_label = 'tickets'
        abstract = dd.is_abstract_model(__name__, 'Ticket')

    def assigned_to_changed(self, ar):
        """Add a star"""
        self.add_change_watcher(self.assigned_to)

    def end_user_changed(self, ar):
        """Add a star"""
        self.add_change_watcher(self.end_user)

    def user_changed(self, ar):
        """Add a star"""
        self.add_change_watcher(self.user)

    def site_changed(self, ar):
        """Leaves a sub-star of old site, but that's OK for now"""
        if self.site is not None:
            # print("Change")
            self.site.add_child_stars(self.site, self)
            # self.add_change_watcher(star.user)

    def after_ui_create(self, ar):
        # print("Create")
        self.site_changed(ar)
        self.assigned_to_changed(ar)
        self.end_user_changed(ar)
        self.user_changed(ar)
        super(Ticket, self).after_ui_create(ar)

        if dd.is_installed('notify'):
            ctx = dict(user=ar.user, what=ar.obj2memo(self))
            def msg(user, mm):
                subject = _("{user} submitted ticket {what}").format(**ctx)
                return (subject , E.tostring(E.span(subject)))

            mt = rt.actors.notify.MessageTypes.change # Maybe something else, but unimporant
            # owner = self.get_change_owner()
            # rt.models.notify.Message.emit_message(
            #     ar, owner, mt, msg, self.get_change_observers())
            rt.models.notify.Message.emit_message(
                ar, self, mt, msg,
                [(u, u.mail_mode) for u in rt.models.users.User.objects.all()
                    if u.user_type is not None and u.user_type.has_required_roles((rt.modules.tickets.Triager,))
                    and u != ar.get_user()
                 ]
            )

class TicketDetail(TicketDetail):
    """Customized detail_layout for Tickets in Noi

    """
    main = "general more history_tab more2 #github.CommitsByTicket"

    general = dd.Panel("""
    general1:60 comments.CommentsByRFC:30
    """, label=_("General"))

    general1 = """
    summary:40 id:6
    user end_user assigned_to deadline
    site topic project
    workflow_buttons:30  ticket_type:10 priority:10 private
    bottom_box
    """

    bottom_box = """
    #faculties.DemandsByDemander:20 #votes.VotesByVotable:20
    deploy.DeploymentsByTicket:20 clocking.SessionsByTicket:20
    github.CommitsByTicket
    """

    more = dd.Panel("""
    more1 DuplicatesByTicket:20 #WishesByTicket
    description:30 upgrade_notes:20 LinksByTicket:20
    """, label=_("More"))

    more1 = """
    #nickname:10     created modified reported_for #fixed_for
    state ref duplicate_of planned_time
    # standby feedback closed
    """

    more2 = dd.Panel("""
    # deploy.DeploymentsByTicket
    # faculties.DemandsByDemander
    stars.AllStarsByController
    uploads.UploadsByController
    """, label=_("Even more"))

class TicketInsertLayout(dd.InsertLayout):
    main = """
           summary private:3
           left right:20
           """

    right = """
           end_user
           assigned_to
           site
           """

    left = """
            ticket_type priority
            description
            """
    window_size = (70, 20)

Tickets.detail_layout = 'tickets.TicketDetail'
Tickets.insert_layout = 'tickets.TicketInsertLayout'
Tickets.params_layout = """user end_user assigned_to not_assigned_to interesting_for site project state priority
    deployed_to has_project show_assigned show_active show_deployed show_todo show_private
    start_date end_date observed_event topic #feasable_by has_ref"""
Tickets.column_names = 'id summary:50 #user:10 #topic #faculty priority ' \
                       'workflow_buttons:30 site:10 #project:10'
Tickets.order_by = ["-id"]

MyTickets.params_layout = """
    user end_user site project state priority
    start_date end_date observed_event topic #feasable_by show_active"""
# Sites.detail_layout = """
# id name partner #responsible_user
# remark
# #InterestsBySite TicketsBySite deploy.MilestonesBySite
# """

<commit_msg>Notify user when assigned to ticket<commit_after># -*- coding: UTF-8 -*-
# Copyright 2016-2017 Luc Saffre
# License: BSD (see file COPYING for details)

"""Database models specific for the Team variant of Lino Noi.

Defines a customized :class:`TicketDetail`.

"""
from __future__ import print_function
from lino_xl.lib.tickets.models import *
from lino.modlib.users.mixins import Assignable
from lino.api import _

class Ticket(Ticket, Assignable):
    class Meta(Ticket.Meta):
        # app_label = 'tickets'
        abstract = dd.is_abstract_model(__name__, 'Ticket')

    def assigned_to_changed(self, ar):
        """Add a star and send notification of Assignment"""
        self.add_change_watcher(self.assigned_to)

        if (self.assigned_to is not None and
                self.assigned_to != ar.user and
                dd.is_installed('notify')):
            ctx = dict(user=ar.user, what=ar.obj2memo(self))
            def msg(user, mm):
                subject = _("{user} has assigned you to ticket: {what}").format(**ctx)
                return (subject , E.tostring(E.span(subject)))

            mt = rt.actors.notify.MessageTypes.action

            rt.models.notify.Message.emit_message(
                ar, self, mt, msg,
                [(self.assigned_to, self.assigned_to.mail_mode)]
            )
    def end_user_changed(self, ar):
        """Add a star"""
        self.add_change_watcher(self.end_user)

    def user_changed(self, ar):
        """Add a star"""
        self.add_change_watcher(self.user)

    def site_changed(self, ar):
        """Leaves a sub-star of old site, but that's OK for now"""
        if self.site is not None:
            # print("Change")
            self.site.add_child_stars(self.site, self)
            # self.add_change_watcher(star.user)

    def after_ui_create(self, ar):
        # print("Create")
        self.site_changed(ar)
        self.assigned_to_changed(ar)
        self.end_user_changed(ar)
        self.user_changed(ar)
        super(Ticket, self).after_ui_create(ar)

        if dd.is_installed('notify'):
            ctx = dict(user=ar.user, what=ar.obj2memo(self))
            def msg(user, mm):
                subject = _("{user} submitted ticket {what}").format(**ctx)
                return (subject , E.tostring(E.span(subject)))

            mt = rt.actors.notify.MessageTypes.change # Maybe something else, but unimporant
            # owner = self.get_change_owner()
            # rt.models.notify.Message.emit_message(
            #     ar, owner, mt, msg, self.get_change_observers())
            rt.models.notify.Message.emit_message(
                ar, self, mt, msg,
                [(u, u.mail_mode) for u in rt.models.users.User.objects.all()
                    if u.user_type is not None and u.user_type.has_required_roles((rt.modules.tickets.Triager,))
                    and u != ar.get_user()
                 ]
            )

class TicketDetail(TicketDetail):
    """Customized detail_layout for Tickets in Noi

    """
    main = "general more history_tab more2 #github.CommitsByTicket"

    general = dd.Panel("""
    general1:60 comments.CommentsByRFC:30
    """, label=_("General"))

    general1 = """
    summary:40 id:6
    user end_user assigned_to deadline
    site topic project
    workflow_buttons:30  ticket_type:10 priority:10 private
    bottom_box
    """

    bottom_box = """
    #faculties.DemandsByDemander:20 #votes.VotesByVotable:20
    deploy.DeploymentsByTicket:20 clocking.SessionsByTicket:20
    github.CommitsByTicket
    """

    more = dd.Panel("""
    more1 DuplicatesByTicket:20 #WishesByTicket
    description:30 upgrade_notes:20 LinksByTicket:20
    """, label=_("More"))

    more1 = """
    #nickname:10     created modified reported_for #fixed_for
    state ref duplicate_of planned_time
    # standby feedback closed
    """

    more2 = dd.Panel("""
    # deploy.DeploymentsByTicket
    # faculties.DemandsByDemander
    stars.AllStarsByController
    uploads.UploadsByController
    """, label=_("Even more"))

class TicketInsertLayout(dd.InsertLayout):
    main = """
           summary private:3
           left right:20
           """

    right = """
           end_user
           assigned_to
           site
           """

    left = """
            ticket_type priority
            description
            """
    window_size = (70, 20)

Tickets.detail_layout = 'tickets.TicketDetail'
Tickets.insert_layout = 'tickets.TicketInsertLayout'
Tickets.params_layout = """user end_user assigned_to not_assigned_to interesting_for site project state priority
    deployed_to has_project show_assigned show_active show_deployed show_todo show_private
    start_date end_date observed_event topic #feasable_by has_ref"""
Tickets.column_names = 'id summary:50 #user:10 #topic #faculty priority ' \
                       'workflow_buttons:30 site:10 #project:10'
Tickets.order_by = ["-id"]

MyTickets.params_layout = """
    user end_user site project state priority
    start_date end_date observed_event topic #feasable_by show_active"""
# Sites.detail_layout = """
# id name partner #responsible_user
# remark
# #InterestsBySite TicketsBySite deploy.MilestonesBySite
# """

<|endoftext|>


<commit_before>from __future__ import absolute_import
import logging
import os
from tempfile import NamedTemporaryFile
import jprops
import lintreview.docker as docker
from lintreview.review import IssueComment
from lintreview.tools import Tool, process_checkstyle

log = logging.getLogger(__name__)

class Checkstyle(Tool):

    name = 'checkstyle'

    def check_dependencies(self):
        """
        See if checkstyle image exists
        """
        return docker.image_exists('checkstyle')

    def match_file(self, filename):
        base = os.path.basename(filename)
        name, ext = os.path.splitext(base)
        return ext == '.java'

    def process_files(self, files):
        """
        Run code checks with checkstyle.
        Only a single process is made for all files
        to save resources.
        """
        log.debug('Processing %s files with %s', files, self.name)
        if 'config' not in self.options:
            msg = ("We could not run `checkstyle` you did not set "
                   "the `config` option to a valid checkstyle XML file.")
            return self.problems.add(IssueComment(msg))

        with NamedTemporaryFile(dir=self.base_path,
                                suffix='.properties') as f:
            self.setup_properties(f)
            properties_filename = os.path.basename(f.name)
            command = self.create_command(properties_filename, files)
            output = docker.run('checkstyle', command, self.base_path)

        # Only one line is generally a config error. Replay the error
        # to the user.
        lines = output.strip().split('\n')
        if not lines[0].startswith('<'):
            msg = ("Running `checkstyle` failed with:\n"
                   "```\n"
                   "%s\n"
                   "```\n"
                   "Ensure your config file exists and is valid XML.")
            return self.problems.add(IssueComment(msg % (lines[0],)))

        # Remove the last line if it is not XML
        # Checkstyle outputs text after the XML if there are errors.
        if not lines[-1].strip().startswith('<'):
            lines = lines[0:-1]
        output = ''.join(lines)

        process_checkstyle(self.problems, output, docker.strip_base)

    def setup_properties(self, properties_file):
        config_loc = os.path.dirname(docker.apply_base(self.options['config']))
        project_loc = docker.apply_base('/')

        properties = {
            'config_loc': config_loc,
            'samedir': config_loc,
            'project_loc': project_loc,
            'basedir': project_loc,
        }

        jprops.store_properties(properties_file, properties)

    def create_command(self, properties_filename, files):
        command = [
            'checkstyle',
            '-f', 'xml',
            '-p', docker.apply_base(properties_filename),
            '-c', docker.apply_base(self.options['config'])
        ]
        command += files
        return command
<commit_msg>Add some documentation for checkstyle options.<commit_after>from __future__ import absolute_import
import logging
import os
from tempfile import NamedTemporaryFile
import jprops
import lintreview.docker as docker
from lintreview.review import IssueComment
from lintreview.tools import Tool, process_checkstyle

log = logging.getLogger(__name__)

class Checkstyle(Tool):
    """
    Integrates with checkstyle.

    When checkstyle is run a properties file will be generated, that
    defines the following keys:

    - config_loc
    - samedir
    - project_loc
    - basedir

    All of these keys will resolve to your repository's root directory.
    """

    name = 'checkstyle'

    def check_dependencies(self):
        """
        See if checkstyle image exists
        """
        return docker.image_exists('checkstyle')

    def match_file(self, filename):
        base = os.path.basename(filename)
        name, ext = os.path.splitext(base)
        return ext == '.java'

    def process_files(self, files):
        """
        Run code checks with checkstyle.
        Only a single process is made for all files
        to save resources.
        """
        log.debug('Processing %s files with %s', files, self.name)
        if 'config' not in self.options:
            msg = ("We could not run `checkstyle` you did not set "
                   "the `config` option to a valid checkstyle XML file.")
            return self.problems.add(IssueComment(msg))

        with NamedTemporaryFile(dir=self.base_path,
                                suffix='.properties') as f:
            self.setup_properties(f)
            properties_filename = os.path.basename(f.name)
            command = self.create_command(properties_filename, files)
            output = docker.run('checkstyle', command, self.base_path)

        # Only one line is generally a config error. Replay the error
        # to the user.
        lines = output.strip().split('\n')
        if not lines[0].startswith('<'):
            msg = ("Running `checkstyle` failed with:\n"
                   "```\n"
                   "%s\n"
                   "```\n"
                   "Ensure your config file exists and is valid XML.")
            return self.problems.add(IssueComment(msg % (lines[0],)))

        # Remove the last line if it is not XML
        # Checkstyle outputs text after the XML if there are errors.
        if not lines[-1].strip().startswith('<'):
            lines = lines[0:-1]
        output = ''.join(lines)

        process_checkstyle(self.problems, output, docker.strip_base)

    def setup_properties(self, properties_file):
        config_loc = os.path.dirname(docker.apply_base(self.options['config']))
        project_loc = docker.apply_base('/')

        properties = {
            'config_loc': config_loc,
            'samedir': config_loc,
            'project_loc': project_loc,
            'basedir': project_loc,
        }

        jprops.store_properties(properties_file, properties)

    def create_command(self, properties_filename, files):
        command = [
            'checkstyle',
            '-f', 'xml',
            '-p', docker.apply_base(properties_filename),
            '-c', docker.apply_base(self.options['config'])
        ]
        command += files
        return command
<|endoftext|>


<commit_before># SPDX-License-Identifier: Apache-2.0
#
# Copyright (C) 2018, Arm Limited and contributors.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from math import ceil

import pandas as pd

from devlib.module.sched import SchedDomain, SchedDomainFlag

from lisa.utils import memoized, ArtifactPath
from lisa.datautils import df_squash, df_add_delta
from lisa.trace import Trace, FtraceConf, FtraceCollector, requires_events
from lisa.wlgen.rta import Periodic
from lisa.tests.base import RTATestBundle, Result, ResultBundle, CannotCreateError, TestMetric
from lisa.target import Target
from lisa.analysis.tasks import TasksAnalysis, TaskState
from lisa.analysis.idle import IdleAnalysis
from lisa.analysis.rta import RTAEventsAnalysis

class MisfitMigrationBase(RTATestBundle):
    """
    Abstract class for Misfit behavioural testing

    This class provides some helpers for features related to Misfit.
    """

    @classmethod
    def _has_asym_cpucapacity(cls, target):
        """
        :returns: Whether the target has asymmetric CPU capacities
        """
        return len(set(target.plat_info["cpu-capacities"]['orig'].values())) > 1

    @classmethod
    def _get_max_lb_interval(cls, plat_info):
        """
        Get the value of maximum_load_balance_interval.

        The kernel computes it so:
            HZ*num_online_cpus()/10;
        (https://elixir.bootlin.com/linux/v4.15/source/kernel/sched/fair.c#L9101)

        Here we don't do any hotplugging so we consider all CPUs to be online.

        :returns: The absolute maximum load-balance interval in seconds
        """
        HZ = plat_info['kernel']['config']['CONFIG_HZ']
        return ((HZ * plat_info['cpus-count']) // 10) * (1. / HZ)

class StaggeredFinishes(MisfitMigrationBase):
    """
    One 100% task per CPU, with staggered completion times.

    By spawning one task per CPU on an asymmetric system, we expect the tasks
    running on the higher-performance CPUs to complete first. At this point,
    the misfit logic should kick in and they should pull tasks from
    lower-performance CPUs.

    The tasks have staggered completion times to prevent having several of them
    completing at the same time, which can cause some unwanted noise (e.g. some
    sshd or systemd activity at the end of the task).

    The end result should look something like this on big.LITTLE::

      a,b,c,d are CPU-hogging tasks
      _ signifies idling

      LITTLE_0 | a a a a _ _ _
      LITTLE_1 | b b b b b _ _
      ---------|--------------
        big_0  | c c c c a a a
        big_1  | d d d d d b b

    """

    task_prefix = "msft"

    PIN_DELAY_S = 0.001
    """
    How long the tasks will be pinned to their "starting" CPU. Doesn't have
    to be long (we just have to ensure they spawn there), so arbitrary value
    """

    # Let us handle things ourselves
    _BUFFER_PHASE_DURATION_S=0

    IDLING_DELAY_S = 1
    """
    A somewhat arbitray delay - long enough to ensure
    rq->avg_idle > sysctl_sched_migration_cost
    """

    @property
    def src_cpus(self):
        return self.plat_info['capacity-classes'][0]

    @property
    def dst_cpus(self):
        cpu_classes = self.plat_info['capacity-classes']

        # XXX: Might need to check the tasks can fit on all of those, rather
        # than just pick all but the smallest CPUs
        dst_cpus = []
        for group in cpu_classes[1:]:
            dst_cpus += group
        return dst_cpus

    @property
    def end_time(self):
        return self.trace.end

    @property
    def duration(self):
        return self.end_time - self.start_time

    @property
    @memoized
    @RTAEventsAnalysis.df_rtapp_phases_start.used_events
    def start_time(self):
        """
        The tasks don't wake up at the same exact time, find the task that is
        the last to wake up. We don't want to redefine trace_window() here
        because we still need the first wakeups to be visible.
        """
        # First phase is idling time, ignore it
        phase_df = self.trace.analysis.rta.df_rtapp_phases_start()
        phase_df = phase_df.reset_index()
        phase_df.set_index("Time", inplace=True)
        phase_df.sort_index(inplace=True)

        return phase_df[phase_df.phase == 1].index[-1]

    @classmethod
    def check_from_target(cls, target):
        if not cls._has_asym_cpucapacity(target):
            raise CannotCreateError(
                "Target doesn't have asymmetric CPU capacities")

    @classmethod
    def get_rtapp_profile(cls, plat_info):
        cpus = list(range(plat_info['cpus-count']))

        # We're pinning stuff in the first phase, so give it ample time to
        # clean the pinned logic out of balance_interval
        free_time_s = 1.1 * cls._get_max_lb_interval(plat_info)
        stagger_s = free_time_s // (10 * len(cpus))

        profile = {}

        for cpu in cpus:
            profile["{}{}".format(cls.task_prefix, cpu)] = (
                Periodic(
                    duty_cycle_pct=0,
                    duration_s=cls.IDLING_DELAY_S,
                    period_ms=cls.TASK_PERIOD_MS,
                    cpus=[cpu]
                ) + Periodic(
                    duty_cycle_pct=100,
                    duration_s=cls.PIN_DELAY_S,
                    period_ms=cls.TASK_PERIOD_MS,
                    cpus=[cpu]
                ) + Periodic(
                    duty_cycle_pct=100,
                    # Introduce staggered task completions
                    duration_s=free_time_s + cpu * stagger_s,
                    period_ms=cls.TASK_PERIOD_MS,
                    cpus=cpus
                )
            )

        return profile

    def _trim_state_df(self, state_df):
        if state_df.empty:
            return state_df

        return df_squash(state_df, self.start_time,
                         state_df.index[-1] + state_df['delta'].iloc[-1], "delta")

    @requires_events('sched_switch', TasksAnalysis.df_task_states.used_events)
    def test_preempt_time(self, allowed_preempt_pct=1) -> ResultBundle:
        """
        Test that tasks are not being preempted too much
        """

        sdf = self.trace.df_event('sched_switch')
        task_state_dfs = {
            task: self.trace.analysis.tasks.df_task_states(task)
            for task in self.rtapp_tasks
        }

        res = ResultBundle.from_bool(True)
        for task, state_df in task_state_dfs.items():
            # The sched_switch dataframe where the misfit task
            # is replaced by another misfit task
            preempt_sdf = sdf[
                (sdf.prev_comm == task) &
                (sdf.next_comm.str.startswith(self.task_prefix))
            ]

            state_df = self._trim_state_df(state_df)
            state_df = state_df[
                (state_df.index.isin(preempt_sdf.index)) &
                # Ensure this is a preemption and not just the task ending
                (state_df.curr_state == TaskState.TASK_INTERRUPTIBLE)
            ]

            preempt_time = state_df.delta.sum()
            preempt_pct = (preempt_time / self.duration) * 100

            res.add_metric("{} preemption".format(task), {
                "ratio": TestMetric(preempt_pct, "%"),
                "time": TestMetric(preempt_time, "seconds")})

            if preempt_pct > allowed_preempt_pct:
                res.result = Result.FAILED

        return res

    @memoized
    @IdleAnalysis.signal_cpu_active.used_events
    def _get_active_df(self, cpu):
        """
        :returns: A dataframe that describes the idle status (on/off) of 'cpu'
        """
        active_df = pd.DataFrame(
            self.trace.analysis.idle.signal_cpu_active(cpu), columns=['state']
        )
        df_add_delta(active_df, inplace=True, window=self.trace.window)
        return active_df

    @_get_active_df.used_events
    def _max_idle_time(self, start, end, cpus):
        """
        :returns: The maximum idle time of 'cpus' in the [start, end] interval
        """
        max_time = 0
        max_cpu = 0

        for cpu in cpus:
            busy_df = self._get_active_df(cpu)
            busy_df = df_squash(busy_df, start, end)
            busy_df = busy_df[busy_df.state == 0]

            if busy_df.empty:
                continue

            local_max = busy_df.delta.max()
            if local_max > max_time:
                max_time = local_max
                max_cpu = cpu

        return max_time, max_cpu

    @_max_idle_time.used_events
    def _test_cpus_busy(self, task_state_dfs, cpus, allowed_idle_time_s):
        """
        Test that for every window in which the tasks are running, :attr:`cpus`
        are not idle for more than :attr:`allowed_idle_time_s`
        """
        if allowed_idle_time_s is None:
            # Regular interval is 1 ms * nr_cpus, rounded to closest jiffy multiple
            jiffy = 1 / self.plat_info['kernel']['config']['CONFIG_HZ']
            interval = 1e-3 * self.plat_info["cpus-count"]

            allowed_idle_time_s = ceil(interval / jiffy) * jiffy

        res = ResultBundle.from_bool(True)

        for task, state_df in task_state_dfs.items():
            # Have a look at every task activation
            task_idle_times = [self._max_idle_time(index, index + row.delta, cpus)
                               for index, row in state_df.iterrows()]

            if not task_idle_times:
                continue

            max_time, max_cpu = max(task_idle_times)
            res.add_metric("{} max idle".format(task), data={
                "time": TestMetric(max_time, "seconds"), "cpu": TestMetric(max_cpu)})

            if max_time > allowed_idle_time_s:
                res.result = Result.FAILED

        return res

    @TasksAnalysis.df_task_states.used_events
    @_test_cpus_busy.used_events
    @RTATestBundle.test_noisy_tasks.undecided_filter(noise_threshold_pct=1)
    def test_throughput(self, allowed_idle_time_s=None) -> ResultBundle:
        """
        Test that big CPUs are not idle when there are misfit tasks to upmigrate

        :param allowed_idle_time_s: How much time should be allowed between a
          big CPU going idle and a misfit task ending on that CPU. In theory
          a newidle balance should lead to a null delay, but in practice
          there's a tiny one, so don't set that to 0 and expect the test to
          pass.

          Furthermore, we're not always guaranteed to get a newidle pull, so
          allow time for a regular load balance to happen.

          When ``None``, this defaults to (1ms x number_of_cpus) to mimic the
          default balance_interval (balance_interval = sd_weight), see
          kernel/sched/topology.c:sd_init().
        :type allowed_idle_time_s: int
        """
        task_state_dfs = {}
        for task in self.rtapp_tasks:
            # This test is all about throughput: check that every time a task
            # runs on a little it's because bigs are busy
            df = self.trace.analysis.tasks.df_task_states(task)
            # Trim first to keep coherent deltas
            df = self._trim_state_df(df)
            task_state_dfs[task] = df[
                # Task is active
                (df.curr_state == TaskState.TASK_ACTIVE) &
                # Task needs to be upmigrated
                (df.cpu.isin(self.src_cpus))
            ]

        return self._test_cpus_busy(task_state_dfs, self.dst_cpus, allowed_idle_time_s)
<commit_msg>lisa.tests.scheduler.misfit: Fix stagger delay computation<commit_after># SPDX-License-Identifier: Apache-2.0
#
# Copyright (C) 2018, Arm Limited and contributors.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from math import ceil

import pandas as pd

from devlib.module.sched import SchedDomain, SchedDomainFlag

from lisa.utils import memoized, ArtifactPath
from lisa.datautils import df_squash, df_add_delta
from lisa.trace import Trace, FtraceConf, FtraceCollector, requires_events
from lisa.wlgen.rta import Periodic
from lisa.tests.base import RTATestBundle, Result, ResultBundle, CannotCreateError, TestMetric
from lisa.target import Target
from lisa.analysis.tasks import TasksAnalysis, TaskState
from lisa.analysis.idle import IdleAnalysis
from lisa.analysis.rta import RTAEventsAnalysis

class MisfitMigrationBase(RTATestBundle):
    """
    Abstract class for Misfit behavioural testing

    This class provides some helpers for features related to Misfit.
    """

    @classmethod
    def _has_asym_cpucapacity(cls, target):
        """
        :returns: Whether the target has asymmetric CPU capacities
        """
        return len(set(target.plat_info["cpu-capacities"]['orig'].values())) > 1

    @classmethod
    def _get_max_lb_interval(cls, plat_info):
        """
        Get the value of maximum_load_balance_interval.

        The kernel computes it so:
            HZ*num_online_cpus()/10;
        (https://elixir.bootlin.com/linux/v4.15/source/kernel/sched/fair.c#L9101)

        Here we don't do any hotplugging so we consider all CPUs to be online.

        :returns: The absolute maximum load-balance interval in seconds
        """
        HZ = plat_info['kernel']['config']['CONFIG_HZ']
        return ((HZ * plat_info['cpus-count']) // 10) * (1. / HZ)

    @classmethod
    def _get_lb_interval(cls, plat_info):
        # Regular interval is 1 ms * nr_cpus, rounded to closest jiffy multiple
        jiffy = 1 / plat_info['kernel']['config']['CONFIG_HZ']
        interval = 1e-3 * plat_info["cpus-count"]

        return ceil(interval / jiffy) * jiffy

class StaggeredFinishes(MisfitMigrationBase):
    """
    One 100% task per CPU, with staggered completion times.

    By spawning one task per CPU on an asymmetric system, we expect the tasks
    running on the higher-performance CPUs to complete first. At this point,
    the misfit logic should kick in and they should pull tasks from
    lower-performance CPUs.

    The tasks have staggered completion times to prevent having several of them
    completing at the same time, which can cause some unwanted noise (e.g. some
    sshd or systemd activity at the end of the task).

    The end result should look something like this on big.LITTLE::

      a,b,c,d are CPU-hogging tasks
      _ signifies idling

      LITTLE_0 | a a a a _ _ _
      LITTLE_1 | b b b b b _ _
      ---------|--------------
        big_0  | c c c c a a a
        big_1  | d d d d d b b

    """

    task_prefix = "msft"

    PIN_DELAY_S = 0.001
    """
    How long the tasks will be pinned to their "starting" CPU. Doesn't have
    to be long (we just have to ensure they spawn there), so arbitrary value
    """

    # Let us handle things ourselves
    _BUFFER_PHASE_DURATION_S=0

    IDLING_DELAY_S = 1
    """
    A somewhat arbitray delay - long enough to ensure
    rq->avg_idle > sysctl_sched_migration_cost
    """

    @property
    def src_cpus(self):
        return self.plat_info['capacity-classes'][0]

    @property
    def dst_cpus(self):
        cpu_classes = self.plat_info['capacity-classes']

        # XXX: Might need to check the tasks can fit on all of those, rather
        # than just pick all but the smallest CPUs
        dst_cpus = []
        for group in cpu_classes[1:]:
            dst_cpus += group
        return dst_cpus

    @property
    def end_time(self):
        return self.trace.end

    @property
    def duration(self):
        return self.end_time - self.start_time

    @property
    @memoized
    @RTAEventsAnalysis.df_rtapp_phases_start.used_events
    def start_time(self):
        """
        The tasks don't wake up at the same exact time, find the task that is
        the last to wake up. We don't want to redefine trace_window() here
        because we still need the first wakeups to be visible.
        """
        # First phase is idling time, ignore it
        phase_df = self.trace.analysis.rta.df_rtapp_phases_start()
        phase_df = phase_df.reset_index()
        phase_df.set_index("Time", inplace=True)
        phase_df.sort_index(inplace=True)

        return phase_df[phase_df.phase == 1].index[-1]

    @classmethod
    def check_from_target(cls, target):
        if not cls._has_asym_cpucapacity(target):
            raise CannotCreateError(
                "Target doesn't have asymmetric CPU capacities")

    @classmethod
    def get_rtapp_profile(cls, plat_info):
        cpus = list(range(plat_info['cpus-count']))

        # We're pinning stuff in the first phase, so give it ample time to
        # clean the pinned logic out of balance_interval
        free_time_s = 1.1 * cls._get_max_lb_interval(plat_info)

        # Ideally we'd like the different tasks not to complete at the same time
        # (hence the "staggered" name), but this depends on a lot of factors
        # (capacity ratios, available frequencies, thermal conditions...) so the
        # best we can do is wing it.
        stagger_s = cls._get_lb_interval(plat_info) * 1.5

        profile = {}

        for cpu in cpus:
            profile["{}{}".format(cls.task_prefix, cpu)] = (
                Periodic(
                    duty_cycle_pct=0,
                    duration_s=cls.IDLING_DELAY_S,
                    period_ms=cls.TASK_PERIOD_MS,
                    cpus=[cpu]
                ) + Periodic(
                    duty_cycle_pct=100,
                    duration_s=cls.PIN_DELAY_S,
                    period_ms=cls.TASK_PERIOD_MS,
                    cpus=[cpu]
                ) + Periodic(
                    duty_cycle_pct=100,
                    # Introduce staggered task completions
                    duration_s=free_time_s + cpu * stagger_s,
                    period_ms=cls.TASK_PERIOD_MS,
                    cpus=cpus
                )
            )

        return profile

    def _trim_state_df(self, state_df):
        if state_df.empty:
            return state_df

        return df_squash(state_df, self.start_time,
                         state_df.index[-1] + state_df['delta'].iloc[-1], "delta")

    @requires_events('sched_switch', TasksAnalysis.df_task_states.used_events)
    def test_preempt_time(self, allowed_preempt_pct=1) -> ResultBundle:
        """
        Test that tasks are not being preempted too much
        """

        sdf = self.trace.df_event('sched_switch')
        task_state_dfs = {
            task: self.trace.analysis.tasks.df_task_states(task)
            for task in self.rtapp_tasks
        }

        res = ResultBundle.from_bool(True)
        for task, state_df in task_state_dfs.items():
            # The sched_switch dataframe where the misfit task
            # is replaced by another misfit task
            preempt_sdf = sdf[
                (sdf.prev_comm == task) &
                (sdf.next_comm.str.startswith(self.task_prefix))
            ]

            state_df = self._trim_state_df(state_df)
            state_df = state_df[
                (state_df.index.isin(preempt_sdf.index)) &
                # Ensure this is a preemption and not just the task ending
                (state_df.curr_state == TaskState.TASK_INTERRUPTIBLE)
            ]

            preempt_time = state_df.delta.sum()
            preempt_pct = (preempt_time / self.duration) * 100

            res.add_metric("{} preemption".format(task), {
                "ratio": TestMetric(preempt_pct, "%"),
                "time": TestMetric(preempt_time, "seconds")})

            if preempt_pct > allowed_preempt_pct:
                res.result = Result.FAILED

        return res

    @memoized
    @IdleAnalysis.signal_cpu_active.used_events
    def _get_active_df(self, cpu):
        """
        :returns: A dataframe that describes the idle status (on/off) of 'cpu'
        """
        active_df = pd.DataFrame(
            self.trace.analysis.idle.signal_cpu_active(cpu), columns=['state']
        )
        df_add_delta(active_df, inplace=True, window=self.trace.window)
        return active_df

    @_get_active_df.used_events
    def _max_idle_time(self, start, end, cpus):
        """
        :returns: The maximum idle time of 'cpus' in the [start, end] interval
        """
        max_time = 0
        max_cpu = 0

        for cpu in cpus:
            busy_df = self._get_active_df(cpu)
            busy_df = df_squash(busy_df, start, end)
            busy_df = busy_df[busy_df.state == 0]

            if busy_df.empty:
                continue

            local_max = busy_df.delta.max()
            if local_max > max_time:
                max_time = local_max
                max_cpu = cpu

        return max_time, max_cpu

    @_max_idle_time.used_events
    def _test_cpus_busy(self, task_state_dfs, cpus, allowed_idle_time_s):
        """
        Test that for every window in which the tasks are running, :attr:`cpus`
        are not idle for more than :attr:`allowed_idle_time_s`
        """
        if allowed_idle_time_s is None:
            allowed_idle_time_s = self._get_lb_interval(self.plat_info)

        res = ResultBundle.from_bool(True)

        for task, state_df in task_state_dfs.items():
            # Have a look at every task activation
            task_idle_times = [self._max_idle_time(index, index + row.delta, cpus)
                               for index, row in state_df.iterrows()]

            if not task_idle_times:
                continue

            max_time, max_cpu = max(task_idle_times)
            res.add_metric("{} max idle".format(task), data={
                "time": TestMetric(max_time, "seconds"), "cpu": TestMetric(max_cpu)})

            if max_time > allowed_idle_time_s:
                res.result = Result.FAILED

        return res

    @TasksAnalysis.df_task_states.used_events
    @_test_cpus_busy.used_events
    @RTATestBundle.test_noisy_tasks.undecided_filter(noise_threshold_pct=1)
    def test_throughput(self, allowed_idle_time_s=None) -> ResultBundle:
        """
        Test that big CPUs are not idle when there are misfit tasks to upmigrate

        :param allowed_idle_time_s: How much time should be allowed between a
          big CPU going idle and a misfit task ending on that CPU. In theory
          a newidle balance should lead to a null delay, but in practice
          there's a tiny one, so don't set that to 0 and expect the test to
          pass.

          Furthermore, we're not always guaranteed to get a newidle pull, so
          allow time for a regular load balance to happen.

          When ``None``, this defaults to (1ms x number_of_cpus) to mimic the
          default balance_interval (balance_interval = sd_weight), see
          kernel/sched/topology.c:sd_init().
        :type allowed_idle_time_s: int
        """
        task_state_dfs = {}
        for task in self.rtapp_tasks:
            # This test is all about throughput: check that every time a task
            # runs on a little it's because bigs are busy
            df = self.trace.analysis.tasks.df_task_states(task)
            # Trim first to keep coherent deltas
            df = self._trim_state_df(df)
            task_state_dfs[task] = df[
                # Task is active
                (df.curr_state == TaskState.TASK_ACTIVE) &
                # Task needs to be upmigrated
                (df.cpu.isin(self.src_cpus))
            ]

        return self._test_cpus_busy(task_state_dfs, self.dst_cpus, allowed_idle_time_s)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright (c) 2012 Roberto Alsina y otros.

# Permission is hereby granted, free of charge, to any
# person obtaining a copy of this software and associated
# documentation files (the "Software"), to deal in the
# Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the
# Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice
# shall be included in all copies or substantial portions of
# the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

from __future__ import unicode_literals, print_function

import codecs
from collections import defaultdict
import locale
import os
import re
import string

import lxml.html

from .utils import to_datetime, slugify, bytes_str

__all__ = ['Post']

TEASER_REGEXP = re.compile('', re.IGNORECASE)

class Post(object):

    """Represents a blog post or web page."""

    def __init__(
        self, source_path, cache_folder, destination, use_in_feeds,
        translations, default_lang, base_url, messages, template_name,
        file_metadata_regexp=None, tzinfo=None
    ):
        """Initialize post.

        The source path is the .txt post file. From it we calculate
        the meta file, as well as any translations available, and
        the .html fragment file path.
        """
        self.translated_to = set([default_lang])
        self.prev_post = None
        self.next_post = None
        self.base_url = base_url
        self.is_draft = False
        self.is_mathjax = False
        self.source_path = source_path  # posts/blah.txt
        self.post_name = os.path.splitext(source_path)[0]  # posts/blah
        # cache/posts/blah.html
        self.base_path = os.path.join(cache_folder, self.post_name + ".html")
        self.metadata_path = self.post_name + ".meta"  # posts/blah.meta
        self.folder = destination
        self.translations = translations
        self.default_lang = default_lang
        self.messages = messages
        self.template_name = template_name

        default_metadata = get_meta(self, file_metadata_regexp)

        self.meta = {}
        self.meta[default_lang] = default_metadata

        if 'title' not in default_metadata or 'slug' not in default_metadata \
                or 'date' not in default_metadata:
            raise OSError("You must set a title (found '{0}'), a slug (found "
                          "'{1}') and a date (found '{2}')! [in file "
                          "{3}]".format(default_metadata.get('title', None),
                                        default_metadata.get('slug', None),
                                        default_metadata.get('date', None),
                                        source_path))

        # If timezone is set, build localized datetime.
        self.date = to_datetime(self.meta[default_lang]['date'], tzinfo)
        self.tags = [x.strip() for x in self.meta[default_lang]['tags'].split(',')]
        self.tags = [_f for _f in self.tags if _f]

        # While draft comes from the tags, it's not really a tag
        self.use_in_feeds = use_in_feeds and "draft" not in self.tags
        self.is_draft = 'draft' in self.tags
        self.tags = [t for t in self.tags if t != 'draft']

        # If mathjax is a tag, then enable mathjax rendering support
        self.is_mathjax = 'mathjax' in self.tags

        # Load internationalized metadata
        for lang in translations:
            if lang != default_lang:
                if os.path.isfile(self.source_path + "." + lang):
                    self.translated_to.add(lang)

                meta = defaultdict(lambda: '')
                meta.update(default_metadata)
                meta.update(get_meta(self, file_metadata_regexp, lang))
                self.meta[lang] = meta

    def _add_old_metadata(self):
        # Compatibility for themes up to Nikola 5.4.1
        # TODO: remove before Nikola 6
        self.pagenames = {}
        self.titles = {}
        for lang in self.translations:
            self.pagenames[lang] = self.meta[lang]['slug']
            self.titles[lang] = self.meta[lang]['title']

    def formatted_date(self, date_format):
        """Return the formatted date, as unicode."""
        fmt_date = self.date.strftime(date_format)
        # Issue #383, this changes from py2 to py3
        if isinstance(fmt_date, bytes_str):
            fmt_date = fmt_date.decode('utf8')
        return fmt_date

    def current_lang(self):
        """Return the currently set locale, if it's one of the
        available translations, or default_lang."""
        lang = locale.getlocale()[0]
        if lang:
            if lang in self.translations:
                return lang
            lang = lang.split('_')[0]
            if lang in self.translations:
                return lang
        # whatever
        return self.default_lang

    def title(self, lang=None):
        """Return localized title.

        If lang is not specified, it will use the currently set locale,
        because templates set it.
        """
        if lang is None:
            lang = self.current_lang()
        return self.meta[lang]['title']

    def description(self, lang=None):
        """Return localized description."""
        if lang is None:
            lang = self.current_lang()
        return self.meta[lang]['description']

    def deps(self, lang):
        """Return a list of dependencies to build this post's page."""
        deps = [self.base_path]
        if lang != self.default_lang:
            deps += [self.base_path + "." + lang]
        deps += self.fragment_deps(lang)
        return deps

    def fragment_deps(self, lang):
        """Return a list of dependencies to build this post's fragment."""
        deps = [self.source_path]
        if os.path.isfile(self.metadata_path):
            deps.append(self.metadata_path)
        if lang != self.default_lang:
            lang_deps = list(filter(os.path.exists, [x + "." + lang for x in
                                                     deps]))
            deps += lang_deps
        return deps

    def is_translation_available(self, lang):
        """Return true if the translation actually exists."""
        return lang in self.translated_to

    def _translated_file_path(self, lang):
        """Return path to the translation's file, or to the original."""
        file_name = self.base_path
        if lang != self.default_lang:
            file_name_lang = '.'.join((file_name, lang))
            if os.path.exists(file_name_lang):
                file_name = file_name_lang
        return file_name

    def text(self, lang=None, teaser_only=False, strip_html=False):
        """Read the post file for that language and return its contents."""
        if lang is None:
            lang = self.current_lang()
        file_name = self._translated_file_path(lang)

        with codecs.open(file_name, "r", "utf8") as post_file:
            data = post_file.read().strip()

        try:
            document = lxml.html.document_fromstring(data)
        except lxml.etree.ParserError as e:
            # if we don't catch this, it breaks later (Issue #374)
            if str(e) == "Document is empty":
                return ""
            # let other errors raise
            raise(e)
        document.make_links_absolute(self.permalink(lang=lang))
        data = lxml.html.tostring(document, encoding='unicode')
        if teaser_only:
            teaser = TEASER_REGEXP.split(data)[0]
            teaser_str = self.messages[lang]["Read more"] + '...'
            teaser += '
{1}
'.format(
                self.permalink(lang), teaser_str)
            # This closes all open tags and sanitizes the broken HTML
            document = lxml.html.fromstring(teaser)
            data = lxml.html.tostring(document, encoding='unicode')

        if data and strip_html:
            content = lxml.html.fromstring(data)
            data = content.text_content().strip()  # No whitespace wanted.
        return data

    def destination_path(self, lang, extension='.html'):
        path = os.path.join(self.translations[lang],
                            self.folder, self.meta[lang]['slug'] + extension)
        return path

    def permalink(self, lang=None, absolute=False, extension='.html'):
        if lang is None:
            lang = self.current_lang()
        pieces = list(os.path.split(self.translations[lang]))
        pieces += list(os.path.split(self.folder))
        pieces += [self.meta[lang]['slug'] + extension]
        pieces = [_f for _f in pieces if _f and _f != '.']
        if absolute:
            pieces = [self.base_url] + pieces
        else:
            pieces = [""] + pieces
        link = "/".join(pieces)
        return link

    def source_ext(self):
        return os.path.splitext(self.source_path)[1]

# Code that fetches metadata from different places

def re_meta(line, match=None):
    """re.compile for meta"""
    if match:
        reStr = re.compile('^\.\. {0}: (.*)'.format(re.escape(match)))
    else:
        reStr = re.compile('^\.\. (.*?): (.*)')
    result = reStr.findall(line.strip())
    if match and result:
        return (match, result[0])
    elif not match and result:
        return (result[0][0], result[0][1].strip())
    else:
        return (None,)

def _get_metadata_from_filename_by_regex(filename, metadata_regexp):
    """
    Tries to ried the metadata from the filename based on the given re.
    This requires to use symbolic group names in the pattern.

    The part to read the metadata from the filename based on a regular
    expression is taken from Pelican - pelican/readers.py
    """
    match = re.match(metadata_regexp, filename)
    meta = {}

    if match:
        # .items() for py3k compat.
        for key, value in match.groupdict().items():
            meta[key.lower()] = value  # metadata must be lowercase

    return meta

def get_metadata_from_file(source_path, lang=None):
    """Extracts metadata from the file itself, by parsing contents."""
    try:
        if lang:
            source_path = "{0}.{1}".format(source_path, lang)
        with codecs.open(source_path, "r", "utf8") as meta_file:
            meta_data = [x.strip() for x in meta_file.readlines()]
        return _get_metadata_from_file(meta_data)
    except Exception:  # The file may not exist, for multilingual sites
        return {}

def _get_metadata_from_file(meta_data):
    """Parse file contents and obtain metadata.

    >>> g = _get_metadata_from_file
    >>> list(g([]).values())
    []
    >>> str(g(["FooBar","======"])["title"])
    'FooBar'
    >>> str(g(["#FooBar"])["title"])
    'FooBar'
    >>> str(g([".. title: FooBar"])["title"])
    'FooBar'
    >>> 'title' in g(["",".. title: FooBar"])
    False

    """
    meta = {}

    re_md_title = re.compile(r'^{0}([^{0}].*)'.format(re.escape('#')))
    # Assuming rst titles are going to be at least 4 chars long
    # otherwise this detects things like ''' wich breaks other markups.
    re_rst_title = re.compile(r'^([{0}]{{4,}})'.format(re.escape(
        string.punctuation)))

    for i, line in enumerate(meta_data):
        if not line:
            break
        if 'title' not in meta:
            match = re_meta(line, 'title')
            if match[0]:
                meta['title'] = match[1]
        if 'title' not in meta:
            if re_rst_title.findall(line) and i > 0:
                meta['title'] = meta_data[i - 1].strip()
        if 'title' not in meta:
            if re_md_title.findall(line):
                meta['title'] = re_md_title.findall(line)[0]

        match = re_meta(line)
        if match[0]:
            meta[match[0]] = match[1]

    return meta

def get_metadata_from_meta_file(path, lang=None):
    """Takes a post path, and gets data from a matching .meta file."""
    meta_path = os.path.splitext(path)[0] + '.meta'
    if lang:
        meta_path += '.' + lang
    if os.path.isfile(meta_path):
        with codecs.open(meta_path, "r", "utf8") as meta_file:
            meta_data = meta_file.readlines()
        while len(meta_data) < 6:
            meta_data.append("")
        (title, slug, date, tags, link, description) = [
            x.strip() for x in meta_data][:6]

        meta = {}

        if title:
            meta['title'] = title
        if slug:
            meta['slug'] = slug
        if date:
            meta['date'] = date
        if tags:
            meta['tags'] = tags
        if link:
            meta['link'] = link
        if description:
            meta['description'] = description

        return meta
    else:
        return {}

def get_meta(post, file_metadata_regexp=None, lang=None):
    """Get post's meta from source.

    If ``file_metadata_regexp`` is given it will be tried to read
    metadata from the filename.
    If any metadata is then found inside the file the metadata from the
    file will override previous findings.
    """
    meta = defaultdict(lambda: '')

    meta.update(get_metadata_from_meta_file(post.metadata_path, lang))

    if meta:
        return meta

    if file_metadata_regexp is not None:
        meta.update(_get_metadata_from_filename_by_regex(post.source_path,
                                                         file_metadata_regexp))

    meta.update(get_metadata_from_file(post.source_path, lang))

    if lang is None:
        # Only perform these checks for the default language

        if 'slug' not in meta:
            # If no slug is found in the metadata use the filename
            meta['slug'] = slugify(os.path.splitext(
                os.path.basename(post.source_path))[0])

        if 'title' not in meta:
            # If no title is found, use the filename without extension
            meta['title'] = os.path.splitext(
                os.path.basename(post.source_path))[0]

    return meta
<commit_msg>don't add Read more if there is no actual teaser<commit_after># -*- coding: utf-8 -*-
# Copyright (c) 2012 Roberto Alsina y otros.

# Permission is hereby granted, free of charge, to any
# person obtaining a copy of this software and associated
# documentation files (the "Software"), to deal in the
# Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the
# Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice
# shall be included in all copies or substantial portions of
# the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

from __future__ import unicode_literals, print_function

import codecs
from collections import defaultdict
import locale
import os
import re
import string

import lxml.html

from .utils import to_datetime, slugify, bytes_str

__all__ = ['Post']

TEASER_REGEXP = re.compile('', re.IGNORECASE)

class Post(object):

    """Represents a blog post or web page."""

    def __init__(
        self, source_path, cache_folder, destination, use_in_feeds,
        translations, default_lang, base_url, messages, template_name,
        file_metadata_regexp=None, tzinfo=None
    ):
        """Initialize post.

        The source path is the .txt post file. From it we calculate
        the meta file, as well as any translations available, and
        the .html fragment file path.
        """
        self.translated_to = set([default_lang])
        self.prev_post = None
        self.next_post = None
        self.base_url = base_url
        self.is_draft = False
        self.is_mathjax = False
        self.source_path = source_path  # posts/blah.txt
        self.post_name = os.path.splitext(source_path)[0]  # posts/blah
        # cache/posts/blah.html
        self.base_path = os.path.join(cache_folder, self.post_name + ".html")
        self.metadata_path = self.post_name + ".meta"  # posts/blah.meta
        self.folder = destination
        self.translations = translations
        self.default_lang = default_lang
        self.messages = messages
        self.template_name = template_name

        default_metadata = get_meta(self, file_metadata_regexp)

        self.meta = {}
        self.meta[default_lang] = default_metadata

        if 'title' not in default_metadata or 'slug' not in default_metadata \
                or 'date' not in default_metadata:
            raise OSError("You must set a title (found '{0}'), a slug (found "
                          "'{1}') and a date (found '{2}')! [in file "
                          "{3}]".format(default_metadata.get('title', None),
                                        default_metadata.get('slug', None),
                                        default_metadata.get('date', None),
                                        source_path))

        # If timezone is set, build localized datetime.
        self.date = to_datetime(self.meta[default_lang]['date'], tzinfo)
        self.tags = [x.strip() for x in self.meta[default_lang]['tags'].split(',')]
        self.tags = [_f for _f in self.tags if _f]

        # While draft comes from the tags, it's not really a tag
        self.use_in_feeds = use_in_feeds and "draft" not in self.tags
        self.is_draft = 'draft' in self.tags
        self.tags = [t for t in self.tags if t != 'draft']

        # If mathjax is a tag, then enable mathjax rendering support
        self.is_mathjax = 'mathjax' in self.tags

        # Load internationalized metadata
        for lang in translations:
            if lang != default_lang:
                if os.path.isfile(self.source_path + "." + lang):
                    self.translated_to.add(lang)

                meta = defaultdict(lambda: '')
                meta.update(default_metadata)
                meta.update(get_meta(self, file_metadata_regexp, lang))
                self.meta[lang] = meta

    def _add_old_metadata(self):
        # Compatibility for themes up to Nikola 5.4.1
        # TODO: remove before Nikola 6
        self.pagenames = {}
        self.titles = {}
        for lang in self.translations:
            self.pagenames[lang] = self.meta[lang]['slug']
            self.titles[lang] = self.meta[lang]['title']

    def formatted_date(self, date_format):
        """Return the formatted date, as unicode."""
        fmt_date = self.date.strftime(date_format)
        # Issue #383, this changes from py2 to py3
        if isinstance(fmt_date, bytes_str):
            fmt_date = fmt_date.decode('utf8')
        return fmt_date

    def current_lang(self):
        """Return the currently set locale, if it's one of the
        available translations, or default_lang."""
        lang = locale.getlocale()[0]
        if lang:
            if lang in self.translations:
                return lang
            lang = lang.split('_')[0]
            if lang in self.translations:
                return lang
        # whatever
        return self.default_lang

    def title(self, lang=None):
        """Return localized title.

        If lang is not specified, it will use the currently set locale,
        because templates set it.
        """
        if lang is None:
            lang = self.current_lang()
        return self.meta[lang]['title']

    def description(self, lang=None):
        """Return localized description."""
        if lang is None:
            lang = self.current_lang()
        return self.meta[lang]['description']

    def deps(self, lang):
        """Return a list of dependencies to build this post's page."""
        deps = [self.base_path]
        if lang != self.default_lang:
            deps += [self.base_path + "." + lang]
        deps += self.fragment_deps(lang)
        return deps

    def fragment_deps(self, lang):
        """Return a list of dependencies to build this post's fragment."""
        deps = [self.source_path]
        if os.path.isfile(self.metadata_path):
            deps.append(self.metadata_path)
        if lang != self.default_lang:
            lang_deps = list(filter(os.path.exists, [x + "." + lang for x in
                                                     deps]))
            deps += lang_deps
        return deps

    def is_translation_available(self, lang):
        """Return true if the translation actually exists."""
        return lang in self.translated_to

    def _translated_file_path(self, lang):
        """Return path to the translation's file, or to the original."""
        file_name = self.base_path
        if lang != self.default_lang:
            file_name_lang = '.'.join((file_name, lang))
            if os.path.exists(file_name_lang):
                file_name = file_name_lang
        return file_name

    def text(self, lang=None, teaser_only=False, strip_html=False):
        """Read the post file for that language and return its contents."""
        if lang is None:
            lang = self.current_lang()
        file_name = self._translated_file_path(lang)

        with codecs.open(file_name, "r", "utf8") as post_file:
            data = post_file.read().strip()

        try:
            document = lxml.html.document_fromstring(data)
        except lxml.etree.ParserError as e:
            # if we don't catch this, it breaks later (Issue #374)
            if str(e) == "Document is empty":
                return ""
            # let other errors raise
            raise(e)
        document.make_links_absolute(self.permalink(lang=lang))
        data = lxml.html.tostring(document, encoding='unicode')
        if teaser_only:
            teaser = TEASER_REGEXP.split(data)[0]
            if teaser != data:
                teaser_str = self.messages[lang]["Read more"] + '...'
                teaser += '
{1}
'.format(
                    self.permalink(lang), teaser_str)
                # This closes all open tags and sanitizes the broken HTML
                document = lxml.html.fromstring(teaser)
                data = lxml.html.tostring(document, encoding='unicode')

        if data and strip_html:
            content = lxml.html.fromstring(data)
            data = content.text_content().strip()  # No whitespace wanted.
        return data

    def destination_path(self, lang, extension='.html'):
        path = os.path.join(self.translations[lang],
                            self.folder, self.meta[lang]['slug'] + extension)
        return path

    def permalink(self, lang=None, absolute=False, extension='.html'):
        if lang is None:
            lang = self.current_lang()
        pieces = list(os.path.split(self.translations[lang]))
        pieces += list(os.path.split(self.folder))
        pieces += [self.meta[lang]['slug'] + extension]
        pieces = [_f for _f in pieces if _f and _f != '.']
        if absolute:
            pieces = [self.base_url] + pieces
        else:
            pieces = [""] + pieces
        link = "/".join(pieces)
        return link

    def source_ext(self):
        return os.path.splitext(self.source_path)[1]

# Code that fetches metadata from different places

def re_meta(line, match=None):
    """re.compile for meta"""
    if match:
        reStr = re.compile('^\.\. {0}: (.*)'.format(re.escape(match)))
    else:
        reStr = re.compile('^\.\. (.*?): (.*)')
    result = reStr.findall(line.strip())
    if match and result:
        return (match, result[0])
    elif not match and result:
        return (result[0][0], result[0][1].strip())
    else:
        return (None,)

def _get_metadata_from_filename_by_regex(filename, metadata_regexp):
    """
    Tries to ried the metadata from the filename based on the given re.
    This requires to use symbolic group names in the pattern.

    The part to read the metadata from the filename based on a regular
    expression is taken from Pelican - pelican/readers.py
    """
    match = re.match(metadata_regexp, filename)
    meta = {}

    if match:
        # .items() for py3k compat.
        for key, value in match.groupdict().items():
            meta[key.lower()] = value  # metadata must be lowercase

    return meta

def get_metadata_from_file(source_path, lang=None):
    """Extracts metadata from the file itself, by parsing contents."""
    try:
        if lang:
            source_path = "{0}.{1}".format(source_path, lang)
        with codecs.open(source_path, "r", "utf8") as meta_file:
            meta_data = [x.strip() for x in meta_file.readlines()]
        return _get_metadata_from_file(meta_data)
    except Exception:  # The file may not exist, for multilingual sites
        return {}

def _get_metadata_from_file(meta_data):
    """Parse file contents and obtain metadata.

    >>> g = _get_metadata_from_file
    >>> list(g([]).values())
    []
    >>> str(g(["FooBar","======"])["title"])
    'FooBar'
    >>> str(g(["#FooBar"])["title"])
    'FooBar'
    >>> str(g([".. title: FooBar"])["title"])
    'FooBar'
    >>> 'title' in g(["",".. title: FooBar"])
    False

    """
    meta = {}

    re_md_title = re.compile(r'^{0}([^{0}].*)'.format(re.escape('#')))
    # Assuming rst titles are going to be at least 4 chars long
    # otherwise this detects things like ''' wich breaks other markups.
    re_rst_title = re.compile(r'^([{0}]{{4,}})'.format(re.escape(
        string.punctuation)))

    for i, line in enumerate(meta_data):
        if not line:
            break
        if 'title' not in meta:
            match = re_meta(line, 'title')
            if match[0]:
                meta['title'] = match[1]
        if 'title' not in meta:
            if re_rst_title.findall(line) and i > 0:
                meta['title'] = meta_data[i - 1].strip()
        if 'title' not in meta:
            if re_md_title.findall(line):
                meta['title'] = re_md_title.findall(line)[0]

        match = re_meta(line)
        if match[0]:
            meta[match[0]] = match[1]

    return meta

def get_metadata_from_meta_file(path, lang=None):
    """Takes a post path, and gets data from a matching .meta file."""
    meta_path = os.path.splitext(path)[0] + '.meta'
    if lang:
        meta_path += '.' + lang
    if os.path.isfile(meta_path):
        with codecs.open(meta_path, "r", "utf8") as meta_file:
            meta_data = meta_file.readlines()
        while len(meta_data) < 6:
            meta_data.append("")
        (title, slug, date, tags, link, description) = [
            x.strip() for x in meta_data][:6]

        meta = {}

        if title:
            meta['title'] = title
        if slug:
            meta['slug'] = slug
        if date:
            meta['date'] = date
        if tags:
            meta['tags'] = tags
        if link:
            meta['link'] = link
        if description:
            meta['description'] = description

        return meta
    else:
        return {}

def get_meta(post, file_metadata_regexp=None, lang=None):
    """Get post's meta from source.

    If ``file_metadata_regexp`` is given it will be tried to read
    metadata from the filename.
    If any metadata is then found inside the file the metadata from the
    file will override previous findings.
    """
    meta = defaultdict(lambda: '')

    meta.update(get_metadata_from_meta_file(post.metadata_path, lang))

    if meta:
        return meta

    if file_metadata_regexp is not None:
        meta.update(_get_metadata_from_filename_by_regex(post.source_path,
                                                         file_metadata_regexp))

    meta.update(get_metadata_from_file(post.source_path, lang))

    if lang is None:
        # Only perform these checks for the default language

        if 'slug' not in meta:
            # If no slug is found in the metadata use the filename
            meta['slug'] = slugify(os.path.splitext(
                os.path.basename(post.source_path))[0])

        if 'title' not in meta:
            # If no title is found, use the filename without extension
            meta['title'] = os.path.splitext(
                os.path.basename(post.source_path))[0]

    return meta
<|endoftext|>