<commit_before>from django.shortcuts import render
from django.http import HttpResponse, HttpResponseNotFound, \
        HttpResponseServerError, HttpResponseRedirect
from django.conf import settings
from django.db.models import Q
import operator
import re
import json
import requests
from readmore.content.models import *
from readmore.main.models import *
import django.dispatch
import datetime
from django.contrib.auth.decorators import login_required

article_read = django.dispatch.Signal(
        providing_args=["user", "category", "article_id", "article"])

def update_feeds(request):
    for category in RSSCategory.objects.all():
        category.update_feed()
    for category in SevenDaysCategory.objects.all():
        category.update_feed()
    return HttpResponse()

@login_required
def index(request):
    """Return response containing overview of categories and articles."""
    # Only show top categories on the index page
    categories = Category.objects.filter(parent=None)
    articles = []
    for category in categories:
        articles += category.get_articles()
    # Render response
    if request.is_ajax():
        # Return JSON list of categories with their properties
        categories = [{'url': c.get_absolute_url(), 'title': c.title,
            'image': c.image} for c in categories]
        articles = [{'url': a.get_absolute_url(), 'title': a.title,
            'image': a.image} for a in articles]
        return HttpResponse(
                json.dumps({'categories': categories, 'articles': articles}),
                content_type='application/json')
    else:
        # Render HTML of the landing page containing top categories
        return render(request, 'articleOverview.html',{
                "articles": articles,
                "categories": categories})

@login_required
def query(request):
    """Return response containing articles matching the query."""
    query_string = request.GET.get('q', None)
    if query_string is None:
        # Fetch any subcategories and articles contained in the category.
        articles = []
        for article in RSSArticle.objects.order_by('-publication_date').all()[:50]:
            category = article.get_categories()[0]
            articles.append({
                'url': article.get_absolute_url(),
                'title': article.title,
                'category-color': category.color,
                'image': article.image if article.image else category.image})
        return HttpResponse(json.dumps({'articles': articles}), content_type='application/json')
    elif query_string.strip():
        matching = []
        articles = []
        querylist = [Q(body__icontains=query) for query in normalize_query(query_string)]
        querylist += [Q(title__icontains=query) for query in normalize_query(query_string)]
        matching = Article.objects.filter(reduce(operator.or_, querylist))
        matching = sorted(matching, key=(
            lambda x: x.publication_date if
                isinstance(x, RSSArticle) else datetime.datetime.now()),
                reverse=True)
        for article in matching:
            articles.append({
                'url': article.get_absolute_url(),
                'title': article.title,
                'category-color': article.categories.all().first().color,
                'image': article.image if article.image else article.categories.all().first().image})
        return HttpResponse(json.dumps({'articles': articles}), content_type="application/json")
    else:
        return HttpResponse(json.dumps({'articles': []}), content_type="application/json")

def normalize_query(query_string,
                    findterms=re.compile(r'"([^"]+)"|(\S+)').findall,
                    normspace=re.compile(r'\s{2,}').sub):
    return [normspace(' ', (t[0] or t[1]).strip()) for t in findterms(query_string)]

@login_required
def category(request, identifier, source='local'):
    """Return response containing contents of identified category.

    The category is identified by a identifier and optionally a source. When
    the source is set to 'local', the identifier must be the primary key of a
    Category object. When the source is set to 'wikipedia', the identifier must
    correspond to either a pageid or a title of a wikipedia page.

    Named keywords:
    identifier -- Number or string identifying the category
    source -- The source the identifier belongs to ( default 'local' )
    """
    if not request.is_ajax():
        return HttpResponseRedirect('/')
    # Resolve identified category
    if source == "wikipedia":
        # Get/set identifier type
        identifier_type = request.GET.get('type','auto')
        category = WikiCategory.factory(identifier, identifier_type)
        if category is None:
            return HttpResponseRedirect('/')
    else:
        try:
            category = Category.objects.get(pk=int(identifier))
        except Category.DoesNotExist:
            return HttpResponseRedirect('/')
    # Fetch any subcategories and articles contained in the category.
    articles = []
    for article in category.get_articles():
        d = {'url': article.get_absolute_url(),
            'title': article.title,
            'category-color': category.color,
            'image': article.image if article.image else category.image}
        if isinstance(article, RSSArticle):
            d.update({
                'publication_date': str(article.publication_date)
            })
        articles.append(d)
    subcategories = [{'url': c.get_absolute_url(), 'title': c.title,
        'image': c.image} for c in category.get_subcategories()]
    # Return JSON list of topics with their properties
    return HttpResponse(json.dumps({'articles': articles,
            'subcategories': subcategories}), content_type='application/json')

@login_required
def article(request, identifier, source='local'):
    """Return response containing the identified article.

    The article is identified by a identifier and optionally a source. When
    the source is set to 'local', the identifier must be the primary key of a
    Category object. When the source is set to 'wikipedia', the identifier must
    correspond to either a pageid or a title of a wikipedia page.

    Named keywords:
    identifier -- Number or string identifying the category
    source -- The source the identifier belongs to ( default 'local' )
    """
    # Resolve identified article
    if source == 'wikipedia':
        # Get/set identifier type
        identifier_type = request.GET.get('type','auto')
        # Attempt to retrieve the article
        article = WikiArticle.factory(identifier, identifier_type)
        if article is None:
            return HttpResponseRedirect('/')
    else:
        try:
            # Attempt to retrieve the article
            article = Article.objects.get(pk=int(identifier))
        except Article.DoesNotExist:
            return HttpResponseRedirect('/')

    # Retrieve the categories that this article belongs to.
    categories = article.get_categories()

    if not request.is_ajax():
        # Fetch random articles from the same categories for reading suggestions.
        random_articles = []
        read_articles = ArticleHistoryItem.objects.filter(user=request.user)
        read_articles = list(set([history.article.pk for history in read_articles] + [article.pk]))
        for category in categories:
            random_articles += category.get_random_articles(3, read_articles)

        # Ensure the current article is not suggested again
        if article in random_articles:
            random_articles.remove(article)

        difficulty_items = ArticleDifficultyItem.objects.filter(user=request.user, article=article)
        rating_items = ArticleRatingItem.objects.filter(user=request.user, article=article)

        recommendations = []
        for rand_article in random_articles:
            if rand_article.image:
                recommendations.append(rand_article)

        # For all categories this article belongs to
        for category in categories:
            # If the category was stored in the database
            if category.pk is not None:
                article_read.send(
                        sender=Article,
                        user=request.user,
                        category=category,
                        article_id=identifier,
                        article=article)
        return render(request, 'article_page.html',
                {
                "article": article,
                "random_articles": recommendations,
                "rating_given": len(rating_items)>0,
                "difficulty_given": len(difficulty_items)>0,
                })
    else:
        # Return JSON with article properties
        return HttpResponse(
            json.dumps({
                'title': article.title,
                'image': article.image,
                'body': article.get_body()
            }),
            content_type='application/json')
<commit_msg>Make default date timezone aware<commit_after>from django.shortcuts import render
from django.http import HttpResponse, HttpResponseNotFound, \
        HttpResponseServerError, HttpResponseRedirect
from django.conf import settings
from django.db.models import Q
import operator
import re
import json
import requests
from readmore.content.models import *
from readmore.main.models import *
import django.dispatch
import datetime
from django.utils import timezone
from django.contrib.auth.decorators import login_required

article_read = django.dispatch.Signal(
        providing_args=["user", "category", "article_id", "article"])

def update_feeds(request):
    for category in RSSCategory.objects.all():
        category.update_feed()
    for category in SevenDaysCategory.objects.all():
        category.update_feed()
    return HttpResponse()

@login_required
def index(request):
    """Return response containing overview of categories and articles."""
    # Only show top categories on the index page
    categories = Category.objects.filter(parent=None)
    articles = []
    for category in categories:
        articles += category.get_articles()
    # Render response
    if request.is_ajax():
        # Return JSON list of categories with their properties
        categories = [{'url': c.get_absolute_url(), 'title': c.title,
            'image': c.image} for c in categories]
        articles = [{'url': a.get_absolute_url(), 'title': a.title,
            'image': a.image} for a in articles]
        return HttpResponse(
                json.dumps({'categories': categories, 'articles': articles}),
                content_type='application/json')
    else:
        # Render HTML of the landing page containing top categories
        return render(request, 'articleOverview.html',{
                "articles": articles,
                "categories": categories})

@login_required
def query(request):
    """Return response containing articles matching the query."""
    query_string = request.GET.get('q', None)
    if query_string is None:
        # Fetch any subcategories and articles contained in the category.
        articles = []
        for article in RSSArticle.objects.order_by('-publication_date').all()[:50]:
            category = article.get_categories()[0]
            articles.append({
                'url': article.get_absolute_url(),
                'title': article.title,
                'category-color': category.color,
                'image': article.image if article.image else category.image})
        return HttpResponse(json.dumps({'articles': articles}), content_type='application/json')
    elif query_string.strip():
        matching = []
        articles = []
        querylist = [Q(body__icontains=query) for query in normalize_query(query_string)]
        querylist += [Q(title__icontains=query) for query in normalize_query(query_string)]
        matching = Article.objects.filter(reduce(operator.or_, querylist))
        default = timezone.make_aware(datetime.datetime.now(), timezone.utc)
        matching = sorted(matching, key=(
            lambda x: x.publication_date if
                isinstance(x, RSSArticle) else default), reverse=True)
        for article in matching:
            articles.append({
                'url': article.get_absolute_url(),
                'title': article.title,
                'category-color': article.categories.all().first().color,
                'image': article.image if article.image else article.categories.all().first().image})
        return HttpResponse(json.dumps({'articles': articles}), content_type="application/json")
    else:
        return HttpResponse(json.dumps({'articles': []}), content_type="application/json")

def normalize_query(query_string,
                    findterms=re.compile(r'"([^"]+)"|(\S+)').findall,
                    normspace=re.compile(r'\s{2,}').sub):
    return [normspace(' ', (t[0] or t[1]).strip()) for t in findterms(query_string)]

@login_required
def category(request, identifier, source='local'):
    """Return response containing contents of identified category.

    The category is identified by a identifier and optionally a source. When
    the source is set to 'local', the identifier must be the primary key of a
    Category object. When the source is set to 'wikipedia', the identifier must
    correspond to either a pageid or a title of a wikipedia page.

    Named keywords:
    identifier -- Number or string identifying the category
    source -- The source the identifier belongs to ( default 'local' )
    """
    if not request.is_ajax():
        return HttpResponseRedirect('/')
    # Resolve identified category
    if source == "wikipedia":
        # Get/set identifier type
        identifier_type = request.GET.get('type','auto')
        category = WikiCategory.factory(identifier, identifier_type)
        if category is None:
            return HttpResponseRedirect('/')
    else:
        try:
            category = Category.objects.get(pk=int(identifier))
        except Category.DoesNotExist:
            return HttpResponseRedirect('/')
    # Fetch any subcategories and articles contained in the category.
    articles = []
    for article in category.get_articles():
        d = {'url': article.get_absolute_url(),
            'title': article.title,
            'category-color': category.color,
            'image': article.image if article.image else category.image}
        if isinstance(article, RSSArticle):
            d.update({
                'publication_date': str(article.publication_date)
            })
        articles.append(d)
    subcategories = [{'url': c.get_absolute_url(), 'title': c.title,
        'image': c.image} for c in category.get_subcategories()]
    # Return JSON list of topics with their properties
    return HttpResponse(json.dumps({'articles': articles,
            'subcategories': subcategories}), content_type='application/json')

@login_required
def article(request, identifier, source='local'):
    """Return response containing the identified article.

    The article is identified by a identifier and optionally a source. When
    the source is set to 'local', the identifier must be the primary key of a
    Category object. When the source is set to 'wikipedia', the identifier must
    correspond to either a pageid or a title of a wikipedia page.

    Named keywords:
    identifier -- Number or string identifying the category
    source -- The source the identifier belongs to ( default 'local' )
    """
    # Resolve identified article
    if source == 'wikipedia':
        # Get/set identifier type
        identifier_type = request.GET.get('type','auto')
        # Attempt to retrieve the article
        article = WikiArticle.factory(identifier, identifier_type)
        if article is None:
            return HttpResponseRedirect('/')
    else:
        try:
            # Attempt to retrieve the article
            article = Article.objects.get(pk=int(identifier))
        except Article.DoesNotExist:
            return HttpResponseRedirect('/')

    # Retrieve the categories that this article belongs to.
    categories = article.get_categories()

    if not request.is_ajax():
        # Fetch random articles from the same categories for reading suggestions.
        random_articles = []
        read_articles = ArticleHistoryItem.objects.filter(user=request.user)
        read_articles = list(set([history.article.pk for history in read_articles] + [article.pk]))
        for category in categories:
            random_articles += category.get_random_articles(3, read_articles)

        # Ensure the current article is not suggested again
        if article in random_articles:
            random_articles.remove(article)

        difficulty_items = ArticleDifficultyItem.objects.filter(user=request.user, article=article)
        rating_items = ArticleRatingItem.objects.filter(user=request.user, article=article)

        recommendations = []
        for rand_article in random_articles:
            if rand_article.image:
                recommendations.append(rand_article)

        # For all categories this article belongs to
        for category in categories:
            # If the category was stored in the database
            if category.pk is not None:
                article_read.send(
                        sender=Article,
                        user=request.user,
                        category=category,
                        article_id=identifier,
                        article=article)
        return render(request, 'article_page.html',
                {
                "article": article,
                "random_articles": recommendations,
                "rating_given": len(rating_items)>0,
                "difficulty_given": len(difficulty_items)>0,
                })
    else:
        # Return JSON with article properties
        return HttpResponse(
            json.dumps({
                'title': article.title,
                'image': article.image,
                'body': article.get_body()
            }),
            content_type='application/json')
<|endoftext|>


<commit_before>import sys
import inspect

from ...vendor.Qt import QtWidgets, QtCore, QtGui
from ...vendor import qtawesome
from ...vendor import six
from ... import api, io, style

from .. import lib

module = sys.modules[__name__]
module.window = None
module.root = api.registered_root()

HelpRole = QtCore.Qt.UserRole + 2
FamilyRole = QtCore.Qt.UserRole + 3
ExistsRole = QtCore.Qt.UserRole + 4
PluginRole = QtCore.Qt.UserRole + 5

Separator = "---separator---"

class Window(QtWidgets.QDialog):

    stateChanged = QtCore.Signal(bool)

    def __init__(self, parent=None):
        super(Window, self).__init__(parent)
        self.setWindowTitle("Instance Creator")
        self.setFocusPolicy(QtCore.Qt.StrongFocus)

        # Store the widgets for lookup in here
        self.data = dict()

        # Store internal states in here
        self.state = {
            "valid": False
        }

        body = QtWidgets.QWidget()
        lists = QtWidgets.QWidget()
        footer = QtWidgets.QWidget()

        container = QtWidgets.QWidget()

        listing = QtWidgets.QListWidget()
        asset = QtWidgets.QLineEdit()
        name = QtWidgets.QLineEdit()
        result = QtWidgets.QLineEdit()
        result.setStyleSheet("color: gray;")
        result.setEnabled(False)

        # region Menu for default subset names

        subset_button = QtWidgets.QPushButton()
        subset_button.setFixedWidth(18)
        subset_button.setFixedHeight(20)
        subset_menu = QtWidgets.QMenu(subset_button)
        subset_button.setMenu(subset_menu)

        # endregion

        name_layout = QtWidgets.QHBoxLayout()
        name_layout.addWidget(name)
        name_layout.addWidget(subset_button)
        name_layout.setContentsMargins(0, 0, 0, 0)

        layout = QtWidgets.QVBoxLayout(container)

        header = FamilyDescriptionWidget(parent=self)
        layout.addWidget(header)

        layout.addWidget(QtWidgets.QLabel("Family"))
        layout.addWidget(listing)
        layout.addWidget(QtWidgets.QLabel("Asset"))
        layout.addWidget(asset)
        layout.addWidget(QtWidgets.QLabel("Subset"))
        layout.addLayout(name_layout)
        layout.addWidget(result)
        layout.setContentsMargins(0, 0, 0, 0)

        options = QtWidgets.QWidget()

        useselection_chk = QtWidgets.QCheckBox("Use selection")
        useselection_chk.setCheckState(QtCore.Qt.Checked)

        layout = QtWidgets.QGridLayout(options)
        layout.setContentsMargins(0, 0, 0, 0)
        layout.addWidget(useselection_chk, 1, 1)

        layout = QtWidgets.QHBoxLayout(lists)
        layout.addWidget(container)
        layout.setContentsMargins(0, 0, 0, 0)

        layout = QtWidgets.QVBoxLayout(body)
        layout.addWidget(lists)
        layout.addWidget(options, 0, QtCore.Qt.AlignLeft)
        layout.setContentsMargins(0, 0, 0, 0)

        create_btn = QtWidgets.QPushButton("Create")
        error_msg = QtWidgets.QLabel()
        error_msg.setFixedHeight(20)

        layout = QtWidgets.QVBoxLayout(footer)
        layout.addWidget(create_btn)
        layout.addWidget(error_msg)
        layout.setContentsMargins(0, 0, 0, 0)

        layout = QtWidgets.QVBoxLayout(self)
        layout.addWidget(body)
        layout.addWidget(footer)

        self.data = {
            "Create Button": create_btn,
            "Listing": listing,
            "Use Selection Checkbox": useselection_chk,
            "Subset": name,
            "Subset Menu": subset_menu,
            "Result": result,
            "Asset": asset,
            "Error Message": error_msg,
        }

        for _name, widget in self.data.items():
            widget.setObjectName(_name)

        create_btn.clicked.connect(self.on_create)
        name.returnPressed.connect(self.on_create)
        name.textChanged.connect(self.on_data_changed)
        asset.textChanged.connect(self.on_data_changed)
        listing.currentItemChanged.connect(self.on_selection_changed)
        listing.currentItemChanged.connect(header.set_item)

        self.stateChanged.connect(self._on_state_changed)

        # Defaults
        self.resize(300, 500)
        name.setFocus()
        create_btn.setEnabled(False)

    def _on_state_changed(self, state):
        self.state["valid"] = state
        self.data["Create Button"].setEnabled(state)

    def _build_menu(self, default_names):
        """Create optional predefined subset names

        Args:
            default_names(list): all predefined names

        Returns:
             None
        """

        menu = self.data["Subset Menu"]
        button = menu.parent()

        # Get and destroy the action group
        group = button.findChild(QtWidgets.QActionGroup)
        if group:
            group.deleteLater()

        state = any(default_names)
        button.setEnabled(state)
        if state is False:
            return

        # Build new action group
        group = QtWidgets.QActionGroup(button)
        for name in default_names:
            if name == Separator:
                menu.addSeparator()
                continue
            action = group.addAction(name)
            menu.addAction(action)

        group.triggered.connect(self._on_action_clicked)

    def _on_action_clicked(self, action):
        name = self.data["Subset"]
        name.setText(action.text())

    def _on_data_changed(self):

        listing = self.data["Listing"]
        asset_name = self.data["Asset"]
        subset = self.data["Subset"]
        result = self.data["Result"]

        item = listing.currentItem()
        subset_name = subset.text()
        asset_name = asset_name.text()

        # Early exit if no asset name
        if not asset_name.strip():
            self._build_menu([])
            item.setData(ExistsRole, False)
            self.echo("Asset name is required ..")
            self.stateChanged.emit(False)
            return

        # Get the assets from the database which match with the name
        assets_db = io.find(filter={"type": "asset"}, projection={"name": 1})
        assets = [asset for asset in assets_db if asset_name == asset["name"]]

        if assets:
            # Get plugin and family
            plugin = item.data(PluginRole)
            family = plugin.family.rsplit(".", 1)[-1]

            # Get all subsets of the current asset
            asset_ids = [asset["_id"] for asset in assets]
            subsets = io.find(filter={"type": "subset",
                                      "name": {"$regex": "{}*".format(family),
                                               "$options": "i"},
                                      "parent": {"$in": asset_ids}}) or []

            # Get all subsets' their subset name, "Default", "High", "Low"
            existed_subsets = [sub["name"].split(family)[-1]
                               for sub in subsets]

            if plugin.defaults and isinstance(plugin.defaults, list):
                defaults = plugin.defaults[:] + [Separator]
                lowered = [d.lower() for d in plugin.defaults]
                for sub in [s for s in existed_subsets
                            if s.lower() not in lowered]:
                    defaults.append(sub)
            else:
                defaults = existed_subsets

            self._build_menu(defaults)

            # Update the result
            if subset_name:
                subset_name = subset_name[0].upper() + subset_name[1:]
            result.setText("{}{}".format(family, subset_name))

            item.setData(ExistsRole, True)
            self.echo("Ready ..")
        else:
            self._build_menu([])
            item.setData(ExistsRole, False)
            self.echo("Asset '%s' not found .." % asset_name)

        # Update the valid state
        valid = (
            subset_name.strip() != "" and
            item.data(QtCore.Qt.ItemIsEnabled) and
            item.data(ExistsRole)
        )
        self.stateChanged.emit(valid)

    def on_data_changed(self, *args):

        # Set invalid state until it's reconfirmed to be valid by the
        # scheduled callback so any form of creation is held back until
        # valid again
        self.stateChanged.emit(False)

        lib.schedule(self._on_data_changed, 500, channel="gui")

    def on_selection_changed(self, *args):
        name = self.data["Subset"]
        item = self.data["Listing"].currentItem()

        plugin = item.data(PluginRole)
        if plugin is None:
            return

        if plugin.defaults and isinstance(plugin.defaults, list):
            default = plugin.defaults[0]
        else:
            default = "Default"

        name.setText(default)

        self.on_data_changed()

    def keyPressEvent(self, event):
        """Custom keyPressEvent.

        Override keyPressEvent to do nothing so that Maya's panels won't
        take focus when pressing "SHIFT" whilst mouse is over viewport or
        outliner. This way users don't accidently perform Maya commands
        whilst trying to name an instance.

        """

    def refresh(self):

        listing = self.data["Listing"]
        asset = self.data["Asset"]
        asset.setText(api.Session["AVALON_ASSET"])

        has_families = False

        creators = api.discover(api.Creator)

        for creator in creators:
            label = creator.label or creator.family
            item = QtWidgets.QListWidgetItem(label)
            item.setData(QtCore.Qt.ItemIsEnabled, True)
            item.setData(HelpRole, creator.__doc__)
            item.setData(FamilyRole, creator.family)
            item.setData(PluginRole, creator)
            item.setData(ExistsRole, False)
            listing.addItem(item)

            has_families = True

        if not has_families:
            item = QtWidgets.QListWidgetItem("No registered families")
            item.setData(QtCore.Qt.ItemIsEnabled, False)
            listing.addItem(item)

        listing.setCurrentItem(listing.item(0))

    def on_create(self):

        # Do not allow creation in an invalid state
        if not self.state["valid"]:
            return

        asset = self.data["Asset"]
        listing = self.data["Listing"]
        result = self.data["Result"]

        item = listing.currentItem()
        if item is None:
            return

        subset_name = result.text()
        asset = asset.text()
        family = item.data(FamilyRole)
        use_selection = self.data["Use Selection Checkbox"].isChecked()

        try:
            api.create(subset_name,
                       asset,
                       family,
                       options={"useSelection": use_selection})

        except NameError as e:
            self.echo(e)
            raise

        except (TypeError, RuntimeError, KeyError, AssertionError) as e:
            self.echo("Program error: %s" % str(e))
            raise

        self.echo("Created %s .." % subset_name)

    def echo(self, message):
        widget = self.data["Error Message"]
        widget.setText(str(message))
        widget.show()

        lib.schedule(lambda: widget.setText(""), 5000, channel="message")

class FamilyDescriptionWidget(QtWidgets.QWidget):
    """A family description widget.

    Shows a family icon, family name and a help description.
    Used in creator header.

     _________________
    |  ____           |
    | |icon| FAMILY   |
    | |____| help     |
    |_________________|

    """

    SIZE = 35

    def __init__(self, parent=None):
        super(FamilyDescriptionWidget, self).__init__(parent=parent)

        # Header font
        font = QtGui.QFont()
        font.setBold(True)
        font.setPointSize(14)

        layout = QtWidgets.QHBoxLayout(self)
        layout.setContentsMargins(0, 0, 0, 0)

        icon = QtWidgets.QLabel()
        icon.setSizePolicy(QtWidgets.QSizePolicy.Maximum,
                           QtWidgets.QSizePolicy.Maximum)

        # Add 4 pixel padding to avoid icon being cut off
        icon.setFixedWidth(self.SIZE + 4)
        icon.setFixedHeight(self.SIZE + 4)
        icon.setStyleSheet("""
        QLabel {
            padding-right: 5px;
        }
        """)

        label_layout = QtWidgets.QVBoxLayout()
        label_layout.setSpacing(0)

        family = QtWidgets.QLabel("family")
        family.setFont(font)
        family.setAlignment(QtCore.Qt.AlignBottom | QtCore.Qt.AlignLeft)

        help = QtWidgets.QLabel("help")
        help.setAlignment(QtCore.Qt.AlignTop | QtCore.Qt.AlignLeft)

        label_layout.addWidget(family)
        label_layout.addWidget(help)

        layout.addWidget(icon)
        layout.addLayout(label_layout)

        self.help = help
        self.family = family
        self.icon = icon

    def set_item(self, item):
        """Update elements to display information of a family item.

        Args:
            item (dict): A family item as registered with name, help and icon

        Returns:
            None

        """
        if not item:
            return

        # Support a font-awesome icon
        plugin = item.data(PluginRole)
        icon = getattr(plugin, "icon", "info-circle")
        assert isinstance(icon, six.string_types)
        icon = qtawesome.icon("fa.{}".format(icon), color="white")
        pixmap = icon.pixmap(self.SIZE, self.SIZE)
        pixmap = pixmap.scaled(self.SIZE, self.SIZE)

        # Parse a clean line from the Creator's docstring
        docstring = inspect.getdoc(plugin)
        help = docstring.splitlines()[0] if docstring else ""

        self.icon.setPixmap(pixmap)
        self.family.setText(item.data(FamilyRole))
        self.help.setText(help)

def show(debug=False, parent=None):
    """Display asset creator GUI

    Arguments:
        debug (bool, optional): Run loader in debug-mode,
            defaults to False
        parent (QtCore.QObject, optional): When provided parent the interface
            to this QObject.

    """

    if module.window:
        module.window.close()
        del(module.window)

    if debug:
        from avalon import mock
        for creator in mock.creators:
            api.register_plugin(api.Creator, creator)

        import traceback
        sys.excepthook = lambda typ, val, tb: traceback.print_last()

        io.install()

        any_project = next(
            project for project in io.projects()
            if project.get("active", True) is not False
        )

        api.Session["AVALON_PROJECT"] = any_project["name"]
        module.project = any_project["name"]

    with lib.application():
        window = Window(parent)
        window.setStyleSheet(style.load_stylesheet())
        window.refresh()
        window.show()

        module.window = window
<commit_msg>Simplify database query for finding asset<commit_after>import sys
import inspect

from ...vendor.Qt import QtWidgets, QtCore, QtGui
from ...vendor import qtawesome
from ...vendor import six
from ... import api, io, style

from .. import lib

module = sys.modules[__name__]
module.window = None
module.root = api.registered_root()

HelpRole = QtCore.Qt.UserRole + 2
FamilyRole = QtCore.Qt.UserRole + 3
ExistsRole = QtCore.Qt.UserRole + 4
PluginRole = QtCore.Qt.UserRole + 5

Separator = "---separator---"

class Window(QtWidgets.QDialog):

    stateChanged = QtCore.Signal(bool)

    def __init__(self, parent=None):
        super(Window, self).__init__(parent)
        self.setWindowTitle("Instance Creator")
        self.setFocusPolicy(QtCore.Qt.StrongFocus)

        # Store the widgets for lookup in here
        self.data = dict()

        # Store internal states in here
        self.state = {
            "valid": False
        }

        body = QtWidgets.QWidget()
        lists = QtWidgets.QWidget()
        footer = QtWidgets.QWidget()

        container = QtWidgets.QWidget()

        listing = QtWidgets.QListWidget()
        asset = QtWidgets.QLineEdit()
        name = QtWidgets.QLineEdit()
        result = QtWidgets.QLineEdit()
        result.setStyleSheet("color: gray;")
        result.setEnabled(False)

        # region Menu for default subset names

        subset_button = QtWidgets.QPushButton()
        subset_button.setFixedWidth(18)
        subset_button.setFixedHeight(20)
        subset_menu = QtWidgets.QMenu(subset_button)
        subset_button.setMenu(subset_menu)

        # endregion

        name_layout = QtWidgets.QHBoxLayout()
        name_layout.addWidget(name)
        name_layout.addWidget(subset_button)
        name_layout.setContentsMargins(0, 0, 0, 0)

        layout = QtWidgets.QVBoxLayout(container)

        header = FamilyDescriptionWidget(parent=self)
        layout.addWidget(header)

        layout.addWidget(QtWidgets.QLabel("Family"))
        layout.addWidget(listing)
        layout.addWidget(QtWidgets.QLabel("Asset"))
        layout.addWidget(asset)
        layout.addWidget(QtWidgets.QLabel("Subset"))
        layout.addLayout(name_layout)
        layout.addWidget(result)
        layout.setContentsMargins(0, 0, 0, 0)

        options = QtWidgets.QWidget()

        useselection_chk = QtWidgets.QCheckBox("Use selection")
        useselection_chk.setCheckState(QtCore.Qt.Checked)

        layout = QtWidgets.QGridLayout(options)
        layout.setContentsMargins(0, 0, 0, 0)
        layout.addWidget(useselection_chk, 1, 1)

        layout = QtWidgets.QHBoxLayout(lists)
        layout.addWidget(container)
        layout.setContentsMargins(0, 0, 0, 0)

        layout = QtWidgets.QVBoxLayout(body)
        layout.addWidget(lists)
        layout.addWidget(options, 0, QtCore.Qt.AlignLeft)
        layout.setContentsMargins(0, 0, 0, 0)

        create_btn = QtWidgets.QPushButton("Create")
        error_msg = QtWidgets.QLabel()
        error_msg.setFixedHeight(20)

        layout = QtWidgets.QVBoxLayout(footer)
        layout.addWidget(create_btn)
        layout.addWidget(error_msg)
        layout.setContentsMargins(0, 0, 0, 0)

        layout = QtWidgets.QVBoxLayout(self)
        layout.addWidget(body)
        layout.addWidget(footer)

        self.data = {
            "Create Button": create_btn,
            "Listing": listing,
            "Use Selection Checkbox": useselection_chk,
            "Subset": name,
            "Subset Menu": subset_menu,
            "Result": result,
            "Asset": asset,
            "Error Message": error_msg,
        }

        for _name, widget in self.data.items():
            widget.setObjectName(_name)

        create_btn.clicked.connect(self.on_create)
        name.returnPressed.connect(self.on_create)
        name.textChanged.connect(self.on_data_changed)
        asset.textChanged.connect(self.on_data_changed)
        listing.currentItemChanged.connect(self.on_selection_changed)
        listing.currentItemChanged.connect(header.set_item)

        self.stateChanged.connect(self._on_state_changed)

        # Defaults
        self.resize(300, 500)
        name.setFocus()
        create_btn.setEnabled(False)

    def _on_state_changed(self, state):
        self.state["valid"] = state
        self.data["Create Button"].setEnabled(state)

    def _build_menu(self, default_names):
        """Create optional predefined subset names

        Args:
            default_names(list): all predefined names

        Returns:
             None
        """

        menu = self.data["Subset Menu"]
        button = menu.parent()

        # Get and destroy the action group
        group = button.findChild(QtWidgets.QActionGroup)
        if group:
            group.deleteLater()

        state = any(default_names)
        button.setEnabled(state)
        if state is False:
            return

        # Build new action group
        group = QtWidgets.QActionGroup(button)
        for name in default_names:
            if name == Separator:
                menu.addSeparator()
                continue
            action = group.addAction(name)
            menu.addAction(action)

        group.triggered.connect(self._on_action_clicked)

    def _on_action_clicked(self, action):
        name = self.data["Subset"]
        name.setText(action.text())

    def _on_data_changed(self):

        listing = self.data["Listing"]
        asset_name = self.data["Asset"]
        subset = self.data["Subset"]
        result = self.data["Result"]

        item = listing.currentItem()
        subset_name = subset.text()
        asset_name = asset_name.text()

        # Early exit if no asset name
        if not asset_name.strip():
            self._build_menu([])
            item.setData(ExistsRole, False)
            self.echo("Asset name is required ..")
            self.stateChanged.emit(False)
            return

        # Get the asset from the database which match with the name
        asset = io.find_one({"name": asset_name, "type": "asset"},
                            projection={"_id": 1})

        if asset:
            # Get plugin and family
            plugin = item.data(PluginRole)
            family = plugin.family.rsplit(".", 1)[-1]

            # Get all subsets of the current asset
            subsets = io.find(filter={"type": "subset",
                                      "name": {"$regex": "{}*".format(family),
                                               "$options": "i"},
                                      "parent": asset["_id"]}) or []

            # Get all subsets' their subset name, "Default", "High", "Low"
            existed_subsets = [sub["name"].split(family)[-1]
                               for sub in subsets]

            if plugin.defaults and isinstance(plugin.defaults, list):
                defaults = plugin.defaults[:] + [Separator]
                lowered = [d.lower() for d in plugin.defaults]
                for sub in [s for s in existed_subsets
                            if s.lower() not in lowered]:
                    defaults.append(sub)
            else:
                defaults = existed_subsets

            self._build_menu(defaults)

            # Update the result
            if subset_name:
                subset_name = subset_name[0].upper() + subset_name[1:]
            result.setText("{}{}".format(family, subset_name))

            item.setData(ExistsRole, True)
            self.echo("Ready ..")
        else:
            self._build_menu([])
            item.setData(ExistsRole, False)
            self.echo("Asset '%s' not found .." % asset_name)

        # Update the valid state
        valid = (
            subset_name.strip() != "" and
            item.data(QtCore.Qt.ItemIsEnabled) and
            item.data(ExistsRole)
        )
        self.stateChanged.emit(valid)

    def on_data_changed(self, *args):

        # Set invalid state until it's reconfirmed to be valid by the
        # scheduled callback so any form of creation is held back until
        # valid again
        self.stateChanged.emit(False)

        lib.schedule(self._on_data_changed, 500, channel="gui")

    def on_selection_changed(self, *args):
        name = self.data["Subset"]
        item = self.data["Listing"].currentItem()

        plugin = item.data(PluginRole)
        if plugin is None:
            return

        if plugin.defaults and isinstance(plugin.defaults, list):
            default = plugin.defaults[0]
        else:
            default = "Default"

        name.setText(default)

        self.on_data_changed()

    def keyPressEvent(self, event):
        """Custom keyPressEvent.

        Override keyPressEvent to do nothing so that Maya's panels won't
        take focus when pressing "SHIFT" whilst mouse is over viewport or
        outliner. This way users don't accidently perform Maya commands
        whilst trying to name an instance.

        """

    def refresh(self):

        listing = self.data["Listing"]
        asset = self.data["Asset"]
        asset.setText(api.Session["AVALON_ASSET"])

        has_families = False

        creators = api.discover(api.Creator)

        for creator in creators:
            label = creator.label or creator.family
            item = QtWidgets.QListWidgetItem(label)
            item.setData(QtCore.Qt.ItemIsEnabled, True)
            item.setData(HelpRole, creator.__doc__)
            item.setData(FamilyRole, creator.family)
            item.setData(PluginRole, creator)
            item.setData(ExistsRole, False)
            listing.addItem(item)

            has_families = True

        if not has_families:
            item = QtWidgets.QListWidgetItem("No registered families")
            item.setData(QtCore.Qt.ItemIsEnabled, False)
            listing.addItem(item)

        listing.setCurrentItem(listing.item(0))

    def on_create(self):

        # Do not allow creation in an invalid state
        if not self.state["valid"]:
            return

        asset = self.data["Asset"]
        listing = self.data["Listing"]
        result = self.data["Result"]

        item = listing.currentItem()
        if item is None:
            return

        subset_name = result.text()
        asset = asset.text()
        family = item.data(FamilyRole)
        use_selection = self.data["Use Selection Checkbox"].isChecked()

        try:
            api.create(subset_name,
                       asset,
                       family,
                       options={"useSelection": use_selection})

        except NameError as e:
            self.echo(e)
            raise

        except (TypeError, RuntimeError, KeyError, AssertionError) as e:
            self.echo("Program error: %s" % str(e))
            raise

        self.echo("Created %s .." % subset_name)

    def echo(self, message):
        widget = self.data["Error Message"]
        widget.setText(str(message))
        widget.show()

        lib.schedule(lambda: widget.setText(""), 5000, channel="message")

class FamilyDescriptionWidget(QtWidgets.QWidget):
    """A family description widget.

    Shows a family icon, family name and a help description.
    Used in creator header.

     _________________
    |  ____           |
    | |icon| FAMILY   |
    | |____| help     |
    |_________________|

    """

    SIZE = 35

    def __init__(self, parent=None):
        super(FamilyDescriptionWidget, self).__init__(parent=parent)

        # Header font
        font = QtGui.QFont()
        font.setBold(True)
        font.setPointSize(14)

        layout = QtWidgets.QHBoxLayout(self)
        layout.setContentsMargins(0, 0, 0, 0)

        icon = QtWidgets.QLabel()
        icon.setSizePolicy(QtWidgets.QSizePolicy.Maximum,
                           QtWidgets.QSizePolicy.Maximum)

        # Add 4 pixel padding to avoid icon being cut off
        icon.setFixedWidth(self.SIZE + 4)
        icon.setFixedHeight(self.SIZE + 4)
        icon.setStyleSheet("""
        QLabel {
            padding-right: 5px;
        }
        """)

        label_layout = QtWidgets.QVBoxLayout()
        label_layout.setSpacing(0)

        family = QtWidgets.QLabel("family")
        family.setFont(font)
        family.setAlignment(QtCore.Qt.AlignBottom | QtCore.Qt.AlignLeft)

        help = QtWidgets.QLabel("help")
        help.setAlignment(QtCore.Qt.AlignTop | QtCore.Qt.AlignLeft)

        label_layout.addWidget(family)
        label_layout.addWidget(help)

        layout.addWidget(icon)
        layout.addLayout(label_layout)

        self.help = help
        self.family = family
        self.icon = icon

    def set_item(self, item):
        """Update elements to display information of a family item.

        Args:
            item (dict): A family item as registered with name, help and icon

        Returns:
            None

        """
        if not item:
            return

        # Support a font-awesome icon
        plugin = item.data(PluginRole)
        icon = getattr(plugin, "icon", "info-circle")
        assert isinstance(icon, six.string_types)
        icon = qtawesome.icon("fa.{}".format(icon), color="white")
        pixmap = icon.pixmap(self.SIZE, self.SIZE)
        pixmap = pixmap.scaled(self.SIZE, self.SIZE)

        # Parse a clean line from the Creator's docstring
        docstring = inspect.getdoc(plugin)
        help = docstring.splitlines()[0] if docstring else ""

        self.icon.setPixmap(pixmap)
        self.family.setText(item.data(FamilyRole))
        self.help.setText(help)

def show(debug=False, parent=None):
    """Display asset creator GUI

    Arguments:
        debug (bool, optional): Run loader in debug-mode,
            defaults to False
        parent (QtCore.QObject, optional): When provided parent the interface
            to this QObject.

    """

    if module.window:
        module.window.close()
        del(module.window)

    if debug:
        from avalon import mock
        for creator in mock.creators:
            api.register_plugin(api.Creator, creator)

        import traceback
        sys.excepthook = lambda typ, val, tb: traceback.print_last()

        io.install()

        any_project = next(
            project for project in io.projects()
            if project.get("active", True) is not False
        )

        api.Session["AVALON_PROJECT"] = any_project["name"]
        module.project = any_project["name"]

    with lib.application():
        window = Window(parent)
        window.setStyleSheet(style.load_stylesheet())
        window.refresh()
        window.show()

        module.window = window
<|endoftext|>


<commit_before>import os
import sys

def main():

	return True

if __name__ == '__main__':
	main()
<commit_msg>Iterate over all files.<commit_after>#!/usr/bin/python3

import os
import shutil
import sys

def parse_file(f):

	print(f)

	return True

def main():

	for rootDir, folders, files in os.walk(os.getcwd()):
		for f in files:
			parse_file(f)

	return True

if __name__ == '__main__':
	main()
<|endoftext|>


<commit_before>"""KALangStatistic Module

Main Module to create Khan Academy Language translation report
HTML Reports are generated in the ouputDirectory

For every Language setLanguage and generateStatistic are called in this order.

Project Website
http://www.kadeutsch.org

Definition of Crowdin Files
https://docs.google.com/document/d/12xylj_z3BbjMF57WdXJ2Mjr2DduhD5FiVKgb5GQBfvs/edit
"""
from __future__ import unicode_literals
from django.core.management.base import BaseCommand, CommandError
from django.conf import settings
from django.db.models import Q
from django.template import Context, loader

from LanguageStatistic.models import Video,Subtitle,Language,LanguageStatistic
from LanguageStatistic.utils import ftime

import _CrowdinAnalyzer
import _VideoStatistic

import codecs
import sys
import pickle
import os.path
import time

class Command(BaseCommand):
    """
    Class to generate all Report Files for Language Statistic report
     - Creates Instances of CrowdinAnalyzer and 2 instances of VideoStatistic
     - Uses Django Templates to generate the report Files

    Args:
    outputDir (string): OutputDirectory for Report Files

    """

    args = '<language1 language2 ...>'
    help = 'Generates the Lanugage Progress Report'

    statistics = {}
    lang = {}
    target = 'Test'
    outputDirectory = ""
    langSummary = []
    Targets = ('Test', 'Live', 'Rockstar' )

    def setLanguage(self,lang):
        self.lang = lang
        self.setTargetPlatform(lang.target)
        self.crowdin.setLang(lang)
        self.dubVideo.setLang(lang)
        self.subVideo.setLang(lang)

    def setTargetPlatform(self,target):
        self.target = target

    def writeHTMLFile(self,fileName,html):
        import os
        directory = self.outputDirectory + self.lang.code
        fileName = directory+'/'+fileName
        if not os.path.exists(directory):
            os.makedirs(directory)

        file = codecs.open(fileName, 'wt', encoding='utf-8')
        file.write(html)
        file.close()

    def generateStatistic(self):
        print("Generating Translation Statistic for " + self.lang.ename + " " + self.target + " Site")
        self.crowdin.prepareStatistics(self.target)

        self.dubVideo.prepareStatistics()
        self.subVideo.prepareStatistics()

        self.writeMissingVideoStatistics()
        self.writeDoneVideoStatistics()
        self.writeStatisticFile()

    def writeStatisticFile(self):
        html = ""

        menuItems = []
        menuItems.append(['missingVideos','Missing Videos'])
        menuItems.append(['missingSubtitles','Missing Subtitles'])
        menuItems.append(['activity', 'Activity'])
        menuItems.append(['charts','Charts'])

        context = Context( {
            'title':u"Khan Academy " + self.lang.name + self.target + " Progess Report",
            'base': "http://www.kadeutsch.org/report/static",
            'updated': time.strftime("%d.%m.%Y - %H:%M %Z"),
            'lang': self.lang,
            'menuItems':menuItems,
            'summary': self.printSummaryTable(),
            'crowdin': self.crowdin.StatisticTable,
            'dubVideo': self.dubVideo.StatisticTable,
            'subVideo': self.subVideo.StatisticTable,
            'crowdinActivity': self.crowdin.UserStatisticTable,
            'amaraActivity': self.subVideo.getActivity(),
            'parentPage': self.target+'SiteStatistic.html'
        })
        template = loader.get_template('LanguageStatistic/languageProgress.html')
        self.writeHTMLFile(self.target+'SiteStatistic.html',template.render(context));

    def writeMissingVideoStatistics(self):
        html = ""
        html += self.dubVideo.generateMissingVideoTable(False)
        html += self.subVideo.generateMissingVideoTable(False)

        self.generateVideoTable(html,'SiteMissingVideos.html')

    def writeDoneVideoStatistics(self):

        html = ""
        html += self.dubVideo.generateCompletedVideoTable(True)
        html += self.subVideo.generateCompletedVideoTable(True)

        self.generateVideoTable(html,'SiteDoneVideos.html')

    def generateVideoTable(self,html,fileName):
        menuItems = []
        #only call generateMenu on one VideoObject
        self.dubVideo.generateMenu(menuItems)

        context = Context( {
            'title':"Completed Videos for '+self.lang.name+ ' ' +self.target+ ' site of Khanacademy",
            'base': "http://www.kadeutsch.org/report/static",
            'updated': time.strftime("%d.%m.%Y - %H:%M %Z"),
            'lang': self.lang,
            'menuItems':menuItems,
            'html': html,
            'parentPage': self.target+'SiteStatistic.html'
        })
        template = loader.get_template('LanguageStatistic/siteVideos.html')
        self.writeHTMLFile(self.target+fileName,template.render(context));

    def printSummaryTable(self):
        totalAmount = self.crowdin.TotalStrings + self.dubVideo.TotalStrings + self.subVideo.TotalStrings
        totalLeft = self.crowdin.Left + self.dubVideo.LeftString + self.subVideo.LeftString

        progress =  ( self.crowdin.Percent + self.subVideo.Percent + self.dubVideo.Percent ) / 3

        print( "Language {0} is at {1:0.2f}% for {2}".format(self.lang.name,progress * 100,self.lang.target ))

        #self.createDataFiles()

        output = u'
Status Khan Academy '+self.lang.name+ " " + self.target + ' platform
'

        output += u'
'
        output += u'
{0:0.2f}%<progress value="{0:0.2f}" max="100"></progress>
'.format(progress*100)
        output += u"
	Completed	# left	Total
"
        output += u"
Crowdin Strings	{0:0.2f}%	{1}	{2}
".format(self.crowdin.Percent*100,self.crowdin.Left,self.crowdin.TotalStrings)
        output += u"
Video Subtitles{0:0.2f}%	{1}	{3}
".format(self.subVideo.Percent*100, ftime(self.subVideo.Left),self.subVideo.LeftString,self.subVideo.Total)
        output += u"
Video Synchronisation{0:0.2f}%	{1}	{3}
".format(self.dubVideo.Percent*100, ftime(self.dubVideo.Left),self.dubVideo.LeftString,self.dubVideo.Total)
        output += u"
Total	{0:0.2f}%			".format(progress*100,'',totalLeft,totalAmount)
        output += u"
Aktualisiert " + time.strftime("Updated %d.%m.%Y - %H:%M %Z") +self.crowdin.velocityString

        sum = {}
        sum['target']       = self.target
        sum['language']     = self.lang.name
        sum['code']         = self.lang.code
        sum['target']         = self.lang.target
        sum['hours']        = self.dubVideo.Left
        sum['subtitles']    = self.subVideo.Left
        sum['left']         = self.crowdin.Left #totalLeft
        sum['progress'] = progress
        sum['speed'] = self.crowdin.speed
        sum['eta'] = self.crowdin.eta
        self.langSummary.append(sum)
        return output

    #Main Entry Point
    def handle(self, *args, **options):
        self.outputDirectory = settings.OUTPUTDIRECTORY

        #videoListTemplate = loader.get_template('LanguageStatistic/videoList.html')
        self.crowdin = _CrowdinAnalyzer.CrowdinAnalyzer()
        self.dubVideo = _VideoStatistic.VideoStatistic(True)
        self.subVideo = _VideoStatistic.VideoStatistic(False)

        #load the languages specified
        languages = Language.objects.filter(enabled=True)
        for lang in args:
            languages = languages.filter(ename=lang)

        for lang in languages:
            self.setLanguage(lang)
            self.generateStatistic()

<commit_msg>fix html title on report<commit_after>"""KALangStatistic Module

Main Module to create Khan Academy Language translation report
HTML Reports are generated in the ouputDirectory

For every Language setLanguage and generateStatistic are called in this order.

Project Website
http://www.kadeutsch.org

Definition of Crowdin Files
https://docs.google.com/document/d/12xylj_z3BbjMF57WdXJ2Mjr2DduhD5FiVKgb5GQBfvs/edit
"""
from __future__ import unicode_literals
from django.core.management.base import BaseCommand, CommandError
from django.conf import settings
from django.db.models import Q
from django.template import Context, loader

from LanguageStatistic.models import Video,Subtitle,Language,LanguageStatistic
from LanguageStatistic.utils import ftime

import _CrowdinAnalyzer
import _VideoStatistic

import codecs
import sys
import pickle
import os.path
import time

class Command(BaseCommand):
    """
    Class to generate all Report Files for Language Statistic report
     - Creates Instances of CrowdinAnalyzer and 2 instances of VideoStatistic
     - Uses Django Templates to generate the report Files

    Args:
    outputDir (string): OutputDirectory for Report Files

    """

    args = '<language1 language2 ...>'
    help = 'Generates the Lanugage Progress Report'

    statistics = {}
    lang = {}
    target = 'Test'
    outputDirectory = ""
    langSummary = []
    Targets = ('Test', 'Live', 'Rockstar' )

    def setLanguage(self,lang):
        self.lang = lang
        self.setTargetPlatform(lang.target)
        self.crowdin.setLang(lang)
        self.dubVideo.setLang(lang)
        self.subVideo.setLang(lang)

    def setTargetPlatform(self,target):
        self.target = target

    def writeHTMLFile(self,fileName,html):
        import os
        directory = self.outputDirectory + self.lang.code
        fileName = directory+'/'+fileName
        if not os.path.exists(directory):
            os.makedirs(directory)

        file = codecs.open(fileName, 'wt', encoding='utf-8')
        file.write(html)
        file.close()

    def generateStatistic(self):
        print("Generating Translation Statistic for " + self.lang.ename + " " + self.target + " Site")
        self.crowdin.prepareStatistics(self.target)

        self.dubVideo.prepareStatistics()
        self.subVideo.prepareStatistics()

        self.writeMissingVideoStatistics()
        self.writeDoneVideoStatistics()
        self.writeStatisticFile()

    def writeStatisticFile(self):
        html = ""

        menuItems = []
        menuItems.append(['missingVideos','Missing Videos'])
        menuItems.append(['missingSubtitles','Missing Subtitles'])
        menuItems.append(['activity', 'Activity'])
        menuItems.append(['charts','Charts'])

        context = Context( {
            'title':u"Khan Academy " + self.lang.name + self.target + " Progess Report",
            'base': "http://www.kadeutsch.org/report/static",
            'updated': time.strftime("%d.%m.%Y - %H:%M %Z"),
            'lang': self.lang,
            'menuItems':menuItems,
            'summary': self.printSummaryTable(),
            'crowdin': self.crowdin.StatisticTable,
            'dubVideo': self.dubVideo.StatisticTable,
            'subVideo': self.subVideo.StatisticTable,
            'crowdinActivity': self.crowdin.UserStatisticTable,
            'amaraActivity': self.subVideo.getActivity(),
            'parentPage': self.target+'SiteStatistic.html'
        })
        template = loader.get_template('LanguageStatistic/languageProgress.html')
        self.writeHTMLFile(self.target+'SiteStatistic.html',template.render(context));

    def writeMissingVideoStatistics(self):
        html = ""
        html += self.dubVideo.generateMissingVideoTable(False)
        html += self.subVideo.generateMissingVideoTable(False)

        self.generateVideoTable(html,'SiteMissingVideos.html')

    def writeDoneVideoStatistics(self):

        html = ""
        html += self.dubVideo.generateCompletedVideoTable(True)
        html += self.subVideo.generateCompletedVideoTable(True)

        self.generateVideoTable(html,'SiteDoneVideos.html')

    def generateVideoTable(self,html,fileName):
        menuItems = []
        #only call generateMenu on one VideoObject
        self.dubVideo.generateMenu(menuItems)

        context = Context( {
            'title':"Completed Videos for "+self.lang.name+ ' ' +self.target+ " site of Khanacademy",
            'base': "http://www.kadeutsch.org/report/static",
            'updated': time.strftime("%d.%m.%Y - %H:%M %Z"),
            'lang': self.lang,
            'menuItems':menuItems,
            'html': html,
            'parentPage': self.target+'SiteStatistic.html'
        })
        template = loader.get_template('LanguageStatistic/siteVideos.html')
        self.writeHTMLFile(self.target+fileName,template.render(context));

    def printSummaryTable(self):
        totalAmount = self.crowdin.TotalStrings + self.dubVideo.TotalStrings + self.subVideo.TotalStrings
        totalLeft = self.crowdin.Left + self.dubVideo.LeftString + self.subVideo.LeftString

        progress =  ( self.crowdin.Percent + self.subVideo.Percent + self.dubVideo.Percent ) / 3

        print( "Language {0} is at {1:0.2f}% for {2}".format(self.lang.name,progress * 100,self.lang.target ))

        #self.createDataFiles()

        output = u'
Status Khan Academy '+self.lang.name+ " " + self.target + ' platform
'

        output += u'
'
        output += u'
{0:0.2f}%<progress value="{0:0.2f}" max="100"></progress>
'.format(progress*100)
        output += u"
	Completed	# left	Total
"
        output += u"
Crowdin Strings	{0:0.2f}%	{1}	{2}
".format(self.crowdin.Percent*100,self.crowdin.Left,self.crowdin.TotalStrings)
        output += u"
Video Subtitles{0:0.2f}%	{1}	{3}
".format(self.subVideo.Percent*100, ftime(self.subVideo.Left),self.subVideo.LeftString,self.subVideo.Total)
        output += u"
Video Synchronisation{0:0.2f}%	{1}	{3}
".format(self.dubVideo.Percent*100, ftime(self.dubVideo.Left),self.dubVideo.LeftString,self.dubVideo.Total)
        output += u"
Total	{0:0.2f}%			".format(progress*100,'',totalLeft,totalAmount)
        output += u"
Aktualisiert " + time.strftime("Updated %d.%m.%Y - %H:%M %Z") +self.crowdin.velocityString

        sum = {}
        sum['target']       = self.target
        sum['language']     = self.lang.name
        sum['code']         = self.lang.code
        sum['target']         = self.lang.target
        sum['hours']        = self.dubVideo.Left
        sum['subtitles']    = self.subVideo.Left
        sum['left']         = self.crowdin.Left #totalLeft
        sum['progress'] = progress
        sum['speed'] = self.crowdin.speed
        sum['eta'] = self.crowdin.eta
        self.langSummary.append(sum)
        return output

    #Main Entry Point
    def handle(self, *args, **options):
        self.outputDirectory = settings.OUTPUTDIRECTORY

        #videoListTemplate = loader.get_template('LanguageStatistic/videoList.html')
        self.crowdin = _CrowdinAnalyzer.CrowdinAnalyzer()
        self.dubVideo = _VideoStatistic.VideoStatistic(True)
        self.subVideo = _VideoStatistic.VideoStatistic(False)

        #load the languages specified
        languages = Language.objects.filter(enabled=True)
        for lang in args:
            languages = languages.filter(ename=lang)

        for lang in languages:
            self.setLanguage(lang)
            self.generateStatistic()

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Defines unit tests for :mod:`colour.models.rgb.transfer_functions.filmic_pro`
module.
"""

from __future__ import division, unicode_literals

import numpy as np
import unittest

from colour.models.rgb.transfer_functions import (log_encoding_FilmicPro6,
                                                  log_decoding_FilmicPro6)
from colour.utilities import domain_range_scale, ignore_numpy_errors

__author__ = 'Colour Developers'
__copyright__ = 'Copyright (C) 2013-2018 - Colour Developers'
__license__ = 'New BSD License - http://opensource.org/licenses/BSD-3-Clause'
__maintainer__ = 'Colour Developers'
__email__ = 'colour-science@googlegroups.com'
__status__ = 'Production'

__all__ = ['TestLogEncoding_FilmicPro6', 'TestLogDecoding_FilmicPro6']

class TestLogEncoding_FilmicPro6(unittest.TestCase):
    """
    Defines :func:`colour.models.rgb.transfer_functions.filmic_pro.\
log_encoding_FilmicPro6` definition unit tests methods.
    """

    def test_log_encoding_FilmicPro6(self):
        """
        Tests :func:`colour.models.rgb.transfer_functions.filmic_pro.\
log_encoding_FilmicPro6` definition.
        """

        self.assertAlmostEqual(log_encoding_FilmicPro6(0.0), -np.inf, places=7)

        self.assertAlmostEqual(
            log_encoding_FilmicPro6(0.18), 0.606634519924703, places=7)

        self.assertAlmostEqual(
            log_encoding_FilmicPro6(1.0), 1.000000819999999, places=7)

    def test_n_dimensional_log_encoding_FilmicPro6(self):
        """
        Tests :func:`colour.models.rgb.transfer_functions.filmic_pro.\
log_encoding_FilmicPro6` definition n-dimensional arrays support.
        """

        x = 0.18
        y = 0.606634519924703
        np.testing.assert_almost_equal(
            log_encoding_FilmicPro6(x), y, decimal=7)

        x = np.tile(x, 6)
        y = np.tile(y, 6)
        np.testing.assert_almost_equal(
            log_encoding_FilmicPro6(x), y, decimal=7)

        x = np.reshape(x, (2, 3))
        y = np.reshape(y, (2, 3))
        np.testing.assert_almost_equal(
            log_encoding_FilmicPro6(x), y, decimal=7)

        x = np.reshape(x, (2, 3, 1))
        y = np.reshape(y, (2, 3, 1))
        np.testing.assert_almost_equal(
            log_encoding_FilmicPro6(x), y, decimal=7)

    def test_domain_range_scale_log_encoding_FilmicPro6(self):
        """
        Tests :func:`colour.models.rgb.transfer_functions.filmic_pro.\
log_encoding_FilmicPro6` definition domain and range scale support.
        """

        x = 0.18
        y = log_encoding_FilmicPro6(x)

        d_r = (('reference', 1), (1, 1), (100, 100))
        for scale, factor in d_r:
            with domain_range_scale(scale):
                np.testing.assert_almost_equal(
                    log_encoding_FilmicPro6(x * factor), y * factor, decimal=7)

    @ignore_numpy_errors
    def test_nan_log_encoding_FilmicPro6(self):
        """
        Tests :func:`colour.models.rgb.transfer_functions.filmic_pro.\
log_encoding_FilmicPro6` definition nan support.
        """

        log_encoding_FilmicPro6(
            np.array([-1.0, 0.0, 1.0, -np.inf, np.inf, np.nan]))

class TestLogDecoding_FilmicPro6(unittest.TestCase):
    """
    Defines :func:`colour.models.rgb.transfer_functions.filmic_pro.\
log_decoding_FilmicPro6` definition unit tests methods.
    """

    def test_log_decoding_FilmicPro6(self):
        """
        Tests :func:`colour.models.rgb.transfer_functions.filmic_pro.\
log_decoding_FilmicPro6` definition.
        """

        np.testing.assert_array_equal(log_decoding_FilmicPro6(-np.inf), np.nan)

        self.assertAlmostEqual(
            log_decoding_FilmicPro6(0.606634519924703), 0.18, places=7)

        self.assertAlmostEqual(
            log_decoding_FilmicPro6(1.000000819999999), 1.0, places=7)

    def test_n_dimensional_log_decoding_FilmicPro6(self):
        """
        Tests :func:`colour.models.rgb.transfer_functions.filmic_pro.\
log_decoding_FilmicPro6` definition n-dimensional arrays support.
        """

        y = 0.606634519924703
        x = 0.18
        np.testing.assert_almost_equal(
            log_decoding_FilmicPro6(y), x, decimal=7)

        y = np.tile(y, 6)
        x = np.tile(x, 6)
        np.testing.assert_almost_equal(
            log_decoding_FilmicPro6(y), x, decimal=7)

        y = np.reshape(y, (2, 3))
        x = np.reshape(x, (2, 3))
        np.testing.assert_almost_equal(
            log_decoding_FilmicPro6(y), x, decimal=7)

        y = np.reshape(y, (2, 3, 1))
        x = np.reshape(x, (2, 3, 1))
        np.testing.assert_almost_equal(
            log_decoding_FilmicPro6(y), x, decimal=7)

    def test_domain_range_scale_log_decoding_FilmicPro6(self):
        """
        Tests :func:`colour.models.rgb.transfer_functions.filmic_pro.\
log_decoding_FilmicPro6` definition domain and range scale support.
        """

        y = 0.606634519924703
        x = log_decoding_FilmicPro6(y)

        d_r = (('reference', 1), (1, 1), (100, 100))
        for scale, factor in d_r:
            with domain_range_scale(scale):
                np.testing.assert_almost_equal(
                    log_decoding_FilmicPro6(y * factor), x * factor, decimal=7)

    @ignore_numpy_errors
    def test_nan_log_decoding_FilmicPro6(self):
        """
        Tests :func:`colour.models.rgb.transfer_functions.filmic_pro.\
log_decoding_FilmicPro6` definition nan support.
        """

        log_decoding_FilmicPro6(
            np.array([-1.0, 0.0, 1.0, -np.inf, np.inf, np.nan]))

if __name__ == '__main__':
    unittest.main()
<commit_msg>Fix broken unit tests in "colour.models.rgb.transfer_functions.tests.test_filmic_pro.TestLogDecoding_FilmicPro6.test_log_decoding_FilmicPro6" method.<commit_after># -*- coding: utf-8 -*-
"""
Defines unit tests for :mod:`colour.models.rgb.transfer_functions.filmic_pro`
module.
"""

from __future__ import division, unicode_literals

import numpy as np
import unittest

from colour.models.rgb.transfer_functions import (log_encoding_FilmicPro6,
                                                  log_decoding_FilmicPro6)
from colour.utilities import domain_range_scale, ignore_numpy_errors

__author__ = 'Colour Developers'
__copyright__ = 'Copyright (C) 2013-2018 - Colour Developers'
__license__ = 'New BSD License - http://opensource.org/licenses/BSD-3-Clause'
__maintainer__ = 'Colour Developers'
__email__ = 'colour-science@googlegroups.com'
__status__ = 'Production'

__all__ = ['TestLogEncoding_FilmicPro6', 'TestLogDecoding_FilmicPro6']

class TestLogEncoding_FilmicPro6(unittest.TestCase):
    """
    Defines :func:`colour.models.rgb.transfer_functions.filmic_pro.\
log_encoding_FilmicPro6` definition unit tests methods.
    """

    def test_log_encoding_FilmicPro6(self):
        """
        Tests :func:`colour.models.rgb.transfer_functions.filmic_pro.\
log_encoding_FilmicPro6` definition.
        """

        self.assertAlmostEqual(log_encoding_FilmicPro6(0.0), -np.inf, places=7)

        self.assertAlmostEqual(
            log_encoding_FilmicPro6(0.18), 0.606634519924703, places=7)

        self.assertAlmostEqual(
            log_encoding_FilmicPro6(1.0), 1.000000819999999, places=7)

    def test_n_dimensional_log_encoding_FilmicPro6(self):
        """
        Tests :func:`colour.models.rgb.transfer_functions.filmic_pro.\
log_encoding_FilmicPro6` definition n-dimensional arrays support.
        """

        x = 0.18
        y = 0.606634519924703
        np.testing.assert_almost_equal(
            log_encoding_FilmicPro6(x), y, decimal=7)

        x = np.tile(x, 6)
        y = np.tile(y, 6)
        np.testing.assert_almost_equal(
            log_encoding_FilmicPro6(x), y, decimal=7)

        x = np.reshape(x, (2, 3))
        y = np.reshape(y, (2, 3))
        np.testing.assert_almost_equal(
            log_encoding_FilmicPro6(x), y, decimal=7)

        x = np.reshape(x, (2, 3, 1))
        y = np.reshape(y, (2, 3, 1))
        np.testing.assert_almost_equal(
            log_encoding_FilmicPro6(x), y, decimal=7)

    def test_domain_range_scale_log_encoding_FilmicPro6(self):
        """
        Tests :func:`colour.models.rgb.transfer_functions.filmic_pro.\
log_encoding_FilmicPro6` definition domain and range scale support.
        """

        x = 0.18
        y = log_encoding_FilmicPro6(x)

        d_r = (('reference', 1), (1, 1), (100, 100))
        for scale, factor in d_r:
            with domain_range_scale(scale):
                np.testing.assert_almost_equal(
                    log_encoding_FilmicPro6(x * factor), y * factor, decimal=7)

    @ignore_numpy_errors
    def test_nan_log_encoding_FilmicPro6(self):
        """
        Tests :func:`colour.models.rgb.transfer_functions.filmic_pro.\
log_encoding_FilmicPro6` definition nan support.
        """

        log_encoding_FilmicPro6(
            np.array([-1.0, 0.0, 1.0, -np.inf, np.inf, np.nan]))

class TestLogDecoding_FilmicPro6(unittest.TestCase):
    """
    Defines :func:`colour.models.rgb.transfer_functions.filmic_pro.\
log_decoding_FilmicPro6` definition unit tests methods.
    """

    def test_log_decoding_FilmicPro6(self):
        """
        Tests :func:`colour.models.rgb.transfer_functions.filmic_pro.\
log_decoding_FilmicPro6` definition.
        """

        np.testing.assert_array_equal(log_decoding_FilmicPro6(-np.inf), 0.0)

        self.assertAlmostEqual(
            log_decoding_FilmicPro6(0.606634519924703), 0.18, places=7)

        self.assertAlmostEqual(
            log_decoding_FilmicPro6(1.000000819999999), 1.0, places=7)

    def test_n_dimensional_log_decoding_FilmicPro6(self):
        """
        Tests :func:`colour.models.rgb.transfer_functions.filmic_pro.\
log_decoding_FilmicPro6` definition n-dimensional arrays support.
        """

        y = 0.606634519924703
        x = 0.18
        np.testing.assert_almost_equal(
            log_decoding_FilmicPro6(y), x, decimal=7)

        y = np.tile(y, 6)
        x = np.tile(x, 6)
        np.testing.assert_almost_equal(
            log_decoding_FilmicPro6(y), x, decimal=7)

        y = np.reshape(y, (2, 3))
        x = np.reshape(x, (2, 3))
        np.testing.assert_almost_equal(
            log_decoding_FilmicPro6(y), x, decimal=7)

        y = np.reshape(y, (2, 3, 1))
        x = np.reshape(x, (2, 3, 1))
        np.testing.assert_almost_equal(
            log_decoding_FilmicPro6(y), x, decimal=7)

    def test_domain_range_scale_log_decoding_FilmicPro6(self):
        """
        Tests :func:`colour.models.rgb.transfer_functions.filmic_pro.\
log_decoding_FilmicPro6` definition domain and range scale support.
        """

        y = 0.606634519924703
        x = log_decoding_FilmicPro6(y)

        d_r = (('reference', 1), (1, 1), (100, 100))
        for scale, factor in d_r:
            with domain_range_scale(scale):
                np.testing.assert_almost_equal(
                    log_decoding_FilmicPro6(y * factor), x * factor, decimal=7)

    @ignore_numpy_errors
    def test_nan_log_decoding_FilmicPro6(self):
        """
        Tests :func:`colour.models.rgb.transfer_functions.filmic_pro.\
log_decoding_FilmicPro6` definition nan support.
        """

        log_decoding_FilmicPro6(
            np.array([-1.0, 0.0, 1.0, -np.inf, np.inf, np.nan]))

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before># converts vim documentation to html

import re, urllib
from itertools import chain

HEAD = """\
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
    "http://www.w3.org/TR/html4/loose.dtd">

<meta http-equiv="Content-type" content="text/html; charset={encoding}"/>
<title>Vim: {filename}</title>
<link rel="shortcut icon" href="favicon.ico">

<link rel="stylesheet" href="vimhelp.css" type="text/css">

"""

SEARCH_SCRIPT = """

"""

HEAD_END = ''

INTRO = """

Vim help files

This is an HTML version of the Vim help pages{vers-note}. They are kept up-to-date automatically from the Vim source repository. Also included is the Vim FAQ, kept up to date from its github
repository.

"""

VERSION_NOTE = ", current as of Vim {version}"

SITENAVI_LINKS = """
Quick links:
help overview ·
quick reference ·
user manual toc ·
reference manual toc ·
faq
"""

SITENAVI_PLAIN = '
' + SITENAVI_LINKS + '
'

SITENAVI_SEARCH = '

' + SITENAVI_LINKS + \
'

' \
'

'

TEXTSTART = """

"""

FOOTER = '
'

FOOTER2 = """

This site is maintained by Carlo Teubner ((my first name) dot (my last name) at gmail dot com).

"""

VIM_FAQ_LINE = '' \
	       'vim_faq.txt   Frequently Asked Questions\n'

RE_TAGLINE = re.compile(r'(\S+)\s+(\S+)')

PAT_WORDCHAR = '[!#-)+-{}~\xC0-\xFF]'

PAT_HEADER   = r'(^.*~$)'
PAT_GRAPHIC  = r'(^.* `$)'
PAT_PIPEWORD = r'(?<!\\)\|([#-)!+-~]+)\|'
PAT_STARWORD = r'\*([#-)!+-~]+)\*(?:(?=\s)|$)'
PAT_OPTWORD  = r"('(?:[a-z]{2,}|t_..)')"
PAT_CTRL     = r'(CTRL-(?:W_)?(?:[\w\[\]^+-<>=@]|<[A-Za-z]+?>)?)'
PAT_SPECIAL  = r'(<.*?>|\{.*?}|' \
	       r'\[(?:range|line|count|offset|\+?cmd|[-+]?num|\+\+opt|' \
	       r'arg|arguments|ident|addr|group)]|' \
	       r'(?<=\s)\[[-a-z^A-Z0-9_]{2,}])'
PAT_TITLE    = r'(Vim version [0-9.a-z]+|VIM REFERENCE.*)'
PAT_NOTE     = r'((?<!' + PAT_WORDCHAR + r')(?:note|NOTE|Notes?):?' \
                 r'(?!' + PAT_WORDCHAR + r'))'
PAT_URL      = r'((?:https?|ftp)://[^\'"<> \t]+[a-zA-Z0-9/])'
PAT_WORD     = r'((?<!' + PAT_WORDCHAR + r')' + PAT_WORDCHAR + r'+' \
                 r'(?!' + PAT_WORDCHAR + r'))'

RE_LINKWORD = re.compile(
	PAT_OPTWORD  + '|' +
	PAT_CTRL     + '|' +
	PAT_SPECIAL)
RE_TAGWORD = re.compile(
	PAT_HEADER   + '|' +
        PAT_GRAPHIC  + '|' +
	PAT_PIPEWORD + '|' +
	PAT_STARWORD + '|' +
	PAT_OPTWORD  + '|' +
	PAT_CTRL     + '|' +
	PAT_SPECIAL  + '|' +
	PAT_TITLE    + '|' +
	PAT_NOTE     + '|' +
	PAT_URL      + '|' +
	PAT_WORD)
RE_NEWLINE   = re.compile(r'[\r\n]')
RE_HRULE     = re.compile(r'[-=]{3,}.*[-=]{3,3}$')
RE_EG_START  = re.compile(r'(?:.* )?>$')
RE_EG_END    = re.compile(r'\S')
RE_SECTION   = re.compile(r'[-A-Z .][-A-Z0-9 .()]*(?=\s+\*)')
RE_STARTAG   = re.compile(r'\s\*([^ \t|]+)\*(?:\s|$)')
RE_LOCAL_ADD = re.compile(r'LOCAL ADDITIONS:\s+\*local-additions\*$')

class Link(object):
    __slots__ = 'link_pipe', 'link_plain'

    def __init__(self, link_pipe, link_plain):
	self.link_pipe = link_pipe
	self.link_plain = link_plain

class VimH2H(object):
    def __init__(self, tags, version=None):
        self._urls = { }
        self._version = version
	for line in RE_NEWLINE.split(tags):
	    m = RE_TAGLINE.match(line)
	    if m:
		tag, filename = m.group(1, 2)
		self.do_add_tag(filename, tag)

    def add_tags(self, filename, contents):
	for match in RE_STARTAG.finditer(contents):
	    tag = match.group(1).replace('\\', '\\\\').replace('/', '\\/')
	    self.do_add_tag(str(filename), tag)

    def do_add_tag(self, filename, tag):
	part1 = '' + html_escape[tag] + ''
	link_pipe = part1 + ' class="l"' + part2
	classattr = ' class="d"'
	m = RE_LINKWORD.match(tag)
	if m:
	    opt, ctrl, special = m.groups()
	    if opt is not None: classattr = ' class="o"'
	    elif ctrl is not None: classattr = ' class="k"'
	    elif special is not None: classattr = ' class="s"'
	link_plain = part1 + classattr + part2
	self._urls[tag] = Link(link_pipe, link_plain)

    def maplink(self, tag, css_class=None):
	links = self._urls.get(tag)
	if links is not None:
	    if css_class == 'l': return links.link_pipe
	    else: return links.link_plain
	elif css_class is not None:
	    return '' + html_escape[tag] + \
		    ''
	else: return html_escape[tag]

    def to_html(self, filename, contents, encoding, web_version=True):
	out = [ ]

	inexample = 0
        filename = str(filename)
        is_help_txt = (filename == 'help.txt')
	faq_line = False
	for line in RE_NEWLINE.split(contents):
	    line = line.rstrip('\r\n')
	    line_tabs = line
	    line = line.expandtabs()
	    if RE_HRULE.match(line):
		out.extend(('', line, '\n'))
		continue
	    if inexample == 2:
		if RE_EG_END.match(line):
		    inexample = 0
		    if line[0] == '<': line = line[1:]
		else:
                    out.extend(('', html_escape[line],
                               '\n'))
		    continue
	    if RE_EG_START.match(line_tabs):
		inexample = 1
		line = line[0:-1]
	    if RE_SECTION.match(line_tabs):
		m = RE_SECTION.match(line)
                out.extend((r'"', m.group(0), r''))
		line = line[m.end():]
	    if is_help_txt and RE_LOCAL_ADD.match(line_tabs):
		faq_line = True
	    lastpos = 0
	    for match in RE_TAGWORD.finditer(line):
		pos = match.start()
		if pos > lastpos:
		    out.append(html_escape[line[lastpos:pos]])
		lastpos = match.end()
                header, graphic, pipeword, starword, opt, ctrl, special, \
                        title, note, url, word = match.groups()
		if pipeword is not None:
		    out.append(self.maplink(pipeword, 'l'))
		elif starword is not None:
		    out.extend(('', html_escape[starword], ''))
		elif opt is not None:
		    out.append(self.maplink(opt, 'o'))
		elif ctrl is not None:
		    out.append(self.maplink(ctrl, 'k'))
		elif special is not None:
		    out.append(self.maplink(special, 's'))
		elif title is not None:
                    out.extend(('', html_escape[title],
                                ''))
		elif note is not None:
                    out.extend(('', html_escape[note],
                                ''))
		elif header is not None:
                    out.extend(('', html_escape[header[:-1]],
                                ''))
                elif graphic is not None:
                    out.append(html_escape[graphic[:-2]])
		elif url is not None:
                    out.extend(('' +
                                html_escape[url], ''))
		elif word is not None:
		    out.append(self.maplink(word))
	    if lastpos < len(line):
		out.append(html_escape[line[lastpos:]])
	    out.append('\n')
	    if inexample == 1: inexample = 2
	    if faq_line:
		out.append(VIM_FAQ_LINE)
		faq_line = False

        header = []
        header.append(HEAD.format(encoding=encoding, filename=filename))
        if web_version:
            header.append(SEARCH_SCRIPT)
        header.append(HEAD_END)
        if web_version and is_help_txt:
            vers_note = VERSION_NOTE.replace('{version}', self._version) \
                    if self._version else ''
            header.append(INTRO.replace('{vers-note}', vers_note))
        if web_version:
            header.append(SITENAVI_SEARCH)
        else:
            header.append(SITENAVI_PLAIN)
        header.append(TEXTSTART)
        return ''.join(chain(header, out, (FOOTER, SITENAVI_PLAIN, FOOTER2)))

class HtmlEscCache(dict):
    def __missing__(self, key):
        r = key.replace('&', '&') \
               .replace('<', '<') \
               .replace('>', '>')
        self[key] = r
        return r

html_escape = HtmlEscCache()
<commit_msg>vimh2h.py: backtick highlighting<commit_after># converts vim documentation to html

import re, urllib
from itertools import chain

HEAD = """\
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
    "http://www.w3.org/TR/html4/loose.dtd">

<meta http-equiv="Content-type" content="text/html; charset={encoding}"/>
<title>Vim: {filename}</title>
<link rel="shortcut icon" href="favicon.ico">

<link rel="stylesheet" href="vimhelp.css" type="text/css">

"""

SEARCH_SCRIPT = """

"""

HEAD_END = ''

INTRO = """

Vim help files

This is an HTML version of the Vim help pages{vers-note}. They are kept up-to-date automatically from the Vim source repository. Also included is the Vim FAQ, kept up to date from its github
repository.

"""

VERSION_NOTE = ", current as of Vim {version}"

SITENAVI_LINKS = """
Quick links:
help overview ·
quick reference ·
user manual toc ·
reference manual toc ·
faq
"""

SITENAVI_PLAIN = '
' + SITENAVI_LINKS + '
'

SITENAVI_SEARCH = '

' + SITENAVI_LINKS + \
'

' \
'

'

TEXTSTART = """

"""

FOOTER = '
'

FOOTER2 = """

This site is maintained by Carlo Teubner ((my first name) dot (my last name) at gmail dot com).

"""

VIM_FAQ_LINE = '' \
	       'vim_faq.txt   Frequently Asked Questions\n'

RE_TAGLINE = re.compile(r'(\S+)\s+(\S+)')

PAT_WORDCHAR = '[!#-)+-{}~\xC0-\xFF]'

PAT_HEADER   = r'(^.*~$)'
PAT_GRAPHIC  = r'(^.* `$)'
PAT_PIPEWORD = r'(?<!\\)\|([#-)!+-~]+)\|'
PAT_STARWORD = r'\*([#-)!+-~]+)\*(?:(?=\s)|$)'
PAT_COMMAND  = r'`([^` ]+)`'
PAT_OPTWORD  = r"('(?:[a-z]{2,}|t_..)')"
PAT_CTRL     = r'(CTRL-(?:W_)?(?:[\w\[\]^+-<>=@]|<[A-Za-z]+?>)?)'
PAT_SPECIAL  = r'(<.*?>|\{.*?}|' \
	       r'\[(?:range|line|count|offset|\+?cmd|[-+]?num|\+\+opt|' \
	       r'arg|arguments|ident|addr|group)]|' \
	       r'(?<=\s)\[[-a-z^A-Z0-9_]{2,}])'
PAT_TITLE    = r'(Vim version [0-9.a-z]+|VIM REFERENCE.*)'
PAT_NOTE     = r'((?<!' + PAT_WORDCHAR + r')(?:note|NOTE|Notes?):?' \
                 r'(?!' + PAT_WORDCHAR + r'))'
PAT_URL      = r'((?:https?|ftp)://[^\'"<> \t]+[a-zA-Z0-9/])'
PAT_WORD     = r'((?<!' + PAT_WORDCHAR + r')' + PAT_WORDCHAR + r'+' \
                 r'(?!' + PAT_WORDCHAR + r'))'

RE_LINKWORD = re.compile(
	PAT_OPTWORD  + '|' +
	PAT_CTRL     + '|' +
	PAT_SPECIAL)
RE_TAGWORD = re.compile(
	PAT_HEADER   + '|' +
        PAT_GRAPHIC  + '|' +
	PAT_PIPEWORD + '|' +
	PAT_STARWORD + '|' +
    	PAT_COMMAND  + '|' +
	PAT_OPTWORD  + '|' +
	PAT_CTRL     + '|' +
	PAT_SPECIAL  + '|' +
	PAT_TITLE    + '|' +
	PAT_NOTE     + '|' +
	PAT_URL      + '|' +
	PAT_WORD)
RE_NEWLINE   = re.compile(r'[\r\n]')
RE_HRULE     = re.compile(r'[-=]{3,}.*[-=]{3,3}$')
RE_EG_START  = re.compile(r'(?:.* )?>$')
RE_EG_END    = re.compile(r'\S')
RE_SECTION   = re.compile(r'[-A-Z .][-A-Z0-9 .()]*(?=\s+\*)')
RE_STARTAG   = re.compile(r'\s\*([^ \t|]+)\*(?:\s|$)')
RE_LOCAL_ADD = re.compile(r'LOCAL ADDITIONS:\s+\*local-additions\*$')

class Link(object):
    __slots__ = 'link_pipe', 'link_plain'

    def __init__(self, link_pipe, link_plain):
	self.link_pipe = link_pipe
	self.link_plain = link_plain

class VimH2H(object):
    def __init__(self, tags, version=None):
        self._urls = { }
        self._version = version
	for line in RE_NEWLINE.split(tags):
	    m = RE_TAGLINE.match(line)
	    if m:
		tag, filename = m.group(1, 2)
		self.do_add_tag(filename, tag)

    def add_tags(self, filename, contents):
	for match in RE_STARTAG.finditer(contents):
	    tag = match.group(1).replace('\\', '\\\\').replace('/', '\\/')
	    self.do_add_tag(str(filename), tag)

    def do_add_tag(self, filename, tag):
	part1 = '' + html_escape[tag] + ''
	link_pipe = part1 + ' class="l"' + part2
	classattr = ' class="d"'
	m = RE_LINKWORD.match(tag)
	if m:
	    opt, ctrl, special = m.groups()
	    if opt is not None: classattr = ' class="o"'
	    elif ctrl is not None: classattr = ' class="k"'
	    elif special is not None: classattr = ' class="s"'
	link_plain = part1 + classattr + part2
	self._urls[tag] = Link(link_pipe, link_plain)

    def maplink(self, tag, css_class=None):
	links = self._urls.get(tag)
	if links is not None:
	    if css_class == 'l': return links.link_pipe
	    else: return links.link_plain
	elif css_class is not None:
	    return '' + html_escape[tag] + \
		    ''
	else: return html_escape[tag]

    def to_html(self, filename, contents, encoding, web_version=True):
	out = [ ]

	inexample = 0
        filename = str(filename)
        is_help_txt = (filename == 'help.txt')
	faq_line = False
	for line in RE_NEWLINE.split(contents):
	    line = line.rstrip('\r\n')
	    line_tabs = line
	    line = line.expandtabs()
	    if RE_HRULE.match(line):
		out.extend(('', line, '\n'))
		continue
	    if inexample == 2:
		if RE_EG_END.match(line):
		    inexample = 0
		    if line[0] == '<': line = line[1:]
		else:
                    out.extend(('', html_escape[line],
                               '\n'))
		    continue
	    if RE_EG_START.match(line_tabs):
		inexample = 1
		line = line[0:-1]
	    if RE_SECTION.match(line_tabs):
		m = RE_SECTION.match(line)
                out.extend((r'"', m.group(0), r''))
		line = line[m.end():]
	    if is_help_txt and RE_LOCAL_ADD.match(line_tabs):
		faq_line = True
	    lastpos = 0
	    for match in RE_TAGWORD.finditer(line):
		pos = match.start()
		if pos > lastpos:
		    out.append(html_escape[line[lastpos:pos]])
		lastpos = match.end()
                header, graphic, pipeword, starword, command, opt, ctrl, \
                        special, title, note, url, word = match.groups()
		if pipeword is not None:
		    out.append(self.maplink(pipeword, 'l'))
		elif starword is not None:
		    out.extend(('', html_escape[starword], ''))
                elif command is not None:
                    out.extend(('', html_escape[command],
                                ''))
		elif opt is not None:
		    out.append(self.maplink(opt, 'o'))
		elif ctrl is not None:
		    out.append(self.maplink(ctrl, 'k'))
		elif special is not None:
		    out.append(self.maplink(special, 's'))
		elif title is not None:
                    out.extend(('', html_escape[title],
                                ''))
		elif note is not None:
                    out.extend(('', html_escape[note],
                                ''))
		elif header is not None:
                    out.extend(('', html_escape[header[:-1]],
                                ''))
                elif graphic is not None:
                    out.append(html_escape[graphic[:-2]])
		elif url is not None:
                    out.extend(('' +
                                html_escape[url], ''))
		elif word is not None:
		    out.append(self.maplink(word))
	    if lastpos < len(line):
		out.append(html_escape[line[lastpos:]])
	    out.append('\n')
	    if inexample == 1: inexample = 2
	    if faq_line:
		out.append(VIM_FAQ_LINE)
		faq_line = False

        header = []
        header.append(HEAD.format(encoding=encoding, filename=filename))
        if web_version:
            header.append(SEARCH_SCRIPT)
        header.append(HEAD_END)
        if web_version and is_help_txt:
            vers_note = VERSION_NOTE.replace('{version}', self._version) \
                    if self._version else ''
            header.append(INTRO.replace('{vers-note}', vers_note))
        if web_version:
            header.append(SITENAVI_SEARCH)
        else:
            header.append(SITENAVI_PLAIN)
        header.append(TEXTSTART)
        return ''.join(chain(header, out, (FOOTER, SITENAVI_PLAIN, FOOTER2)))

class HtmlEscCache(dict):
    def __missing__(self, key):
        r = key.replace('&', '&') \
               .replace('<', '<') \
               .replace('>', '>')
        self[key] = r
        return r

html_escape = HtmlEscCache()
<|endoftext|>