<commit_before># -*- coding: utf-8 -*-
# Copyright (c) 2009 gocept gmbh & co. kg
# See also LICENSE.txt

import datetime
import cjson
import pysolr
import zeit.cms.browser.view
import zc.resourcelibrary
import zope.component
import zope.interface
import zope.app.appsetup.product
import zope.viewlet.interfaces
import zeit.cms.interfaces
import zeit.cms.clipboard.interfaces
import zeit.cms.content.interfaces
import zeit.cms.browser.interfaces
import zeit.cms.browser.preview
import zc.iso8601.parse
from zeit.find import lucenequery as lq

def resources(request):
    return zope.component.getAdapter(
        request, zope.interface.Interface, name='zeit.find')

def get_solr():
    config = zope.app.appsetup.product.getProductConfiguration('zeit.find')
    solr_url = config.get('solr_url')
    return pysolr.Solr(solr_url)

class Find(zeit.cms.browser.view.Base):

    def __call__(self):
        zc.resourcelibrary.need('zeit.find')
        return super(Find, self).__call__()

class JSONView(zeit.cms.browser.view.Base):
    template = None

    def __init__(self, context, request):
        super(JSONView, self).__init__(context, request)
        self.resources = resources(request)

    def __call__(self):
        self.request.response.setHeader('Content-Type', 'text/json')
        result = self.json()

        # use template indicated in JSON if it's there,
        # otherwise use class template
        template = result.pop('template', None)
        if template is None:
            template = self.template
        if template is not None:
            result['template_url'] = self.resources[template]()
        return cjson.encode(result)

    def json(self):
        return {}

    def url(self, view, uniqueId):
        return super(JSONView, self).url(
            self.context, '%s?uniqueId=%s' % (view, uniqueId))

class SearchForm(JSONView):
    template = 'search_form.jsont'

class SearchResult(JSONView):
    template = 'search_result.jsont'

    def _get(self, name, default=None):
        value = self.request.get(name, default)
        if value is default:
            return value
        value = value.strip()
        if value:
            return value
        return default

    def form(self):
        """Given the request, create search form contents.
        """
        g = self._get
        fulltext = g('fulltext')
        if fulltext is None:
            return None
        from_ = parse_input_date(g('from', 'TT.MM.JJJJ'))
        until = parse_input_date(g('until', 'TT.MM.JJJJ'))
        topic = g('topic', None)
        authors = g('author', None)
        keywords = g('keywords', None)
        # three states: want all published, want all unpublished, don't care
        published = g('published', None)
        if published == 'published':
            published = True
        elif published == 'unpublished':
            published = False
        else:
            published = None
        types = set()
        for t in ['article', 'gallery', 'video', 'teaser', 'centerpage']:
            if g(t, '') == 'on':
                types.add(t)
        return dict(
            fulltext=fulltext,
            from_=from_,
            until=until,
            topic=topic,
            authors=authors,
            keywords=keywords,
            published=published,
            types=types)

    def sort_order(self):
        return self.request.get('sort_order', 'relevance')

    def query(self, fulltext, from_, until, topic, authors, keywords,
                   published, types):
        """Given parameters, create solr query string.
        """
        terms = []
        terms.append(lq.field('text', fulltext))
        if from_ is not None or until is not None:
            terms.append(
                lq.datetime_range('last-semantic-change', from_, until))
        if topic is not None:
            terms.append(lq.field('ressort', topic))
        if authors is not None:
            terms.append(lq.multi_field('authors', authors))
        if keywords is not None:
            terms.append(lq.multi_field('keywords', keywords))
        if published is not None:
            terms.append(lq.bool_field('published', published))
        return lq.and_(*terms)

    def form_query(self):
        """Create solr query for request.
        """
        form = self.form()
        if form is None:
            return None
        return self.query(**form)

    def json(self):
        q = self.form_query()
        if q is None:
            return {"results":[]}
        #print q
        r = self.resources
        results = []
        conn = get_solr()

        # record any known favorites
        # XXX this isn't that pleasant to do every search,
        # but we need to match the uniqueIds quickly during
        # the search results
        favorite_uniqueIds = set()
        for favorite in get_favorites(self.request).values():
            uniqueId = favorite.references.uniqueId
            if not uniqueId:
                continue
            favorite_uniqueIds.add(uniqueId)

        result_fields = ['uniqueId', 'published',
                         'teaser_title', 'teaser_text',
                         'last-semantic-change', 'ressort',
                         'authors']

        sort_order = self.request.get('sort_order', 'score desc')
        if sort_order == 'relevance':
            sort_order = 'score desc'
        elif sort_order == 'date':
            sort_order = 'last-semantic-change desc'

        for result in conn.search(q,
                                  sort=sort_order,
                                  fl=' '.join(result_fields)):
            uniqueId = result.get('uniqueId', '')
            if uniqueId in favorite_uniqueIds:
                favorited_icon = r['favorite.png']()
            else:
                favorited_icon = r['not_favorite.png']()

            last_semantic_change = result.get('last-semantic-change')
            if last_semantic_change is not None:
                dt = zc.iso8601.parse.datetimetz(result['last-semantic-change'])
            else:
                dt = None

            published = result.get('published', True)
            if published:
                publication_status = r['published.png']()
            else:
                publication_status = r['unpublished.png']()

            preview_url = zeit.cms.browser.preview.get_preview_url(
                'preview-prefix', uniqueId)

            results.append({
                    'uniqueId': uniqueId,
                    'icon': '/@@/zeit-content-article-interfaces-IArticle-zmi_icon.png',
                    'favorited': favorited_icon,
                    'publication_status': publication_status,
                    'arrow': r['arrow_right.png'](),
                    'teaser_title': result.get('teaser_title', ''),
                    'teaser_text': result.get('teaser_text', ''),
                    'preview_url': preview_url,
                    'date': format_date(dt),
                    'date_filter': '',
                    'week': format_week(dt),
                    'week_filter': '',
                    'topics': result.get('ressort', ''),
                    'topics_filter': '',
                    'author': result.get('authors', ''),
                    'author_filter': '',
                    'related_url': self.url('expanded_search_result', uniqueId),
                    'favorite_url': self.url('toggle_favorited', uniqueId),
                    })
        if not results:
            return {'template': 'no_search_result.jsont'}
        return {'results': results}

class ExtendedSearchForm(JSONView):
    template = 'extended_search_form.jsont'

class ResultFilters(JSONView):
    template = 'result_filters.jsont'

    def time_entries(self):
        return [{"title": "heute", "amount": "20", "query": ""},
                {"title": "7 Tage", "amount": "1000+", "query": ""}]

    def author_entries(self):
        return [{"title": "Martijn Faassen", "amount": "45", "query": ""},
                {"title": "Christian Zagrodnick", "amount": "124", "query": ""}]

    def topic_entries(self):
        return [{"title": "Politik", "amount": "10", "query": ""},
                {"title": "Kultur", "amount": "7", "query": ""},
                {"title": "Kultur", "amount": "7", "query": ""}]

    def content_types_entries(self):
        return [ {"title": "Artikel", "amount": "1000+", "query": ""}]

    def json(self):
        return {"results": [
            {"row": [{"title": "Zeit", "entries": self.time_entries()},
                     {"title": "Ressort", "entries": self.topic_entries()}]},
            {"row": [{"title": "Autor", "entries": self.author_entries()},
                     {"title": "Inhaltstyp", "entries": self.content_types_entries()}]}]}

class ExpandedSearchResult(JSONView):
    template = 'expanded_search_result.jsont'

    def json(self):
        uniqueId = self.request.get('uniqueId')
        if not uniqueId:
            return {'template': 'no_expanded_search_result.jsont'}

        content = zeit.cms.interfaces.ICMSContent(uniqueId)
        related = zeit.cms.related.interfaces.IRelatedContent(content).related

        r = self.resources
        results = []
        for content in related:
            metadata = zeit.cms.content.interfaces.ICommonMetadata(
                content, None)
            if metadata is None:
                continue

            date = zeit.cms.content.interfaces.ISemanticChange(
                content).last_semantic_change
            publication_status = self.render_publication_status(content)
            results.append({
                    'uniqueId': content.uniqueId,
                    'publication_status': publication_status,
                    'short_teaser_title': metadata.shortTeaserTitle or '',
                    'short_teaser_text': metadata.shortTeaserText or '',
                    'date': format_date(date),
                    })
        if not results:
            return {'template': 'no_expanded_search_result.jsont'}

        return {'results': results}

    def render_publication_status(self, content):
        viewlet_manager = zope.component.getMultiAdapter(
            (content, self.request, self),
            zope.viewlet.interfaces.IViewletManager,
            name='zeit.cms.workflow-indicator')
        viewlet_manager.update()
        return viewlet_manager.render()

class ToggleFavorited(JSONView):
    template = 'toggle_favorited.jsont'

    def json(self):
        r = self.resources
        content = zeit.cms.interfaces.ICMSContent(
            self.request.get('uniqueId'))

        favorites = get_favorites(self.request)
        if content.__name__ in favorites:
            del favorites[content.__name__]
            return {'favorited': r['not_favorite.png']()}
        favorites[content.__name__] = (zeit.cms.clipboard.
                                       interfaces.IClipboardEntry(content))
        return {'favorited': r['favorite.png']()}

class Favorites(JSONView):
    template = 'search_result.jsont'

    def result_entry(self, content):
        r = self.resources
        uniqueId = content.uniqueId
        favorited_icon = r['favorite.png']()

        metadata = zeit.cms.content.interfaces.ICommonMetadata(content, None)

        return {
            'uniqueId': uniqueId,
            'icon': '/@@/zeit-content-article-interfaces-IArticle-zmi_icon.png',
            'favorited': favorited_icon,
            'publication_status': r['published.png'](),
            'arrow': r['arrow_right.png'](),
            'teaser_title': (metadata.teaserTitle or '') if metadata else '',
            'teaser_text': (metadata.teaserText or '') if metadata else '',
            'preview_url': '',
            'date': '13.02.2009',
            'date_filter': '',
            'week': '13/2009',
            'week_filter': '',
            'topics': 'Politik',
            'topics_filter': '',
            'author': (metadata.authors or '') if metadata else '',
            'author_filter': '',
            'related_url': self.url('expanded_search_result', uniqueId),
            'favorite_url': self.url('toggle_favorited', uniqueId),
            }

    def json(self):
        favorites = get_favorites(self.request)
        return {"results": [
            self.result_entry(a) for a in [
                zeit.cms.interfaces.ICMSContent(c.referenced_unique_id)
                for c in favorites.values()]]}

def get_favorites(request):
    favorites_id = u'Favoriten'
    clipboard = zeit.cms.clipboard.interfaces.IClipboard(request.principal)
    if not favorites_id in clipboard:
        clipboard.addClip(favorites_id)
    return clipboard[favorites_id]

def format_date(dt):
    if dt is None:
        return ''
    return dt.strftime('%d.%m.%Y')

def format_week(dt):
    if dt is None:
        return ''
    iso_year, iso_week, iso_weekday = dt.isocalendar()
    return '%s/%s' % (iso_week, iso_year)

class InputDateParseError(Exception):
    pass

def parse_input_date(s):
    """Given a date input string, return datetime.datetime object

    Date is of format DD.MM.YYYY. Special value 'TT.MM.JJJJ' is
    equivalent to no date at all.
    """
    s = s.strip()
    if not s:
        return None
    if s == 'TT.MM.JJJJ':
        return None
    try:
        day, month, year = s.split('.')
    except ValueError:
        raise InputDateParseError("Missing periods in date")
    try:
        day = int(day)
    except ValueError:
        raise InputDateParseError("Day is not a proper number")
    try:
        month = int(month)
    except ValueError:
        raise InputDateParseError("Month is not a proper number")
    try:
        year = int(year)
    except ValueError:
        raise InputDateParseError("Year is not a proper number")
    return datetime.datetime(year, month, day)
<commit_msg>Show filtered results (amount of hits) for time now. This required a refactoring of how search results are constructed into functions so it is more easy to reuse for the filters.<commit_after># -*- coding: utf-8 -*-
# Copyright (c) 2009 gocept gmbh & co. kg
# See also LICENSE.txt

from datetime import datetime, timedelta
import cjson
import pysolr
import zeit.cms.browser.view
import zc.resourcelibrary
import zope.component
import zope.interface
import zope.app.appsetup.product
import zope.viewlet.interfaces
import zeit.cms.interfaces
import zeit.cms.clipboard.interfaces
import zeit.cms.content.interfaces
import zeit.cms.browser.interfaces
import zeit.cms.browser.preview
import zc.iso8601.parse
from zeit.find import lucenequery as lq

def resources(request):
    return zope.component.getAdapter(
        request, zope.interface.Interface, name='zeit.find')

def get_solr():
    config = zope.app.appsetup.product.getProductConfiguration('zeit.find')
    solr_url = config.get('solr_url')
    return pysolr.Solr(solr_url)

class Find(zeit.cms.browser.view.Base):

    def __call__(self):
        zc.resourcelibrary.need('zeit.find')
        return super(Find, self).__call__()

class JSONView(zeit.cms.browser.view.Base):
    template = None

    def __init__(self, context, request):
        super(JSONView, self).__init__(context, request)
        self.resources = resources(request)

    def __call__(self):
        self.request.response.setHeader('Content-Type', 'text/json')
        result = self.json()

        # use template indicated in JSON if it's there,
        # otherwise use class template
        template = result.pop('template', None)
        if template is None:
            template = self.template
        if template is not None:
            result['template_url'] = self.resources[template]()
        return cjson.encode(result)

    def json(self):
        return {}

    def url(self, view, uniqueId):
        return super(JSONView, self).url(
            self.context, '%s?uniqueId=%s' % (view, uniqueId))

class SearchForm(JSONView):
    template = 'search_form.jsont'

class SearchResult(JSONView):
    template = 'search_result.jsont'

    def sort_order(self):
        return self.request.get('sort_order', 'relevance')

    def json(self):
        q = search_form_query(self.request)
        if q is None:
            return {"results":[]}

        # record any known favorites
        # XXX this isn't that pleasant to do every search,
        # but we need to match the uniqueIds quickly during
        # the search results
        favorite_uniqueIds = set()
        for favorite in get_favorites(self.request).values():
            uniqueId = favorite.references.uniqueId
            if not uniqueId:
                continue
            favorite_uniqueIds.add(uniqueId)

        result_fields = ['uniqueId', 'published',
                         'teaser_title', 'teaser_text',
                         'last-semantic-change', 'ressort',
                         'authors']

        sort_order = self.request.get('sort_order', 'score desc')
        if sort_order == 'relevance':
            sort_order = 'score desc'
        elif sort_order == 'date':
            sort_order = 'last-semantic-change desc'

        r = self.resources
        results = []
        conn = get_solr()

        for result in conn.search(q,
                                  sort=sort_order,
                                  fl=' '.join(result_fields)):
            uniqueId = result.get('uniqueId', '')
            if uniqueId in favorite_uniqueIds:
                favorited_icon = r['favorite.png']()
            else:
                favorited_icon = r['not_favorite.png']()

            last_semantic_change = result.get('last-semantic-change')
            if last_semantic_change is not None:
                dt = zc.iso8601.parse.datetimetz(result['last-semantic-change'])
            else:
                dt = None

            published = result.get('published', True)
            if published:
                publication_status = r['published.png']()
            else:
                publication_status = r['unpublished.png']()

            preview_url = zeit.cms.browser.preview.get_preview_url(
                'preview-prefix', uniqueId)

            results.append({
                    'uniqueId': uniqueId,
                    'icon': '/@@/zeit-content-article-interfaces-IArticle-zmi_icon.png',
                    'favorited': favorited_icon,
                    'publication_status': publication_status,
                    'arrow': r['arrow_right.png'](),
                    'teaser_title': result.get('teaser_title', ''),
                    'teaser_text': result.get('teaser_text', ''),
                    'preview_url': preview_url,
                    'date': format_date(dt),
                    'date_filter': '',
                    'week': format_week(dt),
                    'week_filter': '',
                    'topics': result.get('ressort', ''),
                    'topics_filter': '',
                    'author': result.get('authors', ''),
                    'author_filter': '',
                    'related_url': self.url('expanded_search_result', uniqueId),
                    'favorite_url': self.url('toggle_favorited', uniqueId),
                    })
        if not results:
            return {'template': 'no_search_result.jsont'}
        return {'results': results}

def _get(request, name, default=None):
    value = request.get(name, default)
    if value is default:
        return value
    value = value.strip()
    if value:
        return value
    return default

def search_form(request):
    g = lambda name, default=None: _get(request, name, default)
    fulltext = g('fulltext')
    if fulltext is None:
        return None
    from_ = parse_input_date(g('from', 'TT.MM.JJJJ'))
    until = parse_input_date(g('until', 'TT.MM.JJJJ'))
    topic = g('topic', None)
    authors = g('author', None)
    keywords = g('keywords', None)
    # three states: want all published, want all unpublished, don't care
    published = g('published', None)
    if published == 'published':
        published = True
    elif published == 'unpublished':
        published = False
    else:
        published = None
    types = set()
    for t in ['article', 'gallery', 'video', 'teaser', 'centerpage']:
        if g(t, '') == 'on':
            types.add(t)
    return dict(
        fulltext=fulltext,
        from_=from_,
        until=until,
        topic=topic,
        authors=authors,
        keywords=keywords,
        published=published,
        types=types)

def search_query(fulltext, from_, until, topic, authors, keywords,
                 published, types, filter_terms=None):
    """Given parameters, create solr query string.
    """
    filter_terms = filter_terms or []

    terms = []
    terms.append(lq.field('text', fulltext))
    if from_ is not None or until is not None:
        terms.append(
            lq.datetime_range('last-semantic-change', from_, until))
    if topic is not None:
        terms.append(lq.field('ressort', topic))
    if authors is not None:
        terms.append(lq.multi_field('authors', authors))
    if keywords is not None:
        terms.append(lq.multi_field('keywords', keywords))
    if published is not None:
        terms.append(lq.bool_field('published', published))

    terms.extend(filter_terms)
    return lq.and_(*terms)

def search_form_query(request, filter_terms=None):
    form = search_form(request)
    if form is None:
        return None
    form['filter_terms'] = filter_terms
    return search_query(**form)

class ExtendedSearchForm(JSONView):
    template = 'extended_search_form.jsont'

class ResultFilters(JSONView):
    template = 'result_filters.jsont'

    def topic_entries(self):
        return [{"title": "Politik", "amount": "10", "query": ""},
                {"title": "Kultur", "amount": "7", "query": ""},
                {"title": "Kultur", "amount": "7", "query": ""}]

    def time_entries(self):
        conn = get_solr()
        result = []
        for name, filter in [("heute", today_filter),
                             ("gestern", yesterday_filter),
                             ("7 Tage",  seven_day_filter),
                             ("letzter Monat", month_filter),
                             ("letztes halbes Jahr", half_year_filter),
                             ("letztes Jahr", year_filter)]:
            q = search_form_query(self.request, filter_terms=[filter()])
            amount = conn.search(q, rows=0).hits
            if amount > 0:
                result.append(dict(title=name,
                                   amount=format_amount(amount),
                                   query=""))
        return result

    def content_types_entries(self):
        return [ {"title": "Artikel", "amount": "1000+", "query": ""}]

    def author_entries(self):
        return [{"title": "Martijn Faassen", "amount": "45", "query": ""},
                {"title": "Christian Zagrodnick", "amount": "124", "query": ""}]

    def json(self):
        return {"results": [
                {"row": [{"title": "Ressort", "entries": self.topic_entries()},
                         {"title": "Zeit", "entries": self.time_entries()},
                         ]},
                {"row": [{"title": "Inhaltstyp", "entries": self.content_types_entries()},
                         {"title": "Autor", "entries": self.author_entries()},
                         ]}]}

def today_filter():
    start = datetime.now().replace(
        hour=0, minute=0, second=0, microsecond=0)
    end = start + timedelta(days=1)
    return lq.datetime_range('last-semantic-change', start, end)

def yesterday_filter():
    start = datetime.now().replace(
        hour=0, minute=0, second=0, microsecond=0) - timedelta(days=1)
    end = start + timedelta(days=1)
    return lq.datetime_range('last-semantic-change', start, end)

def seven_day_filter():
    end = datetime.now()
    start = end - timedelta(days=7)
    return lq.datetime_range('last-semantic-change', start, end)

def month_filter():
    end = datetime.now()
    # XXX last month period if 31 days?
    start = end - timedelta(days=31)
    return lq.datetime_range('last-semantic-change', start, end)

def half_year_filter():
    end = datetime.now()
    # last half year, about 183 days
    start = end - timedelta(days=183)
    return lq.datetime_range('last-semantic-change', start, end)

def year_filter():
    end = datetime.now()
    # last year, about 366 days (to be on the safe side)
    start = end - timedelta(days=366)
    return lq.datetime_range('last-semantic-change', start, end)

class ExpandedSearchResult(JSONView):
    template = 'expanded_search_result.jsont'

    def json(self):
        uniqueId = self.request.get('uniqueId')
        if not uniqueId:
            return {'template': 'no_expanded_search_result.jsont'}

        content = zeit.cms.interfaces.ICMSContent(uniqueId)
        related = zeit.cms.related.interfaces.IRelatedContent(content).related

        r = self.resources
        results = []
        for content in related:
            metadata = zeit.cms.content.interfaces.ICommonMetadata(
                content, None)
            if metadata is None:
                continue

            date = zeit.cms.content.interfaces.ISemanticChange(
                content).last_semantic_change
            publication_status = self.render_publication_status(content)
            results.append({
                    'uniqueId': content.uniqueId,
                    'publication_status': publication_status,
                    'short_teaser_title': metadata.shortTeaserTitle or '',
                    'short_teaser_text': metadata.shortTeaserText or '',
                    'date': format_date(date),
                    })
        if not results:
            return {'template': 'no_expanded_search_result.jsont'}

        return {'results': results}

    def render_publication_status(self, content):
        viewlet_manager = zope.component.getMultiAdapter(
            (content, self.request, self),
            zope.viewlet.interfaces.IViewletManager,
            name='zeit.cms.workflow-indicator')
        viewlet_manager.update()
        return viewlet_manager.render()

class ToggleFavorited(JSONView):
    template = 'toggle_favorited.jsont'

    def json(self):
        r = self.resources
        content = zeit.cms.interfaces.ICMSContent(
            self.request.get('uniqueId'))

        favorites = get_favorites(self.request)
        if content.__name__ in favorites:
            del favorites[content.__name__]
            return {'favorited': r['not_favorite.png']()}
        favorites[content.__name__] = (zeit.cms.clipboard.
                                       interfaces.IClipboardEntry(content))
        return {'favorited': r['favorite.png']()}

class Favorites(JSONView):
    template = 'search_result.jsont'

    def result_entry(self, content):
        r = self.resources
        uniqueId = content.uniqueId
        favorited_icon = r['favorite.png']()

        metadata = zeit.cms.content.interfaces.ICommonMetadata(content, None)

        return {
            'uniqueId': uniqueId,
            'icon': '/@@/zeit-content-article-interfaces-IArticle-zmi_icon.png',
            'favorited': favorited_icon,
            'publication_status': r['published.png'](),
            'arrow': r['arrow_right.png'](),
            'teaser_title': (metadata.teaserTitle or '') if metadata else '',
            'teaser_text': (metadata.teaserText or '') if metadata else '',
            'preview_url': '',
            'date': '13.02.2009',
            'date_filter': '',
            'week': '13/2009',
            'week_filter': '',
            'topics': 'Politik',
            'topics_filter': '',
            'author': (metadata.authors or '') if metadata else '',
            'author_filter': '',
            'related_url': self.url('expanded_search_result', uniqueId),
            'favorite_url': self.url('toggle_favorited', uniqueId),
            }

    def json(self):
        favorites = get_favorites(self.request)
        return {"results": [
            self.result_entry(a) for a in [
                zeit.cms.interfaces.ICMSContent(c.referenced_unique_id)
                for c in favorites.values()]]}

def get_favorites(request):
    favorites_id = u'Favoriten'
    clipboard = zeit.cms.clipboard.interfaces.IClipboard(request.principal)
    if not favorites_id in clipboard:
        clipboard.addClip(favorites_id)
    return clipboard[favorites_id]

def format_date(dt):
    if dt is None:
        return ''
    return dt.strftime('%d.%m.%Y')

def format_week(dt):
    if dt is None:
        return ''
    iso_year, iso_week, iso_weekday = dt.isocalendar()
    return '%s/%s' % (iso_week, iso_year)

def format_amount(amount):
    if amount >= 1000:
        return "1000+"
    else:
        return str(amount)

class InputDateParseError(Exception):
    pass

def parse_input_date(s):
    """Given a date input string, return datetime.datetime object

    Date is of format DD.MM.YYYY. Special value 'TT.MM.JJJJ' is
    equivalent to no date at all.
    """
    s = s.strip()
    if not s:
        return None
    if s == 'TT.MM.JJJJ':
        return None
    try:
        day, month, year = s.split('.')
    except ValueError:
        raise InputDateParseError("Missing periods in date")
    try:
        day = int(day)
    except ValueError:
        raise InputDateParseError("Day is not a proper number")
    try:
        month = int(month)
    except ValueError:
        raise InputDateParseError("Month is not a proper number")
    try:
        year = int(year)
    except ValueError:
        raise InputDateParseError("Year is not a proper number")
    return datetime(year, month, day)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import lang
from code import process, parse_deps, generate_accessors

def get_package(name):
	return ".".join(name.split(".")[:-1])

def split_name(name):
	r = name.split(".")
	return ".".join(r[:-1]), r[-1]

class component_generator(object):
	def __init__(self, name, component):
		self.name = name
		self.component = component
		self.aliases = {}
		self.properties = {}
		self.assignments = {}
		self.animations = {}
		self.package = get_package(name)
		self.base_type = None
		self.children = []
		self.methods = {}
		self.event_handlers = {}
		self.changed_handlers = {}
		self.key_handlers = {}
		self.events = set()
		self.id = None

		for child in component.children:
			self.add_child(child)

	def collect_id(self, id_set):
		if self.id is not None:
			id_set.add(self.id)
		for g in self.assignments.itervalues():
			if type(g) is component_generator and g.id:
				g.collect_id(id_set)
		for g in self.animations.itervalues():
			if type(g) is component_generator and g.id:
				g.collect_id(id_set)
		for g in self.children:
			g.collect_id(id_set)

	def assign(self, target, value):
		t = type(value)
		if t is lang.Component:
			value = component_generator(self.package + ".<anonymous>", value)
		self.assignments[target] = value

	def add_child(self, child):
		t = type(child)
		if t is lang.Property:
			if child.name in self.properties or child.name in self.aliases:
				raise Exception("duplicate property " + child.name)
			self.properties[child.name] = child.type
			if child.value is not None:
				self.assign(child.name, child.value)
		elif t is lang.AliasProperty:
			if child.name in self.properties or child.name in self.aliases:
				raise Exception("duplicate property " + child.name)
			self.aliases[child.name] = child.target
		elif t is lang.Assignment:
			self.assign(child.target, child.value)
		elif t is lang.IdAssignment:
			self.id = child.name
			self.assign("id", child.name)
		elif t is lang.Component:
			self.children.append(component_generator(self.package + ".<anonymous>", child))
		elif t is lang.Behavior:
			if child.target in self.animations:
				raise Exception("duplicate animation on property " + child.target);
			self.animations[child.target] = component_generator(self.package + ".<anonymous-animation>", child.animation)
		elif t is lang.Method:
			name, code = child.name, child.code
			if len(name) > 2 and name.startswith("on") and name[2].isupper(): #onXyzzy
				name = name[2].lower() + name[3:]
				if name.endswith("Pressed"):
					name = name[0].upper() + name[1:-7]
					if name in self.key_handlers:
						raise Exception("duplicate key handler " + child.name)
					self.key_handlers[name] = code
				elif name.endswith("Changed"):
					name = name[:-7]
					if name in self.changed_handlers:
						raise Exception("duplicate event handler " + child.name)
					self.changed_handlers[name] = code
				else:
					if name in self.event_handlers:
						raise Exception("duplicate event handler " + child.name)
					self.event_handlers[name] = code
			else:
				if name in self.methods:
					raise Exception("duplicate method " + name)
				self.methods[name] = code
		elif t is lang.Event:
			name = child.name
			if name in self.events:
				raise Exception("duplicate event " + name)
			self.events.add(name)
		else:
			print "unhandled", child

	def generate_properties(self):
		r = []
		for name, target in self.aliases.iteritems():
			get, pname = generate_accessors(target)
			r.append("""\
	core.addAliasProperty(this, '%s',
		(function() { return %s; }).bind(this),
		(function() { return %s.get('%s'); }).bind(this),
		(function(value) { %s.%s = value; }).bind(this));
""" %(name, get, get, pname, get, pname))
		for name, type in self.properties.iteritems():
			r.append("\tcore.addProperty(this, '%s', '%s');" %(type, name))
		return "\n".join(r)

	def generate_ctor(self, registry):
		return "\texports.%s.apply(this, arguments);\n" %(registry.find_component(self.package, self.component.name))

	def generate(self, registry):
		ctor  = "\texports.%s = function() {\n%s\n%s\n%s\n%s\n\tcore._bootstrap(this, '%s');\n}\n" %(self.name, self.generate_ctor(registry), "\n".join(self.generate_creators(registry, "this")), self.generate_properties(), self.generate_setup_code(registry, "this"), self.name)
		return ctor

	def generate_animations(self, registry, parent):
		r = []
		for name, animation in self.animations.iteritems():
			var = "behavior_on_" + name
			r.append("\tvar %s = new _globals.%s(%s);" %(var, registry.find_component(self.package, animation.component.name), parent))
			r.append(self.wrap_creator("create", var, "\n".join(animation.generate_creators(registry, var, 2))))
			r.append(self.wrap_creator("setup", var, animation.generate_setup_code(registry, var, 2)))
			r.append("\tthis.setAnimation('%s', %s);\n" %(name, var))
		return "\n".join(r)

	def wrap_creator(self, prefix, var, code):
		if not code.strip():
			return ""
		safe_var = var.replace('.', '__')
		return "\tfunction %s_%s () {\n%s\n\t}\n\t%s_%s.call(%s)" %(prefix, safe_var, code, prefix, safe_var, var)

	def generate_creators(self, registry, parent, ident_n = 1):
		prologue = []
		r = []
		ident = "\t" * ident_n

		for name in self.events:
			r.append("%sthis.%s = function() { var args = Array.prototype.slice.call(arguments); args.splice(0, 0, '%s'); this._emitEvent.apply(this, args);/*fixme*/ }" %(ident, name, name))

		idx = 0
		for gen in self.children:
			var = "%s_child%d" %(parent, idx)
			component = registry.find_component(self.package, gen.component.name)
			prologue.append("\tvar %s;" %var)
			r.append("\t%s = new _globals.%s(%s);" %(var, component, parent))
			p, code = gen.generate_creators(registry, var, ident_n + 1)
			prologue.append(p)
			r.append(self.wrap_creator("create", var, code))
			idx += 1

		for target, value in self.assignments.iteritems():
			if target == "id":
				if "." in value:
					raiseException("expected identifier, not expression")
				r.append("%sthis._setId('%s')" %(ident, value))
			elif target.endswith(".id"):
				raise Exception("setting id of the remote object is prohibited")

		return "\n".join(prologue), "\n".join(r)

	def generate_setup_code(self, registry, parent, ident_n = 1):
		r = []
		ident = "\t" * ident_n
		for target, value in self.assignments.iteritems():
			if target == "id":
				continue
			t = type(value)
			#print self.name, target, value
			if t is str:
				deps = parse_deps(value)
				if deps:
					suffix = "_var_%s__%s" %(parent.replace('.', '_'), target.replace('.', '_'))
					var = "_update" + suffix
					r.append("%svar %s = (function() { this.%s = %s; }).bind(this);" %(ident, var, target, value))
					r.append("%s%s();" %(ident, var))
					undep = []
					for path, dep in deps:
						r.append("%s%s.onChanged('%s', %s);" %(ident, path, dep, var))
						undep.append("%s.removeOnChanged('%s', _update%s)" %(path, dep, suffix))
					r.append("%sthis._removeUpdater('%s', (function() { %s }).bind(this));" %(ident, target, ";".join(undep)))
				else:
					r.append("%sthis._removeUpdater('%s'); this.%s = %s;" %(ident, target, target, value))

			elif t is component_generator:
				var = "this.%s" %target
				r.append("\t%s = new _globals.%s(%s);" %(var, registry.find_component(value.package, value.component.name), parent))
				r.append(self.wrap_creator("create", var, "\n".join(value.generate_creators(registry, var, ident_n + 1)))) #fixme: split me
				r.append(self.wrap_creator("setup", var, value.generate_setup_code(registry, var, ident_n + 1)))
			else:
				raise Exception("skip assignment %s = %s" %(target, value))

		idx = 0
		for gen in self.children:
			var = "%s_child%d" %(parent, idx)
			component = registry.find_component(self.package, gen.component.name)
			r.append(self.wrap_creator("setup", var, gen.generate_setup_code(registry, var, 2)))
			r.append("\tthis.addChild(%s);" %var);
			r.append("")
			idx += 1
		for name, code in self.methods.iteritems():
			code = process(code, registry)
			r.append("%sthis.%s = (function() %s ).bind(this);" %(ident, name, code))
		for name, code in self.event_handlers.iteritems():
			code = process(code, registry)
			if name != "completed":
				r.append("%sthis.on('%s', (function() %s ).bind(this));" %(ident, name, code))
			else:
				r.append("%sqml._context._onCompleted((function() %s ).bind(this));" %(ident, code))
		for name, code in self.changed_handlers.iteritems():
			code = process(code, registry)
			r.append("%sthis.onChanged('%s', (function() %s ).bind(this));" %(ident, name, code))
		for name, code in self.key_handlers.iteritems():
			code = process(code, registry)
			r.append("%sthis.onPressed('%s', (function() %s ).bind(this));" %(ident, name, code))
		r.append(self.generate_animations(registry, parent))
		return "\n".join(r)

class generator(object):
	def __init__(self):
		self.components = {}
		self.imports = {}
		self.packages = {}
		self.startup = []
		self.id_set = set(['renderer'])
		self.startup.append("qml.core.core._setup();")
		self.startup.append("qml._context = new qml.core.core.Context();")

	def add_component(self, name, component, declaration):
		if name in self.components:
			raise Exception("duplicate component " + name)

		package, component_name = split_name(name)

		if not declaration:
			name = "%s.Ui%s" %(package, component_name[0].upper() + component_name[1:])
			self.startup.append("qml._context.start(qml.%s)" %name)

		if package not in self.packages:
			self.packages[package] = set()
		self.packages[package].add(component_name)

		gen = component_generator(name, component)
		self.components[name] = gen

	def add_js(self, name, data):
		if name in self.imports:
			raise Exception("duplicate js name " + name)
		self.imports[name] = data

	def add_components(self, name, components, declaration):
		for component in components:
			self.add_component(name, component, declaration)

	def wrap(self, code):
		return "(function() {\nvar exports = {};\n%s\nreturn exports;\n} )" %code

	def find_component(self, package, name):
		if name == "Object":
			return "core.Object"

		if package in self.packages and name in self.packages[package]:
			return "%s.%s" %(package, name)
		for package_name, components in self.packages.iteritems():
			if name in components:
				return "%s.%s" %(package_name, name)
		raise Exception("component %s was not found" %name)

	def generate_components(self):
		r, deps = [], []

		for gen in self.components.itervalues():
			gen.collect_id(self.id_set)

		for name, gen in self.components.iteritems():
			code = "//=====[component %s]=====================\n\n" %name
			code += gen.generate(self)
			base_type = self.find_component(gen.package, gen.component.name)

			for i in xrange(0, len(deps)):
				t, b = deps[i]
				if base_type == t: #my base class, append after
					deps.insert(i + 1, (name, base_type))
					break
				if name == b: #me is base class, prepend before
					deps.insert(i, (name, base_type))
					break
			else:
				deps.append((name, base_type))

			r.append(code)

		for type, base_type in deps:
			code = ""
			code += "\texports.%s.prototype = Object.create(exports.%s.prototype);\n" %(type, base_type)
			code += "\texports.%s.prototype.constructor = exports.%s;\n" %(type, type)
			r.append(code)

		return "\n".join(r)

	def generate_imports(self):
		r = []
		for package in sorted(self.packages.keys()):
			r.append("if (!exports.%s) exports.%s = {};" %(package, package))

		for name, code in self.imports.iteritems():
			safe_name = name
			if safe_name.endswith(".js"):
				safe_name = safe_name[:-3]
			safe_name = safe_name.replace('/', '.')
			code = "//=====[import %s]=====================\n\n" %name + code
			r.append("_globals.%s = %s()" %(safe_name, self.wrap(code)))
		return "\n".join(r)

	def generate(self, ns):
		text = ""
		text += "var _globals = exports;\n"
		text += "%s\n" %self.generate_imports()
		text += "//========================================\n\n"
		text += "var core = _globals.core.core;\n"
		text += "%s\n" %self.generate_components()
		return "%s = %s();\n" %(ns, self.wrap(text))

	def generate_startup(self):
		return "\n".join(self.startup)
<commit_msg>generate property-assigned components with child (creation stage)<commit_after>#!/usr/bin/env python

import lang
from code import process, parse_deps, generate_accessors

def get_package(name):
	return ".".join(name.split(".")[:-1])

def split_name(name):
	r = name.split(".")
	return ".".join(r[:-1]), r[-1]

class component_generator(object):
	def __init__(self, name, component):
		self.name = name
		self.component = component
		self.aliases = {}
		self.properties = {}
		self.assignments = {}
		self.animations = {}
		self.package = get_package(name)
		self.base_type = None
		self.children = []
		self.methods = {}
		self.event_handlers = {}
		self.changed_handlers = {}
		self.key_handlers = {}
		self.events = set()
		self.id = None

		for child in component.children:
			self.add_child(child)

	def collect_id(self, id_set):
		if self.id is not None:
			id_set.add(self.id)
		for g in self.assignments.itervalues():
			if type(g) is component_generator and g.id:
				g.collect_id(id_set)
		for g in self.animations.itervalues():
			if type(g) is component_generator and g.id:
				g.collect_id(id_set)
		for g in self.children:
			g.collect_id(id_set)

	def assign(self, target, value):
		t = type(value)
		if t is lang.Component:
			value = component_generator(self.package + ".<anonymous>", value)
		self.assignments[target] = value

	def add_child(self, child):
		t = type(child)
		if t is lang.Property:
			if child.name in self.properties or child.name in self.aliases:
				raise Exception("duplicate property " + child.name)
			self.properties[child.name] = child.type
			if child.value is not None:
				self.assign(child.name, child.value)
		elif t is lang.AliasProperty:
			if child.name in self.properties or child.name in self.aliases:
				raise Exception("duplicate property " + child.name)
			self.aliases[child.name] = child.target
		elif t is lang.Assignment:
			self.assign(child.target, child.value)
		elif t is lang.IdAssignment:
			self.id = child.name
			self.assign("id", child.name)
		elif t is lang.Component:
			self.children.append(component_generator(self.package + ".<anonymous>", child))
		elif t is lang.Behavior:
			if child.target in self.animations:
				raise Exception("duplicate animation on property " + child.target);
			self.animations[child.target] = component_generator(self.package + ".<anonymous-animation>", child.animation)
		elif t is lang.Method:
			name, code = child.name, child.code
			if len(name) > 2 and name.startswith("on") and name[2].isupper(): #onXyzzy
				name = name[2].lower() + name[3:]
				if name.endswith("Pressed"):
					name = name[0].upper() + name[1:-7]
					if name in self.key_handlers:
						raise Exception("duplicate key handler " + child.name)
					self.key_handlers[name] = code
				elif name.endswith("Changed"):
					name = name[:-7]
					if name in self.changed_handlers:
						raise Exception("duplicate event handler " + child.name)
					self.changed_handlers[name] = code
				else:
					if name in self.event_handlers:
						raise Exception("duplicate event handler " + child.name)
					self.event_handlers[name] = code
			else:
				if name in self.methods:
					raise Exception("duplicate method " + name)
				self.methods[name] = code
		elif t is lang.Event:
			name = child.name
			if name in self.events:
				raise Exception("duplicate event " + name)
			self.events.add(name)
		else:
			print "unhandled", child

	def generate_properties(self):
		r = []
		for name, type in self.properties.iteritems():
			r.append("\tcore.addProperty(this, '%s', '%s');" %(type, name))
		return "\n".join(r)

	def generate_ctor(self, registry):
		return "\texports.%s.apply(this, arguments);\n" %(registry.find_component(self.package, self.component.name))

	def generate(self, registry):
		ctor  = "\texports.%s = function() {\n%s\n%s\n%s\n%s\n\tcore._bootstrap(this, '%s');\n}\n" %(self.name, self.generate_ctor(registry), self.generate_properties(), "\n".join(self.generate_creators(registry, "this")), self.generate_setup_code(registry, "this"), self.name)
		return ctor

	def generate_animations(self, registry, parent):
		r = []
		for name, animation in self.animations.iteritems():
			var = "behavior_on_" + name
			r.append("\tvar %s = new _globals.%s(%s);" %(var, registry.find_component(self.package, animation.component.name), parent))
			r.append(self.wrap_creator("create", var, "\n".join(animation.generate_creators(registry, var, 2))))
			r.append(self.wrap_creator("setup", var, animation.generate_setup_code(registry, var, 2)))
			r.append("\tthis.setAnimation('%s', %s);\n" %(name, var))
		return "\n".join(r)

	def wrap_creator(self, prefix, var, code):
		if not code.strip():
			return ""
		safe_var = var.replace('.', '__')
		return "\tfunction %s_%s () {\n%s\n\t}\n\t%s_%s.call(%s)" %(prefix, safe_var, code, prefix, safe_var, var)

	def generate_creators(self, registry, parent, ident_n = 1):
		prologue = []
		r = []
		ident = "\t" * ident_n

		for name in self.events:
			r.append("%sthis.%s = function() { var args = Array.prototype.slice.call(arguments); args.splice(0, 0, '%s'); this._emitEvent.apply(this, args);/*fixme*/ }" %(ident, name, name))

		idx = 0
		for gen in self.children:
			var = "%s_child%d" %(parent, idx)
			component = registry.find_component(self.package, gen.component.name)
			prologue.append("\tvar %s;" %var)
			r.append("\t%s = new _globals.%s(%s);" %(var, component, parent))
			p, code = gen.generate_creators(registry, var, ident_n + 1)
			prologue.append(p)
			r.append(self.wrap_creator("create", var, code))
			idx += 1

		for target, value in self.assignments.iteritems():
			if target == "id":
				if "." in value:
					raiseException("expected identifier, not expression")
				r.append("%sthis._setId('%s')" %(ident, value))
			elif target.endswith(".id"):
				raise Exception("setting id of the remote object is prohibited")

		for target, value in self.assignments.iteritems():
			if target == "id":
				continue
			if isinstance(value, component_generator):
				var = "%s_%s" %(parent, target.replace('.', '__'))
				prologue.append("%svar %s;" %(ident, var))
				r.append("%s%s = new _globals.%s(%s);" %(ident, var, registry.find_component(value.package, value.component.name), parent))
				p, code = value.generate_creators(registry, var, ident_n + 1)
				prologue.append(p)
				r.append(self.wrap_creator("create", var, code))
				r.append("%sthis.%s = %s" %(ident, target, var))

		return "\n".join(prologue), "\n".join(r)

	def generate_setup_code(self, registry, parent, ident_n = 1):
		r = []
		ident = "\t" * ident_n
		for name, target in self.aliases.iteritems():
			get, pname = generate_accessors(target)
			r.append("""\
	core.addAliasProperty(this, '%s',
		(function() { return %s; }).bind(this),
		(function() { return %s.get('%s'); }).bind(this),
		(function(value) { %s.%s = value; }).bind(this));
""" %(name, get, get, pname, get, pname))
		for target, value in self.assignments.iteritems():
			if target == "id":
				continue
			t = type(value)
			#print self.name, target, value
			if t is str:
				deps = parse_deps(value)
				if deps:
					suffix = "_var_%s__%s" %(parent.replace('.', '_'), target.replace('.', '_'))
					var = "_update" + suffix
					r.append("%svar %s = (function() { this.%s = %s; }).bind(this);" %(ident, var, target, value))
					r.append("%s%s();" %(ident, var))
					undep = []
					for path, dep in deps:
						r.append("%s%s.onChanged('%s', %s);" %(ident, path, dep, var))
						undep.append("%s.removeOnChanged('%s', _update%s)" %(path, dep, suffix))
					r.append("%sthis._removeUpdater('%s', (function() { %s }).bind(this));" %(ident, target, ";".join(undep)))
				else:
					r.append("%sthis._removeUpdater('%s'); this.%s = %s;" %(ident, target, target, value))

			elif t is component_generator:
				var = "%s_%s" %(parent, target.replace('.', '__'))
				r.append(self.wrap_creator("setup", var, value.generate_setup_code(registry, var, ident_n + 1)))
			else:
				raise Exception("skip assignment %s = %s" %(target, value))

		idx = 0
		for gen in self.children:
			var = "%s_child%d" %(parent, idx)
			component = registry.find_component(self.package, gen.component.name)
			r.append(self.wrap_creator("setup", var, gen.generate_setup_code(registry, var, 2)))
			r.append("\tthis.addChild(%s);" %var);
			r.append("")
			idx += 1
		for name, code in self.methods.iteritems():
			code = process(code, registry)
			r.append("%sthis.%s = (function() %s ).bind(this);" %(ident, name, code))
		for name, code in self.event_handlers.iteritems():
			code = process(code, registry)
			if name != "completed":
				r.append("%sthis.on('%s', (function() %s ).bind(this));" %(ident, name, code))
			else:
				r.append("%sqml._context._onCompleted((function() %s ).bind(this));" %(ident, code))
		for name, code in self.changed_handlers.iteritems():
			code = process(code, registry)
			r.append("%sthis.onChanged('%s', (function() %s ).bind(this));" %(ident, name, code))
		for name, code in self.key_handlers.iteritems():
			code = process(code, registry)
			r.append("%sthis.onPressed('%s', (function() %s ).bind(this));" %(ident, name, code))
		r.append(self.generate_animations(registry, parent))
		return "\n".join(r)

class generator(object):
	def __init__(self):
		self.components = {}
		self.imports = {}
		self.packages = {}
		self.startup = []
		self.id_set = set(['renderer'])
		self.startup.append("qml.core.core._setup();")
		self.startup.append("qml._context = new qml.core.core.Context();")

	def add_component(self, name, component, declaration):
		if name in self.components:
			raise Exception("duplicate component " + name)

		package, component_name = split_name(name)

		if not declaration:
			name = "%s.Ui%s" %(package, component_name[0].upper() + component_name[1:])
			self.startup.append("qml._context.start(qml.%s)" %name)

		if package not in self.packages:
			self.packages[package] = set()
		self.packages[package].add(component_name)

		gen = component_generator(name, component)
		self.components[name] = gen

	def add_js(self, name, data):
		if name in self.imports:
			raise Exception("duplicate js name " + name)
		self.imports[name] = data

	def add_components(self, name, components, declaration):
		for component in components:
			self.add_component(name, component, declaration)

	def wrap(self, code):
		return "(function() {\nvar exports = {};\n%s\nreturn exports;\n} )" %code

	def find_component(self, package, name):
		if name == "Object":
			return "core.Object"

		if package in self.packages and name in self.packages[package]:
			return "%s.%s" %(package, name)
		for package_name, components in self.packages.iteritems():
			if name in components:
				return "%s.%s" %(package_name, name)
		raise Exception("component %s was not found" %name)

	def generate_components(self):
		r, deps = [], []

		for gen in self.components.itervalues():
			gen.collect_id(self.id_set)

		for name, gen in self.components.iteritems():
			code = "//=====[component %s]=====================\n\n" %name
			code += gen.generate(self)
			base_type = self.find_component(gen.package, gen.component.name)

			for i in xrange(0, len(deps)):
				t, b = deps[i]
				if base_type == t: #my base class, append after
					deps.insert(i + 1, (name, base_type))
					break
				if name == b: #me is base class, prepend before
					deps.insert(i, (name, base_type))
					break
			else:
				deps.append((name, base_type))

			r.append(code)

		for type, base_type in deps:
			code = ""
			code += "\texports.%s.prototype = Object.create(exports.%s.prototype);\n" %(type, base_type)
			code += "\texports.%s.prototype.constructor = exports.%s;\n" %(type, type)
			r.append(code)

		return "\n".join(r)

	def generate_imports(self):
		r = []
		for package in sorted(self.packages.keys()):
			r.append("if (!exports.%s) exports.%s = {};" %(package, package))

		for name, code in self.imports.iteritems():
			safe_name = name
			if safe_name.endswith(".js"):
				safe_name = safe_name[:-3]
			safe_name = safe_name.replace('/', '.')
			code = "//=====[import %s]=====================\n\n" %name + code
			r.append("_globals.%s = %s()" %(safe_name, self.wrap(code)))
		return "\n".join(r)

	def generate(self, ns):
		text = ""
		text += "var _globals = exports;\n"
		text += "%s\n" %self.generate_imports()
		text += "//========================================\n\n"
		text += "var core = _globals.core.core;\n"
		text += "%s\n" %self.generate_components()
		return "%s = %s();\n" %(ns, self.wrap(text))

	def generate_startup(self):
		return "\n".join(self.startup)
<|endoftext|>


<commit_before>"""democracylab URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/1.11/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.conf.urls import url, include
    2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
"""
from django.conf.urls import url
from django.views.generic import TemplateView
from django.contrib.sitemaps.views import sitemap
from .sitemaps import ProjectSitemap

from . import views

urlpatterns = [

    url(
        r'^sitemap\.xml$',
        sitemap,
        {'sitemaps': {'projects': ProjectSitemap()}},
        name='django.contrib.sitemaps.views.sitemap'
    ),
    url(r'^googlebb20bcf8545e7046.html$', TemplateView.as_view(template_name="googlebb20bcf8545e7046.html")),
    url(r'^groups/create/$', views.group_create, name='group_create'),
    url(r'^groups/edit/(?P<group_id>[0-9]+)/$', views.group_edit, name='group_edit'),
    url(r'^groups/delete/(?P<group_id>[0-9]+)/$', views.group_delete, name='group_delete'),
    url(r'^events/create/$', views.event_create, name='event_create'),
    url(r'^events/edit/(?P<event_id>[0-9]+)/$', views.event_edit, name='event_edit'),
    url(r'^events/delete/(?P<event_id>[0-9]+)/$', views.event_delete, name='event_delete'),
    url(r'^projects/edit/(?P<project_id>[0-9]+)/$', views.project_edit, name='project_edit'),
    url(r'^projects/delete/(?P<project_id>[0-9]+)/$', views.project_delete, name='project_delete'),
    url(r'^projects/signup/$', views.project_create, name='project_create'),
    url(r'^projects/approve/(?P<project_id>[0-9]+)/$', views.approve_project, name='approve_project'),
    url(
        r'^presign_s3/upload/project/thumbnail/$',
        views.presign_project_thumbnail_upload,
    ),
    url(
        r'^delete_s3/(?P<s3_key>.*)$',
        views.delete_uploaded_file,
    ),
    url(r'^api/projects', views.projects_list),
    url(r'^api/my_projects', views.my_projects),
    url(r'^api/tags', views.tags),
    url(r'^index/$', views.index),
    url(r'^api/stats$', views.get_site_stats, name='get_site_stats'),
    url(r'^api/project/(?P<project_id>[0-9]+)/$', views.get_project, name='get_project'),
    url(r'^api/group/(?P<group_id>[0-9]+/$', views.get_group, name='get_group'),
    url(r'^contact/project/(?P<project_id>[0-9]+)/$', views.contact_project_owner, name='contact_project_owner'),
    url(r'^contact/volunteers/(?P<project_id>[0-9]+)/$', views.contact_project_volunteers, name='contact_project_volunteers'),
    url(r'^contact/volunteer/(?P<application_id>[0-9]+)/$', views.contact_project_volunteer, name='contact_project_volunteer'),
    url(r'^volunteer/(?P<project_id>[0-9]+)/$', views.volunteer_with_project, name='volunteer_with_project'),
    url(r'^volunteer/leave/(?P<project_id>[0-9]+)/$', views.leave_project, name='leave_project'),
    url(r'^volunteer/approve/(?P<application_id>[0-9]+)/$', views.accept_project_volunteer, name='accept_project_volunteer'),
    url(r'^volunteer/reject/(?P<application_id>[0-9]+)/$', views.reject_project_volunteer, name='reject_project_volunteer'),
    url(r'^volunteer/dismiss/(?P<application_id>[0-9]+)/$', views.dismiss_project_volunteer, name='dismiss_project_volunteer'),
    url(r'^volunteer/promote/(?P<application_id>[0-9]+)/$', views.promote_project_volunteer, name='promote_project_volunteer'),
    url(r'^volunteer/demote/(?P<application_id>[0-9]+)/$', views.demote_project_volunteer, name='demote_project_volunteer'),
    url(r'^volunteer/renew/(?P<application_id>[0-9]+)/$', views.renew_volunteering_with_project, name='renew_volunteering_with_project'),
    url(r'^volunteer/conclude/(?P<application_id>[0-9]+)/$', views.conclude_volunteering_with_project, name='conclude_volunteering_with_project'),
    url(r'^alert/create/$', views.add_alert, name='add_alert')

]
<commit_msg>Fix: missing paren in regex<commit_after>"""democracylab URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/1.11/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.conf.urls import url, include
    2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
"""
from django.conf.urls import url
from django.views.generic import TemplateView
from django.contrib.sitemaps.views import sitemap
from .sitemaps import ProjectSitemap

from . import views

urlpatterns = [

    url(
        r'^sitemap\.xml$',
        sitemap,
        {'sitemaps': {'projects': ProjectSitemap()}},
        name='django.contrib.sitemaps.views.sitemap'
    ),
    url(r'^googlebb20bcf8545e7046.html$', TemplateView.as_view(template_name="googlebb20bcf8545e7046.html")),
    url(r'^groups/create/$', views.group_create, name='group_create'),
    url(r'^groups/edit/(?P<group_id>[0-9]+)/$', views.group_edit, name='group_edit'),
    url(r'^groups/delete/(?P<group_id>[0-9]+)/$', views.group_delete, name='group_delete'),
    url(r'^events/create/$', views.event_create, name='event_create'),
    url(r'^events/edit/(?P<event_id>[0-9]+)/$', views.event_edit, name='event_edit'),
    url(r'^events/delete/(?P<event_id>[0-9]+)/$', views.event_delete, name='event_delete'),
    url(r'^projects/edit/(?P<project_id>[0-9]+)/$', views.project_edit, name='project_edit'),
    url(r'^projects/delete/(?P<project_id>[0-9]+)/$', views.project_delete, name='project_delete'),
    url(r'^projects/signup/$', views.project_create, name='project_create'),
    url(r'^projects/approve/(?P<project_id>[0-9]+)/$', views.approve_project, name='approve_project'),
    url(
        r'^presign_s3/upload/project/thumbnail/$',
        views.presign_project_thumbnail_upload,
    ),
    url(
        r'^delete_s3/(?P<s3_key>.*)$',
        views.delete_uploaded_file,
    ),
    url(r'^api/projects', views.projects_list),
    url(r'^api/my_projects', views.my_projects),
    url(r'^api/tags', views.tags),
    url(r'^index/$', views.index),
    url(r'^api/stats$', views.get_site_stats, name='get_site_stats'),
    url(r'^api/project/(?P<project_id>[0-9]+)/$', views.get_project, name='get_project'),
    url(r'^api/group/(?P<group_id>[0-9]+)/$', views.get_group, name='get_group'),
    url(r'^contact/project/(?P<project_id>[0-9]+)/$', views.contact_project_owner, name='contact_project_owner'),
    url(r'^contact/volunteers/(?P<project_id>[0-9]+)/$', views.contact_project_volunteers, name='contact_project_volunteers'),
    url(r'^contact/volunteer/(?P<application_id>[0-9]+)/$', views.contact_project_volunteer, name='contact_project_volunteer'),
    url(r'^volunteer/(?P<project_id>[0-9]+)/$', views.volunteer_with_project, name='volunteer_with_project'),
    url(r'^volunteer/leave/(?P<project_id>[0-9]+)/$', views.leave_project, name='leave_project'),
    url(r'^volunteer/approve/(?P<application_id>[0-9]+)/$', views.accept_project_volunteer, name='accept_project_volunteer'),
    url(r'^volunteer/reject/(?P<application_id>[0-9]+)/$', views.reject_project_volunteer, name='reject_project_volunteer'),
    url(r'^volunteer/dismiss/(?P<application_id>[0-9]+)/$', views.dismiss_project_volunteer, name='dismiss_project_volunteer'),
    url(r'^volunteer/promote/(?P<application_id>[0-9]+)/$', views.promote_project_volunteer, name='promote_project_volunteer'),
    url(r'^volunteer/demote/(?P<application_id>[0-9]+)/$', views.demote_project_volunteer, name='demote_project_volunteer'),
    url(r'^volunteer/renew/(?P<application_id>[0-9]+)/$', views.renew_volunteering_with_project, name='renew_volunteering_with_project'),
    url(r'^volunteer/conclude/(?P<application_id>[0-9]+)/$', views.conclude_volunteering_with_project, name='conclude_volunteering_with_project'),
    url(r'^alert/create/$', views.add_alert, name='add_alert')

]
<|endoftext|>


<commit_before>import os
import hashlib
import httplib
import requests
import json
import urllib
import urlparse
import tempfile
import traceback
from datetime import datetime
from ckan.lib.celery_app import celery
from webstore_upload import upload_content
from webstore_upload import DATA_FORMATS as WEBSTORE_DATA_FORMATS

try:
    from ckanext.archiver import settings
except ImportError:
    from ckanext.archiver import default_settings as settings

HTTP_ERROR_CODES = {
    httplib.MULTIPLE_CHOICES: "300 Multiple Choices not implemented",
    httplib.USE_PROXY: "305 Use Proxy not implemented",
    httplib.INTERNAL_SERVER_ERROR: "Internal server error on the remote server",
    httplib.BAD_GATEWAY: "Bad gateway",
    httplib.SERVICE_UNAVAILABLE: "Service unavailable",
    httplib.GATEWAY_TIMEOUT: "Gateway timeout",
    httplib.METHOD_NOT_ALLOWED: "405 Method Not Allowed"
}

DATA_FORMATS = [
    'csv',
    'text/csv',
    'txt',
    'text/plain',
    'text/html',
    'html',
    'rdf',
    'text/rdf',
    'xml',
    'xls',
    'application/ms-excel',
    'application/vnd.ms-excel',
    'application/xls',
    'text/xml',
    'tar',
    'application/x-tar',
    'zip',
    'application/zip'
    'gz',
    'application/gzip',
    'application/x-gzip',
    'application/octet-stream'
]

class ArchiverError(Exception):
    pass
class DownloadError(Exception):
    pass
class LinkCheckerError(Exception):
    pass
class CkanError(Exception):
    pass

def _clean_content_type(ct):
    # For now we should remove the charset from the content type and
    # handle it better, differently, later on.
    if 'charset' in ct:
        return ct[:ct.index(';')]
    return ct

def download(context, resource, url_timeout=30,
             max_content_length=settings.MAX_CONTENT_LENGTH,
             data_formats=DATA_FORMATS):

    link_context = "{}"
    link_data = json.dumps({
        'url': resource['url'],
        'url_timeout': url_timeout
    })
    headers = json.loads(link_checker(link_context, link_data))

    resource_format = resource['format'].lower()
    ct = _clean_content_type( headers.get('content-type', '').lower() )
    cl = headers.get('content-length')

    if resource.get('mimetype') != ct:
        resource_changed = True
        resource['mimetype'] = ct

    # this is to store the size in case there is an error, but the real size check
    # is done after dowloading the data file, with its real length
    if cl is not None and (resource.get('size') != cl):
        resource_changed = True
        resource['size'] = cl

    # make sure resource content-length does not exceed our maximum
    if cl and int(cl) >= max_content_length:
        if resource_changed:
            _update_resource(context, resource)
        # record fact that resource is too large to archive
        raise DownloadError("Content-length %s exceeds maximum allowed value %s" %
            (cl, max_content_length))

    # check that resource is a data file
    if not (resource_format in data_formats or ct.lower() in data_formats):
        if resource_changed:
            _update_resource(context, resource)
        raise DownloadError("Of content type %s, not downloading" % ct)

    # get the resource and archive it
    res = requests.get(resource['url'], timeout = url_timeout)
    length, hash, saved_file = _save_resource(resource, res, max_content_length)

    # check if resource size changed
    if unicode(length) != resource.get('size'):
        resource_changed = True
        resource['size'] = unicode(length)

    # check that resource did not exceed maximum size when being saved
    # (content-length header could have been invalid/corrupted, or not accurate
    # if resource was streamed)
    #
    # TODO: remove partially archived file in this case
    if length >= max_content_length:
        if resource_changed:
            _update_resource(context, resource)
        # record fact that resource is too large to archive
        raise DownloadError("Content-length after streaming reached maximum allowed value of %s" %
            max_content_length)

     # update the resource metadata in CKAN if the resource has changed
     if resource.get('hash') != hash:
         resource['hash'] = hash
         try:
             # This may fail for archiver.update() as a result of the resource
             # not yet existing, but is necessary for dependant extensions.
             _update_resource(context, resource)
         except:
             pass

    return {'length': length,
            'hash' : hash,
            'headers': headers,
            'saved_file': saved_file}

@celery.task(name = "archiver.clean")
def clean():
    """
    Remove all archived resources.
    """
    logger = clean.get_logger()
    logger.error("clean task not implemented yet")

@celery.task(name = "archiver.update")
def update(context, data):
    try:
        data = json.loads(data)
        context = json.loads(context)
        result = _update(context, data)
        return result
    except Exception, e:
        update_task_status(context, {
            'entity_id': data['id'],
            'entity_type': u'resource',
            'task_type': 'archiver',
            'key': u'celery_task_id',
            'value': unicode(update.request.id),
            'error': '%s: %s' % (e.__class__.__name__,  unicode(e)),
            'stack': traceback.format_exc(),
            'last_updated': datetime.now().isoformat()
        })
        raise

def _update(context, data):
    """
    Link check and archive the given resource.

    Returns a JSON dict:

        {
            'resource': the updated resource dict,
            'file_path': path to archived file (if archive successful), or None
        }
    """
    logger = update.get_logger()
    rid = data.pop(u'revision_id')
    api_url = urlparse.urljoin(context['site_url'], 'api/action')

    # check that archive directory exists
    if not os.path.exists(settings.ARCHIVE_DIR):
        logger.info("Creating archive directory: %s" % settings.ARCHIVE_DIR)
        os.mkdir(settings.ARCHIVE_DIR)

    if not data:
        raise ArchiverError('Resource not found')

    result = None
    try:
        result = download(context, data)
        if result is None:
            raise Exception("Download failed")
    except Exception, downloaderr:
        if hasattr(settings, 'RETRIES') and settings.RETRIES:
            update.retry(args=(json.dumps(context), json.dumps(data)), exc=downloaderr)
        else:
            return

    # Check here whether we want to upload this content to webstore before
    # archiving
    # TODO: Possibly run this as a sub-task
    if settings.UPLOAD_TO_WEBSTORE and settings.WEBSTORE_URL:
        content_type = _clean_content_type( result['headers'].get('content-type', '') )
        if content_type in WEBSTORE_DATA_FORMATS or context.get('format', '') in WEBSTORE_DATA_FORMATS:
            try:
                context['webstore_url'] = settings.WEBSTORE_URL
                logger.info("Attempting to upload content to webstore: %s" % context['webstore_url'])
                upload_content( context, data, result )
            except Exception, eUpload:
                logger.error( eUpload )

                if hasattr(settings, 'RETRIES') and settings.RETRIES:
                    data[u'revision_id'] = rid # put this back as we'll need it next time
                    update.retry(args=(json.dumps(context), json.dumps(data)), exc=e)

    logger.info("Attempting to archive resource: %s" % data['url'])
    file_path = archive_resource(context, data, logger, result)

    return json.dumps({
        'resource': data,
        'file_path': file_path
    })

@celery.task(name = "archiver.link_checker")
def link_checker(context, data):
    """
    Check that the resource's url is valid, and accepts a HEAD request.

    Returns a json dict of the headers of the request
    """
    data = json.loads(data)
    url_timeout = data.get('url_timeout', 30)

    success = True
    error_message = ''
    headers = {}

    # Find out if it has unicode characters, and if it does, quote them
    # so we are left with an ascii string
    try:
        url = data['url'].decode('ascii')
    except:
        parts = list(urlparse.urlparse(url))
        parts[2] = urllib.quote(parts[2].encode('utf-8'))
        url = urlparse.urlunparse(parts)
    url = str(url)

    # parse url
    parsed_url = urlparse.urlparse(url)
    # Check we aren't using any schemes we shouldn't be
    allowed_schemes = ['http', 'https', 'ftp']
    if not parsed_url.scheme in allowed_schemes:
        raise LinkCheckerError("Invalid url scheme")
    # check that query string is valid
    # see: http://trac.ckan.org/ticket/318
    # TODO: check urls with a better validator?
    #       eg: ll.url (http://www.livinglogic.de/Python/url/Howto.html)?
    elif any(['/' in parsed_url.query, ':' in parsed_url.query]):
        raise LinkCheckerError("Invalid URL")
    else:
        # Send a head request
        try:
            res = requests.head(url, timeout = url_timeout)
            headers = res.headers
        except ValueError, ve:
            raise LinkCheckerError("Invalid URL")
        if res.error:
            if res.status_code in HTTP_ERROR_CODES:
                error_message = HTTP_ERROR_CODES[res.status_code]
            else:
                error_message = "URL unobtainable"
            raise LinkCheckerError(error_message)

    return json.dumps(headers)

def archive_resource(context, resource, logger, result=None, url_timeout = 30):
    """
    Archive the given resource.
    """
    if result['length']:
        dir = os.path.join(settings.ARCHIVE_DIR, resource['id'])
        if not os.path.exists(dir):
            os.mkdir(dir)
        # try to get a file name from the url
        parsed_url = urlparse.urlparse(resource.get('url'))
        try:
            file_name = parsed_url.path.split('/')[-1] or 'resource'
        except:
            file_name = "resource"
        saved_file = os.path.join(dir, file_name)
        os.rename(result['saved_file'], saved_file)

        # update the resource object: set cache_url and cache_last_updated
        if context.get('cache_url_root'):
            cache_url = urlparse.urljoin(
                context['cache_url_root'], '%s/%s' % (resource['id'], file_name)
            )
            if resource.get('cache_url') != cache_url:
                resource['cache_url'] = cache_url
                resource['cache_last_updated'] = datetime.now().isoformat()
                _update_resource(context, resource)

    return saved_file

def _save_resource(resource, response, max_file_size, chunk_size = 1024*16):
    """
    Write the response content to disk.

    Returns a tuple:

        (file length: int, content hash: string, saved file path: string)
    """
    resource_hash = hashlib.sha1()
    length = 0
    saved_file = None

    #tmp_resource_file = os.path.join(settings.ARCHIVE_DIR, 'archive_%s' % os.getpid())
    fd, tmp_resource_file_path = tempfile.mkstemp()

    with open(tmp_resource_file_path, 'wb') as fp:
        for chunk in response.iter_content(chunk_size = chunk_size, decode_unicode=False):
            fp.write(chunk)
            length += len(chunk)
            resource_hash.update(chunk)

            if length >= max_file_size:
                break

    os.close(fd)

    content_hash = unicode(resource_hash.hexdigest())
    return length, content_hash, tmp_resource_file_path

def _update_resource(context, resource):
    """
    Use CKAN API to update the given resource.
    If cannot update, records this fact in the task_status table.

    Returns the content of the response.

    """
    api_url = urlparse.urljoin(context['site_url'], 'api/action')
    resource['last_modified'] = datetime.now().isoformat()
    post_data = json.dumps(resource)
    res = requests.post(
        api_url + '/resource_update', post_data,
        headers = {'Authorization': context['apikey'],
                   'Content-Type': 'application/json'}
    )

    if res.status_code == 200:
        return res.content
    else:
        raise CkanError('ckan failed to update resource, status_code (%s)'
                        % res.status_code)

def update_task_status(context, data):
    """
    Use CKAN API to update the task status. The data parameter
    should be a dict representing one row in the task_status table.

    Returns the content of the response.
    """
    api_url = urlparse.urljoin(context['site_url'], 'api/action')
    res = requests.post(
        api_url + '/task_status_update', json.dumps(data),
        headers = {'Authorization': context['site_user_apikey'],
                   'Content-Type': 'application/json'}
    )
    if res.status_code == 200:
        return res.content
    else:
        raise CkanError('ckan failed to update task_status, status_code (%s), error %s'  % (res.status_code, res.content))

<commit_msg>Added a brief comment about [#1684]<commit_after>import os
import hashlib
import httplib
import requests
import json
import urllib
import urlparse
import tempfile
import traceback
from datetime import datetime
from ckan.lib.celery_app import celery
from webstore_upload import upload_content
from webstore_upload import DATA_FORMATS as WEBSTORE_DATA_FORMATS

try:
    from ckanext.archiver import settings
except ImportError:
    from ckanext.archiver import default_settings as settings

HTTP_ERROR_CODES = {
    httplib.MULTIPLE_CHOICES: "300 Multiple Choices not implemented",
    httplib.USE_PROXY: "305 Use Proxy not implemented",
    httplib.INTERNAL_SERVER_ERROR: "Internal server error on the remote server",
    httplib.BAD_GATEWAY: "Bad gateway",
    httplib.SERVICE_UNAVAILABLE: "Service unavailable",
    httplib.GATEWAY_TIMEOUT: "Gateway timeout",
    httplib.METHOD_NOT_ALLOWED: "405 Method Not Allowed"
}

DATA_FORMATS = [
    'csv',
    'text/csv',
    'txt',
    'text/plain',
    'text/html',
    'html',
    'rdf',
    'text/rdf',
    'xml',
    'xls',
    'application/ms-excel',
    'application/vnd.ms-excel',
    'application/xls',
    'text/xml',
    'tar',
    'application/x-tar',
    'zip',
    'application/zip'
    'gz',
    'application/gzip',
    'application/x-gzip',
    'application/octet-stream'
]

class ArchiverError(Exception):
    pass
class DownloadError(Exception):
    pass
class LinkCheckerError(Exception):
    pass
class CkanError(Exception):
    pass

def _clean_content_type(ct):
    # For now we should remove the charset from the content type and
    # handle it better, differently, later on.
    if 'charset' in ct:
        return ct[:ct.index(';')]
    return ct

def download(context, resource, url_timeout=30,
             max_content_length=settings.MAX_CONTENT_LENGTH,
             data_formats=DATA_FORMATS):

    link_context = "{}"
    link_data = json.dumps({
        'url': resource['url'],
        'url_timeout': url_timeout
    })
    headers = json.loads(link_checker(link_context, link_data))

    resource_format = resource['format'].lower()
    ct = _clean_content_type( headers.get('content-type', '').lower() )
    cl = headers.get('content-length')

    if resource.get('mimetype') != ct:
        resource_changed = True
        resource['mimetype'] = ct

    # this is to store the size in case there is an error, but the real size check
    # is done after dowloading the data file, with its real length
    if cl is not None and (resource.get('size') != cl):
        resource_changed = True
        resource['size'] = cl

    # make sure resource content-length does not exceed our maximum
    if cl and int(cl) >= max_content_length:
        if resource_changed:
            _update_resource(context, resource)
        # record fact that resource is too large to archive
        raise DownloadError("Content-length %s exceeds maximum allowed value %s" %
            (cl, max_content_length))

    # check that resource is a data file
    if not (resource_format in data_formats or ct.lower() in data_formats):
        if resource_changed:
            _update_resource(context, resource)
        raise DownloadError("Of content type %s, not downloading" % ct)

    # get the resource and archive it
    res = requests.get(resource['url'], timeout = url_timeout)
    length, hash, saved_file = _save_resource(resource, res, max_content_length)

    # check if resource size changed
    if unicode(length) != resource.get('size'):
        resource_changed = True
        resource['size'] = unicode(length)

    # check that resource did not exceed maximum size when being saved
    # (content-length header could have been invalid/corrupted, or not accurate
    # if resource was streamed)
    #
    # TODO: remove partially archived file in this case
    if length >= max_content_length:
        if resource_changed:
            _update_resource(context, resource)
        # record fact that resource is too large to archive
        raise DownloadError("Content-length after streaming reached maximum allowed value of %s" %
            max_content_length)

     # update the resource metadata in CKAN if the resource has changed
     if resource.get('hash') != hash:
         resource['hash'] = hash
         try:
             # This may fail for archiver.update() as a result of the resource
             # not yet existing, but is necessary for dependant extensions.
             _update_resource(context, resource)
         except:
             pass

    return {'length': length,
            'hash' : hash,
            'headers': headers,
            'saved_file': saved_file}

@celery.task(name = "archiver.clean")
def clean():
    """
    Remove all archived resources.
    """
    logger = clean.get_logger()
    logger.error("clean task not implemented yet")

@celery.task(name = "archiver.update")
def update(context, data):
    try:
        data = json.loads(data)
        context = json.loads(context)
        result = _update(context, data)
        return result
    except Exception, e:
        update_task_status(context, {
            'entity_id': data['id'],
            'entity_type': u'resource',
            'task_type': 'archiver',
            'key': u'celery_task_id',
            'value': unicode(update.request.id),
            'error': '%s: %s' % (e.__class__.__name__,  unicode(e)),
            'stack': traceback.format_exc(),
            'last_updated': datetime.now().isoformat()
        })
        raise

def _update(context, data):
    """
    Link check and archive the given resource.

    Returns a JSON dict:

        {
            'resource': the updated resource dict,
            'file_path': path to archived file (if archive successful), or None
        }
    """
    logger = update.get_logger()
    rid = data.pop(u'revision_id')
    api_url = urlparse.urljoin(context['site_url'], 'api/action')

    # check that archive directory exists
    if not os.path.exists(settings.ARCHIVE_DIR):
        logger.info("Creating archive directory: %s" % settings.ARCHIVE_DIR)
        os.mkdir(settings.ARCHIVE_DIR)

    if not data:
        raise ArchiverError('Resource not found')

    result = None
    try:
        result = download(context, data)
        if result is None:
            raise Exception("Download failed")
    except Exception, downloaderr:
        if hasattr(settings, 'RETRIES') and settings.RETRIES:
            update.retry(args=(json.dumps(context), json.dumps(data)), exc=downloaderr)
        else:
            return

    # Check here whether we want to upload this content to webstore before
    # archiving
    # TODO: Possibly run this as a sub-task
    # Ticket #1684 suggests we should receive this config rather than having it in the
    # archiver configuration. In this case the URL would be passed through in the context
    # and this should be enough to decide to archive the data there.
    if settings.UPLOAD_TO_WEBSTORE and settings.WEBSTORE_URL:
        content_type = _clean_content_type( result['headers'].get('content-type', '') )
        if content_type in WEBSTORE_DATA_FORMATS or context.get('format', '') in WEBSTORE_DATA_FORMATS:
            try:
                context['webstore_url'] = settings.WEBSTORE_URL
                logger.info("Attempting to upload content to webstore: %s" % context['webstore_url'])
                upload_content( context, data, result )
            except Exception, eUpload:
                logger.error( eUpload )

                if hasattr(settings, 'RETRIES') and settings.RETRIES:
                    data[u'revision_id'] = rid # put this back as we'll need it next time
                    update.retry(args=(json.dumps(context), json.dumps(data)), exc=e)

    logger.info("Attempting to archive resource: %s" % data['url'])
    file_path = archive_resource(context, data, logger, result)

    return json.dumps({
        'resource': data,
        'file_path': file_path
    })

@celery.task(name = "archiver.link_checker")
def link_checker(context, data):
    """
    Check that the resource's url is valid, and accepts a HEAD request.

    Returns a json dict of the headers of the request
    """
    data = json.loads(data)
    url_timeout = data.get('url_timeout', 30)

    success = True
    error_message = ''
    headers = {}

    # Find out if it has unicode characters, and if it does, quote them
    # so we are left with an ascii string
    try:
        url = data['url'].decode('ascii')
    except:
        parts = list(urlparse.urlparse(url))
        parts[2] = urllib.quote(parts[2].encode('utf-8'))
        url = urlparse.urlunparse(parts)
    url = str(url)

    # parse url
    parsed_url = urlparse.urlparse(url)
    # Check we aren't using any schemes we shouldn't be
    allowed_schemes = ['http', 'https', 'ftp']
    if not parsed_url.scheme in allowed_schemes:
        raise LinkCheckerError("Invalid url scheme")
    # check that query string is valid
    # see: http://trac.ckan.org/ticket/318
    # TODO: check urls with a better validator?
    #       eg: ll.url (http://www.livinglogic.de/Python/url/Howto.html)?
    elif any(['/' in parsed_url.query, ':' in parsed_url.query]):
        raise LinkCheckerError("Invalid URL")
    else:
        # Send a head request
        try:
            res = requests.head(url, timeout = url_timeout)
            headers = res.headers
        except ValueError, ve:
            raise LinkCheckerError("Invalid URL")
        if res.error:
            if res.status_code in HTTP_ERROR_CODES:
                error_message = HTTP_ERROR_CODES[res.status_code]
            else:
                error_message = "URL unobtainable"
            raise LinkCheckerError(error_message)

    return json.dumps(headers)

def archive_resource(context, resource, logger, result=None, url_timeout = 30):
    """
    Archive the given resource.
    """
    if result['length']:
        dir = os.path.join(settings.ARCHIVE_DIR, resource['id'])
        if not os.path.exists(dir):
            os.mkdir(dir)
        # try to get a file name from the url
        parsed_url = urlparse.urlparse(resource.get('url'))
        try:
            file_name = parsed_url.path.split('/')[-1] or 'resource'
        except:
            file_name = "resource"
        saved_file = os.path.join(dir, file_name)
        os.rename(result['saved_file'], saved_file)

        # update the resource object: set cache_url and cache_last_updated
        if context.get('cache_url_root'):
            cache_url = urlparse.urljoin(
                context['cache_url_root'], '%s/%s' % (resource['id'], file_name)
            )
            if resource.get('cache_url') != cache_url:
                resource['cache_url'] = cache_url
                resource['cache_last_updated'] = datetime.now().isoformat()
                _update_resource(context, resource)

    return saved_file

def _save_resource(resource, response, max_file_size, chunk_size = 1024*16):
    """
    Write the response content to disk.

    Returns a tuple:

        (file length: int, content hash: string, saved file path: string)
    """
    resource_hash = hashlib.sha1()
    length = 0
    saved_file = None

    #tmp_resource_file = os.path.join(settings.ARCHIVE_DIR, 'archive_%s' % os.getpid())
    fd, tmp_resource_file_path = tempfile.mkstemp()

    with open(tmp_resource_file_path, 'wb') as fp:
        for chunk in response.iter_content(chunk_size = chunk_size, decode_unicode=False):
            fp.write(chunk)
            length += len(chunk)
            resource_hash.update(chunk)

            if length >= max_file_size:
                break

    os.close(fd)

    content_hash = unicode(resource_hash.hexdigest())
    return length, content_hash, tmp_resource_file_path

def _update_resource(context, resource):
    """
    Use CKAN API to update the given resource.
    If cannot update, records this fact in the task_status table.

    Returns the content of the response.

    """
    api_url = urlparse.urljoin(context['site_url'], 'api/action')
    resource['last_modified'] = datetime.now().isoformat()
    post_data = json.dumps(resource)
    res = requests.post(
        api_url + '/resource_update', post_data,
        headers = {'Authorization': context['apikey'],
                   'Content-Type': 'application/json'}
    )

    if res.status_code == 200:
        return res.content
    else:
        raise CkanError('ckan failed to update resource, status_code (%s)'
                        % res.status_code)

def update_task_status(context, data):
    """
    Use CKAN API to update the task status. The data parameter
    should be a dict representing one row in the task_status table.

    Returns the content of the response.
    """
    api_url = urlparse.urljoin(context['site_url'], 'api/action')
    res = requests.post(
        api_url + '/task_status_update', json.dumps(data),
        headers = {'Authorization': context['site_user_apikey'],
                   'Content-Type': 'application/json'}
    )
    if res.status_code == 200:
        return res.content
    else:
        raise CkanError('ckan failed to update task_status, status_code (%s), error %s'  % (res.status_code, res.content))

<|endoftext|>


<commit_before>"""
An extension to detect blacklisted status and restart Scrapoxy instances.

You must fill :
API_SCRAPOXY with the commander URL
API_SCRAPOXY_PASSWORD with credential
BLACKLIST_HTTP_STATUS_CODES with the list of blacklisted HTTP codes
"""

# -*- coding: utf-8 -*-

from scrapy.exceptions import IgnoreRequest
from scrapoxy.commander import Commander

import logging
import random
import time

class BlacklistError(Exception):
    def __init__(self, response, message, *args, **kwargs):
        super(BlacklistError, self).__init__(*args, **kwargs)

        self.response = response
        self.message = message

    def __str__(self):
        return self.message

class BlacklistDownloaderMiddleware(object):

    def __init__(self, crawler):
        """Access the settings of the crawler to connect to Scrapoxy.
        """
        self._http_status_codes = crawler.settings.get('BLACKLIST_HTTP_STATUS_CODES', [503])
        self._sleep_min = crawler.settings.get('SCRAPOXY_SLEEP_MIN', 60)
        self._sleep_max = crawler.settings.get('SCRAPOXY_SLEEP_MAX', 180)

        self._commander = Commander(
            crawler.settings.get('API_SCRAPOXY'),
            crawler.settings.get('API_SCRAPOXY_PASSWORD')
        )

    @classmethod
    def from_crawler(cls, crawler):
        """Call constructor with crawler parameters
        """
        return cls(crawler)

    def process_response(self, request, response, spider):
        """Detect blacklisted response and stop the instance if necessary.
        """
        try:
            if response.status in self._http_status_codes:
                raise BlacklistError(response, u'HTTP status '.format(response.status))

            return response

        except BlacklistError as ex:
            spider.log(u'Ignoring Blacklisted response {0}: {1}'.format(response.url, ex.message), level=logging.DEBUG)

            name = response.headers.get(u'x-cache-proxyname')
            self._stop_and_sleep(spider, name)

            raise IgnoreRequest()

    def _stop_and_sleep(self, spider, name):
        if name:
            alive = self._commander.stop_instance(name)
            if alive < 0:
                spider.log(u'Remove: cannot find instance {}'.format(name), level=logging.ERROR)
            elif alive == 0:
                spider.log(u'Remove: instance removed (no instance remaining)', level=logging.WARNING)
            else:
                spider.log(u'Remove: instance removed ({} instances remaining)'.format(alive), level=logging.DEBUG)
        else:
            spider.log(u'Cannot find instance name in headers', level=logging.ERROR)

        delay = random.randrange(self._sleep_min, self._sleep_max)
        spider.log(u'Sleeping {} seconds'.format(delay), level=logging.INFO)
        time.sleep(delay)
<commit_msg>Logging fix on BlacklistError<commit_after>"""
An extension to detect blacklisted status and restart Scrapoxy instances.

You must fill :
API_SCRAPOXY with the commander URL
API_SCRAPOXY_PASSWORD with credential
BLACKLIST_HTTP_STATUS_CODES with the list of blacklisted HTTP codes
"""

# -*- coding: utf-8 -*-

from scrapy.exceptions import IgnoreRequest
from scrapoxy.commander import Commander

import logging
import random
import time

class BlacklistError(Exception):
    def __init__(self, response, message, *args, **kwargs):
        super(BlacklistError, self).__init__(*args, **kwargs)

        self.response = response
        self.message = message

    def __str__(self):
        return self.message

class BlacklistDownloaderMiddleware(object):

    def __init__(self, crawler):
        """Access the settings of the crawler to connect to Scrapoxy.
        """
        self._http_status_codes = crawler.settings.get('BLACKLIST_HTTP_STATUS_CODES', [503])
        self._sleep_min = crawler.settings.get('SCRAPOXY_SLEEP_MIN', 60)
        self._sleep_max = crawler.settings.get('SCRAPOXY_SLEEP_MAX', 180)

        self._commander = Commander(
            crawler.settings.get('API_SCRAPOXY'),
            crawler.settings.get('API_SCRAPOXY_PASSWORD')
        )

    @classmethod
    def from_crawler(cls, crawler):
        """Call constructor with crawler parameters
        """
        return cls(crawler)

    def process_response(self, request, response, spider):
        """Detect blacklisted response and stop the instance if necessary.
        """
        try:
            if response.status in self._http_status_codes:
                raise BlacklistError(response, u'HTTP status {}'.format(response.status))

            return response

        except BlacklistError as ex:
            spider.log(u'Ignoring Blacklisted response {0}: {1}'.format(response.url, ex.message), level=logging.DEBUG)

            name = response.headers.get(u'x-cache-proxyname')
            self._stop_and_sleep(spider, name)

            raise IgnoreRequest()

    def _stop_and_sleep(self, spider, name):
        if name:
            alive = self._commander.stop_instance(name)
            if alive < 0:
                spider.log(u'Remove: cannot find instance {}'.format(name), level=logging.ERROR)
            elif alive == 0:
                spider.log(u'Remove: instance removed (no instance remaining)', level=logging.WARNING)
            else:
                spider.log(u'Remove: instance removed ({} instances remaining)'.format(alive), level=logging.DEBUG)
        else:
            spider.log(u'Cannot find instance name in headers', level=logging.ERROR)

        delay = random.randrange(self._sleep_min, self._sleep_max)
        spider.log(u'Sleeping {} seconds'.format(delay), level=logging.INFO)
        time.sleep(delay)
<|endoftext|>