<commit_before># -*- coding: utf-8 -*-
import json
import os
import re
import sys
from datetime import datetime, timedelta

import mwparserfromhell
os.environ['PYWIKIBOT2_DIR'] = os.path.dirname(os.path.realpath(__file__))
import pywikibot
from pywikibot.data.api import Request

from config import *

site = pywikibot.Site()
site.login()

config_page = pywikibot.Page(site, config_page_name)
cfg = config_page.text
cfg = json.loads(cfg)
print(json.dumps(cfg, indent=4, ensure_ascii=False))

if not cfg['enable']:
    exit('disabled\n')

def converttitle(title):
    oldtitle = title
    r = Request(site=site, parameters={
        'action': 'query',
        'titles': title,
        'redirects': 1,
        'converttitles': 1
    })
    data = r.submit()
    title = list(data['query']['pages'].values())[0]['title']
    mode = []
    if 'redirects' in data['query']:  # 重定向
        mode.append('redirects')
    if 'converted' in data['query']:  # 繁簡轉換
        mode.append('converted')
    if 'normalized' in data['query']:  # 命名空間
        mode.append('normalized')
    if 'redirects' not in mode:
        page = pywikibot.Page(site, title)
        if page.exists() and (page.content_model != 'wikitext' or re.search(r'{{\s*[vaictumr]fd', page.text, flags=re.I)):
            mode.append('vfd_on_source')
    else:
        page = pywikibot.Page(site, oldtitle)
        if page.exists() and (page.content_model != 'wikitext' or re.search(r'{{\s*[vaictumr]fd', page.text, flags=re.I)):
            mode.append('vfd_on_source')
        page = pywikibot.Page(site, title)
        if page.exists() and (page.content_model != 'wikitext' or re.search(r'{{\s*[vaictumr]fd', page.text, flags=re.I)):
            mode.append('vfd_on_target')
    if not 'vfd_on_source' in mode and not 'vfd_on_target' in mode:
        mode.append('no_vfd')
    return {'title': title, 'mode': mode}

def appendComment(text, mode):
    text = text.strip()
    if 'A2093064-bot' not in text:
        if 'fix' in mode:
            comment = []
            if 'redirects' in mode and isinstance(cfg['comment_fix']['redirects'], str):
                comment.append(cfg['comment_fix']['redirects'])
                print('\tcomment_fix - redirects')
            if 'converted' in mode and isinstance(cfg['comment_fix']['converted'], str):
                comment.append(cfg['comment_fix']['converted'])
                print('\tcomment_fix - converted')
            if 'normalized' in mode and isinstance(cfg['comment_fix']['normalized'], str):
                comment.append(cfg['comment_fix']['normalized'])
                print('\tcomment_fix - normalized')
            if len(comment) > 0:
                text += '\n' + \
                    cfg['comment_fix']['main'].format(''.join(comment))
                print('\tcomment_fix - redirects')
        if 'no_vfd' in mode:
            text += '\n' + cfg['comment_vfd']
            print('\tcomment_vfd')
    text += '\n\n'
    return text

def fix(pagename):
    if re.search(r'\d{4}/\d{2}/\d{2}', pagename):
        pagename = 'Wikipedia:頁面存廢討論/記錄/' + pagename

    print('-' * 50)
    print('running for ' + pagename)

    afdpage = pywikibot.Page(site, pagename)
    text = afdpage.text

    wikicode = mwparserfromhell.parse(text)

    changes = []
    for secid, section in enumerate(list(wikicode.get_sections())):
        if secid == 0:
            continue
        title = str(section.get(0).title)
        print(secid, title)
        if re.search(r'{{\s*(delh|TalkendH)\s*\|', str(section), re.IGNORECASE) != None:
            print('  closed, skip')
            continue

        m = re.search(r'^\[\[([^\]]+)\]\]$', title, re.IGNORECASE)
        if m != None:
            title = m.group(1)
            start = ''
            if title[0] == ':':
                start = ':'
                title = title[1:]

            mode = []

            convert = converttitle(title)
            if (('redirects' in convert['mode'] and 'vfd_on_target' in convert['mode'])
                    or (not 'redirects' in convert['mode'])):
                title = convert['title']
                mode.append('fix')
            mode += convert['mode']
            print('    ', mode)

            title = '[[' + start + title + ']]'

            if str(section.get(0).title) != title:
                if str(section.get(0).title).replace('_', ' ') != title:
                    section.insert(
                        1, '\n{{formerly|' + str(section.get(0).title) + '}}')
                    pass
                print('  set new title = ' + title)
                section.get(0).title = title
            newtext = appendComment(str(section), mode)
            changes.append([secid, newtext])
            continue

        m = re.search(
            r'^(\[\[[^\]]+\]\][、，])+\[\[[^\]]+\]\]$', title, re.IGNORECASE)
        if m != None:
            titlelist = m.group(0).replace(']]，[[', ']]、[[').split('、')
            newtitlelist = []
            mode = []
            for title in titlelist:
                if title.startswith('[[') and title.endswith(']]'):
                    title = title[2:-2]

                    if title[0] == ':':
                        title = title[1:]

                    convert = converttitle(title)
                    if (('redirects' in convert['mode'] and 'vfd_on_target' in convert['mode'])
                            or (not 'redirects' in convert['mode'])):
                        title = convert['title']
                        mode.append('fix')
                    mode += convert['mode']

                    newtitlelist.append(title)
                else:
                    print('  wrong title: ' + title)
                    return
            title = '{{al|' + '|'.join(newtitlelist) + '}}'
            if str(section.get(0).title) != title:
                print('  set new title = ' + title)
                section.get(0).title = title
            newtext = appendComment(str(section), mode)
            changes.append([secid, newtext])
            continue

        m = re.search(r'^{{al\|([^\]]+\|)+[^\]]+}}$', title, re.IGNORECASE)
        if m != None:
            titlelist = m.group(0)[5:-2].split('|')
            newtitlelist = []
            mode = []
            for title in titlelist:
                if title[0] == ':':
                    title = title[1:]

                convert = converttitle(title)
                if (('redirects' in convert['mode'] and 'vfd_on_target' in convert['mode'])
                        or (not 'redirects' in convert['mode'])):
                    title = convert['title']
                    mode.append('fix')
                mode += convert['mode']

                newtitlelist.append(title)
            title = '{{al|' + '|'.join(newtitlelist) + '}}'
            if str(section.get(0).title) != title:
                print('  set new title = ' + title)
                section.get(0).title = title
            newtext = appendComment(str(section), mode)
            changes.append([secid, newtext])
            continue

        print('  unknown format, skip')

    for change in changes:
        wikicode = mwparserfromhell.parse(text)
        sections = list(wikicode.get_sections())
        secid, newtext = change
        wikicode.replace(sections[secid], newtext)
        text = str(wikicode)

    if re.sub(r'\s+', '', afdpage.text) == re.sub(r'\s+', '', text):
        print('  nothing changed')
        return

    pywikibot.showDiff(afdpage.text, text)
    summary = cfg['summary']
    print(summary)
    afdpage.text = text
    afdpage.save(summary=summary, minor=False)

if len(sys.argv) >= 2:
    pagename = sys.argv[1]
    fix(pagename)
else:
    print('run past {} days'.format(cfg['run_past_days']))
    for delta in range(cfg['run_past_days']):
        rundate = datetime.now() - timedelta(days=delta)
        pagename = rundate.strftime('%Y/%m/%d')
        fix(pagename)
<commit_msg>afd-section-title: vfd_on_source for non-existent page<commit_after># -*- coding: utf-8 -*-
import json
import os
import re
import sys
from datetime import datetime, timedelta

import mwparserfromhell
os.environ['PYWIKIBOT2_DIR'] = os.path.dirname(os.path.realpath(__file__))
import pywikibot
from pywikibot.data.api import Request

from config import *

site = pywikibot.Site()
site.login()

config_page = pywikibot.Page(site, config_page_name)
cfg = config_page.text
cfg = json.loads(cfg)
print(json.dumps(cfg, indent=4, ensure_ascii=False))

if not cfg['enable']:
    exit('disabled\n')

def converttitle(title):
    oldtitle = title
    r = Request(site=site, parameters={
        'action': 'query',
        'titles': title,
        'redirects': 1,
        'converttitles': 1
    })
    data = r.submit()
    title = list(data['query']['pages'].values())[0]['title']
    mode = []
    if 'redirects' in data['query']:  # 重定向
        mode.append('redirects')
    if 'converted' in data['query']:  # 繁簡轉換
        mode.append('converted')
    if 'normalized' in data['query']:  # 命名空間
        mode.append('normalized')
    if 'redirects' not in mode:
        page = pywikibot.Page(site, title)
        if not page.exists():
            mode.append('vfd_on_source')
        if page.exists() and (page.content_model != 'wikitext' or re.search(r'{{\s*[vaictumr]fd', page.text, flags=re.I)):
            mode.append('vfd_on_source')
    else:
        page = pywikibot.Page(site, oldtitle)
        if page.exists() and (page.content_model != 'wikitext' or re.search(r'{{\s*[vaictumr]fd', page.text, flags=re.I)):
            mode.append('vfd_on_source')
        page = pywikibot.Page(site, title)
        if page.exists() and (page.content_model != 'wikitext' or re.search(r'{{\s*[vaictumr]fd', page.text, flags=re.I)):
            mode.append('vfd_on_target')
    if not 'vfd_on_source' in mode and not 'vfd_on_target' in mode:
        mode.append('no_vfd')
    return {'title': title, 'mode': mode}

def appendComment(text, mode):
    text = text.strip()
    if 'A2093064-bot' not in text:
        if 'fix' in mode:
            comment = []
            if 'redirects' in mode and isinstance(cfg['comment_fix']['redirects'], str):
                comment.append(cfg['comment_fix']['redirects'])
                print('\tcomment_fix - redirects')
            if 'converted' in mode and isinstance(cfg['comment_fix']['converted'], str):
                comment.append(cfg['comment_fix']['converted'])
                print('\tcomment_fix - converted')
            if 'normalized' in mode and isinstance(cfg['comment_fix']['normalized'], str):
                comment.append(cfg['comment_fix']['normalized'])
                print('\tcomment_fix - normalized')
            if len(comment) > 0:
                text += '\n' + \
                    cfg['comment_fix']['main'].format(''.join(comment))
                print('\tcomment_fix - redirects')
        if 'no_vfd' in mode:
            text += '\n' + cfg['comment_vfd']
            print('\tcomment_vfd')
    text += '\n\n'
    return text

def fix(pagename):
    if re.search(r'\d{4}/\d{2}/\d{2}', pagename):
        pagename = 'Wikipedia:頁面存廢討論/記錄/' + pagename

    print('-' * 50)
    print('running for ' + pagename)

    afdpage = pywikibot.Page(site, pagename)
    text = afdpage.text

    wikicode = mwparserfromhell.parse(text)

    changes = []
    for secid, section in enumerate(list(wikicode.get_sections())):
        if secid == 0:
            continue
        title = str(section.get(0).title)
        print(secid, title)
        if re.search(r'{{\s*(delh|TalkendH)\s*\|', str(section), re.IGNORECASE) != None:
            print('  closed, skip')
            continue

        m = re.search(r'^\[\[([^\]]+)\]\]$', title, re.IGNORECASE)
        if m != None:
            title = m.group(1)
            start = ''
            if title[0] == ':':
                start = ':'
                title = title[1:]

            mode = []

            convert = converttitle(title)
            if (('redirects' in convert['mode'] and 'vfd_on_target' in convert['mode'])
                    or (not 'redirects' in convert['mode'])):
                title = convert['title']
                mode.append('fix')
            mode += convert['mode']
            print('    ', mode)

            title = '[[' + start + title + ']]'

            if str(section.get(0).title) != title:
                if str(section.get(0).title).replace('_', ' ') != title:
                    section.insert(
                        1, '\n{{formerly|' + str(section.get(0).title) + '}}')
                    pass
                print('  set new title = ' + title)
                section.get(0).title = title
            newtext = appendComment(str(section), mode)
            changes.append([secid, newtext])
            continue

        m = re.search(
            r'^(\[\[[^\]]+\]\][、，])+\[\[[^\]]+\]\]$', title, re.IGNORECASE)
        if m != None:
            titlelist = m.group(0).replace(']]，[[', ']]、[[').split('、')
            newtitlelist = []
            mode = []
            for title in titlelist:
                if title.startswith('[[') and title.endswith(']]'):
                    title = title[2:-2]

                    if title[0] == ':':
                        title = title[1:]

                    convert = converttitle(title)
                    if (('redirects' in convert['mode'] and 'vfd_on_target' in convert['mode'])
                            or (not 'redirects' in convert['mode'])):
                        title = convert['title']
                        mode.append('fix')
                    mode += convert['mode']

                    newtitlelist.append(title)
                else:
                    print('  wrong title: ' + title)
                    return
            title = '{{al|' + '|'.join(newtitlelist) + '}}'
            if str(section.get(0).title) != title:
                print('  set new title = ' + title)
                section.get(0).title = title
            newtext = appendComment(str(section), mode)
            changes.append([secid, newtext])
            continue

        m = re.search(r'^{{al\|([^\]]+\|)+[^\]]+}}$', title, re.IGNORECASE)
        if m != None:
            titlelist = m.group(0)[5:-2].split('|')
            newtitlelist = []
            mode = []
            for title in titlelist:
                if title[0] == ':':
                    title = title[1:]

                convert = converttitle(title)
                if (('redirects' in convert['mode'] and 'vfd_on_target' in convert['mode'])
                        or (not 'redirects' in convert['mode'])):
                    title = convert['title']
                    mode.append('fix')
                mode += convert['mode']

                newtitlelist.append(title)
            title = '{{al|' + '|'.join(newtitlelist) + '}}'
            if str(section.get(0).title) != title:
                print('  set new title = ' + title)
                section.get(0).title = title
            newtext = appendComment(str(section), mode)
            changes.append([secid, newtext])
            continue

        print('  unknown format, skip')

    for change in changes:
        wikicode = mwparserfromhell.parse(text)
        sections = list(wikicode.get_sections())
        secid, newtext = change
        wikicode.replace(sections[secid], newtext)
        text = str(wikicode)

    if re.sub(r'\s+', '', afdpage.text) == re.sub(r'\s+', '', text):
        print('  nothing changed')
        return

    pywikibot.showDiff(afdpage.text, text)
    summary = cfg['summary']
    print(summary)
    afdpage.text = text
    afdpage.save(summary=summary, minor=False)

if len(sys.argv) >= 2:
    pagename = sys.argv[1]
    fix(pagename)
else:
    print('run past {} days'.format(cfg['run_past_days']))
    for delta in range(cfg['run_past_days']):
        rundate = datetime.now() - timedelta(days=delta)
        pagename = rundate.strftime('%Y/%m/%d')
        fix(pagename)
<|endoftext|>


<commit_before>import asyncio
import datetime
import logging
from abc import abstractmethod

import collections

from ..core.base import AbstractNamedEntity
from ..utils import import_name

class AbstractWorker(AbstractNamedEntity):
    @abstractmethod  # pragma: no cover
    async def start(self):
        raise NotImplementedError()

    @abstractmethod  # pragma: no cover
    async def stop(self):
        raise NotImplementedError()

    @abstractmethod  # pragma: no cover
    async def status(self):
        raise NotImplementedError()

class Worker(AbstractWorker):
    """ Worker
        config:
            run: str optional coroutine
            autorun: bool with self.init
            persist: bool if need rerun
            logger: str
            sleep: int time in seconds for sleep between rerun
            sleep_start: int time in seconds for sleep before run
            crontab: str rule as cron. Every 5 minutes "*/5 * * * *"
            input: str.path to instance of AbstractReader
            output: str.path to instance of AbstractWriter
    """
    async def init(self):
        self._started_at = None
        self._stopped_at = None
        self._future = None
        self.counter = collections.Counter()

        if self.config.get('run'):
            run = import_name(self.config.run)
            self.run = lambda value=None: run(self, value)

        if self.config.get('input') or self.config.get('output'):
            self._persist = True
        else:
            self._persist = self.config.get('persist')

        self._sleep = self.config.get('sleep')
        self._sleep_start = self.config.get('sleep_start')

        self._crontab = self.config.get('crontab')
        if self._crontab:
            CronTab = import_name('crontab.CronTab')
            self._crontab = CronTab(self._crontab)
            self._persist = True

        self.logger = logging.getLogger(self.config.get('logger', __name__))

        if self.config.get('autorun'):
            self.context.on_start.append(self.start)
        self.context.on_stop.append(self.stop)

    @property
    def input(self):
        return self.context[self.config.get('input')]

    @property
    def output(self):
        return self.context[self.config.get('output')]

    async def runner(self):
        try:
            if self._sleep_start:
                await asyncio.sleep(self._sleep_start, loop=self.loop)
            if self._crontab is not None:
                await asyncio.sleep(self._crontab.next(), loop=self.loop)
            while True:
                try:
                    if self.input is not None:
                        args = (await self.input.get(),)
                    else:
                        args = ()
                    self.counter['run'] += 1
                    result = await self.run(*args)
                    self.counter['done'] += 1
                    if self.output is not None:
                        await self.output.put(result)
                except asyncio.CancelledError:
                    raise
                except BaseException:
                    self.counter['error'] += 1
                    self.logger.exception('ERROR {} {}'.format(
                        self.name,
                        self.config.get('run', type(self)),
                    ))
                if not self._persist:
                    return
                if self._sleep:
                    await asyncio.sleep(self._sleep, loop=self.loop)
                if self._crontab is not None:
                    await asyncio.sleep(self._crontab.next(), loop=self.loop)
                if not self._persist:  # not forced stop
                    return
        finally:
            self._stopped_at = datetime.datetime.now()

    async def run(self, value=None):
        raise NotImplementedError()

    @property
    def started_at(self):
        return getattr(self, '_started_at', None)

    @property
    def stopped_at(self):
        return getattr(self, '_stopped_at', None)

    def running(self):
        if not hasattr(self, '_future'):
            return False
        elif self._future is None:
            return False
        else:
            return not self._future.done()

    async def start(self):
        if not self.running():
            self._started_at = datetime.datetime.now()
            self._stopped_at = None
            self._future = self.loop.create_task(self.runner())

    async def stop(self, force=True):
        if not self.running():
            pass
        elif force:
            self._future.cancel()
            self._stopped_at = datetime.datetime.now()
            try:
                await self._future
            except asyncio.CancelledError:
                pass
        else:
            self._persist = False
            await self._future

    async def status(self):
        return {
            'started_at': self.started_at,
            'stopped_at': self.stopped_at,
            'running': self.running(),
            **self.counter,
        }
<commit_msg>flag is_sleep in worker<commit_after>import asyncio
import datetime
import logging
from abc import abstractmethod

import collections

from ..core.base import AbstractNamedEntity
from ..utils import import_name

class AbstractWorker(AbstractNamedEntity):
    @abstractmethod  # pragma: no cover
    async def start(self):
        raise NotImplementedError()

    @abstractmethod  # pragma: no cover
    async def stop(self):
        raise NotImplementedError()

    @abstractmethod  # pragma: no cover
    async def status(self):
        raise NotImplementedError()

class Worker(AbstractWorker):
    """ Worker
        config:
            run: str optional coroutine
            autorun: bool with self.init
            persist: bool if need rerun
            logger: str
            sleep: int time in seconds for sleep between rerun
            sleep_start: int time in seconds for sleep before run
            crontab: str rule as cron. Every 5 minutes "*/5 * * * *"
            input: str.path to instance of AbstractReader
            output: str.path to instance of AbstractWriter
    """
    async def init(self):
        self._started_at = None
        self._stopped_at = None
        self._is_sleep = None
        self._future = None
        self.counter = collections.Counter()

        if self.config.get('run'):
            run = import_name(self.config.run)
            self.run = lambda value=None: run(self, value)

        if self.config.get('input') or self.config.get('output'):
            self._persist = True
        else:
            self._persist = self.config.get('persist')

        self._sleep = self.config.get('sleep')
        self._sleep_start = self.config.get('sleep_start')

        self._crontab = self.config.get('crontab')
        if self._crontab:
            CronTab = import_name('crontab.CronTab')
            self._crontab = CronTab(self._crontab)
            self._persist = True

        self.logger = logging.getLogger(self.config.get('logger', __name__))

        if self.config.get('autorun'):
            self.context.on_start.append(self.start)
        self.context.on_stop.append(self.stop)

    @property
    def input(self):
        return self.context[self.config.get('input')]

    @property
    def output(self):
        return self.context[self.config.get('output')]

    async def runner(self):
        self._is_sleep = True
        try:
            if self._sleep_start:
                await asyncio.sleep(self._sleep_start, loop=self.loop)
            while True:
                if self._crontab is not None:
                    await asyncio.sleep(self._crontab.next(), loop=self.loop)
                try:
                    self._is_sleep = False
                    if self.input is not None:
                        args = (await self.input.get(),)
                    else:
                        args = ()
                    self.counter['run'] += 1
                    result = await self.run(*args)
                    self.counter['done'] += 1
                    if self.output is not None:
                        await self.output.put(result)
                except asyncio.CancelledError:
                    raise
                except BaseException:
                    self.counter['error'] += 1
                    self.logger.exception('ERROR {} {}'.format(
                        self.name,
                        self.config.get('run', type(self)),
                    ))
                self._is_sleep = True
                if not self._persist:
                    return
                if self._sleep:
                    await asyncio.sleep(self._sleep, loop=self.loop)
        finally:
            self._stopped_at = datetime.datetime.now()

    async def run(self, value=None):
        raise NotImplementedError()

    @property
    def started_at(self):
        return getattr(self, '_started_at', None)

    @property
    def stopped_at(self):
        return getattr(self, '_stopped_at', None)

    def running(self):
        if not hasattr(self, '_future'):
            return False
        elif self._future is None:
            return False
        else:
            return not self._future.done()

    async def start(self):
        if not self.running():
            self._started_at = datetime.datetime.now()
            self._stopped_at = None
            self._future = self.loop.create_task(self.runner())

    async def stop(self, force=True):
        if not self.running():
            pass
        elif force or self._is_sleep:
            self._future.cancel()
            self._stopped_at = datetime.datetime.now()
            try:
                await self._future
            except asyncio.CancelledError:
                pass
        else:
            self._persist = False
            await self._future
        self._is_sleep = None

    async def status(self):
        return {
            'started_at': self.started_at,
            'stopped_at': self.stopped_at,
            'running': self.running(),
            'is_sleep': self._is_sleep,
            **self.counter,
        }
<|endoftext|>


<commit_before>import os
import json
from flask import Blueprint, request
from werkzeug import secure_filename
from werkzeug.exceptions import BadRequest
from apikit import obj_or_404, jsonify

from aleph.core import upload_folder
from aleph.events import log_event
from aleph.ingest import ingest_document
from aleph.model import Collection, Document
from aleph.model.common import make_textid
from aleph.model.validate import validate
from aleph.util import checksum

blueprint = Blueprint('ingest_api', __name__)

@blueprint.route('/api/1/collections/<int:collection_id>/ingest',
                 methods=['POST', 'PUT'])
def ingest_upload(collection_id):
    collection = obj_or_404(Collection.by_id(collection_id))
    request.authz.require(request.authz.collection_write(collection.id))
    log_event(request)
    crawler_run = make_textid()

    try:
        meta = json.loads(request.form.get('meta', '{}'))
    except Exception as ex:
        raise BadRequest(unicode(ex))

    documents = []
    for storage in request.files.values():
        sec_fn = os.path.join(upload_folder, secure_filename(storage.filename))
        storage.save(sec_fn)
        content_hash = checksum(sec_fn)
        document = Document.by_keys(collection_id=collection.id,
                                    content_hash=content_hash)
        document.crawler = 'user_upload:%s' % request.authz.role.id
        document.crawler_run = crawler_run
        document.mime_type = storage.mimetype
        document.file_name = storage.filename

        try:
            meta = json.loads(request.form.get('meta', '{}'))
            validate(meta, 'metadata.json#')
            document.meta.update(meta)
        except Exception as ex:
            raise BadRequest(unicode(ex))

        ingest_document(document, sec_fn, user_queue=True)
        os.unlink(sec_fn)
        documents.append(document)
    return jsonify({'status': 'ok', 'documents': documents})
<commit_msg>Missed one :( <commit_after>import os
import json
from flask import Blueprint, request
from werkzeug import secure_filename
from werkzeug.exceptions import BadRequest
from apikit import obj_or_404, jsonify

from aleph.core import upload_folder
from aleph.events import log_event
from aleph.ingest import ingest_document
from aleph.model import Collection, Document
from aleph.model.common import make_textid
from aleph.model.validate import validate
from aleph.util import checksum

blueprint = Blueprint('ingest_api', __name__)

@blueprint.route('/api/1/collections/<int:collection_id>/ingest',
                 methods=['POST', 'PUT'])
def ingest_upload(collection_id):
    collection = obj_or_404(Collection.by_id(collection_id))
    request.authz.require(request.authz.collection_write(collection.id))
    log_event(request)
    crawler_run = make_textid()

    try:
        meta = json.loads(request.form.get('meta', '{}'))
    except Exception as ex:
        raise BadRequest(unicode(ex))

    documents = []
    for storage in request.files.values():
        sec_fn = os.path.join(upload_folder, secure_filename(storage.filename))
        storage.save(sec_fn)
        content_hash = checksum(sec_fn)
        document = Document.by_keys(collection=collection,
                                    content_hash=content_hash)
        document.crawler = 'user_upload:%s' % request.authz.role.id
        document.crawler_run = crawler_run
        document.mime_type = storage.mimetype
        document.file_name = storage.filename

        try:
            meta = json.loads(request.form.get('meta', '{}'))
            validate(meta, 'metadata.json#')
            document.meta.update(meta)
        except Exception as ex:
            raise BadRequest(unicode(ex))

        ingest_document(document, sec_fn, user_queue=True)
        os.unlink(sec_fn)
        documents.append(document)
    return jsonify({'status': 'ok', 'documents': documents})
<|endoftext|>


<commit_before>class Algorithm():
	def __init__(self,data):
		self.data = data
		self.output = self.main()

	def main(self):
		return []<commit_msg>Schema now are received as init param<commit_after>class Algorithm():
	def __init__(self,data,schema):
		self.data = data
		self.schema = schema
		self.output = self.main()

	def main(self):
		return []<|endoftext|>


<commit_before>import socket
import threading
import logging

class GraphiteStore(object):
    def __init__(self,host,port,attempts=3):
        """
        Implements a metrics store interface that allows metrics to
        be persisted to Graphite. Raises a :class:`ValueError` on bad arguments.

        :Parameters:
            - `host` : The hostname of the graphite server.
            - `port` : The port of the graphite server
            - `attempts` (optional) : The number of re-connect retries before failing.
        """
        if not isinstance(host, (str,unicode)): raise ValueError, "Host must be a string!"
        if not isinstance(port, int): raise ValueError, "Port must be an integer!"
        if port <= 0: raise ValueError, "Port must be positive!"
        if attempts <= 1: raise ValueError, "Must have at least 1 attempt!"

        self.host = host
        self.port = port
        self.attempts = attempts
        self.sock_lock = threading.Lock()
        self.sock = self._create_socket()
        self.logger = logging.getLogger("statsite.graphite")

    def _create_socket(self):
        """Creates a socket and connects to the graphite server"""
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        sock.connect((self.host,self.port))
        return sock

    def _write_metric(self, metric):
        """Tries to write a string to the socket, reconnecting on any errors"""
        for attempt in xrange(self.attempts):
            try:
                self.sock.send(metric)
                break
            except socket.error:
                self.logger.exception("Failed to flush to Graphite!")
                self.sock = self._create_socket()

    def metrics_store(self, metrics):
        """
        Flushes the metrics provided to Graphite.

       :Parameters:
        - `metrics` : A list of (key,value,timestamp) tuples.
        """
        self.sock_lock.acquire()
        try:
            data = "\n".join(["%s %s %s" % metric for metric in metrics]) + "\n"
            self._write_metric(data)
        except:
            self.logger.exception("Failed to write out the metrics!")
        finally:
            self.sock_lock.release()

<commit_msg>Move constructing the graphite data outside the lock to minimize the time we spend holding the lock.<commit_after>import socket
import threading
import logging

class GraphiteStore(object):
    def __init__(self,host,port,attempts=3):
        """
        Implements a metrics store interface that allows metrics to
        be persisted to Graphite. Raises a :class:`ValueError` on bad arguments.

        :Parameters:
            - `host` : The hostname of the graphite server.
            - `port` : The port of the graphite server
            - `attempts` (optional) : The number of re-connect retries before failing.
        """
        if not isinstance(host, (str,unicode)): raise ValueError, "Host must be a string!"
        if not isinstance(port, int): raise ValueError, "Port must be an integer!"
        if port <= 0: raise ValueError, "Port must be positive!"
        if attempts <= 1: raise ValueError, "Must have at least 1 attempt!"

        self.host = host
        self.port = port
        self.attempts = attempts
        self.sock_lock = threading.Lock()
        self.sock = self._create_socket()
        self.logger = logging.getLogger("statsite.graphite")

    def _create_socket(self):
        """Creates a socket and connects to the graphite server"""
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        sock.connect((self.host,self.port))
        return sock

    def _write_metric(self, metric):
        """Tries to write a string to the socket, reconnecting on any errors"""
        for attempt in xrange(self.attempts):
            try:
                self.sock.send(metric)
                break
            except socket.error:
                self.logger.exception("Failed to flush to Graphite!")
                self.sock = self._create_socket()

    def metrics_store(self, metrics):
        """
        Flushes the metrics provided to Graphite.

       :Parameters:
        - `metrics` : A list of (key,value,timestamp) tuples.
        """
        # Construct the output
        data = "\n".join(["%s %s %s" % metric for metric in metrics]) + "\n"

        # Serialize writes to the socket
        self.sock_lock.acquire()
        try:
            self._write_metric(data)
        except:
            self.logger.exception("Failed to write out the metrics!")
        finally:
            self.sock_lock.release()

<|endoftext|>


<commit_before>from rest_framework import serializers
from drf_extra_fields.geo_fields import PointField as PointFieldSerializer

from django.contrib.auth.models import User
from django.core.exceptions import PermissionDenied
import django.db.models as models

from .models import Profile, Hospital, Ambulance, \
    AmbulancePermission, HospitalPermission

# Profile serializers

class AmbulancePermissionSerializer(serializers.ModelSerializer):

    ambulance_id = serializers.IntegerField(source='ambulance.id')
    ambulance_identifier = serializers.CharField(source='ambulance.identifier')

    class Meta:
        model = AmbulancePermission
        fields = ('ambulance_id', 'ambulance_identifier', 'can_read', 'can_write')
        read_only_fields = ('ambulance_id', 'ambulance__identifier', 'can_read', 'can_write')

class HospitalPermissionSerializer(serializers.ModelSerializer):

    hospital_id = serializers.IntegerField(source='hospital.id')
    hospital_name = serializers.CharField(source='hospital.name')

    class Meta:
        model = HospitalPermission
        fields = ('hospital_id', 'hospital_name', 'can_read', 'can_write')
        read_only_fields = ('hospital_id', 'hospital_name', 'can_read', 'can_write')

class ProfileSerializer(serializers.ModelSerializer):

    ambulances = AmbulancePermissionSerializer(read_only=True, many=True)
    hospitals = HospitalPermissionSerializer(read_only=True, many=True)

    #all_ambulances = serializers.SerializerMethodField()

    class Meta:
        model = Profile
        fields = ('ambulances', 'hospitals')

class ExtendedProfileSerializer(serializers.ModelSerializer):

    ambulances = serializers.SerializerMethodField()
    hospitals = HospitalPermissionSerializer(read_only=True, many=True)

    class Meta:
        model = Profile
        fields = ('ambulances', 'hospitals')

    def get_ambulances(self, obj):
        if obj.user.is_superuser:
            return list(Ambulance.objects.all().values('id', 'identifier').annotate(can_read=models.Value(True,models.BooleanField()), can_write=models.Value(True,models.BooleanField())))
        else:
            return list(obj.ambulances.values('ambulance_id', 'ambulance__identifier', 'can_read', 'can_write'))

# Ambulance serializers
class AmbulanceSerializer(serializers.ModelSerializer):

    class Meta:
        model = Ambulance
        fields = '__all__'
        read_only_fields = ('updated_by',)

    def validate(self, data):

        # location and location_timestamp must be defined together
        if 'location' in data or 'location_timestamp' in data:

            if not ('location' in data and 'location_timestamp' in data):
                raise serializers.ValidationError('location and location_timestamp must be set together')

            if data['location'] and not data['location_timestamp']:
                raise serializers.ValidationError('location cannot be set without location_timestamp')

            if not data['location'] and data['location_timestamp']:
                raise serializers.ValidationError('location_timestamp cannot be set without location')

        return data

    def create(self, validated_data):

        # get current user
        user = validated_data['updated_by']

        # check credentials
        # only super can create
        if not user.is_superuser:
            raise PermissionDenied()

        return super().create(validated_data)

    def update(self, instance, validated_data):

        # get current user
        user = validated_data['updated_by']

        # check credentials
        if not user.is_superuser:

            # serializer.instance will always exist!
            if not user.profile.ambulances.filter(can_write=True,
                                                  ambulance=instance.id):
                raise PermissionDenied()

        return super().update(instance, validated_data)
<commit_msg>Tests<commit_after>from rest_framework import serializers
from drf_extra_fields.geo_fields import PointField as PointFieldSerializer

from django.contrib.auth.models import User
from django.core.exceptions import PermissionDenied
import django.db.models as models

from .models import Profile, Hospital, Ambulance, \
    AmbulancePermission, HospitalPermission

# Profile serializers

class AmbulancePermissionSerializer(serializers.ModelSerializer):

    ambulance_id = serializers.IntegerField(source='ambulance.id')
    ambulance_identifier = serializers.CharField(source='ambulance.identifier')

    class Meta:
        model = AmbulancePermission
        fields = ('ambulance_id', 'ambulance__identifier', 'can_read', 'can_write')
        read_only_fields = ('ambulance_id', 'ambulance__identifier', 'can_read', 'can_write')

class HospitalPermissionSerializer(serializers.ModelSerializer):

    hospital_id = serializers.IntegerField(source='hospital.id')
    hospital_name = serializers.CharField(source='hospital.name')

    class Meta:
        model = HospitalPermission
        fields = ('hospital_id', 'hospital__name', 'can_read', 'can_write')
        read_only_fields = ('hospital_id', 'hospital__name', 'can_read', 'can_write')

class ProfileSerializer(serializers.ModelSerializer):

    ambulances = AmbulancePermissionSerializer(read_only=True, many=True)
    hospitals = HospitalPermissionSerializer(read_only=True, many=True)

    #all_ambulances = serializers.SerializerMethodField()

    class Meta:
        model = Profile
        fields = ('ambulances', 'hospitals')

class ExtendedProfileSerializer(serializers.ModelSerializer):

    ambulances = serializers.SerializerMethodField()
    hospitals = HospitalPermissionSerializer(read_only=True, many=True)

    class Meta:
        model = Profile
        fields = ('ambulances', 'hospitals')

    def get_ambulances(self, obj):
        if obj.user.is_superuser:
            return list(Ambulance.objects.all().values('id', 'identifier').annotate(can_read=models.Value(True,models.BooleanField()), can_write=models.Value(True,models.BooleanField())))
        else:
            return list(obj.ambulances.values('ambulance_id', 'ambulance__identifier', 'can_read', 'can_write'))

# Ambulance serializers
class AmbulanceSerializer(serializers.ModelSerializer):

    class Meta:
        model = Ambulance
        fields = '__all__'
        read_only_fields = ('updated_by',)

    def validate(self, data):

        # location and location_timestamp must be defined together
        if 'location' in data or 'location_timestamp' in data:

            if not ('location' in data and 'location_timestamp' in data):
                raise serializers.ValidationError('location and location_timestamp must be set together')

            if data['location'] and not data['location_timestamp']:
                raise serializers.ValidationError('location cannot be set without location_timestamp')

            if not data['location'] and data['location_timestamp']:
                raise serializers.ValidationError('location_timestamp cannot be set without location')

        return data

    def create(self, validated_data):

        # get current user
        user = validated_data['updated_by']

        # check credentials
        # only super can create
        if not user.is_superuser:
            raise PermissionDenied()

        return super().create(validated_data)

    def update(self, instance, validated_data):

        # get current user
        user = validated_data['updated_by']

        # check credentials
        if not user.is_superuser:

            # serializer.instance will always exist!
            if not user.profile.ambulances.filter(can_write=True,
                                                  ambulance=instance.id):
                raise PermissionDenied()

        return super().update(instance, validated_data)
<|endoftext|>


<commit_before>from rest_framework import serializers

from django.contrib.auth.models import User

from .models import Profile, Hospital, Ambulance, \
    UserLocation, AmbulanceLocation

class UserHospitalSerializer(serializers.ModelSerializer):

    class Meta:
        model = Hospital
        fields = ('id', 'identifier')
        read_only_fields = ('id', 'identifier')

class UserHospitalsSerializer(serializers.ModelSerializer):

    hospitals = UserHospitalSerializer(read_only=True, many=True)

    class Meta:
        model = Profile
        fields = ('hospitals',)

class UserAmbulanceSerializer(serializers.ModelSerializer):

    class Meta:
        model = Ambulance
        fields = ('id', 'identifier')
        read_only_fields = ('id', 'identifier')

class UserAmbulancesSerializer(serializers.ModelSerializer):

    ambulances = UserAmbulanceSerializer(read_only=True, many=True)

    class Meta:
        model = Profile
        fields = ('ambulances',)

class UserLocationSerializer(serializers.ModelSerializer):

    class Meta:
        model = UserLocation
        fields = ('location', 'timestamp')

class AmbulanceLocationSerializer(serializers.ModelSerializer):

    location = UserLocationSerializer()

    class Meta:
        model = AmbulanceLocation
        fields = ('location','status','orientation')
<commit_msg>Tests<commit_after>from rest_framework import serializers

from django.contrib.auth.models import User

from .models import Profile, Hospital, Ambulance, \
    UserLocation, AmbulanceLocation

class UserHospitalSerializer(serializers.ModelSerializer):

    class Meta:
        model = Hospital
        fields = ('id', 'name')
        read_only_fields = ('id', 'name')

class UserHospitalsSerializer(serializers.ModelSerializer):

    hospitals = UserHospitalSerializer(read_only=True, many=True)

    class Meta:
        model = Profile
        fields = ('hospitals',)

class UserAmbulanceSerializer(serializers.ModelSerializer):

    class Meta:
        model = Ambulance
        fields = ('id', 'identifier')
        read_only_fields = ('id', 'identifier')

class UserAmbulancesSerializer(serializers.ModelSerializer):

    ambulances = UserAmbulanceSerializer(read_only=True, many=True)

    class Meta:
        model = Profile
        fields = ('ambulances',)

class UserLocationSerializer(serializers.ModelSerializer):

    class Meta:
        model = UserLocation
        fields = ('location', 'timestamp')

class AmbulanceLocationSerializer(serializers.ModelSerializer):

    location = UserLocationSerializer()

    class Meta:
        model = AmbulanceLocation
        fields = ('location','status','orientation')
<|endoftext|>


<commit_before># coding=utf-8
import types
import datetime

from support.items import get_kind, get_item_thumb
from support.helpers import get_video_display_title
from support.ignore import ignore_list
from support.lib import get_intent
from subzero.constants import ICON

default_thumb = R(ICON)

def should_display_ignore(items, previous=None):
    kind = get_kind(items)
    return items and (
        (kind in ("show", "season")) or
        (kind == "episode" and previous != "season")
    )

def add_ignore_options(oc, kind, callback_menu=None, title=None, rating_key=None, add_kind=True):
    """

    :param oc: oc to add our options to
    :param kind: movie, show, episode ... - gets translated to the ignore key (sections, series, items)
    :param callback_menu: menu to inject
    :param title:
    :param rating_key:
    :return:
    """
    # try to translate kind to the ignore key
    use_kind = kind
    if kind not in ignore_list:
        use_kind = ignore_list.translate_key(kind)
    if not use_kind or use_kind not in ignore_list:
        return

    in_list = rating_key in ignore_list[use_kind]

    oc.add(DirectoryObject(
        key=Callback(callback_menu, kind=use_kind, rating_key=rating_key, title=title),
        title=u"%s %s \"%s\" %s the ignore list" % (
            "Remove" if in_list else "Add", ignore_list.verbose(kind) if add_kind else "", unicode(title), "from" if in_list else "to")
    )
    )

def dig_tree(oc, items, menu_callback, menu_determination_callback=None, force_rating_key=None, fill_args=None, pass_kwargs=None,
             thumb=default_thumb):
    for kind, title, key, dig_deeper, item in items:
        thumb = get_item_thumb(item) or thumb

        add_kwargs = {}
        if fill_args:
            add_kwargs = dict((name, getattr(item, k)) for k, name in fill_args.iteritems() if item and hasattr(item, k))
        if pass_kwargs:
            add_kwargs.update(pass_kwargs)

        oc.add(DirectoryObject(
            key=Callback(menu_callback or menu_determination_callback(kind, item), title=title,
                         rating_key=force_rating_key or key, **add_kwargs),
            title=title, thumb=thumb
        ))
    return oc

def set_refresh_menu_state(state_or_media, media_type="movies"):
    """

    :param state_or_media: string, None, or Media argument from Agent.update()
    :param media_type: movies or series
    :return:
    """
    if not state_or_media:
        # store it in last state and remove the current
        Dict["last_refresh_state"] = Dict["current_refresh_state"]
        Dict["current_refresh_state"] = None
        return

    if isinstance(state_or_media, types.StringTypes):
        Dict["current_refresh_state"] = state_or_media
        return

    media = state_or_media
    media_id = media.id
    title = None
    if media_type == "series":
        for season in media.seasons:
            for episode in media.seasons[season].episodes:
                ep = media.seasons[season].episodes[episode]
                media_id = ep.id
                title = get_video_display_title("show", ep.title, parent_title=media.title, season=int(season), episode=int(episode))
    else:
        title = get_video_display_title("movie", media.title)

    intent = get_intent()
    force_refresh = intent.get("force", media_id)

    Dict["current_refresh_state"] = u"%sRefreshing %s" % ("Force-" if force_refresh else "", unicode(title))

def enable_channel_wrapper(func):
    """
    returns the original wrapper :func: (route or handler) if applicable, else the plain to-be-wrapped function
    :param func: original wrapper
    :return: original wrapper or wrapped function
    """
    def noop(*args, **kwargs):
        def inner(*a, **k):
            """
            :param a: args
            :param k: kwargs
            :return: originally to-be-wrapped function
            """
            return a[0]

        return inner

    def wrap(*args, **kwargs):
        enforce_route = kwargs.pop("enforce_route", None)
        return (func if Prefs["enable_channel"] or enforce_route else noop)(*args, **kwargs)

    return wrap

def debounce(func):
    """
    prevent func from being called twice with the same arguments
    :param func:
    :return:
    """
    def get_lookup_key(args, kwargs):
        func_name = list(args).pop(0).__name__
        return tuple([func_name] + [(key, value) for key, value in kwargs.iteritems()])

    def wrap(*args, **kwargs):
        if "randomize" in kwargs:
            if not "menu_history" in Dict:
                Dict["menu_history"] = {}

            key = get_lookup_key([func] + list(args), kwargs)
            if key in Dict["menu_history"]:
                Log.Debug("not triggering %s twice with %s, %s" % (func, args, kwargs))
                return ObjectContainer()
            else:
                Dict["menu_history"][key] = datetime.datetime.now() + datetime.timedelta(days=1)
                Dict.Save()
        return func(*args, **kwargs)

    return wrap

class SZObjectContainer(ObjectContainer):
    def __init__(self, *args, **kwargs):
        super(SZObjectContainer, self).__init__(*args, **kwargs)
        from interface.menu import fatality
        from support.helpers import pad_title, timestamp
        self.add(DirectoryObject(
            key=Callback(fatality, force_title=" ", randomize=timestamp()),
            title=pad_title("<< Back to home"),
            summary="Current state: %s; Last state: %s" % (
                (Dict["current_refresh_state"] or "Idle") if "current_refresh_state" in Dict else "Idle",
                (Dict["last_refresh_state"] or "None") if "last_refresh_state" in Dict else "None"
            )
        ))<commit_msg>force details view for show/season<commit_after># coding=utf-8
import types
import datetime

from support.items import get_kind, get_item_thumb
from support.helpers import get_video_display_title
from support.ignore import ignore_list
from support.lib import get_intent
from subzero.constants import ICON

default_thumb = R(ICON)

def should_display_ignore(items, previous=None):
    kind = get_kind(items)
    return items and (
        (kind in ("show", "season")) or
        (kind == "episode" and previous != "season")
    )

def add_ignore_options(oc, kind, callback_menu=None, title=None, rating_key=None, add_kind=True):
    """

    :param oc: oc to add our options to
    :param kind: movie, show, episode ... - gets translated to the ignore key (sections, series, items)
    :param callback_menu: menu to inject
    :param title:
    :param rating_key:
    :return:
    """
    # try to translate kind to the ignore key
    use_kind = kind
    if kind not in ignore_list:
        use_kind = ignore_list.translate_key(kind)
    if not use_kind or use_kind not in ignore_list:
        return

    in_list = rating_key in ignore_list[use_kind]

    oc.add(DirectoryObject(
        key=Callback(callback_menu, kind=use_kind, rating_key=rating_key, title=title),
        title=u"%s %s \"%s\" %s the ignore list" % (
            "Remove" if in_list else "Add", ignore_list.verbose(kind) if add_kind else "", unicode(title), "from" if in_list else "to")
    )
    )

def dig_tree(oc, items, menu_callback, menu_determination_callback=None, force_rating_key=None, fill_args=None,
             pass_kwargs=None, thumb=default_thumb):
    for kind, title, key, dig_deeper, item in items:
        thumb = get_item_thumb(item) or thumb

        add_kwargs = {}
        if fill_args:
            add_kwargs = dict((name, getattr(item, k)) for k, name in fill_args.iteritems() if item and hasattr(item, k))
        if pass_kwargs:
            add_kwargs.update(pass_kwargs)

        # force details view for show/season
        summary = " " if kind in ("show", "season") else None

        oc.add(DirectoryObject(
            key=Callback(menu_callback or menu_determination_callback(kind, item), title=title,
                         rating_key=force_rating_key or key, **add_kwargs),
            title=title, thumb=thumb, summary=summary
        ))
    return oc

def set_refresh_menu_state(state_or_media, media_type="movies"):
    """

    :param state_or_media: string, None, or Media argument from Agent.update()
    :param media_type: movies or series
    :return:
    """
    if not state_or_media:
        # store it in last state and remove the current
        Dict["last_refresh_state"] = Dict["current_refresh_state"]
        Dict["current_refresh_state"] = None
        return

    if isinstance(state_or_media, types.StringTypes):
        Dict["current_refresh_state"] = state_or_media
        return

    media = state_or_media
    media_id = media.id
    title = None
    if media_type == "series":
        for season in media.seasons:
            for episode in media.seasons[season].episodes:
                ep = media.seasons[season].episodes[episode]
                media_id = ep.id
                title = get_video_display_title("show", ep.title, parent_title=media.title, season=int(season), episode=int(episode))
    else:
        title = get_video_display_title("movie", media.title)

    intent = get_intent()
    force_refresh = intent.get("force", media_id)

    Dict["current_refresh_state"] = u"%sRefreshing %s" % ("Force-" if force_refresh else "", unicode(title))

def enable_channel_wrapper(func):
    """
    returns the original wrapper :func: (route or handler) if applicable, else the plain to-be-wrapped function
    :param func: original wrapper
    :return: original wrapper or wrapped function
    """
    def noop(*args, **kwargs):
        def inner(*a, **k):
            """
            :param a: args
            :param k: kwargs
            :return: originally to-be-wrapped function
            """
            return a[0]

        return inner

    def wrap(*args, **kwargs):
        enforce_route = kwargs.pop("enforce_route", None)
        return (func if Prefs["enable_channel"] or enforce_route else noop)(*args, **kwargs)

    return wrap

def debounce(func):
    """
    prevent func from being called twice with the same arguments
    :param func:
    :return:
    """
    def get_lookup_key(args, kwargs):
        func_name = list(args).pop(0).__name__
        return tuple([func_name] + [(key, value) for key, value in kwargs.iteritems()])

    def wrap(*args, **kwargs):
        if "randomize" in kwargs:
            if not "menu_history" in Dict:
                Dict["menu_history"] = {}

            key = get_lookup_key([func] + list(args), kwargs)
            if key in Dict["menu_history"]:
                Log.Debug("not triggering %s twice with %s, %s" % (func, args, kwargs))
                return ObjectContainer()
            else:
                Dict["menu_history"][key] = datetime.datetime.now() + datetime.timedelta(days=1)
                Dict.Save()
        return func(*args, **kwargs)

    return wrap

class SZObjectContainer(ObjectContainer):
    def __init__(self, *args, **kwargs):
        super(SZObjectContainer, self).__init__(*args, **kwargs)
        from interface.menu import fatality
        from support.helpers import pad_title, timestamp
        self.add(DirectoryObject(
            key=Callback(fatality, force_title=" ", randomize=timestamp()),
            title=pad_title("<< Back to home"),
            summary="Current state: %s; Last state: %s" % (
                (Dict["current_refresh_state"] or "Idle") if "current_refresh_state" in Dict else "Idle",
                (Dict["last_refresh_state"] or "None") if "last_refresh_state" in Dict else "None"
            )
        ))<|endoftext|>


<commit_before>"""
Testing utilities.
"""

from __future__ import with_statement
from django.conf import settings, UserSettingsHolder
from django.core.management import call_command
from django.db.models import loading
from django.template import Template, Context, RequestContext
from django.test.simple import run_tests as django_run_tests
from django.test.testcases import TestCase as DjangoTestCase
from django.utils.functional import wraps

class override_settings(object):
    """
    Acts as either a decorator, or a context manager.  If it's a decorator it
    takes a function and returns a wrapped function.  If it's a contextmanager
    it's used with the ``with`` statement.  In either event entering/exiting
    are called before and after, respectively, the function/block is executed.

    Via: http://djangosnippets.org/snippets/2437/
    """
    def __init__(self, **kwargs):
        self.options = kwargs
        self.wrapped = settings._wrapped

    def __enter__(self):
        self.enable()

    def __exit__(self, exc_type, exc_value, traceback):
        self.disable()

    def __call__(self, func):
        @wraps(func)
        def inner(*args, **kwargs):
            with self:
                return func(*args, **kwargs)
        return inner

    def enable(self):
        override = UserSettingsHolder(settings._wrapped)
        for key, new_value in self.options.items():
            setattr(override, key, new_value)
        settings._wrapped = override

    def disable(self):
        settings._wrapped = self.wrapped

def run_tests(labels=()):
    """
    Use the Django test runner to run the tests.
    """
    django_run_tests(labels, verbosity=1, interactive=True)

class TestCase(DjangoTestCase):
    """
    Base test case for the django-analytical tests.

    Includes the settings manager.
    """

    def setUp(self):
        self.settings_manager = TestSettingsManager()

    def tearDown(self):
        self.settings_manager.revert()

class TagTestCase(TestCase):
    """
    Tests for a template tag.

    Includes the settings manager.
    """

    def render_tag(self, library, tag, vars=None, request=None):
        if vars is None:
            vars = {}
        t = Template("{%% load %s %%}{%% %s %%}" % (library, tag))
        if request is not None:
            context = RequestContext(request, vars)
        else:
            context = Context(vars)
        return t.render(context)

class TestSettingsManager(object):
    """
    From: http://www.djangosnippets.org/snippets/1011/

    A class which can modify some Django settings temporarily for a
    test and then revert them to their original values later.

    Automatically handles resyncing the DB if INSTALLED_APPS is
    modified.
    """

    NO_SETTING = ('!', None)

    def __init__(self):
        self._original_settings = {}

    def set(self, **kwargs):
        for k, v in kwargs.iteritems():
            self._original_settings.setdefault(k, getattr(settings, k,
                    self.NO_SETTING))
            setattr(settings, k, v)
        if 'INSTALLED_APPS' in kwargs:
            self.syncdb()

    def delete(self, *args):
        for k in args:
            try:
                self._original_settings.setdefault(k, getattr(settings, k,
                        self.NO_SETTING))
                delattr(settings, k)
            except AttributeError:
                pass  # setting did not exist

    def syncdb(self):
        loading.cache.loaded = False
        call_command('syncdb', verbosity=0, interactive=False)

    def revert(self):
        for k,v in self._original_settings.iteritems():
            if v == self.NO_SETTING:
                if hasattr(settings, k):
                    delattr(settings, k)
            else:
                setattr(settings, k, v)
        if 'INSTALLED_APPS' in self._original_settings:
            self.syncdb()
        self._original_settings = {}
<commit_msg>run_tests is depreciated.  Use DjangoTestSuiteRunner<commit_after>"""
Testing utilities.
"""

from __future__ import with_statement
from django.conf import settings, UserSettingsHolder
from django.core.management import call_command
from django.db.models import loading
from django.template import Template, Context, RequestContext
from django.test.testcases import TestCase as DjangoTestCase
from django.utils.functional import wraps

class override_settings(object):
    """
    Acts as either a decorator, or a context manager.  If it's a decorator it
    takes a function and returns a wrapped function.  If it's a contextmanager
    it's used with the ``with`` statement.  In either event entering/exiting
    are called before and after, respectively, the function/block is executed.

    Via: http://djangosnippets.org/snippets/2437/
    """
    def __init__(self, **kwargs):
        self.options = kwargs
        self.wrapped = settings._wrapped

    def __enter__(self):
        self.enable()

    def __exit__(self, exc_type, exc_value, traceback):
        self.disable()

    def __call__(self, func):
        @wraps(func)
        def inner(*args, **kwargs):
            with self:
                return func(*args, **kwargs)
        return inner

    def enable(self):
        override = UserSettingsHolder(settings._wrapped)
        for key, new_value in self.options.items():
            setattr(override, key, new_value)
        settings._wrapped = override

    def disable(self):
        settings._wrapped = self.wrapped

def run_tests():
    """
    Use the Django test runner to run the tests.
    """
    from django.test.simple import DjangoTestSuiteRunner
    runner = DjangoTestSuiteRunner(verbosity=2)
    runner.run_tests(None)

class TestCase(DjangoTestCase):
    """
    Base test case for the django-analytical tests.

    Includes the settings manager.
    """

    def setUp(self):
        self.settings_manager = TestSettingsManager()

    def tearDown(self):
        self.settings_manager.revert()

class TagTestCase(TestCase):
    """
    Tests for a template tag.

    Includes the settings manager.
    """

    def render_tag(self, library, tag, vars=None, request=None):
        if vars is None:
            vars = {}
        t = Template("{%% load %s %%}{%% %s %%}" % (library, tag))
        if request is not None:
            context = RequestContext(request, vars)
        else:
            context = Context(vars)
        return t.render(context)

class TestSettingsManager(object):
    """
    From: http://www.djangosnippets.org/snippets/1011/

    A class which can modify some Django settings temporarily for a
    test and then revert them to their original values later.

    Automatically handles resyncing the DB if INSTALLED_APPS is
    modified.
    """

    NO_SETTING = ('!', None)

    def __init__(self):
        self._original_settings = {}

    def set(self, **kwargs):
        for k, v in kwargs.iteritems():
            self._original_settings.setdefault(k, getattr(settings, k,
                    self.NO_SETTING))
            setattr(settings, k, v)
        if 'INSTALLED_APPS' in kwargs:
            self.syncdb()

    def delete(self, *args):
        for k in args:
            try:
                self._original_settings.setdefault(k, getattr(settings, k,
                        self.NO_SETTING))
                delattr(settings, k)
            except AttributeError:
                pass  # setting did not exist

    def syncdb(self):
        loading.cache.loaded = False
        call_command('syncdb', verbosity=0, interactive=False)

    def revert(self):
        for k,v in self._original_settings.iteritems():
            if v == self.NO_SETTING:
                if hasattr(settings, k):
                    delattr(settings, k)
            else:
                setattr(settings, k, v)
        if 'INSTALLED_APPS' in self._original_settings:
            self.syncdb()
        self._original_settings = {}
<|endoftext|>


<commit_before>import json
import os
import collections
import zipfile
import tempfile
from cudatext import *

README_NAMES = (
    'readme.txt',
    'readme.html',
    'readme.htm',
    'readme.md',
    'README.md',
    'readme.rst',
    'README.rst',
    )

HISTORY_NAMES = (
    'history.txt',
    'history.md',
    'history.rst',
    'history',
)

DATA_DIRS = (
    ('autocomplete', '.acp'),
    ('lang', '.ini'),
    ('newdoc', ''),
    ('snippets', ''),
    ('themes', ''),
    )

def _root_item(s):

    n = s.rfind('/')
    return n<0 or n==len(s)-1

def get_props_of_zip_filename(zip_fn):

    temp_dir = tempfile.gettempdir()
    z = zipfile.ZipFile(zip_fn, 'r')

    files = z.namelist()
    if 'install.inf' in files:
        files.remove('install.inf')
    files = [f for f in files if _root_item(f)]

    z.extract('install.inf', temp_dir)
    z.close()
    fn = os.path.join(temp_dir, 'install.inf')

    if os.path.isfile(fn):
        typ = ini_read(fn, 'info', 'type', '')
        subdir = ini_read(fn, 'info', 'subdir', '')

        if typ=='cudatext-plugin':
            d = 'py'
            files = [subdir+'/']
        elif typ=='cudatext-data':
            d = 'data/'+subdir
        elif typ=='lexer':
            d = 'data/lexlib'
        elif typ=='lexer-lite':
            d = 'data/lexliblite'
        else:
            d = ''

        os.remove(fn)
        #print('prop', (d, files, subdir))
        return (d, files, subdir)

def get_readme_of_module(mod):
    for name in README_NAMES:
        fn = os.path.join(app_path(APP_DIR_PY), mod, 'readme', name)
        if os.path.isfile(fn):
            return fn
        fn = os.path.join(app_path(APP_DIR_PY), mod, name)
        if os.path.isfile(fn):
            return fn

def get_history_of_module(mod):
    for name in HISTORY_NAMES:
        fn = os.path.join(app_path(APP_DIR_PY), mod, 'readme', name)
        if os.path.isfile(fn):
            return fn
        fn = os.path.join(app_path(APP_DIR_PY), mod, name)
        if os.path.isfile(fn):
            return fn

def get_installinf_of_module(mod):
    return os.path.join(app_path(APP_DIR_PY), mod, 'install.inf')

def get_initpy_of_module(mod):
    return os.path.join(app_path(APP_DIR_PY), mod, '__init__.py')

def get_name_of_module(mod):
    fn_ini = get_installinf_of_module(mod)
    return ini_read(fn_ini, 'info', 'title', mod)

def get_homepage_of_module(mod):
    fn_ini = get_installinf_of_module(mod)
    return ini_read(fn_ini, 'info', 'homepage', '')

def do_remove_dir(dir):
    """
    move folder to py/__trash
    (make copy with _ suffix if nessesary)
    """
    print('Deleting folder:', dir)
    if not os.path.isdir(dir):
        return

    dir_trash = os.path.join(app_path(APP_DIR_PY), '__trash')
    dir_dest = os.path.join(dir_trash, os.path.basename(dir))
    while os.path.isdir(dir_dest):
        dir_dest += '_'

    if not os.path.isdir(dir_trash):
        os.mkdir(dir_trash)

    try:
        os.rename(dir, dir_dest)
    except OSError:
        msg_box('Cannot remove folder:\n'+dir, MB_OK+MB_ICONERROR)
        return
    return True

def get_installed_modules():
    """
    gets list of py-modules inside "py"
    """
    d = app_path(APP_DIR_PY)
    l = os.listdir(d)
    l = [s for s in l if not s.startswith('__')]
    l = [s for s in l if os.path.isfile(os.path.join(d, s, 'install.inf'))]
    return sorted(l)

def get_installed_choice(caption, exclude_list=None):
    """
    gets module of addon, from menu of installed addons
    """
    lmod = get_installed_modules()
    if exclude_list:
        lmod = [i for i in lmod if not i in exclude_list]
    ldesc = [get_name_of_module(l) for l in lmod]
    res = dlg_menu(MENU_LIST, ldesc, caption=caption)
    if res is None:
        return None
    return lmod[res]

def get_installed_addons(ignore={}):

    exclude_modules = ignore.get('plugins', [])
    exclude_lexers = ignore.get('lexers', [])
    exclude_lexers_lite = ignore.get('lexers_lite', [])
    exclude_themes = ignore.get('themes', [])
    exclude_translations = ignore.get('lang', [])
    exclude_snippets = ignore.get('snippets', [])

    d = app_path(APP_DIR_PY)
    l = get_installed_modules()
    l = [i for i in l if not i in exclude_modules]
    res = [{
        'kind': 'plugin',
        'name': get_name_of_module(i),
        'module': i,
        'files': [
            os.path.join(d, i)+'/',
            ],
        } for i in l]

    d = os.path.join(app_path(APP_DIR_DATA), 'lexlib')
    d_acp = os.path.join(app_path(APP_DIR_DATA), 'autocomplete')
    l = os.listdir(d)
    l = [i.split('.')[0] for i in l if i.endswith('.lcf')]
    l = [i for i in l if not i in exclude_lexers]
    l = sorted(l)
    res += [{
        'kind': 'lexer',
        'name': i,
        'files': [
            os.path.join(d, i+'.lcf'),
            os.path.join(d, i+'.cuda-lexmap'),
            os.path.join(d_acp, i+'.acp'),
            ],
        } for i in l]

    d = os.path.join(app_path(APP_DIR_DATA), 'lexliblite')
    l = os.listdir(d)
    l = [i.split('.')[0] for i in l if i.endswith('.cuda-litelexer')]
    l = [i for i in l if not i in exclude_lexers_lite]
    l = sorted(l)
    res += [{
        'kind': 'lexer',
        'name': i+' ^',
        'files': [
            os.path.join(d, i+'.cuda-litelexer'),
            ],
        } for i in l]

    d = os.path.join(app_path(APP_DIR_DATA), 'snippets')
    l = os.listdir(d)
    l = [i for i in l if not i in exclude_snippets]
    l = sorted(l)
    res += [{
        'kind': 'snippets',
        'name': i,
        'files': [
            os.path.join(d, i)+'/',
            ],
        } for i in l]

    d = os.path.join(app_path(APP_DIR_DATA), 'themes')
    l = os.listdir(d)
    l = [i.split('.')[0] for i in l if i.endswith('.cuda-theme-syntax') or i.endswith('.cuda-theme-ui')]
    l = [i for i in l if not i in exclude_themes]
    l = list(set(l)) # del duplicates
    l = sorted(l)
    res += [{
        'kind': 'theme',
        'name': i,
        'files': [
            os.path.join(d, i+'.cuda-theme-syntax'),
            os.path.join(d, i+'.cuda-theme-ui'),
            ],
        } for i in l]

    d = os.path.join(app_path(APP_DIR_DATA), 'lang')
    l = os.listdir(d)
    l = [i.split('.')[0] for i in l if i.endswith('.ini')]
    l = [i for i in l if not i in exclude_translations]
    l = sorted(l)
    res += [{
        'kind': 'translation',
        'name': i,
        'files': [
            os.path.join(d, i+'.ini'),
            ],
        } for i in l]

    return res

def get_packages_ini():

    return os.path.join(app_path(APP_DIR_SETTINGS), 'packages.ini')

def do_save_version(url, fn, version):

    props = get_props_of_zip_filename(fn)
    if props:
        d, f, m = props
        fn = get_packages_ini()
        sec = os.path.basename(url)
        ini_write(fn, sec, 'd', d)
        ini_write(fn, sec, 'f', ';'.join(f))
        ini_write(fn, sec, 'v', version)
        return props

def get_addon_version(url):

    fn = get_packages_ini()
    return ini_read(fn, os.path.basename(url), 'v', '')

def do_remove_version_of_plugin(mod):

    import configparser
    fn = get_packages_ini()
    config = configparser.ConfigParser()
    config.read(fn)
    for sec in config.sections():
        if config[sec]['d'] == 'py' and config[sec]['f'] == mod+'/':
            del config[sec]
            with open(fn, 'w') as f:
                config.write(f, False)
<commit_msg>AddonMan: remove msg 'deleting folder'<commit_after>import json
import os
import collections
import zipfile
import tempfile
from cudatext import *

README_NAMES = (
    'readme.txt',
    'readme.html',
    'readme.htm',
    'readme.md',
    'README.md',
    'readme.rst',
    'README.rst',
    )

HISTORY_NAMES = (
    'history.txt',
    'history.md',
    'history.rst',
    'history',
)

DATA_DIRS = (
    ('autocomplete', '.acp'),
    ('lang', '.ini'),
    ('newdoc', ''),
    ('snippets', ''),
    ('themes', ''),
    )

def _root_item(s):

    n = s.rfind('/')
    return n<0 or n==len(s)-1

def get_props_of_zip_filename(zip_fn):

    temp_dir = tempfile.gettempdir()
    z = zipfile.ZipFile(zip_fn, 'r')

    files = z.namelist()
    if 'install.inf' in files:
        files.remove('install.inf')
    files = [f for f in files if _root_item(f)]

    z.extract('install.inf', temp_dir)
    z.close()
    fn = os.path.join(temp_dir, 'install.inf')

    if os.path.isfile(fn):
        typ = ini_read(fn, 'info', 'type', '')
        subdir = ini_read(fn, 'info', 'subdir', '')

        if typ=='cudatext-plugin':
            d = 'py'
            files = [subdir+'/']
        elif typ=='cudatext-data':
            d = 'data/'+subdir
        elif typ=='lexer':
            d = 'data/lexlib'
        elif typ=='lexer-lite':
            d = 'data/lexliblite'
        else:
            d = ''

        os.remove(fn)
        #print('prop', (d, files, subdir))
        return (d, files, subdir)

def get_readme_of_module(mod):
    for name in README_NAMES:
        fn = os.path.join(app_path(APP_DIR_PY), mod, 'readme', name)
        if os.path.isfile(fn):
            return fn
        fn = os.path.join(app_path(APP_DIR_PY), mod, name)
        if os.path.isfile(fn):
            return fn

def get_history_of_module(mod):
    for name in HISTORY_NAMES:
        fn = os.path.join(app_path(APP_DIR_PY), mod, 'readme', name)
        if os.path.isfile(fn):
            return fn
        fn = os.path.join(app_path(APP_DIR_PY), mod, name)
        if os.path.isfile(fn):
            return fn

def get_installinf_of_module(mod):
    return os.path.join(app_path(APP_DIR_PY), mod, 'install.inf')

def get_initpy_of_module(mod):
    return os.path.join(app_path(APP_DIR_PY), mod, '__init__.py')

def get_name_of_module(mod):
    fn_ini = get_installinf_of_module(mod)
    return ini_read(fn_ini, 'info', 'title', mod)

def get_homepage_of_module(mod):
    fn_ini = get_installinf_of_module(mod)
    return ini_read(fn_ini, 'info', 'homepage', '')

def do_remove_dir(dir):
    """
    move folder to py/__trash
    (make copy with _ suffix if nessesary)
    """
    #print('Deleting folder:', dir)
    if not os.path.isdir(dir):
        return

    dir_trash = os.path.join(app_path(APP_DIR_PY), '__trash')
    dir_dest = os.path.join(dir_trash, os.path.basename(dir))
    while os.path.isdir(dir_dest):
        dir_dest += '_'

    if not os.path.isdir(dir_trash):
        os.mkdir(dir_trash)

    try:
        os.rename(dir, dir_dest)
    except OSError:
        msg_box('Cannot remove folder:\n'+dir, MB_OK+MB_ICONERROR)
        return
    return True

def get_installed_modules():
    """
    gets list of py-modules inside "py"
    """
    d = app_path(APP_DIR_PY)
    l = os.listdir(d)
    l = [s for s in l if not s.startswith('__')]
    l = [s for s in l if os.path.isfile(os.path.join(d, s, 'install.inf'))]
    return sorted(l)

def get_installed_choice(caption, exclude_list=None):
    """
    gets module of addon, from menu of installed addons
    """
    lmod = get_installed_modules()
    if exclude_list:
        lmod = [i for i in lmod if not i in exclude_list]
    ldesc = [get_name_of_module(l) for l in lmod]
    res = dlg_menu(MENU_LIST, ldesc, caption=caption)
    if res is None:
        return None
    return lmod[res]

def get_installed_addons(ignore={}):

    exclude_modules = ignore.get('plugins', [])
    exclude_lexers = ignore.get('lexers', [])
    exclude_lexers_lite = ignore.get('lexers_lite', [])
    exclude_themes = ignore.get('themes', [])
    exclude_translations = ignore.get('lang', [])
    exclude_snippets = ignore.get('snippets', [])

    d = app_path(APP_DIR_PY)
    l = get_installed_modules()
    l = [i for i in l if not i in exclude_modules]
    res = [{
        'kind': 'plugin',
        'name': get_name_of_module(i),
        'module': i,
        'files': [
            os.path.join(d, i)+'/',
            ],
        } for i in l]

    d = os.path.join(app_path(APP_DIR_DATA), 'lexlib')
    d_acp = os.path.join(app_path(APP_DIR_DATA), 'autocomplete')
    l = os.listdir(d)
    l = [i.split('.')[0] for i in l if i.endswith('.lcf')]
    l = [i for i in l if not i in exclude_lexers]
    l = sorted(l)
    res += [{
        'kind': 'lexer',
        'name': i,
        'files': [
            os.path.join(d, i+'.lcf'),
            os.path.join(d, i+'.cuda-lexmap'),
            os.path.join(d_acp, i+'.acp'),
            ],
        } for i in l]

    d = os.path.join(app_path(APP_DIR_DATA), 'lexliblite')
    l = os.listdir(d)
    l = [i.split('.')[0] for i in l if i.endswith('.cuda-litelexer')]
    l = [i for i in l if not i in exclude_lexers_lite]
    l = sorted(l)
    res += [{
        'kind': 'lexer',
        'name': i+' ^',
        'files': [
            os.path.join(d, i+'.cuda-litelexer'),
            ],
        } for i in l]

    d = os.path.join(app_path(APP_DIR_DATA), 'snippets')
    l = os.listdir(d)
    l = [i for i in l if not i in exclude_snippets]
    l = sorted(l)
    res += [{
        'kind': 'snippets',
        'name': i,
        'files': [
            os.path.join(d, i)+'/',
            ],
        } for i in l]

    d = os.path.join(app_path(APP_DIR_DATA), 'themes')
    l = os.listdir(d)
    l = [i.split('.')[0] for i in l if i.endswith('.cuda-theme-syntax') or i.endswith('.cuda-theme-ui')]
    l = [i for i in l if not i in exclude_themes]
    l = list(set(l)) # del duplicates
    l = sorted(l)
    res += [{
        'kind': 'theme',
        'name': i,
        'files': [
            os.path.join(d, i+'.cuda-theme-syntax'),
            os.path.join(d, i+'.cuda-theme-ui'),
            ],
        } for i in l]

    d = os.path.join(app_path(APP_DIR_DATA), 'lang')
    l = os.listdir(d)
    l = [i.split('.')[0] for i in l if i.endswith('.ini')]
    l = [i for i in l if not i in exclude_translations]
    l = sorted(l)
    res += [{
        'kind': 'translation',
        'name': i,
        'files': [
            os.path.join(d, i+'.ini'),
            ],
        } for i in l]

    return res

def get_packages_ini():

    return os.path.join(app_path(APP_DIR_SETTINGS), 'packages.ini')

def do_save_version(url, fn, version):

    props = get_props_of_zip_filename(fn)
    if props:
        d, f, m = props
        fn = get_packages_ini()
        sec = os.path.basename(url)
        ini_write(fn, sec, 'd', d)
        ini_write(fn, sec, 'f', ';'.join(f))
        ini_write(fn, sec, 'v', version)
        return props

def get_addon_version(url):

    fn = get_packages_ini()
    return ini_read(fn, os.path.basename(url), 'v', '')

def do_remove_version_of_plugin(mod):

    import configparser
    fn = get_packages_ini()
    config = configparser.ConfigParser()
    config.read(fn)
    for sec in config.sections():
        if config[sec]['d'] == 'py' and config[sec]['f'] == mod+'/':
            del config[sec]
            with open(fn, 'w') as f:
                config.write(f, False)
<|endoftext|>


<commit_before># Copyright 2011 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Module containing the classes which lets us construct readonly
pages for models.
"""

import collections
import re

from django.template import loader
from django.utils.datastructures import SortedDict

from soc.views.helper.surveys import SurveySchema

class ModelReadOnlyTemplateOptions(object):
  """Class holding options specified in the Meta class of the model template.

  Holds the following options:
    model: an Appengine db.Model class, or None
    css_prefix: the prefix that needs to be added to CSS attributes
        while rendering
    fields: list of appengine model property names to be rendered in the
        template, or None
    hidden_fields: list of appengine model property names to be rendered in the
        template with 'display: none' style, or None
    exclude: list of appengine model property names to be skipped from the
        template, or None
    renderers: dictionary of field name and function pairs where the supplied
        function specifies how the field should be rendered.
  """

  def __init__(self, options=None):
    """
    """
    self.model = getattr(options, 'model', None)
    self.css_prefix = getattr(options, 'css_prefix',
                              getattr(self.model, '__name__', None))
    self.fields = getattr(options, 'fields', None)
    self.hidden_fields = getattr(options, 'hidden_fields', None)
    self.exclude = getattr(options, 'exclude', None)
    self.renderers = getattr(options, 'renderers', None)

class ModelReadOnlyTemplateMetaclass(type):
  """Meta class that alters the creation of model readonly template classes.

  During the creation of the model readonly template class we filter out
  the needed and unnecessary properties from the model using the attributes
  specified in the Meta class of the model readonly template class.
  """

  def __new__(cls, class_name, bases, dict):
    """Method that alters the class attributes in the model readonly template.

    This method alters the creation of the object before initialization by
    adding in the model's attributes that needs to be rendered. It also
    adds other necessary options extracted from the Meta class inside the
    model's readonly template class.
    """
    opts = ModelReadOnlyTemplateOptions(dict.get('Meta', None))

    if opts.model is not None:
      model_fields = SortedDict()
      model_hidden_fields = {}
      for name, prop in sorted(opts.model.properties().iteritems(),
                               key=lambda prop: prop[1].creation_counter):
        if opts.fields and name not in opts.fields:
          if opts.hidden_fields and name in opts.hidden_fields:
            model_hidden_fields[name] = prop
          continue
        if opts.exclude and name in opts.exclude:
          continue

        model_fields[name] = prop

      dict['fields'] = model_fields
      dict['hidden_fields'] = model_hidden_fields
      dict['renderers'] = opts.renderers

    if opts.css_prefix:
      dict['css_prefix'] = opts.css_prefix

    return super(ModelReadOnlyTemplateMetaclass, cls).__new__(
        cls, class_name, bases, dict)

class ModelReadOnlyTemplate(object):
  """A base class that constructs the readonly template for a given model.

  This uses the same notion that Django's forms APIs use to generate forms
  for given models. The idea is completely inspired from Django's ModelForm
  APIs and tries to mimic the same names that is used there in order to
  provide consistency.

  In addition, to provide more consistency with the Melange's framework
  itself we also add a render method that render this class into an html
  template.
  """

  __metaclass__ = ModelReadOnlyTemplateMetaclass

  def __init__(self, instance=None):
    """Constructor to initialize the model instance.

    The readonly template will be rendered for the data in this model instance.
    """
    self.instance = instance

    for name, field in self.hidden_fields.items():
      self.hidden_fields[name] = getattr(self.instance, name)

  def __iter__(self):
    """Iterator yielding groups of model instance's properties to be rendered.
    """
    grouping = collections.defaultdict(list)

    for name, field in self.fields.items():
      group = getattr(field, 'group', '0. ')
      grouping[group].append((field.verbose_name, getattr(
          self.instance, name)))

    rexp = re.compile(r"\d+. ")

    for group, fields in sorted(grouping.items()):
      yield rexp.sub('', group), fields

  def render(self):
    """Renders the html collecting the attributes available in this class.
    """

    context = {
      'model': self,
      'hidden_fields': self.hidden_fields,
      'css_prefix': self.css_prefix,
    }
    rendered = loader.render_to_string(self.template_path,
                                       dictionary=context)
    return rendered

class SurveyRecordReadOnlyTemplate(ModelReadOnlyTemplate):
  """A base class that constructs the readonly template for given survey record.

  This uses the same notion that is used to build the model based readonly
  templates but the schema read from the survey schema rather than the model.
  """

  template_path = 'modules/gsoc/_survey/readonly_template.html'

  def __init__(self, instance=None):
    """Constructor to initialize the model instance.

    The readonly template will be rendered for the data in this model instance.
    """
    self.instance = instance
    self.schema = None
    if self.instance:
      self.schema = SurveySchema(self.instance.survey)

  def __iter__(self):
    """Iterator yielding groups of record instance's properties to be rendered.
    """
    for name, field in self.fields.items():
      renderer = self.renderers.get(name)
      if renderer:
        val = renderer(self.instance)
      else:
        val = getattr(self.instance, name)
      yield field.verbose_name, val

    if self.schema:
      for field in self.schema:
        field_id = field.getFieldName()
        label = field.getLabel()
        value = getattr(self.instance, field_id, 'N/A')
        if isinstance(value, list):
          value = ', '.join(value)

        yield label, value

  def render(self):
    """Renders the html collecting the attributes available in this class.
    """

    context = {
      'record': self,
      'hidden_fields': self.hidden_fields,
      'css_prefix': self.css_prefix,
    }

    meta = getattr(self, 'Meta', None)
    context['survey_name'] = getattr(meta, 'survey_name', '')

    rendered = loader.render_to_string(self.template_path,
                                       dictionary=context)
    return rendered
<commit_msg>Empty dictionary used for classes that do not define any renderers.<commit_after># Copyright 2011 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Module containing the classes which lets us construct readonly
pages for models.
"""

import collections
import re

from django.template import loader
from django.utils.datastructures import SortedDict

from soc.views.helper.surveys import SurveySchema

class ModelReadOnlyTemplateOptions(object):
  """Class holding options specified in the Meta class of the model template.

  Holds the following options:
    model: an Appengine db.Model class, or None
    css_prefix: the prefix that needs to be added to CSS attributes
        while rendering
    fields: list of appengine model property names to be rendered in the
        template, or None
    hidden_fields: list of appengine model property names to be rendered in the
        template with 'display: none' style, or None
    exclude: list of appengine model property names to be skipped from the
        template, or None
    renderers: dictionary of field name and function pairs where the supplied
        function specifies how the field should be rendered.
  """

  def __init__(self, options=None):
    """
    """
    self.model = getattr(options, 'model', None)
    self.css_prefix = getattr(options, 'css_prefix',
                              getattr(self.model, '__name__', None))
    self.fields = getattr(options, 'fields', None)
    self.hidden_fields = getattr(options, 'hidden_fields', None)
    self.exclude = getattr(options, 'exclude', None)
    self.renderers = getattr(options, 'renderers', None)

class ModelReadOnlyTemplateMetaclass(type):
  """Meta class that alters the creation of model readonly template classes.

  During the creation of the model readonly template class we filter out
  the needed and unnecessary properties from the model using the attributes
  specified in the Meta class of the model readonly template class.
  """

  def __new__(cls, class_name, bases, dict):
    """Method that alters the class attributes in the model readonly template.

    This method alters the creation of the object before initialization by
    adding in the model's attributes that needs to be rendered. It also
    adds other necessary options extracted from the Meta class inside the
    model's readonly template class.
    """
    opts = ModelReadOnlyTemplateOptions(dict.get('Meta', None))

    if opts.model is not None:
      model_fields = SortedDict()
      model_hidden_fields = {}
      for name, prop in sorted(opts.model.properties().iteritems(),
                               key=lambda prop: prop[1].creation_counter):
        if opts.fields and name not in opts.fields:
          if opts.hidden_fields and name in opts.hidden_fields:
            model_hidden_fields[name] = prop
          continue
        if opts.exclude and name in opts.exclude:
          continue

        model_fields[name] = prop

      dict['fields'] = model_fields
      dict['hidden_fields'] = model_hidden_fields
      dict['renderers'] = opts.renderers or {}

    if opts.css_prefix:
      dict['css_prefix'] = opts.css_prefix

    return super(ModelReadOnlyTemplateMetaclass, cls).__new__(
        cls, class_name, bases, dict)

class ModelReadOnlyTemplate(object):
  """A base class that constructs the readonly template for a given model.

  This uses the same notion that Django's forms APIs use to generate forms
  for given models. The idea is completely inspired from Django's ModelForm
  APIs and tries to mimic the same names that is used there in order to
  provide consistency.

  In addition, to provide more consistency with the Melange's framework
  itself we also add a render method that render this class into an html
  template.
  """

  __metaclass__ = ModelReadOnlyTemplateMetaclass

  def __init__(self, instance=None):
    """Constructor to initialize the model instance.

    The readonly template will be rendered for the data in this model instance.
    """
    self.instance = instance

    for name, field in self.hidden_fields.items():
      self.hidden_fields[name] = getattr(self.instance, name)

  def __iter__(self):
    """Iterator yielding groups of model instance's properties to be rendered.
    """
    grouping = collections.defaultdict(list)

    for name, field in self.fields.items():
      group = getattr(field, 'group', '0. ')
      grouping[group].append((field.verbose_name, getattr(
          self.instance, name)))

    rexp = re.compile(r"\d+. ")

    for group, fields in sorted(grouping.items()):
      yield rexp.sub('', group), fields

  def render(self):
    """Renders the html collecting the attributes available in this class.
    """

    context = {
      'model': self,
      'hidden_fields': self.hidden_fields,
      'css_prefix': self.css_prefix,
    }
    rendered = loader.render_to_string(self.template_path,
                                       dictionary=context)
    return rendered

class SurveyRecordReadOnlyTemplate(ModelReadOnlyTemplate):
  """A base class that constructs the readonly template for given survey record.

  This uses the same notion that is used to build the model based readonly
  templates but the schema read from the survey schema rather than the model.
  """

  template_path = 'modules/gsoc/_survey/readonly_template.html'

  def __init__(self, instance=None):
    """Constructor to initialize the model instance.

    The readonly template will be rendered for the data in this model instance.
    """
    self.instance = instance
    self.schema = None
    if self.instance:
      self.schema = SurveySchema(self.instance.survey)

  def __iter__(self):
    """Iterator yielding groups of record instance's properties to be rendered.
    """
    for name, field in self.fields.items():
      renderer = self.renderers.get(name)
      if renderer:
        val = renderer(self.instance)
      else:
        val = getattr(self.instance, name)
      yield field.verbose_name, val

    if self.schema:
      for field in self.schema:
        field_id = field.getFieldName()
        label = field.getLabel()
        value = getattr(self.instance, field_id, 'N/A')
        if isinstance(value, list):
          value = ', '.join(value)

        yield label, value

  def render(self):
    """Renders the html collecting the attributes available in this class.
    """

    context = {
      'record': self,
      'hidden_fields': self.hidden_fields,
      'css_prefix': self.css_prefix,
    }

    meta = getattr(self, 'Meta', None)
    context['survey_name'] = getattr(meta, 'survey_name', '')

    rendered = loader.render_to_string(self.template_path,
                                       dictionary=context)
    return rendered
<|endoftext|>