<commit_before>from flask import json
from jinja2 import escape
from wtforms.widgets import HTMLString, html_params

from flask_admin._compat import as_unicode, text_type
from flask_admin.babel import gettext
from flask_admin.helpers import get_url
from flask_admin.form import RenderTemplateWidget

class InlineFieldListWidget(RenderTemplateWidget):
    def __init__(self):
        super(InlineFieldListWidget, self).__init__('admin/model/inline_field_list.html')

class InlineFormWidget(RenderTemplateWidget):
    def __init__(self):
        super(InlineFormWidget, self).__init__('admin/model/inline_form.html')

    def __call__(self, field, **kwargs):
        kwargs.setdefault('form_opts', getattr(field, 'form_opts', None))
        return super(InlineFormWidget, self).__call__(field, **kwargs)

class AjaxSelect2Widget(object):
    def __init__(self, multiple=False):
        self.multiple = multiple

    def __call__(self, field, **kwargs):
        kwargs.setdefault('data-role', 'select2-ajax')
        kwargs.setdefault('data-url', get_url('.ajax_lookup', name=field.loader.name))

        allow_blank = getattr(field, 'allow_blank', False)
        if allow_blank and not self.multiple:
            kwargs['data-allow-blank'] = u'1'

        kwargs.setdefault('id', field.id)
        kwargs.setdefault('type', 'hidden')

        if self.multiple:
            result = []
            ids = []

            for value in field.data:
                data = field.loader.format(value)
                result.append(data)
                ids.append(as_unicode(data[0]))

            separator = getattr(field, 'separator', ',')

            kwargs['value'] = separator.join(ids)
            kwargs['data-json'] = json.dumps(result)
            kwargs['data-multiple'] = u'1'
        else:
            data = field.loader.format(field.data)

            if data:
                kwargs['value'] = data[0]
                kwargs['data-json'] = json.dumps(data)

        placeholder = gettext(field.loader.options.get('placeholder', 'Please select model'))
        kwargs.setdefault('data-placeholder', placeholder)

        return HTMLString('<input %s>' % html_params(name=field.name, **kwargs))

class XEditableWidget(object):
    """
        WTForms widget that provides in-line editing for the list view.

        Determines how to display the x-editable/ajax form based on the
        field inside of the FieldList (StringField, IntegerField, etc).
    """
    def __call__(self, field, **kwargs):
        kwargs.setdefault('data-value', kwargs.pop('display_value', ''))

        kwargs.setdefault('data-role', 'x-editable')
        kwargs.setdefault('data-url', './ajax/update/')

        kwargs.setdefault('id', field.id)
        kwargs.setdefault('name', field.name)
        kwargs.setdefault('href', '#')

        if not kwargs.get('pk'):
            raise Exception('pk required')
        kwargs['data-pk'] = str(kwargs.pop("pk"))

        kwargs['data-csrf'] = kwargs.pop("csrf", "")

        kwargs = self.get_kwargs(field, kwargs)

        return HTMLString(
            '%s' % (html_params(**kwargs),
                              escape(kwargs['data-value']))
        )

    def get_kwargs(self, field, kwargs):
        """
            Return extra kwargs based on the field type.
        """
        if field.type == 'StringField':
            kwargs['data-type'] = 'text'
        elif field.type == 'TextAreaField':
            kwargs['data-type'] = 'textarea'
            kwargs['data-rows'] = '5'
        elif field.type == 'BooleanField':
            kwargs['data-type'] = 'select'
            # data-source = dropdown options
            kwargs['data-source'] = json.dumps([
                {'value': '', 'text': gettext('No')},
                {'value': '1', 'text': gettext('Yes')}
            ])
            kwargs['data-role'] = 'x-editable-boolean'
        elif field.type in ['Select2Field', 'SelectField']:
            kwargs['data-type'] = 'select'
            choices = [{'value': x, 'text': y} for x, y in field.choices]

            # prepend a blank field to choices if allow_blank = True
            if getattr(field, 'allow_blank', False):
                choices.insert(0, {'value': '__None', 'text': ''})

            # json.dumps fixes issue with unicode strings not loading correctly
            kwargs['data-source'] = json.dumps(choices)
        elif field.type == 'DateField':
            kwargs['data-type'] = 'combodate'
            kwargs['data-format'] = 'YYYY-MM-DD'
            kwargs['data-template'] = 'YYYY-MM-DD'
        elif field.type == 'DateTimeField':
            kwargs['data-type'] = 'combodate'
            kwargs['data-format'] = 'YYYY-MM-DD HH:mm:ss'
            kwargs['data-template'] = 'YYYY-MM-DD  HH:mm:ss'
            # x-editable-combodate uses 1 minute increments
            kwargs['data-role'] = 'x-editable-combodate'
        elif field.type == 'TimeField':
            kwargs['data-type'] = 'combodate'
            kwargs['data-format'] = 'HH:mm:ss'
            kwargs['data-template'] = 'HH:mm:ss'
            kwargs['data-role'] = 'x-editable-combodate'
        elif field.type == 'IntegerField':
            kwargs['data-type'] = 'number'
        elif field.type in ['FloatField', 'DecimalField']:
            kwargs['data-type'] = 'number'
            kwargs['data-step'] = 'any'
        elif field.type in ['QuerySelectField', 'ModelSelectField',
                            'QuerySelectMultipleField', 'KeyPropertyField']:
            # QuerySelectField and ModelSelectField are for relations
            kwargs['data-type'] = 'select'

            choices = []
            selected_ids = []
            for value, label, selected in field.iter_choices():
                try:
                    label = text_type(label)
                except TypeError:
                    # unable to display text value
                    label = ''
                choices.append({'value': text_type(value), 'text': label})
                if selected:
                    selected_ids.append(value)

            # blank field is already included if allow_blank
            kwargs['data-source'] = json.dumps(choices)

            if field.type == 'QuerySelectMultipleField':
                kwargs['data-type'] = 'select2'
                kwargs['data-role'] = 'x-editable-select2-multiple'

                # must use id instead of text or prefilled values won't work
                separator = getattr(field, 'separator', ',')
                kwargs['data-value'] = separator.join(selected_ids)
        else:
            raise Exception('Unsupported field type: %s' % (type(field),))

        return kwargs
<commit_msg>fix gettext for 'Please select model'<commit_after>from flask import json
from jinja2 import escape
from wtforms.widgets import HTMLString, html_params

from flask_admin._compat import as_unicode, text_type
from flask_admin.babel import gettext
from flask_admin.helpers import get_url
from flask_admin.form import RenderTemplateWidget

class InlineFieldListWidget(RenderTemplateWidget):
    def __init__(self):
        super(InlineFieldListWidget, self).__init__('admin/model/inline_field_list.html')

class InlineFormWidget(RenderTemplateWidget):
    def __init__(self):
        super(InlineFormWidget, self).__init__('admin/model/inline_form.html')

    def __call__(self, field, **kwargs):
        kwargs.setdefault('form_opts', getattr(field, 'form_opts', None))
        return super(InlineFormWidget, self).__call__(field, **kwargs)

class AjaxSelect2Widget(object):
    def __init__(self, multiple=False):
        self.multiple = multiple

    def __call__(self, field, **kwargs):
        kwargs.setdefault('data-role', 'select2-ajax')
        kwargs.setdefault('data-url', get_url('.ajax_lookup', name=field.loader.name))

        allow_blank = getattr(field, 'allow_blank', False)
        if allow_blank and not self.multiple:
            kwargs['data-allow-blank'] = u'1'

        kwargs.setdefault('id', field.id)
        kwargs.setdefault('type', 'hidden')

        if self.multiple:
            result = []
            ids = []

            for value in field.data:
                data = field.loader.format(value)
                result.append(data)
                ids.append(as_unicode(data[0]))

            separator = getattr(field, 'separator', ',')

            kwargs['value'] = separator.join(ids)
            kwargs['data-json'] = json.dumps(result)
            kwargs['data-multiple'] = u'1'
        else:
            data = field.loader.format(field.data)

            if data:
                kwargs['value'] = data[0]
                kwargs['data-json'] = json.dumps(data)

        placeholder = field.loader.options.get('placeholder', gettext('Please select model'))
        kwargs.setdefault('data-placeholder', placeholder)

        return HTMLString('<input %s>' % html_params(name=field.name, **kwargs))

class XEditableWidget(object):
    """
        WTForms widget that provides in-line editing for the list view.

        Determines how to display the x-editable/ajax form based on the
        field inside of the FieldList (StringField, IntegerField, etc).
    """
    def __call__(self, field, **kwargs):
        kwargs.setdefault('data-value', kwargs.pop('display_value', ''))

        kwargs.setdefault('data-role', 'x-editable')
        kwargs.setdefault('data-url', './ajax/update/')

        kwargs.setdefault('id', field.id)
        kwargs.setdefault('name', field.name)
        kwargs.setdefault('href', '#')

        if not kwargs.get('pk'):
            raise Exception('pk required')
        kwargs['data-pk'] = str(kwargs.pop("pk"))

        kwargs['data-csrf'] = kwargs.pop("csrf", "")

        kwargs = self.get_kwargs(field, kwargs)

        return HTMLString(
            '%s' % (html_params(**kwargs),
                              escape(kwargs['data-value']))
        )

    def get_kwargs(self, field, kwargs):
        """
            Return extra kwargs based on the field type.
        """
        if field.type == 'StringField':
            kwargs['data-type'] = 'text'
        elif field.type == 'TextAreaField':
            kwargs['data-type'] = 'textarea'
            kwargs['data-rows'] = '5'
        elif field.type == 'BooleanField':
            kwargs['data-type'] = 'select'
            # data-source = dropdown options
            kwargs['data-source'] = json.dumps([
                {'value': '', 'text': gettext('No')},
                {'value': '1', 'text': gettext('Yes')}
            ])
            kwargs['data-role'] = 'x-editable-boolean'
        elif field.type in ['Select2Field', 'SelectField']:
            kwargs['data-type'] = 'select'
            choices = [{'value': x, 'text': y} for x, y in field.choices]

            # prepend a blank field to choices if allow_blank = True
            if getattr(field, 'allow_blank', False):
                choices.insert(0, {'value': '__None', 'text': ''})

            # json.dumps fixes issue with unicode strings not loading correctly
            kwargs['data-source'] = json.dumps(choices)
        elif field.type == 'DateField':
            kwargs['data-type'] = 'combodate'
            kwargs['data-format'] = 'YYYY-MM-DD'
            kwargs['data-template'] = 'YYYY-MM-DD'
        elif field.type == 'DateTimeField':
            kwargs['data-type'] = 'combodate'
            kwargs['data-format'] = 'YYYY-MM-DD HH:mm:ss'
            kwargs['data-template'] = 'YYYY-MM-DD  HH:mm:ss'
            # x-editable-combodate uses 1 minute increments
            kwargs['data-role'] = 'x-editable-combodate'
        elif field.type == 'TimeField':
            kwargs['data-type'] = 'combodate'
            kwargs['data-format'] = 'HH:mm:ss'
            kwargs['data-template'] = 'HH:mm:ss'
            kwargs['data-role'] = 'x-editable-combodate'
        elif field.type == 'IntegerField':
            kwargs['data-type'] = 'number'
        elif field.type in ['FloatField', 'DecimalField']:
            kwargs['data-type'] = 'number'
            kwargs['data-step'] = 'any'
        elif field.type in ['QuerySelectField', 'ModelSelectField',
                            'QuerySelectMultipleField', 'KeyPropertyField']:
            # QuerySelectField and ModelSelectField are for relations
            kwargs['data-type'] = 'select'

            choices = []
            selected_ids = []
            for value, label, selected in field.iter_choices():
                try:
                    label = text_type(label)
                except TypeError:
                    # unable to display text value
                    label = ''
                choices.append({'value': text_type(value), 'text': label})
                if selected:
                    selected_ids.append(value)

            # blank field is already included if allow_blank
            kwargs['data-source'] = json.dumps(choices)

            if field.type == 'QuerySelectMultipleField':
                kwargs['data-type'] = 'select2'
                kwargs['data-role'] = 'x-editable-select2-multiple'

                # must use id instead of text or prefilled values won't work
                separator = getattr(field, 'separator', ',')
                kwargs['data-value'] = separator.join(selected_ids)
        else:
            raise Exception('Unsupported field type: %s' % (type(field),))

        return kwargs
<|endoftext|>


<commit_before>r"""
uBiome fastq data extraction.

Copyright (C) 2016 PersonalGenomes.org

This software is shared under the "MIT License" license (aka "Expat License"),
see LICENSE.TXT for full license text.

May be used on the command line. For example, the following command:

    python -m sources.ubiome ~/Downloads/uBiome.zip files

Will assemble processed data sets in files/
"""

import os
import shutil
import tempfile
import zipfile

import requests

from boto.s3.connection import S3Connection

from data_retrieval.files import get_remote_file, mv_tempfile_to_output

def s3_connection():
    """
    Get an S3 connection using environment variables.
    """
    key = os.getenv('AWS_ACCESS_KEY_ID')
    secret = os.getenv('AWS_SECRET_ACCESS_KEY')

    if not (key and secret):
        raise Exception('You must specify AWS credentials.')

    return S3Connection(key, secret)

def verify_ubiome(input_filepath, sentry=None, username=None):
    """
    Verify that this is a uBiome file.
    """
    if input_filepath.endswith('.zip'):
        zip_file = zipfile.ZipFile(input_filepath)

        file_list = [f for f in zip_file.namelist()
                     if not f.startswith('__MACOSX/')]

        for filename in file_list:
            if not filename.endswith('.fastq.gz'):
                if sentry:
                    sentry_msg = ('uBiome file did not conform to expected '
                                  'format.')

                    if username:
                        sentry_msg += ' Username: {}'.format(username)

                    sentry.captureMessage(sentry_msg)

                raise ValueError(
                    'Found a filename that did not end with ".fastq.gz": '
                    '"{}"'.format(filename))
    else:
        raise ValueError('Input file is expected to be a ZIP archive')

def create_datafiles(username, samples=None, task_id=None, update_url=None,
                     sentry=None, **kwargs):
    """
    Create Open Humans Dataset from uploaded uBiome data.

    Optional arguments:
        samples: path to an online copy of the input file
        filedir: Local filepath, folder in which to place the resulting file.
        s3_bucket_name: S3 bucket to write resulting file.
        s3_key_dir: S3 key "directory" to write resulting file. The full S3 key
                    name will add a filename to the end of s3_key_dir.

    For output: iither 'filedir' (and no S3 arguments), or both
    's3_bucket_name' and 's3_key_dir' (and no 'filedir') must be specified.
    """
    tempdir = tempfile.mkdtemp()
    temp_files = []
    data_files = []

    if not samples:
        raise Exception('`samples` parameter missing')

    for sample in samples:
        filename = get_remote_file(sample['sequence_file']['url'], tempdir)
        input_file = os.path.join(tempdir, filename)

        verify_ubiome(input_file, sentry, username)

        shutil.copyfile(input_file, os.path.join(tempdir, 'uBiome-fastq.zip'))

        temp_files.append({
            'temp_filename': 'uBiome-fastq.zip',
            'tempdir': tempdir,
            'metadata': {
                'description': 'uBiome data, original format',
                'tags': ['uBiome', 'FASTQ'],
            },
        })

        print 'Finished creating all datasets locally.'

        for file_info in temp_files:
            print 'File info: {}'.format(str(file_info))

            filename = file_info['temp_filename']
            file_tempdir = file_info['tempdir']

            output_path = mv_tempfile_to_output(
                os.path.join(file_tempdir, filename), filename, **kwargs)

            if 's3_key_dir' in kwargs and 's3_bucket_name' in kwargs:
                data_files.append({
                    's3_key': output_path,
                    'metadata': file_info['metadata'],
                })

        if samples:
            os.remove(input_file)

        os.rmdir(tempdir)

        print 'Finished moving all datasets to permanent storage.'

    if not (task_id and update_url):
        return

    task_data = {
        'task_id': task_id,
        's3_keys': [data_file['s3_key'] for data_file in data_files],
        'data_files': data_files
    }

    requests.post(update_url, json={'task_data': task_data})

# if __name__ == '__main__':
#     if len(sys.argv) != 4:
#         print ('Please specify a remote file URL, target local directory, '
#                'and username.')

#         sys.exit(1)

#     create_datafiles(input_file=sys.argv[1], filedir=sys.argv[2],
#                      username=sys.argv[3])
<commit_msg>Add taxonomy file to uBiome<commit_after>r"""
uBiome fastq data extraction.

Copyright (C) 2016 PersonalGenomes.org

This software is shared under the "MIT License" license (aka "Expat License"),
see LICENSE.TXT for full license text.

May be used on the command line. For example, the following command:

    python -m sources.ubiome ~/Downloads/uBiome.zip files

Will assemble processed data sets in files/
"""

import os
import shutil
import tempfile
import zipfile

from cStringIO import StringIO

import requests

from boto.s3.connection import S3Connection

from data_retrieval.files import get_remote_file, mv_tempfile_to_output

def s3_connection():
    """
    Get an S3 connection using environment variables.
    """
    key = os.getenv('AWS_ACCESS_KEY_ID')
    secret = os.getenv('AWS_SECRET_ACCESS_KEY')

    if not (key and secret):
        raise Exception('You must specify AWS credentials.')

    return S3Connection(key, secret)

def verify_ubiome(input_filepath, sentry=None, username=None):
    """
    Verify that this is a uBiome file.
    """
    if input_filepath.endswith('.zip'):
        zip_file = zipfile.ZipFile(input_filepath)

        file_list = [f for f in zip_file.namelist()
                     if not f.startswith('__MACOSX/')]

        for filename in file_list:
            if not filename.endswith('.fastq.gz'):
                if sentry:
                    sentry_msg = ('uBiome file did not conform to expected '
                                  'format.')

                    if username:
                        sentry_msg += ' Username: {}'.format(username)

                    sentry.captureMessage(sentry_msg)

                raise ValueError(
                    'Found a filename that did not end with ".fastq.gz": '
                    '"{}"'.format(filename))
    else:
        raise ValueError('Input file is expected to be a ZIP archive')

def create_datafiles(username, samples=None, task_id=None, update_url=None,
                     sentry=None, **kwargs):
    """
    Create Open Humans Dataset from uploaded uBiome data.

    Optional arguments:
        samples: path to an online copy of the input file
        filedir: Local filepath, folder in which to place the resulting file.
        s3_bucket_name: S3 bucket to write resulting file.
        s3_key_dir: S3 key "directory" to write resulting file. The full S3 key
                    name will add a filename to the end of s3_key_dir.

    For output: iither 'filedir' (and no S3 arguments), or both
    's3_bucket_name' and 's3_key_dir' (and no 'filedir') must be specified.
    """
    tempdir = tempfile.mkdtemp()
    temp_files = []
    data_files = []

    if not samples:
        raise Exception('`samples` parameter missing')

    for sample in samples:
        filename = get_remote_file(sample['sequence_file']['url'], tempdir)
        input_file = os.path.join(tempdir, filename)

        verify_ubiome(input_file, sentry, username)

        taxonomy = StringIO(sample['taxonomy'])

        shutil.copyfile(input_file, os.path.join(tempdir, 'uBiome-fastq.zip'))
        shutil.copyfileobj(taxonomy,
                           file(os.path.join(tempdir, 'taxonomy.json'), 'w'))

        temp_files.append({
            'temp_filename': 'taxonomy.json',
            'tempdir': tempdir,
            'metadata': {
                'desription': 'uBiome taxonomy data, original format',
                'tags': ['uBiome', 'JSON'],
            },
        })

        temp_files.append({
            'temp_filename': 'uBiome-fastq.zip',
            'tempdir': tempdir,
            'metadata': {
                'description': 'uBiome data, original format',
                'tags': ['uBiome', 'FASTQ'],
            },
        })

        print 'Finished creating all datasets locally.'

        for file_info in temp_files:
            print 'File info: {}'.format(str(file_info))

            filename = file_info['temp_filename']
            file_tempdir = file_info['tempdir']

            output_path = mv_tempfile_to_output(
                os.path.join(file_tempdir, filename), filename, **kwargs)

            if 's3_key_dir' in kwargs and 's3_bucket_name' in kwargs:
                data_files.append({
                    's3_key': output_path,
                    'metadata': file_info['metadata'],
                })

        if samples:
            os.remove(input_file)

        os.rmdir(tempdir)

        print 'Finished moving all datasets to permanent storage.'

    if not (task_id and update_url):
        return

    task_data = {
        'task_id': task_id,
        's3_keys': [data_file['s3_key'] for data_file in data_files],
        'data_files': data_files
    }

    requests.post(update_url, json={'task_data': task_data})

# if __name__ == '__main__':
#     if len(sys.argv) != 4:
#         print ('Please specify a remote file URL, target local directory, '
#                'and username.')

#         sys.exit(1)

#     create_datafiles(input_file=sys.argv[1], filedir=sys.argv[2],
#                      username=sys.argv[3])
<|endoftext|>


<commit_before>from flask import Blueprint
from flask import abort, flash, redirect, render_template, request, url_for
from flask.ext.login import current_user

from viaduct import application, db
from viaduct.helpers import flash_form_errors

from api import PageAPI
from models import Page, PageRevision, PagePermission
from forms import ChangePathForm

module = Blueprint('page', __name__)

@application.errorhandler(403)
@application.errorhandler(404)
def get_error_page(error):
	return render_template('page/view_page.htm')

@module.route('/', methods=['GET', 'POST'])
@module.route('/<path:path>', methods=['GET', 'POST'])
def get_page(path=''):
	return render_template('page/view_page.htm', page=True, path=path)

@module.route('/delete/')
@module.route('/delete/<path:page_path>')
def delete_page(page_path='', revision=''):
	page = Page.query.filter(Page.path==page_path).first()
	revisions = PageRevision.query.filter(PageRevision.page_id==page.id).all()
	for revision in revisions:
		db.session.delete(revision)
	db.session.delete(page)
	db.session.commit()
	return redirect('/')

@module.route('/edit/', methods=['GET', 'POST'])
@module.route('/edit/<path:path>', methods=['GET', 'POST'])
def edit_page(path=''):
	rights = PagePermission.get_user_rights(current_user, path)

	if not rights['safe_edit'] or not rights['unsafe_edit']:
		abort(403)

	form = EditPageForm()

	if rights['unsafe_edit']:
		form.content_type.choices.append(('1', 'HTML'))

	if rights['safe_edit']:
		form.content_type.choices.append(('2', 'Markdown'))

	page = Page.query.filter(Page.path==path).first()
	revision = None

	if page:
		revision = page.revisions.order_by(PageRevision.timestamp.desc()).first()

	if request.method == 'POST':
		title = request.form['title'].strip()
		content = request.form['content'].strip()
		try:
			priority = int(request.form['priority'].strip())
		except:
			priority = 0

		# A new page is being created
		if not page:
			page = Page(page_path)

			db.session.add(page)
			db.session.commit()

		revision = PageRevision(page, current_user, title, content, priority)

		db.session.add(revision)
		db.session.commit()

		flash('The page has been saved.', 'success')

		return redirect(url_for('page.get_page', page=True, path=page_path))

	return render_template('page/edit_page.htm', form=form,
		revision=revision, page=path)

<commit_msg>Added the import.<commit_after>from flask import Blueprint
from flask import abort, flash, redirect, render_template, request, url_for
from flask.ext.login import current_user

from viaduct import application, db
from viaduct.helpers import flash_form_errors

from api import PageAPI
from models import Page, PageRevision, PagePermission
from forms import ChangePathForm, EditPageForm

module = Blueprint('page', __name__)

@application.errorhandler(403)
@application.errorhandler(404)
def get_error_page(error):
	return render_template('page/view_page.htm')

@module.route('/', methods=['GET', 'POST'])
@module.route('/<path:path>', methods=['GET', 'POST'])
def get_page(path=''):
	return render_template('page/view_page.htm', page=True, path=path)

@module.route('/delete/')
@module.route('/delete/<path:page_path>')
def delete_page(page_path='', revision=''):
	page = Page.query.filter(Page.path==page_path).first()
	revisions = PageRevision.query.filter(PageRevision.page_id==page.id).all()
	for revision in revisions:
		db.session.delete(revision)
	db.session.delete(page)
	db.session.commit()
	return redirect('/')

@module.route('/edit/', methods=['GET', 'POST'])
@module.route('/edit/<path:path>', methods=['GET', 'POST'])
def edit_page(path=''):
	rights = PagePermission.get_user_rights(current_user, path)

	if not rights['safe_edit'] or not rights['unsafe_edit']:
		abort(403)

	form = EditPageForm()

	if rights['unsafe_edit']:
		form.content_type.choices.append(('1', 'HTML'))

	if rights['safe_edit']:
		form.content_type.choices.append(('2', 'Markdown'))

	page = Page.query.filter(Page.path==path).first()
	revision = None

	if page:
		revision = page.revisions.order_by(PageRevision.timestamp.desc()).first()

	if request.method == 'POST':
		title = request.form['title'].strip()
		content = request.form['content'].strip()
		try:
			priority = int(request.form['priority'].strip())
		except:
			priority = 0

		# A new page is being created
		if not page:
			page = Page(page_path)

			db.session.add(page)
			db.session.commit()

		revision = PageRevision(page, current_user, title, content, priority)

		db.session.add(revision)
		db.session.commit()

		flash('The page has been saved.', 'success')

		return redirect(url_for('page.get_page', page=True, path=page_path))

	return render_template('page/edit_page.htm', form=form,
		revision=revision, page=path)

<|endoftext|>


<commit_before># Copyright 2019 Omar Castiñeira, Comunitea Servicios Tecnológicos S.L.
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
from odoo import models, fields

class AccountInvoiceReport(models.Model):

    _inherit = 'account.invoice.report'

    payment_mode_id = fields.Many2one('account.payment.mode', 'Payment mode')
    number = fields.Char('Number')
    benefit = fields.Float('Benefit')
    brand_name = fields.Char('Brand name')
    area_id = fields.Many2one('res.partner.area', 'Area')
    commercial_region_ids = fields.\
        Many2many(related='area_id.commercial_region_ids')
    contact_id = fields.Many2one('res.partner', 'Partner Contact',
                                 readonly=True)
    manufacturer = fields.Char('Product Manufacturer')
    parent_product_categ_id = fields.Many2one('product.category','Parent product category')

    def _select(self):
        select_str = super()._select()
        select_str += ', sub.payment_mode_id as payment_mode_id,' \
                      ' sub.number as number' \
                      ", CASE WHEN sub.type IN ('out_refund') " \
                      "THEN -sub.benefit " \
                      " WHEN sub.type IN ('out_invoice') THEN sub.benefit " \
                      " ELSE 0 END as benefit" \
                      ', sub.name as brand_name, sub.area_id' \
                      ', sub.contact_id as contact_id' \
                      ', sub.manufacturer as manufacturer'\
                      ', sub.parent_product_categ_id as parent_product_categ_id'
        return select_str

    def _sub_select(self):
        select_str = super()._sub_select()
        select_str += ', ai.payment_mode_id,' \
                      ' ai.number , sum(ail.quantity * ail.price_unit * ' \
                      '(100.0-ail.discount) / 100.0) - ' \
                      'sum(coalesce(ail.cost_unit, 0)*ail.quantity) ' \
                      'as benefit, pb.name, rpa.id as area_id, ' \
                      'ai.partner_shipping_id as contact_id, ' \
                      'rpm.name as manufacturer, '\
                      'pc.parent_id as parent_product_categ_id'
        return select_str

    def _from(self):
        from_str = super()._from()
        from_str += ' LEFT JOIN product_brand pb ON pt.product_brand_id = ' \
                    'pb.id LEFT JOIN res_partner_area rpa ON rpa.id = ' \
                    'partner.area_id LEFT JOIN res_partner rpc on rpc.id = ' \
                    'ai.partner_shipping_id LEFT JOIN res_partner rpm ON rpm.id = ' \
                    'pt.manufacturer '\
                    'LEFT JOIN product_category pc ON pt.categ_id = pc.id  '
        return from_str

    def _group_by(self):
        group_by_str = super()._group_by()
        group_by_str += ', ai.payment_mode_id,' \
                        ' ai.number,' \
                        ' pb.name, ' \
                        ' rpa.id, ai.partner_shipping_id, ' \
                        'rpm.name, '\
                        'pc.parent_id'

        return group_by_str
<commit_msg>[IMP] custom_account: Añadido poder filtrar por tag de producto en análisis de facturas<commit_after># Copyright 2019 Omar Castiñeira, Comunitea Servicios Tecnológicos S.L.
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
from odoo import models, fields

class AccountInvoiceReport(models.Model):

    _inherit = 'account.invoice.report'

    payment_mode_id = fields.Many2one('account.payment.mode', 'Payment mode')
    number = fields.Char('Number')
    benefit = fields.Float('Benefit')
    brand_name = fields.Char('Brand name')
    area_id = fields.Many2one('res.partner.area', 'Area')
    commercial_region_ids = fields.\
        Many2many(related='area_id.commercial_region_ids')
    contact_id = fields.Many2one('res.partner', 'Partner Contact',
                                 readonly=True)
    manufacturer = fields.Char('Product Manufacturer')
    parent_product_categ_id = fields.Many2one('product.category','Parent product category')
    tag_ids = fields. \
        Many2many(related='product_id.tag_ids')

    def _select(self):
        select_str = super()._select()
        select_str += ', sub.payment_mode_id as payment_mode_id,' \
                      ' sub.number as number' \
                      ", CASE WHEN sub.type IN ('out_refund') " \
                      "THEN -sub.benefit " \
                      " WHEN sub.type IN ('out_invoice') THEN sub.benefit " \
                      " ELSE 0 END as benefit" \
                      ', sub.name as brand_name, sub.area_id' \
                      ', sub.contact_id as contact_id' \
                      ', sub.manufacturer as manufacturer'\
                      ', sub.parent_product_categ_id as parent_product_categ_id'
        return select_str

    def _sub_select(self):
        select_str = super()._sub_select()
        select_str += ', ai.payment_mode_id,' \
                      ' ai.number , sum(ail.quantity * ail.price_unit * ' \
                      '(100.0-ail.discount) / 100.0) - ' \
                      'sum(coalesce(ail.cost_unit, 0)*ail.quantity) ' \
                      'as benefit, pb.name, rpa.id as area_id, ' \
                      'ai.partner_shipping_id as contact_id, ' \
                      'rpm.name as manufacturer, '\
                      'pc.parent_id as parent_product_categ_id'
        return select_str

    def _from(self):
        from_str = super()._from()
        from_str += ' LEFT JOIN product_brand pb ON pt.product_brand_id = ' \
                    'pb.id LEFT JOIN res_partner_area rpa ON rpa.id = ' \
                    'partner.area_id LEFT JOIN res_partner rpc on rpc.id = ' \
                    'ai.partner_shipping_id LEFT JOIN res_partner rpm ON rpm.id = ' \
                    'pt.manufacturer '\
                    'LEFT JOIN product_category pc ON pt.categ_id = pc.id  '
        return from_str

    def _group_by(self):
        group_by_str = super()._group_by()
        group_by_str += ', ai.payment_mode_id,' \
                        ' ai.number,' \
                        ' pb.name, ' \
                        ' rpa.id, ai.partner_shipping_id, ' \
                        'rpm.name, '\
                        'pc.parent_id'

        return group_by_str
<|endoftext|>


<commit_before>class AliasDict(dict):
    def __init__(self, *args, **kwargs):
        super(AliasDict, self).__init__(*args, **kwargs)
        self.aliases = {}

    def alias(self, from_, to):
        self.aliases[from_] = to

    def unalias(self, from_):
        del self.aliases[from_]

    def _handle(self, key, value, single, multi, unaliased):
        # Attribute existence test required to not blow up when deepcopy'd
        if hasattr(self, 'aliases') and key in self.aliases:
            target = self.aliases[key]
            # Single-string targets
            if isinstance(target, basestring):
                return single(self, target, value)
            # Multi-string targets
            else:
                return multi(self, target, value)
        else:
            return unaliased(self, key, value)

    def _single(self, target):
        return isinstance(target, basestring)

    def __setitem__(self, key, value):
        def single(d, target, value): d[target] = value

        def multi(d, target, value):
            for subkey in target:
                d[subkey] = value

        def unaliased(d, key, value): super(AliasDict, d).__setitem__(key, value)

        return self._handle(key, value, single, multi, unaliased)

    def __getitem__(self, key):
        def single(d, target, value): return d[target]

        def multi(d, target, value):
            msg = "Multi-target aliases have no well-defined value and can't be read."
            raise ValueError(msg)

        def unaliased(d, key, value): return super(AliasDict, d).__getitem__(key)

        return self._handle(key, None, single, multi, unaliased)

    def __contains__(self, key):
        def single(d, target, value): return target in d

        def multi(d, target, value):
            return all(subkey in self for subkey in self.aliases[key])

        def unaliased(d, key, value):
            return super(AliasDict, d).__contains__(key)

        return self._handle(key, None, single, multi, unaliased)

    def __delitem__(self, key):
        def single(d, target, value): del d[target]

        def multi(d, target, value):
            for subkey in self.aliases[key]:
                del self[subkey]

        def unaliased(d, key, value):
            return super(AliasDict, d).__delitem__(key)

        return self._handle(key, None, single, multi, unaliased)
<commit_msg>Another minor refactor<commit_after>class AliasDict(dict):
    def __init__(self, *args, **kwargs):
        super(AliasDict, self).__init__(*args, **kwargs)
        self.aliases = {}

    def alias(self, from_, to):
        self.aliases[from_] = to

    def unalias(self, from_):
        del self.aliases[from_]

    def _handle(self, key, value, single, multi, unaliased):
        # Attribute existence test required to not blow up when deepcopy'd
        if hasattr(self, 'aliases') and key in self.aliases:
            target = self.aliases[key]
            # Single-string targets
            if isinstance(target, basestring):
                return single(self, target, value)
            # Multi-string targets
            else:
                if multi:
                    return multi(self, target, value)
                else:
                    for subkey in target:
                        single(self, subkey, value)
        else:
            return unaliased(self, key, value)

    def _single(self, target):
        return isinstance(target, basestring)

    def __setitem__(self, key, value):
        def single(d, target, value): d[target] = value
        def unaliased(d, key, value): super(AliasDict, d).__setitem__(key, value)
        return self._handle(key, value, single, None, unaliased)

    def __getitem__(self, key):
        def single(d, target, value): return d[target]
        def unaliased(d, key, value): return super(AliasDict, d).__getitem__(key)

        def multi(d, target, value):
            msg = "Multi-target aliases have no well-defined value and can't be read."
            raise ValueError(msg)

        return self._handle(key, None, single, multi, unaliased)

    def __contains__(self, key):
        def single(d, target, value): return target in d

        def multi(d, target, value):
            return all(subkey in self for subkey in self.aliases[key])

        def unaliased(d, key, value):
            return super(AliasDict, d).__contains__(key)

        return self._handle(key, None, single, multi, unaliased)

    def __delitem__(self, key):
        def single(d, target, value): del d[target]

        def unaliased(d, key, value):
            return super(AliasDict, d).__delitem__(key)

        return self._handle(key, None, single, None, unaliased)
<|endoftext|>


<commit_before># MySQL-specific implementations for south
# Original author: Andrew Godwin
# Patches by: F. Gabriel Gosselin <gabrielNOSPAM@evidens.ca>

from south.db import generic
from south.db.generic import DryRunError, INVALID
from south.logger import get_logger

def delete_column_constraints(func):
    """
    Decorates column operation functions for MySQL.
    Deletes the constraints from the database and clears local cache.
    """
    def _column_rm(self, table_name, column_name, *args, **opts):
        # Delete foreign key constraints
        try:
            self.delete_foreign_key(table_name, column_name)
        except ValueError:
            pass  # If no foreign key on column, OK because it checks first
        # Delete constraints referring to this column
        try:
            reverse = self._lookup_reverse_constraint(table_name, column_name)
            for cname, rtable, rcolumn in reverse:
                self.delete_foreign_key(rtable, rcolumn)
        except DryRunError:
            pass
        return func(self, table_name, column_name, *args, **opts)
    return _column_rm

def copy_column_constraints(func):
    """
    Decorates column operation functions for MySQL.
    Determines existing constraints and copies them to a new column
    """
    def _column_cp(self, table_name, column_old, column_new, *args, **opts):
        # Copy foreign key constraint
        try:
            constraint = self._find_foreign_constraints(table_name, column_old)[0]
            (ftable, fcolumn) = self._lookup_constraint_references(table_name, constraint)
            if ftable and fcolumn:
                fk_sql = self.foreign_key_sql(
                            table_name, column_new, ftable, fcolumn)
                get_logger().debug("Foreign key SQL: " + fk_sql)
                self.add_deferred_sql(fk_sql)
        except IndexError:
            pass  # No constraint exists so ignore
        except DryRunError:
            pass
        # Copy constraints referring to this column
        try:
            reverse = self._lookup_reverse_constraint(table_name, column_old)
            for cname, rtable, rcolumn in reverse:
                fk_sql = self.foreign_key_sql(
                        rtable, rcolumn, table_name, column_new)
                self.add_deferred_sql(fk_sql)
        except DryRunError:
            pass
        return func(self, table_name, column_old, column_new, *args, **opts)
    return _column_cp

def invalidate_table_constraints(func):
    """
    For MySQL we grab all table constraints simultaneously, so this is
    effective.
    It further solves the issues of invalidating referred table constraints.
    """
    def _cache_clear(self, table, *args, **opts):
        db_name = self._get_setting('NAME')
        if db_name in self._constraint_cache:
            del self._constraint_cache[db_name]
        if db_name in self._reverse_cache:
            del self._reverse_cache[db_name]
        if db_name in self._constraint_references:
            del self._constraint_references[db_name]
        return func(self, table, *args, **opts)
    return _cache_clear

class DatabaseOperations(generic.DatabaseOperations):
    """
    MySQL implementation of database operations.

    MySQL has no DDL transaction support This can confuse people when they ask
    how to roll back - hence the dry runs, etc., found in the migration code.
    """

    backend_name = "mysql"
    alter_string_set_type = ''
    alter_string_set_null = 'MODIFY %(column)s %(type)s NULL;'
    alter_string_drop_null = 'MODIFY %(column)s %(type)s NOT NULL;'
    drop_index_string = 'DROP INDEX %(index_name)s ON %(table_name)s'
    delete_primary_key_sql = "ALTER TABLE %(table)s DROP PRIMARY KEY"
    delete_foreign_key_sql = "ALTER TABLE %(table)s DROP FOREIGN KEY %(constraint)s"
    delete_unique_sql = "ALTER TABLE %s DROP INDEX %s"
    rename_table_sql = "RENAME TABLE %s TO %s;"

    allows_combined_alters = False
    has_check_constraints = False

    geom_types = ['geometry', 'point', 'linestring', 'polygon']
    text_types = ['text', 'blob']

    def __init__(self, db_alias):
        self._constraint_references = {}
        self._reverse_cache = {}
        super(DatabaseOperations, self).__init__(db_alias)
        if self._has_setting('STORAGE_ENGINE') and self._get_setting('STORAGE_ENGINE'):
            self.create_table_sql += ' ENGINE=%s' % self._get_setting('STORAGE_ENGINE')

    def _is_valid_cache(self, db_name, table_name):
        cache = self._constraint_cache
        # we cache the whole db so if there are any tables table_name is valid
        return db_name in cache and cache[db_name].get(table_name, None) is not INVALID

    def _fill_constraint_cache(self, db_name, table_name):
        # for MySQL grab all constraints for this database.  It's just as cheap as a single column.
        self._constraint_cache[db_name] = {}
        self._constraint_cache[db_name][table_name] = {}
        self._reverse_cache[db_name] = {}
        self._constraint_references[db_name] = {}

        name_query = """
            SELECT kc.`constraint_name`, kc.`column_name`, kc.`table_name`,
                kc.`referenced_table_name`, kc.`referenced_column_name`
            FROM information_schema.key_column_usage AS kc
            WHERE
                kc.table_schema = %s
        """
        rows = self.execute(name_query, [db_name])
        if not rows:
            return
        cnames = {}
        for constraint, column, table, ref_table, ref_column in rows:
            key = (table, constraint)
            cnames.setdefault(key, set())
            cnames[key].add((column, ref_table, ref_column))

        type_query = """
            SELECT c.constraint_name, c.table_name, c.constraint_type
            FROM information_schema.table_constraints AS c
            WHERE
                c.table_schema = %s
        """
        rows = self.execute(type_query, [db_name])
        for constraint, table, kind in rows:
            key = (table, constraint)
            self._constraint_cache[db_name].setdefault(table, {})
            try:
                cols = cnames[key]
            except KeyError:
                cols = set()
            for column_set in cols:
                (column, ref_table, ref_column) = column_set
                self._constraint_cache[db_name][table].setdefault(column, set())
                if kind == 'FOREIGN KEY':
                    self._constraint_cache[db_name][table][column].add((kind,
                        constraint))
                    # Create constraint lookup, see constraint_references
                    self._constraint_references[db_name][(table,
                        constraint)] = (ref_table, ref_column)
                    # Create reverse table lookup, reverse_lookup
                    self._reverse_cache[db_name].setdefault(ref_table, {})
                    self._reverse_cache[db_name][ref_table].setdefault(ref_column,
                            set())
                    self._reverse_cache[db_name][ref_table][ref_column].add(
                            (constraint, table, column))
                else:
                    self._constraint_cache[db_name][table][column].add((kind,
                    constraint))

    def connection_init(self):
        """
        Run before any SQL to let database-specific config be sent as a command,
        e.g. which storage engine (MySQL) or transaction serialisability level.
        """
        cursor = self._get_connection().cursor()
        if self._has_setting('STORAGE_ENGINE') and self._get_setting('STORAGE_ENGINE'):
            cursor.execute("SET storage_engine=%s;" % self._get_setting('STORAGE_ENGINE'))

    def start_transaction(self):
        super(DatabaseOperations, self).start_transaction()
        self.execute("SET FOREIGN_KEY_CHECKS=0;")

    @copy_column_constraints
    @delete_column_constraints
    @invalidate_table_constraints
    def rename_column(self, table_name, old, new):
        if old == new or self.dry_run:
            return []

        rows = [x for x in self.execute('DESCRIBE %s' % (self.quote_name(table_name),)) if x[0] == old]

        if not rows:
            raise ValueError("No column '%s' in '%s'." % (old, table_name))

        params = (
            self.quote_name(table_name),
            self.quote_name(old),
            self.quote_name(new),
            rows[0][1],
            rows[0][2] == "YES" and "NULL" or "NOT NULL",
            rows[0][4] and "DEFAULT " or "",
            rows[0][4] and "%s" or "",
            rows[0][5] or "",
        )

        sql = 'ALTER TABLE %s CHANGE COLUMN %s %s %s %s %s %s %s;' % params

        if rows[0][4]:
            self.execute(sql, (rows[0][4],))
        else:
            self.execute(sql)

    @delete_column_constraints
    def delete_column(self, table_name, name):
        super(DatabaseOperations, self).delete_column(table_name, name)

    @invalidate_table_constraints
    def create_table(self, table_name, fields):
        super(DatabaseOperations, self).create_table(table_name, fields)

    @invalidate_table_constraints
    def rename_table(self, old_table_name, table_name):
        super(DatabaseOperations, self).rename_table(old_table_name,
                table_name)

    @invalidate_table_constraints
    def delete_table(self, table_name):
        super(DatabaseOperations, self).delete_table(table_name)

    def _lookup_constraint_references(self, table_name, cname):
        """
        Provided an existing table and constraint, returns tuple of (foreign
        table, column)
        """
        db_name = self._get_setting('NAME')
        try:
            return self._constraint_references[db_name][(table_name, cname)]
        except KeyError:
            return None

    def _lookup_reverse_constraint(self, table_name, column_name=None):
        """Look for the column referenced by a foreign constraint"""
        db_name = self._get_setting('NAME')
        if self.dry_run:
            raise DryRunError("Cannot get constraints for columns.")

        if not self._is_valid_cache(db_name, table_name):
            # Piggy-back on lookup_constraint, ensures cache exists
            self.lookup_constraint(db_name, table_name)

        try:
            table = self._reverse_cache[db_name][table_name]
            if column_name == None:
                return [(y, tuple(y)) for x, y in table.items()]
            else:
                return tuple(table[column_name])
        except KeyError:
            return []

    def _field_sanity(self, field):
        """
        This particular override stops us sending DEFAULTs for BLOB/TEXT columns.
        """
        #  MySQL does not support defaults for geometry columns also
        type = self._db_type_for_alter_column(field).lower()
        is_geom = True in [type.find(t) > -1 for t in self.geom_types]
        is_text = True in [type.find(t) > -1 for t in self.text_types]

        if is_geom or is_text:
            field._suppress_default = True
        return field

    def _alter_set_defaults(self, field, name, params, sqls):
        """
        MySQL does not support defaults on text or blob columns.
        """
        type = params['type']
        #  MySQL does not support defaults for geometry columns also
        is_geom = True in [type.find(t) > -1 for t in self.geom_types]
        is_text = True in [type.find(t) > -1 for t in self.text_types]
        if not is_geom and not is_text:
            super(DatabaseOperations, self)._alter_set_defaults(field, name, params, sqls)
<commit_msg>Removing extraneous create_table<commit_after># MySQL-specific implementations for south
# Original author: Andrew Godwin
# Patches by: F. Gabriel Gosselin <gabrielNOSPAM@evidens.ca>

from south.db import generic
from south.db.generic import DryRunError, INVALID
from south.logger import get_logger

def delete_column_constraints(func):
    """
    Decorates column operation functions for MySQL.
    Deletes the constraints from the database and clears local cache.
    """
    def _column_rm(self, table_name, column_name, *args, **opts):
        # Delete foreign key constraints
        try:
            self.delete_foreign_key(table_name, column_name)
        except ValueError:
            pass  # If no foreign key on column, OK because it checks first
        # Delete constraints referring to this column
        try:
            reverse = self._lookup_reverse_constraint(table_name, column_name)
            for cname, rtable, rcolumn in reverse:
                self.delete_foreign_key(rtable, rcolumn)
        except DryRunError:
            pass
        return func(self, table_name, column_name, *args, **opts)
    return _column_rm

def copy_column_constraints(func):
    """
    Decorates column operation functions for MySQL.
    Determines existing constraints and copies them to a new column
    """
    def _column_cp(self, table_name, column_old, column_new, *args, **opts):
        # Copy foreign key constraint
        try:
            constraint = self._find_foreign_constraints(table_name, column_old)[0]
            (ftable, fcolumn) = self._lookup_constraint_references(table_name, constraint)
            if ftable and fcolumn:
                fk_sql = self.foreign_key_sql(
                            table_name, column_new, ftable, fcolumn)
                get_logger().debug("Foreign key SQL: " + fk_sql)
                self.add_deferred_sql(fk_sql)
        except IndexError:
            pass  # No constraint exists so ignore
        except DryRunError:
            pass
        # Copy constraints referring to this column
        try:
            reverse = self._lookup_reverse_constraint(table_name, column_old)
            for cname, rtable, rcolumn in reverse:
                fk_sql = self.foreign_key_sql(
                        rtable, rcolumn, table_name, column_new)
                self.add_deferred_sql(fk_sql)
        except DryRunError:
            pass
        return func(self, table_name, column_old, column_new, *args, **opts)
    return _column_cp

def invalidate_table_constraints(func):
    """
    For MySQL we grab all table constraints simultaneously, so this is
    effective.
    It further solves the issues of invalidating referred table constraints.
    """
    def _cache_clear(self, table, *args, **opts):
        db_name = self._get_setting('NAME')
        if db_name in self._constraint_cache:
            del self._constraint_cache[db_name]
        if db_name in self._reverse_cache:
            del self._reverse_cache[db_name]
        if db_name in self._constraint_references:
            del self._constraint_references[db_name]
        return func(self, table, *args, **opts)
    return _cache_clear

class DatabaseOperations(generic.DatabaseOperations):
    """
    MySQL implementation of database operations.

    MySQL has no DDL transaction support This can confuse people when they ask
    how to roll back - hence the dry runs, etc., found in the migration code.
    """

    backend_name = "mysql"
    alter_string_set_type = ''
    alter_string_set_null = 'MODIFY %(column)s %(type)s NULL;'
    alter_string_drop_null = 'MODIFY %(column)s %(type)s NOT NULL;'
    drop_index_string = 'DROP INDEX %(index_name)s ON %(table_name)s'
    delete_primary_key_sql = "ALTER TABLE %(table)s DROP PRIMARY KEY"
    delete_foreign_key_sql = "ALTER TABLE %(table)s DROP FOREIGN KEY %(constraint)s"
    delete_unique_sql = "ALTER TABLE %s DROP INDEX %s"
    rename_table_sql = "RENAME TABLE %s TO %s;"

    allows_combined_alters = False
    has_check_constraints = False

    geom_types = ['geometry', 'point', 'linestring', 'polygon']
    text_types = ['text', 'blob']

    def __init__(self, db_alias):
        self._constraint_references = {}
        self._reverse_cache = {}
        super(DatabaseOperations, self).__init__(db_alias)
        if self._has_setting('STORAGE_ENGINE') and self._get_setting('STORAGE_ENGINE'):
            self.create_table_sql = self.create_table_sql + ' ENGINE=%s' % self._get_setting('STORAGE_ENGINE')

    def _is_valid_cache(self, db_name, table_name):
        cache = self._constraint_cache
        # we cache the whole db so if there are any tables table_name is valid
        return db_name in cache and cache[db_name].get(table_name, None) is not INVALID

    def _fill_constraint_cache(self, db_name, table_name):
        # for MySQL grab all constraints for this database.  It's just as cheap as a single column.
        self._constraint_cache[db_name] = {}
        self._constraint_cache[db_name][table_name] = {}
        self._reverse_cache[db_name] = {}
        self._constraint_references[db_name] = {}

        name_query = """
            SELECT kc.`constraint_name`, kc.`column_name`, kc.`table_name`,
                kc.`referenced_table_name`, kc.`referenced_column_name`
            FROM information_schema.key_column_usage AS kc
            WHERE
                kc.table_schema = %s
        """
        rows = self.execute(name_query, [db_name])
        if not rows:
            return
        cnames = {}
        for constraint, column, table, ref_table, ref_column in rows:
            key = (table, constraint)
            cnames.setdefault(key, set())
            cnames[key].add((column, ref_table, ref_column))

        type_query = """
            SELECT c.constraint_name, c.table_name, c.constraint_type
            FROM information_schema.table_constraints AS c
            WHERE
                c.table_schema = %s
        """
        rows = self.execute(type_query, [db_name])
        for constraint, table, kind in rows:
            key = (table, constraint)
            self._constraint_cache[db_name].setdefault(table, {})
            try:
                cols = cnames[key]
            except KeyError:
                cols = set()
            for column_set in cols:
                (column, ref_table, ref_column) = column_set
                self._constraint_cache[db_name][table].setdefault(column, set())
                if kind == 'FOREIGN KEY':
                    self._constraint_cache[db_name][table][column].add((kind,
                        constraint))
                    # Create constraint lookup, see constraint_references
                    self._constraint_references[db_name][(table,
                        constraint)] = (ref_table, ref_column)
                    # Create reverse table lookup, reverse_lookup
                    self._reverse_cache[db_name].setdefault(ref_table, {})
                    self._reverse_cache[db_name][ref_table].setdefault(ref_column,
                            set())
                    self._reverse_cache[db_name][ref_table][ref_column].add(
                            (constraint, table, column))
                else:
                    self._constraint_cache[db_name][table][column].add((kind,
                    constraint))

    def connection_init(self):
        """
        Run before any SQL to let database-specific config be sent as a command,
        e.g. which storage engine (MySQL) or transaction serialisability level.
        """
        cursor = self._get_connection().cursor()
        if self._has_setting('STORAGE_ENGINE') and self._get_setting('STORAGE_ENGINE'):
            cursor.execute("SET storage_engine=%s;" % self._get_setting('STORAGE_ENGINE'))

    def start_transaction(self):
        super(DatabaseOperations, self).start_transaction()
        self.execute("SET FOREIGN_KEY_CHECKS=0;")

    @copy_column_constraints
    @delete_column_constraints
    @invalidate_table_constraints
    def rename_column(self, table_name, old, new):
        if old == new or self.dry_run:
            return []

        rows = [x for x in self.execute('DESCRIBE %s' % (self.quote_name(table_name),)) if x[0] == old]

        if not rows:
            raise ValueError("No column '%s' in '%s'." % (old, table_name))

        params = (
            self.quote_name(table_name),
            self.quote_name(old),
            self.quote_name(new),
            rows[0][1],
            rows[0][2] == "YES" and "NULL" or "NOT NULL",
            rows[0][4] and "DEFAULT " or "",
            rows[0][4] and "%s" or "",
            rows[0][5] or "",
        )

        sql = 'ALTER TABLE %s CHANGE COLUMN %s %s %s %s %s %s %s;' % params

        if rows[0][4]:
            self.execute(sql, (rows[0][4],))
        else:
            self.execute(sql)

    @delete_column_constraints
    def delete_column(self, table_name, name):
        super(DatabaseOperations, self).delete_column(table_name, name)

    @invalidate_table_constraints
    def rename_table(self, old_table_name, table_name):
        super(DatabaseOperations, self).rename_table(old_table_name,
                table_name)

    @invalidate_table_constraints
    def delete_table(self, table_name):
        super(DatabaseOperations, self).delete_table(table_name)

    def _lookup_constraint_references(self, table_name, cname):
        """
        Provided an existing table and constraint, returns tuple of (foreign
        table, column)
        """
        db_name = self._get_setting('NAME')
        try:
            return self._constraint_references[db_name][(table_name, cname)]
        except KeyError:
            return None

    def _lookup_reverse_constraint(self, table_name, column_name=None):
        """Look for the column referenced by a foreign constraint"""
        db_name = self._get_setting('NAME')
        if self.dry_run:
            raise DryRunError("Cannot get constraints for columns.")

        if not self._is_valid_cache(db_name, table_name):
            # Piggy-back on lookup_constraint, ensures cache exists
            self.lookup_constraint(db_name, table_name)

        try:
            table = self._reverse_cache[db_name][table_name]
            if column_name == None:
                return [(y, tuple(y)) for x, y in table.items()]
            else:
                return tuple(table[column_name])
        except KeyError:
            return []

    def _field_sanity(self, field):
        """
        This particular override stops us sending DEFAULTs for BLOB/TEXT columns.
        """
        #  MySQL does not support defaults for geometry columns also
        type = self._db_type_for_alter_column(field).lower()
        is_geom = True in [type.find(t) > -1 for t in self.geom_types]
        is_text = True in [type.find(t) > -1 for t in self.text_types]

        if is_geom or is_text:
            field._suppress_default = True
        return field

    def _alter_set_defaults(self, field, name, params, sqls):
        """
        MySQL does not support defaults on text or blob columns.
        """
        type = params['type']
        #  MySQL does not support defaults for geometry columns also
        is_geom = True in [type.find(t) > -1 for t in self.geom_types]
        is_text = True in [type.find(t) > -1 for t in self.text_types]
        if not is_geom and not is_text:
            super(DatabaseOperations, self)._alter_set_defaults(field, name, params, sqls)
<|endoftext|>


<commit_before>"""Plotting functions
- makeMTX: Generate 3D-matrix data
- visualize3D: Draw matrix data in 3D
"""
import numpy as _np
from vispy import scene, color
from .process import pdc
from .sph import sph2cart

def makeMTX(Pnm, dn, Nviz=3, krIndex=1, oversize=1):
    """mtxData = makeMTX(Nviz=3, Pnm, dn, krIndex)
    ------------------------------------------------------------------------
    mtxData   3D-matrix-data in 1[deg] steps
    ------------------------------------------------------------------------
    Pnm       Spatial Fourier Coefficients (from S/T/C)
    dn        Modal Radial Filters (from M/F)
    N         Order of the spatial fourier transform     [default = 3]
    krIndex   Index of kr Vector                         [default = 1]
    oversize  Integer Factor to increase the resolution. Set oversize = 1
              (default) to use the mtxData matrix for visual3D(), map3D().

    The file generates a SOFiA mtxData Matrix of 181x360 pixels for the
    visualisation with visualize3D() in 1[deg] Steps (65160 plane waves).
    The HD version generally allows to raise the resolution (oversize > 1).
    (visual3D(), map3D() admit 1[deg] data only, oversize = 1)
    """
    if oversize < 1:
        raise ValueError('Oversize parameter must be >= 1')

    # Generate angles for sphere with 1[deg] spacing
    angles = _np.mgrid[0:360, 0:181].T.reshape((-1, 2)) * _np.pi / 180

    # Compute plane wave decomposition for all angles at given kr
    Y = pdc(Nviz, angles, Pnm[:, krIndex], dn[:, krIndex])

    return Y.reshape((181, -1))  # Return pwd data as [181, 360] matrix

def normalizeMTX(vizMTX):
    vizMTX -= vizMTX.min()
    return vizMTX / vizMTX.max()

def genShape(vizMTX, offset=0, scale=1.0, colorize=False):
    thetas, phis = _np.meshgrid(_np.linspace(0, _np.pi, 181), _np.linspace(0, 2 * _np.pi, 360))
    rs = offset + scale * vizMTX.reshape((181, -1)).T
    xs = rs * _np.sin(thetas) * _np.cos(phis)
    ys = rs * _np.sin(thetas) * _np.sin(phis)
    zs = rs * _np.cos(thetas)
    if colorize:
        cm = color.get_colormap('viridis')
        colors = cm[vizMTX]
        return scene.visuals.GridMesh(xs, ys, zs, colors=colors.rgba.reshape((181, -1, 4)))
    else:
        return scene.visuals.GridMesh(xs, ys, zs)

def genSphere(vizMTX, colorize=False):
    #  visObj = scene.visuals.Sphere(radius=1, rows=362, cols=91, method='latitude', face_colors=colors)
    thetas, phis = _np.meshgrid(_np.linspace(0, _np.pi, 181), _np.linspace(0, 2 * _np.pi, 360))
    rs = 1
    xs = rs * _np.sin(thetas) * _np.cos(phis)
    ys = rs * _np.sin(thetas) * _np.sin(phis)
    zs = rs * _np.cos(thetas)
    if colorize:
        cm = color.get_colormap('viridis')
        colors = cm[vizMTX]
        return scene.visuals.GridMesh(xs, ys, zs, colors=colors.rgba.reshape((181, -1, 4)))
    else:
        return scene.visuals.GridMesh(xs, ys, zs)

def genScatter(vizMTX, colorize=False):
    vizMTX = vizMTX.reshape([-1, 1])
    # Recreate angles
    angles = _np.array(generateAngles())

    sphCoords = _np.concatenate((angles, _np.atleast_2d(vizMTX)), axis=1)
    xyzCoords = _np.array(sph2cart(*sphCoords.T))
    scatterObj = scene.visuals.Markers()

    if colorize:
        cm = color.get_colormap('viridis')
        colors = cm[_np.squeeze(vizMTX)]
        scatterObj.set_data(xyzCoords.T, size=10, edge_color=None, face_color=colors)
    else:
        scatterObj.set_data(xyzCoords.T, size=10, edge_color=None, face_color='black')

    return scatterObj

def genVisual(vizMTX, style='shape', colorize=False, offset=0, scale=1.0, normalize=True):
    if normalize:
        vizMTX = normalizeMTX(vizMTX)

    if style == 'shape':
        return genShape(vizMTX, colorize=colorize, offset=offset, scale=scale)
    elif style == 'sphere':
        return genSphere(vizMTX, colorize=colorize)
    elif style == 'scatter':
        return genScatter(vizMTX, colorize=colorize)
    else:
        raise ValueError('Provided style "' + style + '" not available. Try sphere, shape or scatter.')

def visualize3D(vizMTX, style='sphere', colorize=True, offset=0., scale=1., **kargs):
    """Visualize matrix data, such as from makeMTX(Pnm, dn)
    vizMTX     SOFiA 3D-matrix-data [1[deg] steps]
    style       'sphere',   surface colors indicate the intensity (default)
                'flat',     surface colors indicate the intensity (TODO)
                'scatter',  extension indicates the intensity
                'shape',    extension indicates the intensity
    offset      linear offset for shape (Default: 0)
    scale       scaling factor for shape (Default: 1)
    TODO: Implement flat style, fix color position in sphere and shape, make colormap selectable, move grid generation into function
    """

    # Prepare data: reshape to [65160 x 1], take abs, normalize
    vizMTX = _np.abs(vizMTX.reshape((65160)))
    vizMTX = normalizeMTX(vizMTX)

    if style not in ('sphere', 'flat', 'shape', 'scatter'):
        raise ValueError('Provided style "' + style + '" not available. Try sphere, flat, shape or scatter.')

    # Create scene
    canvas = scene.SceneCanvas(keys='interactive', bgcolor='white')

    # Create view with camera on target
    view = canvas.central_widget.add_view()
    view.camera = 'turntable'
    view.camera.set_range(x=[-0.1, 0.1])

    # Create correct visual object from mtxData
    visObj = genVisual(vizMTX, style=style, colorize=colorize, offset=offset, scale=scale)

    # Add visual object and show canvas
    view.add(visObj)
    canvas.show()

    return canvas

def plotGrid(rows, cols, vizMTX, bgcolor='white', style='shape', colorize=False, normalize=True):
    canvas = scene.SceneCanvas(keys='interactive', bgcolor=bgcolor)

    vizMTX = _np.atleast_3d(vizMTX)
    N = vizMTX.shape[0]
    if rows * cols != N:
        raise ValueError('rows (' + str(rows) + ') * cols (' + str(cols) + ') must be number of objects (' + str(N) + ').')

    # Top-level grid that holds subfigures
    grid = canvas.central_widget.add_grid()

    # Add ViewBoxes to the grid
    for row in range(0, rows):
        for col in range(0, cols):
            temp = grid.add_view(row=row, col=col, border_color=(0.5, 0.5, 0.5, 1), camera='turntable')
            temp.add(genVisual(vizMTX[col + cols * row], style=style, colorize=colorize, normalize=normalize))

    canvas.show()
    return canvas

def generateAngles():
    """Returns a [65160 x 1] grid of all radiant angles in 1 deg steps"""
    return _np.mgrid[0:360, 0:181].T.reshape((-1, 2)) * _np.pi / 180
<commit_msg>fix spacing<commit_after>"""Plotting functions
- makeMTX: Generate 3D-matrix data
- visualize3D: Draw matrix data in 3D
"""
import numpy as _np
from vispy import scene, color
from .process import pdc
from .sph import sph2cart

def makeMTX(Pnm, dn, Nviz=3, krIndex=1, oversize=1):
    """mtxData = makeMTX(Nviz=3, Pnm, dn, krIndex)
    ------------------------------------------------------------------------
    mtxData   3D-matrix-data in 1[deg] steps
    ------------------------------------------------------------------------
    Pnm       Spatial Fourier Coefficients (from S/T/C)
    dn        Modal Radial Filters (from M/F)
    N         Order of the spatial fourier transform     [default = 3]
    krIndex   Index of kr Vector                         [default = 1]
    oversize  Integer Factor to increase the resolution. Set oversize = 1
              (default) to use the mtxData matrix for visual3D(), map3D().

    The file generates a SOFiA mtxData Matrix of 181x360 pixels for the
    visualisation with visualize3D() in 1[deg] Steps (65160 plane waves).
    The HD version generally allows to raise the resolution (oversize > 1).
    (visual3D(), map3D() admit 1[deg] data only, oversize = 1)
    """
    if oversize < 1:
        raise ValueError('Oversize parameter must be >= 1')

    # Generate angles for sphere with 1[deg] spacing
    angles = _np.mgrid[0:360, 0:181].T.reshape((-1, 2)) * _np.pi / 180

    # Compute plane wave decomposition for all angles at given kr
    Y = pdc(Nviz, angles, Pnm[:, krIndex], dn[:, krIndex])

    return Y.reshape((181, -1))  # Return pwd data as [181, 360] matrix

def normalizeMTX(vizMTX):
    vizMTX -= vizMTX.min()
    return vizMTX / vizMTX.max()

def genShape(vizMTX, offset=0, scale=1.0, colorize=False):
    thetas, phis = _np.meshgrid(_np.linspace(0, _np.pi, 181), _np.linspace(0, 2 * _np.pi, 360))
    rs = offset + scale * vizMTX.reshape((181, -1)).T
    xs = rs * _np.sin(thetas) * _np.cos(phis)
    ys = rs * _np.sin(thetas) * _np.sin(phis)
    zs = rs * _np.cos(thetas)
    if colorize:
        cm = color.get_colormap('viridis')
        colors = cm[vizMTX]
        return scene.visuals.GridMesh(xs, ys, zs, colors=colors.rgba.reshape((181, -1, 4)))
    else:
        return scene.visuals.GridMesh(xs, ys, zs)

def genSphere(vizMTX, colorize=False):
    #  visObj = scene.visuals.Sphere(radius=1, rows=362, cols=91, method='latitude', face_colors=colors)
    thetas, phis = _np.meshgrid(_np.linspace(0, _np.pi, 181), _np.linspace(0, 2 * _np.pi, 360))
    rs = 1
    xs = rs * _np.sin(thetas) * _np.cos(phis)
    ys = rs * _np.sin(thetas) * _np.sin(phis)
    zs = rs * _np.cos(thetas)
    if colorize:
        cm = color.get_colormap('viridis')
        colors = cm[vizMTX]
        return scene.visuals.GridMesh(xs, ys, zs, colors=colors.rgba.reshape((181, -1, 4)))
    else:
        return scene.visuals.GridMesh(xs, ys, zs)

def genScatter(vizMTX, colorize=False):
    vizMTX = vizMTX.reshape([-1, 1])
    # Recreate angles
    angles = _np.array(generateAngles())

    sphCoords = _np.concatenate((angles, _np.atleast_2d(vizMTX)), axis=1)
    xyzCoords = _np.array(sph2cart(*sphCoords.T))
    scatterObj = scene.visuals.Markers()

    if colorize:
        cm = color.get_colormap('viridis')
        colors = cm[_np.squeeze(vizMTX)]
        scatterObj.set_data(xyzCoords.T, size=10, edge_color=None, face_color=colors)
    else:
        scatterObj.set_data(xyzCoords.T, size=10, edge_color=None, face_color='black')

    return scatterObj

def genVisual(vizMTX, style='shape', colorize=False, offset=0, scale=1.0, normalize=True):
    if normalize:
        vizMTX = normalizeMTX(vizMTX)

    if style == 'shape':
        return genShape(vizMTX, colorize=colorize, offset=offset, scale=scale)
    elif style == 'sphere':
        return genSphere(vizMTX, colorize=colorize)
    elif style == 'scatter':
        return genScatter(vizMTX, colorize=colorize)
    else:
        raise ValueError('Provided style "' + style + '" not available. Try sphere, shape or scatter.')

def visualize3D(vizMTX, style='sphere', colorize=True, offset=0., scale=1., **kargs):
    """Visualize matrix data, such as from makeMTX(Pnm, dn)
    vizMTX     SOFiA 3D-matrix-data [1[deg] steps]
    style       'sphere',   surface colors indicate the intensity (default)
                'flat',     surface colors indicate the intensity (TODO)
                'scatter',  extension indicates the intensity
                'shape',    extension indicates the intensity
    offset      linear offset for shape (Default: 0)
    scale       scaling factor for shape (Default: 1)
    TODO: Implement flat style, fix color position in sphere and shape, make colormap selectable, move grid generation into function
    """

    # Prepare data: reshape to [65160 x 1], take abs, normalize
    vizMTX = _np.abs(vizMTX.reshape((65160)))
    vizMTX = normalizeMTX(vizMTX)

    if style not in ('sphere', 'flat', 'shape', 'scatter'):
        raise ValueError('Provided style "' + style + '" not available. Try sphere, flat, shape or scatter.')

    # Create scene
    canvas = scene.SceneCanvas(keys='interactive', bgcolor='white')

    # Create view with camera on target
    view = canvas.central_widget.add_view()
    view.camera = 'turntable'
    view.camera.set_range(x=[-0.1, 0.1])

    # Create correct visual object from mtxData
    visObj = genVisual(vizMTX, style=style, colorize=colorize, offset=offset, scale=scale)

    # Add visual object and show canvas
    view.add(visObj)
    canvas.show()

    return canvas

def plotGrid(rows, cols, vizMTX, bgcolor='white', style='shape', colorize=False, normalize=True):
    canvas = scene.SceneCanvas(keys='interactive', bgcolor=bgcolor)

    vizMTX = _np.atleast_3d(vizMTX)
    N = vizMTX.shape[0]
    if rows * cols != N:
        raise ValueError('rows (' + str(rows) + ') * cols (' + str(cols) + ') must be number of objects (' + str(N) + ').')

    # Top-level grid that holds subfigures
    grid = canvas.central_widget.add_grid()

    # Add ViewBoxes to the grid
    for row in range(0, rows):
        for col in range(0, cols):
            temp = grid.add_view(row=row, col=col, border_color=(0.5, 0.5, 0.5, 1), camera='turntable')
            temp.add(genVisual(vizMTX[col + cols * row], style=style, colorize=colorize, normalize=normalize))

    canvas.show()
    return canvas

def generateAngles():
    """Returns a [65160 x 1] grid of all radiant angles in 1 deg steps"""
    return _np.mgrid[0:360, 0:181].T.reshape((-1, 2)) * _np.pi / 180
<|endoftext|>


<commit_before>import json
import mongoengine
from flask import request
from bson.dbref import DBRef
from mongoengine.fields import EmbeddedDocumentField, ListField, ReferenceField, DateTimeField
from flask.ext.mongorest.exceptions import ValidationError
import dateutil.parser

class ResourceMeta(type):
    def __init__(cls, name, bases, classdict):
        if classdict.get('__metaclass__') is not ResourceMeta:
            for document,resource in cls.child_document_resources.iteritems():
                if resource == name:
                    cls.child_document_resources[document] = cls
        type.__init__(cls, name, bases, classdict)

class Resource(object):
    document = None # required
    fields = None
    readonly_fields = ['id']
    form = None
    related_resources = {}
    rename_fields = {}
    child_document_resources = {}
    paginate = True
    select_related = False
    allowed_ordering = []

    __metaclass__ = ResourceMeta

    def __init__(self):
        doc_fields = self.document._fields.keys()
        if self.fields == None:
            self.fields = doc_fields
        self._related_resources = self.get_related_resources()
        self._rename_fields = self.get_rename_fields()
        self._filters = self.get_filters()
        self._child_document_resources = self.get_child_document_resources()
        self.data = {}
        if request.method in ('PUT', 'POST'):
            self.data = json.loads(request.data)

    def get_fields(self):
        return self.fields

    def get_related_resources(self):
        return self.related_resources

    def get_rename_fields(self):
        """
        Return (k,v) and (v,k) for all renamed fields to support serialization and deserialization
        @TODO should automatically support model_id for reference fields (only) and model for related_resources
        """
        rename_fields = {}
        for k, v in self.rename_fields.iteritems():
            rename_fields[k] = v
            if not self.rename_fields.has_key(v):
                rename_fields[v] = k
        return rename_fields

    def get_child_document_resources(self):
        return self.child_document_resources

    def get_filters(self):
        filters = {}
        for field, operators in getattr(self, 'filters', {}).iteritems():
            field_filters = {}
            for op in operators:
                if op.op == 'exact':
                    field_filters[''] = op
                field_filters[op.op] = op
            filters[field] = field_filters
        return filters

    def serialize(self, obj, params=None):
        if not obj:
            return {}

        if obj.__class__ in self._child_document_resources \
        and self._child_document_resources[obj.__class__] != self.__class__:
            return obj and self._child_document_resources[obj.__class__]().serialize(obj)

        def get(obj, field_name, field_instance=None):
            """
            @TODO needs significant cleanup
            """
            field_value = obj if field_instance else getattr(obj, field_name)
            field_instance = field_instance or getattr(self.document, field_name)
            if isinstance(field_instance, (ReferenceField, EmbeddedDocumentField)):
                if field_name in self._related_resources:
                    return field_value and self._related_resources[field_name]().serialize(field_value)
                else:
                    if isinstance(field_value, DBRef):
                        return field_value
                    return field_value and field_value.to_dbref()
            elif isinstance(field_instance, ListField):
                return [get(elem, field_name, field_instance=field_instance.field) for elem in field_value]
            elif callable(field_instance):
                value = field_value()
                if field_name in self._related_resources:
                    if isinstance(value, mongoengine.queryset.QuerySet):
                        return [self._related_resources[field_name]().serialize(o) for o in value]
                return value
            return field_value

        fields = self.get_fields()

        if params and '_fields' in params:
            only_fields = set(params['_fields'].split(','))
        else:
            only_fields = None

        data = {}
        for field in fields:
            renamed_field = self._rename_fields.get(field, field)
            if only_fields != None and renamed_field not in only_fields:
                continue
            if hasattr(self, field):
                data[renamed_field] = getattr(self, field)(obj)
            else:
                data[renamed_field] = get(obj, field)

        return data

    def validate_request(self, obj=None):
        # @TODO this should rename form fields otherwise in a resource you could say "model_id" and in a form still have to use "model".
        for k, v in self._rename_fields.iteritems():
            if self.data.has_key(v):
                self.data[k] = self.data[v]
                del self.data[v]

        if self.form:
            from werkzeug.datastructures import MultiDict

            if request.method == 'PUT' and obj != None:
                # We treat 'PUT' like 'PATCH', i.e. when fields are not
                # specified, existing values are used.

                # TODO: This is not implemented properly for nested objects yet.

                obj_data = self.serialize(obj)
                obj_data.update(self.data)

                self.data = obj_data

            # We need to convert JSON data into form data.
            # e.g. { "people": [ { "name": "A" } ] } into { "people-0-name": "A" }
            def json_to_form_data(prefix, json_data):
                form_data = {}
                for k, v in json_data.iteritems():
                    if isinstance(v, list): # FieldList
                        for n, el in enumerate(v):
                            form_data.update(json_to_form_data('%s%s-%d-' % (prefix, k, n), el))
                    else:
                        if isinstance(v, dict): # DictField
                            v = json.dumps(v)
                        form_data['%s%s' % (prefix, k)] = v
                return form_data

            json_data = json_to_form_data('', self.data)
            data = MultiDict(json_data)
            form = self.form(data, csrf_enabled=False)

            if not form.validate():
                raise ValidationError({'field-errors': form.errors})

            self.data = form.data

    def get_queryset(self):
        return self.document.objects

    def get_object(self, pk):
        return self.get_queryset().get(pk=pk)

    def get_objects(self):
        params = request.args
        qs = self.get_queryset()
        for key in params:
            value = params[key]
            operator = None
            negate = False
            op_name = ''
            parts = key.split('__')
            if len(parts) > 1:
                op_name = parts.pop()
                if parts[-1] == 'not':
                    negate = True
                    parts.pop()
            field = parts.pop()
            allowed_operators = self._filters.get(field, {})
            if op_name not in allowed_operators.keys():
                continue
            operator = allowed_operators[op_name]
            field = self._rename_fields.get(field, field)
            qs = operator().apply(qs, field, value, negate)
        if self.paginate:
            qs = qs.skip(int(params.get('_skip', 0))).limit(int(params.get('_limit', 100)))
        if self.allowed_ordering and params.get('_order_by', None) in self.allowed_ordering:
            qs = qs.order_by(params['_order_by'])
        # Needs to be at the end as it returns a list.
        if self.select_related:
            qs = qs.select_related()
        return qs

    def _get(self, method, data, field_name, field_instance=None, parent_resources=None):
        """
        @TODO needs significant cleanup
        """
        if not parent_resources:
            parent_resources = []

        field_data_value = data if field_instance else data[field_name]
        field_instance = field_instance or getattr(self.document, field_name)

        if isinstance(field_instance, ReferenceField):
            if field_name in self._related_resources:
                return self.related_resources[field_name]().create_object(data=field_data_value, save=True, parent_resources=parent_resources+[self])
            else:
                if isinstance(field_data_value, mongoengine.Document):
                    return field_data_value
                return field_data_value and field_instance.document_type.objects.get(pk=field_data_value).to_dbref()

        elif isinstance(field_instance, DateTimeField):
            return field_data_value and dateutil.parser.parse(field_data_value) or field_data_value

        elif isinstance(field_instance, EmbeddedDocumentField):
            if field_name in self._related_resources:
                return self.related_resources[field_name]().create_object(data=field_data_value, save=False, parent_resources=parent_resources+[self])
            else:
                return {} # dummy embedded document

        elif isinstance(field_instance, ListField):
            def expand_list(inner_field, inner_data):
                if isinstance(inner_field, ListField):
                    return [expand_list(inner_field.field, elem) for elem in inner_data]
                elif isinstance(inner_field, EmbeddedDocumentField):
                    return self.related_resources[field_name]().create_object(data=inner_data, save=False, parent_resources=parent_resources+[self])
                else:
                    return self._get(method, inner_data, field_name, field_instance=inner_field, parent_resources=parent_resources)
            return [expand_list(field_instance.field, elem) for elem in field_data_value]

        else:
            return field_data_value

    def _save(self, obj):
        try:
            obj.save()
        except mongoengine.ValidationError, e:
            def serialize_errors(errors):
                if hasattr(errors, 'iteritems'):
                    return {k: serialize_errors(v) for k, v in errors.iteritems()}
                else:
                    return str(errors)
            raise ValidationError({'field-errors': serialize_errors(e.errors)})

    def create_object(self, data=None, save=True, parent_resources=None):
        kwargs = {}
        data = data or self.data
        for field in self.get_fields():
            if field in self.document._fields.keys() and field not in self.readonly_fields and (type(data) is list or (type(data) is dict and data.has_key(field))):
                kwargs[field] = self._get('create_object', data, field, parent_resources=parent_resources)
        obj = self.document(**kwargs)
        if save:
            self._save(obj)
        return obj

    def update_object(self, obj, data=None, save=True, parent_resources=None):
        data = data or self.data
        for field in self.get_fields():
            if field in self.document._fields.keys() and field not in self.readonly_fields and field in data:
                if field in self._related_resources:
                    field_instance = getattr(self.document, field)
                    if isinstance(field_instance, ReferenceField) or (isinstance(field_instance, ListField) and isinstance(field_instance.field, ReferenceField)):
                        continue # Not implemented.
                setattr(obj, field, self._get('update_object', data, field, parent_resources=parent_resources))
        if save:
            self._save(obj)
        return obj

    def delete_object(self, obj, parent_resources=None):
        obj.delete()
<commit_msg>Tidying up<commit_after>import json
import mongoengine
from flask import request
from bson.dbref import DBRef
from mongoengine.fields import EmbeddedDocumentField, ListField, ReferenceField, DateTimeField
from flask.ext.mongorest.exceptions import ValidationError
import dateutil.parser

class ResourceMeta(type):
    def __init__(cls, name, bases, classdict):
        if classdict.get('__metaclass__') is not ResourceMeta:
            for document,resource in cls.child_document_resources.iteritems():
                if resource == name:
                    cls.child_document_resources[document] = cls
        type.__init__(cls, name, bases, classdict)

class Resource(object):
    document = None # required
    fields = None
    readonly_fields = ['id']
    form = None
    related_resources = {}
    rename_fields = {}
    child_document_resources = {}
    paginate = True
    select_related = False
    allowed_ordering = []

    __metaclass__ = ResourceMeta

    def __init__(self):
        doc_fields = self.document._fields.keys()
        if self.fields == None:
            self.fields = doc_fields
        self._related_resources = self.get_related_resources()
        self._rename_fields = self.get_rename_fields()
        self._filters = self.get_filters()
        self._child_document_resources = self.get_child_document_resources()
        self.data = {}
        if request.method in ('PUT', 'POST'):
            self.data = json.loads(request.data)

    def get_fields(self):
        return self.fields

    def get_related_resources(self):
        return self.related_resources

    def get_rename_fields(self):
        """
        Return (k,v) and (v,k) for all renamed fields to support serialization and deserialization
        @TODO should automatically support model_id for reference fields (only) and model for related_resources
        """
        rename_fields = {}
        for k, v in self.rename_fields.iteritems():
            rename_fields[k] = v
            if not self.rename_fields.has_key(v):
                rename_fields[v] = k
        return rename_fields

    def get_child_document_resources(self):
        return self.child_document_resources

    def get_filters(self):
        filters = {}
        for field, operators in getattr(self, 'filters', {}).iteritems():
            field_filters = {}
            for op in operators:
                if op.op == 'exact':
                    field_filters[''] = op
                field_filters[op.op] = op
            filters[field] = field_filters
        return filters

    def serialize(self, obj, params=None):
        if not obj:
            return {}

        if obj.__class__ in self._child_document_resources \
        and self._child_document_resources[obj.__class__] != self.__class__:
            return obj and self._child_document_resources[obj.__class__]().serialize(obj)

        def get(obj, field_name, field_instance=None):
            """
            @TODO needs significant cleanup
            """
            field_value = obj if field_instance else getattr(obj, field_name)
            field_instance = field_instance or getattr(self.document, field_name)
            if isinstance(field_instance, (ReferenceField, EmbeddedDocumentField)):
                if field_name in self._related_resources:
                    return field_value and self._related_resources[field_name]().serialize(field_value)
                else:
                    if isinstance(field_value, DBRef):
                        return field_value
                    return field_value and field_value.to_dbref()
            elif isinstance(field_instance, ListField):
                return [get(elem, field_name, field_instance=field_instance.field) for elem in field_value]
            elif callable(field_instance):
                value = field_value()
                if field_name in self._related_resources:
                    if isinstance(value, mongoengine.queryset.QuerySet):
                        return [self._related_resources[field_name]().serialize(o) for o in value]
                return value
            return field_value

        fields = self.get_fields()

        if params and '_fields' in params:
            only_fields = set(params['_fields'].split(','))
        else:
            only_fields = None

        data = {}
        for field in fields:
            renamed_field = self._rename_fields.get(field, field)
            if only_fields != None and renamed_field not in only_fields:
                continue
            if hasattr(self, field):
                data[renamed_field] = getattr(self, field)(obj)
            else:
                data[renamed_field] = get(obj, field)

        return data

    def validate_request(self, obj=None):
        # @TODO this should rename form fields otherwise in a resource you could say "model_id" and in a form still have to use "model".
        for k, v in self._rename_fields.iteritems():
            if self.data.has_key(v):
                self.data[k] = self.data[v]
                del self.data[v]

        if self.form:
            from werkzeug.datastructures import MultiDict

            if request.method == 'PUT' and obj != None:
                # We treat 'PUT' like 'PATCH', i.e. when fields are not
                # specified, existing values are used.

                # TODO: This is not implemented properly for nested objects yet.

                obj_data = self.serialize(obj)
                obj_data.update(self.data)

                self.data = obj_data

            # We need to convert JSON data into form data.
            # e.g. { "people": [ { "name": "A" } ] } into { "people-0-name": "A" }
            def json_to_form_data(prefix, json_data):
                form_data = {}
                for k, v in json_data.iteritems():
                    if isinstance(v, list): # FieldList
                        for n, el in enumerate(v):
                            form_data.update(json_to_form_data('%s%s-%d-' % (prefix, k, n), el))
                    else:
                        if isinstance(v, dict): # DictField
                            v = json.dumps(v)
                        form_data['%s%s' % (prefix, k)] = v
                return form_data

            json_data = json_to_form_data('', self.data)
            data = MultiDict(json_data)
            form = self.form(data, csrf_enabled=False)

            if not form.validate():
                raise ValidationError({'field-errors': form.errors})

            self.data = form.data

    def get_queryset(self):
        return self.document.objects

    def get_object(self, pk):
        return self.get_queryset().get(pk=pk)

    def get_objects(self):
        params = request.args
        qs = self.get_queryset()
        for key in params:
            value = params[key]
            operator = None
            negate = False
            op_name = ''
            parts = key.split('__')
            if len(parts) > 1:
                op_name = parts.pop()
                if parts[-1] == 'not':
                    negate = True
                    parts.pop()
            field = parts.pop()
            allowed_operators = self._filters.get(field, {})
            if op_name not in allowed_operators.keys():
                continue
            operator = allowed_operators[op_name]
            field = self._rename_fields.get(field, field)
            qs = operator().apply(qs, field, value, negate)
        if self.paginate:
            qs = qs.skip(int(params.get('_skip', 0))).limit(int(params.get('_limit', 100)))
        if self.allowed_ordering and params.get('_order_by', None) in self.allowed_ordering:
            qs = qs.order_by(params['_order_by'])
        # Needs to be at the end as it returns a list.
        if self.select_related:
            qs = qs.select_related()
        return qs

    def _get(self, method, data, field_name, field_instance=None, parent_resources=None):
        """
        @TODO needs significant cleanup
        """
        if not parent_resources:
            parent_resources = []

        field_data_value = data if field_instance else data[field_name]
        field_instance = field_instance or getattr(self.document, field_name)

        if isinstance(field_instance, ReferenceField):
            if field_name in self._related_resources:
                return self.related_resources[field_name]().create_object(data=field_data_value, save=True, parent_resources=parent_resources+[self])
            else:
                if isinstance(field_data_value, mongoengine.Document):
                    return field_data_value
                return field_data_value and field_instance.document_type.objects.get(pk=field_data_value).to_dbref()

        elif isinstance(field_instance, DateTimeField):
            return field_data_value and dateutil.parser.parse(field_data_value)

        elif isinstance(field_instance, EmbeddedDocumentField):
            if field_name in self._related_resources:
                return self.related_resources[field_name]().create_object(data=field_data_value, save=False, parent_resources=parent_resources+[self])
            else:
                return {} # dummy embedded document

        elif isinstance(field_instance, ListField):
            def expand_list(inner_field, inner_data):
                if isinstance(inner_field, ListField):
                    return [expand_list(inner_field.field, elem) for elem in inner_data]
                elif isinstance(inner_field, EmbeddedDocumentField):
                    return self.related_resources[field_name]().create_object(data=inner_data, save=False, parent_resources=parent_resources+[self])
                else:
                    return self._get(method, inner_data, field_name, field_instance=inner_field, parent_resources=parent_resources)
            return [expand_list(field_instance.field, elem) for elem in field_data_value]

        else:
            return field_data_value

    def _save(self, obj):
        try:
            obj.save()
        except mongoengine.ValidationError, e:
            def serialize_errors(errors):
                if hasattr(errors, 'iteritems'):
                    return {k: serialize_errors(v) for k, v in errors.iteritems()}
                else:
                    return str(errors)
            raise ValidationError({'field-errors': serialize_errors(e.errors)})

    def create_object(self, data=None, save=True, parent_resources=None):
        kwargs = {}
        data = data or self.data
        for field in self.get_fields():
            if field in self.document._fields.keys() and field not in self.readonly_fields and (type(data) is list or (type(data) is dict and data.has_key(field))):
                kwargs[field] = self._get('create_object', data, field, parent_resources=parent_resources)
        obj = self.document(**kwargs)
        if save:
            self._save(obj)
        return obj

    def update_object(self, obj, data=None, save=True, parent_resources=None):
        data = data or self.data
        for field in self.get_fields():
            if field in self.document._fields.keys() and field not in self.readonly_fields and field in data:
                if field in self._related_resources:
                    field_instance = getattr(self.document, field)
                    if isinstance(field_instance, ReferenceField) or (isinstance(field_instance, ListField) and isinstance(field_instance.field, ReferenceField)):
                        continue # Not implemented.
                setattr(obj, field, self._get('update_object', data, field, parent_resources=parent_resources))
        if save:
            self._save(obj)
        return obj

    def delete_object(self, obj, parent_resources=None):
        obj.delete()
<|endoftext|>