<commit_before>import os
import tempfile
import xmltool
from xmltool import dtd_parser, render as xt_render
from lxml import etree
import json
import importlib

from urllib2 import HTTPError, URLError
from pyramid.view import view_config
from pyramid.renderers import render, Response
import pyramid.httpexceptions as exc
from waxe.core import browser, utils, resource, events
from waxe.core.views.base import BaseUserView
import pyramid_logging

log = pyramid_logging.getLogger(__name__)

import waxe.xml

EXTENSIONS = waxe.xml.EXTENSIONS
ROUTE_PREFIX = waxe.xml.ROUTE_PREFIX

def _get_tags(dtd_url):
    dic = xmltool.dtd_parser.parse(dtd_url=dtd_url)
    lis = []
    for k, v in dic.items():
        if issubclass(v, xmltool.elements.TextElement):
            continue
        lis += [k]
    lis.sort()
    return lis

# Basic plugin system
def match(request, str_id, dtd_url):
    for plugin in request.xml_plugins:
        if plugin.match(request, str_id, dtd_url):
            return plugin

def add_element(request, str_id, dtd_url):
    plugin = match(request, str_id, dtd_url)
    if not plugin:
        return None
    return plugin.add_element(request, str_id, dtd_url)
# end Basic plugin system

def is_valid_filecontent(view, path, filecontent):
    if not os.path.splitext(path)[1] == '.xml':
        return filecontent

    try:
        xmltool.load_string(filecontent)
        transform = view.request.xmltool_transform
        if transform:
            return view, path, transform(filecontent)
        return view, path, filecontent
    except Exception, e:
        raise exc.HTTPInternalServerError(str(e))

class EditorView(BaseUserView):

    def _get_html_renderer(self):
        if 'waxe.xml.xmltool.renderer_func' not in self.request.registry.settings:
            return None

        func = self.request.registry.settings['waxe.xml.xmltool.renderer_func']
        mod, func = func.rsplit('.', 1)
        return getattr(importlib.import_module(mod), func)(
            self.current_user.login)

    @view_config(route_name='edit_json')
    def edit(self):
        filename = self.request.GET.get('path')
        if not filename:
            raise exc.HTTPClientError('No filename given')
        root_path = self.root_path
        absfilename = browser.absolute_path(filename, root_path)
        iframe = 'iframe' in self.request.GET
        try:
            obj = xmltool.load(absfilename)
            if iframe:
                obj.root.html_renderer = xt_render.ReadonlyRender()
                html = obj.to_html()
            else:
                obj.root.html_renderer = self._get_html_renderer()
                html = xmltool.generate_form_from_obj(
                    obj,
                    form_filename=filename,
                    form_attrs={
                        'data-add-href': self.request.custom_route_path('add_element_json'),
                        'data-comment-href': self.request.custom_route_path('get_comment_modal_json'),
                        'data-action': self.request.custom_route_path('update_json'),
                        'data-copy-href': self.request.custom_route_path('copy_json'),
                        'data-paste-href': self.request.custom_route_path('paste_json'),
                    }
                )
            jstree_data = obj.to_jstree_dict()
        except (HTTPError, URLError), e:
            log.exception(e, request=self.request)
            raise exc.HTTPInternalServerError(
                "The dtd of %s can't be loaded." % filename)
        except etree.XMLSyntaxError, e:
            log.exception(e, request=self.request)
            return self.edit_text(e)
        except Exception, e:
            log.exception(e, request=self.request)
            raise exc.HTTPInternalServerError(str(e))

        if 'iframe' in self.request.GET:
            # TODO: we should remove this logic when adding a window to edit
            # tag directly
            return Response(
                render('iframe.mak',
                       {
                           'content': html,
                           'jstree_data': json.dumps(jstree_data),
                       },
                       self.request))

        self.add_opened_file(filename)
        return {
            'content': html,
            'jstree_data': jstree_data,
        }

    @view_config(route_name='edit_text_json')
    def edit_text(self, exception=None):
        filename = self.request.GET.get('path')
        if not filename:
            raise exc.HTTPClientError('No filename given')
        root_path = self.root_path
        absfilename = browser.absolute_path(filename, root_path)
        try:
            content = open(absfilename, 'r').read()
            content = content.decode('utf-8')
        except Exception, e:
            log.exception(e, request=self.request)
            raise exc.HTTPInternalServerError(str(e))

        content = utils.escape_entities(content)

        html = u'<form id="xmltool-form" class="no-tree" data-action="%s" method="POST">' % (
            self.request.custom_route_path('update_text_json'),
        )
        html += u'<input type="hidden" id="_xml_filename" name="filename" value="%s" />' % filename
        html += u'<textarea class="codemirror" name="filecontent">%s</textarea>' % content
        html += u'</form>'

        dic = {
            'content': html,
        }
        # TODO: Support to display message in angular or change this logic
        if exception:
            dic['error_msg'] = str(exception)
        return dic

    @view_config(route_name='get_tags_json')
    def get_tags(self):
        dtd_url = self.request.GET.get('dtd_url', None)
        if not dtd_url:
            raise exc.HTTPClientError('No dtd url given')
        return _get_tags(dtd_url)

    @view_config(route_name='new_json')
    def new(self):
        relpath = self.request.GET.get('path')
        dtd_url = self.request.GET.get('dtd_url')
        dtd_tag = self.request.GET.get('dtd_tag')

        obj = None
        if dtd_tag and dtd_url:
            # Create new object from the dtd url and tag
            try:
                dic = dtd_parser.parse(dtd_url=dtd_url)
            except (HTTPError, URLError), e:
                log.exception(e, request=self.request)
                raise exc.HTTPInternalServerError(
                    "The dtd file %s can't be loaded." % dtd_url)
            if dtd_tag not in dic:
                raise exc.HTTPInternalServerError(
                    'Invalid dtd element: %s (%s)' % (dtd_tag, dtd_url))
            obj = dic[dtd_tag]()
            obj.dtd_url = dtd_url
        elif relpath:
            # Create new object from a template
            absfilename = browser.absolute_path(relpath, self.root_path)
            try:
                obj = xmltool.load(absfilename)
            except Exception, e:
                log.exception(e, request=self.request)
                raise exc.HTTPInternalServerError(str(e))

        if obj:
            obj.root.html_renderer = self._get_html_renderer()
            html = xmltool.generate_form_from_obj(
                obj,
                form_attrs={
                    'data-add-href': self.request.custom_route_path('add_element_json'),
                    'data-comment-href': self.request.custom_route_path('get_comment_modal_json'),
                    'data-action': self.request.custom_route_path('update_json'),
                    'data-copy-href': self.request.custom_route_path('copy_json'),
                    'data-paste-href': self.request.custom_route_path('paste_json'),
                }
            )
            jstree_data = obj.to_jstree_dict()
            return {
                'content': html,
                'jstree_data': jstree_data,
            }

        content = render('blocks/new.mak',
                         {'dtd_urls': self.request.dtd_urls,
                          'tags': _get_tags(self.request.dtd_urls[0])},
                         self.request)
        return {'modal': content}

    @view_config(route_name='update_json')
    def update(self):
        data = self.req_post
        filename = data.pop('_xml_filename', None)
        if not filename:
            raise exc.HTTPClientError('No filename given')

        root, ext = os.path.splitext(filename)
        if ext != '.xml':
            error_msg = 'No filename extension.'
            if ext:
                error_msg = "Bad filename extension '%s'." % ext
            error_msg += " It should be '.xml'"
            raise exc.HTTPClientError(error_msg)

        root_path = self.root_path
        absfilename = browser.absolute_path(filename, root_path)
        try:
            xmltool.update(absfilename, data,
                           transform=self.request.xmltool_transform)
        except (HTTPError, URLError), e:
            log.exception(e, request=self.request)
            raise exc.HTTPInternalServerError(
                "The dtd of %s can't be loaded." % filename)
        except Exception, e:
            log.exception(e, request=self.request)
            raise exc.HTTPInternalServerError(str(e))

        events.trigger('updated.xml',
                       view=self,
                       path=filename)
        return 'File updated'

    @view_config(route_name='update_text_json')
    def update_text(self):
        filecontent = self.req_post.get('filecontent')
        filename = self.req_post.get('filename') or ''
        if not filecontent or not filename:
            raise exc.HTTPClientError('Missing parameters!')
        root_path = self.root_path
        absfilename = browser.absolute_path(filename, root_path)
        try:
            obj = xmltool.load_string(filecontent)
            obj.write(absfilename, transform=self.request.xmltool_transform())
        except Exception, e:
            raise exc.HTTPInternalServerError(str(e))

        # TODO: the modal logic should be in angular
        modal = None
        msg = 'File updated'
        if self.request.POST.get('commit'):
            modal = render('blocks/commit_modal.mak',
                           {}, self.request)

        events.trigger('updated', view=self, paths=filename)
        if modal:
            return {
                'modal': modal
            }

        return msg

    @view_config(route_name='add_element_json')
    def add_element_json(self):
        elt_id = self.request.GET.get('elt_id')
        dtd_url = self.request.GET.get('dtd_url')
        if not elt_id or not dtd_url:
            return {'error_msg': 'Bad parameter'}

        res = add_element(self.request, elt_id, dtd_url)
        if res:
            return res

        dic = xmltool.factory.get_data_from_str_id_for_html_display(
            elt_id,
            dtd_url=dtd_url,
            html_renderer=self._get_html_renderer()
        )
        return dic

    @view_config(route_name='copy_json')
    def copy_json(self):
        if 'elt_id' not in self.request.POST:
            return {'error_msg': 'Bad parameter'}
        data = xmltool.factory.getElementData(self.request.POST['elt_id'],
                                              self.request.POST)
        # Write the content to paste in a temporary file
        filename = tempfile.mktemp()
        open(filename, 'w').write(json.dumps(data))
        self.request.session['clipboard'] = {
            'filename': filename,
            'elt_id': self.request.POST['elt_id'],
        }
        return {'info_msg': 'Copied'}

    @view_config(route_name='paste_json')
    def paste_json(self):
        # TODO: Validate it's the same dtd
        elt_id = self.request.POST.pop('elt_id', None)
        dtd_url = self.request.POST.pop('_xml_dtd_url', None)
        data = xmltool.utils.unflatten_params(self.request.POST)

        if not elt_id or not dtd_url:
            return {'error_msg': 'Bad parameter'}

        clipboard = self.request.session.get('clipboard')
        if not clipboard:
            return {
                'error_msg': 'Empty clipboard'
            }
        filename = clipboard['filename']
        clipboard_data = json.loads(open(filename, 'r').read())

        dic = xmltool.factory.get_new_element_data_for_html_display(
            elt_id, data,
            clipboard_data, dtd_url,
            # Don't keep the attributes nor the comments
            skip_extra=True,
            html_renderer=self._get_html_renderer()
        )
        if not dic:
            return {
                'error_msg': 'The element can\'t be pasted here'
            }

        return dic

    @view_config(route_name='get_comment_modal_json')
    def get_comment_modal_json(self):
        # TODO: remove this function, it should be done in angular
        comment = self.request.GET.get('comment') or ''
        content = render('blocks/comment_modal.mak',
                         {'comment': comment}, self.request)
        return {'content': content}

def get_dtd_urls(request):
    if 'dtd_urls' not in request.registry.settings:
        raise AttributeError('No dtd_urls defined in the ini file.')
    return filter(bool, request.registry.settings['dtd_urls'].split('\n'))

def get_xml_plugins(request):
    if 'waxe.xml.plugins' not in request.registry.settings:
        return []
    lis = filter(bool,
                 request.registry.settings['waxe.xml.plugins'].split('\n'))

    mods = []
    for s in lis:
        mods.append(importlib.import_module(s))
    return mods

def get_xmltool_transform(request):
    """Before writing XML, we can call a function to transform it.
    """
    if 'waxe.xml.xmltool.transform' not in request.registry.settings:
        return None
    func = request.registry.settings['waxe.xml.xmltool.transform']
    mod, func = func.rsplit('.', 1)
    return getattr(importlib.import_module(mod), func)

def includeme(config):
    settings = config.registry.settings
    cache_timeout = settings.get('xmltool.cache_timeout')
    if cache_timeout:
        xmltool.cache.CACHE_TIMEOUT = cache_timeout

    config.set_request_property(get_dtd_urls, 'dtd_urls', reify=True)
    config.set_request_property(get_xml_plugins, 'xml_plugins', reify=True)
    config.set_request_property(get_xmltool_transform, 'xmltool_transform',
                                reify=True)

    config.add_route('edit_json', '/edit.json')
    config.add_route('edit_text_json', '/edit-text.json')
    config.add_route('new_json', '/new.json')
    config.add_route('update_json', '/update.json')
    config.add_route('update_text_json', '/update-text.json')
    config.add_route('add_element_json', '/add-element.json')
    config.add_route('get_comment_modal_json', '/get-comment-modal.json')
    config.add_route('copy_json', '/copy.json')
    config.add_route('paste_json', '/paste.json')
    config.add_route('get_tags_json', '/get-tags.json')
    config.scan(__name__)

    # We have to be sure we don't have any prefix
    config.route_prefix = None
    config.add_static_view(
        'static-xml',
        'waxe.xml:static',
        cache_max_age=3600,
    )

    resource.add_js_resource('waxe.xml:static/jstree.min.js')
    resource.add_js_resource('waxe.xml:static/xmltool.js')
    resource.add_css_resource('waxe.xml:static/xmltool.min.css')
    resource.add_css_resource('waxe.xml:static/themes/default/style.min.css')

    # When we update a file as txt, we validate it if it's an XML.
    events.on('before_update.txt', is_valid_filecontent)
<commit_msg>Use minified version<commit_after>import os
import tempfile
import xmltool
from xmltool import dtd_parser, render as xt_render
from lxml import etree
import json
import importlib

from urllib2 import HTTPError, URLError
from pyramid.view import view_config
from pyramid.renderers import render, Response
import pyramid.httpexceptions as exc
from waxe.core import browser, utils, resource, events
from waxe.core.views.base import BaseUserView
import pyramid_logging

log = pyramid_logging.getLogger(__name__)

import waxe.xml

EXTENSIONS = waxe.xml.EXTENSIONS
ROUTE_PREFIX = waxe.xml.ROUTE_PREFIX

def _get_tags(dtd_url):
    dic = xmltool.dtd_parser.parse(dtd_url=dtd_url)
    lis = []
    for k, v in dic.items():
        if issubclass(v, xmltool.elements.TextElement):
            continue
        lis += [k]
    lis.sort()
    return lis

# Basic plugin system
def match(request, str_id, dtd_url):
    for plugin in request.xml_plugins:
        if plugin.match(request, str_id, dtd_url):
            return plugin

def add_element(request, str_id, dtd_url):
    plugin = match(request, str_id, dtd_url)
    if not plugin:
        return None
    return plugin.add_element(request, str_id, dtd_url)
# end Basic plugin system

def is_valid_filecontent(view, path, filecontent):
    if not os.path.splitext(path)[1] == '.xml':
        return filecontent

    try:
        xmltool.load_string(filecontent)
        transform = view.request.xmltool_transform
        if transform:
            return view, path, transform(filecontent)
        return view, path, filecontent
    except Exception, e:
        raise exc.HTTPInternalServerError(str(e))

class EditorView(BaseUserView):

    def _get_html_renderer(self):
        if 'waxe.xml.xmltool.renderer_func' not in self.request.registry.settings:
            return None

        func = self.request.registry.settings['waxe.xml.xmltool.renderer_func']
        mod, func = func.rsplit('.', 1)
        return getattr(importlib.import_module(mod), func)(
            self.current_user.login)

    @view_config(route_name='edit_json')
    def edit(self):
        filename = self.request.GET.get('path')
        if not filename:
            raise exc.HTTPClientError('No filename given')
        root_path = self.root_path
        absfilename = browser.absolute_path(filename, root_path)
        iframe = 'iframe' in self.request.GET
        try:
            obj = xmltool.load(absfilename)
            if iframe:
                obj.root.html_renderer = xt_render.ReadonlyRender()
                html = obj.to_html()
            else:
                obj.root.html_renderer = self._get_html_renderer()
                html = xmltool.generate_form_from_obj(
                    obj,
                    form_filename=filename,
                    form_attrs={
                        'data-add-href': self.request.custom_route_path('add_element_json'),
                        'data-comment-href': self.request.custom_route_path('get_comment_modal_json'),
                        'data-action': self.request.custom_route_path('update_json'),
                        'data-copy-href': self.request.custom_route_path('copy_json'),
                        'data-paste-href': self.request.custom_route_path('paste_json'),
                    }
                )
            jstree_data = obj.to_jstree_dict()
        except (HTTPError, URLError), e:
            log.exception(e, request=self.request)
            raise exc.HTTPInternalServerError(
                "The dtd of %s can't be loaded." % filename)
        except etree.XMLSyntaxError, e:
            log.exception(e, request=self.request)
            return self.edit_text(e)
        except Exception, e:
            log.exception(e, request=self.request)
            raise exc.HTTPInternalServerError(str(e))

        if 'iframe' in self.request.GET:
            # TODO: we should remove this logic when adding a window to edit
            # tag directly
            return Response(
                render('iframe.mak',
                       {
                           'content': html,
                           'jstree_data': json.dumps(jstree_data),
                       },
                       self.request))

        self.add_opened_file(filename)
        return {
            'content': html,
            'jstree_data': jstree_data,
        }

    @view_config(route_name='edit_text_json')
    def edit_text(self, exception=None):
        filename = self.request.GET.get('path')
        if not filename:
            raise exc.HTTPClientError('No filename given')
        root_path = self.root_path
        absfilename = browser.absolute_path(filename, root_path)
        try:
            content = open(absfilename, 'r').read()
            content = content.decode('utf-8')
        except Exception, e:
            log.exception(e, request=self.request)
            raise exc.HTTPInternalServerError(str(e))

        content = utils.escape_entities(content)

        html = u'<form id="xmltool-form" class="no-tree" data-action="%s" method="POST">' % (
            self.request.custom_route_path('update_text_json'),
        )
        html += u'<input type="hidden" id="_xml_filename" name="filename" value="%s" />' % filename
        html += u'<textarea class="codemirror" name="filecontent">%s</textarea>' % content
        html += u'</form>'

        dic = {
            'content': html,
        }
        # TODO: Support to display message in angular or change this logic
        if exception:
            dic['error_msg'] = str(exception)
        return dic

    @view_config(route_name='get_tags_json')
    def get_tags(self):
        dtd_url = self.request.GET.get('dtd_url', None)
        if not dtd_url:
            raise exc.HTTPClientError('No dtd url given')
        return _get_tags(dtd_url)

    @view_config(route_name='new_json')
    def new(self):
        relpath = self.request.GET.get('path')
        dtd_url = self.request.GET.get('dtd_url')
        dtd_tag = self.request.GET.get('dtd_tag')

        obj = None
        if dtd_tag and dtd_url:
            # Create new object from the dtd url and tag
            try:
                dic = dtd_parser.parse(dtd_url=dtd_url)
            except (HTTPError, URLError), e:
                log.exception(e, request=self.request)
                raise exc.HTTPInternalServerError(
                    "The dtd file %s can't be loaded." % dtd_url)
            if dtd_tag not in dic:
                raise exc.HTTPInternalServerError(
                    'Invalid dtd element: %s (%s)' % (dtd_tag, dtd_url))
            obj = dic[dtd_tag]()
            obj.dtd_url = dtd_url
        elif relpath:
            # Create new object from a template
            absfilename = browser.absolute_path(relpath, self.root_path)
            try:
                obj = xmltool.load(absfilename)
            except Exception, e:
                log.exception(e, request=self.request)
                raise exc.HTTPInternalServerError(str(e))

        if obj:
            obj.root.html_renderer = self._get_html_renderer()
            html = xmltool.generate_form_from_obj(
                obj,
                form_attrs={
                    'data-add-href': self.request.custom_route_path('add_element_json'),
                    'data-comment-href': self.request.custom_route_path('get_comment_modal_json'),
                    'data-action': self.request.custom_route_path('update_json'),
                    'data-copy-href': self.request.custom_route_path('copy_json'),
                    'data-paste-href': self.request.custom_route_path('paste_json'),
                }
            )
            jstree_data = obj.to_jstree_dict()
            return {
                'content': html,
                'jstree_data': jstree_data,
            }

        content = render('blocks/new.mak',
                         {'dtd_urls': self.request.dtd_urls,
                          'tags': _get_tags(self.request.dtd_urls[0])},
                         self.request)
        return {'modal': content}

    @view_config(route_name='update_json')
    def update(self):
        data = self.req_post
        filename = data.pop('_xml_filename', None)
        if not filename:
            raise exc.HTTPClientError('No filename given')

        root, ext = os.path.splitext(filename)
        if ext != '.xml':
            error_msg = 'No filename extension.'
            if ext:
                error_msg = "Bad filename extension '%s'." % ext
            error_msg += " It should be '.xml'"
            raise exc.HTTPClientError(error_msg)

        root_path = self.root_path
        absfilename = browser.absolute_path(filename, root_path)
        try:
            xmltool.update(absfilename, data,
                           transform=self.request.xmltool_transform)
        except (HTTPError, URLError), e:
            log.exception(e, request=self.request)
            raise exc.HTTPInternalServerError(
                "The dtd of %s can't be loaded." % filename)
        except Exception, e:
            log.exception(e, request=self.request)
            raise exc.HTTPInternalServerError(str(e))

        events.trigger('updated.xml',
                       view=self,
                       path=filename)
        return 'File updated'

    @view_config(route_name='update_text_json')
    def update_text(self):
        filecontent = self.req_post.get('filecontent')
        filename = self.req_post.get('filename') or ''
        if not filecontent or not filename:
            raise exc.HTTPClientError('Missing parameters!')
        root_path = self.root_path
        absfilename = browser.absolute_path(filename, root_path)
        try:
            obj = xmltool.load_string(filecontent)
            obj.write(absfilename, transform=self.request.xmltool_transform())
        except Exception, e:
            raise exc.HTTPInternalServerError(str(e))

        # TODO: the modal logic should be in angular
        modal = None
        msg = 'File updated'
        if self.request.POST.get('commit'):
            modal = render('blocks/commit_modal.mak',
                           {}, self.request)

        events.trigger('updated', view=self, paths=filename)
        if modal:
            return {
                'modal': modal
            }

        return msg

    @view_config(route_name='add_element_json')
    def add_element_json(self):
        elt_id = self.request.GET.get('elt_id')
        dtd_url = self.request.GET.get('dtd_url')
        if not elt_id or not dtd_url:
            return {'error_msg': 'Bad parameter'}

        res = add_element(self.request, elt_id, dtd_url)
        if res:
            return res

        dic = xmltool.factory.get_data_from_str_id_for_html_display(
            elt_id,
            dtd_url=dtd_url,
            html_renderer=self._get_html_renderer()
        )
        return dic

    @view_config(route_name='copy_json')
    def copy_json(self):
        if 'elt_id' not in self.request.POST:
            return {'error_msg': 'Bad parameter'}
        data = xmltool.factory.getElementData(self.request.POST['elt_id'],
                                              self.request.POST)
        # Write the content to paste in a temporary file
        filename = tempfile.mktemp()
        open(filename, 'w').write(json.dumps(data))
        self.request.session['clipboard'] = {
            'filename': filename,
            'elt_id': self.request.POST['elt_id'],
        }
        return {'info_msg': 'Copied'}

    @view_config(route_name='paste_json')
    def paste_json(self):
        # TODO: Validate it's the same dtd
        elt_id = self.request.POST.pop('elt_id', None)
        dtd_url = self.request.POST.pop('_xml_dtd_url', None)
        data = xmltool.utils.unflatten_params(self.request.POST)

        if not elt_id or not dtd_url:
            return {'error_msg': 'Bad parameter'}

        clipboard = self.request.session.get('clipboard')
        if not clipboard:
            return {
                'error_msg': 'Empty clipboard'
            }
        filename = clipboard['filename']
        clipboard_data = json.loads(open(filename, 'r').read())

        dic = xmltool.factory.get_new_element_data_for_html_display(
            elt_id, data,
            clipboard_data, dtd_url,
            # Don't keep the attributes nor the comments
            skip_extra=True,
            html_renderer=self._get_html_renderer()
        )
        if not dic:
            return {
                'error_msg': 'The element can\'t be pasted here'
            }

        return dic

    @view_config(route_name='get_comment_modal_json')
    def get_comment_modal_json(self):
        # TODO: remove this function, it should be done in angular
        comment = self.request.GET.get('comment') or ''
        content = render('blocks/comment_modal.mak',
                         {'comment': comment}, self.request)
        return {'content': content}

def get_dtd_urls(request):
    if 'dtd_urls' not in request.registry.settings:
        raise AttributeError('No dtd_urls defined in the ini file.')
    return filter(bool, request.registry.settings['dtd_urls'].split('\n'))

def get_xml_plugins(request):
    if 'waxe.xml.plugins' not in request.registry.settings:
        return []
    lis = filter(bool,
                 request.registry.settings['waxe.xml.plugins'].split('\n'))

    mods = []
    for s in lis:
        mods.append(importlib.import_module(s))
    return mods

def get_xmltool_transform(request):
    """Before writing XML, we can call a function to transform it.
    """
    if 'waxe.xml.xmltool.transform' not in request.registry.settings:
        return None
    func = request.registry.settings['waxe.xml.xmltool.transform']
    mod, func = func.rsplit('.', 1)
    return getattr(importlib.import_module(mod), func)

def includeme(config):
    settings = config.registry.settings
    cache_timeout = settings.get('xmltool.cache_timeout')
    if cache_timeout:
        xmltool.cache.CACHE_TIMEOUT = cache_timeout

    config.set_request_property(get_dtd_urls, 'dtd_urls', reify=True)
    config.set_request_property(get_xml_plugins, 'xml_plugins', reify=True)
    config.set_request_property(get_xmltool_transform, 'xmltool_transform',
                                reify=True)

    config.add_route('edit_json', '/edit.json')
    config.add_route('edit_text_json', '/edit-text.json')
    config.add_route('new_json', '/new.json')
    config.add_route('update_json', '/update.json')
    config.add_route('update_text_json', '/update-text.json')
    config.add_route('add_element_json', '/add-element.json')
    config.add_route('get_comment_modal_json', '/get-comment-modal.json')
    config.add_route('copy_json', '/copy.json')
    config.add_route('paste_json', '/paste.json')
    config.add_route('get_tags_json', '/get-tags.json')
    config.scan(__name__)

    # We have to be sure we don't have any prefix
    config.route_prefix = None
    config.add_static_view(
        'static-xml',
        'waxe.xml:static',
        cache_max_age=3600,
    )

    resource.add_js_resource('waxe.xml:static/jstree.min.js')
    resource.add_js_resource('waxe.xml:static/xmltool.min.js')
    resource.add_css_resource('waxe.xml:static/xmltool.min.css')
    resource.add_css_resource('waxe.xml:static/themes/default/style.min.css')

    # When we update a file as txt, we validate it if it's an XML.
    events.on('before_update.txt', is_valid_filecontent)
<|endoftext|>


<commit_before>"""Riak protocol buffers client.

Provides a couple interfaces to working with a Riak database.

The first is a lower-level Client object. It has methods for interacting with
the server, buckets, keys and values. The methods are more verbose and return
dense dictionaries of data.

The other interface is via Buckets. These use a Client to allow you to work
with keys and values in the context of a Riak bucket. They offer a simpler API
and hooks for conflict resolution and custom loading/dumping of values.

"""
import struct

import diesel

from diesel.protocols import riak_pb2
from contextlib import contextmanager

# The commented-out message codes and types below are for requests and/or
# responses that don't have a body.

MESSAGE_CODES = [
(0, riak_pb2.RpbErrorResp),
#(1, riak_pb2.RpbPingReq),
#(2, riak_pb2.RpbPingResp),
#(3, riak_pb2.RpbGetClientIdReq),
(4, riak_pb2.RpbGetClientIdResp),
(5, riak_pb2.RpbSetClientIdReq),
#(6, riak_pb2.RpbSetClientIdResp),
#(7, riak_pb2.RpbGetServerInfoReq),
(8, riak_pb2.RpbGetServerInfoResp),
(9, riak_pb2.RpbGetReq ),
(10, riak_pb2.RpbGetResp),
(11, riak_pb2.RpbPutReq ),
(12, riak_pb2.RpbPutResp),
(13, riak_pb2.RpbDelReq ),
#(14, riak_pb2.RpbDelResp),
#(15, riak_pb2.RpbListBucketsReq),
(16, riak_pb2.RpbListBucketsResp),
(17, riak_pb2.RpbListKeysReq),
(18, riak_pb2.RpbListKeysResp),
(19, riak_pb2.RpbGetBucketReq),
(20, riak_pb2.RpbGetBucketResp),
(21, riak_pb2.RpbSetBucketReq),
#(22, riak_pb2.RpbSetBucketResp),
(23, riak_pb2.RpbMapRedReq),
(24, riak_pb2.RpbMapRedResp),
]

PB_TO_MESSAGE_CODE = dict((cls, code) for code, cls in MESSAGE_CODES)
MESSAGE_CODE_TO_PB = dict(MESSAGE_CODES)

class Bucket(object):
    """A Bucket of keys/values in a Riak database.

    Buckets are intended to be cheap, non-shared objects for easily performing
    common key/value operations with Riak.

    Buckets should not be shared amongst concurrent consumers. Each consumer
    should have its own Bucket instance. This is due to the way vector clocks
    are tracked on the Bucket instances.

    """
    def __init__(self, name, client=None, make_client_context=None, resolver=None):
        """Return a new Bucket for the named bucket, using the given client.

        If your bucket allows sibling content (conflicts) you should supply a
        conflict resolver function or subclass and override ``resolve``. It
        should take four arguments and return a resolved value. Here is an
        example::

            def resolve_random(timestamp_1, value_1, timestamp_2, value_2):
                '''The worlds worst resolver function!'''
                return random.choice([value_1, value_2])

        """
        assert client is not None or make_client_context is not None,\
        "Must specify either client or make_client_context"
        assert not (client is not None and make_client_context is not None),\
        "Cannot specify both client and make_client_context"
        self.name = name
        if make_client_context:
            self.make_client_context = make_client_context
        else:
            @contextmanager
            def noop_cm():
                yield client
            self.make_client_context = noop_cm
        self._vclocks = {}
        if resolver:
            self.resolve = resolver

    def get(self, key):
        """Get the value for key from the bucket.

        Records the vector clock for the value for future modification
        using ``put`` with this same Bucket instance.

        """
        with self.make_client_context() as client:
            response = client.get(self.name, key)
        if response:
            return self._handle_response(key, response)

    def put(self, key, value, safe=True, **params):
        """Puts the given key/value into the bucket.

        If safe==True the response will be read back and conflict resolution
        might take place as well if the put triggers multiple sibling values
        for the key.

        Extra params are passed to the ``Client.put`` method.
        """
        if safe:
            params['return_body'] = True
            if 'vclock' not in params and key in self._vclocks:
                params['vclock'] = self._vclocks.pop(key)
        with self.make_client_context() as client:
            response = client.put(self.name, key, self.dumps(value), **params)
        if response:
            return self._handle_response(key, response)

    def delete(self, key):
        """Deletes all values for the given key from the bucket."""
        with self.make_client_context() as client:
            client.delete(self.name, key)

    def _handle_response(self, key, response):
        # Returns responses for non-conflicting content. Resolves conflicts
        # if there are multiple values for a key.
        if len(response['content']) == 1:
            self._vclocks[key] = response['vclock']
            return self.loads(response['content'][0]['value'])
        else:
            return self._resolve(key, response)

    def _resolve(self, key, response):
        # Performs conflict resolution for the given key and response. If all
        # goes well a new harmonized value for the key will be put up to the
        # bucket. If things go wrong, expect ... exceptions. :-|
        res = response['content'].pop(0)
        while response['content']:
            other = response['content'].pop(0)
            other['value'] = self.dumps(
                self.resolve(
                    res['last_mod'],
                    self.loads(res['value']),
                    other['last_mod'],
                    self.loads(other['value']),
                )
            )
            res = other
        resolved_value = self.loads(res['value'])
        params = dict(vclock=response['vclock'], return_body=True)
        return self.put(key, resolved_value, **params)

    def resolve(self, timestamp1, value1, timestamp2, value2):
        """Subclass to support custom conflict resolution."""
        msg = "Pass in a resolver or override this method."
        raise NotImplementedError(msg)

    def loads(self, raw_value):
        """Subclass to support loading rich values."""
        return raw_value

    def dumps(self, rich_value):
        """Subclass to support dumping rich values."""
        return rich_value

class RiakClient(diesel.Client):
    """A client for the Riak distributed key/value database.

    Instances can be used stand-alone or passed to a Bucket constructor
    (which has a simpler API).

    """
    def __init__(self, host='127.0.0.1', port=8087, **kw):
        """Creates a new Riak Client connection object."""
        diesel.Client.__init__(self, host, port, **kw)

    @diesel.call
    def get(self, bucket, key):
        """Get the value of key from named bucket.

        Returns a dictionary with a list of the content for the key
        and the vector clock (vclock) for the key.

        """
        request = riak_pb2.RpbGetReq(bucket=bucket, key=key)
        self._send(request)
        response = self._receive()
        if response:
            return _to_dict(response)

    @diesel.call
    def put(self, bucket, key, value, **params):
        """Puts the value to the key in the named bucket.

        If an ``extra_content`` dictionary parameter is present, its content
        is merged into the RpbContent object.

        All other parameters are merged into the RpbPutReq object.

        """
        dict_content={'value':value}
        if 'extra_content' in params:
            dict_content.update(params.pop('extra_content'))
        content = riak_pb2.RpbContent(**dict_content)
        request = riak_pb2.RpbPutReq(
            bucket=bucket,
            key=key,
            content=content,
        )
        for name, value in params.iteritems():
            setattr(request, name, value)
        self._send(request)
        response = self._receive()
        if response:
            return _to_dict(response)

    @diesel.call
    def delete(self, bucket, key):
        """Deletes the given key from the named bucket, including all values."""
        request = riak_pb2.RpbDelReq(bucket=bucket, key=key)
        self._send(request)
        return self._receive()

    @diesel.call
    def info(self):
        # No protocol buffer object to build or send.
        message_code = 7
        total_size = 1
        fmt = "!iB"
        diesel.send(struct.pack(fmt, total_size, message_code))
        return self._receive()

    @diesel.call
    def set_bucket_props(self, bucket, props):
        """Sets some properties on the named bucket.

        ``props`` should be a dictionary of properties supported by the
        RpbBucketProps protocol buffer.

        """
        request = riak_pb2.RpbSetBucketReq(bucket=bucket)
        for name, value in props.iteritems():
            setattr(request.props, name, value)
        self._send(request)
        return self._receive()

    def _send(self, pb):
        # Send a protocol buffer on the wire as a request.
        message_code = PB_TO_MESSAGE_CODE[pb.__class__]
        message = pb.SerializeToString()
        message_size = len(message)
        total_size = message_size + 1 # plus 1 mc byte
        fmt = "!iB%ds" % message_size
        diesel.send(struct.pack(fmt, total_size, message_code, message))

    def _receive(self):
        # Receive a protocol buffer from the wire as a response.
        response_size, = struct.unpack('!i', diesel.receive(4))
        raw_response = diesel.receive(response_size)
        message_code, = struct.unpack('B',raw_response[0])
        response = raw_response[1:]
        if response:
            pb = MESSAGE_CODE_TO_PB[message_code]()
            pb.ParseFromString(response)
            return pb

def _to_dict(pb):
    # Takes a protocol buffer (pb) and transforms it into a dict of more
    # common Python types.
    out = {}
    for descriptor, value in pb.ListFields():
        # Perform a couple sniff tests to see if a value is:
        # a) A protocol buffer
        # b) An iterable protocol buffer
        # c) Neither
        # Handles all of those situations.
        try:
            value.MergeFrom
            try:
                value = [_to_dict(v) for v in iter(value)]
            except TypeError:
                value = _to_dict(v)
        except AttributeError:
            pass
        out[descriptor.name] = value
    return out

# Testing Code Below
# ==================
# XXX hack - can't define it in __main__ below
class Point(object):
    def __init__(self, x, y):
        self.x = x
        self.y = y

if __name__ == '__main__':
    import cPickle

    def test_client():
        c = RiakClient('riak-1')

        # Do some cleanup from a previous run.
        c.delete('testing', 'bar')
        c.delete('testing', 'foo')
        c.delete('testing', 'lala')
        c.delete('testing', 'baz')
        c.delete('testing.pickles', 'here')
        c.delete('testing.pickles', 'there')

        assert not c.get('testing', 'foo')
        assert c.put('testing', 'foo', '1', return_body=True)
        assert c.get('testing', 'foo')

        # Create a conflict for the 'bar' key in 'testing'.
        assert not c.set_bucket_props('testing', {'allow_mult':True})
        assert c.put('testing', 'bar', 'hi', return_body=True)
        assert c.put('testing', 'bar', 'bye', return_body=True)
        assert len(c.get('testing', 'bar')['content']) > 1

        def resolve_longest(t1, v1, t2, v2):
            if len(v1) > len(v2):
                return v1
            return v2

        # Test that the conflict is resolved, this time using the Bucket
        # interface.
        b = Bucket('testing', c, resolver=resolve_longest)
        resolved = b.get('bar')
        assert resolved == 'bye', resolved
        assert len(c.get('testing', 'bar')['content']) == 1

        # put/get/delete with a Bucket
        assert b.put('lala', 'g'*1024)
        assert b.get('lala') == 'g'*1024
        b.delete('lala')
        assert not b.get('lala')

        # Multiple changes to a key using a Bucket should be a-ok.
        assert b.put('baz', 'zzzz')
        assert b.put('baz', 'ffff')
        assert b.put('baz', 'tttt')
        assert len(c.get('testing', 'baz')['content']) == 1
        assert b.get('baz') == 'tttt'

        # Custom Bucket.
        class PickleBucket(Bucket): # lol
            def loads(self, raw_value):
                return cPickle.loads(raw_value)

            def dumps(self, rich_value):
                return cPickle.dumps(rich_value)

            def resolve(self, t1, v1, t2, v2):
                # Returns the value with the smallest different between points.
                d1 = abs(v1.x - v1.y)
                d2 = abs(v2.x - v2.y)
                if d1 < d2:
                    return v1
                return v2

        assert not c.set_bucket_props('testing.pickles', {'allow_mult':True})
        p = PickleBucket('testing.pickles', c)
        assert p.put('here', Point(4,2))
        out = p.get('here')
        assert (4,2) == (out.x, out.y)
        assert isinstance(out, Point)

        # Resolve Point conflicts.
        p.put('there', Point(4,12), safe=False)
        p.put('there', Point(3,7), safe=False)
        there = p.get('there')
        assert (3,7) == (there.x, there.y), (there.x, there.y)
        assert isinstance(there, Point)

        diesel.quickstop()
    diesel.quickstart(test_client)

del Point
<commit_msg>Support parallel subrequests for get_many.<commit_after>"""Riak protocol buffers client.

Provides a couple interfaces to working with a Riak database.

The first is a lower-level Client object. It has methods for interacting with
the server, buckets, keys and values. The methods are more verbose and return
dense dictionaries of data.

The other interface is via Buckets. These use a Client to allow you to work
with keys and values in the context of a Riak bucket. They offer a simpler API
and hooks for conflict resolution and custom loading/dumping of values.

"""
import struct

import diesel
from diesel.util.queue import Queue, QueueEmpty

from diesel.protocols import riak_pb2
from contextlib import contextmanager

# The commented-out message codes and types below are for requests and/or
# responses that don't have a body.

MESSAGE_CODES = [
(0, riak_pb2.RpbErrorResp),
#(1, riak_pb2.RpbPingReq),
#(2, riak_pb2.RpbPingResp),
#(3, riak_pb2.RpbGetClientIdReq),
(4, riak_pb2.RpbGetClientIdResp),
(5, riak_pb2.RpbSetClientIdReq),
#(6, riak_pb2.RpbSetClientIdResp),
#(7, riak_pb2.RpbGetServerInfoReq),
(8, riak_pb2.RpbGetServerInfoResp),
(9, riak_pb2.RpbGetReq ),
(10, riak_pb2.RpbGetResp),
(11, riak_pb2.RpbPutReq ),
(12, riak_pb2.RpbPutResp),
(13, riak_pb2.RpbDelReq ),
#(14, riak_pb2.RpbDelResp),
#(15, riak_pb2.RpbListBucketsReq),
(16, riak_pb2.RpbListBucketsResp),
(17, riak_pb2.RpbListKeysReq),
(18, riak_pb2.RpbListKeysResp),
(19, riak_pb2.RpbGetBucketReq),
(20, riak_pb2.RpbGetBucketResp),
(21, riak_pb2.RpbSetBucketReq),
#(22, riak_pb2.RpbSetBucketResp),
(23, riak_pb2.RpbMapRedReq),
(24, riak_pb2.RpbMapRedResp),
]

PB_TO_MESSAGE_CODE = dict((cls, code) for code, cls in MESSAGE_CODES)
MESSAGE_CODE_TO_PB = dict(MESSAGE_CODES)

class BucketSubrequestException(Exception):
    def __init__(self, s, sub_exceptions):
        Exception.__init__(s)
        self.sub_exceptions = sub_exceptions

class Bucket(object):
    """A Bucket of keys/values in a Riak database.

    Buckets are intended to be cheap, non-shared objects for easily performing
    common key/value operations with Riak.

    Buckets should not be shared amongst concurrent consumers. Each consumer
    should have its own Bucket instance. This is due to the way vector clocks
    are tracked on the Bucket instances.

    """
    def __init__(self, name, client=None, make_client_context=None, resolver=None):
        """Return a new Bucket for the named bucket, using the given client.

        If your bucket allows sibling content (conflicts) you should supply a
        conflict resolver function or subclass and override ``resolve``. It
        should take four arguments and return a resolved value. Here is an
        example::

            def resolve_random(timestamp_1, value_1, timestamp_2, value_2):
                '''The worlds worst resolver function!'''
                return random.choice([value_1, value_2])

        """
        assert client is not None or make_client_context is not None,\
        "Must specify either client or make_client_context"
        assert not (client is not None and make_client_context is not None),\
        "Cannot specify both client and make_client_context"
        self.name = name
        if make_client_context:
            self.make_client_context = make_client_context
            self.used_client_context = True
        else:
            @contextmanager
            def noop_cm():
                yield client
            self.make_client_context = noop_cm
            self.used_client_context = False
        self._vclocks = {}
        if resolver:
            self.resolve = resolver

    def get(self, key):
        """Get the value for key from the bucket.

        Records the vector clock for the value for future modification
        using ``put`` with this same Bucket instance.

        """
        with self.make_client_context() as client:
            response = client.get(self.name, key)
        if response:
            return self._handle_response(key, response)

    def _subrequest(self, inq, outq):
        while True:
            try:
                key = inq.get(waiting=False)
            except QueueEmpty:
                break
            else:
                try:
                    res = self.get(key)
                except Exception, e:
                    outq.put((key, False, e))
                else:
                    outq.put((key, True, res))

    def get_many(self, keys, concurrency_limit=100, no_failures=False):
        assert self.used_client_context,\
        "Cannot fetch in parallel without a pooled make_client_context!"
        inq = Queue()
        outq = Queue()
        for k in keys:
            inq.put(k)

        for x in xrange(min(len(keys), concurrency_limit)):
            diesel.fork(self._subrequest, inq, outq)

        failure = False
        okay, err = [], []
        for k in keys:
            (key, success, val) = outq.get()
            if success:
                okay.append((key, val))
            else:
                err.append((key, val))

        if no_failures:
            raise BucketSubrequestException(
            "Error in parallel subrequests", err)
        return okay, err

    def put(self, key, value, safe=True, **params):
        """Puts the given key/value into the bucket.

        If safe==True the response will be read back and conflict resolution
        might take place as well if the put triggers multiple sibling values
        for the key.

        Extra params are passed to the ``Client.put`` method.
        """
        if safe:
            params['return_body'] = True
            if 'vclock' not in params and key in self._vclocks:
                params['vclock'] = self._vclocks.pop(key)
        with self.make_client_context() as client:
            response = client.put(self.name, key, self.dumps(value), **params)
        if response:
            return self._handle_response(key, response)

    def delete(self, key):
        """Deletes all values for the given key from the bucket."""
        with self.make_client_context() as client:
            client.delete(self.name, key)

    def _handle_response(self, key, response):
        # Returns responses for non-conflicting content. Resolves conflicts
        # if there are multiple values for a key.
        if len(response['content']) == 1:
            self._vclocks[key] = response['vclock']
            return self.loads(response['content'][0]['value'])
        else:
            return self._resolve(key, response)

    def _resolve(self, key, response):
        # Performs conflict resolution for the given key and response. If all
        # goes well a new harmonized value for the key will be put up to the
        # bucket. If things go wrong, expect ... exceptions. :-|
        res = response['content'].pop(0)
        while response['content']:
            other = response['content'].pop(0)
            other['value'] = self.dumps(
                self.resolve(
                    res['last_mod'],
                    self.loads(res['value']),
                    other['last_mod'],
                    self.loads(other['value']),
                )
            )
            res = other
        resolved_value = self.loads(res['value'])
        params = dict(vclock=response['vclock'], return_body=True)
        return self.put(key, resolved_value, **params)

    def resolve(self, timestamp1, value1, timestamp2, value2):
        """Subclass to support custom conflict resolution."""
        msg = "Pass in a resolver or override this method."
        raise NotImplementedError(msg)

    def loads(self, raw_value):
        """Subclass to support loading rich values."""
        return raw_value

    def dumps(self, rich_value):
        """Subclass to support dumping rich values."""
        return rich_value

class RiakClient(diesel.Client):
    """A client for the Riak distributed key/value database.

    Instances can be used stand-alone or passed to a Bucket constructor
    (which has a simpler API).

    """
    def __init__(self, host='127.0.0.1', port=8087, **kw):
        """Creates a new Riak Client connection object."""
        diesel.Client.__init__(self, host, port, **kw)

    @diesel.call
    def get(self, bucket, key):
        """Get the value of key from named bucket.

        Returns a dictionary with a list of the content for the key
        and the vector clock (vclock) for the key.

        """
        request = riak_pb2.RpbGetReq(bucket=bucket, key=key)
        self._send(request)
        response = self._receive()
        if response:
            return _to_dict(response)

    @diesel.call
    def put(self, bucket, key, value, **params):
        """Puts the value to the key in the named bucket.

        If an ``extra_content`` dictionary parameter is present, its content
        is merged into the RpbContent object.

        All other parameters are merged into the RpbPutReq object.

        """
        dict_content={'value':value}
        if 'extra_content' in params:
            dict_content.update(params.pop('extra_content'))
        content = riak_pb2.RpbContent(**dict_content)
        request = riak_pb2.RpbPutReq(
            bucket=bucket,
            key=key,
            content=content,
        )
        for name, value in params.iteritems():
            setattr(request, name, value)
        self._send(request)
        response = self._receive()
        if response:
            return _to_dict(response)

    @diesel.call
    def delete(self, bucket, key):
        """Deletes the given key from the named bucket, including all values."""
        request = riak_pb2.RpbDelReq(bucket=bucket, key=key)
        self._send(request)
        return self._receive()

    @diesel.call
    def info(self):
        # No protocol buffer object to build or send.
        message_code = 7
        total_size = 1
        fmt = "!iB"
        diesel.send(struct.pack(fmt, total_size, message_code))
        return self._receive()

    @diesel.call
    def set_bucket_props(self, bucket, props):
        """Sets some properties on the named bucket.

        ``props`` should be a dictionary of properties supported by the
        RpbBucketProps protocol buffer.

        """
        request = riak_pb2.RpbSetBucketReq(bucket=bucket)
        for name, value in props.iteritems():
            setattr(request.props, name, value)
        self._send(request)
        return self._receive()

    def _send(self, pb):
        # Send a protocol buffer on the wire as a request.
        message_code = PB_TO_MESSAGE_CODE[pb.__class__]
        message = pb.SerializeToString()
        message_size = len(message)
        total_size = message_size + 1 # plus 1 mc byte
        fmt = "!iB%ds" % message_size
        diesel.send(struct.pack(fmt, total_size, message_code, message))

    def _receive(self):
        # Receive a protocol buffer from the wire as a response.
        response_size, = struct.unpack('!i', diesel.receive(4))
        raw_response = diesel.receive(response_size)
        message_code, = struct.unpack('B',raw_response[0])
        response = raw_response[1:]
        if response:
            pb = MESSAGE_CODE_TO_PB[message_code]()
            pb.ParseFromString(response)
            return pb

def _to_dict(pb):
    # Takes a protocol buffer (pb) and transforms it into a dict of more
    # common Python types.
    out = {}
    for descriptor, value in pb.ListFields():
        # Perform a couple sniff tests to see if a value is:
        # a) A protocol buffer
        # b) An iterable protocol buffer
        # c) Neither
        # Handles all of those situations.
        try:
            value.MergeFrom
            try:
                value = [_to_dict(v) for v in iter(value)]
            except TypeError:
                value = _to_dict(v)
        except AttributeError:
            pass
        out[descriptor.name] = value
    return out

# Testing Code Below
# ==================
# XXX hack - can't define it in __main__ below
class Point(object):
    def __init__(self, x, y):
        self.x = x
        self.y = y

if __name__ == '__main__':
    import cPickle

    def test_client():
        c = RiakClient('riak-1')

        # Do some cleanup from a previous run.
        c.delete('testing', 'bar')
        c.delete('testing', 'foo')
        c.delete('testing', 'lala')
        c.delete('testing', 'baz')
        c.delete('testing.pickles', 'here')
        c.delete('testing.pickles', 'there')

        assert not c.get('testing', 'foo')
        assert c.put('testing', 'foo', '1', return_body=True)
        assert c.get('testing', 'foo')

        # Create a conflict for the 'bar' key in 'testing'.
        assert not c.set_bucket_props('testing', {'allow_mult':True})
        assert c.put('testing', 'bar', 'hi', return_body=True)
        assert c.put('testing', 'bar', 'bye', return_body=True)
        assert len(c.get('testing', 'bar')['content']) > 1

        def resolve_longest(t1, v1, t2, v2):
            if len(v1) > len(v2):
                return v1
            return v2

        # Test that the conflict is resolved, this time using the Bucket
        # interface.
        b = Bucket('testing', c, resolver=resolve_longest)
        resolved = b.get('bar')
        assert resolved == 'bye', resolved
        assert len(c.get('testing', 'bar')['content']) == 1

        # put/get/delete with a Bucket
        assert b.put('lala', 'g'*1024)
        assert b.get('lala') == 'g'*1024
        b.delete('lala')
        assert not b.get('lala')

        # Multiple changes to a key using a Bucket should be a-ok.
        assert b.put('baz', 'zzzz')
        assert b.put('baz', 'ffff')
        assert b.put('baz', 'tttt')
        assert len(c.get('testing', 'baz')['content']) == 1
        assert b.get('baz') == 'tttt'

        # Custom Bucket.
        class PickleBucket(Bucket): # lol
            def loads(self, raw_value):
                return cPickle.loads(raw_value)

            def dumps(self, rich_value):
                return cPickle.dumps(rich_value)

            def resolve(self, t1, v1, t2, v2):
                # Returns the value with the smallest different between points.
                d1 = abs(v1.x - v1.y)
                d2 = abs(v2.x - v2.y)
                if d1 < d2:
                    return v1
                return v2

        assert not c.set_bucket_props('testing.pickles', {'allow_mult':True})
        p = PickleBucket('testing.pickles', c)
        assert p.put('here', Point(4,2))
        out = p.get('here')
        assert (4,2) == (out.x, out.y)
        assert isinstance(out, Point)

        # Resolve Point conflicts.
        p.put('there', Point(4,12), safe=False)
        p.put('there', Point(3,7), safe=False)
        there = p.get('there')
        assert (3,7) == (there.x, there.y), (there.x, there.y)
        assert isinstance(there, Point)

        diesel.quickstop()
    diesel.quickstart(test_client)

del Point
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright (C) 2016 KMEE (http://www.kmee.com.br)
# License AGPL-3 - See http://www.gnu.org/licenses/agpl-3.0.html

import logging

from openerp import fields, models, exceptions, _
from openerp.tools.safe_eval import safe_eval

_logger = logging.getLogger(__name__)

try:
    from pybrasil.python_pt_BR import python_pt_BR
    # from pybrasil.valor.decimal import Decimal

except ImportError:
    _logger.info('Cannot import pybrasil')

CALCULO_FOLHA_PT_BR = {
    u'resultado': 'result',
    u'valor': 'result',
    u'taxa': 'result_rate',
    u'aliquota': 'result_rate',
    u'alíquota': 'result_rate',
    u'quantidade': 'result_qty',
    u'quant': 'result_qty',
    u'qtd': 'result_qty',
    u'qtde': 'result_qty',
}

class HrSalaryRule(models.Model):
    _inherit = 'hr.salary.rule'

    compoe_base_INSS = fields.Boolean(
        string=u'Compõe Base INSS',
    )

    compoe_base_IR = fields.Boolean(
        string=u'Compõe Base IR',
    )

    compoe_base_FGTS = fields.Boolean(
        string=u'Compõe Base FGTS',
    )

    def compute_rule(self, cr, uid, rule_id, localdict, context=None):
        rule = self.browse(cr, uid, rule_id, context=context)

        if rule.amount_select != 'code':
            return super(HrSalaryRule, self).compute_rule(cr, uid, rule_id,
                                                          localdict,
                                                          context=context)

        codigo_python = python_pt_BR(rule.amount_python_compute or '',
                                     CALCULO_FOLHA_PT_BR)

        try:
            safe_eval(codigo_python, localdict, mode='exec', nocopy=True)
            result = localdict['result']

            if 'result_qty' in localdict:
                result_qty = localdict['result_qty']
            else:
                result_qty = 1

            if 'result_rate' in localdict:
                result_rate = localdict['result_rate']
            else:
                result_rate = 100

            return result, result_qty, result_rate

        except:
            raise exceptions.UserError(
                _('Wrong python code defined for salary rule %s (%s).') % \
                (rule.name, rule.code))

    def satisfy_condition(self, cr, uid, rule_id, localdict, context=None):
        rule = self.browse(cr, uid, rule_id, context=context)

        if rule.condition_select != 'python':
            return super(HrSalaryRule, self).satisfy_condition(cr, uid, rule_id,
                                                               localdict,
                                                               context=context)

        codigo_python = python_pt_BR(rule.condition_python or '',
                                     CALCULO_FOLHA_PT_BR)

        try:
            safe_eval(codigo_python, localdict, mode='exec', nocopy=True)
            return 'result' in localdict and localdict['result'] or False

        except:
            raise exceptions.UserError(
                _('Wrong python condition defined for salary rule %s (%s).') % \
                 (rule.name, rule.code))
<commit_msg>[FIX] Mais PEP8<commit_after># -*- coding: utf-8 -*-
# Copyright (C) 2016 KMEE (http://www.kmee.com.br)
# License AGPL-3 - See http://www.gnu.org/licenses/agpl-3.0.html

import logging

from openerp import fields, models, exceptions, _
from openerp.tools.safe_eval import safe_eval

_logger = logging.getLogger(__name__)

try:
    from pybrasil.python_pt_BR import python_pt_BR
    # from pybrasil.valor.decimal import Decimal

except ImportError:
    _logger.info('Cannot import pybrasil')

CALCULO_FOLHA_PT_BR = {
    u'resultado': 'result',
    u'valor': 'result',
    u'taxa': 'result_rate',
    u'aliquota': 'result_rate',
    u'alíquota': 'result_rate',
    u'quantidade': 'result_qty',
    u'quant': 'result_qty',
    u'qtd': 'result_qty',
    u'qtde': 'result_qty',
}

class HrSalaryRule(models.Model):
    _inherit = 'hr.salary.rule'

    compoe_base_INSS = fields.Boolean(
        string=u'Compõe Base INSS',
    )

    compoe_base_IR = fields.Boolean(
        string=u'Compõe Base IR',
    )

    compoe_base_FGTS = fields.Boolean(
        string=u'Compõe Base FGTS',
    )

    def compute_rule(self, cr, uid, rule_id, localdict, context=None):
        rule = self.browse(cr, uid, rule_id, context=context)

        if rule.amount_select != 'code':
            return super(HrSalaryRule, self).compute_rule(cr, uid, rule_id,
                                                          localdict,
                                                          context=context)

        codigo_python = python_pt_BR(rule.amount_python_compute or '',
                                     CALCULO_FOLHA_PT_BR)

        try:
            safe_eval(codigo_python, localdict, mode='exec', nocopy=True)
            result = localdict['result']

            if 'result_qty' in localdict:
                result_qty = localdict['result_qty']
            else:
                result_qty = 1

            if 'result_rate' in localdict:
                result_rate = localdict['result_rate']
            else:
                result_rate = 100

            return result, result_qty, result_rate

        except:
            raise exceptions.UserError(
                _('Wrong python code defined for salary rule %s (%s).') % \
                (rule.name, rule.code))

    def satisfy_condition(self, cr, uid, rule_id, localdict, context=None):
        rule = self.browse(cr, uid, rule_id, context=context)

        if rule.condition_select != 'python':
            return super(HrSalaryRule, self).satisfy_condition(cr, uid,
                rule_id, localdict, context=context)

        codigo_python = python_pt_BR(rule.condition_python or '',
                                     CALCULO_FOLHA_PT_BR)

        try:
            safe_eval(codigo_python, localdict, mode='exec', nocopy=True)
            return 'result' in localdict and localdict['result'] or False

        except:
            raise exceptions.UserError(
                _('Wrong python condition defined for salary rule %s (%s).') %\
                 (rule.name, rule.code))
<|endoftext|>


<commit_before>VERSION = (0, 3, 4)

try:
    from django_url_framework.site import Site
    from django_url_framework.controller import ActionController
    from django_url_framework.exceptions import InvalidActionError
    from django_url_framework.exceptions import InvalidControllerError
    from django_url_framework.helper import ApplicationHelper
    site = Site()
except ImportError:
    #todo this is an ugly hack for setuptools to load version, fix
    pass

# Dynamically calculate the version based on VERSION tuple
if len(VERSION)>2 and VERSION[2] is not None:
    str_version = "%d.%d.%s" % VERSION[:3]
else:
    str_version = "%d.%d" % VERSION[:2]

__version__ = str_version

def reraise(exception, info=None):
    import sys
    raise exception, None, sys.exc_info()[-1]

<commit_msg>version bump<commit_after>VERSION = (0, 3, 5)

try:
    from django_url_framework.site import Site
    from django_url_framework.controller import ActionController
    from django_url_framework.exceptions import InvalidActionError
    from django_url_framework.exceptions import InvalidControllerError
    from django_url_framework.helper import ApplicationHelper
    site = Site()
except ImportError:
    #todo this is an ugly hack for setuptools to load version, fix
    pass

# Dynamically calculate the version based on VERSION tuple
if len(VERSION)>2 and VERSION[2] is not None:
    str_version = "%d.%d.%s" % VERSION[:3]
else:
    str_version = "%d.%d" % VERSION[:2]

__version__ = str_version

def reraise(exception, info=None):
    import sys
    raise exception, None, sys.exc_info()[-1]

<|endoftext|>


<commit_before>from __future__ import unicode_literals

from django.db import models
from django.utils.translation import ugettext_lazy as _
from django.utils.functional import cached_property
from django.core import serializers
from django.conf import settings
from django.core.urlresolvers import reverse
from django.utils.encoding import python_2_unicode_compatible
from elasticsearch import ConnectionError

from modelcluster.fields import M2MField

from wagtail.wagtailcore.models import Page, Site
from wagtail.wagtailcore.fields import RichTextField

from wagtail.wagtailadmin.edit_handlers import FieldPanel, \
    MultiFieldPanel, InlinePanel, PageChooserPanel, FieldRowPanel
from wagtail.wagtailimages.edit_handlers import ImageChooserPanel
from wagtail.wagtailsearch import index
from wagtail.wagtailsearch.backends import get_search_backend
from wagtail.wagtailsearch.backends.elasticsearch import ElasticSearchMapping, \
    ElasticSearchResults

from core.edit_handlers import M2MFieldPanel

# Override the url property of the Page model
# to accommodate for child pages
from core.utils import get_translations_for_page, SearchBoost

Page.wg_url = Page.url

@cached_property
def url_property(self):
    instance = self.specific
    if getattr(instance, 'get_absolute_url', None):
        return instance.get_absolute_url()
    return self.wg_url

Page.url = url_property

@python_2_unicode_compatible
class Article(Page):
    authors = M2MField("author.Author", related_name="articles_by_author")
    translators = M2MField("author.Author",
                           related_name="translations_by_author",
                           blank=True)
    strap = models.TextField(blank=True)
    content = RichTextField()
    language = models.CharField(max_length=7, choices=settings.LANGUAGES)
    original_published_date = models.DateField(null=True, blank=True)
    show_day = models.BooleanField(default=True)
    show_month = models.BooleanField(default=True)
    show_year = models.BooleanField(default=True)
    featured_image = models.ForeignKey('core.AffixImage',
                                       null=True, blank=True,
                                       on_delete=models.SET_NULL)
    show_featured_image = models.BooleanField(default=True)

    categories = M2MField("category.Category", related_name="articles_by_category")
    locations = M2MField("location.Location", related_name="articles_by_location", blank=True)
    content_panels = Page.content_panels + [
        FieldPanel('strap'),
        M2MFieldPanel('authors'),
        M2MFieldPanel('translators'),
        FieldPanel('language'),
        MultiFieldPanel(
            [
                FieldPanel('original_published_date'),
                FieldRowPanel(
                    [
                        FieldPanel('show_day',classname="col4"),
                        FieldPanel('show_month',classname="col4"),
                        FieldPanel('show_year',classname="col4")
                    ])
            ],'Date'),
        FieldPanel('content'),
        MultiFieldPanel(
            [
                ImageChooserPanel('featured_image'),
                FieldPanel('show_featured_image'),
            ], 'Cover Image'),
        FieldPanel('categories'),
        M2MFieldPanel('locations'),
    ]

    search_fields = Page.search_fields + [
        index.SearchField('title', partial_match=True, boost=SearchBoost.TITLE),
        index.SearchField('authors', partial_match=True, boost=SearchBoost.AUTHOR),
        index.SearchField('translators', partial_match=True, boost=SearchBoost.AUTHOR),
        index.SearchField('strap', partial_match=True, boost=SearchBoost.DESCRIPTION),
        index.SearchField('content', partial_match=True, boost=SearchBoost.CONTENT),
        index.FilterField('categories'),
        index.SearchField('locations', partial_match=True, boost=SearchBoost.LOCATION),
        index.FilterField('language'),
    ]

    def __str__(self):
        return self.title

    def get_context(self, request, *args, **kwargs):
        try:
            site = Site.objects.get(hostname=request.get_host())
        except Site.DoesNotExist:
            site = Site.objects.filter(is_default_site=True)[0]
        return {
            'article': self,
            'request': request,
            'site': site
        }

    def get_absolute_url(self):
        name = "article-detail"
        return reverse(name, kwargs={"slug": self.slug})

    def get_translation(self):
        return get_translations_for_page(self)

    def related_articles(self):
        if not self.pk:
            # In preview mode
            return []
        max_results = getattr(settings, "MAX_RELATED_RESULTS", 4)
        es_backend = get_search_backend()
        mapping = ElasticSearchMapping(self.__class__)
        search_fields = []
        for ii in self.search_fields:
            if getattr(ii, "boost", None):
                search_fields.append("{0}^{1}".format(ii.field_name, ii.boost))
            else:
                search_fields.append(ii.field_name)
        query = {
            "query": {
                "filtered": {
                    "filter": {
                        "term": {
                            "live_filter": True
                        }
                    },
                    "query": {
                        "more_like_this": {
                            "docs": [
                                {
                                    "_id": mapping.get_document_id(self),
                                    "_type": mapping.get_document_type()
                                }
                            ],
                            "min_doc_freq": 1,
                            "min_term_freq": 2,
                            "max_query_terms": 500,
                            "min_word_length": 4,
                            "fields": search_fields
                        }
                    }
                }
            }
        }

        try:
            mlt = es_backend.es.search(
                index=es_backend.index_name,
                doc_type=mapping.get_document_type(),
                body=query
            )
        except ConnectionError:
            return []
        # Get pks from results
        pks = [hit['_source']['pk'] for hit in mlt['hits']['hits']][:max_results]

        # Initialise results dictionary
        results = dict((str(pk), None) for pk in pks)

        # Find objects in database and add them to dict
        queryset = self._default_manager.filter(pk__in=pks)
        for obj in queryset:
            results[str(obj.pk)] = obj

        # Return results in order given by ElasticSearch
        return [results[str(pk)] for pk in pks if results[str(pk)]]
<commit_msg>Jayakumar/Lokesh #142 Fix related-articles for articles<commit_after>from __future__ import unicode_literals

from django.db import models
from django.utils.translation import ugettext_lazy as _
from django.utils.functional import cached_property
from django.core import serializers
from django.conf import settings
from django.core.urlresolvers import reverse
from django.utils.encoding import python_2_unicode_compatible
from elasticsearch import ConnectionError

from modelcluster.fields import M2MField

from wagtail.wagtailcore.models import Page, Site
from wagtail.wagtailcore.fields import RichTextField

from wagtail.wagtailadmin.edit_handlers import FieldPanel, \
    MultiFieldPanel, InlinePanel, PageChooserPanel, FieldRowPanel
from wagtail.wagtailimages.edit_handlers import ImageChooserPanel
from wagtail.wagtailsearch import index
from wagtail.wagtailsearch.backends import get_search_backend
from wagtail.wagtailsearch.backends.elasticsearch import ElasticSearchMapping, \
    ElasticSearchResults

from core.edit_handlers import M2MFieldPanel

# Override the url property of the Page model
# to accommodate for child pages
from core.utils import get_translations_for_page, SearchBoost

Page.wg_url = Page.url

@cached_property
def url_property(self):
    instance = self.specific
    if getattr(instance, 'get_absolute_url', None):
        return instance.get_absolute_url()
    return self.wg_url

Page.url = url_property

@python_2_unicode_compatible
class Article(Page):
    authors = M2MField("author.Author", related_name="articles_by_author")
    translators = M2MField("author.Author",
                           related_name="translations_by_author",
                           blank=True)
    strap = models.TextField(blank=True)
    content = RichTextField()
    language = models.CharField(max_length=7, choices=settings.LANGUAGES)
    original_published_date = models.DateField(null=True, blank=True)
    show_day = models.BooleanField(default=True)
    show_month = models.BooleanField(default=True)
    show_year = models.BooleanField(default=True)
    featured_image = models.ForeignKey('core.AffixImage',
                                       null=True, blank=True,
                                       on_delete=models.SET_NULL)
    show_featured_image = models.BooleanField(default=True)

    categories = M2MField("category.Category", related_name="articles_by_category")
    locations = M2MField("location.Location", related_name="articles_by_location", blank=True)
    content_panels = Page.content_panels + [
        FieldPanel('strap'),
        M2MFieldPanel('authors'),
        M2MFieldPanel('translators'),
        FieldPanel('language'),
        MultiFieldPanel(
            [
                FieldPanel('original_published_date'),
                FieldRowPanel(
                    [
                        FieldPanel('show_day',classname="col4"),
                        FieldPanel('show_month',classname="col4"),
                        FieldPanel('show_year',classname="col4")
                    ])
            ],'Date'),
        FieldPanel('content'),
        MultiFieldPanel(
            [
                ImageChooserPanel('featured_image'),
                FieldPanel('show_featured_image'),
            ], 'Cover Image'),
        FieldPanel('categories'),
        M2MFieldPanel('locations'),
    ]

    search_fields = Page.search_fields + [
        index.SearchField('title', partial_match=True, boost=SearchBoost.TITLE),
        index.SearchField('authors', partial_match=True, boost=SearchBoost.AUTHOR),
        index.SearchField('translators', partial_match=True, boost=SearchBoost.AUTHOR),
        index.SearchField('strap', partial_match=True, boost=SearchBoost.DESCRIPTION),
        index.SearchField('content', partial_match=True, boost=SearchBoost.CONTENT),
        index.FilterField('categories'),
        index.SearchField('locations', partial_match=True, boost=SearchBoost.LOCATION),
        index.FilterField('language'),
    ]

    def __str__(self):
        return self.title

    def get_context(self, request, *args, **kwargs):
        try:
            site = Site.objects.get(hostname=request.get_host())
        except Site.DoesNotExist:
            site = Site.objects.filter(is_default_site=True)[0]
        return {
            'article': self,
            'request': request,
            'site': site
        }

    def get_absolute_url(self):
        name = "article-detail"
        return reverse(name, kwargs={"slug": self.slug})

    def get_translation(self):
        return get_translations_for_page(self)

    def related_articles(self):
        if not self.pk:
            # In preview mode
            return []

        lookup_fields = ['authors', 'translators', 'locations']
        higher_boost = 2
        max_results = getattr(settings, "MAX_RELATED_RESULTS", 4)

        es_backend = get_search_backend()
        mapping = ElasticSearchMapping(self.__class__)
        search_fields = []

        for ii in self.search_fields:
            if ii.field_name in lookup_fields:
                search_fields.append("{0}^{1}".format(ii.field_name, higher_boost))
            else:
                search_fields.append(ii.field_name)
        query = {
            "query": {
                "filtered": {
                    "filter": {
                        "term": {
                            "live_filter": True
                        }
                    },
                    "query": {
                        "more_like_this": {
                            "docs": [
                                {
                                    "_id": mapping.get_document_id(self),
                                    "_type": mapping.get_document_type()
                                }
                            ],
                            "min_doc_freq": 1,
                            "min_term_freq": 2,
                            "max_query_terms": 500,
                            "min_word_length": 4,
                            "fields": search_fields
                        }
                    }
                }
            }
        }

        try:
            mlt = es_backend.es.search(
                index=es_backend.index_name,
                doc_type=mapping.get_document_type(),
                body=query
            )
        except ConnectionError:
            return []
        # Get pks from results
        pks = [hit['_source']['pk'] for hit in mlt['hits']['hits']][:max_results]

        # Initialise results dictionary
        results = dict((str(pk), None) for pk in pks)

        # Find objects in database and add them to dict
        queryset = self._default_manager.filter(pk__in=pks)
        for obj in queryset:
            results[str(obj.pk)] = obj

        # Return results in order given by ElasticSearch
        return [results[str(pk)] for pk in pks if results[str(pk)]]
<|endoftext|>


<commit_before>"""
Module for creating trees
"""
from math import pi, log, sqrt
from random import gauss
from Tree.utils import Node

class Tree:
    """The standard tree.

    Attributes:
        length (float): The start length.
        age (int): A counter increasing every time grow() is called by 1.
        nodes (list): A 2d-list holding the grown nodes for every age.
    """
    def __init__(self, pos=(0, 0, 0, -100), scale=0.5, complexity=2, angle=(pi, 0), sigma=(0, 0)):
        """The contructor.

        Args:
            pos (tupel): A tupel, holding the start and end point of the tree. (x1, y1, x2, y2)
            scale (float): Indicating how the branch length develops from age to age.
            complexity (int): Indicating how many new branches/nodes sprout from an older node.
            angle (tuple): Holding the branch and shift angle in radians. e.g.(pi, p1/2)
            sigma (tuple): Holding the branch and angle sigma. e.g.(0.1, 0.2)
        """
        self.pos = pos
        self.length = sqrt((pos[2]-pos[0])**2+(pos[3]-pos[1])**2)
        self.scale = scale
        self.comp = complexity
        self.angle = angle
        self.sigma = sigma

        self.age = 0

        self.nodes = [
            [Node(pos[2:])]
        ]

    def get_rectangle(self):
        """Get the coordinates of the rectangle, in which the tree can be put.

        Returns:
            tupel: (x1, y1, x2, y2)
        """
        min_x = self.pos[0]
        max_x = self.pos[0]
        min_y = self.pos[1]
        max_y = self.pos[1]
        for age in self.nodes:
            for node in age:
                if min_x > node.pos[0]:
                    min_x = node.pos[0]
                if max_x < node.pos[0]:
                    max_x = node.pos[0]
                if min_y > node.pos[1]:
                    min_y = node.pos[1]
                if max_y < node.pos[1]:
                    max_y = node.pos[1]
        return (min_x, min_y, max_x, max_y)

    def get_size(self):
        """Get the size of the tree.

        Returns:
            tupel: (width, height)
        """
        rec = self.get_rectangle()
        return (int(rec[2]-rec[0]), int(rec[3]-rec[1]))

    def get_branch_length(self, age=None):
        """Get the length of a branch.

        This method calculates the length of a branch in specific age.
        The used formula: length * scale^age.

        Args:
            age (int): The age, for which you want to know the branch length.
        Returns:
            float: The length of the branch
        """
        if age is None:
            age = self.age

        return self.length * pow(self.scale, age)

    def get_steps_branch_len(self, length):
        """Get, how much steps will needed for a given branch length.

        Returns:
            float: The age the tree must achieve to reach the given branch length.
        """
        return log(length/self.length, self.scale)

    def get_node_sum(self, age=None):
        """Get sum of all branches in the tree.

        Returns:
            int: The sum of all nodes grown until the age.
        """
        if age is None:
            age = self.age

        return age if self.comp == 1 else int((pow(self.comp, age+1) - 1) / (self.comp - 1))

    def get_node_age_sum(self, age=None):
        """Get the sum of branches grown in an specific age.

        Returns:
            int: The sum of all nodes grown in an age.
        """
        if age is None:
            age = self.age

        return pow(self.comp, age)

    def get_nodes(self):
        """Get the tree nodes as list.

        Returns:
            list: A 2d-list holding the grown nodes coordinates as tupel for every age.
                Example:
                [
                [(10, 40)],
                [(20, 80), (100, 30)],
                [(100, 90), (120, 40), ...],
                ...
                ]
        """
        nodes = []
        for age, level in enumerate(self.nodes):
            nodes.append([])
            for node in level:
                nodes[age].append(node.get_tuple())
        return nodes

    def get_branches(self):
        """Get the tree branches as list.

        Returns:
            list: A 2d-list holding the grown branches coordinates as tupel for every age.
                Example:
                [
                [(10, 40, 90, 30)],
                [(90, 30, 100, 40), (90, 30, 300, 60)],
                [(100, 40, 120, 70), (100, 40, 150, 90), ...],
                ...
                ]
        """
        branches = []
        for age, level in enumerate(self.nodes):
            branches.append([])
            for n, node in enumerate(level):
                if age == 0:
                    p_node = Node(self.pos[:2])
                else:
                    p_node = self._get_node_parent(age-1, n)
                branches[age].append(p_node.get_tuple() + node.get_tuple())

        return branches

    def move(self, delta):
        """Move the tree.

        Args:
            delta (tupel): The adjustment of the position.
        """
        pos = self.pos
        self.pos = (pos[0]+delta[0], pos[1]+delta[1], pos[2]+delta[0], pos[3]+delta[1])

        # Move all nodes
        for age in self.nodes:
            for node in age:
                node.move(delta)

    def move_in_rectangle(self):
        """Move the tree so that the tree fits in the rectangle."""
        rec = self.get_rectangle()
        self.move((-rec[0], -rec[1]))

    def grow(self, times=1):
        """Let the tree grow.

        Args:
            times (integer): Indicate how many times the tree will grow.
        """
        self.nodes.append([])

        for n, node in enumerate(self.nodes[self.age]):
            if self.age == 0:
                p_node = Node(self.pos[:2])
            else:
                p_node = self._get_node_parent(self.age-1, n)
            angle = node.get_node_angle(p_node)
            for i in range(self.comp):
                pos = (self.comp-1) / 2 - i
                tot_angle = self.__get_total_angle(angle, pos)
                length = self.__get_total_length(self.age+1)
                self.nodes[self.age+1].append(node.make_new_node(length, tot_angle))

        self.age += 1

        if times > 1:
            self.grow(times-1)

    def __get_total_angle(self, angle, pos):
        """Get the total angle."""
        tot_angle = angle + self.angle[0] * pos - self.angle[1]
        if self.sigma[1] != 0:
            tot_angle += gauss(0, self.sigma[1]) * pi
        return tot_angle

    def __get_total_length(self, age):
        length = self.get_branch_length(age)
        if self.sigma[0] != 0:
            length *= (1+gauss(0, self.sigma[0]))
        return length

    def _get_node_parent(self, age, pos):
        """Get the parent node of node, whch is located in tree's node list.

        Returns:
            object: The parent node.
        """
        return self.nodes[age][int(pos / self.comp)]
<commit_msg>choose smallest scale as calculation base<commit_after>"""
Module for creating trees
"""
from math import pi, log, sqrt
from random import gauss
from Tree.utils import Node

class Tree:
    """The standard tree.

    Attributes:
        length (float): The start length.
        age (int): A counter increasing every time grow() is called by 1.
        nodes (list): A 2d-list holding the grown nodes for every age.
    """
    def __init__(self, pos=(0, 0, 0, -100), scale=0.5, complexity=2, angle=(pi, 0), sigma=(0, 0)):
        """The contructor.

        Args:
            pos (tupel): A tupel, holding the start and end point of the tree. (x1, y1, x2, y2)
            scale (float): Indicating how the branch length develops from age to age.
            complexity (int): Indicating how many new branches/nodes sprout from an older node.
            angle (tuple): Holding the branch and shift angle in radians. e.g.(pi, p1/2)
            sigma (tuple): Holding the branch and angle sigma. e.g.(0.1, 0.2)
        """
        self.pos = pos
        self.length = sqrt((pos[2]-pos[0])**2+(pos[3]-pos[1])**2)
        self.scale = scale
        self.comp = complexity
        self.angle = angle
        self.sigma = sigma

        self.age = 0

        self.nodes = [
            [Node(pos[2:])]
        ]

    def get_rectangle(self):
        """Get the coordinates of the rectangle, in which the tree can be put.

        Returns:
            tupel: (x1, y1, x2, y2)
        """
        min_x = self.pos[0]
        max_x = self.pos[0]
        min_y = self.pos[1]
        max_y = self.pos[1]
        for age in self.nodes:
            for node in age:
                if min_x > node.pos[0]:
                    min_x = node.pos[0]
                if max_x < node.pos[0]:
                    max_x = node.pos[0]
                if min_y > node.pos[1]:
                    min_y = node.pos[1]
                if max_y < node.pos[1]:
                    max_y = node.pos[1]
        return (min_x, min_y, max_x, max_y)

    def get_size(self):
        """Get the size of the tree.

        Returns:
            tupel: (width, height)
        """
        rec = self.get_rectangle()
        return (int(rec[2]-rec[0]), int(rec[3]-rec[1]))

    def get_branch_length(self, age=None):
        """Get the length of a branch.

        This method calculates the length of a branch in specific age.
        The used formula: length * scale^age.

        Args:
            age (int): The age, for which you want to know the branch length.
        Returns:
            float: The length of the branch
        """
        if age is None:
            age = self.age

        return self.length * pow(min(self.scale), age)

    def get_steps_branch_len(self, length):
        """Get, how much steps will needed for a given branch length.

        Returns:
            float: The age the tree must achieve to reach the given branch length.
        """
        return log(length/self.length, self.scale)

    def get_node_sum(self, age=None):
        """Get sum of all branches in the tree.

        Returns:
            int: The sum of all nodes grown until the age.
        """
        if age is None:
            age = self.age

        return age if self.comp == 1 else int((pow(self.comp, age+1) - 1) / (self.comp - 1))

    def get_node_age_sum(self, age=None):
        """Get the sum of branches grown in an specific age.

        Returns:
            int: The sum of all nodes grown in an age.
        """
        if age is None:
            age = self.age

        return pow(self.comp, age)

    def get_nodes(self):
        """Get the tree nodes as list.

        Returns:
            list: A 2d-list holding the grown nodes coordinates as tupel for every age.
                Example:
                [
                [(10, 40)],
                [(20, 80), (100, 30)],
                [(100, 90), (120, 40), ...],
                ...
                ]
        """
        nodes = []
        for age, level in enumerate(self.nodes):
            nodes.append([])
            for node in level:
                nodes[age].append(node.get_tuple())
        return nodes

    def get_branches(self):
        """Get the tree branches as list.

        Returns:
            list: A 2d-list holding the grown branches coordinates as tupel for every age.
                Example:
                [
                [(10, 40, 90, 30)],
                [(90, 30, 100, 40), (90, 30, 300, 60)],
                [(100, 40, 120, 70), (100, 40, 150, 90), ...],
                ...
                ]
        """
        branches = []
        for age, level in enumerate(self.nodes):
            branches.append([])
            for n, node in enumerate(level):
                if age == 0:
                    p_node = Node(self.pos[:2])
                else:
                    p_node = self._get_node_parent(age-1, n)
                branches[age].append(p_node.get_tuple() + node.get_tuple())

        return branches

    def move(self, delta):
        """Move the tree.

        Args:
            delta (tupel): The adjustment of the position.
        """
        pos = self.pos
        self.pos = (pos[0]+delta[0], pos[1]+delta[1], pos[2]+delta[0], pos[3]+delta[1])

        # Move all nodes
        for age in self.nodes:
            for node in age:
                node.move(delta)

    def move_in_rectangle(self):
        """Move the tree so that the tree fits in the rectangle."""
        rec = self.get_rectangle()
        self.move((-rec[0], -rec[1]))

    def grow(self, times=1):
        """Let the tree grow.

        Args:
            times (integer): Indicate how many times the tree will grow.
        """
        self.nodes.append([])

        for n, node in enumerate(self.nodes[self.age]):
            if self.age == 0:
                p_node = Node(self.pos[:2])
            else:
                p_node = self._get_node_parent(self.age-1, n)
            angle = node.get_node_angle(p_node)
            for i in range(self.comp):
                pos = (self.comp-1) / 2 - i
                tot_angle = self.__get_total_angle(angle, pos)
                length = self.__get_total_length(self.age+1)
                self.nodes[self.age+1].append(node.make_new_node(length, tot_angle))

        self.age += 1

        if times > 1:
            self.grow(times-1)

    def __get_total_angle(self, angle, pos):
        """Get the total angle."""
        tot_angle = angle + self.angle[0] * pos - self.angle[1]
        if self.sigma[1] != 0:
            tot_angle += gauss(0, self.sigma[1]) * pi
        return tot_angle

    def __get_total_length(self, age):
        length = self.get_branch_length(age)
        if self.sigma[0] != 0:
            length *= (1+gauss(0, self.sigma[0]))
        return length

    def _get_node_parent(self, age, pos):
        """Get the parent node of node, whch is located in tree's node list.

        Returns:
            object: The parent node.
        """
        return self.nodes[age][int(pos / self.comp)]
<|endoftext|>