<commit_before>from django.conf import settings as django_settings
from django.contrib import admin
from django.db.models.query import QuerySet
from django.http import HttpResponse, HttpResponseBadRequest, HttpResponseForbidden, HttpResponseNotFound, HttpResponseServerError
from django.utils import simplejson
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext_lazy as _

from feincms import settings

import logging

# ------------------------------------------------------------------------
def django_boolean_icon(field_val, alt_text=None, title=None):
    """
    Return HTML code for a nice representation of true/false.
    """

    # Origin: contrib/admin/templatetags/admin_list.py
    BOOLEAN_MAPPING = { True: 'yes', False: 'no', None: 'unknown' }
    alt_text = alt_text or BOOLEAN_MAPPING[field_val]
    if title is not None:
        title = 'title="%s" ' % title
    else:
        title = ''
    return mark_safe(u'' %
            (django_settings.ADMIN_MEDIA_PREFIX, BOOLEAN_MAPPING[field_val], alt_text, title))

def _build_tree_structure(cls):
    """
    Build an in-memory representation of the item tree, trying to keep
    database accesses down to a minimum. The returned dictionary looks like
    this (as json dump):

        {"6": {"id": 6, "children": [7, 8, 10], "parent": null, "descendants": [7, 12, 13, 8, 10]},
         "7": {"id": 7, "children": [12], "parent": 6, "descendants": [12, 13]},
         "8": {"id": 8, "children": [], "parent": 6, "descendants": []},
         ...

    """
    all_nodes = { }
    def add_as_descendant(n, p):
        all_nodes[n]['descendants'].append(p)

        n_parent_id = all_nodes[n]['parent']

        if n_parent_id:
            add_as_descendant(n_parent_id, p)

    for p_id, parent_id in cls.objects.order_by(cls._meta.tree_id_attr, cls._meta.left_attr).values_list("pk", "%s_id" % cls._meta.parent_attr):

        all_nodes[p_id] = { 'id': p_id, 'children' : [ ], 'descendants' : [ ], 'parent' : parent_id }

        if parent_id:
            all_nodes[parent_id]['children'].append(p_id)
            add_as_descendant(parent_id, p_id)

    return all_nodes

# ------------------------------------------------------------------------
def ajax_editable_boolean_cell(item, attr, text='', override=None):
    """
    Generate a html snippet for showing a boolean value on the admin page.
    Item is an object, attr is the attribute name we should display. Text
    is an optional explanatory text to be included in the output.

    This function will emit code to produce a checkbox input with its state
    corresponding to the item.attr attribute if no override value is passed.
    This input is wired to run a JS ajax updater to toggle the value.

    If override is passed in, ignores the attr attribute and returns a
    static image for the override boolean with no user interaction possible
    (useful for "disabled and you can't change it" situations).
    """
    if text:
        text = ' (%s)' % unicode(text)

    if override is not None:
        a = [ django_boolean_icon(override, text), text ]
    else:
        value = getattr(item, attr)
        a = [
              '<input type="checkbox"',
              value and ' checked="checked"' or '',
              ' onclick="return inplace_toggle_boolean(%d, \'%s\')";' % (item.id, attr),
              ' />',
              text,
            ]

    a.insert(0, '
' % ( attr, item.id ))
    a.append('
')
    #print a
    return unicode(''.join(a))

# ------------------------------------------------------------------------
def ajax_editable_boolean(attr, short_description):
    """
    Convenience function: Assign the return value of this method to a variable
    of your ModelAdmin class and put the variable name into list_display.

    Example:
        class MyTreeEditor(TreeEditor):
            list_display = ('__unicode__', 'active_toggle')

            active_toggle = ajax_editable_boolean('active', _('is active'))
    """
    def _fn(self, item):
        return ajax_editable_boolean_cell(item, attr)
    _fn.allow_tags = True
    _fn.short_description = short_description
    _fn.editable_boolean_field = attr
    return _fn

# ------------------------------------------------------------------------
class TreeEditorQuerySet(QuerySet):
    """
    The TreeEditorQuerySet is a special query set used only in the TreeEditor
    ChangeList page. The only difference to a regular QuerySet is that it
    will enforce:

        (a) The result is ordered in correct tree order so that
            the TreeAdmin works all right.

        (b) It ensures that all ancestors of selected items are included
            in the result set, so the resulting tree display actually
            makes sense.
    """
    def iterator(self):
        qs = self
        # Reaching into the bowels of query sets to find out whether the qs is
        # actually filtered and we need to do the INCLUDE_ANCESTORS dance at all.
        # INCLUDE_ANCESTORS is quite expensive, so don't do it if not needed.
        is_filtered = bool(qs.query.where.children)
        if settings.FEINCMS_TREE_EDITOR_INCLUDE_ANCESTORS and is_filtered:
            include_pages = set()
            # Order by 'rght' will return the tree deepest nodes first;
            # this cuts down the number of queries considerably since all ancestors
            # will already be in include_pages when they are checked, thus not
            # trigger additional queries.
            for p in super(TreeEditorQuerySet, self.order_by('rght')).iterator():
                if p.parent_id and p.parent_id not in include_pages and \
                                   p.id not in include_pages:
                    ancestor_id_list = p.get_ancestors().values_list('id', flat=True)
                    include_pages.update(ancestor_id_list)

            if include_pages:
                qs = qs | self.model._default_manager.filter(id__in=include_pages)

            qs = qs.distinct()

        for obj in super(TreeEditorQuerySet, qs).iterator():
            yield obj

    if settings.FEINCMS_TREE_EDITOR_INCLUDE_ANCESTORS:
        def __getitem__(self, index):
            return self   # Don't even try to slice

        def get(self, *args, **kwargs):
            """
            Quick and dirty hack to fix change_view and delete_view; they use
            self.queryset(request).get(...) to get the object they should work
            with. Our modifications to the queryset when INCLUDE_ANCESTORS is
            enabled make get() fail often with a MultipleObjectsReturned
            exception.
            """
            return self.model._default_manager.get(*args, **kwargs)

# !!!: Hack alert! Patching ChangeList, check whether this still applies post Django 1.1
# If the ChangeList is used by a TreEditor, we always need to order by 'tree_id' and 'lft'.
from django.contrib.admin.views import main
class ChangeList(main.ChangeList):
    def get_query_set(self):
        qs = super(ChangeList, self).get_query_set()
        if isinstance(self.model_admin, TreeEditor):
            return qs.order_by('tree_id', 'lft')
        return qs
main.ChangeList = ChangeList

# ------------------------------------------------------------------------
# MARK: -
# ------------------------------------------------------------------------

class TreeEditor(admin.ModelAdmin):
    if settings.FEINCMS_TREE_EDITOR_INCLUDE_ANCESTORS:
        # Make sure that no pagination is displayed. Slicing is disabled anyway,
        # therefore this value does not have an influence on the queryset
        list_per_page = 999999999

    def __init__(self, *args, **kwargs):
        super(TreeEditor, self).__init__(*args, **kwargs)

        self.list_display = list(self.list_display)

        if 'indented_short_title' not in self.list_display:
            if self.list_display[0] == 'action_checkbox':
                self.list_display[1] = 'indented_short_title'
            else:
                self.list_display[0] = 'indented_short_title'
        self.list_display_links = ('indented_short_title',)

        opts = self.model._meta
        self.change_list_template = [
            'admin/feincms/%s/%s/tree_editor.html' % (opts.app_label, opts.object_name.lower()),
            'admin/feincms/%s/tree_editor.html' % opts.app_label,
            'admin/feincms/tree_editor.html',
            ]

    def indented_short_title(self, item):
        """
        Generate a short title for a page, indent it depending on
        the page's depth in the hierarchy.
        """
        r = '''<input type="hidden" class="medialibrary_file_path" value="%s">  ''' % (
                item.get_absolute_url(),
                item.id, item.id, 14+item.level*18)

#        r += ''
        if hasattr(item, 'short_title'):
            r += item.short_title()
        else:
            r += unicode(item)
#        r += ''
        return mark_safe(r)
    indented_short_title.short_description = _('title')
    indented_short_title.allow_tags = True

    def _collect_editable_booleans(self):
        """
        Collect all fields marked as editable booleans. We do not
        want the user to be able to edit arbitrary fields by crafting
        an AJAX request by hand.
        """
        if hasattr(self, '_ajax_editable_booleans'):
            return

        self._ajax_editable_booleans = {}

        for field in self.list_display:
            # The ajax_editable_boolean return value has to be assigned
            # to the ModelAdmin class
            item = getattr(self.__class__, field, None)
            if not item:
                continue

            attr = getattr(item, 'editable_boolean_field', None)
            if attr:
                def _fn(self, page):
                    return [ ajax_editable_boolean_cell(page, _fn.attr) ]
                _fn.attr = attr
                result_func = getattr(item, 'editable_boolean_result', _fn)
                self._ajax_editable_booleans[attr] = result_func

    def _refresh_changelist_caches(self):
        """
        Refresh information used to show the changelist tree structure such as
        inherited active/inactive states etc.

        XXX: This is somewhat hacky, but since it's an internal method, so be it.
        """

        pass

    def _toggle_boolean(self, request):
        """
        Handle an AJAX toggle_boolean request
        """
        try:
            item_id = int(request.POST.get('item_id', None))
            attr = str(request.POST.get('attr', None))
        except:
            return HttpResponseBadRequest("Malformed request")

        if not request.user.is_staff:
            logging.warning("Denied AJAX request by non-staff %s to toggle boolean %s for page #%s", request.user, attr, item_id)
            return HttpResponseForbidden("You do not have permission to access this page")

        self._collect_editable_booleans()

        if not self._ajax_editable_booleans.has_key(attr):
            return HttpResponseBadRequest("not a valid attribute %s" % attr)

        try:
            obj = self.model._default_manager.get(pk=item_id)
        except self.model.DoesNotExist:
            return HttpResponseNotFound("Object does not exist")

        can_change = False

        if hasattr(obj, "user_can") and obj.user_can(request.user, change_page=True):
            can_change = True
        else:
            can_change = request.user.has_perm("page.change_page")

        if not can_change:
            logging.warning("Denied AJAX request by %s to toggle boolean %s for page %s", request.user, attr, item_id)
            return HttpResponseForbidden("You do not have permission to access this page")

        logging.info("Processing request by %s to toggle %s on %s", request.user, attr, obj)

        try:
            before_data = self._ajax_editable_booleans[attr](self, obj)

            setattr(obj, attr, not getattr(obj, attr))
            obj.save()

            self._refresh_changelist_caches() # ???: Perhaps better a post_save signal?

            # Construct html snippets to send back to client for status update
            data = self._ajax_editable_booleans[attr](self, obj)

        except Exception, e:
            logging.exception("Unhandled exception while toggling %s on %s", attr, obj)
            return HttpResponseServerError("Unable to toggle %s on %s" % (attr, obj))

        # Weed out unchanged cells to keep the updates small. This assumes
        # that the order a possible get_descendents() returns does not change
        # before and after toggling this attribute. Unlikely, but still...
        d = []
        for a, b in zip(before_data, data):
            if a != b:
                d.append(b)

        return HttpResponse(simplejson.dumps(d), mimetype="application/json")

    def changelist_view(self, request, extra_context=None, *args, **kwargs):
        """
        Handle the changelist view, the django view for the model instances
        change list/actions page.
        """

        if 'actions_column' not in self.list_display:
            self.list_display.append('actions_column')

        # handle common AJAX requests
        if request.is_ajax():
            cmd = request.POST.get('__cmd')
            if cmd == 'toggle_boolean':
                return self._toggle_boolean(request)
            elif cmd == 'move_node':
                return self._move_node(request)
            else:
                return HttpResponseBadRequest('Oops. AJAX request not understood.')

        self._refresh_changelist_caches()

        extra_context = extra_context or {}
        extra_context['FEINCMS_ADMIN_MEDIA'] = settings.FEINCMS_ADMIN_MEDIA
        extra_context['FEINCMS_ADMIN_MEDIA_HOTLINKING'] = settings.FEINCMS_ADMIN_MEDIA_HOTLINKING
        extra_context['tree_structure'] = mark_safe(simplejson.dumps(
                                                    _build_tree_structure(self.model)))

        return super(TreeEditor, self).changelist_view(request, extra_context, *args, **kwargs)

    def _move_node(self, request):
        cut_item = self.model._tree_manager.get(pk=request.POST.get('cut_item'))
        pasted_on = self.model._tree_manager.get(pk=request.POST.get('pasted_on'))
        position = request.POST.get('position')

        if position in ('last-child', 'left'):
            self.model._tree_manager.move_node(cut_item, pasted_on, position)

            # Ensure that model save has been run
            source = self.model._tree_manager.get(pk=request.POST.get('cut_item'))
            source.save()

            return HttpResponse('OK')
        return HttpResponse('FAIL')

    def queryset(self, request):
        """
        Returns a QuerySet of all model instances that can be edited by the
        admin site. This is used by changelist_view.
        """
        qs = self.model._default_manager.get_query_set()
        qs.__class__ = TreeEditorQuerySet
        return qs

    def _actions_column(self, page):
        actions = []
        actions.append(u'<big>✂</big>' % (
            page.pk, _('Cut')))

        actions.append(u'↳' % (
            page.pk, _('Insert as child')))
        actions.append(u'↱' % (
            page.pk, _('Insert before')))
        return actions

    def actions_column(self, page):
        return u' '.join(self._actions_column(page))
    actions_column.allow_tags = True
    actions_column.short_description = _('actions')

<commit_msg>make url embedding in change list conditional on the presence of get_absolute_url() in the model<commit_after>from django.conf import settings as django_settings
from django.contrib import admin
from django.db.models.query import QuerySet
from django.http import HttpResponse, HttpResponseBadRequest, HttpResponseForbidden, HttpResponseNotFound, HttpResponseServerError
from django.utils import simplejson
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext_lazy as _

from feincms import settings

import logging

# ------------------------------------------------------------------------
def django_boolean_icon(field_val, alt_text=None, title=None):
    """
    Return HTML code for a nice representation of true/false.
    """

    # Origin: contrib/admin/templatetags/admin_list.py
    BOOLEAN_MAPPING = { True: 'yes', False: 'no', None: 'unknown' }
    alt_text = alt_text or BOOLEAN_MAPPING[field_val]
    if title is not None:
        title = 'title="%s" ' % title
    else:
        title = ''
    return mark_safe(u'' %
            (django_settings.ADMIN_MEDIA_PREFIX, BOOLEAN_MAPPING[field_val], alt_text, title))

def _build_tree_structure(cls):
    """
    Build an in-memory representation of the item tree, trying to keep
    database accesses down to a minimum. The returned dictionary looks like
    this (as json dump):

        {"6": {"id": 6, "children": [7, 8, 10], "parent": null, "descendants": [7, 12, 13, 8, 10]},
         "7": {"id": 7, "children": [12], "parent": 6, "descendants": [12, 13]},
         "8": {"id": 8, "children": [], "parent": 6, "descendants": []},
         ...

    """
    all_nodes = { }
    def add_as_descendant(n, p):
        all_nodes[n]['descendants'].append(p)

        n_parent_id = all_nodes[n]['parent']

        if n_parent_id:
            add_as_descendant(n_parent_id, p)

    for p_id, parent_id in cls.objects.order_by(cls._meta.tree_id_attr, cls._meta.left_attr).values_list("pk", "%s_id" % cls._meta.parent_attr):

        all_nodes[p_id] = { 'id': p_id, 'children' : [ ], 'descendants' : [ ], 'parent' : parent_id }

        if parent_id:
            all_nodes[parent_id]['children'].append(p_id)
            add_as_descendant(parent_id, p_id)

    return all_nodes

# ------------------------------------------------------------------------
def ajax_editable_boolean_cell(item, attr, text='', override=None):
    """
    Generate a html snippet for showing a boolean value on the admin page.
    Item is an object, attr is the attribute name we should display. Text
    is an optional explanatory text to be included in the output.

    This function will emit code to produce a checkbox input with its state
    corresponding to the item.attr attribute if no override value is passed.
    This input is wired to run a JS ajax updater to toggle the value.

    If override is passed in, ignores the attr attribute and returns a
    static image for the override boolean with no user interaction possible
    (useful for "disabled and you can't change it" situations).
    """
    if text:
        text = ' (%s)' % unicode(text)

    if override is not None:
        a = [ django_boolean_icon(override, text), text ]
    else:
        value = getattr(item, attr)
        a = [
              '<input type="checkbox"',
              value and ' checked="checked"' or '',
              ' onclick="return inplace_toggle_boolean(%d, \'%s\')";' % (item.id, attr),
              ' />',
              text,
            ]

    a.insert(0, '
' % ( attr, item.id ))
    a.append('
')
    #print a
    return unicode(''.join(a))

# ------------------------------------------------------------------------
def ajax_editable_boolean(attr, short_description):
    """
    Convenience function: Assign the return value of this method to a variable
    of your ModelAdmin class and put the variable name into list_display.

    Example:
        class MyTreeEditor(TreeEditor):
            list_display = ('__unicode__', 'active_toggle')

            active_toggle = ajax_editable_boolean('active', _('is active'))
    """
    def _fn(self, item):
        return ajax_editable_boolean_cell(item, attr)
    _fn.allow_tags = True
    _fn.short_description = short_description
    _fn.editable_boolean_field = attr
    return _fn

# ------------------------------------------------------------------------
class TreeEditorQuerySet(QuerySet):
    """
    The TreeEditorQuerySet is a special query set used only in the TreeEditor
    ChangeList page. The only difference to a regular QuerySet is that it
    will enforce:

        (a) The result is ordered in correct tree order so that
            the TreeAdmin works all right.

        (b) It ensures that all ancestors of selected items are included
            in the result set, so the resulting tree display actually
            makes sense.
    """
    def iterator(self):
        qs = self
        # Reaching into the bowels of query sets to find out whether the qs is
        # actually filtered and we need to do the INCLUDE_ANCESTORS dance at all.
        # INCLUDE_ANCESTORS is quite expensive, so don't do it if not needed.
        is_filtered = bool(qs.query.where.children)
        if settings.FEINCMS_TREE_EDITOR_INCLUDE_ANCESTORS and is_filtered:
            include_pages = set()
            # Order by 'rght' will return the tree deepest nodes first;
            # this cuts down the number of queries considerably since all ancestors
            # will already be in include_pages when they are checked, thus not
            # trigger additional queries.
            for p in super(TreeEditorQuerySet, self.order_by('rght')).iterator():
                if p.parent_id and p.parent_id not in include_pages and \
                                   p.id not in include_pages:
                    ancestor_id_list = p.get_ancestors().values_list('id', flat=True)
                    include_pages.update(ancestor_id_list)

            if include_pages:
                qs = qs | self.model._default_manager.filter(id__in=include_pages)

            qs = qs.distinct()

        for obj in super(TreeEditorQuerySet, qs).iterator():
            yield obj

    if settings.FEINCMS_TREE_EDITOR_INCLUDE_ANCESTORS:
        def __getitem__(self, index):
            return self   # Don't even try to slice

        def get(self, *args, **kwargs):
            """
            Quick and dirty hack to fix change_view and delete_view; they use
            self.queryset(request).get(...) to get the object they should work
            with. Our modifications to the queryset when INCLUDE_ANCESTORS is
            enabled make get() fail often with a MultipleObjectsReturned
            exception.
            """
            return self.model._default_manager.get(*args, **kwargs)

# !!!: Hack alert! Patching ChangeList, check whether this still applies post Django 1.1
# If the ChangeList is used by a TreEditor, we always need to order by 'tree_id' and 'lft'.
from django.contrib.admin.views import main
class ChangeList(main.ChangeList):
    def get_query_set(self):
        qs = super(ChangeList, self).get_query_set()
        if isinstance(self.model_admin, TreeEditor):
            return qs.order_by('tree_id', 'lft')
        return qs
main.ChangeList = ChangeList

# ------------------------------------------------------------------------
# MARK: -
# ------------------------------------------------------------------------

class TreeEditor(admin.ModelAdmin):
    if settings.FEINCMS_TREE_EDITOR_INCLUDE_ANCESTORS:
        # Make sure that no pagination is displayed. Slicing is disabled anyway,
        # therefore this value does not have an influence on the queryset
        list_per_page = 999999999

    def __init__(self, *args, **kwargs):
        super(TreeEditor, self).__init__(*args, **kwargs)

        self.list_display = list(self.list_display)

        if 'indented_short_title' not in self.list_display:
            if self.list_display[0] == 'action_checkbox':
                self.list_display[1] = 'indented_short_title'
            else:
                self.list_display[0] = 'indented_short_title'
        self.list_display_links = ('indented_short_title',)

        opts = self.model._meta
        self.change_list_template = [
            'admin/feincms/%s/%s/tree_editor.html' % (opts.app_label, opts.object_name.lower()),
            'admin/feincms/%s/tree_editor.html' % opts.app_label,
            'admin/feincms/tree_editor.html',
            ]

    def indented_short_title(self, item):
        """
        Generate a short title for a page, indent it depending on
        the page's depth in the hierarchy.
        """
        if hasattr(item, 'get_absolute_url'):
            r = '''<input type="hidden" class="medialibrary_file_path" value="%s">  ''' % (
                item.get_absolute_url(),
                item.id, item.id, 14+item.level*18)
        else:
            r = '''  ''' % (
                item.id, item.id, 14+item.level*18)

#        r += ''
        if hasattr(item, 'short_title'):
            r += item.short_title()
        else:
            r += unicode(item)
#        r += ''
        return mark_safe(r)
    indented_short_title.short_description = _('title')
    indented_short_title.allow_tags = True

    def _collect_editable_booleans(self):
        """
        Collect all fields marked as editable booleans. We do not
        want the user to be able to edit arbitrary fields by crafting
        an AJAX request by hand.
        """
        if hasattr(self, '_ajax_editable_booleans'):
            return

        self._ajax_editable_booleans = {}

        for field in self.list_display:
            # The ajax_editable_boolean return value has to be assigned
            # to the ModelAdmin class
            item = getattr(self.__class__, field, None)
            if not item:
                continue

            attr = getattr(item, 'editable_boolean_field', None)
            if attr:
                def _fn(self, page):
                    return [ ajax_editable_boolean_cell(page, _fn.attr) ]
                _fn.attr = attr
                result_func = getattr(item, 'editable_boolean_result', _fn)
                self._ajax_editable_booleans[attr] = result_func

    def _refresh_changelist_caches(self):
        """
        Refresh information used to show the changelist tree structure such as
        inherited active/inactive states etc.

        XXX: This is somewhat hacky, but since it's an internal method, so be it.
        """

        pass

    def _toggle_boolean(self, request):
        """
        Handle an AJAX toggle_boolean request
        """
        try:
            item_id = int(request.POST.get('item_id', None))
            attr = str(request.POST.get('attr', None))
        except:
            return HttpResponseBadRequest("Malformed request")

        if not request.user.is_staff:
            logging.warning("Denied AJAX request by non-staff %s to toggle boolean %s for page #%s", request.user, attr, item_id)
            return HttpResponseForbidden("You do not have permission to access this page")

        self._collect_editable_booleans()

        if not self._ajax_editable_booleans.has_key(attr):
            return HttpResponseBadRequest("not a valid attribute %s" % attr)

        try:
            obj = self.model._default_manager.get(pk=item_id)
        except self.model.DoesNotExist:
            return HttpResponseNotFound("Object does not exist")

        can_change = False

        if hasattr(obj, "user_can") and obj.user_can(request.user, change_page=True):
            can_change = True
        else:
            can_change = request.user.has_perm("page.change_page")

        if not can_change:
            logging.warning("Denied AJAX request by %s to toggle boolean %s for page %s", request.user, attr, item_id)
            return HttpResponseForbidden("You do not have permission to access this page")

        logging.info("Processing request by %s to toggle %s on %s", request.user, attr, obj)

        try:
            before_data = self._ajax_editable_booleans[attr](self, obj)

            setattr(obj, attr, not getattr(obj, attr))
            obj.save()

            self._refresh_changelist_caches() # ???: Perhaps better a post_save signal?

            # Construct html snippets to send back to client for status update
            data = self._ajax_editable_booleans[attr](self, obj)

        except Exception, e:
            logging.exception("Unhandled exception while toggling %s on %s", attr, obj)
            return HttpResponseServerError("Unable to toggle %s on %s" % (attr, obj))

        # Weed out unchanged cells to keep the updates small. This assumes
        # that the order a possible get_descendents() returns does not change
        # before and after toggling this attribute. Unlikely, but still...
        d = []
        for a, b in zip(before_data, data):
            if a != b:
                d.append(b)

        return HttpResponse(simplejson.dumps(d), mimetype="application/json")

    def changelist_view(self, request, extra_context=None, *args, **kwargs):
        """
        Handle the changelist view, the django view for the model instances
        change list/actions page.
        """

        if 'actions_column' not in self.list_display:
            self.list_display.append('actions_column')

        # handle common AJAX requests
        if request.is_ajax():
            cmd = request.POST.get('__cmd')
            if cmd == 'toggle_boolean':
                return self._toggle_boolean(request)
            elif cmd == 'move_node':
                return self._move_node(request)
            else:
                return HttpResponseBadRequest('Oops. AJAX request not understood.')

        self._refresh_changelist_caches()

        extra_context = extra_context or {}
        extra_context['FEINCMS_ADMIN_MEDIA'] = settings.FEINCMS_ADMIN_MEDIA
        extra_context['FEINCMS_ADMIN_MEDIA_HOTLINKING'] = settings.FEINCMS_ADMIN_MEDIA_HOTLINKING
        extra_context['tree_structure'] = mark_safe(simplejson.dumps(
                                                    _build_tree_structure(self.model)))

        return super(TreeEditor, self).changelist_view(request, extra_context, *args, **kwargs)

    def _move_node(self, request):
        cut_item = self.model._tree_manager.get(pk=request.POST.get('cut_item'))
        pasted_on = self.model._tree_manager.get(pk=request.POST.get('pasted_on'))
        position = request.POST.get('position')

        if position in ('last-child', 'left'):
            self.model._tree_manager.move_node(cut_item, pasted_on, position)

            # Ensure that model save has been run
            source = self.model._tree_manager.get(pk=request.POST.get('cut_item'))
            source.save()

            return HttpResponse('OK')
        return HttpResponse('FAIL')

    def queryset(self, request):
        """
        Returns a QuerySet of all model instances that can be edited by the
        admin site. This is used by changelist_view.
        """
        qs = self.model._default_manager.get_query_set()
        qs.__class__ = TreeEditorQuerySet
        return qs

    def _actions_column(self, page):
        actions = []
        actions.append(u'<big>✂</big>' % (
            page.pk, _('Cut')))

        actions.append(u'↳' % (
            page.pk, _('Insert as child')))
        actions.append(u'↱' % (
            page.pk, _('Insert before')))
        return actions

    def actions_column(self, page):
        return u' '.join(self._actions_column(page))
    actions_column.allow_tags = True
    actions_column.short_description = _('actions')

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from holmes.validators import BASE_VALIDATORS  # NOQA

__version__ = '0.8.14'
<commit_msg>Releasing 0.9<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from holmes.validators import BASE_VALIDATORS  # NOQA

__version__ = '0.9.0'
<|endoftext|>


<commit_before>"""
tasks.py
~~~~~~~~

Handles running of tasks in an asynchronous fashion. Not explicitly tied to Celery. The `run` method simply must
exist here and handle the execution of whatever task is passed to it, whether or not it is handled asynchronously.
"""
from multiprocessing import Process
from time import sleep
from typing import Callable

from apscheduler.schedulers.background import BackgroundScheduler
from celery import Celery
from celery.security import setup_security
from celery.utils.log import get_task_logger
from raven import Client
from raven.contrib.celery import register_signal, register_logger_signal

from home.settings import ASYNC_MODE, SENTRY_URL, BROKER_PATH, BACKEND_PATH

setup_security(allowed_serializers=['pickle', 'json'],
               serializer='pickle')

queue = Celery('home',
               broker=BROKER_PATH,
               backend=BACKEND_PATH,
               serializer='pickle')

queue.conf.update(
    CELERY_TASK_SERIALIZER='pickle',
    CELERY_ACCEPT_CONTENT=['pickle', 'json'],
    CELERY_TASK_SOFT_TIME_LIMIT=30,
)

try:
    import gevent

    from apscheduler.schedulers.gevent import GeventScheduler

    scheduler = GeventScheduler()
except ImportError:
    scheduler = BackgroundScheduler()
scheduler.start()
logger = get_task_logger(__name__)

@queue.task
def _run(method, **kwargs) -> None:
    """
    Run the configured actions in multiple processes.
    """
    logger.info('Running {} on {} with config: {}'.format(method.__name__,
                                                          method.__self__,
                                                          kwargs
                                                          ))
    method(**kwargs)

def run(method: Callable, delay: float = 0, **kwargs):
    if ASYNC_MODE == 'celery':
        return _run.apply_async(args=[method], kwargs=kwargs, countdown=float(delay))
    else:
        return multiprocessing_run(method, delay, **kwargs)

def multiprocessing_run(target: Callable, delay: float = 0, **kwargs):
    if delay:
        sleep(delay)
    Process(target=target, kwargs=kwargs).start()

if SENTRY_URL:
    client = Client(SENTRY_URL)

    # register a custom filter to filter out duplicate logs
    register_logger_signal(client)

    # The register_logger_signal function can also take an optional argument
    # `loglevel` which is the level used for the handler created.
    # Defaults to `logging.ERROR`
    import logging

    register_logger_signal(client, loglevel=logging.INFO)

    # hook into the Celery error handler
    register_signal(client)

    # The register_signal function can also take an optional argument
    # `ignore_expected` which causes exception classes specified in Task.throws
    # to be ignored
    register_signal(client, ignore_expected=True)
<commit_msg>Fix for new celery security<commit_after>"""
tasks.py
~~~~~~~~

Handles running of tasks in an asynchronous fashion. Not explicitly tied to Celery. The `run` method simply must
exist here and handle the execution of whatever task is passed to it, whether or not it is handled asynchronously.
"""
from multiprocessing import Process
from time import sleep
from typing import Callable

from apscheduler.schedulers.background import BackgroundScheduler
from celery import Celery
from celery.security import setup_security
from celery.utils.log import get_task_logger
from raven import Client
from raven.contrib.celery import register_signal, register_logger_signal

from home.settings import ASYNC_MODE, SENTRY_URL, BROKER_PATH, BACKEND_PATH

setup_security(allowed_serializers=['pickle', 'json'],
               serializer='pickle',
               task_serializer='auth',
               accept_content=['auth'])

queue = Celery('home',
               broker=BROKER_PATH,
               backend=BACKEND_PATH,
               serializer='pickle')

queue.conf.update(
    CELERY_TASK_SERIALIZER='pickle',
    CELERY_ACCEPT_CONTENT=['pickle', 'json'],
    CELERY_TASK_SOFT_TIME_LIMIT=30,
)

try:
    import gevent

    from apscheduler.schedulers.gevent import GeventScheduler

    scheduler = GeventScheduler()
except ImportError:
    scheduler = BackgroundScheduler()
scheduler.start()
logger = get_task_logger(__name__)

@queue.task
def _run(method, **kwargs) -> None:
    """
    Run the configured actions in multiple processes.
    """
    logger.info('Running {} on {} with config: {}'.format(method.__name__,
                                                          method.__self__,
                                                          kwargs
                                                          ))
    method(**kwargs)

def run(method: Callable, delay: float = 0, **kwargs):
    if ASYNC_MODE == 'celery':
        return _run.apply_async(args=[method], kwargs=kwargs, countdown=float(delay))
    else:
        return multiprocessing_run(method, delay, **kwargs)

def multiprocessing_run(target: Callable, delay: float = 0, **kwargs):
    if delay:
        sleep(delay)
    Process(target=target, kwargs=kwargs).start()

if SENTRY_URL:
    client = Client(SENTRY_URL)

    # register a custom filter to filter out duplicate logs
    register_logger_signal(client)

    # The register_logger_signal function can also take an optional argument
    # `loglevel` which is the level used for the handler created.
    # Defaults to `logging.ERROR`
    import logging

    register_logger_signal(client, loglevel=logging.INFO)

    # hook into the Celery error handler
    register_signal(client)

    # The register_signal function can also take an optional argument
    # `ignore_expected` which causes exception classes specified in Task.throws
    # to be ignored
    register_signal(client, ignore_expected=True)
<|endoftext|>


<commit_before>#-*- coding: utf-8 -*-
from django.core import urlresolvers
from django.utils.translation import ugettext_lazy as _
from filer.models import mixins
from filer.models.filemodels import File
from filer.models.foldermodels import Folder
from django.conf import settings

class DummyFolder(mixins.IconsMixin):
    file_type = 'DummyFolder'
    name = "Dummy Folder"
    is_root = True
    is_smart_folder = True
    can_have_subfolders = False
    parent = None
    _icon = "plainfolder"

    @property
    def virtual_folders(self):
        return []

    @property
    def children(self):
        return Folder.objects.none()

    @property
    def files(self):
        return File.objects.none()
    parent_url = None

    @property
    def image_files(self):
        return self.files

    @property
    def logical_path(self):
        """
        Gets logical path of the folder in the tree structure.
        Used to generate breadcrumbs
        """
        return []

class UnfiledImages(DummyFolder):
    name = _("unfiled files")
    is_root = True
    _icon = "unfiled_folder"

    def _files(self):
        return File.objects.filter(folder__isnull=True)
    files = property(_files)

    def get_admin_directory_listing_url_path(self):
        return urlresolvers.reverse(
                            'admin:filer-directory_listing-unfiled_images')

class ImagesWithMissingData(DummyFolder):
    name = _("files with missing metadata")
    is_root = True
    _icon = "incomplete_metadata_folder"

    @property
    def files(self):
        return File.objects.filter(has_all_mandatory_data=False)

    def get_admin_directory_listing_url_path(self):
        return urlresolvers.reverse(
                    'admin:filer-directory_listing-images_with_missing_data')

class FolderRoot(DummyFolder):
    name = _('root')
    is_root = True
    is_smart_folder = False
    can_have_subfolders = True

    @property
    def virtual_folders(self):
        return [UnfiledImages()]

    @property
    def children(self):
        if settings.FILER_ENABLE_PERMISSIONS:
            return Folder.objects.all()
        return Folder.objects.filter(parent__isnull=True)
    parent_url = None

    def contains_folder(self, folder_name):
        try:
            self.children.get(name=folder_name)
            return True
        except Folder.DoesNotExist:
            return False

    def get_admin_directory_listing_url_path(self):
        return urlresolvers.reverse('admin:filer-directory_listing-root')
<commit_msg>settings fix<commit_after>#-*- coding: utf-8 -*-
from django.core import urlresolvers
from django.utils.translation import ugettext_lazy as _
from filer.models import mixins
from filer.models.filemodels import File
from filer.models.foldermodels import Folder
from filer import settings as filer_settings

class DummyFolder(mixins.IconsMixin):
    file_type = 'DummyFolder'
    name = "Dummy Folder"
    is_root = True
    is_smart_folder = True
    can_have_subfolders = False
    parent = None
    _icon = "plainfolder"

    @property
    def virtual_folders(self):
        return []

    @property
    def children(self):
        return Folder.objects.none()

    @property
    def files(self):
        return File.objects.none()
    parent_url = None

    @property
    def image_files(self):
        return self.files

    @property
    def logical_path(self):
        """
        Gets logical path of the folder in the tree structure.
        Used to generate breadcrumbs
        """
        return []

class UnfiledImages(DummyFolder):
    name = _("unfiled files")
    is_root = True
    _icon = "unfiled_folder"

    def _files(self):
        return File.objects.filter(folder__isnull=True)
    files = property(_files)

    def get_admin_directory_listing_url_path(self):
        return urlresolvers.reverse(
                            'admin:filer-directory_listing-unfiled_images')

class ImagesWithMissingData(DummyFolder):
    name = _("files with missing metadata")
    is_root = True
    _icon = "incomplete_metadata_folder"

    @property
    def files(self):
        return File.objects.filter(has_all_mandatory_data=False)

    def get_admin_directory_listing_url_path(self):
        return urlresolvers.reverse(
                    'admin:filer-directory_listing-images_with_missing_data')

class FolderRoot(DummyFolder):
    name = _('root')
    is_root = True
    is_smart_folder = False
    can_have_subfolders = True

    @property
    def virtual_folders(self):
        return [UnfiledImages()]

    @property
    def children(self):
        if filer_settings.FILER_ENABLE_PERMISSIONS:
            return Folder.objects.all()
        return Folder.objects.filter(parent__isnull=True)
    parent_url = None

    def contains_folder(self, folder_name):
        try:
            self.children.get(name=folder_name)
            return True
        except Folder.DoesNotExist:
            return False

    def get_admin_directory_listing_url_path(self):
        return urlresolvers.reverse('admin:filer-directory_listing-root')
<|endoftext|>


<commit_before>import datetime

from . import AbstractCamera

from ..utils.logger import has_logger
from ..utils import error

@has_logger
class Camera(AbstractCamera):

    def __init__(self, config):
        config['driver'] = 'indi_gphoto_ccd'
        super().__init__(config)

        self.config['init_commands'] = {
            "artist": {"artist": "Project PANOPTES"},
            "autoexposuremode": {"autoexposuremode4": "On"},
            "autofocusdrive": {"autofocusdrive0": "Off", "autofocusdrive1": "On"},
            "autopoweroff": {"autopoweroff": "0"},
            "CAPTURE_FORMAT": {"FORMAT9": "On"},
            "capturetarget": {"capturetarget1": "On"},
            "CCD_COMPRESSION": {"CCD_RAW": "On"},
            "CCD_INFO": {"CCD_PIXEL_SIZE": "4.3", "CCD_PIXEL_SIZE_X": "4.3", "CCD_PIXEL_SIZE_Y": "4.3"},
            "CCD_ISO": {"ISO1": "On"},
            "continuousaf": {"continuousaf0": "Off", "continuousaf1": "On"},
            "copyright": {"copyright": "Project PANOPTES All Rights Reserved"},
            "imageformatcf": {"imageformatcf9": "On"},
            "imageformatsd": {"imageformatsd9": "On"},
            "ownername": {"ownername": "Project PANOPTES"},
            "picturestyle": {"picturestyle1": "On", },
            "reviewtime": {"reviewtime0": "On", },
            'Transfer Format': {'FITS': 'On', 'Native': 'Off'},
            "UPLOAD_MODE": {"UPLOAD_BOTH": "On"},
            "UPLOAD_SETTINGS": {"UPLOAD_DIR": "/var/panoptes/images/", "UPLOAD_PREFIX": "IMAGE_XXX"},
            "viewfinder": {"viewfinder0": "Off", "viewfinder1": "On"},
            "WCS_CONTROL": {"WCS_ENABLE": "Off"},
            # 'WCS_CONTROL': {'WCS_ENABLE': 'On'},
        }

        try:
            self.connect()
        except error.InvalidCommand:
            self.logger.warning(
                "Problem connecting to {}, camera unavailable. You should probably try to fix this.".format(self.name))
        else:
            self.logger.info("{} connected".format(self.name))

        self.last_start_time = None

    def construct_filename(self):
        '''
        Use the filename_pattern from the camera config file to construct the
        filename for an image from this camera
        '''
        if self.last_start_time:
            filename = self.last_start_time.strftime('{}_%Y%m%dat%H%M%S.cr2'.format(self.name))
        else:
            filename = self.last_start_time.strftime('image.cr2')
        return filename

    def take_exposure(self, exptime=5):
        """ Take an exposure """
        self.logger.info("<<<<<<<< Exposure >>>>>>>>>")
        self.logger.info('Taking {} second exposure'.format(exptime))

        try:
            output = self.set_property('CCD_EXPOSURE', {'CCD_EXPOSURE_VALUE': '{:.03f}'.format(exptime)})
            self.logger.info("Output from exposure: {}".format(output))
            self.last_start_time = datetime.datetime.now()
        except Exception as e:
            raise error.PanError(e)
<commit_msg>Try WCS again<commit_after>import datetime

from . import AbstractCamera

from ..utils.logger import has_logger
from ..utils import error

@has_logger
class Camera(AbstractCamera):

    def __init__(self, config):
        config['driver'] = 'indi_gphoto_ccd'
        super().__init__(config)

        self.config['init_commands'] = {
            "artist": {"artist": "Project PANOPTES"},
            "autoexposuremode": {"autoexposuremode4": "On"},
            "autofocusdrive": {"autofocusdrive0": "Off", "autofocusdrive1": "On"},
            "autopoweroff": {"autopoweroff": "0"},
            "CAPTURE_FORMAT": {"FORMAT9": "On"},
            "capturetarget": {"capturetarget1": "On"},
            "CCD_COMPRESSION": {"CCD_RAW": "On"},
            "CCD_INFO": {"CCD_PIXEL_SIZE": "4.3", "CCD_PIXEL_SIZE_X": "4.3", "CCD_PIXEL_SIZE_Y": "4.3"},
            "CCD_ISO": {"ISO1": "On"},
            "continuousaf": {"continuousaf0": "Off", "continuousaf1": "On"},
            "copyright": {"copyright": "Project PANOPTES All Rights Reserved"},
            "imageformatcf": {"imageformatcf9": "On"},
            "imageformatsd": {"imageformatsd9": "On"},
            "ownername": {"ownername": "Project PANOPTES"},
            "picturestyle": {"picturestyle1": "On", },
            "reviewtime": {"reviewtime0": "On", },
            'Transfer Format': {'FITS': 'On', 'Native': 'Off'},
            "UPLOAD_MODE": {"UPLOAD_BOTH": "On"},
            "UPLOAD_SETTINGS": {"UPLOAD_DIR": "/var/panoptes/images/", "UPLOAD_PREFIX": "{}_XXX".format(self.name)},
            "viewfinder": {"viewfinder0": "Off", "viewfinder1": "On"},
            # "WCS_CONTROL": {"WCS_ENABLE": "Off"},
            'WCS_CONTROL': {'WCS_ENABLE': 'On'},
        }

        try:
            self.connect()
        except error.InvalidCommand:
            self.logger.warning(
                "Problem connecting to {}, camera unavailable. You should probably try to fix this.".format(self.name))
        else:
            self.logger.info("{} connected".format(self.name))

        self.last_start_time = None

    def construct_filename(self):
        '''
        Use the filename_pattern from the camera config file to construct the
        filename for an image from this camera
        '''
        if self.last_start_time:
            filename = self.last_start_time.strftime('{}_%Y%m%dat%H%M%S.cr2'.format(self.name))
        else:
            filename = self.last_start_time.strftime('image.cr2')
        return filename

    def take_exposure(self, exptime=5):
        """ Take an exposure """
        self.logger.info("<<<<<<<< Exposure >>>>>>>>>")
        self.logger.info('Taking {} second exposure'.format(exptime))

        try:
            output = self.set_property('CCD_EXPOSURE', {'CCD_EXPOSURE_VALUE': '{:.03f}'.format(exptime)})
            self.logger.info("Output from exposure: {}".format(output))
            self.last_start_time = datetime.datetime.now()
        except Exception as e:
            raise error.PanError(e)
<|endoftext|>


<commit_before>
'''
Utilities for tests.
'''

import numpy as np
import astropy.units as u
from astropy.io import fits
from astropy.utils import NumpyRNGContext
from astropy.extern.six.moves import zip

from ..spectral_cube import SpectralCube

def generate_header(pixel_scale, spec_scale, beamfwhm, imshape, v0):

    header = {'CDELT1': -(pixel_scale).to(u.deg).value,
              'CDELT2': (pixel_scale).to(u.deg).value,
              'BMAJ': beamfwhm.to(u.deg).value,
              'BMIN': beamfwhm.to(u.deg).value,
              'BPA': 0.0,
              'CRPIX1': imshape[0] / 2.,
              'CRPIX2': imshape[1] / 2.,
              'CRVAL1': 0.0,
              'CRVAL2': 0.0,
              'CTYPE1': 'GLON-CAR',
              'CTYPE2': 'GLAT-CAR',
              'CUNIT1': 'deg',
              'CUNIT2': 'deg',
              'CRVAL3': v0,
              'CUNIT3': spec_scale.unit.to_string(),
              'CDELT3': spec_scale.value,
              'CRPIX3': 1,
              'CTYPE3': 'VRAD',
              'BUNIT': 'K',
              }

    return fits.Header(header)

def generate_hdu(data, pixel_scale, spec_scale, beamfwhm, v0):

    imshape = data.shape[1:]

    header = generate_header(pixel_scale, spec_scale, beamfwhm, imshape, v0)

    return fits.PrimaryHDU(data, header)

def gaussian(x, amp, mean, sigma):
    return amp * np.exp(- (x - mean)**2 / (2 * sigma**2))

def generate_gaussian_cube(shape=(100, 25, 25), sigma=8., amp=1.,
                           noise=None, spec_scale=1 * u.km / u.s,
                           pixel_scale=1 * u.arcsec,
                           beamfwhm=3 * u.arcsec,
                           v0=None,
                           vel_surface=None,
                           seed=247825498):
    '''
    Generate a SpectralCube with Gaussian profiles.

    The peak velocity positions can be given with `vel_surface`. Otherwise,
    the peaks of the profiles are randomly assigned positions in the cubes.
    This is primarily to test shuffling and stacking of spectra, rather than
    trying to be being physically meaningful.

    Returns
    -------
    spec_cube : SpectralCube
        The generated cube.
    mean_positions : array
        The peak positions in the cube.
    '''

    test_cube = np.empty(shape)
    mean_positions = np.empty(shape[1:])

    spec_middle = int(shape[0] / 2)
    spec_quarter = int(shape[0] / 4)

    if v0 is None:
        v0 = 0

    with NumpyRNGContext(seed):

        spec_inds = np.mgrid[-spec_middle:spec_middle] * spec_scale.value
        spat_inds = np.indices(shape[1:])
        for y, x in zip(spat_inds[0].flatten(), spat_inds[1].flatten()):
            # Lock the mean to within 25% from the centre
            if vel_surface is not None:
                mean_pos = vel_surface[y,x]
            else:
                mean_pos = \
                    np.random.uniform(low=spec_inds[spec_quarter],
                                      high=spec_inds[spec_quarter + spec_middle])
            test_cube[:, y, x] = gaussian(spec_inds, amp, mean_pos, sigma)
            mean_positions[y, x] = mean_pos + v0
            if noise is not None:
                test_cube[:, y, x] += np.random.normal(0, noise, shape[0])

    test_hdu = generate_hdu(test_cube, pixel_scale, spec_scale, beamfwhm,
                            spec_inds[0] + v0)

    spec_cube = SpectralCube.read(test_hdu)

    mean_positions = mean_positions * spec_scale.unit

    return spec_cube, mean_positions
<commit_msg>fix test generation for one-length spectral axis<commit_after>
'''
Utilities for tests.
'''

import numpy as np
import astropy.units as u
from astropy.io import fits
from astropy.utils import NumpyRNGContext
from astropy.extern.six.moves import zip

from ..spectral_cube import SpectralCube

def generate_header(pixel_scale, spec_scale, beamfwhm, imshape, v0):

    header = {'CDELT1': -(pixel_scale).to(u.deg).value,
              'CDELT2': (pixel_scale).to(u.deg).value,
              'BMAJ': beamfwhm.to(u.deg).value,
              'BMIN': beamfwhm.to(u.deg).value,
              'BPA': 0.0,
              'CRPIX1': imshape[0] / 2.,
              'CRPIX2': imshape[1] / 2.,
              'CRVAL1': 0.0,
              'CRVAL2': 0.0,
              'CTYPE1': 'GLON-CAR',
              'CTYPE2': 'GLAT-CAR',
              'CUNIT1': 'deg',
              'CUNIT2': 'deg',
              'CRVAL3': v0,
              'CUNIT3': spec_scale.unit.to_string(),
              'CDELT3': spec_scale.value,
              'CRPIX3': 1,
              'CTYPE3': 'VRAD',
              'BUNIT': 'K',
              }

    return fits.Header(header)

def generate_hdu(data, pixel_scale, spec_scale, beamfwhm, v0):

    imshape = data.shape[1:]

    header = generate_header(pixel_scale, spec_scale, beamfwhm, imshape, v0)

    return fits.PrimaryHDU(data, header)

def gaussian(x, amp, mean, sigma):
    return amp * np.exp(- (x - mean)**2 / (2 * sigma**2))

def generate_gaussian_cube(shape=(100, 25, 25), sigma=8., amp=1.,
                           noise=None, spec_scale=1 * u.km / u.s,
                           pixel_scale=1 * u.arcsec,
                           beamfwhm=3 * u.arcsec,
                           v0=None,
                           vel_surface=None,
                           seed=247825498):
    '''
    Generate a SpectralCube with Gaussian profiles.

    The peak velocity positions can be given with `vel_surface`. Otherwise,
    the peaks of the profiles are randomly assigned positions in the cubes.
    This is primarily to test shuffling and stacking of spectra, rather than
    trying to be being physically meaningful.

    Returns
    -------
    spec_cube : SpectralCube
        The generated cube.
    mean_positions : array
        The peak positions in the cube.
    '''

    test_cube = np.empty(shape)
    mean_positions = np.empty(shape[1:])

    spec_middle = int(shape[0] / 2)
    spec_quarter = int(shape[0] / 4)

    if v0 is None:
        v0 = 0

    with NumpyRNGContext(seed):

        spec_inds = np.mgrid[-spec_middle:spec_middle] * spec_scale.value
        if len(spec_inds) == 0:
            spec_inds = np.array([0])
        spat_inds = np.indices(shape[1:])
        for y, x in zip(spat_inds[0].flatten(), spat_inds[1].flatten()):
            # Lock the mean to within 25% from the centre
            if vel_surface is not None:
                mean_pos = vel_surface[y,x]
            else:
                mean_pos = \
                    np.random.uniform(low=spec_inds[spec_quarter],
                                      high=spec_inds[spec_quarter + spec_middle])
            test_cube[:, y, x] = gaussian(spec_inds, amp, mean_pos, sigma)
            mean_positions[y, x] = mean_pos + v0
            if noise is not None:
                test_cube[:, y, x] += np.random.normal(0, noise, shape[0])

    test_hdu = generate_hdu(test_cube, pixel_scale, spec_scale, beamfwhm,
                            spec_inds[0] + v0)

    spec_cube = SpectralCube.read(test_hdu)

    mean_positions = mean_positions * spec_scale.unit

    return spec_cube, mean_positions
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''
everest_cdpp.py
---------------

Computes the 6-hr CDPP for all the `everest` de-trended light curves.

'''

from __future__ import division, print_function, absolute_import, unicode_literals
import os, sys
import everest
from everest.config import EVEREST_SRC, EVEREST_DAT
from everest.utils import RMS
import kplr
from kplr.config import KPLR_ROOT
import random
import numpy as np
import shutil
import subprocess
import warnings
from urllib.error import HTTPError
from scipy.signal import savgol_filter

run_name = 'default'

for campaign in range(8):

  print("\nRunning campaign %02d..." % campaign)

  # Create file if it doesn't exist
  if not os.path.exists(os.path.join('CDPP', 'everest_C%02d.tsv' % campaign)):
    open(os.path.join('CDPP', 'everest_C%02d.tsv' % campaign), 'a').close()

  # Get all EPIC stars
  stars = list(np.loadtxt(os.path.join(EVEREST_SRC, 'tables', 'C%02d.csv' % campaign), dtype = int))
  nstars = len(stars)

  # Remove ones we've done
  with warnings.catch_warnings():
    warnings.simplefilter("ignore")
    try:
      done, _, _ = np.loadtxt(os.path.join('CDPP', 'everest_C%02d.tsv' % campaign), dtype = float, unpack = True)
    except ValueError:
      done = []
  if len(done):
    done = [int(d) for d in done]
  stars = list(set(stars) - set(done))
  n = len(done) + 1

  # Open the output file (1)
  with open(os.path.join('CDPP', 'everest_C%02d.tsv' % campaign), 'a') as feverest:

    # Open the output file (2)
    with open(os.path.join('CDPP', 'k2raw_C%02d.tsv' % campaign), 'a') as fraw:

      # Loop over all to get the CDPP
      for star in stars:

        # Progress
        sys.stdout.write('\rRunning target %d/%d...' % (n, nstars))
        sys.stdout.flush()
        n += 1

        # Get the cdpp
        try:
          data = np.load(os.path.join(EVEREST_DAT, 'output', 'C%02d' % campaign, '%d' % star, run_name, 'data.npz'))
        except KeyboardInterrupt:
          sys.exit()
        #except:
        #  continue
        rms_raw, rms_raw_savgol, rms_evr, rms_evr_savgol, rms_pht = data['rms']
        satsev = data['satsev'][()]
        crwdsev = data['crwdsev'][()]
        kepmag = data['kepmag'][()]

        print("{:>09d} {:>15.3f} {:>15.3f}".format(star, rms_evr, rms_evr_savgol), file = feverest)
        print("{:>09d} {:>15.3f} {:>15.3f} {:>15.3f} {:>15.3f} {:>01d} {:>01d}".format(star, kepmag, rms_raw, rms_raw_savgol, rms_pht, satsev, crwdsev), file = fraw)<commit_msg>debug<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''
everest_cdpp.py
---------------

Computes the 6-hr CDPP for all the `everest` de-trended light curves.

'''

from __future__ import division, print_function, absolute_import, unicode_literals
import os, sys
import everest
from everest.config import EVEREST_SRC, EVEREST_DAT
from everest.utils import RMS
import kplr
from kplr.config import KPLR_ROOT
import random
import numpy as np
import shutil
import subprocess
import warnings
from urllib.error import HTTPError
from scipy.signal import savgol_filter

run_name = 'default'

for campaign in range(8):

  print("\nRunning campaign %02d..." % campaign)

  # Create file if it doesn't exist
  if not os.path.exists(os.path.join('CDPP', 'everest_C%02d.tsv' % campaign)):
    open(os.path.join('CDPP', 'everest_C%02d.tsv' % campaign), 'a').close()

  # Get all EPIC stars
  stars = list(np.loadtxt(os.path.join(EVEREST_SRC, 'tables', 'C%02d.csv' % campaign), dtype = int))
  nstars = len(stars)

  # Remove ones we've done
  with warnings.catch_warnings():
    warnings.simplefilter("ignore")
    try:
      done, _, _ = np.loadtxt(os.path.join('CDPP', 'everest_C%02d.tsv' % campaign), dtype = float, unpack = True)
    except ValueError:
      done = []
  if len(done):
    done = [int(d) for d in done]
  stars = list(set(stars) - set(done))
  n = len(done) + 1

  # Open the output file (1)
  with open(os.path.join('CDPP', 'everest_C%02d.tsv' % campaign), 'a') as feverest:

    # Open the output file (2)
    with open(os.path.join('CDPP', 'k2raw_C%02d.tsv' % campaign), 'a') as fraw:

      # Loop over all to get the CDPP
      for star in stars:

        # Progress
        sys.stdout.write('\rRunning target %d/%d...' % (n, nstars))
        sys.stdout.flush()
        n += 1

        # Get the cdpp
        try:
          data = np.load(os.path.join(EVEREST_DAT, 'output', 'C%02d' % campaign, '%d' % star, run_name, 'data.npz'))
        except KeyboardInterrupt:
          sys.exit()
        except:
          with open(os.path.join('CDPP', 'everest_error.log' % campaign), 'a') as ferror:
            print(star, file = ferror)
          continue
        rms_raw, rms_raw_savgol, rms_evr, rms_evr_savgol, rms_pht = data['rms']
        satsev = data['satsev'][()]
        crwdsev = data['crwdsev'][()]
        kepmag = data['kepmag'][()]

        print("{:>09d} {:>15.3f} {:>15.3f}".format(star, rms_evr, rms_evr_savgol), file = feverest)
        print("{:>09d} {:>15.3f} {:>15.3f} {:>15.3f} {:>15.3f} {:>01d} {:>01d}".format(star, kepmag, rms_raw, rms_raw_savgol, rms_pht, satsev, crwdsev), file = fraw)<|endoftext|>


<commit_before>import re
import os
import traceback
from types import ClassType, MethodType
from datetime import datetime
from celery import result as celeryresult
from celery.contrib.abortable import AbortableAsyncResult
from celery.task.sets import TaskSet, subtask
from django.conf import settings
from django.contrib.auth.decorators import login_required
from django.core import serializers
from django.core.paginator import Paginator, InvalidPage, EmptyPage
from django.core.serializers.json import DjangoJSONEncoder
from django.db import transaction
from django.db.models import Q
from django.http import HttpResponse, HttpResponseRedirect, Http404
from django.shortcuts import render_to_response, get_object_or_404
from django.template import RequestContext
from django.utils import simplejson
from ocradmin.ocr import tasks
from ocradmin.ocr import utils as ocrutils
from ocradmin.ocrmodels.models import OcrModel
from ocradmin.ocrpresets.models import OcrPreset
from ocradmin.ocrtasks.models import OcrTask, OcrBatch

from ocradmin.ocr.views import _get_best_params

PER_PAGE = 10

def batch_query(params):
    """
    Query the batch db.
    """
    order = [x for x in params.getlist("order_by") if x != ""] or ["created_on"]
    query =  Q()
    for key, val in params.items():
        if key.find("__") == -1 and \
                not key in OcrBatch._meta.get_all_field_names():
            continue
        if not val:
            continue
        query = query & Q(**{str(key): str(val)})
    return OcrBatch.objects.select_related().filter(query).order_by(*order)

@login_required
def new(request):
    """
    Present a new batch form.
    """
    template = "batch/new.html"
    # add available seg and bin presets to the context
    context = {
        "binpresets": OcrPreset.objects.filter(
            type="binarize").order_by("name"),
        "segpresets": OcrPreset.objects.filter(
            type="segment").order_by("name"),
    }
    return render_to_response(template, context,
            context_instance=RequestContext(request))

@login_required
def list(request):
    """
    List recent batches.
    """
    excludes = ["args", "kwargs", "traceback", "results",]
    params = request.GET.copy()
    context = {
        "params" : params,
    }

    paginator = Paginator(batch_query(params), PER_PAGE)
    try:
        page = int(request.GET.get('page', '1'))
    except ValueError:
        page = 1

    try:
        batches = paginator.page(page)
    except (EmptyPage, InvalidPage):
        batches = paginator.page(paginator.num_pages)

    pythonserializer = serializers.get_serializer("python")()
    serializedpage = {}
    serializedpage["num_pages"] = paginator.num_pages
    wanted = ("end_index", "has_next", "has_other_pages", "has_previous",
            "next_page_number", "number", "start_index", "previous_page_number")
    for attr in wanted:
        v = getattr(batches, attr)
        if isinstance(v, MethodType):
            serializedpage[attr] = v()
        elif isinstance(v, (str, int)):
            serializedpage[attr] = v
    # This gets rather gnarly, see:
    # http://code.google.com/p/wadofstuff/wiki/DjangoFullSerializers
    serializedpage["params"] = params
    serializedpage["object_list"] = pythonserializer.serialize(
        batches.object_list,
        extras=( "username", ),
    )

    response = HttpResponse(mimetype="application/json")
    simplejson.dump(serializedpage, response, cls=DjangoJSONEncoder)
    return response

@login_required
@transaction.commit_manually
def create(request):
    """
    Create a batch from pre-saved images convert them with Celery.
    """

    tasktype = "Batch"
    celerytask = tasks.ConvertPageTask

    if not request.method == "POST":
        return render_to_response("batch/new.html", context,
                        context_instance=RequestContext(request))

    filenames = request.POST.get("files", "").split(",")
    dirpath = ocrutils.get_ocr_path(user=request.user.username,
            temp=False, subdir=None, timestamp=False)
    paths = [os.path.join(dirpath, f) for f in filenames]

    if not paths:
        return HttpResponse(
                simplejson.dumps({"error": "no valid images found"}),
                mimetype="application/json")

    # wrangle the params - this needs improving
    userparams = _get_best_params(request.POST.copy())

    # create a batch db job
    batch_name = request.POST.get("batch_name", "%s %s" % (tasktype, datetime.now()))
    batch_desc = request.POST.get("batch_desc", "")
    batch = OcrBatch(user=request.user, name=batch_name, description=batch_desc,
            task_type=celerytask.name, batch_type="MULTI")
    batch.save()
    subtasks = []
    try:
        for path in paths:
            tid = ocrutils.get_new_task_id(path)
            ocrtask = OcrTask(task_id=tid, user=request.user, batch=batch,
                    page_name=os.path.basename(path), status="INIT")
            ocrtask.save()
            subtasks.append(
                    celerytask.subtask(args=(path.encode(), userparams),
                        options=dict(task_id=tid, loglevel=60, retries=2)))
        tasksetresults = TaskSet(tasks=subtasks).apply_async()
    except Exception, e:
        transaction.rollback()
        print e.message
        return HttpResponse(e.message, mimetype="application/json")

    # return a serialized result
    transaction.commit()
    response = HttpResponse(mimetype="application/json")
    simplejson.dump(_serialize_batch(batch), response, cls=DjangoJSONEncoder)
    return response

@login_required
@transaction.commit_manually
def batch(request):
    """
    Save a batch of posted image to the DFS and convert them with Celery.
    """

    # add available seg and bin presets to the context
    context = {
        "binpresets": OcrPreset.objects.filter(
            type="binarize").order_by("name"),
        "segpresets": OcrPreset.objects.filter(
            type="segment").order_by("name"),
    }
    tasktype = "Batch"
    celerytask = tasks.ConvertPageTask

    if not request.method == "POST":
        return render_to_response("batch/batch.html", context,
                        context_instance=RequestContext(request))

    try:
        paths = ocrutils.save_ocr_images(request.FILES.iteritems(),
                temp=True, user=request.user.username,
                name=tasktype.lower())
    except AppException, err:
        return HttpResponse(simplejson.dumps({"error": err.message}),
            mimetype="application/json")
    if not paths:
        return HttpResponse(
                simplejson.dumps({"error": "no valid images found"}),
                mimetype="application/json")

    # wrangle the params - this needs improving
    userparams = _get_best_params(request.POST.copy())

    # create a batch db job
    batch_name = request.POST.get("name", "%s %s" % (tasktype, datetime.now()))
    batch = OcrBatch(user=request.user, name=batch_name,
            task_type=celerytask.name, batch_type="MULTI")
    batch.save()
    subtasks = []
    try:
        for path in paths:
            tid = ocrutils.get_new_task_id(path)
            ocrtask = OcrTask(task_id=tid, user=request.user, batch=batch,
                    page_name=os.path.basename(path), status="INIT")
            ocrtask.save()
            subtasks.append(
                    celerytask.subtask(args=(path.encode(), userparams),
                        options=dict(task_id=tid, loglevel=60, retries=2)))
        tasksetresults = TaskSet(tasks=subtasks).apply_async()
    except Exception, e:
        transaction.rollback()
        print e.message
        return HttpResponse(e.message, mimetype="application/json")

    # return a serialized result
    transaction.commit()
    response = HttpResponse(mimetype="application/json")
    simplejson.dump(_serialize_batch(batch), response, cls=DjangoJSONEncoder)
    return response

@login_required
def results(request, pk):
    """
    Get results for a taskset.
    """
    batch = get_object_or_404(OcrBatch, pk=pk)
    try:
        start = max(0, int(request.GET.get("start", 0)))
    except ValueError:
        start = 0
    try:
        limit = max(1, int(request.GET.get("limit", 25)))
    except ValueError:
        limit = 25
    response = HttpResponse(mimetype="application/json")
    simplejson.dump(_serialize_batch(batch, start, limit),
            response, cls=DjangoJSONEncoder)
    return response

@login_required
def page_results(request, pk, page_index):
    """
    Get the results for a single page.
    """
    batch = get_object_or_404(OcrBatch, pk=pk)
    try:
        page = batch.tasks.all()[int(page_index)]
    except OcrBatch.DoesNotExist, e:
        raise e

    pyserializer = serializers.get_serializer("python")()
    response = HttpResponse(mimetype="application/json")
    taskssl = pyserializer.serialize(
        [page],
        excludes=("results",),
    )
    taskssl[0]["results"] = page.results
    simplejson.dump(taskssl, response, cls=DjangoJSONEncoder)
    return response

@login_required
def submit_viewer_binarization(request, pk):
    """
    Trigger a re-binarization of the image for viewing purposes.
    """
    task = get_object_or_404(OcrTask, pk=pk)
    async = tasks.BinarizePageTask.apply_async(args=task.args)
    out = {
        "job_name": async.task_id,
        "status": async.status,
        "results": async.result,
    }
    return HttpResponse(simplejson.dumps(out),
            mimetype="application/json")

@login_required
def viewer_binarization_results(request, task_id):
    """
    Trigger a re-binarization of the image for viewing purposes.
    """
    async = celeryresult.AsyncResult(task_id)
    out = {
        "job_name": async.task_id,
        "status": async.status,
        "results": async.result,
    }
    return HttpResponse(simplejson.dumps(out),
            mimetype="application/json")

@login_required
def latest(request):
    """
    View the latest batch.
    """
    try:
        batch = OcrBatch.objects.filter(user=request.user)\
                .order_by("-created_on")[0]
    except OcrBatch.DoesNotExist:
        raise Http404

    return _show_batch(request, batch)

@login_required
def show(request, pk):
    """
    View a batch.
    """
    batch = get_object_or_404(OcrBatch, pk=pk)
    return _show_batch(request, batch)

@login_required
def upload_files(request):
    """
    Upload files to the server for batch-processing.
    """
    try:
        paths = ocrutils.save_ocr_images(request.FILES.iteritems(),
                temp=False, user=request.user.username,
                name=None, timestamp=False)
    except AppException, err:
        return HttpResponse(simplejson.dumps({"error": err.message}),
            mimetype="application/json")
    if not paths:
        return HttpResponse(
                simplejson.dumps({"error": "no valid images found"}),
                mimetype="application/json")
    pathlist = [os.path.basename(p) for p in paths]
    return HttpResponse(simplejson.dumps(pathlist),
            mimetype="application/json")

@login_required
def transcript(request, pk):
    """
    View the transcription of a batch.
    """
    batch = get_object_or_404(OcrBatch, pk=pk)
    template = "batch/transcript.html"
    context = {"batch": batch}

    return render_to_response(template, context,
            context_instance=RequestContext(request))

def _show_batch(request, batch):
    """
    View a (passed-in) batch.
    """
    template = "batch/show.html"
    context = {"batch": batch}

    return render_to_response(template, context,
            context_instance=RequestContext(request))

@login_required
def retry_task(request, pk):
    """
    Retry a batch task.
    """
    task = get_object_or_404(OcrTask, pk=pk)
    _retry_celery_task(task)
    return HttpResponse(simplejson.dumps({"ok": True}),
            mimetype="application/json")

@login_required
def abort_task(request, pk):
    """
    Abort a batch task.
    """
    task = get_object_or_404(OcrTask, pk=pk)
    return HttpResponse(simplejson.dumps({"ok": _abort_celery_task(task)}),
            mimetype="application/json")

@transaction.commit_manually
@login_required
def abort_batch(request, pk):
    """
    Abort an entire batch.
    """
    batch = get_object_or_404(OcrBatch, pk=pk)
    for task in batch.tasks.all():
        _abort_celery_task(task)
    transaction.commit()
    return HttpResponse(simplejson.dumps({"ok": True}),
            mimetype="application/json")

@transaction.commit_manually
@login_required
def retry_batch(request, pk):
    """
    Retry all tasks in a batch.
    """
    batch = get_object_or_404(OcrBatch, pk=pk)
    for task in batch.tasks.all():
        _retry_celery_task(task)
    transaction.commit()
    return HttpResponse(simplejson.dumps({"ok": True}),
            mimetype="application/json")

def _abort_celery_task(task):
    """
    Abort a task.
    """
    asyncres = AbortableAsyncResult(task.task_id)
    print asyncres
    print asyncres.__dict__
    #asyncres.revoke()
    print "STATUS PRE:", asyncres.backend.get_status(task.task_id)
    print asyncres.abort()
    print "STATUS POST:", asyncres.backend.get_status(task.task_id)
    if asyncres.is_aborted():
        task.status = "ABORTED"
        task.progress = 0
        task.save()
    return asyncres.is_aborted()

def _retry_celery_task(task):
    """
    Set a task re-running.
    """
    celerytask = tasks.ConvertPageTask
    celerytask.retry(args=task.args, kwargs=task.kwargs,
                options=dict(task_id=task.task_id, loglevel=60, retries=2),
                countdown=0, throw=False)
    task.status = "RETRY"
    task.progress = 0
    task.save()

def _serialize_batch(batch, start=0, limit=25):
    """
    Hack around the problem of serializing
    an object AND it's child objects.
    """
    pyserializer = serializers.get_serializer("python")()
    batchsl = pyserializer.serialize(
        [batch],
        extras=("estimate_progress", "is_complete", "task_count",),
        relations={
            "user":  { "fields": ("username") }
        },
    )
    taskssl = pyserializer.serialize(
        batch.tasks.all()[start:start + limit],
        excludes=("args", "kwargs", "traceback", "results",),
    )
    batchsl[0]["fields"]["tasks"] = taskssl
    return batchsl

<commit_msg>Add the results to the 'fields' dictionary, as they'd normally be if I could get serialization to work transparently<commit_after>import re
import os
import traceback
from types import ClassType, MethodType
from datetime import datetime
from celery import result as celeryresult
from celery.contrib.abortable import AbortableAsyncResult
from celery.task.sets import TaskSet, subtask
from django.conf import settings
from django.contrib.auth.decorators import login_required
from django.core import serializers
from django.core.paginator import Paginator, InvalidPage, EmptyPage
from django.core.serializers.json import DjangoJSONEncoder
from django.db import transaction
from django.db.models import Q
from django.http import HttpResponse, HttpResponseRedirect, Http404
from django.shortcuts import render_to_response, get_object_or_404
from django.template import RequestContext
from django.utils import simplejson
from ocradmin.ocr import tasks
from ocradmin.ocr import utils as ocrutils
from ocradmin.ocrmodels.models import OcrModel
from ocradmin.ocrpresets.models import OcrPreset
from ocradmin.ocrtasks.models import OcrTask, OcrBatch

from ocradmin.ocr.views import _get_best_params

PER_PAGE = 10

def batch_query(params):
    """
    Query the batch db.
    """
    order = [x for x in params.getlist("order_by") if x != ""] or ["created_on"]
    query =  Q()
    for key, val in params.items():
        if key.find("__") == -1 and \
                not key in OcrBatch._meta.get_all_field_names():
            continue
        if not val:
            continue
        query = query & Q(**{str(key): str(val)})
    return OcrBatch.objects.select_related().filter(query).order_by(*order)

@login_required
def new(request):
    """
    Present a new batch form.
    """
    template = "batch/new.html"
    # add available seg and bin presets to the context
    context = {
        "binpresets": OcrPreset.objects.filter(
            type="binarize").order_by("name"),
        "segpresets": OcrPreset.objects.filter(
            type="segment").order_by("name"),
    }
    return render_to_response(template, context,
            context_instance=RequestContext(request))

@login_required
def list(request):
    """
    List recent batches.
    """
    excludes = ["args", "kwargs", "traceback", "results",]
    params = request.GET.copy()
    context = {
        "params" : params,
    }

    paginator = Paginator(batch_query(params), PER_PAGE)
    try:
        page = int(request.GET.get('page', '1'))
    except ValueError:
        page = 1

    try:
        batches = paginator.page(page)
    except (EmptyPage, InvalidPage):
        batches = paginator.page(paginator.num_pages)

    pythonserializer = serializers.get_serializer("python")()
    serializedpage = {}
    serializedpage["num_pages"] = paginator.num_pages
    wanted = ("end_index", "has_next", "has_other_pages", "has_previous",
            "next_page_number", "number", "start_index", "previous_page_number")
    for attr in wanted:
        v = getattr(batches, attr)
        if isinstance(v, MethodType):
            serializedpage[attr] = v()
        elif isinstance(v, (str, int)):
            serializedpage[attr] = v
    # This gets rather gnarly, see:
    # http://code.google.com/p/wadofstuff/wiki/DjangoFullSerializers
    serializedpage["params"] = params
    serializedpage["object_list"] = pythonserializer.serialize(
        batches.object_list,
        extras=( "username", ),
    )

    response = HttpResponse(mimetype="application/json")
    simplejson.dump(serializedpage, response, cls=DjangoJSONEncoder)
    return response

@login_required
@transaction.commit_manually
def create(request):
    """
    Create a batch from pre-saved images convert them with Celery.
    """

    tasktype = "Batch"
    celerytask = tasks.ConvertPageTask

    if not request.method == "POST":
        return render_to_response("batch/new.html", context,
                        context_instance=RequestContext(request))

    filenames = request.POST.get("files", "").split(",")
    dirpath = ocrutils.get_ocr_path(user=request.user.username,
            temp=False, subdir=None, timestamp=False)
    paths = [os.path.join(dirpath, f) for f in filenames]

    if not paths:
        return HttpResponse(
                simplejson.dumps({"error": "no valid images found"}),
                mimetype="application/json")

    # wrangle the params - this needs improving
    userparams = _get_best_params(request.POST.copy())

    # create a batch db job
    batch_name = request.POST.get("batch_name", "%s %s" % (tasktype, datetime.now()))
    batch_desc = request.POST.get("batch_desc", "")
    batch = OcrBatch(user=request.user, name=batch_name, description=batch_desc,
            task_type=celerytask.name, batch_type="MULTI")
    batch.save()
    subtasks = []
    try:
        for path in paths:
            tid = ocrutils.get_new_task_id(path)
            ocrtask = OcrTask(task_id=tid, user=request.user, batch=batch,
                    page_name=os.path.basename(path), status="INIT")
            ocrtask.save()
            subtasks.append(
                    celerytask.subtask(args=(path.encode(), userparams),
                        options=dict(task_id=tid, loglevel=60, retries=2)))
        tasksetresults = TaskSet(tasks=subtasks).apply_async()
    except Exception, e:
        transaction.rollback()
        print e.message
        return HttpResponse(e.message, mimetype="application/json")

    # return a serialized result
    transaction.commit()
    response = HttpResponse(mimetype="application/json")
    simplejson.dump(_serialize_batch(batch), response, cls=DjangoJSONEncoder)
    return response

@login_required
@transaction.commit_manually
def batch(request):
    """
    Save a batch of posted image to the DFS and convert them with Celery.
    """

    # add available seg and bin presets to the context
    context = {
        "binpresets": OcrPreset.objects.filter(
            type="binarize").order_by("name"),
        "segpresets": OcrPreset.objects.filter(
            type="segment").order_by("name"),
    }
    tasktype = "Batch"
    celerytask = tasks.ConvertPageTask

    if not request.method == "POST":
        return render_to_response("batch/batch.html", context,
                        context_instance=RequestContext(request))

    try:
        paths = ocrutils.save_ocr_images(request.FILES.iteritems(),
                temp=True, user=request.user.username,
                name=tasktype.lower())
    except AppException, err:
        return HttpResponse(simplejson.dumps({"error": err.message}),
            mimetype="application/json")
    if not paths:
        return HttpResponse(
                simplejson.dumps({"error": "no valid images found"}),
                mimetype="application/json")

    # wrangle the params - this needs improving
    userparams = _get_best_params(request.POST.copy())

    # create a batch db job
    batch_name = request.POST.get("name", "%s %s" % (tasktype, datetime.now()))
    batch = OcrBatch(user=request.user, name=batch_name,
            task_type=celerytask.name, batch_type="MULTI")
    batch.save()
    subtasks = []
    try:
        for path in paths:
            tid = ocrutils.get_new_task_id(path)
            ocrtask = OcrTask(task_id=tid, user=request.user, batch=batch,
                    page_name=os.path.basename(path), status="INIT")
            ocrtask.save()
            subtasks.append(
                    celerytask.subtask(args=(path.encode(), userparams),
                        options=dict(task_id=tid, loglevel=60, retries=2)))
        tasksetresults = TaskSet(tasks=subtasks).apply_async()
    except Exception, e:
        transaction.rollback()
        print e.message
        return HttpResponse(e.message, mimetype="application/json")

    # return a serialized result
    transaction.commit()
    response = HttpResponse(mimetype="application/json")
    simplejson.dump(_serialize_batch(batch), response, cls=DjangoJSONEncoder)
    return response

@login_required
def results(request, pk):
    """
    Get results for a taskset.
    """
    batch = get_object_or_404(OcrBatch, pk=pk)
    try:
        start = max(0, int(request.GET.get("start", 0)))
    except ValueError:
        start = 0
    try:
        limit = max(1, int(request.GET.get("limit", 25)))
    except ValueError:
        limit = 25
    response = HttpResponse(mimetype="application/json")
    simplejson.dump(_serialize_batch(batch, start, limit),
            response, cls=DjangoJSONEncoder)
    return response

@login_required
def page_results(request, pk, page_index):
    """
    Get the results for a single page.
    """
    batch = get_object_or_404(OcrBatch, pk=pk)
    try:
        page = batch.tasks.all()[int(page_index)]
    except OcrBatch.DoesNotExist, e:
        raise e

    pyserializer = serializers.get_serializer("python")()
    response = HttpResponse(mimetype="application/json")
    taskssl = pyserializer.serialize(
        [page],
        excludes=("results",),
    )
    taskssl[0]["fields"]["results"] = page.results
    simplejson.dump(taskssl, response, cls=DjangoJSONEncoder)
    return response

@login_required
def submit_viewer_binarization(request, pk):
    """
    Trigger a re-binarization of the image for viewing purposes.
    """
    task = get_object_or_404(OcrTask, pk=pk)
    async = tasks.BinarizePageTask.apply_async(args=task.args)
    out = {
        "job_name": async.task_id,
        "status": async.status,
        "results": async.result,
    }
    return HttpResponse(simplejson.dumps(out),
            mimetype="application/json")

@login_required
def viewer_binarization_results(request, task_id):
    """
    Trigger a re-binarization of the image for viewing purposes.
    """
    async = celeryresult.AsyncResult(task_id)
    out = {
        "job_name": async.task_id,
        "status": async.status,
        "results": async.result,
    }
    return HttpResponse(simplejson.dumps(out),
            mimetype="application/json")

@login_required
def latest(request):
    """
    View the latest batch.
    """
    try:
        batch = OcrBatch.objects.filter(user=request.user)\
                .order_by("-created_on")[0]
    except OcrBatch.DoesNotExist:
        raise Http404

    return _show_batch(request, batch)

@login_required
def show(request, pk):
    """
    View a batch.
    """
    batch = get_object_or_404(OcrBatch, pk=pk)
    return _show_batch(request, batch)

@login_required
def upload_files(request):
    """
    Upload files to the server for batch-processing.
    """
    try:
        paths = ocrutils.save_ocr_images(request.FILES.iteritems(),
                temp=False, user=request.user.username,
                name=None, timestamp=False)
    except AppException, err:
        return HttpResponse(simplejson.dumps({"error": err.message}),
            mimetype="application/json")
    if not paths:
        return HttpResponse(
                simplejson.dumps({"error": "no valid images found"}),
                mimetype="application/json")
    pathlist = [os.path.basename(p) for p in paths]
    return HttpResponse(simplejson.dumps(pathlist),
            mimetype="application/json")

@login_required
def transcript(request, pk):
    """
    View the transcription of a batch.
    """
    batch = get_object_or_404(OcrBatch, pk=pk)
    template = "batch/transcript.html"
    context = {"batch": batch}

    return render_to_response(template, context,
            context_instance=RequestContext(request))

def _show_batch(request, batch):
    """
    View a (passed-in) batch.
    """
    template = "batch/show.html"
    context = {"batch": batch}

    return render_to_response(template, context,
            context_instance=RequestContext(request))

@login_required
def retry_task(request, pk):
    """
    Retry a batch task.
    """
    task = get_object_or_404(OcrTask, pk=pk)
    _retry_celery_task(task)
    return HttpResponse(simplejson.dumps({"ok": True}),
            mimetype="application/json")

@login_required
def abort_task(request, pk):
    """
    Abort a batch task.
    """
    task = get_object_or_404(OcrTask, pk=pk)
    return HttpResponse(simplejson.dumps({"ok": _abort_celery_task(task)}),
            mimetype="application/json")

@transaction.commit_manually
@login_required
def abort_batch(request, pk):
    """
    Abort an entire batch.
    """
    batch = get_object_or_404(OcrBatch, pk=pk)
    for task in batch.tasks.all():
        _abort_celery_task(task)
    transaction.commit()
    return HttpResponse(simplejson.dumps({"ok": True}),
            mimetype="application/json")

@transaction.commit_manually
@login_required
def retry_batch(request, pk):
    """
    Retry all tasks in a batch.
    """
    batch = get_object_or_404(OcrBatch, pk=pk)
    for task in batch.tasks.all():
        _retry_celery_task(task)
    transaction.commit()
    return HttpResponse(simplejson.dumps({"ok": True}),
            mimetype="application/json")

def _abort_celery_task(task):
    """
    Abort a task.
    """
    asyncres = AbortableAsyncResult(task.task_id)
    print asyncres
    print asyncres.__dict__
    #asyncres.revoke()
    print "STATUS PRE:", asyncres.backend.get_status(task.task_id)
    print asyncres.abort()
    print "STATUS POST:", asyncres.backend.get_status(task.task_id)
    if asyncres.is_aborted():
        task.status = "ABORTED"
        task.progress = 0
        task.save()
    return asyncres.is_aborted()

def _retry_celery_task(task):
    """
    Set a task re-running.
    """
    celerytask = tasks.ConvertPageTask
    celerytask.retry(args=task.args, kwargs=task.kwargs,
                options=dict(task_id=task.task_id, loglevel=60, retries=2),
                countdown=0, throw=False)
    task.status = "RETRY"
    task.progress = 0
    task.save()

def _serialize_batch(batch, start=0, limit=25):
    """
    Hack around the problem of serializing
    an object AND it's child objects.
    """
    pyserializer = serializers.get_serializer("python")()
    batchsl = pyserializer.serialize(
        [batch],
        extras=("estimate_progress", "is_complete", "task_count",),
        relations={
            "user":  { "fields": ("username") }
        },
    )
    taskssl = pyserializer.serialize(
        batch.tasks.all()[start:start + limit],
        excludes=("args", "kwargs", "traceback", "results",),
    )
    batchsl[0]["fields"]["tasks"] = taskssl
    return batchsl

<|endoftext|>


<commit_before>#!/usr/bin/env python3

import os.path
import plistlib
import sqlite3
import time

from subprocess import call
from tempfile import NamedTemporaryFile

# DBFILE = $HOME/.batterylogx.sqlite
DBFILE = os.path.join(os.environ['HOME'], '.batterylogx.sqlite')

def getdata():
    with NamedTemporaryFile() as f:
        call(['system_profiler', '-xml', 'SPPowerDataType'], stdout=f)
        f.seek(0)
        p = plistlib.readPlist(f)
        d = p[0]['_items'][0]
        chargeinfo = d['sppower_battery_charge_info']
        batterymodel = d['sppower_battery_model_info']
        voltage = d['sppower_current_voltage']
        amperage = d['sppower_current_amperage']
        maxcap = chargeinfo['sppower_battery_max_capacity']
        is_charging = chargeinfo['sppower_battery_is_charging']
        capacity = chargeinfo['sppower_battery_current_capacity']
        bserial = batterymodel['sppower_battery_serial_number']
    return {'voltage' : voltage, 'amperage': amperage, 'maxcap' : maxcap,
        'is_charging' : is_charging, 'capacity' : capacity,
        'bserial' : bserial}

def main():
    # Initialize database
    conn = sqlite3.connect(DBFILE, isolation_level=None)
    c = conn.cursor()
    try:
        c.execute('''CREATE TABLE batterylog (
            id SERIAL,
            ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
            bserial TEXT,
            voltage INTEGER,
            amperage INTEGER,
            capacity INTEGER)''')
    except sqlite3.OperationalError:
        pass # table already created

    # Main loop
    while True:
        d = getdata()
        c.execute("""INSERT INTO batterylog
            (bserial, voltage, amperage, capacity) VALUES (?, ?, ?, ?)""",
            [d['bserial'], d['voltage'], d['amperage'], d['capacity']])
        time.sleep(30)

if __name__ == '__main__':
    main()<commit_msg>Add initial optparse and reformat code<commit_after>#!/usr/bin/env python3

import os.path
import plistlib
import sqlite3
import time

from optparse import OptionParser
from subprocess import call
from tempfile import NamedTemporaryFile

# DBFILE = $HOME/.batterylogx.sqlite
DBFILE = os.path.join(os.environ['HOME'], '.batterylogx.sqlite')
# Initialize database
conn = sqlite3.connect(DBFILE, isolation_level=None)
c = conn.cursor()

def getdata():
    with NamedTemporaryFile() as f:
        call(['system_profiler', '-xml', 'SPPowerDataType'], stdout=f)
        f.seek(0)
        p = plistlib.readPlist(f)
        d = p[0]['_items'][0]
        chargeinfo = d['sppower_battery_charge_info']
        batterymodel = d['sppower_battery_model_info']
        voltage = d['sppower_current_voltage']
        amperage = d['sppower_current_amperage']
        maxcap = chargeinfo['sppower_battery_max_capacity']
        is_charging = chargeinfo['sppower_battery_is_charging']
        capacity = chargeinfo['sppower_battery_current_capacity']
        bserial = batterymodel['sppower_battery_serial_number']
    return dict(voltage=voltage, amperage=amperage, maxcap=maxcap,
        is_charging=is_charging, capacity=capacity, bserial=bserial)

def create_table():
    '''Create the logging table if it does not exist'''
    try:
        c.execute('''CREATE TABLE batterylog (
            id SERIAL,
            ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
            bserial TEXT,
            voltage INTEGER,
            amperage INTEGER,
            capacity INTEGER)''')
    except sqlite3.OperationalError:
        pass # table already created

def log_data(verbose=False):
    d = getdata()
    c.execute("""INSERT INTO batterylog
        (bserial, voltage, amperage, capacity) VALUES (?, ?, ?, ?)""",
        [d['bserial'], d['voltage'], d['amperage'], d['capacity']])
    if verbose:
        print("Voltage: %d mV  Amperage %d mA  Capacity %d mAh" % (
            d['voltage'], d['amperage'], d['capacity']))

def main():
    parser = OptionParser()
    parser.add_option("-k", "--kill", dest="kill", action="store_true",
                      default=False, help="Kill a running logger")
    parser.add_option("-d", "--daemon", dest="daemon", action="store_true",
                      default=False, help="Run logger in background")

    (options, args) = parser.parse_args()
    if options.daemon:
        verbose = False
    else:
        verbose = True
        print("Logging to stdout and %s\n" % DBFILE)

    # Create the logging table if it doesn't exist
    create_table()

    # Main loop
    while True:
        log_data(verbose)
        time.sleep(30)

if __name__ == '__main__':
    main()<|endoftext|>


<commit_before>__version__ = '0.3.1'
<commit_msg>Bump version -> v0.4.0<commit_after>__version__ = '0.4.0'
<|endoftext|>