<commit_before>import sys

from sklearn.model_selection import GridSearchCV
from sklearn.svm import SVC

from nalaf.learning.lib.sklsvm import SklSVM
from nalaf.structures.data import Dataset
from loctext.learning.train import read_corpus
from loctext.util import PRO_ID, LOC_ID, ORG_ID, REL_PRO_LOC_ID, repo_path
from loctext.learning.annotators import LocTextDXModelRelationExtractor
from util import *
from loctext.util import *
from sklearn.feature_selection import SelectFromModel
from sklearn.ensemble import RandomForestClassifier
from sklearn.pipeline import Pipeline
from sklearn.svm import LinearSVC

print(__doc__)

sentence_distance = int(sys.argv[1])
use_pred = sys.argv[2].lower() == "true"

annotator, X, y, groups = get_model_and_data(sentence_distance, use_pred)

print(sentence_distance, use_pred)

# ----------------------------------------------------------------------------------------------------

SCORING_NAMES = [
    F05_SCORER,
    'f1',
]

FEATURE_SELECTIONS = [
    ("LinearSVC_C=4.0", SelectFromModel(LinearSVC(C=4.0, penalty="l1", dual=False, random_state=2727, tol=1e-5))),
    ("LinearSVC_C=2.0", SelectFromModel(LinearSVC(C=2.0, penalty="l1", dual=False, random_state=2727, tol=1e-5))),
    ("LinearSVC_C=1.0", SelectFromModel(LinearSVC(C=1.0, penalty="l1", dual=False, random_state=2727, tol=1e-5))),
]

PIPELINE = Pipeline([
    # ("feat_sel", SelectFromModel(LinearSVC(penalty="l1", dual=False, random_state=2727, tol=1e-5))),
    ('classify', SVC(kernel="linear"))
])

SEARCH_SPACE = [
    {
        'classify': [SVC()],
        'classify__kernel': ['rbf'],
        'classify__class_weight': [None, 'balanced', {-1: 1.5}, {-1: 2.0}],
        'classify__C': [2**log2 for log2 in list(range(-3, 10, 1))],
        'classify__gamma': [2**log2 for log2 in list(range(3, -15, -2))],
    },

    {
        'classify': [SVC()],
        'classify__kernel': ['linear'],
        'classify__class_weight': [None, 'balanced', {-1: 1.5}, {-1: 2.0}],
        'classify__C': [2**log2 for log2 in list(range(-3, 10, 1))],
    },

    {
        # see: http://scikit-learn.org/stable/auto_examples/model_selection/randomized_search.html
        'classify': [RandomForestClassifier()],
        'classify__max_features': [None, 'sqrt', 'log2'],
        'classify__max_depth': [None, 3, 5, 10, 20],
        'classify__bootstrap': [True, False],
        'classify__n_jobs': [-1],
        'classify__class_weight': [None, 'balanced', {-1: 2}, {+1: 2}],
    },
]

# ----------------------------------------------------------------------------------------------------

for fsel_name, feature_selection in FEATURE_SELECTIONS:

    X_new = feature_selection.fit_transform(X, y)

    for scoring_name in SCORING_NAMES:
        print()
        print("----------------------------------------------------------------------------------------------------")
        print()
        print("# Tuning hyper-parameters for *** {} *** with {}".format(scoring_name, fsel_name))
        print()

        grid = GridSearchCV(
            estimator=PIPELINE,
            param_grid=SEARCH_SPACE,
            verbose=True,
            cv=my_cv_generator(groups, len(y)),
            scoring=scoring_name,
            refit=False,
            iid=False,
            n_jobs=-1,
        )

        grid.fit(X_new, y)

        print()
        print("Best parameters found for *** {} *** with {}".format(scoring_name, fsel_name))
        print()
        print(grid.best_params_)
        print()
        print("Grid scores:")
        print()

        means = grid.cv_results_['mean_test_score']
        stds = grid.cv_results_['std_test_score']

        desc_sorted_best_indices = sorted(range(len(means)), key=lambda k: (means[k] - stds[k]), reverse=True)

        for index in desc_sorted_best_indices[:20]:
            mean = means[index]
            std = stds[index]
            params = grid.cv_results_['params'][index]

            print("%0.3f (+/-%0.03f) for %r" % (mean, std * 2, params))
            print()

        print()

    print()

    selected_feature_keys = feature_selection.get_support(indices=True)
    fsel_names, _ = print_selected_features(
        selected_feature_keys,
        annotator.pipeline.feature_set,
        file_prefix=("_".join([str(sentence_distance), str(use_pred), fsel_name]))
    )
    print(fsel_names)
<commit_msg>put auto<commit_after>import sys

from sklearn.model_selection import GridSearchCV
from sklearn.svm import SVC

from nalaf.learning.lib.sklsvm import SklSVM
from nalaf.structures.data import Dataset
from loctext.learning.train import read_corpus
from loctext.util import PRO_ID, LOC_ID, ORG_ID, REL_PRO_LOC_ID, repo_path
from loctext.learning.annotators import LocTextDXModelRelationExtractor
from util import *
from loctext.util import *
from sklearn.feature_selection import SelectFromModel
from sklearn.ensemble import RandomForestClassifier
from sklearn.pipeline import Pipeline
from sklearn.svm import LinearSVC

print(__doc__)

sentence_distance = int(sys.argv[1])
use_pred = sys.argv[2].lower() == "true"

annotator, X, y, groups = get_model_and_data(sentence_distance, use_pred)

print(sentence_distance, use_pred)

# ----------------------------------------------------------------------------------------------------

SCORING_NAMES = [
    F05_SCORER,
    'f1',
]

FEATURE_SELECTIONS = [
    ("LinearSVC_C=4.0", SelectFromModel(LinearSVC(C=4.0, penalty="l1", dual=False, random_state=2727, tol=1e-5))),
    ("LinearSVC_C=2.0", SelectFromModel(LinearSVC(C=2.0, penalty="l1", dual=False, random_state=2727, tol=1e-5))),
    ("LinearSVC_C=1.0", SelectFromModel(LinearSVC(C=1.0, penalty="l1", dual=False, random_state=2727, tol=1e-5))),
]

PIPELINE = Pipeline([
    # ("feat_sel", SelectFromModel(LinearSVC(penalty="l1", dual=False, random_state=2727, tol=1e-5))),
    ('classify', SVC(kernel="linear"))
])

SEARCH_SPACE = [
    {
        'classify': [SVC()],
        'classify__kernel': ['rbf'],
        'classify__class_weight': [None, 'balanced', {-1: 1.5}, {-1: 2.0}],
        'classify__C': [2**log2 for log2 in list(range(-3, 10, 1))],
        'classify__gamma': ['auto'] + [2**log2 for log2 in list(range(3, -15, -2))],
    },

    {
        'classify': [SVC()],
        'classify__kernel': ['linear'],
        'classify__class_weight': [None, 'balanced', {-1: 1.5}, {-1: 2.0}],
        'classify__C': [2**log2 for log2 in list(range(-3, 10, 1))],
    },

    {
        # see: http://scikit-learn.org/stable/auto_examples/model_selection/randomized_search.html
        'classify': [RandomForestClassifier()],
        'classify__max_features': [None, 'sqrt', 'log2'],
        'classify__max_depth': [None, 3, 5, 10, 20],
        'classify__bootstrap': [True, False],
        'classify__n_jobs': [-1],
        'classify__class_weight': [None, 'balanced', {-1: 2}, {+1: 2}],
    },
]

# ----------------------------------------------------------------------------------------------------

for fsel_name, feature_selection in FEATURE_SELECTIONS:

    X_new = feature_selection.fit_transform(X, y)

    for scoring_name in SCORING_NAMES:
        print()
        print("----------------------------------------------------------------------------------------------------")
        print()
        print("# Tuning hyper-parameters for *** {} *** with {}".format(scoring_name, fsel_name))
        print()

        grid = GridSearchCV(
            estimator=PIPELINE,
            param_grid=SEARCH_SPACE,
            verbose=True,
            cv=my_cv_generator(groups, len(y)),
            scoring=scoring_name,
            refit=False,
            iid=False,
            n_jobs=-1,
        )

        grid.fit(X_new, y)

        print()
        print("Best parameters found for *** {} *** with {}".format(scoring_name, fsel_name))
        print()
        print(grid.best_params_)
        print()
        print("Grid scores:")
        print()

        means = grid.cv_results_['mean_test_score']
        stds = grid.cv_results_['std_test_score']

        desc_sorted_best_indices = sorted(range(len(means)), key=lambda k: (means[k] - stds[k]), reverse=True)

        for index in desc_sorted_best_indices[:20]:
            mean = means[index]
            std = stds[index]
            params = grid.cv_results_['params'][index]

            print("%0.3f (+/-%0.03f) for %r" % (mean, std * 2, params))
            print()

        print()

    print()

    selected_feature_keys = feature_selection.get_support(indices=True)
    fsel_names, _ = print_selected_features(
        selected_feature_keys,
        annotator.pipeline.feature_set,
        file_prefix=("_".join([str(sentence_distance), str(use_pred), fsel_name]))
    )
    print(fsel_names)
<|endoftext|>


<commit_before>from django.contrib import admin
from django.utils.translation import ugettext_lazy as _
from django.forms import models as modelforms
from django.contrib.contenttypes import generic
from django.contrib.contenttypes.models import ContentType
from django.db.models import Q
from django.forms.util import ValidationError

from ella.ellaadmin import widgets
from ella.core.models import Author, Source, Category, Listing, HitCount, Placement

class PlacementForm(modelforms.ModelForm):
    # create the field here to pass validation
    listings =  modelforms.ModelMultipleChoiceField(Category.objects.all(), label=_('Category'), cache_choices=True, required=False)

    class Meta:
        model = Placement

    def __init__(self, *args, **kwargs):
        initial = []
        if 'initial' in kwargs:
            initial = [ c.pk for c in Category.objects.distinct().filter(listing__placement=kwargs['initial']['id']) ]

        self.base_fields['listings'] = modelforms.ModelMultipleChoiceField(
                Category.objects.all(), label=_('Category'), cache_choices=True, required=False, initial=initial)
        super(PlacementForm, self).__init__(*args, **kwargs)

class PlacementInlineFormset(modelforms.BaseModelFormSet):

    def __init__(self, *args, **kwargs):
        self.can_delete = True
        super(PlacementInlineFormset, self).__init__(*args, **kwargs)

    def save_existing(self, form, instance, commit=True):
        instance = super(PlacementInlineFormset, self).save_existing(form, instance, commit)
        return self.save_listings(form, instance, commit)

    def save_new(self, form, commit=True):
        instance = super(PlacementInlineFormset, self).save_new(form, commit)
        return self.save_listings(form, instance, commit)

    def save_listings(self, form, instance, commit=True):
        list_cats = form.cleaned_data.pop('listings')

        def save_listings():
            listings = dict([ (l.category, l) for l in Listing.objects.filter(placement=instance.pk) ])

            for c in list_cats:
                if not c in listings:
                    # create listing
                    l = Listing(placement=instance, category=c, publish_from=instance.publish_from)
                    l.save()
                else:
                    del listings[c]
            for l in listings.values():
                l.delete()

        if commit:
            save_listings()
        else:
            save_m2m = form.save_m2m
            def save_all():
                save_m2m()
                save_listings()
            form.save_m2m = save_all
        return instance

    def clean (self):
        # no data - nothing to validate
        if not self.is_valid() or not self.cleaned_data or not self.instance or not self.cleaned_data[0]:
            return

        obj = self.instance
        cat = getattr(obj, 'category', None)
        obj_slug = getattr(obj, 'slug', obj.pk)
        target_ct=ContentType.objects.get_for_model(obj)

        main = None
        for d in self.cleaned_data:
            # empty form
            if not d: break

            if cat and cat == cat and cat:
                main = d

            if d['slug'] and d['slug'] != '':
                slug = d['slug']
            else:
                slug = obj_slug

            # try and find conflicting placement
            qset = Placement.objects.filter(
                category=d['category'],
                slug=slug,
                target_ct=target_ct.pk,
                static=d['static']
            )

            if d['static']: # allow placements that do not overlap
                q = Q(publish_to__lt=d['publish_from'])
                if d['publish_to']:
                    q |= Q(publish_from__gt=d['publish_to'])
                qset = qset.exclude(q)

            # check for same date in URL
            if not d['static']:
                qset = qset.filter(
                    publish_from__year=d['publish_from'].year,
                    publish_from__month=d['publish_from'].month,
                    publish_from__day=d['publish_from'].day,
                )

            # exclude current object from search
            if d['id']:
                qset = qset.exclude(pk=d['id'])

            if qset:
                plac = qset[0]
                # raise forms.ValidationError(
                raise ValidationError(
                        _('''There is already a Placement object published in
                        category %(category)s with the same URL referring to %(target)s.
                        Please change the slug or publish date.''') % {
                            'category' : plac.category,
                            'target' : plac.target,
                        }
                )

            '''
            qset = Placement.objects.filter(
                        target_id=obj.pk,
                        target_ct=target_ct,
            )

            if 'id' in d:
                qset = qset.exclude(id=d['id'])

            if qset:
                # raise forms.ValidationError('Chyba')
                raise ValidationError('Chyba')
            '''

        if cat and not main:
            # raise forms.ValidationError(_('If object has a category, it must have a main placement.'))
            raise (_('If object has a category, it must have a main placement.'))

        return

class ListingInlineAdmin(admin.TabularInline):
    model = Listing
    extra = 2
    fieldsets = ((None, {'fields' : ('category','publish_from', 'priority_from', 'priority_to', 'priority_value', 'remove', 'commercial',)}),)

    def formfield_for_dbfield(self, db_field, **kwargs):
        if db_field.name == 'category':
            kwargs['widget'] = widgets.ListingCategoryWidget
        return super(ListingInlineAdmin, self).formfield_for_dbfield(db_field, **kwargs)

class PlacementInlineAdmin(admin.TabularInline):
    model = Placement
    max_num = 1
    '''
    extra = 1
    ct_field = 'target_ct'
    ct_fk_field = 'target_id'
    formset = PlacementInlineFormset
    form = PlacementForm
    fieldsets = ((None, {'fields' : ('category', 'publish_from', 'publish_to', 'slug', 'static', 'listings',)}),)

    def formfield_for_dbfield(self, db_field, **kwargs):
        if db_field.name == 'category':
            kwargs['widget'] = widgets.ListingCategoryWidget
        return super(PlacementInlineOptions, self).formfield_for_dbfield(db_field, **kwargs)
    '''

class HitCountInlineAdmin(admin.TabularInline):
    model = HitCount
    extra = 0

class PlacementAdmin(admin.ModelAdmin):
    pass
    '''
    list_display = ('target_admin', 'category', 'publish_from', 'full_url',)
    list_filter = ('publish_from', 'category', 'target_ct',)
    inlines = (ListingInlineOptions,)
    fieldsets = (
        (_('target'), {'fields': ('target_ct', 'target_id', 'slug', 'category',), 'classes': ('wide',)},),
        (_('time'), {'fields': ('publish_from','publish_to', 'static',), 'classes': ('wide',)},),
    )
    '''

class ListingAdmin(admin.ModelAdmin):
    pass
    '''
    list_display = ('target_admin', 'target_ct', 'publish_from', 'category', 'placement_admin', 'target_hitcounts', 'target_url',)
    list_display_links = ()
    list_filter = ('publish_from', 'category__site', 'category', 'placement__target_ct',)
    raw_id_fields = ('placement',)
    date_hierarchy = 'publish_from'
    '''

class CategoryAdmin(admin.ModelAdmin):
    list_filter = ('site',)
    list_display = ('draw_title', 'tree_path', '__unicode__')
    search_fields = ('title', 'slug',)
    #ordering = ('site', 'tree_path',)
    prepopulated_fields = {'slug': ('title',)}

class HitCountAdmin(admin.ModelAdmin):
    list_display = ('target', 'hits',)
    ordering = ('-hits', '-last_seen',)

class AuthorAdmin(admin.ModelAdmin):
    prepopulated_fields = {'slug': ('name',)}
    search_fields = ('name',)

class SourceAdmin(admin.ModelAdmin):
    list_display = ('name', 'url',)
    search_fields = ('name',)

class PublishableAdmin(admin.ModelAdmin):
    """ Default admin options for all publishables """

    list_display = ('title', 'category',)
    list_filter = ('category', 'authors',)
    search_fields = ('title', 'description', 'slug', 'authors__name', 'authors__slug',) # FIXME: 'tags__tag__name',)
    raw_id_fields = ('photo',)
    prepopulated_fields = {'slug' : ('title',)}
    rich_text_fields = {None: ('description',)}

    suggest_fields = {
        'category': ('tree_path', 'title', 'slug',),
        'authors': ('name', 'slug', 'email',),
        'source': ('name', 'url',),
    }

    inlines = [PlacementInlineAdmin]

admin.site.register(HitCount, HitCountAdmin)
admin.site.register(Category, CategoryAdmin)
admin.site.register(Source, SourceAdmin)
admin.site.register(Author, AuthorAdmin)
admin.site.register(Placement, PlacementAdmin)
admin.site.register(Listing, ListingAdmin)
<commit_msg>Added ListingInlineAdmin into PlacementAdmin<commit_after>from django.contrib import admin
from django.utils.translation import ugettext_lazy as _
from django.forms import models as modelforms
from django.contrib.contenttypes import generic
from django.contrib.contenttypes.models import ContentType
from django.db.models import Q
from django.forms.util import ValidationError

from ella.ellaadmin import widgets
from ella.core.models import Author, Source, Category, Listing, HitCount, Placement

class PlacementForm(modelforms.ModelForm):
    # create the field here to pass validation
    listings =  modelforms.ModelMultipleChoiceField(Category.objects.all(), label=_('Category'), cache_choices=True, required=False)

    class Meta:
        model = Placement

    def __init__(self, *args, **kwargs):
        initial = []
        if 'initial' in kwargs:
            initial = [ c.pk for c in Category.objects.distinct().filter(listing__placement=kwargs['initial']['id']) ]

        self.base_fields['listings'] = modelforms.ModelMultipleChoiceField(
                Category.objects.all(), label=_('Category'), cache_choices=True, required=False, initial=initial)
        super(PlacementForm, self).__init__(*args, **kwargs)

class PlacementInlineFormset(modelforms.BaseModelFormSet):

    def __init__(self, *args, **kwargs):
        self.can_delete = True
        super(PlacementInlineFormset, self).__init__(*args, **kwargs)

    def save_existing(self, form, instance, commit=True):
        instance = super(PlacementInlineFormset, self).save_existing(form, instance, commit)
        return self.save_listings(form, instance, commit)

    def save_new(self, form, commit=True):
        instance = super(PlacementInlineFormset, self).save_new(form, commit)
        return self.save_listings(form, instance, commit)

    def save_listings(self, form, instance, commit=True):
        list_cats = form.cleaned_data.pop('listings')

        def save_listings():
            listings = dict([ (l.category, l) for l in Listing.objects.filter(placement=instance.pk) ])

            for c in list_cats:
                if not c in listings:
                    # create listing
                    l = Listing(placement=instance, category=c, publish_from=instance.publish_from)
                    l.save()
                else:
                    del listings[c]
            for l in listings.values():
                l.delete()

        if commit:
            save_listings()
        else:
            save_m2m = form.save_m2m
            def save_all():
                save_m2m()
                save_listings()
            form.save_m2m = save_all
        return instance

    def clean (self):
        # no data - nothing to validate
        if not self.is_valid() or not self.cleaned_data or not self.instance or not self.cleaned_data[0]:
            return

        obj = self.instance
        cat = getattr(obj, 'category', None)
        obj_slug = getattr(obj, 'slug', obj.pk)
        target_ct=ContentType.objects.get_for_model(obj)

        main = None
        for d in self.cleaned_data:
            # empty form
            if not d: break

            if cat and cat == cat and cat:
                main = d

            if d['slug'] and d['slug'] != '':
                slug = d['slug']
            else:
                slug = obj_slug

            # try and find conflicting placement
            qset = Placement.objects.filter(
                category=d['category'],
                slug=slug,
                target_ct=target_ct.pk,
                static=d['static']
            )

            if d['static']: # allow placements that do not overlap
                q = Q(publish_to__lt=d['publish_from'])
                if d['publish_to']:
                    q |= Q(publish_from__gt=d['publish_to'])
                qset = qset.exclude(q)

            # check for same date in URL
            if not d['static']:
                qset = qset.filter(
                    publish_from__year=d['publish_from'].year,
                    publish_from__month=d['publish_from'].month,
                    publish_from__day=d['publish_from'].day,
                )

            # exclude current object from search
            if d['id']:
                qset = qset.exclude(pk=d['id'])

            if qset:
                plac = qset[0]
                # raise forms.ValidationError(
                raise ValidationError(
                        _('''There is already a Placement object published in
                        category %(category)s with the same URL referring to %(target)s.
                        Please change the slug or publish date.''') % {
                            'category' : plac.category,
                            'target' : plac.target,
                        }
                )

            '''
            qset = Placement.objects.filter(
                        target_id=obj.pk,
                        target_ct=target_ct,
            )

            if 'id' in d:
                qset = qset.exclude(id=d['id'])

            if qset:
                # raise forms.ValidationError('Chyba')
                raise ValidationError('Chyba')
            '''

        if cat and not main:
            # raise forms.ValidationError(_('If object has a category, it must have a main placement.'))
            raise (_('If object has a category, it must have a main placement.'))

        return

class ListingInlineAdmin(admin.TabularInline):
    model = Listing
    extra = 2
    fieldsets = ((None, {'fields' : ('category','publish_from', 'priority_from', 'priority_to', 'priority_value', 'commercial',)}),)

    def formfield_for_dbfield(self, db_field, **kwargs):
        if db_field.name == 'category':
            kwargs['widget'] = widgets.ListingCategoryWidget
        return super(ListingInlineAdmin, self).formfield_for_dbfield(db_field, **kwargs)

class PlacementInlineAdmin(admin.TabularInline):
    model = Placement
    max_num = 1
    formset = PlacementInlineFormset
    form = PlacementForm
    extra = 1
    '''
    extra = 1
    ct_field = 'target_ct'
    ct_fk_field = 'target_id'
    formset = PlacementInlineFormset
    form = PlacementForm
    fieldsets = ((None, {'fields' : ('category', 'publish_from', 'publish_to', 'slug', 'static', 'listings',)}),)

    def formfield_for_dbfield(self, db_field, **kwargs):
        if db_field.name == 'category':
            kwargs['widget'] = widgets.ListingCategoryWidget
        return super(PlacementInlineOptions, self).formfield_for_dbfield(db_field, **kwargs)
    '''

class HitCountInlineAdmin(admin.TabularInline):
    model = HitCount
    extra = 0

class PlacementAdmin(admin.ModelAdmin):
    pass
    '''
    list_display = ('target_admin', 'category', 'publish_from', 'full_url',)
    list_filter = ('publish_from', 'category', 'target_ct',)
    inlines = (ListingInlineOptions,)
    fieldsets = (
        (_('target'), {'fields': ('target_ct', 'target_id', 'slug', 'category',), 'classes': ('wide',)},),
        (_('time'), {'fields': ('publish_from','publish_to', 'static',), 'classes': ('wide',)},),
    )
    '''

class ListingAdmin(admin.ModelAdmin):
    pass
    '''
    list_display = ('target_admin', 'target_ct', 'publish_from', 'category', 'placement_admin', 'target_hitcounts', 'target_url',)
    list_display_links = ()
    list_filter = ('publish_from', 'category__site', 'category', 'placement__target_ct',)
    raw_id_fields = ('placement',)
    date_hierarchy = 'publish_from'
    '''

class CategoryAdmin(admin.ModelAdmin):
    list_filter = ('site',)
    list_display = ('draw_title', 'tree_path', '__unicode__')
    search_fields = ('title', 'slug',)
    #ordering = ('site', 'tree_path',)
    prepopulated_fields = {'slug': ('title',)}

class HitCountAdmin(admin.ModelAdmin):
    list_display = ('target', 'hits',)
    ordering = ('-hits', '-last_seen',)

class AuthorAdmin(admin.ModelAdmin):
    prepopulated_fields = {'slug': ('name',)}
    search_fields = ('name',)

class SourceAdmin(admin.ModelAdmin):
    list_display = ('name', 'url',)
    search_fields = ('name',)

class PublishableAdmin(admin.ModelAdmin):
    """ Default admin options for all publishables """

    list_display = ('title', 'category',)
    list_filter = ('category', 'authors',)
    search_fields = ('title', 'description', 'slug', 'authors__name', 'authors__slug',) # FIXME: 'tags__tag__name',)
    raw_id_fields = ('photo',)
    prepopulated_fields = {'slug' : ('title',)}
    rich_text_fields = {None: ('description',)}

    suggest_fields = {
        'category': ('tree_path', 'title', 'slug',),
        'authors': ('name', 'slug', 'email',),
        'source': ('name', 'url',),
    }

    inlines = [PlacementInlineAdmin]

admin.site.register(HitCount, HitCountAdmin)
admin.site.register(Category, CategoryAdmin)
admin.site.register(Source, SourceAdmin)
admin.site.register(Author, AuthorAdmin)
admin.site.register(Placement, PlacementAdmin)
admin.site.register(Listing, ListingAdmin)
<|endoftext|>


<commit_before>#!/usr/bin/env python3

#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2
# as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
# Authors:
# Caner Candan <caner@candan.fr>, http://caner.candan.fr
#

import sys, subprocess
import argparse, logging
from pprint import pprint

logger = logging.getLogger("mpirun")

def main():
    parser = argparse.ArgumentParser(description='Run mpi program with threads sequentialized.')

    parser.add_argument('program', nargs='+', help='the program arguments')
    parser.add_argument('-np', help='number of nodes used', type=int, required=True)

    args = parser.parse_args()

    # pprint(args)

    cmd = 'mpirun %s' % ' : '.join(["-np 1 hwloc-bind pu:%d %s" % (i, ' '.join(args.program)) for i in range(args.np)])

    # print(cmd.split())

    subprocess.call(cmd.split())

# when executed, just run main():
if __name__ == '__main__':
    logger.debug('### script started ###')
    main()
    logger.debug('### script ended ###')
<commit_msg>* ../../../scripts/mpirunWithSequentialThreads.py: added an exception capture if the user interrupts the program<commit_after>#!/usr/bin/env python3

#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2
# as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
# Authors:
# Caner Candan <caner@candan.fr>, http://caner.candan.fr
#

import sys, subprocess
import argparse, logging
from pprint import pprint

logger = logging.getLogger("mpirun")

def main():
    parser = argparse.ArgumentParser(description='Run mpi program with threads sequentialized.')

    parser.add_argument('program', nargs='+', help='the program arguments')
    parser.add_argument('-np', help='number of nodes used', type=int, required=True)

    args = parser.parse_args()

    # pprint(args)

    cmd = 'mpirun %s' % ' : '.join(["-np 1 hwloc-bind pu:%d %s" % (i, ' '.join(args.program)) for i in range(args.np)])

    # print(cmd.split())

    try:
        subprocess.call(cmd.split())
    except KeyboardInterrupt:
        print("Interrupted by user.")

# when executed, just run main():
if __name__ == '__main__':
    logger.debug('### script started ###')
    main()
    logger.debug('### script ended ###')
<|endoftext|>


<commit_before><commit_msg>logger<commit_after><|endoftext|>


<commit_before>from datetime import datetime

from django.contrib.contenttypes.models import ContentType
from django.template import RequestContext
from django.core.paginator import ObjectPaginator
from django.conf import settings
from django.template.defaultfilters import slugify
from django.db import models
from django.http import Http404

from ella.core.models import Listing, Category, HitCount
from ella.core.cache import get_cached_list, get_cached_object_or_404, cache_this
from ella.core.custom_urls import dispatcher
from ella.core.cache.template_loader import render_to_response

# local cache for get_content_type()
CONTENT_TYPE_MAPPING = {}

def get_content_type(ct_name):
    """
    A helper function that returns ContentType object based on its slugified verbose_name_plural.

    Results of this function is cached to improve performance.

    Params:
        ct_name - slugified verbose_name_plural of the target model.

    Raises:
        Http404 if no matching ContentType is found
    """
    try:
        ct = CONTENT_TYPE_MAPPING[ct_name]
    except KeyError:
        for model in models.get_models():
            if ct_name == slugify(model._meta.verbose_name_plural):
                ct = ContentType.objects.get_for_model(model)
                CONTENT_TYPE_MAPPING[ct_name] = ct
                break
        else:
            raise Http404
    return ct

def object_detail(request, category, year, month, day, content_type, slug, url_remainder=None):
    """
    Detail view that displays a single object based on it's main listing.

    Params:
        request - HttpRequest supplied by Django
        category - base Category tree_path (empty if home category)
        year, month, day - date matching the ``publish_from`` field of the ``Listing`` object
        content_type - slugified verbose_name_plural of the target model
        slug - slug of the object itself

    Raises:
        Http404 if object or listing doesn't exist or dates doesn't match
    """
    ct = get_content_type(content_type)

    if category:
        cat = get_cached_object_or_404(Category, tree_path=category, site__id=settings.SITE_ID)
    else:
        cat = get_cached_object_or_404(Category, tree_parent__isnull=True, site__id=settings.SITE_ID)

    # ger all possible listings
    listings = get_cached_list(
                Listing,
                publish_from__year=year,
                publish_from__month=month,
                publish_from__day=day,
                target_ct=ct,
                category=cat
)

    # get the object
    obj = get_cached_object_or_404(ct, slug=slug, pk__in=[ l.target_id for l in listings ])

    listing = None
    # find main listing
    for l in listings:
        if l.target_id == obj._get_pk_val():
            listing = l
            break
    else:
        raise Http404

    if not listing.is_active():
        # expired listing
        raise Http404

    if not (listing.is_published() or request.user.is_staff):
        # future listing, render if accessed by logged in staff member
        raise Http404

    context = {
            'listing' : listing,
            'object' : obj,
            'category' : cat,
            'content_type_name' : content_type,
            'content_type' : ct,
}

    # check for custom actions
    if url_remainder:
        bits = url_remainder.split('/')
        return dispatcher.call_view(request, bits, context)
    elif dispatcher.has_custom_detail(obj):
        # increment hit counter
        HitCount.objects.hit(obj)
        return dispatcher.call_custom_detail(request, context)

    # increment hit counter
    HitCount.objects.hit(obj)

    return render_to_response(
        get_templates('object.html', slug, cat, ct),
        context,
        context_instance=RequestContext(request)
)

def get_templates(name, slug=None, category=None, ct=None):
    """
    Returns templates in following format and order:
        'page/category/%s/content_type/%s.%s/%s/%s' % (category.path, ct.app_label, ct.model, slug, name),
        'page/category/%s/content_type/%s.%s/%s' % (category.path, ct.app_label, ct.model, name),
        'page/category/%s/%s' % (category.path, name),
        'page/content_type/%s.%s/%s' % (ct.app_label, ct.model, name),
        'page/%s' % name,

    TODO: Allow Placement() as parameter?
    """
    templates = []
    if category:
        if ct:
            if slug:
                templates.append('page/category/%s/content_type/%s.%s/%s/%s' % (category.path, ct.app_label, ct.model, slug, name))
            templates.append('page/category/%s/content_type/%s.%s/%s' % (category.path, ct.app_label, ct.model, name))
        templates.append('page/category/%s/%s' % (category.path, name))
    if ct:
        templates.append('page/content_type/%s.%s/%s' % (ct.app_label, ct.model, name))
    templates.append('page/%s' % name)
    return templates

def list_content_type(request, category=None, year=None, month=None, day=None, content_type=None, paginate_by=20):
    """
    List object's listings according to the parameters.

    Params:
        request - HttpRequest supplied by Django
        category - base Category tree_path, optional - defaults to all categories
        year, month, day - date matching the ``publish_from`` field of the ``Listing`` object
                           All of these parameters are optional, the list will be filtered by the non-empty ones
        content_type - slugified verbose_name_plural of the target model, if omitted all content_types are listed

    Raises:
        Http404 - if the specified category or content_type does not exist or if the given date is malformed.
    """
    kwa = {}
    dates_kwa = {}
    if year:
        current_date = datetime(int(year), 1, 1)
        dates_kwa['publish_from__year'] = int(year)
        date_kind = 'month'

    if month:
        try:
            current_date = datetime(int(year), int(month), 1)
        except ValueError:
            raise Http404
        dates_kwa['publish_from__month'] = int(month)
        date_kind = 'day'

    if day:
        try:
            current_date = datetime(int(year), int(month), int(day))
        except ValueError:
            raise Http404
        dates_kwa['publish_from__day'] = int(day)
        date_kind = 'detail'

    if category:
        cat = get_cached_object_or_404(Category, tree_path=category, site__id=settings.SITE_ID)
    else:
        cat = get_cached_object_or_404(Category, tree_parent__isnull=True, site__id=settings.SITE_ID)
    kwa['category'] = cat

    if content_type:
        ct = get_content_type(content_type)
        kwa['content_types'] = [ ct ]
    else:
        ct = False

    # pagination
    if 'p' in request.GET and request.GET['p'].isdigit():
        page = int(request.GET['p'])
    else:
        page = 1

    base_qset = Listing.objects.get_queryset(**kwa)
    kwa.update(dates_kwa)
    qset = Listing.objects.get_queryset(**kwa)
    paginator = ObjectPaginator(qset.filter(**dates_kwa), paginate_by)

    if page > paginator.pages:
        raise Http404

    kwa['count'] = paginate_by
    kwa['offset'] = (page - 1) * paginate_by + 1
    listings = Listing.objects.get_listing(**kwa)

    template_list = []
    if ct:
        template_list.append('page/category/%s/content_type/%s.%s/listing.html' % (cat.path, ct.app_label, ct.model))
    template_list.append('page/category/%s/listing.html' % (cat.path))

    if ct:
        template_list.append('page/content_type/%s.%s/listing.html' % (ct.app_label, ct.model))

    template_list.append('page/listing.html')

    url_prepend = ''
    url_apend = ''
    if ct:
        url_prepend = '../'
        url_apend = content_type + '/'

    # list of years with published objects
    year_list = [
            (d, d.strftime(DATE_REPR["year"]), url_prepend + d.strftime(YEAR_URLS[date_kind]))
            for d in base_qset.dates('publish_from', "year", order='DESC')
        ]

    # list of dates with published objects
    date_list = None
    if date_kind != 'detail':
        date_list = [
                (d, d.strftime(DATE_REPR[date_kind]), url_prepend + d.strftime(DATE_URLS[date_kind]) + url_apend)
                for d in qset.dates('publish_from', date_kind, order='DESC')
            ]

    return render_to_response(template_list, {
            'is_paginated': paginator.pages > 1,
            'results_per_page': paginate_by,
            'has_next': paginator.has_next_page(page - 1),
            'has_previous': paginator.has_previous_page(page - 1),
            'page': page,
            'next': page + 1,
            'previous': page - 1,
            'last_on_page': paginator.last_on_page(page - 1),
            'first_on_page': paginator.first_on_page(page - 1),
            'pages': paginator.pages,
            'hits' : paginator.hits,

            'year_list' : year_list,
            'date_list' : date_list,
            'current_date' : current_date,
            'current_date_text' : current_date.strftime(CURRENT_DATE_REPR[date_kind]),
            'date_kind' : date_kind,

            'content_type' : content_type,
            'listings' : listings,
            'category' : cat,
}, context_instance=RequestContext(request))

# format lookups for year_list and date_list
DATE_URLS = {'month' : '../%Y/%m/', 'day' : '../../%Y/%m/%d/',}
YEAR_URLS = {'month' : '../%Y/', 'day' : '../../%Y/', 'detail' : '../../../%Y/',}
DATE_REPR = {'year' : '%Y', 'month' : '%m/%Y', 'day' : '%d/%m/%Y', 'detail' : '%d/%m/%Y',}
CURRENT_DATE_REPR = {'month' : '%Y', 'day' : '%m/%Y', 'detail' : '%d/%m/%Y',}

def home(request):
    """
    Homepage of the actual site.

    Params:
        request - HttpRequest from Django

    Raise:
        Http404 if there is no base category
    """
    cat = get_cached_object_or_404(Category, tree_parent__isnull=True, site__id=settings.SITE_ID)
    return render_to_response(
            (
                'page/category/%s/category.html' % (cat.path),
                'page/category.html',
),
            {
                'category' : cat,
},
            context_instance=RequestContext(request)
)

def category_detail(request, category):
    """
    Homepage of a given category.

    Params:
        request - HttpRequest from Django
        category - category's ``tree_path``

    Raise:
        Http404 if the category doesn't exist
    """
    cat = get_cached_object_or_404(Category, tree_path=category, site__id=settings.SITE_ID)
    return render_to_response(
            (
                'page/category/%s/category.html' % (cat.path),
                'page/category.html',
),
            {
                'category' : cat
},
            context_instance=RequestContext(request)
)

def get_export_key(func, request, count, name=''):
    return 'ella.core.views.export:%d:%d:%s' % (
            settings.SITE_ID, count, name
)

@cache_this(get_export_key, timeout=60*60)
def export(request, count, name=''):
    """
    Export banners.

    Params:
        count - number of objects to pass into the template
        name - name of the template (page/export/banner.html is default)
        models - list of Model classes to include
    """
    t_list = []
    if name:
        t_list.append('page/export/%s.html' % name)
    t_list.append('page/export/banner.html')

    cat = get_cached_object_or_404(Category, tree_parent__isnull=True, site__id=settings.SITE_ID)
    listing = Listing.objects.get_listing(count=count, category=cat)
    return render_to_response(
            t_list,
            {'category' : cat, 'listing' : listing},
            context_instance=RequestContext(request)
)

##
# Error handlers
##
def page_not_found(request):
    response = render_to_response('page/404.html', {}, context_instance=RequestContext(request))
    response.status_code = 404
    return response

def handle_error(request):
    response = render_to_response('page/500.html', {}, context_instance=RequestContext(request))
    response.status_code = 500
    return response
<commit_msg>ella.core.views.get_templates_from_listing()<commit_after>from datetime import datetime

from django.contrib.contenttypes.models import ContentType
from django.template import RequestContext
from django.core.paginator import ObjectPaginator
from django.conf import settings
from django.template.defaultfilters import slugify
from django.db import models
from django.http import Http404

from ella.core.models import Listing, Category, HitCount
from ella.core.cache import get_cached_list, get_cached_object_or_404, cache_this
from ella.core.custom_urls import dispatcher
from ella.core.cache.template_loader import render_to_response

# local cache for get_content_type()
CONTENT_TYPE_MAPPING = {}

def get_content_type(ct_name):
    """
    A helper function that returns ContentType object based on its slugified verbose_name_plural.

    Results of this function is cached to improve performance.

    Params:
        ct_name - slugified verbose_name_plural of the target model.

    Raises:
        Http404 if no matching ContentType is found
    """
    try:
        ct = CONTENT_TYPE_MAPPING[ct_name]
    except KeyError:
        for model in models.get_models():
            if ct_name == slugify(model._meta.verbose_name_plural):
                ct = ContentType.objects.get_for_model(model)
                CONTENT_TYPE_MAPPING[ct_name] = ct
                break
        else:
            raise Http404
    return ct

def object_detail(request, category, year, month, day, content_type, slug, url_remainder=None):
    """
    Detail view that displays a single object based on it's main listing.

    Params:
        request - HttpRequest supplied by Django
        category - base Category tree_path (empty if home category)
        year, month, day - date matching the ``publish_from`` field of the ``Listing`` object
        content_type - slugified verbose_name_plural of the target model
        slug - slug of the object itself

    Raises:
        Http404 if object or listing doesn't exist or dates doesn't match
    """
    ct = get_content_type(content_type)

    if category:
        cat = get_cached_object_or_404(Category, tree_path=category, site__id=settings.SITE_ID)
    else:
        cat = get_cached_object_or_404(Category, tree_parent__isnull=True, site__id=settings.SITE_ID)

    # ger all possible listings
    listings = get_cached_list(
                Listing,
                publish_from__year=year,
                publish_from__month=month,
                publish_from__day=day,
                target_ct=ct,
                category=cat
)

    # get the object
    obj = get_cached_object_or_404(ct, slug=slug, pk__in=[ l.target_id for l in listings ])

    listing = None
    # find main listing
    for l in listings:
        if l.target_id == obj._get_pk_val():
            listing = l
            break
    else:
        raise Http404

    if not listing.is_active():
        # expired listing
        raise Http404

    if not (listing.is_published() or request.user.is_staff):
        # future listing, render if accessed by logged in staff member
        raise Http404

    context = {
            'listing' : listing,
            'object' : obj,
            'category' : cat,
            'content_type_name' : content_type,
            'content_type' : ct,
}

    # check for custom actions
    if url_remainder:
        bits = url_remainder.split('/')
        return dispatcher.call_view(request, bits, context)
    elif dispatcher.has_custom_detail(obj):
        # increment hit counter
        HitCount.objects.hit(obj)
        return dispatcher.call_custom_detail(request, context)

    # increment hit counter
    HitCount.objects.hit(obj)

    return render_to_response(
        get_templates('object.html', slug, cat, ct),
        context,
        context_instance=RequestContext(request)
)

def get_templates(name, slug=None, category=None, app_label=None, model_label=None):
    """
    Returns templates in following format and order:
        'page/category/%s/content_type/%s.%s/%s/%s' % (category.path, app_label, model_label, slug, name),
        'page/category/%s/content_type/%s.%s/%s' % (category.path, app_label, model_label, name),
        'page/category/%s/%s' % (category.path, name),
        'page/content_type/%s.%s/%s' % (app_label, model_label, name),
        'page/%s' % name,

    TODO: Allow Placement() as parameter?
    """
    templates = []
    if category:
        if app_label and model_label:
            if slug:
                templates.append('page/category/%s/content_type/%s.%s/%s/%s' % (category.path, app_label, model_label, slug, name))
            templates.append('page/category/%s/content_type/%s.%s/%s' % (category.path, app_label, model_label, name))
        templates.append('page/category/%s/%s' % (category.path, name))
    if app_label and model_label:
        templates.append('page/content_type/%s.%s/%s' % (app_label, model_label, name))
    templates.append('page/%s' % name)
    return templates

def get_templates_from_listing(name, listing, slug=None, category=None, app_label=None, model_label=None):
    """ Returns template list by listing. """
    if slug is None:
        slug = listing.target.slug
    if category is None:
        category = listing.category
    if app_label is None:
        app_label = listing.target._meta.app_label
    if model_label is None:
        model_label = listing.target._meta.module_name
    return get_templates(name, slug, category, app_label, model_label)

def list_content_type(request, category=None, year=None, month=None, day=None, content_type=None, paginate_by=20):
    """
    List object's listings according to the parameters.

    Params:
        request - HttpRequest supplied by Django
        category - base Category tree_path, optional - defaults to all categories
        year, month, day - date matching the ``publish_from`` field of the ``Listing`` object
                           All of these parameters are optional, the list will be filtered by the non-empty ones
        content_type - slugified verbose_name_plural of the target model, if omitted all content_types are listed

    Raises:
        Http404 - if the specified category or content_type does not exist or if the given date is malformed.
    """
    kwa = {}
    dates_kwa = {}
    if year:
        current_date = datetime(int(year), 1, 1)
        dates_kwa['publish_from__year'] = int(year)
        date_kind = 'month'

    if month:
        try:
            current_date = datetime(int(year), int(month), 1)
        except ValueError:
            raise Http404
        dates_kwa['publish_from__month'] = int(month)
        date_kind = 'day'

    if day:
        try:
            current_date = datetime(int(year), int(month), int(day))
        except ValueError:
            raise Http404
        dates_kwa['publish_from__day'] = int(day)
        date_kind = 'detail'

    if category:
        cat = get_cached_object_or_404(Category, tree_path=category, site__id=settings.SITE_ID)
    else:
        cat = get_cached_object_or_404(Category, tree_parent__isnull=True, site__id=settings.SITE_ID)
    kwa['category'] = cat

    if content_type:
        ct = get_content_type(content_type)
        kwa['content_types'] = [ ct ]
    else:
        ct = False

    # pagination
    if 'p' in request.GET and request.GET['p'].isdigit():
        page = int(request.GET['p'])
    else:
        page = 1

    base_qset = Listing.objects.get_queryset(**kwa)
    kwa.update(dates_kwa)
    qset = Listing.objects.get_queryset(**kwa)
    paginator = ObjectPaginator(qset.filter(**dates_kwa), paginate_by)

    if page > paginator.pages:
        raise Http404

    kwa['count'] = paginate_by
    kwa['offset'] = (page - 1) * paginate_by + 1
    listings = Listing.objects.get_listing(**kwa)

    template_list = []
    if ct:
        template_list.append('page/category/%s/content_type/%s.%s/listing.html' % (cat.path, ct.app_label, ct.model))
    template_list.append('page/category/%s/listing.html' % (cat.path))

    if ct:
        template_list.append('page/content_type/%s.%s/listing.html' % (ct.app_label, ct.model))

    template_list.append('page/listing.html')

    url_prepend = ''
    url_apend = ''
    if ct:
        url_prepend = '../'
        url_apend = content_type + '/'

    # list of years with published objects
    year_list = [
            (d, d.strftime(DATE_REPR["year"]), url_prepend + d.strftime(YEAR_URLS[date_kind]))
            for d in base_qset.dates('publish_from', "year", order='DESC')
        ]

    # list of dates with published objects
    date_list = None
    if date_kind != 'detail':
        date_list = [
                (d, d.strftime(DATE_REPR[date_kind]), url_prepend + d.strftime(DATE_URLS[date_kind]) + url_apend)
                for d in qset.dates('publish_from', date_kind, order='DESC')
            ]

    return render_to_response(template_list, {
            'is_paginated': paginator.pages > 1,
            'results_per_page': paginate_by,
            'has_next': paginator.has_next_page(page - 1),
            'has_previous': paginator.has_previous_page(page - 1),
            'page': page,
            'next': page + 1,
            'previous': page - 1,
            'last_on_page': paginator.last_on_page(page - 1),
            'first_on_page': paginator.first_on_page(page - 1),
            'pages': paginator.pages,
            'hits' : paginator.hits,

            'year_list' : year_list,
            'date_list' : date_list,
            'current_date' : current_date,
            'current_date_text' : current_date.strftime(CURRENT_DATE_REPR[date_kind]),
            'date_kind' : date_kind,

            'content_type' : content_type,
            'listings' : listings,
            'category' : cat,
}, context_instance=RequestContext(request))

# format lookups for year_list and date_list
DATE_URLS = {'month' : '../%Y/%m/', 'day' : '../../%Y/%m/%d/',}
YEAR_URLS = {'month' : '../%Y/', 'day' : '../../%Y/', 'detail' : '../../../%Y/',}
DATE_REPR = {'year' : '%Y', 'month' : '%m/%Y', 'day' : '%d/%m/%Y', 'detail' : '%d/%m/%Y',}
CURRENT_DATE_REPR = {'month' : '%Y', 'day' : '%m/%Y', 'detail' : '%d/%m/%Y',}

def home(request):
    """
    Homepage of the actual site.

    Params:
        request - HttpRequest from Django

    Raise:
        Http404 if there is no base category
    """
    cat = get_cached_object_or_404(Category, tree_parent__isnull=True, site__id=settings.SITE_ID)
    return render_to_response(
            (
                'page/category/%s/category.html' % (cat.path),
                'page/category.html',
),
            {
                'category' : cat,
},
            context_instance=RequestContext(request)
)

def category_detail(request, category):
    """
    Homepage of a given category.

    Params:
        request - HttpRequest from Django
        category - category's ``tree_path``

    Raise:
        Http404 if the category doesn't exist
    """
    cat = get_cached_object_or_404(Category, tree_path=category, site__id=settings.SITE_ID)
    return render_to_response(
            (
                'page/category/%s/category.html' % (cat.path),
                'page/category.html',
),
            {
                'category' : cat
},
            context_instance=RequestContext(request)
)

def get_export_key(func, request, count, name=''):
    return 'ella.core.views.export:%d:%d:%s' % (
            settings.SITE_ID, count, name
)

@cache_this(get_export_key, timeout=60*60)
def export(request, count, name=''):
    """
    Export banners.

    Params:
        count - number of objects to pass into the template
        name - name of the template (page/export/banner.html is default)
        models - list of Model classes to include
    """
    t_list = []
    if name:
        t_list.append('page/export/%s.html' % name)
    t_list.append('page/export/banner.html')

    cat = get_cached_object_or_404(Category, tree_parent__isnull=True, site__id=settings.SITE_ID)
    listing = Listing.objects.get_listing(count=count, category=cat)
    return render_to_response(
            t_list,
            {'category' : cat, 'listing' : listing},
            context_instance=RequestContext(request)
)

##
# Error handlers
##
def page_not_found(request):
    response = render_to_response('page/404.html', {}, context_instance=RequestContext(request))
    response.status_code = 404
    return response

def handle_error(request):
    response = render_to_response('page/500.html', {}, context_instance=RequestContext(request))
    response.status_code = 500
    return response
<|endoftext|>


<commit_before>import re
import pathlib
from typing import Dict, Optional

import nltk
import pyap.source_US.data

from . import register_detector
from .sklearn import BIOTokenSklearnDetector
from ..filth import AddressFilth

class AddressTokeniser(nltk.tokenize.destructive.NLTKWordTokenizer):
    PUNCTUATION = [
        (re.compile(r'([^\.])(\.)([\]\)}>"\'' u"»”’ " r"]*)[\t\r\f\v]*$", re.U), r"\1 \2 \3 "),
        (re.compile(r"([:,])([^\d])"), r" \1 \2"),
        (re.compile(r"([:,])$"), r" \1 "),
        (re.compile(r"([:\n])"), r" \1 "),
        (re.compile(r"\.{2,}", re.U), r" \g<0> "),  # See https://github.com/nltk/nltk/pull/2322
        (re.compile(r"[;@#$%&]"), r" \g<0> "),
        (
            re.compile(r'([^\.])(\.)([\]\)}>"\']*)[\t\r\f\v]*$'),
            r"\1 \2\3 ",
        ),  # Handles the final period.
        # Earlier we put a space before the full-stop, if there is something in the format X.X.X .
        # transform this back to X.X.X.
        (re.compile(r'(?<!\w)(\w(?:\.\w)+) +(\.)'), r'\1\2'),
        # Remove special whitespace by replacing anything that's not (^) not whitespace (\S) and that's not a newline
        # This is the same as removing whitespace (\s) except newlines (\n)
        (re.compile(r"[^\S\n]+"), " "),
        (re.compile(r"[?!]"), r" \g<0> "),
        (re.compile(r"([^'])' "), r"\1 ' "),
        (re.compile(r"[*]", re.U), r" \g<0> "),  # See https://github.com/nltk/nltk/pull/2322
    ]

    def tokenize(self, text, convert_parentheses=False, return_str=False):
        for regexp, substitution in self.STARTING_QUOTES:
            text = regexp.sub(substitution, text)

        for regexp, substitution in self.PUNCTUATION:
            text = regexp.sub(substitution, text)

        # Handles parentheses.
        regexp, substitution = self.PARENS_BRACKETS
        text = regexp.sub(substitution, text)
        # Optionally convert parentheses
        if convert_parentheses:
            for regexp, substitution in self.CONVERT_PARENTHESES:
                text = regexp.sub(substitution, text)

        # Handles double dash.
        regexp, substitution = self.DOUBLE_DASHES
        text = regexp.sub(substitution, text)

        # add extra space to make things easier
        text = " " + text + " "

        for regexp, substitution in self.ENDING_QUOTES:
            text = regexp.sub(substitution, text)

        for regexp in self.CONTRACTIONS2:
            text = regexp.sub(r" \1 \2 ", text)
        for regexp in self.CONTRACTIONS3:
            text = regexp.sub(r" \1 \2 ", text)

        # Only splitting by spaces to get newlines as tokens
        return text if return_str else [x for x in text.split(' ') if x != '']

class SklearnAddressDetector(BIOTokenSklearnDetector):
    name = "sklearn_address"
    label_filth_map = {
        'ADD': AddressFilth,
        'ADDRESS': AddressFilth,
        'I-ADD': AddressFilth,
        'B-ADD': AddressFilth,
        'I-ADDRESS': AddressFilth,
        'B-ADDRESS': AddressFilth,
    }

    # models / sklearn_address

    BUILDING_WORDS = [
        'flat', 'flats', 'building', 'buildings', 'bldg', 'bld', 'apartment', 'apartments', 'apt', 'house', 'houses',
        'studio', 'studios', 'suite', 'suites', 'room', 'rooms', 'centre', 'penthouse', 'residence', 'office', 'tower',
        'court',
    ]

    PO_BOX_WORDS = [
        'po', 'box', 'p.o', 'p.o.'
    ]

    DIRECTION_WORDS = [
        'north', 'south', 'east', 'west', 'northern', 'southern', 'eastern', 'western',
    ]

    STREET_WORDS = [x.lower() for x in pyap.source_US.data.street_type_list]

    # TODO: Include these prefixes/suffixes
    # https://en.wikipedia.org/wiki/List_of_generic_forms_in_place_names_in_Ireland_and_the_United_Kingdom
    PLACE_WORDS = {
        'town', 'ton', 'land', 'lands', 'ville', 'berg', 'burgh', 'brough', 'borough', 'bury', 'view', 'port',
        'stad', 'stead', 'furt', 'chester', 'mouth', 'fort', 'haven', 'side', 'shire', 'city', 'cum',
        'cester', 'ford', 'ham', 'worth', 'berry', 'field', 'church,'
        # Include rivers here
        'thames', 'severn', 'trent', 'wye', 'ouse', 'tyne', 'mersey', 'avon', 'aber',
        'upon'
    }
    # TODO: add towns
    # TODO: PO Box address
    # TODO: the word address

    COUNTY_WORDS = {
        'bedfordshire', 'buckinghamshire', 'cambridgeshire', 'cheshire', 'cleveland', 'cornwall', 'cumbria',
        'derbyshire', 'devon', 'dorset', 'durham', 'sussex', 'essex', 'gloucestershire', 'greater', 'london',
        'manchester', 'hampshire', 'hertfordshire', 'kent', 'lancashire', 'leicestershire', 'lincolnshire',
        'merseyside', 'norfolk', 'yorkshire', 'northamptonshire', 'northumberland', 'nottinghamshire', 'oxfordshire',
        'shropshire', 'somerset', 'staffordshire', 'suffolk', 'surrey', 'tyne', 'wear', 'warwickshire',
        'berkshire', 'midlands', 'sussex', 'wiltshire', 'worcestershire', 'flintshire', 'glamorgan', 'merionethshire',
        'monmouthshire', 'montgomeryshire', 'pembrokeshire', 'radnorshire', 'anglesey', 'breconshire',
        'caernarvonshire', 'cardiganshire', 'carmarthenshire', 'denbighshire', 'aberdeen', 'aberdeenshire', 'angus',
        'argyll', 'bute', 'edinburgh', 'clackmannanshire', 'dumfries', 'galloway', 'dundee', 'ayrshire',
        'dunbartonshire', 'lothian', 'renfrewshire', 'eilean', 'siar', 'falkirk', 'fife', 'glasgow', 'highland',
        'inverclyde', 'midlothian', 'moray', 'ayrshire', 'lanarkshire', 'orkney', 'perth', 'kinross', 'renfrewshire',
        'borders', 'shetland', 'ayrshire', 'lanarkshire', 'stirling', 'dunbartonshire', 'lothian', 'antrim', 'armagh',
        'fermanagh', 'derry', 'londonderry', 'tyrone'
    }

    COUNTRY_WORDS = {
        'united', 'kingdom', 'britain', 'england', 'ni', 'uk', 'gb', 'gbr', 'scotland', 'ireland', 'wales',
        'cymry', 'cymru', 'alba'
    }

    # Added as dates looked similar to addresses
    DATE_WORDS = [
        'january', 'february', 'march', 'april', 'may', 'june', 'july', 'august', 'september', 'october', 'november',
        'december', 'jan', 'feb', 'mar', 'apr', 'jun', 'jul', 'aug', 'sep', 'sept', 'oct', 'nov', 'dec',
        'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday',
        'mon', 'tue', 'wed', 'weds', 'thurs', 'fri', 'sat', 'sun',
    ]

    # Regexs allow for 5 <-> S errors in OCR
    POSTCODE_START = re.compile(r"""
        (?:[Ww][Cc][0-9sS][abehmnprvwxyABEHMNPRVWXY])|
        (?:[Ee][Cc][1-4][abehmnprvwxyABEHMNPRVWXY])|
        (?:[Nn][Ww]1[Ww])|
        (?:[Ss][Ee]1[Pp])|
        (?:[Ss][Ww]1[abehmnprvwxyABEHMNPRVWXY])|
        (?:[EeNnWw]1[a-hjkpstuwA-HJKPSTUW5])|
        (?:[BbEeGgLlMmNnSsWw][0-9sS][0-9sS]?)|
        (?:[a-pr-uwyzA-PR-UWYZ5][a-hk-yxA-HK-XY5][0-9sS][0-9sS]?)
    """, re.VERBOSE)

    POSTCODE_END = re.compile(r"""[0-9sS][abd-hjlnp-uw-zABD-HJLNP-UW-Z5]{2}""", re.VERBOSE)

    def __init__(self, model_path_prefix: Optional[str] = None, b_token_required: bool = False,
                 minimum_ntokens: int = 4, number_missing_tokens_allowed: Optional[int] = 3,
                 number_missing_characters_allowed: Optional[int] = None, **kwargs):

        if model_path_prefix is None:
            model_path_prefix = str(pathlib.Path(__file__).parent / 'models' / 'sklearn_address')

        super(SklearnAddressDetector, self).__init__(
            model_path_prefix=model_path_prefix, b_token_required=b_token_required, minimum_ntokens=minimum_ntokens,
            number_missing_tokens_allowed=number_missing_tokens_allowed,
            number_missing_characters_allowed=number_missing_characters_allowed, **kwargs
        )
        self.tokeniser = AddressTokeniser()

    @staticmethod
    def create_features_single_token(token: str, prefix: str = '') -> Dict:
        token = token.strip(' \t\r\v\f')
        address_punctuation = any(punct in token.lower() for punct in ('\n', ','))
        street_word = token.lower() in SklearnAddressDetector.STREET_WORDS
        building_word = token.lower() in SklearnAddressDetector.BUILDING_WORDS
        direction_word = token.lower() in SklearnAddressDetector.DIRECTION_WORDS
        place_word = sum(word in token.lower() for word in SklearnAddressDetector.PLACE_WORDS)
        county_word = token.lower() in SklearnAddressDetector.COUNTY_WORDS
        country_word = token.lower() in SklearnAddressDetector.COUNTRY_WORDS
        date_word = token.lower() in SklearnAddressDetector.DATE_WORDS
        po_box_word = token.lower() in SklearnAddressDetector.PO_BOX_WORDS
        postcode_start = re.match(SklearnAddressDetector.POSTCODE_START, token) is not None
        postcode_end = re.match(SklearnAddressDetector.POSTCODE_END, token) is not None

        features = {
            prefix + 'capitalised': token.istitle(),
            prefix + 'lower': token.islower(),
            prefix + 'upper': token.isupper(),
            prefix + 'numeric': any(c.isdigit() for c in token),
            prefix + 'alphanumeric': any(c.isdigit() for c in token) and any(c.isalpha() for c in token),
            prefix + 'address_punctuation': address_punctuation,
            prefix + 'building_word': building_word,
            prefix + 'direction_word': direction_word,
            prefix + 'street_word': street_word,
            prefix + 'place_word': place_word,
            prefix + 'po_box_word': po_box_word,
            prefix + 'county_word': county_word,
            prefix + 'country_word': country_word,
            prefix + 'date_word': date_word,
            # Limit to 5 to prevent ML from learning fingerprint of common addresses
            prefix + 'length': len(token) if len(token) <= 5 else 5,
            prefix + 'length_short': len(token) <= 4,
            prefix + 'length_long': len(token) > 20,
            prefix + 'postcode_start': postcode_start,
            prefix + 'postcode_end': postcode_end,
        }
        return features

    @classmethod
    def supported_locale(cls, locale: str) -> bool:
        """Returns true if this ``Detector`` supports the given locale.

        :param locale: The locale of the documents in the format: 2 letter lower-case language code followed by an
                       underscore and the two letter upper-case country code, eg "en_GB" or "de_CH".
        :type locale: str
        :return: ``True`` if the locale is supported, otherwise ``False``
        :rtype: bool
        """
        language, region = cls.locale_split(locale)
        return region in ['GB']

register_detector(SklearnAddressDetector, autoload=False)
<commit_msg>tokenise 'hello. a.5.' as ['hello', '.', 'a.5.']<commit_after>import re
import pathlib
from typing import Dict, Optional

import nltk
import pyap.source_US.data

from . import register_detector
from .sklearn import BIOTokenSklearnDetector
from ..filth import AddressFilth

class AddressTokeniser(nltk.tokenize.destructive.NLTKWordTokenizer):
    PUNCTUATION = [
        (re.compile(r'([^\.]{2,})(\.)([\]\)}>"\'' u"»”’ " r"]*)[\t\r\f\v]*$", re.U), r"\1 \2 \3 "),
        (re.compile(r"([:,])([^\d])"), r" \1 \2"),
        (re.compile(r"([:,])$"), r" \1 "),
        (re.compile(r"([:\n])"), r" \1 "),
        (re.compile(r"\.{2,}", re.U), r" \g<0> "),  # See https://github.com/nltk/nltk/pull/2322
        (re.compile(r"[;@#$%&]"), r" \g<0> "),
        (
            re.compile(r'([^\.]{2,})(\.)([\]\)}>"\']*)[\t\r\f\v]*$'),
            r"\1 \2\3 ",
        ),  # Handles the final period.
        # Earlier we put a space before the full-stop, if there is something in the format X.X.X .
        # transform this back to X.X.X.
        # (re.compile(r'(?<!\w)(\w(?:\.\w)+) +(\.)'), r'\1\2'),
        # Remove special whitespace by replacing anything that's not (^) not whitespace (\S) and that's not a newline
        # This is the same as removing whitespace (\s) except newlines (\n)
        (re.compile(r"[^\S\n]+"), " "),
        (re.compile(r"[?!]"), r" \g<0> "),
        (re.compile(r"([^'])' "), r"\1 ' "),
        (re.compile(r"[*]", re.U), r" \g<0> "),  # See https://github.com/nltk/nltk/pull/2322
    ]

    def tokenize(self, text, convert_parentheses=False, return_str=False):
        for regexp, substitution in self.STARTING_QUOTES:
            text = regexp.sub(substitution, text)

        for regexp, substitution in self.PUNCTUATION:
            text = regexp.sub(substitution, text)

        # Handles parentheses.
        regexp, substitution = self.PARENS_BRACKETS
        text = regexp.sub(substitution, text)
        # Optionally convert parentheses
        if convert_parentheses:
            for regexp, substitution in self.CONVERT_PARENTHESES:
                text = regexp.sub(substitution, text)

        # Handles double dash.
        regexp, substitution = self.DOUBLE_DASHES
        text = regexp.sub(substitution, text)

        # add extra space to make things easier
        text = " " + text + " "

        for regexp, substitution in self.ENDING_QUOTES:
            text = regexp.sub(substitution, text)

        for regexp in self.CONTRACTIONS2:
            text = regexp.sub(r" \1 \2 ", text)
        for regexp in self.CONTRACTIONS3:
            text = regexp.sub(r" \1 \2 ", text)

        # Only splitting by spaces to get newlines as tokens
        return text if return_str else [x for x in text.split(' ') if x != '']

class SklearnAddressDetector(BIOTokenSklearnDetector):
    name = "sklearn_address"
    label_filth_map = {
        'ADD': AddressFilth,
        'ADDRESS': AddressFilth,
        'I-ADD': AddressFilth,
        'B-ADD': AddressFilth,
        'I-ADDRESS': AddressFilth,
        'B-ADDRESS': AddressFilth,
    }

    # models / sklearn_address

    BUILDING_WORDS = [
        'flat', 'flats', 'building', 'buildings', 'bldg', 'bld', 'apartment', 'apartments', 'apt', 'house', 'houses',
        'studio', 'studios', 'suite', 'suites', 'room', 'rooms', 'centre', 'penthouse', 'residence', 'office', 'tower',
        'court',
    ]

    PO_BOX_WORDS = [
        'po', 'box', 'p.o', 'p.o.'
    ]

    DIRECTION_WORDS = [
        'north', 'south', 'east', 'west', 'northern', 'southern', 'eastern', 'western',
    ]

    STREET_WORDS = [x.lower() for x in pyap.source_US.data.street_type_list]

    # TODO: Include these prefixes/suffixes
    # https://en.wikipedia.org/wiki/List_of_generic_forms_in_place_names_in_Ireland_and_the_United_Kingdom
    PLACE_WORDS = {
        'town', 'ton', 'land', 'lands', 'ville', 'berg', 'burgh', 'brough', 'borough', 'bury', 'view', 'port',
        'stad', 'stead', 'furt', 'chester', 'mouth', 'fort', 'haven', 'side', 'shire', 'city', 'cum',
        'cester', 'ford', 'ham', 'worth', 'berry', 'field', 'church,'
        # Include rivers here
        'thames', 'severn', 'trent', 'wye', 'ouse', 'tyne', 'mersey', 'avon', 'aber',
        'upon'
    }
    # TODO: add towns
    # TODO: PO Box address
    # TODO: the word address

    COUNTY_WORDS = {
        'bedfordshire', 'buckinghamshire', 'cambridgeshire', 'cheshire', 'cleveland', 'cornwall', 'cumbria',
        'derbyshire', 'devon', 'dorset', 'durham', 'sussex', 'essex', 'gloucestershire', 'greater', 'london',
        'manchester', 'hampshire', 'hertfordshire', 'kent', 'lancashire', 'leicestershire', 'lincolnshire',
        'merseyside', 'norfolk', 'yorkshire', 'northamptonshire', 'northumberland', 'nottinghamshire', 'oxfordshire',
        'shropshire', 'somerset', 'staffordshire', 'suffolk', 'surrey', 'tyne', 'wear', 'warwickshire',
        'berkshire', 'midlands', 'sussex', 'wiltshire', 'worcestershire', 'flintshire', 'glamorgan', 'merionethshire',
        'monmouthshire', 'montgomeryshire', 'pembrokeshire', 'radnorshire', 'anglesey', 'breconshire',
        'caernarvonshire', 'cardiganshire', 'carmarthenshire', 'denbighshire', 'aberdeen', 'aberdeenshire', 'angus',
        'argyll', 'bute', 'edinburgh', 'clackmannanshire', 'dumfries', 'galloway', 'dundee', 'ayrshire',
        'dunbartonshire', 'lothian', 'renfrewshire', 'eilean', 'siar', 'falkirk', 'fife', 'glasgow', 'highland',
        'inverclyde', 'midlothian', 'moray', 'ayrshire', 'lanarkshire', 'orkney', 'perth', 'kinross', 'renfrewshire',
        'borders', 'shetland', 'ayrshire', 'lanarkshire', 'stirling', 'dunbartonshire', 'lothian', 'antrim', 'armagh',
        'fermanagh', 'derry', 'londonderry', 'tyrone'
    }

    COUNTRY_WORDS = {
        'united', 'kingdom', 'britain', 'england', 'ni', 'uk', 'gb', 'gbr', 'scotland', 'ireland', 'wales',
        'cymry', 'cymru', 'alba'
    }

    # Added as dates looked similar to addresses
    DATE_WORDS = [
        'january', 'february', 'march', 'april', 'may', 'june', 'july', 'august', 'september', 'october', 'november',
        'december', 'jan', 'feb', 'mar', 'apr', 'jun', 'jul', 'aug', 'sep', 'sept', 'oct', 'nov', 'dec',
        'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday',
        'mon', 'tue', 'wed', 'weds', 'thurs', 'fri', 'sat', 'sun',
    ]

    # Regexs allow for 5 <-> S errors in OCR
    POSTCODE_START = re.compile(r"""
        (?:[Ww][Cc][0-9sS][abehmnprvwxyABEHMNPRVWXY])|
        (?:[Ee][Cc][1-4][abehmnprvwxyABEHMNPRVWXY])|
        (?:[Nn][Ww]1[Ww])|
        (?:[Ss][Ee]1[Pp])|
        (?:[Ss][Ww]1[abehmnprvwxyABEHMNPRVWXY])|
        (?:[EeNnWw]1[a-hjkpstuwA-HJKPSTUW5])|
        (?:[BbEeGgLlMmNnSsWw][0-9sS][0-9sS]?)|
        (?:[a-pr-uwyzA-PR-UWYZ5][a-hk-yxA-HK-XY5][0-9sS][0-9sS]?)
    """, re.VERBOSE)

    POSTCODE_END = re.compile(r"""[0-9sS][abd-hjlnp-uw-zABD-HJLNP-UW-Z5]{2}""", re.VERBOSE)

    def __init__(self, model_path_prefix: Optional[str] = None, b_token_required: bool = False,
                 minimum_ntokens: int = 4, number_missing_tokens_allowed: Optional[int] = 3,
                 number_missing_characters_allowed: Optional[int] = None, **kwargs):

        if model_path_prefix is None:
            model_path_prefix = str(pathlib.Path(__file__).parent / 'models' / 'sklearn_address')

        super(SklearnAddressDetector, self).__init__(
            model_path_prefix=model_path_prefix, b_token_required=b_token_required, minimum_ntokens=minimum_ntokens,
            number_missing_tokens_allowed=number_missing_tokens_allowed,
            number_missing_characters_allowed=number_missing_characters_allowed, **kwargs
        )
        self.tokeniser = AddressTokeniser()

    @staticmethod
    def create_features_single_token(token: str, prefix: str = '') -> Dict:
        token = token.strip(' \t\r\v\f')
        address_punctuation = any(punct in token.lower() for punct in ('\n', ','))
        street_word = token.lower() in SklearnAddressDetector.STREET_WORDS
        building_word = token.lower() in SklearnAddressDetector.BUILDING_WORDS
        direction_word = token.lower() in SklearnAddressDetector.DIRECTION_WORDS
        place_word = sum(word in token.lower() for word in SklearnAddressDetector.PLACE_WORDS)
        county_word = token.lower() in SklearnAddressDetector.COUNTY_WORDS
        country_word = token.lower() in SklearnAddressDetector.COUNTRY_WORDS
        date_word = token.lower() in SklearnAddressDetector.DATE_WORDS
        po_box_word = token.lower() in SklearnAddressDetector.PO_BOX_WORDS
        postcode_start = re.match(SklearnAddressDetector.POSTCODE_START, token) is not None
        postcode_end = re.match(SklearnAddressDetector.POSTCODE_END, token) is not None

        features = {
            prefix + 'capitalised': token.istitle(),
            prefix + 'lower': token.islower(),
            prefix + 'upper': token.isupper(),
            prefix + 'numeric': any(c.isdigit() for c in token),
            prefix + 'alphanumeric': any(c.isdigit() for c in token) and any(c.isalpha() for c in token),
            prefix + 'address_punctuation': address_punctuation,
            prefix + 'building_word': building_word,
            prefix + 'direction_word': direction_word,
            prefix + 'street_word': street_word,
            prefix + 'place_word': place_word,
            prefix + 'po_box_word': po_box_word,
            prefix + 'county_word': county_word,
            prefix + 'country_word': country_word,
            prefix + 'date_word': date_word,
            # Limit to 5 to prevent ML from learning fingerprint of common addresses
            prefix + 'length': len(token) if len(token) <= 5 else 5,
            prefix + 'length_short': len(token) <= 4,
            prefix + 'length_long': len(token) > 20,
            prefix + 'postcode_start': postcode_start,
            prefix + 'postcode_end': postcode_end,
        }
        return features

    @classmethod
    def supported_locale(cls, locale: str) -> bool:
        """Returns true if this ``Detector`` supports the given locale.

        :param locale: The locale of the documents in the format: 2 letter lower-case language code followed by an
                       underscore and the two letter upper-case country code, eg "en_GB" or "de_CH".
        :type locale: str
        :return: ``True`` if the locale is supported, otherwise ``False``
        :rtype: bool
        """
        language, region = cls.locale_split(locale)
        return region in ['GB']

register_detector(SklearnAddressDetector, autoload=False)
<|endoftext|>


<commit_before>from datetime import datetime
from django.db import models
from apps.authentication.models import OnlineUser as User
from django.utils.translation import ugettext as _
from hashlib import sha256
from django.conf import settings
import operator

# Statics

QUESTION_TYPES = [
    (0, _(u'Ja/Nei')),
    (1, _(u'Multiple Choice')),
]

BOOLEAN_VOTE = 0
MULTIPLE_CHOICE = 1

# General genfors model

class Meeting(models.Model):
    '''
    The Meeting model encapsulates a single Generalforsamling with all cascading one-to-many relationships
    '''
    start_date = models.DateTimeField(_(u'date'), help_text=_('Dato for arrangementsstart'), null=False)
    title = models.CharField(_(u'title'), max_length=150, null=False)
    registration_locked = models.BooleanField(_(u'registration_lock'), help_text=_(u'Steng registrering'), default=False, blank=False, null=False)
    ended = models.BooleanField(_(u'event_lockdown'), help_text=_(u'Avslutt generalforsamlingen'), default=False, blank=False, null=False)

    def __unicode__(self):
        return self.title + ' (' + self.start_date.ctime() + ')'

    # Return the number of attendees
    def num_attendees(self):
        return RegisteredVoter.objects.filter(meeting=self).count()

    # Get attendee list as a list of strings
    def get_attendee_list(self):
        return sorted([u'%s, %s' %(a.user.last_name, a.user.first_name) for a in RegisteredVoter.objects.filter(meeting=self)])

    # Get the queryset of all questions
    def get_questions(self):
        return Question.objects.filter(meeting=self).order_by('-id')

    # Get active question, if there is one
    def get_active_question(self):
        question = self.get_questions()
        if question and not question[0].locked:
            return question[0]
        else:
            return None

    # Get all previous questions
    def get_locked_questions(self):
        return self.get_questions().filter(locked=True)

    # Get the number of questions
    def num_questions(self):
        return Question.objects.filter(meeting=self).count()

    # Get results from a specific question
    def get_results_from_question(self, question):
        return question.get_results()

    class Meta:
        verbose_name = _(u'Generalforsamling')
        verbose_name_plural = _(u'Generalforsamlinger')

# User related models

class RegisteredVoter(models.Model):
    '''
    The RegisteredVoter model is a wrapper for OnlineUser with added fields for attendance registry
    porpuses, as well as an added can_vote field for future use.
    '''
    meeting = models.ForeignKey(Meeting, null=False)
    user = models.ForeignKey(User, null=False)
    can_vote = models.BooleanField(_(u'voting_right'), help_text=_(u'Har stemmerett'), null=False, default=True)

    # Simple hashing function to hide realnames
    def hide_user(self):
        h = sha256()
        h.update(self.user.first_name + self.user.last_name)
        h.update(settings.SECRET_KEY)
        h = h.hexdigest()[:8]
        return h

    def __unicode__(self):
        return u'%s %s' %(self.user.first_name, self.user.last_name)

# Question wrapper

class Question(models.Model):
    '''
    A question is a wrapper to which all votes must be connected.
    '''
    meeting = models.ForeignKey(Meeting, help_text=_(u'Generalforsamling'), null=False)
    anonymous = models.BooleanField(_(u'anonymous'), help_text=_(u'Hemmelig valg'), null=False, blank=False)
    created_time = models.DateTimeField(_(u'added'), auto_now_add=True)
    locked = models.BooleanField(_(u'locked'), help_text=_(u'Steng avstemmingen'), null=False, blank=False, default=False)
    question_type = models.SmallIntegerField(_(u'question_type'), help_text=_(u'Type'), choices=QUESTION_TYPES, null=False, default=0, blank=False)
    description = models.TextField(_(u'description'), help_text=_(u'Beskrivelse av saken som skal stemmes over'), max_length=500, blank=True)
    result = models.CharField(_(u'result'), max_length=100, help_text=_(u'Resultatet av avstemmingen'), null=True, blank=True)
    only_show_winner = models.BooleanField(_(u'onlywinner'), help_text=_(u'Vis kun vinner'), null=False, blank=False, default=False)

    # Returns results as a dictionary, either by alternative or boolean-ish types
    def get_results(self):
        retults = None

        if self.question_type is BOOLEAN_VOTE:
            results = {'JA': 0, 'NEI': 0, 'BLANKT': 0}
            for a in BooleanVote.objects.filter(question=self):
                if a.answer == None:
                    results['BLANKT'] += 1
                elif a.answer == False:
                    results['NEI'] += 1
                elif a.answer == True:
                    results['JA'] += 1

        elif self.question_type is MULTIPLE_CHOICE:
            mc = MultipleChoice.objects.filter(question=self)
            results = {}
            for a in mc:
                if a.answer != None:
                    if a.answer.description not in results:
                        results[a.answer.description] = 0
                    results[a.answer.description] += 1
                else:
                    if 0 not in results:
                        results['Blankt'] = 0
                    results['Blankt'] += 1

        return results

    # Fetches the winner of a vote
    def get_winner(self):
        results = self.get_results()
        winner = max(results.iterkeys(), key=(lambda key: results[key]))
        results = {winner: results[winner]}

        return results

    # Returns the queryset of alternatives connected to this question if it is a multiple choice question
    def get_alternatives(self):
        if self.question_type is MULTIPLE_CHOICE:
            return Alternative.objects.filter(question=self)
        else:
            return None

    # Resets the question if there has not been enough difference to settle the case
    def reset_question(self):
        BooleanVote.objects.filter(question=self).delete()
        MultipleChoice.objects.filter(question=self).delete()

    # Sets final result as a string for later fast lookup and locks question
    def set_result_and_lock(self):
        r = self.get_results()
        if self.question_type is BOOLEAN_VOTE:
            self.result = 'J:' + str(r['JA']) + ' N:' + str(r['NEI']) + ' B:' + str(r['BLANKT'])
        elif self.question_type is MULTIPLE_CHOICE:
            result_string = ''
            if self.only_show_winner:
                r = self.get_winner()
                for k,v in r.items():
                    result_string += u'%s' %(str(k))
            else:
                for k,v in r.items():
                    result_string += u'%s: %s ' %(str(k), str(v))
            self.result = result_string
        self.locked = True
        self.save()

    # Returns all votes connected to this question
    def get_votes(self):
        if self.question_type is BOOLEAN_VOTE:
            return BooleanVote.objects.filter(question=self)
        elif self.question_type is MULTIPLE_CHOICE:
            return MultipleChoice.objects.filter(question=self)

    # Check if a RegisteredVoter already has voted on this question
    def already_voted(self, v):
        if self.question_type is BOOLEAN_VOTE:
            return BooleanVote.objects.filter(question=self, voter=v).count()
        elif self.question_type is MULTIPLE_CHOICE:
            return MultipleChoice.objects.filter(question=self, voter=v).count()

    def __unicode__(self):
        return u'[%d] %s' %(self.id - self.meeting.num_questions(), self.description)

# Individual abstract vote and vote types

class AbstractVote(models.Model):
    '''
    The AbstractVote model holds some key components of a vote to a specific question
    '''
    time = models.DateTimeField(_(u'timestamp'), auto_now_add=True)
    voter = models.ForeignKey(RegisteredVoter, help_text=_(u'Bruker'), null=False)
    question = models.ForeignKey(Question, null=False)

    # Get voter name
    def get_voter_name(self):
        if self.question.anonymous:
            return voter.hide_user()
        else:
            return u'%s %s' %(self.voter.user.first_name, self.voter.user.last_name)

class BooleanVote(AbstractVote):
    '''
    The BooleanVote model holds the yes/no/blank answer to a specific question held in superclass
    '''
    answer = models.NullBooleanField(_(u'answer'), help_text=_(u'Ja/Nei'), null=True, blank=True)

class Alternative(models.Model):
    '''
    The Alternative class represents a single alternative that is connected to a particular multiple choice type question
    '''
    alt_id = models.PositiveIntegerField(null=False, help_text=_(u'Alternativ ID'), blank=False)
    question = models.ForeignKey(Question, null=False, help_text=_(u'Question'))
    description = models.CharField(null=True, blank=True, help_text=_(u'Beskrivelse'), max_length=150)

    def __unicode__(self):
        return self.description

class MultipleChoice(AbstractVote):
    '''
    The MultipleChoice model holds the answered alternative to a specific question held in superclass
    '''
    answer = models.ForeignKey(Alternative, null=True, blank=True, help_text=_(u'Alternativ'))

<commit_msg>Corrected labels and help text for genfors models<commit_after>#-*- coding: utf-8 -*-
from datetime import datetime
from django.db import models
from apps.authentication.models import OnlineUser as User
from django.utils.translation import ugettext as _
from hashlib import sha256
from django.conf import settings
import operator

# Statics

QUESTION_TYPES = [
    (0, _(u'Ja/Nei')),
    (1, _(u'Multiple Choice')),
]

BOOLEAN_VOTE = 0
MULTIPLE_CHOICE = 1

# General genfors model

class Meeting(models.Model):
    '''
    The Meeting model encapsulates a single Generalforsamling with all cascading one-to-many relationships
    '''
    start_date = models.DateTimeField(_(u'Dato'), help_text=_('Dato for arrangementsstart'), null=False)
    title = models.CharField(_(u'Tittel'), max_length=150, null=False)
    registration_locked = models.BooleanField(_(u'registration_lock'), help_text=_(u'Steng registrering'), default=False, blank=False, null=False)
    ended = models.BooleanField(_(u'event_lockdown'), help_text=_(u'Avslutt generalforsamlingen'), default=False, blank=False, null=False)

    def __unicode__(self):
        return self.title + ' (' + self.start_date.ctime() + ')'

    # Return the number of attendees
    def num_attendees(self):
        return RegisteredVoter.objects.filter(meeting=self).count()

    # Get attendee list as a list of strings
    def get_attendee_list(self):
        return sorted([u'%s, %s' %(a.user.last_name, a.user.first_name) for a in RegisteredVoter.objects.filter(meeting=self)])

    # Get the queryset of all questions
    def get_questions(self):
        return Question.objects.filter(meeting=self).order_by('-id')

    # Get active question, if there is one
    def get_active_question(self):
        question = self.get_questions()
        if question and not question[0].locked:
            return question[0]
        else:
            return None

    # Get all previous questions
    def get_locked_questions(self):
        return self.get_questions().filter(locked=True)

    # Get the number of questions
    def num_questions(self):
        return Question.objects.filter(meeting=self).count()

    # Get results from a specific question
    def get_results_from_question(self, question):
        return question.get_results()

    class Meta:
        verbose_name = _(u'Generalforsamling')
        verbose_name_plural = _(u'Generalforsamlinger')

# User related models

class RegisteredVoter(models.Model):
    '''
    The RegisteredVoter model is a wrapper for OnlineUser with added fields for attendance registry
    porpuses, as well as an added can_vote field for future use.
    '''
    meeting = models.ForeignKey(Meeting, null=False)
    user = models.ForeignKey(User, null=False)
    can_vote = models.BooleanField(_(u'voting_right'), help_text=_(u'Har stemmerett'), null=False, default=True)

    # Simple hashing function to hide realnames
    def hide_user(self):
        h = sha256()
        h.update(self.user.first_name + self.user.last_name)
        h.update(settings.SECRET_KEY)
        h = h.hexdigest()[:8]
        return h

    def __unicode__(self):
        return u'%s %s' %(self.user.first_name, self.user.last_name)

# Question wrapper

class Question(models.Model):
    '''
    A question is a wrapper to which all votes must be connected.
    '''
    meeting = models.ForeignKey(Meeting, help_text=_(u'Generalforsamling'), null=False)
    anonymous = models.BooleanField(_(u'Hemmelig valg'), null=False, blank=False)
    created_time = models.DateTimeField(_(u'added'), auto_now_add=True)
    locked = models.BooleanField(_(u'locked'), help_text=_(u'Steng avstemmingen'), null=False, blank=False, default=False)
    question_type = models.SmallIntegerField(_(u'Spørsmålstype'), choices=QUESTION_TYPES, null=False, default=0, blank=False)
    description = models.TextField(_(u'Beskrivelse'), help_text=_(u'Beskrivelse av saken som skal stemmes over'), max_length=500, blank=True)
    result = models.CharField(_(u'result'), max_length=100, help_text=_(u'Resultatet av avstemmingen'), null=True, blank=True)
    only_show_winner = models.BooleanField(_(u'Vis kun vinner'), null=False, blank=False, default=False)

    # Returns results as a dictionary, either by alternative or boolean-ish types
    def get_results(self):
        retults = None

        if self.question_type is BOOLEAN_VOTE:
            results = {'JA': 0, 'NEI': 0, 'BLANKT': 0}
            for a in BooleanVote.objects.filter(question=self):
                if a.answer == None:
                    results['BLANKT'] += 1
                elif a.answer == False:
                    results['NEI'] += 1
                elif a.answer == True:
                    results['JA'] += 1

        elif self.question_type is MULTIPLE_CHOICE:
            mc = MultipleChoice.objects.filter(question=self)
            results = {}
            for a in mc:
                if a.answer != None:
                    if a.answer.description not in results:
                        results[a.answer.description] = 0
                    results[a.answer.description] += 1
                else:
                    if 0 not in results:
                        results['Blankt'] = 0
                    results['Blankt'] += 1

        return results

    # Fetches the winner of a vote
    def get_winner(self):
        results = self.get_results()
        winner = max(results.iterkeys(), key=(lambda key: results[key]))
        results = {winner: results[winner]}

        return results

    # Returns the queryset of alternatives connected to this question if it is a multiple choice question
    def get_alternatives(self):
        if self.question_type is MULTIPLE_CHOICE:
            return Alternative.objects.filter(question=self)
        else:
            return None

    # Resets the question if there has not been enough difference to settle the case
    def reset_question(self):
        BooleanVote.objects.filter(question=self).delete()
        MultipleChoice.objects.filter(question=self).delete()

    # Sets final result as a string for later fast lookup and locks question
    def set_result_and_lock(self):
        r = self.get_results()
        if self.question_type is BOOLEAN_VOTE:
            self.result = 'J:' + str(r['JA']) + ' N:' + str(r['NEI']) + ' B:' + str(r['BLANKT'])
        elif self.question_type is MULTIPLE_CHOICE:
            result_string = ''
            if self.only_show_winner:
                r = self.get_winner()
                for k,v in r.items():
                    result_string += u'%s' %(str(k))
            else:
                for k,v in r.items():
                    result_string += u'%s: %s ' %(str(k), str(v))
            self.result = result_string
        self.locked = True
        self.save()

    # Returns all votes connected to this question
    def get_votes(self):
        if self.question_type is BOOLEAN_VOTE:
            return BooleanVote.objects.filter(question=self)
        elif self.question_type is MULTIPLE_CHOICE:
            return MultipleChoice.objects.filter(question=self)

    # Check if a RegisteredVoter already has voted on this question
    def already_voted(self, v):
        if self.question_type is BOOLEAN_VOTE:
            return BooleanVote.objects.filter(question=self, voter=v).count()
        elif self.question_type is MULTIPLE_CHOICE:
            return MultipleChoice.objects.filter(question=self, voter=v).count()

    def __unicode__(self):
        return u'[%d] %s' %(self.id - self.meeting.num_questions(), self.description)

# Individual abstract vote and vote types

class AbstractVote(models.Model):
    '''
    The AbstractVote model holds some key components of a vote to a specific question
    '''
    time = models.DateTimeField(_(u'timestamp'), auto_now_add=True)
    voter = models.ForeignKey(RegisteredVoter, help_text=_(u'Bruker'), null=False)
    question = models.ForeignKey(Question, null=False)

    # Get voter name
    def get_voter_name(self):
        if self.question.anonymous:
            return voter.hide_user()
        else:
            return u'%s %s' %(self.voter.user.first_name, self.voter.user.last_name)

class BooleanVote(AbstractVote):
    '''
    The BooleanVote model holds the yes/no/blank answer to a specific question held in superclass
    '''
    answer = models.NullBooleanField(_(u'answer'), help_text=_(u'Ja/Nei'), null=True, blank=True)

class Alternative(models.Model):
    '''
    The Alternative class represents a single alternative that is connected to a particular multiple choice type question
    '''
    alt_id = models.PositiveIntegerField(null=False, help_text=_(u'Alternativ ID'), blank=False)
    question = models.ForeignKey(Question, null=False, help_text=_(u'Question'))
    description = models.CharField(_(u'Beskrivelse'), null=True, blank=True, max_length=150)

    def __unicode__(self):
        return self.description

class MultipleChoice(AbstractVote):
    '''
    The MultipleChoice model holds the answered alternative to a specific question held in superclass
    '''
    answer = models.ForeignKey(Alternative, null=True, blank=True, help_text=_(u'Alternativ'))

<|endoftext|>