<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
from django import forms
from rapidsms.models import Contact, Connection
from django.core.paginator import Paginator, Page
from django.contrib.auth.models import Group
from django.db.models import Q
from rapidsms_httprouter.router import get_router, \
    start_sending_mass_messages, stop_sending_mass_messages
from rapidsms_httprouter.models import Message
from rapidsms.messages.outgoing import OutgoingMessage
from generic.forms import ActionForm, FilterForm
from contact.models import MassText, Flag
from django.contrib.sites.models import Site
from rapidsms.contrib.locations.models import Location
from uganda_common.forms import SMSInput
from django.conf import settings
import datetime
from rapidsms_httprouter.models import Message
from django.forms.util import ErrorList

class FlaggedMessageForm(forms.ModelForm):
    class Meta:
        model = Flag

class ReplyForm(forms.Form):
    recipient = forms.CharField(max_length=20)
    message = forms.CharField(max_length=160, widget=forms.TextInput(attrs={'size':'60'}))
    in_response_to = forms.ModelChoiceField(queryset=Message.objects.filter(direction='I'), widget=forms.HiddenInput())

    def clean(self):
        cleaned_data = self.cleaned_data
        text = cleaned_data.get('message')

        #replace common MS-word characters with SMS-friendly characters
        for find, replace in [(u'\u201c', '"'),
                              (u'\u201d', '"'),
                              (u'\u201f', '"'),
                              (u'\u2018', "'"),
                              (u'\u2019', "'"),
                              (u'\u201B', "'"),
                              (u'\u2013', "-"),
                              (u'\u2014', "-"),
                              (u'\u2015', "-"),
                              (u'\xa7', "$"),
                              (u'\xa1', "i"),
                              (u'\xa4', ''),
                              (u'\xc4', 'A')]:
            cleaned_data['message'] = text.replace(find, replace)
            cleaned_data['message'] = text.replace('%', '%%')
        return cleaned_data

class FilterGroupsForm(FilterForm):

    """ concrete implementation of filter form """
    # This may seem like a hack, but this allows time for the Contact model's
    # default manage to be replaced at run-time.  There are many applications
    # for that, such as filtering contacts by site_id (as is done in the
    # authsites app, see github.com/daveycrockett/authsites).
    # This does, however, also make the polling app independent of authsites.
    def __init__(self, data=None, **kwargs):
        self.request = kwargs.pop('request')
        if data:
            forms.Form.__init__(self, data, **kwargs)
        else:
            forms.Form.__init__(self, **kwargs)
        if hasattr(Contact, 'groups'):
            if self.request.user.is_authenticated():
                if self.request.user.groups.order_by('-pk') == Group.objects.order_by('-pk'):
                    choices = ((-1, 'No Group'),) + tuple([(int(g.pk), g.name) for g in Group.objects.all().order_by('name')])
                    self.fields['groups'] = forms.MultipleChoiceField(choices=choices, required=True)
                else:
                    self.fields['groups'] = forms.ModelMultipleChoiceField(queryset=Group.objects.filter(pk__in=self.request.user.groups.values_list('pk', flat=True)), required=True)
            else:
                choices = ((-1, 'No Group'),) + tuple([(int(g.pk), g.name) for g in Group.objects.all().order_by('name')])
                self.fields['groups'] = forms.MultipleChoiceField(choices=choices, required=True)

    def filter(self, request, queryset):
        groups_pk = self.cleaned_data['groups']
        if '-1' in groups_pk:
            groups_pk.remove('-1')
            if len(groups_pk):
                return queryset.filter(Q(groups=None) | Q(groups__in=groups_pk))
            else:
                return queryset.filter(groups=None)
        else:
            return queryset.filter(groups__in=groups_pk)

class NewContactForm(forms.ModelForm):

    class Meta:
        model = Contact

class FreeSearchForm(FilterForm):

    """ concrete implementation of filter form
        TO DO: add ability to search for multiple search terms separated by 'or'
    """

    search = forms.CharField(max_length=100, required=False, label="Free-form search",
                             help_text="Use 'or' to search for multiple names")

    def filter(self, request, queryset):
        search = self.cleaned_data['search']
        if search == "":
            return queryset
        else:
            return queryset.filter(Q(name__icontains=search)
                                   | Q(reporting_location__name__icontains=search)
                                   | Q(connection__identity__icontains=search))

class FreeSearchTextForm(FilterForm):

    """ concrete implementation of filter form """

    search = forms.CharField(max_length=100, required=True, label="Free-form search", help_text="Use 'or' to search for multiple names")

    def filter(self, request, queryset):
        search = self.cleaned_data['search']
        return queryset.filter(text__icontains=search)

class HandledByForm(FilterForm):
    type = forms.ChoiceField(
            choices=(('', '-----'), ('poll', 'Poll Response'), ('rapidsms_xforms', 'Report'), ('*', 'Other'),), \
            required=False)

    def filter(self, request, queryset):
        handled_by = self.cleaned_data['type']
        if handled_by == '':
            return queryset
        elif handled_by == '*':
            return queryset.filter(application=None)
        else:
            return queryset.filter(application=handled_by)

class DistictFilterForm(FilterForm):

    """ filter cvs districs on their districts """

    district = forms.ChoiceField(choices=(('', '-----'), (-1,
                                 'No District')) + tuple([(int(d.pk),
                                 d.name) for d in
                                 Location.objects.filter(type__slug='district'
                                 ).order_by('name')]), required=False)

    def filter(self, request, queryset):
        district_pk = self.cleaned_data['district']
        if district_pk == '':
            return queryset
        elif int(district_pk) == -1:
            return queryset.filter(reporting_location=None)
        else:

            try:
                district = Location.objects.get(pk=district_pk)
            except Location.DoesNotExist:
                district = None
            if district:
                return queryset.filter(reporting_location__in=district.get_descendants(include_self=True))
            else:
                return queryset

class MultipleDistictFilterForm(FilterForm):

    districts = forms.ModelMultipleChoiceField(queryset=
                                 Location.objects.filter(type__slug='district'
                                 ).order_by('name'), required=False)

    def filter(self, request, queryset):
        districts = self.cleaned_data['districts']
        return queryset.filter(reporting_location__in=districts)

class DistictFilterMessageForm(FilterForm):

    """ filter cvs districs on their districts """
    district = forms.ChoiceField(choices=(('', '-----'), (-1,
                                 'No District')) + tuple([(int(d.pk),
                                 d.name) for d in
                                 Location.objects.filter(type__slug='district'
                                 ).order_by('name')]), required=False)

    def filter(self, request, queryset):
        district_pk = self.cleaned_data['district']
        if district_pk == '':
            return queryset
        elif int(district_pk) == -1:
            return queryset.filter(Q(connection__contact=None) | Q(connection__contact__reporting_location=None))
        else:
            try:
                district = Location.objects.get(pk=district_pk)
            except Location.DoesNotExist:
                district = None
            if district:
                return queryset.filter(connection__contact__reporting_location__in=district.get_descendants(include_self=True))
            else:
                return queryset

class MassTextForm(ActionForm):

    text = forms.CharField(max_length=160, required=True, widget=SMSInput())
    action_label = 'Send Message'

    def clean_text(self):
        text = self.cleaned_data['text']

        #replace common MS-word characters with SMS-friendly characters
        for find, replace in [(u'\u201c', '"'),
                              (u'\u201d', '"'),
                              (u'\u201f', '"'),
                              (u'\u2018', "'"),
                              (u'\u2019', "'"),
                              (u'\u201B', "'"),
                              (u'\u2013', "-"),
                              (u'\u2014', "-"),
                              (u'\u2015', "-"),
                              (u'\xa7', "$"),
                              (u'\xa1', "i"),
                              (u'\xa4', ''),
                              (u'\xc4', 'A')]:
            text = text.replace(find, replace)
        return text

    def perform(self, request, results):
        if results is None or len(results) == 0:
            return ('A message must have one or more recipients!', 'error')

        if request.user and request.user.has_perm('contact.can_message'):
            connections = \
                list(Connection.objects.filter(contact__in=results).distinct())

            text = self.cleaned_data.get('text', "")
            text = text.replace('%', u'\u0025')

            messages = Message.mass_text(text, connections)

            MassText.bulk.bulk_insert(send_pre_save=False,
                    user=request.user,
                    text=text,
                    contacts=list(results))
            masstexts = MassText.bulk.bulk_insert_commit(send_post_save=False, autoclobber=True)
            masstext = masstexts[0]
            if settings.SITE_ID:
                masstext.sites.add(Site.objects.get_current())

            return ('Message successfully sent to %d numbers' % len(connections), 'success',)
        else:
            return ("You don't have permission to send messages!", 'error',)

class ReplyTextForm(ActionForm):

    text = forms.CharField(required=True, widget=SMSInput())
    action_label = 'Reply to selected'

    def perform(self, request, results):
        if results is None or len(results) == 0:
            return ('A message must have one or more recipients!', 'error')

        if request.user and request.user.has_perm('contact.can_message'):
            router = get_router()
            text = self.cleaned_data['text']
            start_sending_mass_messages()
            for msg in results:
                outgoing = OutgoingMessage(msg.connection, text)
                router.handle_outgoing(outgoing, msg)
            stop_sending_mass_messages()
            return ('%d messages sent successfully' % results.count(), 'success',)
        else:
            return ("You don't have permission to send messages!", 'error',)

class AssignGroupForm(ActionForm):

    action_label = 'Assign to group(s)'

    # This may seem like a hack, but this allows time for the Contact model's
    # default manage to be replaced at run-time.  There are many applications
    # for that, such as filtering contacts by site_id (as is done in the
    # authsites app, see github.com/daveycrockett/authsites).
    # This does, however, also make the polling app independent of authsites.
    def __init__(self, data=None, **kwargs):
        self.request = kwargs.pop('request')
        if data:
            forms.Form.__init__(self, data, **kwargs)
        else:
            forms.Form.__init__(self, **kwargs)
        if hasattr(Contact, 'groups'):
            if self.request.user.is_authenticated():
                self.fields['groups'] = forms.ModelMultipleChoiceField(queryset=Group.objects.filter(pk__in=self.request.user.groups.values_list('pk', flat=True)), required=False)
            else:
                self.fields['groups'] = forms.ModelMultipleChoiceField(queryset=Group.objects.all(), required=False)

    def perform(self, request, results):
        groups = self.cleaned_data['groups']
        for c in results:
            for g in groups:
                c.groups.add(g)
        return ('%d Contacts assigned to %d groups.' % (len(results), len(groups)), 'success',)

class FlaggedForm(FilterForm):

    """ filter flagged/unflagged messages form """

    flagged = forms.ChoiceField(choices=(('', '-----'), (1, 'Flagged'), (0, 'Not flagged'),))

    def filter(self, request, queryset):
        flagged = self.cleaned_data['flagged']
        if flagged == '':
            return queryset
        elif int(flagged) == 1:
            return queryset.exclude(flags=None)
        else:
            return queryset.filter(flags=None)

class FlagMessageForm(ActionForm):

    """ flag/unflag messages action form """

    flag = forms.ChoiceField(choices=(('', '-----'), ('flag', 'Flag'), ('unflag', 'Unflag'),))
    action_label = 'Flag/Unflag selected'

    def perform(self, request, results):
        if results is None or len(results) == 0:
            return ('You must select one or more messages to Flag or Unflag them!', 'error')
        flag = self.cleaned_data['flag']
        for msg in results:
            if flag == 'flag':
                msg.flags.create()
            else:
                for msg_flag in msg.flags.all():
                    msg_flag.delete()
        return ('%d message(s) have been %sed' % (len(results), flag), 'successfully!',)

class GenderFilterForm(FilterForm):
    """ filter contacts by their gender"""

    gender = forms.ChoiceField(choices=(('', '-----'), ('M', 'Male'), ('F', 'Female'), ('None', 'N/A')))

    def filter(self, request, queryset):

        gender = self.cleaned_data['gender']
        if gender == '':
            return queryset
        elif gender == 'M':
            return queryset.filter(gender='M')
        elif gender == 'F':
            return queryset.filter(gender='F')
        else:
            return queryset.filter(gender=None)

class AgeFilterForm(FilterForm):
    """ filter contacts by their age """
    flag = forms.ChoiceField(label='' , choices=(('', '-----'), ('==', 'Equal to'), ('>', 'Greater than'), ('<', \
                                        'Less than'), ('None', 'N/A')), required=False)
    age = forms.CharField(max_length=20, label="Age", widget=forms.TextInput(attrs={'size':'20'}), required=False)
    def filter(self, request, queryset):

        flag = self.cleaned_data['flag']
        try:
            age = int(self.cleaned_data['age'])
            end = datetime.datetime.now()
            start = end - datetime.timedelta(days=age * 365)
        except ValueError:
            age = None
            start = None
            end = None

        if flag == '':
            return queryset
        elif flag == '==':
            return queryset.exclude(birthdate=None).filter(birthdate__year=start.year)
        elif flag == '>':
            return queryset.exclude(birthdate=None).exclude(birthdate__range=(start, end))
        elif flag == "<":
            return queryset.exclude(birthdate=None).filter(birthdate__range=(start, end))
        else:
            return queryset.filter(birthdate=None)

<commit_msg>fix search<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
from django import forms
from rapidsms.models import Contact, Connection
from django.core.paginator import Paginator, Page
from django.contrib.auth.models import Group
from django.db.models import Q
from rapidsms_httprouter.router import get_router, \
    start_sending_mass_messages, stop_sending_mass_messages
from rapidsms_httprouter.models import Message
from rapidsms.messages.outgoing import OutgoingMessage
from generic.forms import ActionForm, FilterForm
from contact.models import MassText, Flag
from django.contrib.sites.models import Site
from rapidsms.contrib.locations.models import Location
from uganda_common.forms import SMSInput
from django.conf import settings
import datetime
from rapidsms_httprouter.models import Message
from django.forms.util import ErrorList

class FlaggedMessageForm(forms.ModelForm):
    class Meta:
        model = Flag

class ReplyForm(forms.Form):
    recipient = forms.CharField(max_length=20)
    message = forms.CharField(max_length=160, widget=forms.TextInput(attrs={'size':'60'}))
    in_response_to = forms.ModelChoiceField(queryset=Message.objects.filter(direction='I'), widget=forms.HiddenInput())

    def clean(self):
        cleaned_data = self.cleaned_data
        text = cleaned_data.get('message')

        #replace common MS-word characters with SMS-friendly characters
        for find, replace in [(u'\u201c', '"'),
                              (u'\u201d', '"'),
                              (u'\u201f', '"'),
                              (u'\u2018', "'"),
                              (u'\u2019', "'"),
                              (u'\u201B', "'"),
                              (u'\u2013', "-"),
                              (u'\u2014', "-"),
                              (u'\u2015', "-"),
                              (u'\xa7', "$"),
                              (u'\xa1', "i"),
                              (u'\xa4', ''),
                              (u'\xc4', 'A')]:
            cleaned_data['message'] = text.replace(find, replace)
            cleaned_data['message'] = text.replace('%', '%%')
        return cleaned_data

class FilterGroupsForm(FilterForm):

    """ concrete implementation of filter form """
    # This may seem like a hack, but this allows time for the Contact model's
    # default manage to be replaced at run-time.  There are many applications
    # for that, such as filtering contacts by site_id (as is done in the
    # authsites app, see github.com/daveycrockett/authsites).
    # This does, however, also make the polling app independent of authsites.
    def __init__(self, data=None, **kwargs):
        self.request = kwargs.pop('request')
        if data:
            forms.Form.__init__(self, data, **kwargs)
        else:
            forms.Form.__init__(self, **kwargs)
        if hasattr(Contact, 'groups'):
            if self.request.user.is_authenticated():
                if self.request.user.groups.order_by('-pk') == Group.objects.order_by('-pk'):
                    choices = ((-1, 'No Group'),) + tuple([(int(g.pk), g.name) for g in Group.objects.all().order_by('name')])
                    self.fields['groups'] = forms.MultipleChoiceField(choices=choices, required=True)
                else:
                    self.fields['groups'] = forms.ModelMultipleChoiceField(queryset=Group.objects.filter(pk__in=self.request.user.groups.values_list('pk', flat=True)), required=True)
            else:
                choices = ((-1, 'No Group'),) + tuple([(int(g.pk), g.name) for g in Group.objects.all().order_by('name')])
                self.fields['groups'] = forms.MultipleChoiceField(choices=choices, required=True)

    def filter(self, request, queryset):
        groups_pk = self.cleaned_data['groups']
        if '-1' in groups_pk:
            groups_pk.remove('-1')
            if len(groups_pk):
                return queryset.filter(Q(groups=None) | Q(groups__in=groups_pk))
            else:
                return queryset.filter(groups=None)
        else:
            return queryset.filter(groups__in=groups_pk)

class NewContactForm(forms.ModelForm):

    class Meta:
        model = Contact

class FreeSearchForm(FilterForm):

    """ concrete implementation of filter form
        TO DO: add ability to search for multiple search terms separated by 'or'
    """

    search = forms.CharField(max_length=100, required=False, label="Free-form search",
                             help_text="Use 'or' to search for multiple names")

    def filter(self, request, queryset):
        search = self.cleaned_data['search'].strip()
        if search == "":
            return queryset
        else:
            return queryset.filter(Q(name__icontains=search)
                                   | Q(reporting_location__name__icontains=search)
                                   | Q(connection__identity__icontains=search))

class FreeSearchTextForm(FilterForm):

    """ concrete implementation of filter form """

    search = forms.CharField(max_length=100, required=True, label="Free-form search", help_text="Use 'or' to search for multiple names")

    def filter(self, request, queryset):
        search = self.cleaned_data['search']
        return queryset.filter(text__icontains=search)

class HandledByForm(FilterForm):
    type = forms.ChoiceField(
            choices=(('', '-----'), ('poll', 'Poll Response'), ('rapidsms_xforms', 'Report'), ('*', 'Other'),), \
            required=False)

    def filter(self, request, queryset):
        handled_by = self.cleaned_data['type']
        if handled_by == '':
            return queryset
        elif handled_by == '*':
            return queryset.filter(application=None)
        else:
            return queryset.filter(application=handled_by)

class DistictFilterForm(FilterForm):

    """ filter cvs districs on their districts """

    district = forms.ChoiceField(choices=(('', '-----'), (-1,
                                 'No District')) + tuple([(int(d.pk),
                                 d.name) for d in
                                 Location.objects.filter(type__slug='district'
                                 ).order_by('name')]), required=False)

    def filter(self, request, queryset):
        district_pk = self.cleaned_data['district']
        if district_pk == '':
            return queryset
        elif int(district_pk) == -1:
            return queryset.filter(reporting_location=None)
        else:

            try:
                district = Location.objects.get(pk=district_pk)
            except Location.DoesNotExist:
                district = None
            if district:
                return queryset.filter(reporting_location__in=district.get_descendants(include_self=True))
            else:
                return queryset

class MultipleDistictFilterForm(FilterForm):

    districts = forms.ModelMultipleChoiceField(queryset=
                                 Location.objects.filter(type__slug='district'
                                 ).order_by('name'), required=False)

    def filter(self, request, queryset):
        districts = self.cleaned_data['districts']
        if len(districts):
            return queryset.filter(reporting_location__in=districts)
        else:
            return queryset

class DistictFilterMessageForm(FilterForm):

    """ filter cvs districs on their districts """
    district = forms.ChoiceField(choices=(('', '-----'), (-1,
                                 'No District')) + tuple([(int(d.pk),
                                 d.name) for d in
                                 Location.objects.filter(type__slug='district'
                                 ).order_by('name')]), required=False)

    def filter(self, request, queryset):
        district_pk = self.cleaned_data['district']
        if district_pk == '':
            return queryset
        elif int(district_pk) == -1:
            return queryset.filter(Q(connection__contact=None) | Q(connection__contact__reporting_location=None))
        else:
            try:
                district = Location.objects.get(pk=district_pk)
            except Location.DoesNotExist:
                district = None
            if district:
                return queryset.filter(connection__contact__reporting_location__in=district.get_descendants(include_self=True))
            else:
                return queryset

class MassTextForm(ActionForm):

    text = forms.CharField(max_length=160, required=True, widget=SMSInput())
    action_label = 'Send Message'

    def clean_text(self):
        text = self.cleaned_data['text']

        #replace common MS-word characters with SMS-friendly characters
        for find, replace in [(u'\u201c', '"'),
                              (u'\u201d', '"'),
                              (u'\u201f', '"'),
                              (u'\u2018', "'"),
                              (u'\u2019', "'"),
                              (u'\u201B', "'"),
                              (u'\u2013', "-"),
                              (u'\u2014', "-"),
                              (u'\u2015', "-"),
                              (u'\xa7', "$"),
                              (u'\xa1', "i"),
                              (u'\xa4', ''),
                              (u'\xc4', 'A')]:
            text = text.replace(find, replace)
        return text

    def perform(self, request, results):
        if results is None or len(results) == 0:
            return ('A message must have one or more recipients!', 'error')

        if request.user and request.user.has_perm('contact.can_message'):
            connections = \
                list(Connection.objects.filter(contact__in=results).distinct())

            text = self.cleaned_data.get('text', "")
            text = text.replace('%', u'\u0025')

            messages = Message.mass_text(text, connections)

            MassText.bulk.bulk_insert(send_pre_save=False,
                    user=request.user,
                    text=text,
                    contacts=list(results))
            masstexts = MassText.bulk.bulk_insert_commit(send_post_save=False, autoclobber=True)
            masstext = masstexts[0]
            if settings.SITE_ID:
                masstext.sites.add(Site.objects.get_current())

            return ('Message successfully sent to %d numbers' % len(connections), 'success',)
        else:
            return ("You don't have permission to send messages!", 'error',)

class ReplyTextForm(ActionForm):

    text = forms.CharField(required=True, widget=SMSInput())
    action_label = 'Reply to selected'

    def perform(self, request, results):
        if results is None or len(results) == 0:
            return ('A message must have one or more recipients!', 'error')

        if request.user and request.user.has_perm('contact.can_message'):
            router = get_router()
            text = self.cleaned_data['text']
            start_sending_mass_messages()
            for msg in results:
                outgoing = OutgoingMessage(msg.connection, text)
                router.handle_outgoing(outgoing, msg)
            stop_sending_mass_messages()
            return ('%d messages sent successfully' % results.count(), 'success',)
        else:
            return ("You don't have permission to send messages!", 'error',)

class AssignGroupForm(ActionForm):

    action_label = 'Assign to group(s)'

    # This may seem like a hack, but this allows time for the Contact model's
    # default manage to be replaced at run-time.  There are many applications
    # for that, such as filtering contacts by site_id (as is done in the
    # authsites app, see github.com/daveycrockett/authsites).
    # This does, however, also make the polling app independent of authsites.
    def __init__(self, data=None, **kwargs):
        self.request = kwargs.pop('request')
        if data:
            forms.Form.__init__(self, data, **kwargs)
        else:
            forms.Form.__init__(self, **kwargs)
        if hasattr(Contact, 'groups'):
            if self.request.user.is_authenticated():
                self.fields['groups'] = forms.ModelMultipleChoiceField(queryset=Group.objects.filter(pk__in=self.request.user.groups.values_list('pk', flat=True)), required=False)
            else:
                self.fields['groups'] = forms.ModelMultipleChoiceField(queryset=Group.objects.all(), required=False)

    def perform(self, request, results):
        groups = self.cleaned_data['groups']
        for c in results:
            for g in groups:
                c.groups.add(g)
        return ('%d Contacts assigned to %d groups.' % (len(results), len(groups)), 'success',)

class FlaggedForm(FilterForm):

    """ filter flagged/unflagged messages form """

    flagged = forms.ChoiceField(choices=(('', '-----'), (1, 'Flagged'), (0, 'Not flagged'),))

    def filter(self, request, queryset):
        flagged = self.cleaned_data['flagged']
        if flagged == '':
            return queryset
        elif int(flagged) == 1:
            return queryset.exclude(flags=None)
        else:
            return queryset.filter(flags=None)

class FlagMessageForm(ActionForm):

    """ flag/unflag messages action form """

    flag = forms.ChoiceField(choices=(('', '-----'), ('flag', 'Flag'), ('unflag', 'Unflag'),))
    action_label = 'Flag/Unflag selected'

    def perform(self, request, results):
        if results is None or len(results) == 0:
            return ('You must select one or more messages to Flag or Unflag them!', 'error')
        flag = self.cleaned_data['flag']
        for msg in results:
            if flag == 'flag':
                msg.flags.create()
            else:
                for msg_flag in msg.flags.all():
                    msg_flag.delete()
        return ('%d message(s) have been %sed' % (len(results), flag), 'successfully!',)

class GenderFilterForm(FilterForm):
    """ filter contacts by their gender"""

    gender = forms.ChoiceField(choices=(('', '-----'), ('M', 'Male'), ('F', 'Female'), ('None', 'N/A')))

    def filter(self, request, queryset):

        gender = self.cleaned_data['gender']
        if gender == '':
            return queryset
        elif gender == 'M':
            return queryset.filter(gender='M')
        elif gender == 'F':
            return queryset.filter(gender='F')
        else:
            return queryset.filter(gender=None)

class AgeFilterForm(FilterForm):
    """ filter contacts by their age """
    flag = forms.ChoiceField(label='' , choices=(('', '-----'), ('==', 'Equal to'), ('>', 'Greater than'), ('<', \
                                        'Less than'), ('None', 'N/A')), required=False)
    age = forms.CharField(max_length=20, label="Age", widget=forms.TextInput(attrs={'size':'20'}), required=False)
    def filter(self, request, queryset):

        flag = self.cleaned_data['flag']
        try:
            age = int(self.cleaned_data['age'])
            end = datetime.datetime.now()
            start = end - datetime.timedelta(days=age * 365)
        except ValueError:
            age = None
            start = None
            end = None

        if flag == '':
            return queryset
        elif flag == '==':
            return queryset.exclude(birthdate=None).filter(birthdate__year=start.year)
        elif flag == '>':
            return queryset.exclude(birthdate=None).exclude(birthdate__range=(start, end))
        elif flag == "<":
            return queryset.exclude(birthdate=None).filter(birthdate__range=(start, end))
        else:
            return queryset.filter(birthdate=None)

<|endoftext|>


<commit_before>"""Utilities for handling DWI images."""

from time import time
import os
import numpy as np

import fit
import util

def load(filename, nrois=1, varname='ROIdata'):
    """Load images from a file."""
    root, ext = os.path.splitext(filename)
    if ext == '.mat':
        return load_matlab(filename, varname)
    else:
        return load_ascii(filename, nrois)

def load_matlab(filename, varname='ROIdata'):
    """Load images from a MATLAB file."""
    import scipy.io
    mat = scipy.io.loadmat(filename, struct_as_record=False)
    r = []
    for window in mat[varname][0]:
        win = window[0,0]
        sis = win.SIs.T
        bset = win.bset[0]
        dwi = DWImage(sis, bset)
        dwi.filename = filename
        dwi.roislice = '-' # Not implemented.
        dwi.name = '-' # Not implemented.
        dwi.number = int(win.number[0,0])
        try:
            dwi.subwindow = tuple(map(int, win.subwindow[0]))
        except:
            dwi.subwindow = util.fabricate_subwindow(len(sis))
        r.append(dwi)
    return r

def load_ascii(filename, nrois=1):
    """Load images from an ASCII file."""
    import asciifile
    af = asciifile.AsciiFile(filename)
    a = af.a.reshape(nrois,-1,af.a.shape[-1])
    r = []
    for i in range(nrois):
        sis = a[i]
        bset = af.bset()
        dwi = DWImage(sis, bset)
        dwi.filename = filename
        dwi.roislice = af.roislice()
        dwi.name = af.name()
        dwi.number = af.number + i
        dwi.subwindow = af.subwindow()
        r.append(dwi)
    return r

def load_dicom(filenames):
    """Load a 3d image from DICOM files with slices combined.

    If only one filename is given, it is assumed to be a directory.
    """
    import dicomfile
    if len(filenames) == 1:
        d = dicomfile.read_dir(filenames[0]) # Directory.
    else:
        d = dicomfile.read_files(filenames) # File list.
    bset = d['bvalues']
    image = d['image']
    dwi = DWImage(image, bset)
    dwi.filename = os.path.abspath(filenames[0])
    dwi.roislice = '-'
    dwi.name = '-'
    dwi.number = 0
    dwi.subwindow = (0, image.shape[0], 0, image.shape[1], 0, image.shape[2])
    return [dwi]

class DWImage(object):
    """DWI image, a single slice of signal intensities at several b-values.

    Variables
    ---------
    image : ndarray, shape = [depth, height, width, n_bvalues]
        Array of voxels.
    sis : ndarray, shape = [depth * height * width, n_bvalues]
        Flattened view of the image.
    bset : ndarray, shape = [n_bvalues]
        Different b-values.
    """

    def __init__(self, image, bset):
        """Create a new DWI image.

        Parameters
        ----------
        image : array_like, shape = [((depth,) height,) width, n_bvalues]
            Array of signal intensities at different b-values.
        bset : sequence
            Different b-values.
        """
        self.image = np.array(image, dtype=float, ndmin=4)
        self.sis = self.image.view()
        self.sis.shape = (-1,self.image.shape[-1])
        self.bset = np.array(sorted(set(bset)), dtype=float)
        self.start_time = self.end_time = -1
        if len(self.image.shape) != 4:
            raise Exception('Invalid image dimensions.')
        if not self.image.shape[-1] == self.sis.shape[-1] == len(self.bset):
            raise Exception('Image size does not match with b-values.')

    def __repr__(self):
        return '%s:%i' % (self.filename, self.number)

    def __str__(self):
        d = dict(fn=self.filename, n=self.number,
                nb=len(self.bset), b=list(self.bset),
                size=self.size(), shape=self.shape(),
                w=self.subwindow, ws=self.subwindow_shape())
        s = 'File: {fn}\n'\
                'Number: {n}\n'\
                'B-values: {nb}: {b}\n'\
                'Voxels: {size}, {shape}\n'\
                'Window: {w}, {ws}'.format(**d)
        return s

    def subwindow_shape(self):
        return tuple((b-a for a, b in util.chunks(self.subwindow, 2)))

    def shape(self):
        """Return image height and width."""
        return self.image.shape[0:-1]

    def size(self):
        """Return number of voxels."""
        return len(self.sis)

    def get_roi(self, position, bvalues=[], onebased=False):
        """Get a view of a specific ROI (region of interest)."""
        if onebased:
            position = [i-1 for i in position] # One-based indexing.
        z0, z1, y0, y1, x0, x1 = position
        if not bvalues:
            bvalues = range(len(self.bset))
        image = self.image[z0:z1,y0:y1,x0:x1,bvalues]
        bset = self.bset[bvalues]
        dwimage = DWImage(image, bset)
        dwimage.filename = self.filename
        dwimage.roislice = self.roislice
        dwimage.name = self.name
        dwimage.number = self.number
        dwimage.subwindow = (
                self.subwindow[0] + z0,
                self.subwindow[0] + z1,
                self.subwindow[2] + y0,
                self.subwindow[2] + y1,
                self.subwindow[4] + x0,
                self.subwindow[4] + x1)
        if onebased:
            dwimage.subwindow = tuple([i+1 for i in dwimage.subwindow])
        return dwimage

    def start_execution(self):
        """Start taking time for an operation."""
        self.start_time = time()

    def end_execution(self):
        """End taking time for an operation."""
        self.end_time = time()

    def execution_time(self):
        """Return time consumed by previous operation."""
        return self.end_time - self.start_time

    def fit(self, model):
        """Fit model to whole image.

        Parameters
        ----------
        model : fit.Model
            Model used for fitting.

        Returns
        -------
        pmap : ndarray
            Result parameters and RMSE.
        """
        self.start_execution()
        xdata = self.bset
        ydatas = self.sis
        pmap = model.fit(xdata, ydatas)
        self.end_execution()
        return pmap
<commit_msg>Store voxel spacing<commit_after>"""Utilities for handling DWI images."""

from time import time
import os
import numpy as np

import fit
import util

def load(filename, nrois=1, varname='ROIdata'):
    """Load images from a file."""
    root, ext = os.path.splitext(filename)
    if ext == '.mat':
        return load_matlab(filename, varname)
    else:
        return load_ascii(filename, nrois)

def load_matlab(filename, varname='ROIdata'):
    """Load images from a MATLAB file."""
    import scipy.io
    mat = scipy.io.loadmat(filename, struct_as_record=False)
    r = []
    for window in mat[varname][0]:
        win = window[0,0]
        sis = win.SIs.T
        bset = win.bset[0]
        dwi = DWImage(sis, bset)
        dwi.filename = filename
        dwi.roislice = '-' # Not implemented.
        dwi.name = '-' # Not implemented.
        dwi.number = int(win.number[0,0])
        try:
            dwi.subwindow = tuple(map(int, win.subwindow[0]))
        except:
            dwi.subwindow = util.fabricate_subwindow(len(sis))
        r.append(dwi)
    return r

def load_ascii(filename, nrois=1):
    """Load images from an ASCII file."""
    import asciifile
    af = asciifile.AsciiFile(filename)
    a = af.a.reshape(nrois,-1,af.a.shape[-1])
    r = []
    for i in range(nrois):
        sis = a[i]
        bset = af.bset()
        dwi = DWImage(sis, bset)
        dwi.filename = filename
        dwi.roislice = af.roislice()
        dwi.name = af.name()
        dwi.number = af.number + i
        dwi.subwindow = af.subwindow()
        r.append(dwi)
    return r

def load_dicom(filenames):
    """Load a 3d image from DICOM files with slices combined.

    If only one filename is given, it is assumed to be a directory.
    """
    import dicomfile
    if len(filenames) == 1:
        d = dicomfile.read_dir(filenames[0]) # Directory.
    else:
        d = dicomfile.read_files(filenames) # File list.
    bset = d['bvalues']
    image = d['image']
    dwi = DWImage(image, bset)
    dwi.filename = os.path.abspath(filenames[0])
    dwi.roislice = '-'
    dwi.name = '-'
    dwi.number = 0
    dwi.subwindow = (0, image.shape[0], 0, image.shape[1], 0, image.shape[2])
    dwi.voxel_spacing = d['voxel_spacing']
    return [dwi]

class DWImage(object):
    """DWI image, a single slice of signal intensities at several b-values.

    Variables
    ---------
    image : ndarray, shape = [depth, height, width, n_bvalues]
        Array of voxels.
    sis : ndarray, shape = [depth * height * width, n_bvalues]
        Flattened view of the image.
    bset : ndarray, shape = [n_bvalues]
        Different b-values.
    """

    def __init__(self, image, bset):
        """Create a new DWI image.

        Parameters
        ----------
        image : array_like, shape = [((depth,) height,) width, n_bvalues]
            Array of signal intensities at different b-values.
        bset : sequence
            Different b-values.
        """
        self.image = np.array(image, dtype=float, ndmin=4)
        self.sis = self.image.view()
        self.sis.shape = (-1,self.image.shape[-1])
        self.bset = np.array(sorted(set(bset)), dtype=float)
        self.start_time = self.end_time = -1
        if len(self.image.shape) != 4:
            raise Exception('Invalid image dimensions.')
        if not self.image.shape[-1] == self.sis.shape[-1] == len(self.bset):
            raise Exception('Image size does not match with b-values.')

    def __repr__(self):
        return '%s:%i' % (self.filename, self.number)

    def __str__(self):
        d = dict(fn=self.filename, n=self.number,
                nb=len(self.bset), b=list(self.bset),
                size=self.size(), shape=self.shape(),
                w=self.subwindow, ws=self.subwindow_shape())
        s = 'File: {fn}\n'\
                'Number: {n}\n'\
                'B-values: {nb}: {b}\n'\
                'Voxels: {size}, {shape}\n'\
                'Window: {w}, {ws}'.format(**d)
        return s

    def subwindow_shape(self):
        return tuple((b-a for a, b in util.chunks(self.subwindow, 2)))

    def shape(self):
        """Return image height and width."""
        return self.image.shape[0:-1]

    def size(self):
        """Return number of voxels."""
        return len(self.sis)

    def get_roi(self, position, bvalues=[], onebased=False):
        """Get a view of a specific ROI (region of interest)."""
        if onebased:
            position = [i-1 for i in position] # One-based indexing.
        z0, z1, y0, y1, x0, x1 = position
        if not bvalues:
            bvalues = range(len(self.bset))
        image = self.image[z0:z1,y0:y1,x0:x1,bvalues]
        bset = self.bset[bvalues]
        dwimage = DWImage(image, bset)
        dwimage.filename = self.filename
        dwimage.roislice = self.roislice
        dwimage.name = self.name
        dwimage.number = self.number
        dwimage.subwindow = (
                self.subwindow[0] + z0,
                self.subwindow[0] + z1,
                self.subwindow[2] + y0,
                self.subwindow[2] + y1,
                self.subwindow[4] + x0,
                self.subwindow[4] + x1)
        if onebased:
            dwimage.subwindow = tuple([i+1 for i in dwimage.subwindow])
        return dwimage

    def start_execution(self):
        """Start taking time for an operation."""
        self.start_time = time()

    def end_execution(self):
        """End taking time for an operation."""
        self.end_time = time()

    def execution_time(self):
        """Return time consumed by previous operation."""
        return self.end_time - self.start_time

    def fit(self, model):
        """Fit model to whole image.

        Parameters
        ----------
        model : fit.Model
            Model used for fitting.

        Returns
        -------
        pmap : ndarray
            Result parameters and RMSE.
        """
        self.start_execution()
        xdata = self.bset
        ydatas = self.sis
        pmap = model.fit(xdata, ydatas)
        self.end_execution()
        return pmap
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
import sys
import shutil
import subprocess

if len(sys.argv) != 2:
    print 'Usage: release.py version-number'
    sys.exit(1)

version = sys.argv[1]
name = 'goo-' + version

print 'Creating release', name
if os.path.isdir('out'):
    shutil.rmtree('out')

grunt_command = 'node_modules/grunt-cli/bin/grunt'
subprocess.check_call([grunt_command, 'minify', '--goo-version=' + version])

subprocess.check_call([grunt_command, 'jsdoc'])
subprocess.check_call([grunt_command, 'visualtoc'])
subprocess.check_call([grunt_command, 'examplestoc'])

release_dir = os.getenv('RELEASE_DIR', 'out/release/' + name)
if os.path.isdir(release_dir):
	print 'Release directory already exists:', release_dir
else:
	print 'Creating directory for release:', release_dir
	os.makedirs(release_dir)

os.makedirs(release_dir + '/lib')
for directory in (
	'lib/p2',
	'lib/box2d',
	'lib/cannon',
	'lib/soundmanager2',
	'lib/howler',
	'lib/crunch',
	'lib/hammerv2',
	('goojs-jsdoc', 'docs'),
	'visual-test',
	'examples'
):
	if isinstance(directory, basestring):
		source = destination = directory
	else:
		source, destination = directory
	shutil.copytree(source, release_dir + '/' + destination)

shutil.copy('out/goo.js', release_dir + '/lib/goo.js')

# pack files must also be copied
for packName in (
	'fsm',
	'geometry',
	'quad',
	'script',
	'timeline',
	'debug',
	'p2',
	'box2d',
	'terrain',
	'ammo',
	'cannon',
	'howler',
	'water',
	'animation',
	'soundmanager2',
	'gamepad',
	'pass',
	'gizmo'
):
    shutil.copy('out/' + packName + 'pack.js', release_dir + '/lib/' + packName + 'pack.js')

shutil.copy('lib/require.js', release_dir + '/lib/require.js')
shutil.copy('lib/ammo.small.js', release_dir + '/lib/ammo.small.js')
shutil.copy('lib/polyk.js', release_dir + '/lib/polyk.js')
shutil.copy('COPYING', release_dir + '/COPYING')
shutil.copy('LICENSE', release_dir + '/LICENSE')
shutil.copy('CHANGES', release_dir + '/CHANGES')
<commit_msg>Added execute permissions to release.py; fake change to be able to commit<commit_after>#!/usr/bin/env python

import os
import sys
import shutil
import subprocess

if len(sys.argv) != 2:
    print 'Usage: release.py version-number'
    sys.exit(1)

version = sys.argv[1]
name = 'goo-' + version

print 'Creating release', name
if os.path.isdir('out'):
    shutil.rmtree('out')

grunt_command = 'node_modules/grunt-cli/bin/grunt'
subprocess.check_call([grunt_command, 'minify', '--goo-version=' + version])

subprocess.check_call([grunt_command, 'jsdoc'])
subprocess.check_call([grunt_command, 'visualtoc'])
subprocess.check_call([grunt_command, 'examplestoc'])

release_dir = os.getenv('RELEASE_DIR', 'out/release/' + name)
if os.path.isdir(release_dir):
	print 'Release directory already exists:', release_dir
else:
	print 'Creating directory for release:', release_dir
	os.makedirs(release_dir)

os.makedirs(release_dir + '/lib')
for directory in (
	'lib/p2',
	'lib/box2d',
	'lib/cannon',
	'lib/soundmanager2',
	'lib/howler',
	'lib/crunch',
	'lib/hammerv2',
	('goojs-jsdoc', 'docs'),
	'visual-test',
	'examples'
):
	if isinstance(directory, basestring):
		source = destination = directory
	else:
		source, destination = directory
	shutil.copytree(source, release_dir + '/' + destination)

shutil.copy('out/goo.js', release_dir + '/lib/goo.js')

# pack files must also be copied
for packName in (
	'fsm',
	'geometry',
	'quad',
	'script',
	'timeline',
	'debug',
	'p2',
	'box2d',
	'terrain',
	'ammo',
	'cannon',
	'howler',
	'water',
	'animation',
	'soundmanager2',
	'gamepad',
	'pass',
	'gizmo'
):
    shutil.copy('out/' + packName + 'pack.js', release_dir + '/lib/' + packName + 'pack.js')

shutil.copy('lib/require.js', release_dir + '/lib/require.js')
shutil.copy('lib/ammo.small.js', release_dir + '/lib/ammo.small.js')
shutil.copy('lib/polyk.js', release_dir + '/lib/polyk.js')
shutil.copy('COPYING', release_dir + '/COPYING')
shutil.copy('LICENSE', release_dir + '/LICENSE')
shutil.copy('CHANGES', release_dir + '/CHANGES')

#<|endoftext|>


<commit_before> #
 # Copyright 2004 Apache Software Foundation
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You
 # may obtain a copy of the License at
 #
 #      http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 # implied.  See the License for the specific language governing
 # permissions and limitations under the License.
 #
 # Originally developed by Gregory Trubetskoy.
 #
 # This was donated by Nicolas Lehuen, and also posted to the Python Cookbook
 # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/302997
 #
 # $Id$

# -*- coding: CP1252 -*-
from threading import Lock
from os import fstat
from time import time, strptime
from calendar import timegm
import urllib2
import re
import weakref

NOT_INITIALIZED = object()

class Entry(object):
    """ A cache entry, mostly an internal object. """
    def __init__(self, key):
        object.__init__(self)
        self._key=key
        self._value=NOT_INITIALIZED
        self._lock=Lock()

class Cache(object):
    """ An abstract, multi-threaded cache object. """

    def __init__(self, max_size=0):
        """ Builds a cache with a limit of max_size entries.
            If this limit is exceeded, the Least Recently Used entry is discarded.
            if max_size==0, the cache is unbounded (no LRU rule is applied).
        """
        object.__init__(self)
        self._maxsize=max_size
        self._dict={}
        self._lock=Lock()

        # Header of the access list
        if self._maxsize:
            self._head=Entry(None)
            self._head._previous=self._head
            self._head._next=self._head

    def __setitem__(self, name, value):
        """ Populates the cache with a given name and value. """
        self._lock.acquire()
        try:
            key = self.key(name)
            entry = self._dict.get(key)
            if not entry:
                entry = Entry(key)
                self._pack(entry, value)
                self._dict[key]=entry
                if self._maxsize:
                    entry._next = entry._previous = None
                    self._access(entry)
                    self._checklru()
            else:
                self._pack(entry, value)
                if self._maxsize:
                    self._access(entry)
            self.commit()
        finally:
            self._lock.release()

    def __getitem__(self, name):
        """ Gets a value from the cache, builds it if required. """
        self._lock.acquire()
        try:
            key = self.key(name)
            entry = self._dict.get(key)
            if not entry:
                entry = Entry(key)
                self._dict[key]=entry
                if self._maxsize:
                    entry._next = entry._previous = None
                    self._access(entry)
                    self._checklru()
            elif self._maxsize:
                self._access(entry)
        finally:
            self._lock.release()

        entry._lock.acquire()
        try:
            value = self._unpack(entry)
            if value is NOT_INITIALIZED:
                opened = self.check(key, name, entry)
                value = self.build(key, name, opened, entry)
                self._pack(entry, value)
                self.commit()
            else:
                opened = self.check(key, name, entry)
                if opened is not None:
                    value = self.build(key, name, opened, entry)
                    self._pack(entry, value)
                    self.commit()
            return value
        finally:
            entry._lock.release()

    def __delitem__(self, key):
        self._lock.acquire()
        try:
            key = self.key(key)
            del self._dict[key]
        finally:
            self._lock.release()

    def mru(self):
        """ Returns the Most Recently Used key """
        if self._maxsize:
            self._lock.acquire()
            try:
                return self._head._previous._key
            finally:
                self._lock.release()
        else:
            return None

    def lru(self):
        """ Returns the Least Recently Used key """
        if self._maxsize:
            self._lock.acquire()
            try:
                return self._head._next._key
            finally:
                self._lock.release()
        else:
            return None

    def key(self, name):
        """ Override this method to extract a key from the name passed to the [] operator """
        return name

    def commit(self):
        """ Override this method if you want to do something each time the underlying dictionary is modified (e.g. make it persistent). """
        pass

    def clear(self):
        """ Clears the cache """
        self._lock.acquire()
        try:
            self._dict.clear()
            if self._maxsize:
                self._head._next=self._head
                self._head._previous=self._head
        finally:
            self._lock.release()

    def check(self, key, name, entry):
        """ Override this method to check whether the entry with the given name is stale. Return None if it is fresh
            or an opened resource if it is stale. The object returned will be passed to the 'build' method as the 'opened' parameter.
            Use the 'entry' parameter to store meta-data if required. Don't worry about multiple threads accessing the same name,
            as this method is properly isolated.
        """
        return None

    def build(self, key, name, opened, entry):
        """ Build the cached value with the given name from the given opened resource. Use entry to obtain or store meta-data if needed.
             Don't worry about multiple threads accessing the same name, as this method is properly isolated.
        """
        raise NotImplementedError()

    def _access(self, entry):
        " Internal use only, must be invoked within a cache lock. Updates the access list. """
        if entry._next is not self._head:
            if entry._previous is not None:
                # remove the entry from the access list
                entry._previous._next=entry._next
                entry._next._previous=entry._previous
            # insert the entry at the end of the access list
            entry._previous=self._head._previous
            entry._previous._next=entry
            entry._next=self._head
            entry._next._previous=entry
            if self._head._next is self._head:
                self._head._next=entry

    def _checklru(self):
        " Internal use only, must be invoked within a cache lock. Removes the LRU entry if needed. """
        if len(self._dict)>self._maxsize:
            lru=self._head._next
            lru._previous._next=lru._next
            lru._next._previous=lru._previous
            del self._dict[lru._key]

    def _pack(self, entry, value):
        """ Store the value in the entry. """
        entry._value=value

    def _unpack(self, entry):
        """ Recover the value from the entry, returns NOT_INITIALIZED if it is not OK. """
        return entry._value

class WeakCache(Cache):
    """ This cache holds weak references to the values it stores. Whenever a value is not longer
        normally referenced, it is removed from the cache. Useful for sharing the result of long
        computations but letting them go as soon as they are not needed by anybody.
    """

    def _pack(self, entry, value):
        entry._value=weakref.ref(value, lambda ref: self.__delitem__(entry._key))

    def _unpack(self, entry):
        if entry._value is NOT_INITIALIZED:
            return NOT_INITIALIZED

        value = entry._value()
        if value is None:
            return NOT_INITIALIZED
        else:
            return value

class FileCache(Cache):
    """ A file cache. Returns the content of the files as a string, given their filename.
        Whenever the files are modified (according to their modification time) the cache is updated.
        Override the build method to obtain more interesting behaviour.
    """
    def __init__(self, max_size=0, mode='rb'):
        Cache.__init__(self, max_size)
        self.mode=mode

    def check(self, key, name, entry):
        """ Checks the modification time to determine whether a file has changed or not. """
        f = file(key, self.mode)
        fs = fstat(f.fileno())
        ts1 = fs[-2]
        try:
            ts2 = entry._timestamp
        except AttributeError:
            ts2 = ts1-1

        if ts2<ts1:
            entry._timestamp=ts1
            return f
        else:
            return None

    def build(self, key, name, opened, entry):
        """ Return the content of the file as a string. Override this for better behaviour. """
        try:
            return opened.read()
        finally:
            opened.close()

def parseRFC822Time(t):
    return timegm(strptime(t, "%a, %d %b %Y %H:%M:%S %Z"))

re_max_age=re.compile('max-age\s*=\s*(\d+)', re.I)

class HTTPEntity(object):
    def __init__(self, entity, metadata):
        self.entity=entity
        self.metadata=metadata

    def __repr__(self):
        return 'HTTPEntity(%s, %s)'%(repr(self.entity), self.metadata)

    def __str__(self):
        return self.entity

class HTTPCache(Cache):
    """ An HTTP cache. Returns the entity found at the given URL.
        Uses Expires, ETag and Last-Modified headers to minimize bandwidth usage.
        Partial Cache-Control support (only max-age is supported).
    """
    def check(self, key, name, entry):
        request = urllib2.Request(key)

        try:
            if time()<entry._expires:
                return None
        except AttributeError:
            pass
        try:
            header, value = entry._validator
            request.headers[header]=value
        except AttributeError:
            pass
        opened = None
        try:
            opened = urllib2.urlopen(request)
            headers = opened.info()

            # expiration handling
            expiration = False
            try:
                match = re_max_age.match(headers['cache-control'])
                if match:
                        entry._expires=time()+int(match.group(1))
                        expiration = True
            except (KeyError, ValueError):
                pass
            if not expiration:
                try:
                    date = parseRFC822Time(headers['date'])
                    expires = parseRFC822Time(headers['expires'])
                    entry._expires = time()+(expires-date)
                    expiration = True
                except KeyError:
                    pass

            # validator handling
            validation = False
            try:
                entry._validator='If-None-Match', headers['etag']
                validation = True
            except KeyError:
                pass
            if not validation:
                try:
                    entry._validator='If-Modified-Since', headers['last-modified']
                except KeyError:
                    pass

            return opened
        except urllib2.HTTPError, error:
            if opened: opened.close()
            if error.code==304:
                return None
            else:
                raise error

    def build(self, key, name, opened, entry):
        try:
            return HTTPEntity(opened.read(), dict(opened.info()))
        finally:
            opened.close()

class Module(object):
    """ Placeholder object for the module definition. """
    def __init__(self, filename):
        self.__file__=filename

    def __repr__(self):
        return '<%s object at 0x%08x from %s>'%(type(self).__name__, id(self), self.__file__)

class ModuleCache(FileCache):
    """ A module cache. Give it a file name, it returns a module-like object
        which results from the execution of the Python script it contains.
    """
    def __init__(self, max_size=0):
        FileCache.__init__(self, max_size, 'r')

    def build(self, key, name, opened, entry):
        try:
            module = Module(key)
            exec opened in module.__dict__
            return module
            # I used to use imp.load_source but right now I'm trying the stuff above
            # return imp.load_source(re.sub('\W', '_', name), name, opened)
        finally:
            opened.close()

class HttpModuleCache(HTTPCache):
    """ A module cache. Give it a file name, it returns a module-like object
        which results from the execution of the Python script it contains.
    """
    def __init__(self, max_size=0):
        HTTPCache.__init__(self, max_size)

    def build(self, key, name, opened, entry):
        try:
            module = Module(key)
            text = opened.read().replace('\r\n', '\n')
            code = compile(text, name, 'exec')
            exec code in module.__dict__
            return module
            # I used to use imp.load_source but right now I'm trying the stuff above
            # return imp.load_source(re.sub('\W', '_', name), name, opened)
        finally:
            opened.close()

class FunctionCache(Cache):
    def __init__(self, function, max_size=0):
        Cache.__init__(self, max_size)
        self.function=function

    def __call__(self, *args, **kw):
        if kw:
            # a dict is not hashable so we build a tuple of (key, value) pairs
            kw = tuple(kw.iteritems())
            return self[args, kw]
        else:
            return self[args, ()]

    def build(self, key, name, opened, entry):
        args, kw = key
        return self.function(*args, **dict(kw))
<commit_msg>The module cache now uses real module objects (obtained through new.module) instead of custom modules.<commit_after> #
 # Copyright 2004 Apache Software Foundation
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You
 # may obtain a copy of the License at
 #
 #      http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 # implied.  See the License for the specific language governing
 # permissions and limitations under the License.
 #
 # Originally developed by Gregory Trubetskoy.
 #
 # This was donated by Nicolas Lehuen, and also posted to the Python Cookbook
 # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/302997
 #
 # $Id$

# -*- coding: CP1252 -*-
from threading import Lock
from os import fstat
from time import time, strptime
from calendar import timegm
import urllib2
import re
import weakref
import new

NOT_INITIALIZED = object()

class Entry(object):
    """ A cache entry, mostly an internal object. """
    def __init__(self, key):
        object.__init__(self)
        self._key=key
        self._value=NOT_INITIALIZED
        self._lock=Lock()

class Cache(object):
    """ An abstract, multi-threaded cache object. """

    def __init__(self, max_size=0):
        """ Builds a cache with a limit of max_size entries.
            If this limit is exceeded, the Least Recently Used entry is discarded.
            if max_size==0, the cache is unbounded (no LRU rule is applied).
        """
        object.__init__(self)
        self._maxsize=max_size
        self._dict={}
        self._lock=Lock()

        # Header of the access list
        if self._maxsize:
            self._head=Entry(None)
            self._head._previous=self._head
            self._head._next=self._head

    def __setitem__(self, name, value):
        """ Populates the cache with a given name and value. """
        self._lock.acquire()
        try:
            key = self.key(name)
            entry = self._dict.get(key)
            if not entry:
                entry = Entry(key)
                self._pack(entry, value)
                self._dict[key]=entry
                if self._maxsize:
                    entry._next = entry._previous = None
                    self._access(entry)
                    self._checklru()
            else:
                self._pack(entry, value)
                if self._maxsize:
                    self._access(entry)
            self.commit()
        finally:
            self._lock.release()

    def __getitem__(self, name):
        return self.checkitem(name)[1]

    def checkitem(self, name):
        """ Gets a value from the cache, builds it if required.
            Returns a tuple is_new, value. If is_new is True, the dependency had
            to be rebuilt.
        """
        self._lock.acquire()
        try:
            key = self.key(name)
            entry = self._dict.get(key)
            if not entry:
                entry = Entry(key)
                self._dict[key]=entry
                if self._maxsize:
                    entry._next = entry._previous = None
                    self._access(entry)
                    self._checklru()
            elif self._maxsize:
                self._access(entry)
        finally:
            self._lock.release()

        entry._lock.acquire()
        try:
            value = self._unpack(entry)
            is_new = False
            if value is NOT_INITIALIZED:
                opened = self.check(key, name, entry)
                value = self.build(key, name, opened, entry)
                is_new = True
                self._pack(entry, value)
                self.commit()
            else:
                opened = self.check(key, name, entry)
                if opened is not None:
                    value = self.build(key, name, opened, entry)
                    is_new = True
                    self._pack(entry, value)
                    self.commit()
            return is_new, value
        finally:
            entry._lock.release()

    def __delitem__(self, key):
        self._lock.acquire()
        try:
            key = self.key(key)
            del self._dict[key]
        finally:
            self._lock.release()

    def mru(self):
        """ Returns the Most Recently Used key """
        if self._maxsize:
            self._lock.acquire()
            try:
                return self._head._previous._key
            finally:
                self._lock.release()
        else:
            return None

    def lru(self):
        """ Returns the Least Recently Used key """
        if self._maxsize:
            self._lock.acquire()
            try:
                return self._head._next._key
            finally:
                self._lock.release()
        else:
            return None

    def key(self, name):
        """ Override this method to extract a key from the name passed to the [] operator """
        return name

    def commit(self):
        """ Override this method if you want to do something each time the underlying dictionary is modified (e.g. make it persistent). """
        pass

    def clear(self):
        """ Clears the cache """
        self._lock.acquire()
        try:
            self._dict.clear()
            if self._maxsize:
                self._head._next=self._head
                self._head._previous=self._head
        finally:
            self._lock.release()

    def check(self, key, name, entry):
        """ Override this method to check whether the entry with the given name is stale. Return None if it is fresh
            or an opened resource if it is stale. The object returned will be passed to the 'build' method as the 'opened' parameter.
            Use the 'entry' parameter to store meta-data if required. Don't worry about multiple threads accessing the same name,
            as this method is properly isolated.
        """
        return None

    def build(self, key, name, opened, entry):
        """ Build the cached value with the given name from the given opened resource. Use entry to obtain or store meta-data if needed.
             Don't worry about multiple threads accessing the same name, as this method is properly isolated.
        """
        raise NotImplementedError()

    def _access(self, entry):
        " Internal use only, must be invoked within a cache lock. Updates the access list. """
        if entry._next is not self._head:
            if entry._previous is not None:
                # remove the entry from the access list
                entry._previous._next=entry._next
                entry._next._previous=entry._previous
            # insert the entry at the end of the access list
            entry._previous=self._head._previous
            entry._previous._next=entry
            entry._next=self._head
            entry._next._previous=entry
            if self._head._next is self._head:
                self._head._next=entry

    def _checklru(self):
        " Internal use only, must be invoked within a cache lock. Removes the LRU entry if needed. """
        if len(self._dict)>self._maxsize:
            lru=self._head._next
            lru._previous._next=lru._next
            lru._next._previous=lru._previous
            del self._dict[lru._key]

    def _pack(self, entry, value):
        """ Store the value in the entry. """
        entry._value=value

    def _unpack(self, entry):
        """ Recover the value from the entry, returns NOT_INITIALIZED if it is not OK. """
        return entry._value

class WeakCache(Cache):
    """ This cache holds weak references to the values it stores. Whenever a value is not longer
        normally referenced, it is removed from the cache. Useful for sharing the result of long
        computations but letting them go as soon as they are not needed by anybody.
    """

    def _pack(self, entry, value):
        entry._value=weakref.ref(value, lambda ref: self.__delitem__(entry._key))

    def _unpack(self, entry):
        if entry._value is NOT_INITIALIZED:
            return NOT_INITIALIZED

        value = entry._value()
        if value is None:
            return NOT_INITIALIZED
        else:
            return value

class FileCache(Cache):
    """ A file cache. Returns the content of the files as a string, given their filename.
        Whenever the files are modified (according to their modification time) the cache is updated.
        Override the build method to obtain more interesting behaviour.
    """
    def __init__(self, max_size=0, mode='rb'):
        Cache.__init__(self, max_size)
        self.mode=mode

    def check(self, key, name, entry):
        """ Checks the modification time to determine whether a file has changed or not. """
        f = file(key, self.mode)
        fs = fstat(f.fileno())
        ts1 = fs[-2]
        try:
            ts2 = entry._timestamp
        except AttributeError:
            ts2 = ts1-1

        if ts2!=ts1:
            entry._timestamp=ts1
            return f
        else:
            f.close()
            return None

    def build(self, key, name, opened, entry):
        """ Return the content of the file as a string. Override this for better behaviour. """
        try:
            return opened.read()
        finally:
            opened.close()

def parseRFC822Time(t):
    return timegm(strptime(t, "%a, %d %b %Y %H:%M:%S %Z"))

re_max_age=re.compile('max-age\s*=\s*(\d+)', re.I)

class HTTPEntity(object):
    def __init__(self, entity, metadata):
        self.entity=entity
        self.metadata=metadata

    def __repr__(self):
        return 'HTTPEntity(%s, %s)'%(repr(self.entity), self.metadata)

    def __str__(self):
        return self.entity

class HTTPCache(Cache):
    """ An HTTP cache. Returns the entity found at the given URL.
        Uses Expires, ETag and Last-Modified headers to minimize bandwidth usage.
        Partial Cache-Control support (only max-age is supported).
    """
    def check(self, key, name, entry):
        request = urllib2.Request(key)

        try:
            if time()<entry._expires:
                return None
        except AttributeError:
            pass
        try:
            header, value = entry._validator
            request.headers[header]=value
        except AttributeError:
            pass
        opened = None
        try:
            opened = urllib2.urlopen(request)
            headers = opened.info()

            # expiration handling
            expiration = False
            try:
                match = re_max_age.match(headers['cache-control'])
                if match:
                        entry._expires=time()+int(match.group(1))
                        expiration = True
            except (KeyError, ValueError):
                pass
            if not expiration:
                try:
                    date = parseRFC822Time(headers['date'])
                    expires = parseRFC822Time(headers['expires'])
                    entry._expires = time()+(expires-date)
                    expiration = True
                except KeyError:
                    pass

            # validator handling
            validation = False
            try:
                entry._validator='If-None-Match', headers['etag']
                validation = True
            except KeyError:
                pass
            if not validation:
                try:
                    entry._validator='If-Modified-Since', headers['last-modified']
                except KeyError:
                    pass

            return opened
        except urllib2.HTTPError, error:
            if opened: opened.close()
            if error.code==304:
                return None
            else:
                raise error

    def build(self, key, name, opened, entry):
        try:
            return HTTPEntity(opened.read(), dict(opened.info()))
        finally:
            opened.close()

class ModuleCache(FileCache):
    """ A module cache. Give it a file name, it returns a module-like object
        which results from the execution of the Python script it contains.
    """
    def __init__(self, max_size=0):
        FileCache.__init__(self, max_size, 'r')

    def build(self, key, name, opened, entry):
        try:
            module = new.module(key)
            module.__file__ = key
            exec opened in module.__dict__
            return module
        finally:
            opened.close()

class HttpModuleCache(HTTPCache):
    """ A module cache. Give it a file name, it returns a module-like object
        which results from the execution of the Python script it contains.
    """
    def __init__(self, max_size=0):
        HTTPCache.__init__(self, max_size)

    def build(self, key, name, opened, entry):
        try:
            module = new.module(key)
            module.__file__ = key
            text = opened.read().replace('\r\n', '\n')
            code = compile(text, name, 'exec')
            exec code in module.__dict__
            return module
        finally:
            opened.close()

class FunctionCache(Cache):
    def __init__(self, function, max_size=0):
        Cache.__init__(self, max_size)
        self.function=function

    def __call__(self, *args, **kw):
        if kw:
            # a dict is not hashable so we build a tuple of (key, value) pairs
            kw = tuple(kw.iteritems())
            return self[args, kw]
        else:
            return self[args, ()]

    def build(self, key, name, opened, entry):
        args, kw = key
        return self.function(*args, **dict(kw))
<|endoftext|>


<commit_before>import requests
import re
from amonagent.utils.helpers import slugify
from amonagent.settings import settings
from amonagent.log import log

class AmonNginxPlugin(object):

	def __init__(self):
		configuration = settings.PLUGINS.get('nginx', None)
		url = configuration.get('url', None)

		if url is None:
			log.error('Nginx: Empty status URL in /etc/amonagent.conf:plugins:nginx')
		else:
			self.url = configuration.get('url', None)

	def build_report(self):
		# Report format
		# {
		# "plugin": "nginx",
		# "graphs": [
		# 	{
		# 		"name": "Connections",
		# 		"data": {
		# 			"connections": 1,
		# 			"reading": 0,
		# 			"writing": 0,
		# 			"waiting": 0
		# 	}},
		# 	{
		# 		"name": "Requests per second",
		# 		"data":  {
		# 			"requests_per_second": 2
		# 	}}
		# ]
		# }
		report = {"plugin": "nginx", "graphs": []}
		temp_report = {} # Used for internal storage
		graphs = ['Requests per second', 'Connections']

		result = requests.get(self.url)
		whitelist = ['accepts','handled','requests']

		if result.text:
			status = result.text.splitlines()
			for line in status:
				stats_line = re.match('\s*(\d+)\s+(\d+)\s+(\d+)\s*$', line)
				if stats_line:
					for i, key in enumerate(whitelist): # Enumerate start is supported only in Python 2.6+
						temp_report[key] = int(stats_line.group(i+1))

				else:
					for (key,val) in re.findall('(\w+):\s*(\d+)', line):
						filtered_key = slugify(key)
						temp_report[filtered_key] = int(val)

		# Calculate requests per second
		if len(temp_report) > 0:
			total_requests = temp_report.get('requests', 0)
			handled = temp_report.get('handled', 0)

			if total_requests > 0 and handled > 0:
				requests_per_second = total_requests/handled
				if requests_per_second > 0:
					for key in whitelist:
						del temp_report[key]

					temp_report['requests_per_second'] = requests_per_second

		report['graphs'].append({"name": "Connections",
			"data": {
				"connections": temp_report['connections'],
				"reading": temp_report['reading'],
				"writing": temp_report['writing'],
				"waiting": temp_report['waiting']
			}
		})

		report['graphs'].append({"name": "Requests per second",
			"data": {
				"requests_per_second": temp_report['requests_per_second']
			}
		})

		return report

plugin = AmonNginxPlugin()<commit_msg>Nginx updates<commit_after>import requests
import re
from amonagent.utils.helpers import slugify
from amonagent.settings import settings
from amonagent.log import log

class AmonNginxPlugin(object):

	def __init__(self):
		configuration = settings.PLUGINS.get('nginx', None)
		url = configuration.get('url', None)

		if url is None:
			log.error('Nginx: Empty status URL in /etc/amonagent.conf:plugins:nginx')
		else:
			self.url = configuration.get('url', None)

	def build_report(self):
		# Report format
		# {
		# "plugin": "nginx",
		# "graphs": [
		# 	{
		# 		"name": "Connections",
		# 		"data": {
		# 			"connections": 1,
		# 			"reading": 0,
		# 			"writing": 0,
		# 			"waiting": 0
		# 	}},
		# 	{
		# 		"name": "Requests per second",
		# 		"data":  {
		# 			"requests_per_second": 2
		# 	}}
		# ]
		# }
		report = {"plugin": "nginx", "graphs": []}
		temp_report = {} # Used for internal storage
		graphs = ['Requests per second', 'Connections']

		result = requests.get(self.url)
		whitelist = ['accepts','handled','requests']

		if result.text:
			status = result.text.splitlines()
			for line in status:
				stats_line = re.match('\s*(\d+)\s+(\d+)\s+(\d+)\s*$', line)
				if stats_line:
					for i, key in enumerate(whitelist): # Enumerate start is supported only in Python 2.6+
						temp_report[key] = int(stats_line.group(i+1))

				else:
					for (key,val) in re.findall('(\w+):\s*(\d+)', line):
						filtered_key = slugify(key)
						temp_report[filtered_key] = int(val)

		# Calculate requests per second
		if len(temp_report) > 0:
			total_requests = temp_report.get('requests', 0)
			handled = temp_report.get('handled', 0)

			if total_requests > 0 and handled > 0:
				requests_per_second = total_requests/handled
				if requests_per_second > 0:
					for key in whitelist:
						del temp_report[key]

					temp_report['requests_per_second'] = requests_per_second

		report['graphs'].append({"name": "Connections",
			"data": {
				"connections": temp_report['connections'],
				"reading": temp_report['reading'],
				"writing": temp_report['writing'],
				"waiting": temp_report['waiting']
			}
		})

		report['graphs'].append({"name": "Requests per second",
			"data": {
				"requests_per_second": temp_report['requests_per_second']
			}
		})
	l
		return report

plugin = AmonNginxPlugin()<|endoftext|>


<commit_before><commit_msg>Factor out logging about excessive task creation<commit_after><|endoftext|>


<commit_before># -*- coding:utf-8 -*-

import os
import time
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler

from fdfs_client.client import Fdfs_client

client = Fdfs_client('client.conf')

class MyHandler(FileSystemEventHandler):

    def on_created(self,event):
        pass

    def on_modified(self,event):
        if not event.is_directory:
            try:
                print("检测到生成新文件")
                client.upload_by_filename(event.src_path)
                os.remove(event.src_path)
                print("已将新文件放进mongodb")
            except IOError:
                pass

if __name__ == "__main__":
    event_handler = MyHandler()
    observer = Observer()
    observer.schedule(event_handler, path='./www', recursive=True)
    observer.start()
    try:
        print "started myWatch"
        while True:
            time.sleep(1)
    except KeyboardInterrupt:
        observer.stop()
        observer.join()
<commit_msg>修改源文件的异常<commit_after># -*- coding:utf-8 -*-

import os
import time
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler

from fdfs_client.client import Fdfs_client
from fdfs_client.exceptions import DataError

client = Fdfs_client('client.conf')

class MyHandler(FileSystemEventHandler):

    def on_created(self,event):
        pass

    def on_modified(self,event):
        if not event.is_directory:
            try:
                print("检测到生成新文件")
                client.upload_by_filename(event.src_path)
                os.remove(event.src_path)
                print("已将新文件放进fdfs")
            except DataError:
                pass

if __name__ == "__main__":
    event_handler = MyHandler()
    observer = Observer()
    observer.schedule(event_handler, path='./www', recursive=True)
    observer.start()
    try:
        print "started myWatch"
        while True:
            time.sleep(1)
    except KeyboardInterrupt:
        observer.stop()
        observer.join()
<|endoftext|>