<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.conf import settings
from django.conf.urls import patterns, url
from django.contrib import admin
from django.core.urlresolvers import reverse
from django.db.models import Sum
from django.forms import models, widgets, ValidationError
from django.http import HttpResponse
from django.template import RequestContext
from django.template.loader import select_template
from django.utils import timezone
from django.utils.html import format_html
from django.utils.translation import ugettext_lazy as _
from shop.admin.order import OrderItemInline
from shop.models.order import OrderItemModel
from shop.models.delivery import DeliveryModel, DeliveryItemModel
from shop.modifiers.pool import cart_modifiers_pool
from shop.rest import serializers

class OrderItemForm(models.ModelForm):
    """
    This form handles an ordered item, but adds a number field to modify the number of
    items to deliver.
    """
    class Meta:
        model = OrderItemModel
        exclude = ()

    def __init__(self, *args, **kwargs):
        if 'instance' in kwargs:
            kwargs.setdefault('initial', {})
            deliver_quantity = kwargs['instance'].quantity - self.get_delivered(kwargs['instance'])
            kwargs['initial'].update(deliver_quantity=deliver_quantity)
        else:
            deliver_quantity = None
        super(OrderItemForm, self).__init__(*args, **kwargs)
        if deliver_quantity == 0:
            self['deliver_quantity'].field.widget.attrs.update(readonly='readonly')

    @classmethod
    def get_delivered(cls, instance):
        """
        Returns the quantity already delivered for this order item.
        """
        aggr = instance.deliveryitem_set.aggregate(delivered=Sum('quantity'))
        return aggr['delivered'] or 0

    def clean(self):
        cleaned_data = super(OrderItemForm, self).clean()
        if cleaned_data.get('deliver_quantity') is not None:
            if cleaned_data['deliver_quantity'] < 0:
                raise ValidationError(_("Only a positive number of items can be delivered"), code='invalid')
            if cleaned_data['deliver_quantity'] > self.instance.quantity - self.get_delivered(self.instance):
                raise ValidationError(_("The number of items to deliver exceeds the ordered quantity"), code='invalid')
        return cleaned_data

    def has_changed(self):
        """Force form to changed"""
        return True

class OrderItemInlineDelivery(OrderItemInline):
    def get_fields(self, request, obj=None):
        fields = list(super(OrderItemInlineDelivery, self).get_fields(request, obj))
        if obj:
            if obj.status == 'pick_goods' and obj.unfulfilled_items > 0:
                fields[1] += ('deliver_quantity', 'canceled',)
            else:
                fields[1] += ('get_delivered',)
        return fields

    def get_readonly_fields(self, request, obj=None):
        fields = list(super(OrderItemInlineDelivery, self).get_readonly_fields(request, obj))
        if obj:
            if not (obj.status == 'pick_goods' and obj.unfulfilled_items > 0):
                fields.append('get_delivered')
        return fields

    def get_formset(self, request, obj=None, **kwargs):
        """
        Add field `quantity` to the form on the fly, using the same numeric type as `OrderItem.quantity`
        """
        labels = {'quantity': _("Deliver quantity")}
        attrs = models.fields_for_model(obj.items.model, fields=['quantity'], labels=labels)
        # rename to deliver_quantity, since quantity is already used
        attrs['deliver_quantity'] = attrs.pop('quantity')
        if obj.status == 'pick_goods' and obj.unfulfilled_items > 0:
            attrs['deliver_quantity'].widget.attrs.update(style='width: 50px;')
        else:
            attrs['deliver_quantity'].required = False
        form = type(str('OrderItemForm'), (OrderItemForm,), attrs)
        labels = {'canceled': _("Cancel this item")}
        kwargs.update(form=form, labels=labels)
        formset = super(OrderItemInlineDelivery, self).get_formset(request, obj, **kwargs)
        return formset

    def get_delivered(self, obj=None):
        return OrderItemForm.get_delivered(obj)
    get_delivered.short_description = _("Delivered quantity")

class DeliveryInline(admin.TabularInline):
    model = DeliveryModel
    extra = 0
    fields = ('shipping_id', 'shipping_method', 'delivered_items', 'print_out', 'fulfilled',)
    readonly_fields = ('delivered_items', 'print_out', 'fulfilled',)

    def get_formset(self, request, obj=None, **kwargs):
        """
        Convert the field `shipping_method` into a select box with all possible shipping methods.
        """
        choices = [sm.get_choice() for sm in cart_modifiers_pool.get_shipping_modifiers()]
        kwargs.update(widgets={'shipping_method': widgets.Select(choices=choices)})
        formset = super(DeliveryInline, self).get_formset(request, obj, **kwargs)
        return formset

    def get_max_num(self, request, obj=None, **kwargs):
        qs = self.model.objects.filter(order=obj)
        if obj.status != 'pick_goods' or qs.filter(fulfilled_at__isnull=True) or obj.unfulfilled_items == 0:
            return qs.count()
        return qs.count() + 1

    def has_delete_permission(self, request, obj=None):
        return False

    def delivered_items(self, obj):
        aggr = obj.deliveryitem_set.aggregate(quantity=Sum('quantity'))
        aggr.update(items=obj.deliveryitem_set.count())
        return '{quantity}/{items}'.format(**aggr)
    delivered_items.short_description = _("Quantity/Items")

    def print_out(self, obj):
        if obj.fulfilled_at is None:
            return ''
        link = reverse('admin:print_delivery_note', args=(obj.id,)), _("Delivery Note")
        return format_html('{1}', *link)
    print_out.short_description = _("Print out")
    print_out.allow_tags = True

    def fulfilled(self, obj):
        if obj.fulfilled_at:
            return timezone.localtime(obj.fulfilled_at).ctime()  # TODO: find the correct time format
        return _("Pending")
    fulfilled.short_description = _("Fulfilled at")

class DeliveryOrderAdminMixin(object):
    """
    Add this mixin to the class defining the OrderAdmin
    """
    def get_urls(self):
        return patterns('',
            url(r'^(?P<delivery_pk>\d+)/print_delivery_note/$',
                self.admin_site.admin_view(self.render_delivery_note),
                name='print_delivery_note'),
        ) + super(DeliveryOrderAdminMixin, self).get_urls()

    def render_delivery_note(self, request, delivery_pk=None):
        template = select_template([
            '{}/print/delivery-note.html'.format(settings.SHOP_APP_LABEL.lower()),
            'shop/print/delivery-note.html'
        ])
        delivery = DeliveryModel.objects.get(pk=delivery_pk)
        context = {'request': request, 'render_label': 'print'}
        order_serializer = serializers.OrderDetailSerializer(delivery.order, context=context)
        content = template.render(RequestContext(request, {
            'customer': serializers.CustomerSerializer(delivery.order.customer).data,
            'data': order_serializer.data,
            'delivery': delivery,
        }))
        return HttpResponse(content)

    def get_inline_instances(self, request, obj=None):
        inline_instances = [OrderItemInlineDelivery(self.model, self.admin_site)
                            if isinstance(instance, OrderItemInline) else instance
            for instance in super(DeliveryOrderAdminMixin, self).get_inline_instances(request, obj)]
        if obj.status in ('pick_goods', 'pack_goods',):
            inline_instances.append(DeliveryInline(self.model, self.admin_site))
        return inline_instances

    def save_related(self, request, form, formsets, change):
        super(DeliveryOrderAdminMixin, self).save_related(request, form, formsets, change)
        if hasattr(form.instance, '_transition_to_pack_goods') or (
                form.instance.status == 'pick_goods' and 'status' not in form.changed_data):
            # merchant clicked on button: "Pack the Goods", "Save and continue" or "Save"
            self._mark_items_for_delivery(formsets)

    def _mark_items_for_delivery(self, formsets):
        # create a DeliveryItem for each OrderItem marked to be shipped with this Delivery object
        orderitem_formset = [fs for fs in formsets if issubclass(fs.model, OrderItemModel)]
        delivery_formset = [fs for fs in formsets if issubclass(fs.model, DeliveryModel)]
        if orderitem_formset and delivery_formset:
            orderitem_formset = orderitem_formset[0]
            delivery_formset = delivery_formset[0]
            if delivery_formset.new_objects:
                delivery = delivery_formset.new_objects[0]
            else:
                delivery = delivery_formset.queryset.filter(fulfilled_at__isnull=True).last()
            if delivery:
                count_items = 0
                for data in orderitem_formset.cleaned_data:
                    if data['deliver_quantity'] > 0 and not data['canceled']:
                        DeliveryItemModel.objects.create(delivery=delivery, item=data['id'],
                                                         quantity=data['deliver_quantity'])
                        count_items += 1
                if count_items > 0:
                    # mark Delivery object as fulfilled
                    delivery.fulfilled_at = timezone.now()
                    delivery.save()
                else:
                    # since no OrderItem was added to this delivery, discard it
                    delivery.delete()
<commit_msg>Feature: Cancel item from order<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.conf import settings
from django.conf.urls import patterns, url
from django.contrib import admin
from django.core.urlresolvers import reverse
from django.db.models import Sum
from django.forms import models, widgets, ValidationError
from django.http import HttpResponse
from django.template import RequestContext
from django.template.loader import select_template
from django.utils import timezone
from django.utils.html import format_html
from django.utils.translation import ugettext_lazy as _
from shop.admin.order import OrderItemInline
from shop.models.order import OrderItemModel
from shop.models.delivery import DeliveryModel, DeliveryItemModel
from shop.modifiers.pool import cart_modifiers_pool
from shop.rest import serializers

class OrderItemForm(models.ModelForm):
    """
    This form handles an ordered item, but adds a number field to modify the number of
    items to deliver.
    """
    class Meta:
        model = OrderItemModel
        exclude = ()

    def __init__(self, *args, **kwargs):
        if 'instance' in kwargs:
            kwargs.setdefault('initial', {})
            deliver_quantity = kwargs['instance'].quantity - self.get_delivered(kwargs['instance'])
            kwargs['initial'].update(deliver_quantity=deliver_quantity)
        else:
            deliver_quantity = None
        super(OrderItemForm, self).__init__(*args, **kwargs)
        if deliver_quantity == 0:
            self['deliver_quantity'].field.widget.attrs.update(readonly='readonly')

    @classmethod
    def get_delivered(cls, instance):
        """
        Returns the quantity already delivered for this order item.
        """
        aggr = instance.deliveryitem_set.aggregate(delivered=Sum('quantity'))
        return aggr['delivered'] or 0

    def clean(self):
        cleaned_data = super(OrderItemForm, self).clean()
        if cleaned_data.get('deliver_quantity') is not None:
            if cleaned_data['deliver_quantity'] < 0:
                raise ValidationError(_("Only a positive number of items can be delivered"), code='invalid')
            if cleaned_data['deliver_quantity'] > self.instance.quantity - self.get_delivered(self.instance):
                raise ValidationError(_("The number of items to deliver exceeds the ordered quantity"), code='invalid')
        return cleaned_data

    def has_changed(self):
        """Force form to changed"""
        return True

class OrderItemInlineDelivery(OrderItemInline):
    def get_fields(self, request, obj=None):
        fields = list(super(OrderItemInlineDelivery, self).get_fields(request, obj))
        if obj:
            if obj.status == 'pick_goods' and obj.unfulfilled_items > 0:
                fields[1] += ('deliver_quantity', 'canceled',)
            else:
                fields[1] += ('get_delivered', 'show_ready',)
        return fields

    def get_readonly_fields(self, request, obj=None):
        fields = list(super(OrderItemInlineDelivery, self).get_readonly_fields(request, obj))
        if obj:
            if not (obj.status == 'pick_goods' and obj.unfulfilled_items > 0):
                fields.extend(['get_delivered', 'show_ready'])
        return fields

    def get_formset(self, request, obj=None, **kwargs):
        """
        Add field `quantity` to the form on the fly, using the same numeric type as `OrderItem.quantity`
        """
        labels = {'quantity': _("Deliver quantity")}
        attrs = models.fields_for_model(obj.items.model, fields=['quantity'], labels=labels)
        # rename to deliver_quantity, since quantity is already used
        attrs['deliver_quantity'] = attrs.pop('quantity')
        if obj.status == 'pick_goods' and obj.unfulfilled_items > 0:
            attrs['deliver_quantity'].widget.attrs.update(style='width: 50px;')
        else:
            attrs['deliver_quantity'].required = False
        form = type(str('OrderItemForm'), (OrderItemForm,), attrs)
        labels = {'canceled': _("Cancel this item")}
        kwargs.update(form=form, labels=labels)
        formset = super(OrderItemInlineDelivery, self).get_formset(request, obj, **kwargs)
        return formset

    def get_delivered(self, obj=None):
        return OrderItemForm.get_delivered(obj)
    get_delivered.short_description = _("Delivered quantity")

    def show_ready(self, obj=None):
        return not obj.canceled
    show_ready.boolean = True
    show_ready.short_description = _("Ready for delivery")

class DeliveryInline(admin.TabularInline):
    model = DeliveryModel
    extra = 0
    fields = ('shipping_id', 'shipping_method', 'delivered_items', 'print_out', 'fulfilled',)
    readonly_fields = ('delivered_items', 'print_out', 'fulfilled',)

    def get_formset(self, request, obj=None, **kwargs):
        """
        Convert the field `shipping_method` into a select box with all possible shipping methods.
        """
        choices = [sm.get_choice() for sm in cart_modifiers_pool.get_shipping_modifiers()]
        kwargs.update(widgets={'shipping_method': widgets.Select(choices=choices)})
        formset = super(DeliveryInline, self).get_formset(request, obj, **kwargs)
        return formset

    def get_max_num(self, request, obj=None, **kwargs):
        qs = self.model.objects.filter(order=obj)
        if obj.status != 'pick_goods' or qs.filter(fulfilled_at__isnull=True) or obj.unfulfilled_items == 0:
            return qs.count()
        return qs.count() + 1

    def has_delete_permission(self, request, obj=None):
        return False

    def delivered_items(self, obj):
        aggr = obj.deliveryitem_set.aggregate(quantity=Sum('quantity'))
        aggr['quantity'] = aggr['quantity'] or 0
        aggr.update(items=obj.deliveryitem_set.count())
        return '{quantity}/{items}'.format(**aggr)
    delivered_items.short_description = _("Quantity/Items")

    def print_out(self, obj):
        if obj.fulfilled_at is None:
            return ''
        link = reverse('admin:print_delivery_note', args=(obj.id,)), _("Delivery Note")
        return format_html('{1}', *link)
    print_out.short_description = _("Print out")
    print_out.allow_tags = True

    def fulfilled(self, obj):
        if obj.fulfilled_at:
            return timezone.localtime(obj.fulfilled_at).ctime()  # TODO: find the correct time format
        return _("Pending")
    fulfilled.short_description = _("Fulfilled at")

class DeliveryOrderAdminMixin(object):
    """
    Add this mixin to the class defining the OrderAdmin
    """
    def get_urls(self):
        return patterns('',
            url(r'^(?P<delivery_pk>\d+)/print_delivery_note/$',
                self.admin_site.admin_view(self.render_delivery_note),
                name='print_delivery_note'),
        ) + super(DeliveryOrderAdminMixin, self).get_urls()

    def render_delivery_note(self, request, delivery_pk=None):
        template = select_template([
            '{}/print/delivery-note.html'.format(settings.SHOP_APP_LABEL.lower()),
            'shop/print/delivery-note.html'
        ])
        delivery = DeliveryModel.objects.get(pk=delivery_pk)
        context = {'request': request, 'render_label': 'print'}
        order_serializer = serializers.OrderDetailSerializer(delivery.order, context=context)
        content = template.render(RequestContext(request, {
            'customer': serializers.CustomerSerializer(delivery.order.customer).data,
            'data': order_serializer.data,
            'delivery': delivery,
        }))
        return HttpResponse(content)

    def get_inline_instances(self, request, obj=None):
        inline_instances = [OrderItemInlineDelivery(self.model, self.admin_site)
                            if isinstance(instance, OrderItemInline) else instance
            for instance in super(DeliveryOrderAdminMixin, self).get_inline_instances(request, obj)]
        if obj.status in ('pick_goods', 'pack_goods',):
            inline_instances.append(DeliveryInline(self.model, self.admin_site))
        return inline_instances

    def save_related(self, request, form, formsets, change):
        super(DeliveryOrderAdminMixin, self).save_related(request, form, formsets, change)
        if hasattr(form.instance, '_transition_to_pack_goods') or (
                form.instance.status == 'pick_goods' and 'status' not in form.changed_data):
            # merchant clicked on button: "Pack the Goods", "Save and continue" or "Save"
            self._mark_items_for_delivery(formsets)

    def _mark_items_for_delivery(self, formsets):
        # create a DeliveryItem for each OrderItem marked to be shipped with this Delivery object
        orderitem_formset = [fs for fs in formsets if issubclass(fs.model, OrderItemModel)]
        delivery_formset = [fs for fs in formsets if issubclass(fs.model, DeliveryModel)]
        if orderitem_formset and delivery_formset:
            orderitem_formset = orderitem_formset[0]
            delivery_formset = delivery_formset[0]
            if delivery_formset.new_objects:
                delivery = delivery_formset.new_objects[0]
            else:
                delivery = delivery_formset.queryset.filter(fulfilled_at__isnull=True).last()
            if delivery:
                count_items = 0
                for data in orderitem_formset.cleaned_data:
                    if data['deliver_quantity'] > 0 and not data['canceled']:
                        DeliveryItemModel.objects.create(delivery=delivery, item=data['id'],
                                                         quantity=data['deliver_quantity'])
                        count_items += 1
                if count_items > 0:
                    # mark Delivery object as fulfilled
                    delivery.fulfilled_at = timezone.now()
                    delivery.save()
                else:
                    # since no OrderItem was added to this delivery, discard it
                    delivery.delete()
<|endoftext|>


<commit_before>import json
import iso8601
import io

from datetime import datetime, date

from decimal import Decimal

class JsonWriter(object):
    def __init__(self, stream=None, indent="  ", max_indent_depth=3):
        self.stream = stream or io.StringIO()
        self._depth = 0
        self.indent = indent
        self.max_indent_depth = max_indent_depth

    def write_value(self, obj):
        if isinstance(obj, dict):
            self.write_object(obj)
        elif isinstance(obj, (list, tuple)):
            self.write_array(obj)
        else:
            json.dump(obj, self.stream, default=_default)

    def _newline(self, inc=0):
        self._depth += inc
        if (self._depth + max(0, -inc)) <= self.max_indent_depth:
            self.stream.write("\n")
            self.stream.write(self.indent * min(self.max_indent_depth, self._depth))

    def write_object(self, obj):
        self.stream.write("{")
        if len(obj) == 0:
            self.stream.write("}")
            return
        self._newline(1)
        n = len(obj)
        for key, value in obj.items():
            self.write_value(key)
            self.stream.write(": ")
            self.write_value(value)
            n -= 1
            if n > 0:
                self.stream.write(", ")
                self._newline()
        self._newline(-1)
        self.stream.write("}")

    def write_array(self, obj):
        self.stream.write("[")
        if len(obj) == 0:
            self.stream.write("]")
            return
        self._newline(1)
        n = len(obj)
        for value in obj:
            self.write_value(value)
            n -= 1
            if n > 0:
                self.stream.write(", ")
                self._newline()
        self._newline(-1)
        self.stream.write("]")

def _default(obj):
    if isinstance(obj, datetime):
        return {'$datetime': obj.isoformat()}
    elif isinstance(obj, date):
        return {'$date': obj.isoformat()}
    elif isinstance(obj, Decimal):
        return float(obj)
    else:
        return obj

def _object_hook(obj):
    if len(obj) == 1:
        key, = obj.keys()
        if key == '$datetime':
            return iso8601.parse_date(obj[key])
        elif key == '$date':
            return iso8601.parse_date(obj[key]).date()
    return obj

def dump(obj, f):
    jw = JsonWriter(f)
    jw.write_value(obj)

def dumps(obj):
    jw = JsonWriter()
    jw.write_value(obj)
    return jw.stream.getvalue()

def loads(*args, **kwargs):
    return json.loads(*args, object_hook=_object_hook, **kwargs)

def load(*args, **kwargs):
    return json.load(*args, object_hook=_object_hook, **kwargs)<commit_msg>Json now reads as naive datetime - better for Postgres<commit_after>import json
import iso8601
import io
import pytz

from datetime import datetime, date

from decimal import Decimal

class JsonWriter(object):
    def __init__(self, stream=None, indent="  ", max_indent_depth=3):
        self.stream = stream or io.StringIO()
        self._depth = 0
        self.indent = indent
        self.max_indent_depth = max_indent_depth

    def write_value(self, obj):
        if isinstance(obj, dict):
            self.write_object(obj)
        elif isinstance(obj, (list, tuple)):
            self.write_array(obj)
        else:
            json.dump(obj, self.stream, default=_default)

    def _newline(self, inc=0, alt=''):
        self._depth += inc
        if (self._depth + max(0, -inc)) <= self.max_indent_depth:
            self.stream.write("\n")
            self.stream.write(self.indent * min(self.max_indent_depth, self._depth))
        else:
            self.stream.write(alt)

    def write_object(self, obj):
        self.stream.write("{")
        if len(obj) == 0:
            self.stream.write("}")
            return
        self._newline(1)
        n = len(obj)
        for key, value in obj.items():
            self.write_value(key)
            self.stream.write(": ")
            self.write_value(value)
            n -= 1
            if n > 0:
                self.stream.write(", ")
                self._newline()
        self._newline(-1)
        self.stream.write("}")

    def write_array(self, obj):
        self.stream.write("[")
        if len(obj) == 0:
            self.stream.write("]")
            return
        self._newline(1)
        n = len(obj)
        for value in obj:
            self.write_value(value)
            n -= 1
            if n > 0:
                self.stream.write(", ")
                self._newline()
        self._newline(-1)
        self.stream.write("]")

def _default(obj):
    if isinstance(obj, datetime):
        return {'$datetime': obj.isoformat()}
    elif isinstance(obj, date):
        return {'$date': obj.isoformat()}
    elif isinstance(obj, Decimal):
        return float(obj)
    else:
        return obj

def _object_hook(obj):
    if len(obj) == 1:
        key, = obj.keys()
        if key == '$datetime':
            return iso8601.parse_date(obj[key]).replace(tzinfo=None)
        elif key == '$date':
            return iso8601.parse_date(obj[key]).date()
    return obj

def dump(obj, f):
    jw = JsonWriter(f)
    jw.write_value(obj)

def dumps(obj):
    jw = JsonWriter()
    jw.write_value(obj)
    return jw.stream.getvalue()

def loads(*args, **kwargs):
    return json.loads(*args, object_hook=_object_hook, **kwargs)

def load(*args, **kwargs):
    return json.load(*args, object_hook=_object_hook, **kwargs)<|endoftext|>


<commit_before>import flask
import sqlalchemy

def get_group_list_data(fields=None, attrs={}):
    """
    Queries the database and returns list of group data constrained by the
    specified attributes.

    Arguments:
        fields: The fields to return. If None specified, then default_fields
                are used.
        attrs:  The attributes of the group to filter for.
    Returns:
        result: The fields and corresponding values of groups with desired
                attributes. In the form of a list of dicts with key:value of
                columnname:columnvalue.
    """
    all_returnable_fields = [
        "group_id", "group_name", "group_desc", "type", "anyone_can_send",
        "members_can_send", "newsgroups", "visible", "admin_control_members"
    ]
    default_fields = ["group_id", "group_name", "group_desc", "type"]
    if fields == None:
        fields = default_fields
    else:
        if any(f not in all_returnable_fields for f in fields):
            return "Invalid field"

    # Build the SELECT and FROM clauses
    s = sqlalchemy.sql.select(fields).select_from(sqlalchemy.text("groups"))

    # Build the WHERE clause
    for key, value in list(attrs.items()):
        s = s.where(sqlalchemy.text(key + "= :" + key))

    # Execute the query
    result = flask.g.db.execute(s, attrs).fetchall()

    # Return the rows in the form of a list of dicts
    result = [{f: t for f, t in zip(fields, res)} for res in result]
    return result

def get_group_data(group_id, fields=None):
    """
    Queries the databse and returns member data for the specified group_id.

    Arguments:
        group_id: The group to look up
        fields:   The fields to return. If None are specified, then
                  default_fields are used

    Returns:
        result:   The fields and corresponding values of group with group_id.
                  In the form of a dict with key:value of columnname:columnalue
    """
    all_returnable_fields = [
        "group_id", "group_name", "group_desc", "type", "anyone_can_send",
        "members_can_send", "newsgroups", "visible", "admin_control_members"
    ]
    default_fields = ["group_id", "group_name", "group_desc", "type"]
    if fields is None:
        fields = default_fields
    else:
        if any(f not in all_returnable_fields for f in fields):
            return None

    # Build the SELECT and FROM clauses
    s = sqlalchemy.sql.select(fields).select_from(sqlalchemy.text("groups"))

    # Build the WHERE clause
    s = s.where(sqlalchemy.text("group_id = :g"))

    # Execute the query
    result = flask.g.db.execute(s, g=group_id).first()

    # Check to see if query returned anything
    if result is None:
        return {}

    # Return the row in the form of a dict
    result = {f: t for f, t in zip(fields, result)}
    return result
<commit_msg>Go back to using invalid field<commit_after>import flask
import sqlalchemy

def get_group_list_data(fields=None, attrs={}):
    """
    Queries the database and returns list of group data constrained by the
    specified attributes.

    Arguments:
        fields: The fields to return. If None specified, then default_fields
                are used.
        attrs:  The attributes of the group to filter for.
    Returns:
        result: The fields and corresponding values of groups with desired
                attributes. In the form of a list of dicts with key:value of
                columnname:columnvalue.
    """
    all_returnable_fields = [
        "group_id", "group_name", "group_desc", "type", "anyone_can_send",
        "members_can_send", "newsgroups", "visible", "admin_control_members"
    ]
    default_fields = ["group_id", "group_name", "group_desc", "type"]
    if fields == None:
        fields = default_fields
    else:
        if any(f not in all_returnable_fields for f in fields):
            return "Invalid field"

    # Build the SELECT and FROM clauses
    s = sqlalchemy.sql.select(fields).select_from(sqlalchemy.text("groups"))

    # Build the WHERE clause
    for key, value in list(attrs.items()):
        s = s.where(sqlalchemy.text(key + "= :" + key))

    # Execute the query
    result = flask.g.db.execute(s, attrs).fetchall()

    # Return the rows in the form of a list of dicts
    result = [{f: t for f, t in zip(fields, res)} for res in result]
    return result

def get_group_data(group_id, fields=None):
    """
    Queries the databse and returns member data for the specified group_id.

    Arguments:
        group_id: The group to look up
        fields:   The fields to return. If None are specified, then
                  default_fields are used

    Returns:
        result:   The fields and corresponding values of group with group_id.
                  In the form of a dict with key:value of columnname:columnalue
    """
    all_returnable_fields = [
        "group_id", "group_name", "group_desc", "type", "anyone_can_send",
        "members_can_send", "newsgroups", "visible", "admin_control_members"
    ]
    default_fields = ["group_id", "group_name", "group_desc", "type"]
    if fields is None:
        fields = default_fields
    else:
        if any(f not in all_returnable_fields for f in fields):
            return "Invalid field"

    # Build the SELECT and FROM clauses
    s = sqlalchemy.sql.select(fields).select_from(sqlalchemy.text("groups"))

    # Build the WHERE clause
    s = s.where(sqlalchemy.text("group_id = :g"))

    # Execute the query
    result = flask.g.db.execute(s, g=group_id).first()

    # Check to see if query returned anything
    if result is None:
        return {}

    # Return the row in the form of a dict
    result = {f: t for f, t in zip(fields, result)}
    return result
<|endoftext|>


<commit_before>import numpy as np
import cv2
import sys
import scipy
from scipy import io

global frame

vid_name = "ID-EMaTF9-ArJY"

caffe_root = '/Users/mprat/Documents/repos/caffe/'
repo_root = '/Users/mprat/Documents/repos/vid-to-json-deploy/'
model_root = '/Users/mprat/Documents/repos/vid-to-json-cleaned/data/placesCNN/'
weights_root = '/Users/mprat/Documents/repos/vid-to-json-cleaned/data/conv5_weights/'
vid_dir = '/Users/mprat/Desktop/test_vids/'

people_weightfile_name = 'places-people-train-activ-target-3.mat'
people_weight_data = scipy.io.loadmat(weights_root + people_weightfile_name, squeeze_me=True)
people_weight_data = people_weight_data['weights']
nonzero_people_weights = np.nonzero(people_weight_data)[0]

thresh = 0.01

sys.path.insert(0, caffe_root + 'python')
import caffe

# caffe stuff
MODEL_FILE = model_root + 'places205CNN_deploy_FC7_upgraded_one.prototxt'
CAFFEMODEL_FILE = model_root + 'places205CNN_iter_300000_upgraded.caffemodel'
MEAN_FILE = model_root + 'ilsvrc_2012_mean.npy'

caffe.set_mode_cpu()
net = caffe.Net(MODEL_FILE, CAFFEMODEL_FILE, caffe.TEST)

transformer = caffe.io.Transformer({'data': net.blobs['data'].data.shape})
transformer.set_transpose('data', (2, 0, 1)) # channels, then width, then height
transformer.set_mean('data', np.load(model_root + 'ilsvrc_2012_mean.npy').mean(1).mean(1))
# transformer.set_raw_scale('data', 255) # don't need this because opencv reads in 255 scale anyway
transformer.set_channel_swap('data', (2, 1, 0))

def extract_person_box(frame):
	# input the data
	net.blobs['data'].data[...] = transformer.preprocess('data', frame)
	# send it through the net
	out = net.forward()

	# extract the conv5 features
	conv5_features = net.blobs['conv5'].data # the shape is (1, 256, 13, 13)
	conv5_features = conv5_features[0]

	final_mask = np.zeros((13, 13))

	# multiply them by the weights
	for nonzero_index in nonzero_people_weights:
		final_mask += conv5_features[nonzero_index] * people_weight_data[nonzero_index]

	# transform to heatmap
	heatmap = scipy.ndimage.zoom(final_mask, 227/13.0, order=1)
	# plt.subplot(1, 2, 1)
	# plt.imshow(heatmap)
	cv2.imshow('heatmap', heatmap)

	# transform to bounding box
	[y_index, x_index] = np.where(heatmap > thresh)
	if (len(y_index) > 0 and len(x_index) > 0):
		box = [min(x_index), min(y_index), max(x_index), max(y_index)]
		box[2] = box[2] - box[0]
		box[3] = box[3] - box[1]
		# bounding_box_patch.set_bounds(box[0], box[1]+box[3], box[2], -1*box[3])
		return box
	return []

def run_video():
	cap = cv2.VideoCapture(vid_dir + vid_name + '.mp4')

	index = 0
	while (cap.isOpened()):
		ret, frame = cap.read()

		cv2.imshow('frame', frame)

		# only extract the person box every 10 frames
		# do some caffe stuff
		if index%10 == 0:
			person_box = extract_person_box(frame)
			print person_box

		if cv2.waitKey(1) & 0xFF == ord('q'):
			break

		index += 1

	cap.release()
	cv2.destroyAllWindows()

if __name__ == '__main__':
	run_video()<commit_msg>WIP: fix the box corners<commit_after>import numpy as np
import cv2
import sys
import scipy
from scipy import io

global frame

vid_name = "ID-EMaTF9-ArJY"

caffe_root = '/Users/mprat/Documents/repos/caffe/'
repo_root = '/Users/mprat/Documents/repos/vid-to-json-deploy/'
model_root = '/Users/mprat/Documents/repos/vid-to-json-cleaned/data/placesCNN/'
weights_root = '/Users/mprat/Documents/repos/vid-to-json-cleaned/data/conv5_weights/'
vid_dir = '/Users/mprat/Desktop/test_vids/'

people_weightfile_name = 'places-people-train-activ-target-3.mat'
people_weight_data = scipy.io.loadmat(weights_root + people_weightfile_name, squeeze_me=True)
people_weight_data = people_weight_data['weights']
nonzero_people_weights = np.nonzero(people_weight_data)[0]

thresh = 0.01

sys.path.insert(0, caffe_root + 'python')
import caffe

# caffe stuff
MODEL_FILE = model_root + 'places205CNN_deploy_FC7_upgraded_one.prototxt'
CAFFEMODEL_FILE = model_root + 'places205CNN_iter_300000_upgraded.caffemodel'
MEAN_FILE = model_root + 'ilsvrc_2012_mean.npy'

caffe.set_mode_cpu()
net = caffe.Net(MODEL_FILE, CAFFEMODEL_FILE, caffe.TEST)

transformer = caffe.io.Transformer({'data': net.blobs['data'].data.shape})
transformer.set_transpose('data', (2, 0, 1)) # channels, then width, then height
transformer.set_mean('data', np.load(model_root + 'ilsvrc_2012_mean.npy').mean(1).mean(1))
# transformer.set_raw_scale('data', 255) # don't need this because opencv reads in 255 scale anyway
transformer.set_channel_swap('data', (2, 1, 0))

def extract_box(frame):
	# input the data
	net.blobs['data'].data[...] = transformer.preprocess('data', frame)
	# send it through the net
	out = net.forward()

	# extract the conv5 features
	conv5_features = net.blobs['conv5'].data # the shape is (1, 256, 13, 13)
	conv5_features = conv5_features[0]

	final_mask = np.zeros((13, 13))

	# multiply them by the weights
	for nonzero_index in nonzero_people_weights:
		final_mask += conv5_features[nonzero_index] * people_weight_data[nonzero_index]

	# transform to heatmap
	heatmap = scipy.ndimage.zoom(final_mask, 227/13.0, order=1)
	# plt.subplot(1, 2, 1)
	# plt.imshow(heatmap)
	cv2.imshow('heatmap', heatmap)

	# transform to bounding box
	[y_index, x_index] = np.where(heatmap > thresh)
	if (len(y_index) > 0 and len(x_index) > 0):
		box = [min(x_index), min(y_index), max(x_index), max(y_index)]
		# box[2] = box[2] - box[0]
		# box[3] = box[3] - box[1]
		# bounding_box_patch.set_bounds(box[0], box[1]+box[3], box[2], -1*box[3])
		return box
	return []

def run_video():
	cap = cv2.VideoCapture(vid_dir + vid_name + '.mp4')

	index = 0
	while (cap.isOpened()):
		ret, frame = cap.read()

		# only extract the person box every 10 frames
		# do some caffe stuff
		if index%10 == 0:
			person_box = extract_box(frame)
			print person_box
			if len(person_box) > 0:
				# TODO: fix the problem that this right now
				# is drawing to the scale of the 227x227
				cv2.rectangle(frame, (person_box[0]*frame.shape[1]/227, person_box[1]*frame.shape[1]/227), (person_box[2]*frame.shape[0]/227, person_box[3]*frame.shape[0]/227), (255, 0, 0))

		cv2.imshow('frame', frame)

		if cv2.waitKey(1) & 0xFF == ord('q'):
			break

		index += 1

	cap.release()
	cv2.destroyAllWindows()

if __name__ == '__main__':
	run_video()<|endoftext|>


<commit_before>from __future__ import absolute_import
from distutils.version import LooseVersion

from datetime import datetime
from django.http import JsonResponse, Http404, HttpResponse, HttpResponseBadRequest
from django.utils.translation import ugettext as _
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_POST, require_GET

from corehq.apps.app_manager.dbaccessors import get_app_cached
from dimagi.utils.logging import notify_exception
from casexml.apps.case.cleanup import claim_case, get_first_claim
from casexml.apps.case.fixtures import CaseDBFixture
from casexml.apps.case.models import CommCareCase

from corehq import toggles
from corehq.const import OPENROSA_VERSION_MAP
from corehq.middleware import OPENROSA_VERSION_HEADER
from corehq.apps.app_manager.util import LatestAppInfo
from corehq.apps.case_search.models import QueryMergeException
from corehq.apps.case_search.utils import CaseSearchCriteria
from corehq.apps.domain.decorators import (
    mobile_auth,
    check_domain_migration,
    mobile_auth_or_token,
)
from corehq.apps.domain.models import Domain
from corehq.apps.es.case_search import flatten_result
from corehq.apps.users.models import CouchUser, DeviceAppMeta
from corehq.apps.locations.permissions import location_safe
from corehq.form_processor.exceptions import CaseNotFound
from casexml.apps.phone.restore import RestoreConfig, RestoreParams, RestoreCacheSettings

from .models import SerialIdBucket
from .utils import (
    demo_user_restore_response, get_restore_user, is_permitted_to_restore,
    handle_401_response)
from corehq.apps.users.util import update_device_meta

@location_safe
@handle_401_response
@mobile_auth_or_token
@check_domain_migration
def restore(request, domain, app_id=None):
    """
    We override restore because we have to supply our own
    user model (and have the domain in the url)
    """
    response, timing_context = get_restore_response(
        domain, request.couch_user, app_id, **get_restore_params(request))
    return response

@location_safe
@mobile_auth
@check_domain_migration
def search(request, domain):
    """
    Accepts search criteria as GET params, e.g. "https://www.commcarehq.org/a/domain/phone/search/?a=b&c=d"
    Returns results as a fixture with the same structure as a casedb instance.
    """
    criteria = request.GET.dict()
    try:
        case_type = criteria.pop('case_type')
    except KeyError:
        return HttpResponse('Search request must specify case type', status=400)
    try:
        case_search_criteria = CaseSearchCriteria(domain, case_type, criteria)
        search_es = case_search_criteria.search_es
    except QueryMergeException as e:
        return _handle_query_merge_exception(request, e)
    try:
        hits = search_es.run().raw_hits
    except Exception as e:
        return _handle_es_exception(request, e, case_search_criteria.query_addition_debug_details)

    # Even if it's a SQL domain, we just need to render the hits as cases, so CommCareCase.wrap will be fine
    cases = [CommCareCase.wrap(flatten_result(result)) for result in hits]
    fixtures = CaseDBFixture(cases).fixture
    return HttpResponse(fixtures, content_type="text/xml; charset=utf-8")

def _handle_query_merge_exception(request, exception):
    notify_exception(request, exception.message, details=dict(
        exception_type=type(exception),
        original_query=getattr(exception, "original_query", None),
        query_addition=getattr(exception, "query_addition", None)
    ))
    return HttpResponse(status=500)

def _handle_es_exception(request, exception, query_addition_debug_details):
    notify_exception(request, exception, details=dict(
        exception_type=type(exception),
        **query_addition_debug_details
    ))
    return HttpResponse(status=500)

@location_safe
@csrf_exempt
@require_POST
@mobile_auth
@check_domain_migration
def claim(request, domain):
    """
    Allows a user to claim a case that they don't own.
    """
    as_user = request.POST.get('commcare_login_as', None)
    as_user_obj = CouchUser.get_by_username(as_user) if as_user else None
    restore_user = get_restore_user(domain, request.couch_user, as_user_obj)

    case_id = request.POST.get('case_id', None)
    if case_id is None:
        return HttpResponse('A case_id is required', status=400)

    try:
        if get_first_claim(domain, restore_user.user_id, case_id):
            return HttpResponse('You have already claimed that {}'.format(request.POST.get('case_type', 'case')),
                                status=409)

        claim_case(domain, restore_user.user_id, case_id,
                   host_type=request.POST.get('case_type'),
                   host_name=request.POST.get('case_name'),
                   device_id=__name__ + ".claim")
    except CaseNotFound:
        return HttpResponse('The case "{}" you are trying to claim was not found'.format(case_id),
                            status=410)
    return HttpResponse(status=200)

def get_restore_params(request):
    """
    Given a request, get the relevant restore parameters out with sensible defaults
    """
    # not a view just a view util
    try:
        openrosa_headers = getattr(request, 'openrosa_headers', {})
        openrosa_version = openrosa_headers[OPENROSA_VERSION_HEADER]
    except KeyError:
        openrosa_version = request.GET.get('openrosa_version', None)

    return {
        'since': request.GET.get('since'),
        'version': request.GET.get('version', "2.0"),
        'state': request.GET.get('state'),
        'items': request.GET.get('items') == 'true',
        'as_user': request.GET.get('as'),
        'overwrite_cache': request.GET.get('overwrite_cache') == 'true',
        'openrosa_version': openrosa_version,
        'device_id': request.GET.get('device_id'),
        'user_id': request.GET.get('user_id'),
        'case_sync': request.GET.get('case_sync'),
    }

def get_restore_response(domain, couch_user, app_id=None, since=None, version='1.0',
                         state=None, items=False, force_cache=False,
                         cache_timeout=None, overwrite_cache=False,
                         as_user=None, device_id=None, user_id=None,
                         openrosa_version=None,
                         case_sync=None):

    if user_id and user_id != couch_user.user_id:
        # sync with a user that has been deleted but a new
        # user was created with the same username and password
        from couchforms.openrosa_response import get_simple_response_xml
        from couchforms.openrosa_response import ResponseNature
        response = get_simple_response_xml(
            'Attempt to sync with invalid user.',
            ResponseNature.OTA_RESTORE_ERROR
        )
        return HttpResponse(response, content_type="text/xml; charset=utf-8", status=412), None

    is_demo_restore = couch_user.is_commcare_user() and couch_user.is_demo_user
    if is_demo_restore:
        # if user is in demo-mode, return demo restore
        return demo_user_restore_response(couch_user), None

    uses_login_as = bool(as_user)
    as_user_obj = CouchUser.get_by_username(as_user) if uses_login_as else None
    if uses_login_as and not as_user_obj:
        msg = _(u'Invalid restore as user {}').format(as_user)
        return HttpResponse(msg, status=401), None
    is_permitted, message = is_permitted_to_restore(
        domain,
        couch_user,
        as_user_obj,
    )
    if not is_permitted:
        return HttpResponse(message, status=401), None

    couch_restore_user = as_user_obj if uses_login_as else couch_user
    app = app_meta = None
    if app_id:
        app = get_app_cached(domain, app_id)
        app_meta = DeviceAppMeta(
            app_id=app.master_id,
            build_id=app_id if app.copy_of else None,
            last_sync=datetime.utcnow(),
        )
    update_device_meta(couch_restore_user, device_id, device_app_meta=app_meta)

    restore_user = get_restore_user(domain, couch_user, as_user_obj)
    if not restore_user:
        return HttpResponse('Could not find user', status=404), None

    project = Domain.get_by_name(domain)
    async_restore_enabled = (
        toggles.ASYNC_RESTORE.enabled(domain)
        and openrosa_version
        and LooseVersion(openrosa_version) >= LooseVersion(OPENROSA_VERSION_MAP['ASYNC_RESTORE'])
    )
    restore_config = RestoreConfig(
        project=project,
        restore_user=restore_user,
        params=RestoreParams(
            sync_log_id=since,
            version=version,
            state_hash=state,
            include_item_count=items,
            app=app,
            device_id=device_id,
            openrosa_version=openrosa_version,
        ),
        cache_settings=RestoreCacheSettings(
            force_cache=force_cache or async_restore_enabled,
            cache_timeout=cache_timeout,
            overwrite_cache=overwrite_cache
        ),
        async=async_restore_enabled,
        case_sync=case_sync,
    )
    return restore_config.get_response(), restore_config.timing_context

@mobile_auth
@require_GET
def heartbeat(request, domain, app_build_id):
    """
    An endpoint for CommCare mobile to get latest CommCare APK and app version
        info. (Should serve from cache as it's going to be busy view)

    'app_build_id' (that comes from URL) can be id of any version of the app
    'app_id' (urlparam) is usually id of an app that is not a copy
        mobile simply needs it to be resent back in the JSON, and doesn't
        need any validation on it. This is pulled from @uniqueid from profile.xml
    """
    def _safe_int(val):
        try:
            return int(val)
        except:
            pass

    app_id = request.GET.get('app_id', '')

    app_version = _safe_int(request.GET.get('app_version', ''))
    device_id = request.GET.get('device_id', '')
    last_sync_time = request.GET.get('last_sync_time', '')
    num_unsent_forms = _safe_int(request.GET.get('num_unsent_forms', ''))
    num_quarantined_forms = _safe_int(request.GET.get('num_quarantined_forms', ''))
    commcare_version = request.GET.get('cc_version', '')

    info = {"app_id": app_id}
    try:
        # mobile will send brief_app_id
        info.update(LatestAppInfo(app_id, domain).get_info())
    except (Http404, AssertionError):
        # If it's not a valid 'brief' app id, find it by talking to couch
        notify_exception(request, 'Received an invalid heartbeat request')
        app = get_app_cached(domain, app_build_id)
        brief_app_id = app.master_id
        info.update(LatestAppInfo(brief_app_id, domain).get_info())
    else:
        couch_user = request.couch_user
        save_user = update_latest_builds(couch_user, app_id, datetime.utcnow(), app_version)
        try:
            last_sync = adjust_text_to_datetime(last_sync_time)
        except iso8601.ParseError:
            last_sync = None
        else:
            save_user |= update_last_sync(couch_user, app_id, last_sync, app_version)

        app_meta = DeviceAppMeta(
            app_id=app_id,
            build_id=app_build_id,
            build_version=app_version,
            last_heartbeat=datetime.utcnow(),
            last_sync=last_sync,
            num_unsent_forms=num_unsent_forms,
            num_quarantined_forms=num_quarantined_forms
        )
        save_user |= update_device_meta(
            couch_user,
            device_id,
            commcare_version=commcare_version,
            device_app_meta=app_meta,
            save=False
        )

        if save_user:
            couch_user.save(fire_signals=False)

    return JsonResponse(info)

@location_safe
@mobile_auth
@require_GET
def get_next_id(request, domain):
    bucket_id = request.GET.get('pool_id')
    session_id = request.GET.get('session_id')
    if bucket_id is None:
        return HttpResponseBadRequest("You must provide a pool_id parameter")
    return HttpResponse(SerialIdBucket.get_next(domain, bucket_id, session_id))
<commit_msg>add missing imports<commit_after>from __future__ import absolute_import
from distutils.version import LooseVersion

from datetime import datetime
from django.http import JsonResponse, Http404, HttpResponse, HttpResponseBadRequest
from django.utils.translation import ugettext as _
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_POST, require_GET
from iso8601 import iso8601

from corehq.apps.app_manager.dbaccessors import get_app_cached
from corehq.form_processor.utils.xform import adjust_text_to_datetime
from dimagi.utils.logging import notify_exception
from casexml.apps.case.cleanup import claim_case, get_first_claim
from casexml.apps.case.fixtures import CaseDBFixture
from casexml.apps.case.models import CommCareCase

from corehq import toggles
from corehq.const import OPENROSA_VERSION_MAP
from corehq.middleware import OPENROSA_VERSION_HEADER
from corehq.apps.app_manager.util import LatestAppInfo
from corehq.apps.case_search.models import QueryMergeException
from corehq.apps.case_search.utils import CaseSearchCriteria
from corehq.apps.domain.decorators import (
    mobile_auth,
    check_domain_migration,
    mobile_auth_or_token,
)
from corehq.apps.domain.models import Domain
from corehq.apps.es.case_search import flatten_result
from corehq.apps.users.models import CouchUser, DeviceAppMeta
from corehq.apps.locations.permissions import location_safe
from corehq.form_processor.exceptions import CaseNotFound
from casexml.apps.phone.restore import RestoreConfig, RestoreParams, RestoreCacheSettings

from .models import SerialIdBucket
from .utils import (
    demo_user_restore_response, get_restore_user, is_permitted_to_restore,
    handle_401_response)
from corehq.apps.users.util import update_device_meta, update_latest_builds, update_last_sync

@location_safe
@handle_401_response
@mobile_auth_or_token
@check_domain_migration
def restore(request, domain, app_id=None):
    """
    We override restore because we have to supply our own
    user model (and have the domain in the url)
    """
    response, timing_context = get_restore_response(
        domain, request.couch_user, app_id, **get_restore_params(request))
    return response

@location_safe
@mobile_auth
@check_domain_migration
def search(request, domain):
    """
    Accepts search criteria as GET params, e.g. "https://www.commcarehq.org/a/domain/phone/search/?a=b&c=d"
    Returns results as a fixture with the same structure as a casedb instance.
    """
    criteria = request.GET.dict()
    try:
        case_type = criteria.pop('case_type')
    except KeyError:
        return HttpResponse('Search request must specify case type', status=400)
    try:
        case_search_criteria = CaseSearchCriteria(domain, case_type, criteria)
        search_es = case_search_criteria.search_es
    except QueryMergeException as e:
        return _handle_query_merge_exception(request, e)
    try:
        hits = search_es.run().raw_hits
    except Exception as e:
        return _handle_es_exception(request, e, case_search_criteria.query_addition_debug_details)

    # Even if it's a SQL domain, we just need to render the hits as cases, so CommCareCase.wrap will be fine
    cases = [CommCareCase.wrap(flatten_result(result)) for result in hits]
    fixtures = CaseDBFixture(cases).fixture
    return HttpResponse(fixtures, content_type="text/xml; charset=utf-8")

def _handle_query_merge_exception(request, exception):
    notify_exception(request, exception.message, details=dict(
        exception_type=type(exception),
        original_query=getattr(exception, "original_query", None),
        query_addition=getattr(exception, "query_addition", None)
    ))
    return HttpResponse(status=500)

def _handle_es_exception(request, exception, query_addition_debug_details):
    notify_exception(request, exception, details=dict(
        exception_type=type(exception),
        **query_addition_debug_details
    ))
    return HttpResponse(status=500)

@location_safe
@csrf_exempt
@require_POST
@mobile_auth
@check_domain_migration
def claim(request, domain):
    """
    Allows a user to claim a case that they don't own.
    """
    as_user = request.POST.get('commcare_login_as', None)
    as_user_obj = CouchUser.get_by_username(as_user) if as_user else None
    restore_user = get_restore_user(domain, request.couch_user, as_user_obj)

    case_id = request.POST.get('case_id', None)
    if case_id is None:
        return HttpResponse('A case_id is required', status=400)

    try:
        if get_first_claim(domain, restore_user.user_id, case_id):
            return HttpResponse('You have already claimed that {}'.format(request.POST.get('case_type', 'case')),
                                status=409)

        claim_case(domain, restore_user.user_id, case_id,
                   host_type=request.POST.get('case_type'),
                   host_name=request.POST.get('case_name'),
                   device_id=__name__ + ".claim")
    except CaseNotFound:
        return HttpResponse('The case "{}" you are trying to claim was not found'.format(case_id),
                            status=410)
    return HttpResponse(status=200)

def get_restore_params(request):
    """
    Given a request, get the relevant restore parameters out with sensible defaults
    """
    # not a view just a view util
    try:
        openrosa_headers = getattr(request, 'openrosa_headers', {})
        openrosa_version = openrosa_headers[OPENROSA_VERSION_HEADER]
    except KeyError:
        openrosa_version = request.GET.get('openrosa_version', None)

    return {
        'since': request.GET.get('since'),
        'version': request.GET.get('version', "2.0"),
        'state': request.GET.get('state'),
        'items': request.GET.get('items') == 'true',
        'as_user': request.GET.get('as'),
        'overwrite_cache': request.GET.get('overwrite_cache') == 'true',
        'openrosa_version': openrosa_version,
        'device_id': request.GET.get('device_id'),
        'user_id': request.GET.get('user_id'),
        'case_sync': request.GET.get('case_sync'),
    }

def get_restore_response(domain, couch_user, app_id=None, since=None, version='1.0',
                         state=None, items=False, force_cache=False,
                         cache_timeout=None, overwrite_cache=False,
                         as_user=None, device_id=None, user_id=None,
                         openrosa_version=None,
                         case_sync=None):

    if user_id and user_id != couch_user.user_id:
        # sync with a user that has been deleted but a new
        # user was created with the same username and password
        from couchforms.openrosa_response import get_simple_response_xml
        from couchforms.openrosa_response import ResponseNature
        response = get_simple_response_xml(
            'Attempt to sync with invalid user.',
            ResponseNature.OTA_RESTORE_ERROR
        )
        return HttpResponse(response, content_type="text/xml; charset=utf-8", status=412), None

    is_demo_restore = couch_user.is_commcare_user() and couch_user.is_demo_user
    if is_demo_restore:
        # if user is in demo-mode, return demo restore
        return demo_user_restore_response(couch_user), None

    uses_login_as = bool(as_user)
    as_user_obj = CouchUser.get_by_username(as_user) if uses_login_as else None
    if uses_login_as and not as_user_obj:
        msg = _(u'Invalid restore as user {}').format(as_user)
        return HttpResponse(msg, status=401), None
    is_permitted, message = is_permitted_to_restore(
        domain,
        couch_user,
        as_user_obj,
    )
    if not is_permitted:
        return HttpResponse(message, status=401), None

    couch_restore_user = as_user_obj if uses_login_as else couch_user
    app = app_meta = None
    if app_id:
        app = get_app_cached(domain, app_id)
        app_meta = DeviceAppMeta(
            app_id=app.master_id,
            build_id=app_id if app.copy_of else None,
            last_sync=datetime.utcnow(),
        )
    update_device_meta(couch_restore_user, device_id, device_app_meta=app_meta)

    restore_user = get_restore_user(domain, couch_user, as_user_obj)
    if not restore_user:
        return HttpResponse('Could not find user', status=404), None

    project = Domain.get_by_name(domain)
    async_restore_enabled = (
        toggles.ASYNC_RESTORE.enabled(domain)
        and openrosa_version
        and LooseVersion(openrosa_version) >= LooseVersion(OPENROSA_VERSION_MAP['ASYNC_RESTORE'])
    )
    restore_config = RestoreConfig(
        project=project,
        restore_user=restore_user,
        params=RestoreParams(
            sync_log_id=since,
            version=version,
            state_hash=state,
            include_item_count=items,
            app=app,
            device_id=device_id,
            openrosa_version=openrosa_version,
        ),
        cache_settings=RestoreCacheSettings(
            force_cache=force_cache or async_restore_enabled,
            cache_timeout=cache_timeout,
            overwrite_cache=overwrite_cache
        ),
        async=async_restore_enabled,
        case_sync=case_sync,
    )
    return restore_config.get_response(), restore_config.timing_context

@mobile_auth
@require_GET
def heartbeat(request, domain, app_build_id):
    """
    An endpoint for CommCare mobile to get latest CommCare APK and app version
        info. (Should serve from cache as it's going to be busy view)

    'app_build_id' (that comes from URL) can be id of any version of the app
    'app_id' (urlparam) is usually id of an app that is not a copy
        mobile simply needs it to be resent back in the JSON, and doesn't
        need any validation on it. This is pulled from @uniqueid from profile.xml
    """
    def _safe_int(val):
        try:
            return int(val)
        except:
            pass

    app_id = request.GET.get('app_id', '')

    app_version = _safe_int(request.GET.get('app_version', ''))
    device_id = request.GET.get('device_id', '')
    last_sync_time = request.GET.get('last_sync_time', '')
    num_unsent_forms = _safe_int(request.GET.get('num_unsent_forms', ''))
    num_quarantined_forms = _safe_int(request.GET.get('num_quarantined_forms', ''))
    commcare_version = request.GET.get('cc_version', '')

    info = {"app_id": app_id}
    try:
        # mobile will send brief_app_id
        info.update(LatestAppInfo(app_id, domain).get_info())
    except (Http404, AssertionError):
        # If it's not a valid 'brief' app id, find it by talking to couch
        notify_exception(request, 'Received an invalid heartbeat request')
        app = get_app_cached(domain, app_build_id)
        brief_app_id = app.master_id
        info.update(LatestAppInfo(brief_app_id, domain).get_info())
    else:
        couch_user = request.couch_user
        save_user = update_latest_builds(couch_user, app_id, datetime.utcnow(), app_version)
        try:
            last_sync = adjust_text_to_datetime(last_sync_time)
        except iso8601.ParseError:
            last_sync = None
        else:
            save_user |= update_last_sync(couch_user, app_id, last_sync, app_version)

        app_meta = DeviceAppMeta(
            app_id=app_id,
            build_id=app_build_id,
            build_version=app_version,
            last_heartbeat=datetime.utcnow(),
            last_sync=last_sync,
            num_unsent_forms=num_unsent_forms,
            num_quarantined_forms=num_quarantined_forms
        )
        save_user |= update_device_meta(
            couch_user,
            device_id,
            commcare_version=commcare_version,
            device_app_meta=app_meta,
            save=False
        )

        if save_user:
            couch_user.save(fire_signals=False)

    return JsonResponse(info)

@location_safe
@mobile_auth
@require_GET
def get_next_id(request, domain):
    bucket_id = request.GET.get('pool_id')
    session_id = request.GET.get('session_id')
    if bucket_id is None:
        return HttpResponseBadRequest("You must provide a pool_id parameter")
    return HttpResponse(SerialIdBucket.get_next(domain, bucket_id, session_id))
<|endoftext|>


<commit_before># coding=utf-8
from __future__ import print_function
from __future__ import absolute_import

'''
Module for colorfull pretty printing different data.

All functions should return first value passed to them to
allow easy inserting print function into expressions.
'''

import os
from difflib import ndiff
import xml.dom.minidom as minidom
from pygments import highlight
from pygments.lexers import XmlLexer
from pygments.formatters import TerminalFormatter
from pprint import pformat

colors = {
    'red': 91,
    'green': 92,
    '-':91, # for diffs
    '+':92,
    '?':94,
}

def xml(text):
    tree = minidom.parseString(text)
    print(highlight(tree.toprettyxml(), XmlLexer(), TerminalFormatter()))
    return text

def print_diff(a, b):
    raise NotImplementedError
    a_lines = render_json(a).splitlines(1)
    b_lines = render_json(b).splitlines(1)
    diff = ndiff(a_lines,b_lines)

    for line in diff:
        # TODO: may be use some DiffLexer if such exists?
        print('\033[%sm%s\033[0m'% (colors.get(line[0],0), line[:-1]))

def get_console_width():
    rows, columns = os.popen('stty size', 'r').read().split()
    return int(columns)

def wrap(text, max_width=None):
    if max_width is None:
        max_width = get_console_width()

    lines = text.splitlines()
    res = []
    for l in lines:
        while True:
            cut = l[:max_width]
            if cut:
                res.append(cut)
            else:
                break
            l = l[max_width:]
    return '\n'.join(res)

def bordered(text, max_width=None):
    '''
┌─┐  ╔═╗
│ │  ║ ║
└─┘  ╚═╝
    '''
    lines = wrap(text, max_width).splitlines()
    width = max(len(s) for s in lines)
    res = ['┌' + '─' * width + '┐']
    for s in lines:
        res.append('│' + (s + ' ' * width)[:width] + '│')
    res.append('└' + '─' * width + '┘')
    return '\n'.join(res)

def pprint(*args):
    for arg in args:
        print(bordered(pformat(arg)))
    return args[0]
<commit_msg>pprint as a object<commit_after># coding=utf-8
from __future__ import print_function
from __future__ import absolute_import

'''
Module for colorfull pretty printing different data.

All functions should return first value passed to them to
allow easy inserting print function into expressions.
'''

import os
import sys
from difflib import ndiff
import xml.dom.minidom as minidom
from pygments import highlight
from pygments.lexers import XmlLexer
from pygments.formatters import TerminalFormatter
from pprint import pformat

colors = {
    'white': 90,
    'red': 91,
    'green': 92,
    '-':91, # for diffs
    '+':92,
    '?':94,
}

def xml(text):
    tree = minidom.parseString(text)
    print(highlight(tree.toprettyxml(), XmlLexer(), TerminalFormatter()))
    return text

def print_diff(a, b):
    raise NotImplementedError
    a_lines = render_json(a).splitlines(1)
    b_lines = render_json(b).splitlines(1)
    diff = ndiff(a_lines,b_lines)

    for line in diff:
        # TODO: may be use some DiffLexer if such exists?
        print('\033[%sm%s\033[0m'% (colors.get(line[0],0), line[:-1]))

def wrap(text, max_width):
    lines = text.splitlines()
    res = []
    for l in lines:
        while True:
            cut = l[:max_width]
            if cut:
                res.append(cut)
            else:
                break
            l = l[max_width:]
    return '\n'.join(res)

def bordered(text):
    '''
┌─┐  ╔═╗
│ │  ║ ║
└─┘  ╚═╝
    '''
    lines = text.splitlines()
    width = max(len(s) for s in lines)
    res = ['┌' + '─' * width + '┐']
    for s in lines:
        res.append('│' + (s + ' ' * width)[:width] + '│')
    res.append('└' + '─' * width + '┘')
    return '\n'.join(res)

class Printer(object):
    pretty = True
    bordered = False
    max_width = None
    outfile = sys.stdout

    def __call__(self, *args):
        out = []
        for arg in args:
            item = pformat(arg) if self.pretty else str(arg)
            if self.max_width:
                item = wrap(item, self.max_width)
            if self.bordered:
                item = bordered(item)
            out.append(item)
        print(' '.join(out), file=self.outfile)

    def color_code(self, name):
        return '\033[%sm' % colors[name]

    def set_color(self, name):
        self._color = name
        print(self.color_code(name), sep='', end='')

    def get_color(self):
        return self._color

    color = property(get_color, set_color)

    @classmethod
    def get_console_width():
        rows, columns = os.popen('stty size', 'r').read().split()
        return int(columns)

pprint = Printer()
<|endoftext|>


<commit_before>import unittest

from pyiem import datatypes, meteorology

class TestDatatypes(unittest.TestCase):

    def test_heatindex(self):
        ''' Test our heat index calculations '''
        t = datatypes.temperature(80.0, 'F')
        td = datatypes.temperature(70.0, 'F')
        hdx = meteorology.heatindex(t, td)
        self.assertAlmostEqual( hdx.value("F"), 83.93, 2)

        t = datatypes.temperature(30.0, 'F')
        hdx = meteorology.heatindex(t, td)
        self.assertAlmostEqual( hdx.value("F"), 30.00, 2)

    def test_uv(self):
        """ Test calculation of uv wind components """
        speed = datatypes.speed([10,], 'KT')
        mydir = datatypes.direction([0,], 'DEG')
        u,v = meteorology.uv(speed, mydir)
        self.assertEqual(u.value("KT"), 0.)
        self.assertEqual(v.value("KT"), -10.)

        speed = datatypes.speed([10,20,15], 'KT')
        mydir = datatypes.direction([90,180,135], 'DEG')
        u,v = meteorology.uv(speed, mydir)
        self.assertEqual(u.value("KT")[0], -10)
        self.assertEqual(v.value("KT")[1], 20.)
        self.assertAlmostEquals(v.value("KT")[2], 10.6, 1)

    def test_relh(self):
        """ Simple check of bad units in temperature """
        tmp = datatypes.temperature(24, 'C')
        dwp = datatypes.temperature(24, 'C')
        relh = meteorology.relh(tmp, dwp)
        self.assertEquals(100.0, relh.value("%"))

        tmp = datatypes.temperature(32, 'C')
        dwp = datatypes.temperature(10, 'C')
        relh = meteorology.relh(tmp, dwp)
        self.assertAlmostEquals(25.79, relh.value("%"), 2)<commit_msg>add more RH tests to make me feel better<commit_after>import unittest

from pyiem import datatypes, meteorology

class TestDatatypes(unittest.TestCase):

    def test_heatindex(self):
        ''' Test our heat index calculations '''
        t = datatypes.temperature(80.0, 'F')
        td = datatypes.temperature(70.0, 'F')
        hdx = meteorology.heatindex(t, td)
        self.assertAlmostEqual( hdx.value("F"), 83.93, 2)

        t = datatypes.temperature(30.0, 'F')
        hdx = meteorology.heatindex(t, td)
        self.assertAlmostEqual( hdx.value("F"), 30.00, 2)

    def test_uv(self):
        """ Test calculation of uv wind components """
        speed = datatypes.speed([10,], 'KT')
        mydir = datatypes.direction([0,], 'DEG')
        u,v = meteorology.uv(speed, mydir)
        self.assertEqual(u.value("KT"), 0.)
        self.assertEqual(v.value("KT"), -10.)

        speed = datatypes.speed([10,20,15], 'KT')
        mydir = datatypes.direction([90,180,135], 'DEG')
        u,v = meteorology.uv(speed, mydir)
        self.assertEqual(u.value("KT")[0], -10)
        self.assertEqual(v.value("KT")[1], 20.)
        self.assertAlmostEquals(v.value("KT")[2], 10.6, 1)

    def test_relh(self):
        """ Simple check of bad units in temperature """
        tmp = datatypes.temperature(24, 'C')
        dwp = datatypes.temperature(24, 'C')
        relh = meteorology.relh(tmp, dwp)
        self.assertEquals(100.0, relh.value("%"))

        tmp = datatypes.temperature(32, 'C')
        dwp = datatypes.temperature(10, 'C')
        relh = meteorology.relh(tmp, dwp)
        self.assertAlmostEquals(25.79, relh.value("%"), 2)

        tmp = datatypes.temperature(32, 'C')
        dwp = datatypes.temperature(15, 'C')
        relh = meteorology.relh(tmp, dwp)
        self.assertAlmostEquals(35.81, relh.value("%"), 2)

        tmp = datatypes.temperature(5, 'C')
        dwp = datatypes.temperature(4, 'C')
        relh = meteorology.relh(tmp, dwp)
        self.assertAlmostEquals(93.24, relh.value("%"), 2)<|endoftext|>


<commit_before># pylint: disable-msg=W0102, W0704
# Copyright 2008 German Aerospace Center (DLR)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

""" Class to work with transactions. """

import os
import re
import shutil
import tempfile

from repoguard.core import process

class FileNotFoundException(Exception):
    def __init__(self, filename):
        Exception.__init__(self, "No such file in repository or transaction %r." % filename)

class PropertyNotFoundException(Exception):
    def __init__(self, keyword, filename):
        Exception.__init__(self, "Property %r for file %r not set." % (keyword, filename))

class Transaction(object):

    def __init__(self, repos_path, txn_name):
        """
        Initialize the transaction object.
        """

        if txn_name is None: # HEAD revision
            self.type = "revision"
        else:
            txn_name = str(txn_name)
            try:
                int(txn_name)
                self.type = "revision"
            except ValueError:
                self.type = "transaction"

        self.repos_path = repos_path
        self.txn_name = txn_name
        self._profile = re.compile(".*")
        self.tmpdir = tempfile.mkdtemp()
        self.cache = {}

    def _execute_svn(self, command, arg="", split=False, raw_out=False):
        if self.txn_name is None:
            command = 'svnlook %s "%s" %s' % (command, self.repos_path, arg)
        else:
            command = 'svnlook --%s %s %s "%s" %s' % (self.type, self.txn_name, command, self.repos_path, arg)

        if command in self.cache:
            return self.cache[command]

        try:
            output = process.execute(command, raw_out)
        except process.ProcessException, error:
            if "Transaction '(null)'" in error.output: # Nothing bad happened we just have an empty repository
                output = ""
            else:
                raise

        if split:
            output = [x.strip() for x in output.split("\n") if x.strip()]

        self.cache[command] = output
        return self.cache[command]

    def cleanup(self):
        """
        Delete the temporary directory.
        Do NOT use in your checks or handlers.
        """
        shutil.rmtree(self.tmpdir)

    def _get_profile(self):
        """
        Returns the current profile.
        Do NOT use in your checks or handlers.
        """
        return self._profile

    def _set_profile(self, profile):
        """
        Sets the current profile.
        Do NOT use in your checks or handlers.
        """
        self._profile = re.compile(profile)

    def _get_user_id(self):
        """ Returns a string with the username of the current transaction. """
        user = self._execute_svn("author")
        return user.strip()

    def get_files(self, check_list=[".*"], ignore_list=[]):
        """
        Returns a map of all modified files. The keys of the map
        are the filenames. The values of the map is the
        associated attribute, which can be one of the default
        svnlook changed attributes:
        http://svnbook.red-bean.com/en/1.4/svn-book.html#svn.ref.svnlook.c.changed

        @param check_list List of regular expressions for files which should be included.
        @param ignore_list List of regular expressions for files which should be ignored.
        """

        output = self._execute_svn("changed", split=True)
        files = {}
        for entry in output:
            attributes = entry[0:3].strip()
            filename = entry[4:].strip()

            if self._profile.search(filename) and self.__check(filename, check_list) and not self.__check(filename, ignore_list):
                files[filename] = attributes
        return files

    def __check(self, datei, items):
        for item in items:
            regex = re.compile(item)
            if regex.search(datei):
                return True
        return False

    def get_file(self, filename):
        """ Returns the path to a temporary copy of a file in the repository. """
        if not self.file_exists(filename):
            raise FileNotFoundException(filename)

        tmpfilename = os.path.join(self.tmpdir, filename)
        if os.path.exists(tmpfilename):
            return tmpfilename

        content = self._execute_svn("cat", "\"" + filename + "\"", raw_out=True)

        dirname = os.path.dirname(filename)
        tmpdirname = os.path.join(self.tmpdir, dirname)
        if dirname and not os.path.exists(tmpdirname):
            os.makedirs(tmpdirname)

        file_object = open(tmpfilename, "w")
        try:
            file_object.write(content)
        finally:
            file_object.close()
        return tmpfilename

    def file_exists(self, filename, ignore_case=False):
        """
        Returns whether a file exists in the current transaction or revision of
        the repository, optionally case-insensitive.
        """
        exists = False

        if ignore_case:
            filename = filename.lower()
            files = self._execute_svn("tree", "--full-paths", split=True)
            count = 0
            for fname in files:
                if fname.lower() == filename:
                    count += 1
                    if count >= 2:
                        exists = True
                        break

        else:
            try:
                self._execute_svn("proplist", "\"" + filename + "\"", split=True)
                exists = True
            except process.ProcessException:
                pass

        return exists

    def _get_commit_msg(self):
        """
        Returns the commit message.
        """

        output = self._execute_svn("info", split=True)
        temp = output[3:]
        msg = "\n".join(temp)
        return msg.strip()

    def _get_revision(self):
        """
        Returns the id of the revision or transaction.
        """

        revision = "HEAD"
        if not self.txn_name is None:
            try:
                revision = int(self.txn_name.split("-")[0]) + 1
            except ValueError:
                revision = self.txn_name
        return str(revision)

    def get_property(self, keyword, filename):
        """
        Returns a specified property of a file.
        """

        if not self.has_property(keyword, filename):
            raise PropertyNotFoundException(keyword, filename)

        return self._execute_svn("propget", " ".join([keyword, "\"" + filename + "\""]))

    def has_property(self, keyword, filename):
        """
        Checks if a given file has the given property.
        """

        return keyword in self.list_properties(filename)

    def list_properties(self, filename):
        """
        Returns a list of names of the properties for a file.
        """

        if not self.file_exists(filename):
            raise FileNotFoundException(filename)

        return self._execute_svn("proplist", "\"" + filename + "\"", split=True)

    profile = property(_get_profile, _set_profile)
    user_id = property(_get_user_id)
    revision = property(_get_revision)
    commit_msg = property(_get_commit_msg)
<commit_msg>Fixing nasty bug when messing around with encodings in context of svn command output.<commit_after># pylint: disable-msg=W0102, W0704
# Copyright 2008 German Aerospace Center (DLR)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

""" Class to work with transactions. """

import os
import re
import shutil
import tempfile

from repoguard.core import process

class FileNotFoundException(Exception):
    def __init__(self, filename):
        Exception.__init__(self, "No such file in repository or transaction %r." % filename)

class PropertyNotFoundException(Exception):
    def __init__(self, keyword, filename):
        Exception.__init__(self, "Property %r for file %r not set." % (keyword, filename))

class Transaction(object):

    def __init__(self, repos_path, txn_name):
        """
        Initialize the transaction object.
        """

        if txn_name is None: # HEAD revision
            self.type = "revision"
        else:
            txn_name = str(txn_name)
            try:
                int(txn_name)
                self.type = "revision"
            except ValueError:
                self.type = "transaction"

        self.repos_path = repos_path
        self.txn_name = txn_name
        self._profile = re.compile(".*")
        self.tmpdir = tempfile.mkdtemp()
        self.cache = {}

    def _execute_svn(self, command, arg="", split=False):
        if self.txn_name is None:
            command = 'svnlook %s "%s" %s' % (command, self.repos_path, arg)
        else:
            command = 'svnlook --%s %s %s "%s" %s' % (self.type, self.txn_name, command, self.repos_path, arg)

        if command in self.cache:
            return self.cache[command]

        try:
            output = process.execute(command, raw_out=True)
        except process.ProcessException, error:
            if "Transaction '(null)'" in error.output: # Nothing bad happened we just have an empty repository
                output = ""
            else:
                raise

        if split:
            output = [x.strip() for x in output.split("\n") if x.strip()]

        self.cache[command] = output
        return self.cache[command]

    def cleanup(self):
        """
        Delete the temporary directory.
        Do NOT use in your checks or handlers.
        """
        shutil.rmtree(self.tmpdir)

    def _get_profile(self):
        """
        Returns the current profile.
        Do NOT use in your checks or handlers.
        """
        return self._profile

    def _set_profile(self, profile):
        """
        Sets the current profile.
        Do NOT use in your checks or handlers.
        """
        self._profile = re.compile(profile)

    def _get_user_id(self):
        """ Returns a string with the username of the current transaction. """
        user = self._execute_svn("author")
        return user.strip()

    def get_files(self, check_list=[".*"], ignore_list=[]):
        """
        Returns a map of all modified files. The keys of the map
        are the filenames. The values of the map is the
        associated attribute, which can be one of the default
        svnlook changed attributes:
        http://svnbook.red-bean.com/en/1.4/svn-book.html#svn.ref.svnlook.c.changed

        @param check_list List of regular expressions for files which should be included.
        @param ignore_list List of regular expressions for files which should be ignored.
        """

        output = self._execute_svn("changed", split=True)
        files = {}
        for entry in output:
            attributes = entry[0:3].strip()
            filename = entry[4:].strip()

            if self._profile.search(filename) and self.__check(filename, check_list) and not self.__check(filename, ignore_list):
                files[filename] = attributes
        return files

    def __check(self, datei, items):
        for item in items:
            regex = re.compile(item)
            if regex.search(datei):
                return True
        return False

    def get_file(self, filename):
        """ Returns the path to a temporary copy of a file in the repository. """
        if not self.file_exists(filename):
            raise FileNotFoundException(filename)

        tmpfilename = os.path.join(self.tmpdir, filename)
        if os.path.exists(tmpfilename):
            return tmpfilename

        content = self._execute_svn("cat", "\"" + filename + "\"")

        dirname = os.path.dirname(filename)
        tmpdirname = os.path.join(self.tmpdir, dirname)
        if dirname and not os.path.exists(tmpdirname):
            os.makedirs(tmpdirname)

        file_object = open(tmpfilename, "w")
        try:
            file_object.write(content)
        finally:
            file_object.close()
        return tmpfilename

    def file_exists(self, filename, ignore_case=False):
        """
        Returns whether a file exists in the current transaction or revision of
        the repository, optionally case-insensitive.
        """
        exists = False

        if ignore_case:
            filename = filename.lower()
            files = self._execute_svn("tree", "--full-paths", split=True)
            count = 0
            for fname in files:
                if fname.lower() == filename:
                    count += 1
                    if count >= 2:
                        exists = True
                        break

        else:
            try:
                self._execute_svn("proplist", "\"" + filename + "\"", split=True)
                exists = True
            except process.ProcessException:
                pass

        return exists

    def _get_commit_msg(self):
        """
        Returns the commit message.
        """

        output = self._execute_svn("info", split=True)
        temp = output[3:]
        msg = "\n".join(temp)
        return msg.strip()

    def _get_revision(self):
        """
        Returns the id of the revision or transaction.
        """

        revision = "HEAD"
        if not self.txn_name is None:
            try:
                revision = int(self.txn_name.split("-")[0]) + 1
            except ValueError:
                revision = self.txn_name
        return str(revision)

    def get_property(self, keyword, filename):
        """
        Returns a specified property of a file.
        """

        if not self.has_property(keyword, filename):
            raise PropertyNotFoundException(keyword, filename)

        return self._execute_svn("propget", " ".join([keyword, "\"" + filename + "\""]))

    def has_property(self, keyword, filename):
        """
        Checks if a given file has the given property.
        """

        return keyword in self.list_properties(filename)

    def list_properties(self, filename):
        """
        Returns a list of names of the properties for a file.
        """

        if not self.file_exists(filename):
            raise FileNotFoundException(filename)

        return self._execute_svn("proplist", "\"" + filename + "\"", split=True)

    profile = property(_get_profile, _set_profile)
    user_id = property(_get_user_id)
    revision = property(_get_revision)
    commit_msg = property(_get_commit_msg)
<|endoftext|>


<commit_before>"""
concerts/utils.py

Misc. utility functions for the concerts app.
"""

import logging, os

from .scrapers import (
    emptybottle,
    bottomlounge,
    subterranean,
    doubledoor,
    houseofblues,
    thaliahall,
    metro,
    lincolnhall,
    schubas,
)

LOGGER = logging.getLogger('concerts')

# key is Venue objects's pk; used by scrape_shows mgmt command
SCRAPERS = {
    1: emptybottle.EmptyBottle,
    2: bottomlounge.BottomLounge,
    3: subterranean.Subterranean,
    4: doubledoor.DoubleDoor,
    5: houseofblues.HouseOfBlues,
    6: thaliahall.ThaliaHall,
    7: metro.Metro,
    8: lincolnhall.LincolnHall,
    9: schubas.SchubasTavern,
}

FIXTURES_BASE_DIR = os.path.join(
                        os.environ.get('SIFT_HOME'),
                        'concerts',
                        'fixtures'
                    )

FIXTURE_DIRS = {
    'artists': os.path.join(FIXTURES_BASE_DIR, 'artists'),
    'concerts': os.path.join(FIXTURES_BASE_DIR, 'concerts'),
    'venues': os.path.join(FIXTURES_BASE_DIR, 'venues'),
}

def get_spotify_id(artist_name):
    """Returns Spotify artist ID, or empty string if none found."""

    import requests
    from django.utils.http import urlencode

    SPOTIFY_QUERY_BASE = 'https://api.spotify.com/v1/search?'

    # searching quoted artist name seems to give more accurate
    quoted_name = '\"' + artist_name + '\"'
    query_str = urlencode([('q', quoted_name), ('type', 'artist')])
    response = requests.get(SPOTIFY_QUERY_BASE + query_str)
    response.raise_for_status()

    artist_id = ''

    try:
        artist_id = response.json()['artists']['items'][0]['id']
    except IndexError as e:
        LOGGER.debug("Failed to get Spotify artist ID for {}".format(artist_name))

    return artist_id

def make_artist_regex(artist_name):
    """Returns regex string of artist name for storage in DB."""

    from string import punctuation
    import re

    PUNC_RE = re.compile(
        '[{}]'.format(punctuation),
        flags=re.I|re.M|re.DOTALL
    )

    no_punc = re.sub(PUNC_RE, r'.?', artist_name)
    no_end = re.sub(r'$', '\\\\b', no_punc)
    re_string = re.sub(r'^', '\\\\b', no_end)
    return re_string

if __name__ == '__main__':
    pass<commit_msg>Ignore Metro for time being<commit_after>"""
concerts/utils.py

Misc. utility functions for the concerts app.
"""

import logging, os

from .scrapers import (
    emptybottle,
    bottomlounge,
    subterranean,
    doubledoor,
    houseofblues,
    thaliahall,
    metro,
    lincolnhall,
    schubas,
)

LOGGER = logging.getLogger('concerts')

# key is Venue objects's pk; used by scrape_shows mgmt command
SCRAPERS = {
    1: emptybottle.EmptyBottle,
    2: bottomlounge.BottomLounge,
    3: subterranean.Subterranean,
    4: doubledoor.DoubleDoor,
    5: houseofblues.HouseOfBlues,
    6: thaliahall.ThaliaHall,
    #7: metro.Metro,
    8: lincolnhall.LincolnHall,
    9: schubas.SchubasTavern,
}

FIXTURES_BASE_DIR = os.path.join(
                        os.environ.get('SIFT_HOME'),
                        'concerts',
                        'fixtures'
                    )

FIXTURE_DIRS = {
    'artists': os.path.join(FIXTURES_BASE_DIR, 'artists'),
    'concerts': os.path.join(FIXTURES_BASE_DIR, 'concerts'),
    'venues': os.path.join(FIXTURES_BASE_DIR, 'venues'),
}

def get_spotify_id(artist_name):
    """Returns Spotify artist ID, or empty string if none found."""

    import requests
    from django.utils.http import urlencode

    SPOTIFY_QUERY_BASE = 'https://api.spotify.com/v1/search?'

    # searching quoted artist name seems to give more accurate
    quoted_name = '\"' + artist_name + '\"'
    query_str = urlencode([('q', quoted_name), ('type', 'artist')])
    response = requests.get(SPOTIFY_QUERY_BASE + query_str)
    response.raise_for_status()

    artist_id = ''

    try:
        artist_id = response.json()['artists']['items'][0]['id']
    except IndexError as e:
        LOGGER.debug("Failed to get Spotify artist ID for {}".format(artist_name))

    return artist_id

def make_artist_regex(artist_name):
    """Returns regex string of artist name for storage in DB."""

    from string import punctuation
    import re

    PUNC_RE = re.compile(
        '[{}]'.format(punctuation),
        flags=re.I|re.M|re.DOTALL
    )

    no_punc = re.sub(PUNC_RE, r'.?', artist_name)
    no_end = re.sub(r'$', '\\\\b', no_punc)
    re_string = re.sub(r'^', '\\\\b', no_end)
    return re_string

if __name__ == '__main__':
    pass
<|endoftext|>