<commit_before>from django.conf import settings
from django.contrib import admin
from django.utils.translation import ugettext_lazy as _

from django.utils.safestring import mark_safe
from django.urls import reverse

from django.utils import timezone
from django.utils.formats import date_format

from .models import Ticket
from .models import MultiPurchase
from .models import TicketWarning
from .models import TicketSeatHold
from admin_csv import CSVMixin

from .filters import TicketWindowFilter
from .filters import SingleTicketWindowFilter
from windows.models import TicketWindowSale
from django.core.exceptions import MultipleObjectsReturned
from windows.utils import online_sale

from events.models import TicketField

from extended_filters.filters import DateRangeFilter
from congressus.admin import register

def confirm(modeladmin, request, queryset):
    for i in queryset:
        i.confirm()
confirm.short_description = _("Manual confirm")

def unconfirm(modeladmin, request, queryset):
    for i in queryset:
        i.confirmed = False
        i.remove_hold_seats()
        i.save()
unconfirm.short_description = _("Manual unconfirm")

def mark_used(modeladmin, request, queryset):
    now = timezone.now()
    for i in queryset:
        i.used = True
        i.used_date = now
        i.save()
mark_used.short_description = _("Mark like used")

def mark_no_used(modeladmin, request, queryset):
    for i in queryset:
        i.used = False
        i.used_date = None
        i.save()
mark_no_used.short_description = _("Mark like no used")

def delete_selected(modeladmin, request, queryset):
    for obj in queryset:
        obj.delete()
delete_selected.short_description = _("Remove")

class TicketAdmin(CSVMixin, admin.ModelAdmin):
    list_per_page = 20
    list_max_show_all = 800
    list_display = ('order', 'order_tpv2', 'session2', 'cseat', 'twin',
                    'created2', 'confirmed', 'used',
                    'email', 'payment', 'payment_method', 'price2', 'event')
    list_filter = (('created', DateRangeFilter), 'confirmed',
                   'payment_method', 'used', SingleTicketWindowFilter,
                   'event_name',
                   ('session__space', admin.RelatedOnlyFieldListFilter))
    search_fields = ('order', 'order_tpv', 'email', 'mp__order', 'mp__order_tpv')
    date_hierarchy = 'created'
    actions = [delete_selected, confirm, unconfirm, mark_used, mark_no_used]

    readonly_fields = (
        'order_tpv_linked', 'order', 'event_name', 'session2', 'cseat',
        'confirmed', 'confirmed_date', 'price', 'tax',
        'email', 'formated_extra_data',
        'used_date', 'gate_name',
        'start', 'end', 'twin'
    )

    fieldsets = (
        (None, {
            'fields': ('order', 'order_tpv_linked', 'event_name',
                       'session2', 'cseat', 'twin')
        }),
        (_('Personal info'), {
            'fields': ('email', 'formated_extra_data')
        }),
        (_('Extra info'), {
            'fields': (('confirmed', 'confirmed_date'),
                       ('price', 'tax'),
                       ('used', 'used_date', 'gate_name'),
                       'start', 'end')
        }),
    )

    @property
    def csv_fields(self):
        data = [
            'email',

            'order',
            'order_tpv',

            'confirmed',
            'confirmed_date',

            'price2',
            'cseat',
            'mp',
            'twin',

            'event_name',
            'space_name',
            'session_name',
            'created',

            'used',
            'used_at',
        ]

        if getattr(settings, 'CSV_TICKET_FIELDS'):
            data += ['ticket_field_' + i for i in settings.CSV_TICKET_FIELDS]

        return data

    def __getattr__(self, value):
        if value.startswith('ticket_field_'):
            pk = value.split("_")[-1]
            field = TicketField.objects.get(pk=pk)
            def f(obj):
                return obj.get_extra_data(field.label)
            f.short_description = field.label
            return f
        raise AttributeError

    def formated_extra_data(self, obj):
        extras = obj.get_extras_dict()
        html = '
'
        for k in sorted(extras.keys()):
            html += '
' + k + '	' + str(extras[k]) + '
'
        html += '
'
        return mark_safe(html)
    formated_extra_data.short_description = _('extra data')

    def order_tpv_linked(self, obj):
        if not obj.mp:
            return obj.order_tpv

        url = reverse('admin:tickets_multipurchase_change', args=(obj.mp.id,))
        html = '' + obj.mp.order_tpv + ''
        return mark_safe(html)
    order_tpv_linked.short_description = _('order TPV')

    def price2(self, obj):
        return obj.get_real_price()
    price2.short_description = _('price')

    def twin(self, obj):
        try:
            tws = TicketWindowSale.objects.get(purchase__tickets=obj)
        except:
            return '-'
        prefix = tws.window.code
        return prefix
    twin.short_description = _('ticket window')

    def order_tpv2(self, obj):
        if obj.mp:
            return obj.mp.order_tpv
        else:
            return obj.order_tpv
    order_tpv2.short_description = _('order TPV')

    def session2(self, obj):
        return obj.session.short()
    session2.short_description = _('session')

    def payment(self, obj):
        if not obj.mp:
            return '-'
        try:
            tws = TicketWindowSale.objects.get(purchase=obj.mp)
        except:
            return '-'
        return tws.get_payment_display()
    payment.short_description = _('payment')

    def created2(self, obj):
        fmt='d/m/y H:i:s'
        d1 = timezone.localtime(obj.created)
        return date_format(d1, fmt)
    created2.short_description = _('date')

    def used_at(self, obj):
        if not obj.used_date:
            return '-'
        fmt='d/m/y H:i:s'
        d1 = timezone.localtime(obj.used_date)
        return date_format(d1, fmt)
    used_at.short_description = _('used at')

    def event_filter(self, request, slug):
        qs = super().get_queryset(request)
        return qs.filter(session__space__event__slug=slug)

class TicketInline(admin.TabularInline):
    model = Ticket
    fields = ('link_order', 'session', 'price', 'seat_layout', 'seat')
    readonly_fields = fields

    def link_order(self, obj):
        url = reverse('admin:tickets_ticket_change', args=(obj.id,))
        html = '' + obj.order + ''
        return mark_safe(html)
    link_order.short_description = _('order')

    def has_delete_permission(self, request, obj):
        return False

    def has_add_permission(self, request):
        return False

def link_online_sale(modeladmin, request, queryset):
    for i in queryset:
        online_sale(i)
        i.save()
link_online_sale.short_description = _("Link online sale")

class MPAdmin(CSVMixin, admin.ModelAdmin):
    list_per_page = 20
    list_max_show_all = 800
    list_display = ('order_tpv', 'twin', 'created', 'confirmed2', 'email', 'ntickets', 'price', 'payment_method', 'event')
    list_filter = (('created', DateRangeFilter), 'confirmed', 'payment_method',
                   TicketWindowFilter,
                   ('ev', admin.RelatedOnlyFieldListFilter), 'tpv_error')

    search_fields = ('order', 'order_tpv', 'email', 'extra_data')
    date_hierarchy = 'created'
    actions = [delete_selected, confirm, unconfirm, link_online_sale]
    inlines = [TicketInline, ]

    @property
    def csv_fields(self):
        data = [
            'email',

            'order',
            'order_tpv',

            'confirmed',
            'confirmed_date',

            'price',
            'ntickets',
            'twin',

            'ev',
            'created',
        ]

        if getattr(settings, 'CSV_TICKET_FIELDS'):
            data += ['ticket_field_' + i for i in settings.CSV_TICKET_FIELDS]

        return data

    def __getattr__(self, value):
        if value.startswith('ticket_field_'):
            pk = value.split("_")[-1]
            field = TicketField.objects.get(pk=pk)
            def f(obj):
                return obj.get_extra_data(field.label)
            f.short_description = field.label
            return f
        raise AttributeError

    readonly_fields = (
        'order_tpv', 'order', 'ev',
        'confirmed', 'confirmed_date',
        'ntickets', 'price', 'payment',
        'formated_extra_data',
        'tpv_error', 'tpv_error_info', 'twin'
    )

    fieldsets = (
        (None, {
            'fields': ('order_tpv', 'order', 'ev', 'twin')
        }),
        (_('Personal info'), {
            'fields': (('email', 'confirm_sent'), 'formated_extra_data')
        }),
        (_('Extra info'), {
            'fields': (('confirmed', 'confirmed_date'),
                       ('ntickets', 'price'),
                       'discount', 'payment', 'payment_method',
                       'tpv_error', 'tpv_error_info',
                      )
        }),
    )

    def confirmed2(self, obj):
        icon = 'icon-no.svg'
        if obj.confirmed:
            icon = 'icon-yes.svg'
        elif obj.tpv_error:
            icon = 'icon-alert.svg'
        html = ''.format(settings.STATIC_URL, icon)
        return mark_safe(html)
    confirmed2.short_description = _('confirmed')

    def payment(self, obj):
        try:
            tws = TicketWindowSale.objects.get(purchase=obj)
        except:
            return '-'
        return tws.get_payment_display()
    payment.short_description = _('payment')

    def ntickets(self, obj):
        return obj.tickets.count()
    ntickets.short_description = _('ntickets')

    def twin(self, obj):
        try:
            tws = TicketWindowSale.objects.get(purchase=obj)
        except MultipleObjectsReturned:
            tws = TicketWindowSale.objects.filter(purchase=obj).first()
            tws.id = None

            TicketWindowSale.objects.filter(purchase=obj).delete()
            tws.save()
        except:
            return '-'
        prefix = tws.window.code
        return prefix
    twin.short_description = _('ticket window')

    def price(self, obj):
        return obj.get_real_price()
    price.short_description = _('price')

    def formated_extra_data(self, obj):
        extras = obj.get_extras_dict()
        html = '
'
        for k in sorted(extras.keys()):
            html += '
' + k + '	' + str(extras[k]) + '
'
        html += '
'
        return mark_safe(html)
    formated_extra_data.short_description = _('extra data')

    def event_filter(self, request, slug):
        qs = super().get_queryset(request)
        return qs.filter(ev__slug=slug)

class TicketWarningAdmin(admin.ModelAdmin):
    list_display = ('name', 'ev',  'csessions1', 'csessions2', 'message')
    list_filter = ('name',)
    filter_horizontal = ('sessions1', 'sessions2')

    def csessions1(self, obj):
        return ', '.join(str(s) for s in obj.sessions1.all())

    def csessions2(self, obj):
        return ', '.join(str(s) for s in obj.sessions2.all())

    def event_filter(self, request, slug):
        qs = super().get_queryset(request)
        return qs.filter(ev__slug=slug)

class TicketSeatHoldAdmin(admin.ModelAdmin):
    search_fields = ('client', )
    date_hierarchy = 'date'
    list_display = ('client', 'session',  'layout', 'seat', 'date', 'type')
    list_filter = (('session', admin.RelatedOnlyFieldListFilter),
                   'type',
                   ('layout', admin.RelatedOnlyFieldListFilter))

    def event_filter(self, request, slug):
        qs = super().get_queryset(request)
        return qs.filter(session__space__event__slug=slug)

register(Ticket, TicketAdmin)
register(TicketWarning, TicketWarningAdmin)
register(MultiPurchase, MPAdmin)
register(TicketSeatHold, TicketSeatHoldAdmin)

# tinymce for flatpages

from django import forms
from django.contrib.flatpages.admin import FlatpageForm, FlatPageAdmin
from django.contrib.flatpages.models import FlatPage
from tinymce.widgets import TinyMCE

class PageForm(FlatpageForm):

    content = forms.CharField(widget=TinyMCE(attrs={'cols': 80, 'rows': 30}))

    class Meta:
        model = FlatPage
        fields = ['content']

class PageAdmin(FlatPageAdmin):
    form = PageForm

admin.site.unregister(FlatPage)
admin.site.register(FlatPage, PageAdmin)
<commit_msg>Payment method to the csv export<commit_after>from django.conf import settings
from django.contrib import admin
from django.utils.translation import ugettext_lazy as _

from django.utils.safestring import mark_safe
from django.urls import reverse

from django.utils import timezone
from django.utils.formats import date_format

from .models import Ticket
from .models import MultiPurchase
from .models import TicketWarning
from .models import TicketSeatHold
from admin_csv import CSVMixin

from .filters import TicketWindowFilter
from .filters import SingleTicketWindowFilter
from windows.models import TicketWindowSale
from django.core.exceptions import MultipleObjectsReturned
from windows.utils import online_sale

from events.models import TicketField

from extended_filters.filters import DateRangeFilter
from congressus.admin import register

def confirm(modeladmin, request, queryset):
    for i in queryset:
        i.confirm()
confirm.short_description = _("Manual confirm")

def unconfirm(modeladmin, request, queryset):
    for i in queryset:
        i.confirmed = False
        i.remove_hold_seats()
        i.save()
unconfirm.short_description = _("Manual unconfirm")

def mark_used(modeladmin, request, queryset):
    now = timezone.now()
    for i in queryset:
        i.used = True
        i.used_date = now
        i.save()
mark_used.short_description = _("Mark like used")

def mark_no_used(modeladmin, request, queryset):
    for i in queryset:
        i.used = False
        i.used_date = None
        i.save()
mark_no_used.short_description = _("Mark like no used")

def delete_selected(modeladmin, request, queryset):
    for obj in queryset:
        obj.delete()
delete_selected.short_description = _("Remove")

class TicketAdmin(CSVMixin, admin.ModelAdmin):
    list_per_page = 20
    list_max_show_all = 800
    list_display = ('order', 'order_tpv2', 'session2', 'cseat', 'twin',
                    'created2', 'confirmed', 'used',
                    'email', 'payment', 'payment_method', 'price2', 'event')
    list_filter = (('created', DateRangeFilter), 'confirmed',
                   'payment_method', 'used', SingleTicketWindowFilter,
                   'event_name',
                   ('session__space', admin.RelatedOnlyFieldListFilter))
    search_fields = ('order', 'order_tpv', 'email', 'mp__order', 'mp__order_tpv')
    date_hierarchy = 'created'
    actions = [delete_selected, confirm, unconfirm, mark_used, mark_no_used]

    readonly_fields = (
        'order_tpv_linked', 'order', 'event_name', 'session2', 'cseat',
        'confirmed', 'confirmed_date', 'price', 'tax',
        'email', 'formated_extra_data',
        'used_date', 'gate_name',
        'start', 'end', 'twin'
    )

    fieldsets = (
        (None, {
            'fields': ('order', 'order_tpv_linked', 'event_name',
                       'session2', 'cseat', 'twin')
        }),
        (_('Personal info'), {
            'fields': ('email', 'formated_extra_data')
        }),
        (_('Extra info'), {
            'fields': (('confirmed', 'confirmed_date'),
                       ('price', 'tax'),
                       ('used', 'used_date', 'gate_name'),
                       'start', 'end')
        }),
    )

    @property
    def csv_fields(self):
        data = [
            'email',

            'order',
            'order_tpv',

            'confirmed',
            'confirmed_date',

            'price2',
            'cseat',
            'mp',
            'twin',

            'event_name',
            'space_name',
            'session_name',
            'created',

            'used',
            'used_at',
        ]

        if getattr(settings, 'CSV_TICKET_FIELDS'):
            data += ['ticket_field_' + i for i in settings.CSV_TICKET_FIELDS]

        return data

    def __getattr__(self, value):
        if value.startswith('ticket_field_'):
            pk = value.split("_")[-1]
            field = TicketField.objects.get(pk=pk)
            def f(obj):
                return obj.get_extra_data(field.label)
            f.short_description = field.label
            return f
        raise AttributeError

    def formated_extra_data(self, obj):
        extras = obj.get_extras_dict()
        html = '
'
        for k in sorted(extras.keys()):
            html += '
' + k + '	' + str(extras[k]) + '
'
        html += '
'
        return mark_safe(html)
    formated_extra_data.short_description = _('extra data')

    def order_tpv_linked(self, obj):
        if not obj.mp:
            return obj.order_tpv

        url = reverse('admin:tickets_multipurchase_change', args=(obj.mp.id,))
        html = '' + obj.mp.order_tpv + ''
        return mark_safe(html)
    order_tpv_linked.short_description = _('order TPV')

    def price2(self, obj):
        return obj.get_real_price()
    price2.short_description = _('price')

    def twin(self, obj):
        try:
            tws = TicketWindowSale.objects.get(purchase__tickets=obj)
        except:
            return '-'
        prefix = tws.window.code
        return prefix
    twin.short_description = _('ticket window')

    def order_tpv2(self, obj):
        if obj.mp:
            return obj.mp.order_tpv
        else:
            return obj.order_tpv
    order_tpv2.short_description = _('order TPV')

    def session2(self, obj):
        return obj.session.short()
    session2.short_description = _('session')

    def payment(self, obj):
        if not obj.mp:
            return '-'
        try:
            tws = TicketWindowSale.objects.get(purchase=obj.mp)
        except:
            return '-'
        return tws.get_payment_display()
    payment.short_description = _('payment')

    def created2(self, obj):
        fmt='d/m/y H:i:s'
        d1 = timezone.localtime(obj.created)
        return date_format(d1, fmt)
    created2.short_description = _('date')

    def used_at(self, obj):
        if not obj.used_date:
            return '-'
        fmt='d/m/y H:i:s'
        d1 = timezone.localtime(obj.used_date)
        return date_format(d1, fmt)
    used_at.short_description = _('used at')

    def event_filter(self, request, slug):
        qs = super().get_queryset(request)
        return qs.filter(session__space__event__slug=slug)

class TicketInline(admin.TabularInline):
    model = Ticket
    fields = ('link_order', 'session', 'price', 'seat_layout', 'seat')
    readonly_fields = fields

    def link_order(self, obj):
        url = reverse('admin:tickets_ticket_change', args=(obj.id,))
        html = '' + obj.order + ''
        return mark_safe(html)
    link_order.short_description = _('order')

    def has_delete_permission(self, request, obj):
        return False

    def has_add_permission(self, request):
        return False

def link_online_sale(modeladmin, request, queryset):
    for i in queryset:
        online_sale(i)
        i.save()
link_online_sale.short_description = _("Link online sale")

class MPAdmin(CSVMixin, admin.ModelAdmin):
    list_per_page = 20
    list_max_show_all = 800
    list_display = ('order_tpv', 'twin', 'created', 'confirmed2', 'email', 'ntickets', 'price', 'payment_method', 'event')
    list_filter = (('created', DateRangeFilter), 'confirmed', 'payment_method',
                   TicketWindowFilter,
                   ('ev', admin.RelatedOnlyFieldListFilter), 'tpv_error')

    search_fields = ('order', 'order_tpv', 'email', 'extra_data')
    date_hierarchy = 'created'
    actions = [delete_selected, confirm, unconfirm, link_online_sale]
    inlines = [TicketInline, ]

    @property
    def csv_fields(self):
        data = [
            'email',

            'order',
            'order_tpv',

            'confirmed',
            'confirmed_date',

            'price',
            'ntickets',
            'twin',
            'payment_method',

            'ev',
            'created',
        ]

        if getattr(settings, 'CSV_TICKET_FIELDS'):
            data += ['ticket_field_' + i for i in settings.CSV_TICKET_FIELDS]

        return data

    def __getattr__(self, value):
        if value.startswith('ticket_field_'):
            pk = value.split("_")[-1]
            field = TicketField.objects.get(pk=pk)
            def f(obj):
                return obj.get_extra_data(field.label)
            f.short_description = field.label
            return f
        raise AttributeError

    readonly_fields = (
        'order_tpv', 'order', 'ev',
        'confirmed', 'confirmed_date',
        'ntickets', 'price', 'payment',
        'formated_extra_data',
        'tpv_error', 'tpv_error_info', 'twin'
    )

    fieldsets = (
        (None, {
            'fields': ('order_tpv', 'order', 'ev', 'twin')
        }),
        (_('Personal info'), {
            'fields': (('email', 'confirm_sent'), 'formated_extra_data')
        }),
        (_('Extra info'), {
            'fields': (('confirmed', 'confirmed_date'),
                       ('ntickets', 'price'),
                       'discount', 'payment', 'payment_method',
                       'tpv_error', 'tpv_error_info',
                      )
        }),
    )

    def confirmed2(self, obj):
        icon = 'icon-no.svg'
        if obj.confirmed:
            icon = 'icon-yes.svg'
        elif obj.tpv_error:
            icon = 'icon-alert.svg'
        html = ''.format(settings.STATIC_URL, icon)
        return mark_safe(html)
    confirmed2.short_description = _('confirmed')

    def payment(self, obj):
        try:
            tws = TicketWindowSale.objects.get(purchase=obj)
        except:
            return '-'
        return tws.get_payment_display()
    payment.short_description = _('payment')

    def ntickets(self, obj):
        return obj.tickets.count()
    ntickets.short_description = _('ntickets')

    def twin(self, obj):
        try:
            tws = TicketWindowSale.objects.get(purchase=obj)
        except MultipleObjectsReturned:
            tws = TicketWindowSale.objects.filter(purchase=obj).first()
            tws.id = None

            TicketWindowSale.objects.filter(purchase=obj).delete()
            tws.save()
        except:
            return '-'
        prefix = tws.window.code
        return prefix
    twin.short_description = _('ticket window')

    def price(self, obj):
        return obj.get_real_price()
    price.short_description = _('price')

    def formated_extra_data(self, obj):
        extras = obj.get_extras_dict()
        html = '
'
        for k in sorted(extras.keys()):
            html += '
' + k + '	' + str(extras[k]) + '
'
        html += '
'
        return mark_safe(html)
    formated_extra_data.short_description = _('extra data')

    def event_filter(self, request, slug):
        qs = super().get_queryset(request)
        return qs.filter(ev__slug=slug)

class TicketWarningAdmin(admin.ModelAdmin):
    list_display = ('name', 'ev',  'csessions1', 'csessions2', 'message')
    list_filter = ('name',)
    filter_horizontal = ('sessions1', 'sessions2')

    def csessions1(self, obj):
        return ', '.join(str(s) for s in obj.sessions1.all())

    def csessions2(self, obj):
        return ', '.join(str(s) for s in obj.sessions2.all())

    def event_filter(self, request, slug):
        qs = super().get_queryset(request)
        return qs.filter(ev__slug=slug)

class TicketSeatHoldAdmin(admin.ModelAdmin):
    search_fields = ('client', )
    date_hierarchy = 'date'
    list_display = ('client', 'session',  'layout', 'seat', 'date', 'type')
    list_filter = (('session', admin.RelatedOnlyFieldListFilter),
                   'type',
                   ('layout', admin.RelatedOnlyFieldListFilter))

    def event_filter(self, request, slug):
        qs = super().get_queryset(request)
        return qs.filter(session__space__event__slug=slug)

register(Ticket, TicketAdmin)
register(TicketWarning, TicketWarningAdmin)
register(MultiPurchase, MPAdmin)
register(TicketSeatHold, TicketSeatHoldAdmin)

# tinymce for flatpages

from django import forms
from django.contrib.flatpages.admin import FlatpageForm, FlatPageAdmin
from django.contrib.flatpages.models import FlatPage
from tinymce.widgets import TinyMCE

class PageForm(FlatpageForm):

    content = forms.CharField(widget=TinyMCE(attrs={'cols': 80, 'rows': 30}))

    class Meta:
        model = FlatPage
        fields = ['content']

class PageAdmin(FlatPageAdmin):
    form = PageForm

admin.site.unregister(FlatPage)
admin.site.register(FlatPage, PageAdmin)
<|endoftext|>


<commit_before>from .es_query import HQESQuery
from . import filters

class FormES(HQESQuery):
    index = 'forms'
    default_filters = {
        'is_xform_instance': {"term": {"doc_type": "xforminstance"}},
        'has_xmlns': {"not": {"missing": {"field": "xmlns"}}},
        'has_user': {"not": {"missing": {"field": "form.meta.userID"}}},
    }
    @property
    def builtin_filters(self):
        return [
            xmlns,
            app,
            submitted,
            completed,
        ] + super(FormES, self).builtin_filters

    def user_facet(self):
        return self.terms_facet('form.meta.userID', 'user')

def xmlns(xmlns):
    return filters.term('xmlns.exact', xmlns)

def app(app_id):
    return filters.term('app_id', app_id)

def submitted(gt=None, gte=None, lt=None, lte=None):
    return filters.date_range('received_on', gt, gte, lt, lte)

def completed(gt=None, gte=None, lt=None, lte=None):
    return filters.date_range('form.meta.timeEnd', gt, gte, lt, lte)
<commit_msg>Add domain facet for forms<commit_after>from .es_query import HQESQuery
from . import filters

class FormES(HQESQuery):
    index = 'forms'
    default_filters = {
        'is_xform_instance': {"term": {"doc_type": "xforminstance"}},
        'has_xmlns': {"not": {"missing": {"field": "xmlns"}}},
        'has_user': {"not": {"missing": {"field": "form.meta.userID"}}},
    }
    @property
    def builtin_filters(self):
        return [
            xmlns,
            app,
            submitted,
            completed,
        ] + super(FormES, self).builtin_filters

    def user_facet(self):
        return self.terms_facet('form.meta.userID', 'user')

    def domain_facet(self):
        return self.terms_facet('domain', 'domain')

def xmlns(xmlns):
    return filters.term('xmlns.exact', xmlns)

def app(app_id):
    return filters.term('app_id', app_id)

def submitted(gt=None, gte=None, lt=None, lte=None):
    return filters.date_range('received_on', gt, gte, lt, lte)

def completed(gt=None, gte=None, lt=None, lte=None):
    return filters.date_range('form.meta.timeEnd', gt, gte, lt, lte)
<|endoftext|>


<commit_before>"""Migrate attachments from couchdb to an external blob store

## Running migrations

Use the `run_blob_migration` management command to transfer attachments
on selected models from couchdb to the external blob database. Run the
command without arguments to get a list of possible migrations that
can be run:

    ./manage.py run_blob_migration

The migration may be done automatically when running `./manage.py migrate`
if there are only a few documents to be migrated. However, if there are
many documents to be migrated then the normal migration process will
stop and propmpt you to run the blob migration manually before
continuing with the normal migration process.

## Writing new migrations

Complete each of the following steps when migrating a new set of couch
models' attachments to the blob database:

1. Add `BlobMixin` as a base class of each couch model with attachments
   to be migrated to the blob database. For each new `BlobMixin` model:

   a. The mixin must come before `Document` in the list of base classes.
   b. Add `migrating_blobs_from_couch = True` to the class.
   c. Adapt any uses of the `_attachments` property to use `blobs`
      instead (this is more than a simple find and replace; see
      `corehq.blobs.mixin` for details).

2. Add a `Migrator` instance to the list of `MIGRATIONS` in
   `corehq/blobs/migrate.py`. It will look something like
   `Migrator("<your_slug>", [<list of new BlobMixin couch models>])`.
   Don't forget to add a test to verify that your migration actually
   works.

3. Deploy.

4. Run the management command to migrate attachments out of couch:
   `./manage.py run_blob_migration <your_slug>`

5. Remove `migrating_blobs_from_couch = True` from each of your
   `BlobMixin` models.

6. Create a new django migration with the `makemigrations` management
   command:
   ```
   ./manage.py makemigrations --empty blobs
   ```
   Then modify the new migration, adding an operation:
   ```
   operations = [
       HqRunPython(*assert_migration_complete("<your_slug>"))
   ]
   ```
   Don't forget to put
   ```
   from corehq.blobs.migrate import assert_migration_complete
   from corehq.sql_db.operations import HqRunPython
   ```
   at the top of the file.

7. Deploy.

That's it, you're done!
"""
import json
import os
import traceback
from tempfile import mkdtemp

from django.conf import settings
from couchexport.models import SavedBasicExport
from corehq.blobs import get_blob_db
from corehq.blobs.exceptions import NotFound
from corehq.blobs.migratingdb import MigratingBlobDB
from corehq.blobs.mixin import BlobMixin
from corehq.blobs.models import BlobMigrationState
from corehq.dbaccessors.couchapps.all_docs import (
    get_all_docs_with_doc_types,
    get_doc_count_by_type,
)
from couchdbkit import ResourceConflict, ResourceNotFound

MIGRATION_INSTRUCTIONS = """
There are {total} documents that may have attachments, and they must be
migrated to a new blob database.

Run these commands to procede with migrations:

./manage.py run_blob_migration {slug} --file=FILE
./manage.py migrate

Note: --file=FILE is optional and can be omitted if you do not want to
keep a copy of the couch documents that were migrated. Also note that
the copy of the couch documents will not include attachment content
because `get_all_docs_with_doc_types()` does not support that.

See also:
https://github.com/dimagi/commcare-hq/blob/master/corehq/blobs/migrate.py
"""

BLOB_DB_NOT_CONFIGURED = """
Cannot get the blob database.

This often means that settings.SHARED_DRIVE_ROOT is not configured.
It should be set to a real directory. Update localsettings.py and
retry the migration.
"""

MIGRATIONS_SKIPPED_WARNING = """
WARNING {} documents were not migrated due to concurrent modification
during migration. Run the migration again until you do not see this
message.
"""

def migrate_from_couch_to_blobdb(filename, type_map, total):
    """Migrate attachments from couchdb to blob storage
    """
    skips = 0
    with open(filename, 'r') as f:
        for n, line in enumerate(f):
            if n % 100 == 0:
                print_status(n + 1, total)
            doc = json.loads(line)
            obj = type_map[doc["doc_type"]](doc)
            fetch_attachment = super(BlobMixin, obj).fetch_attachment
            try:
                with obj.atomic_blobs():
                    for name, meta in list(obj._attachments.iteritems()):
                        try:
                            content = fetch_attachment(name, stream=True)
                        except ResourceNotFound:
                            # ignore attachment that has been removed
                            continue
                        obj.put_attachment(
                            content, name, content_type=meta["content_type"])
            except ResourceConflict:
                # Do not migrate document if `atomic_blobs()` fails.
                # This is an unlikely state, but could happen if the
                # document is (externally) modified between when the
                # migration fetches and processes the document.
                skips += 1
    return total, skips

migrate_from_couch_to_blobdb.blobs_key = "_attachments"

def migrate_blob_db_backend(filename, type_map, total):
    """Copy blobs from old blob db to new blob db

    This is an idempotent operation. It is safe to interrupt and/or run
    it more than once for a given pair of blob dbs. Running it more than
    once may be very innefficient (it may copy the full content of every
    blob each time), depending on the implementation of the databases
    being migrated.
    """
    db = get_blob_db()
    if not isinstance(db, MigratingBlobDB):
        raise MigrationError(
            "Expected to find migrating blob db backend (got %r)" % db)

    not_found = 0
    with open(filename, 'r') as f:
        for n, line in enumerate(f):
            if n % 100 == 0:
                print_status(n + 1, total)
            doc = json.loads(line)
            obj = type_map[doc["doc_type"]](doc)
            bucket = obj._blobdb_bucket()
            assert obj.external_blobs == obj.blobs, (obj.external_blobs, obj.blobs)
            for name, meta in obj.blobs.iteritems():
                try:
                    content = db.old_db.get(meta.id, bucket)
                except NotFound:
                    not_found += 1
                else:
                    with content:
                        db.copy_blob(content, meta.info, bucket)
    if not_found:
        print("{} documents were not found in the old blob database. It is "
              "possible that some blobs were deleted as part of normal "
              "operation during the migration if the migration took a long "
              "time. However, it may be cause for concern if a majority of "
              "the total number of migrated blobs were not found."
              .format(not_found))
    return total - not_found, 0

migrate_blob_db_backend.blobs_key = "external_blobs"

class Migrator(object):

    def __init__(self, slug, doc_types, migrate_func):
        self.slug = slug
        self.doc_types = doc_types
        self.migrate_func = migrate_func

    def migrate(self, filename=None):
        return migrate(self.slug, self.doc_types, self.migrate_func, filename)

MIGRATIONS = {m.slug: m for m in [
    Migrator("saved_exports", [SavedBasicExport], migrate_from_couch_to_blobdb),
    Migrator("migrate_backend", [SavedBasicExport], migrate_blob_db_backend),
]}

def migrate(slug, doc_types, migrate_func, filename=None):
    """Migrate blobs

    :param doc_types: List of couch model classes to be migrated.
    :param filename: File path for intermediate storage of migration data.
    :param migrate_func: A function `func(filename, type_map, total)`
    returning a tuple `(<num migrated>, <num skipped>)`. If `<num skipped>`
    is non-zero the migration will be considered failed (a migration state
    record will not be saved). `<num migrated>` need not match the original
    `total` passed in. This could happen, for example, if a document is
    deleted during the migration (and should not cause migration failure).
    :returns: A tuple `(<num migrated>, <num skipped>)`
    """
    couchdb = doc_types[0].get_db()
    assert all(t.get_db() is couchdb for t in doc_types[1:]), repr(doc_types)
    type_map = {cls.__name__: cls for cls in doc_types}

    dirpath = None
    if filename is None:
        dirpath = mkdtemp()
        filename = os.path.join(dirpath, "export.txt")

    print("Loading documents: {}...".format(", ".join(type_map)))
    total = 0
    with open(filename, 'w') as f:
        for doc in get_all_docs_with_doc_types(couchdb, list(type_map)):
            if doc.get(migrate_func.blobs_key):
                f.write('{}\n'.format(json.dumps(doc)))
                total += 1

    migrated, skips = migrate_func(filename, type_map, total)

    if dirpath is not None:
        os.remove(filename)
        os.rmdir(dirpath)

    print("Migrated {} documents.".format(migrated - skips))
    if skips:
        print(MIGRATIONS_SKIPPED_WARNING.format(skips))
    else:
        BlobMigrationState.objects.get_or_create(slug=slug)[0].save()
    return migrated - skips, skips

def print_status(num, total):
    print("Migrating {} of {} documents".format(num, total))

def assert_migration_complete(slug):

    def forwards(apps, schema_editor):
        if settings.UNIT_TESTING:
            return

        try:
            get_blob_db()
        except Exception:
            raise MigrationError(
                "Cannot get blob db:\n{error}{message}".format(
                    error=traceback.format_exc(),
                    message=BLOB_DB_NOT_CONFIGURED,
                ))

        try:
            BlobMigrationState.objects.get(slug=slug)
            return  # already migrated
        except BlobMigrationState.DoesNotExist:
            pass

        migrator = MIGRATIONS[slug]
        total = 0
        for doc_type in migrator.doc_types:
            total += get_doc_count_by_type(doc_type.get_db(), doc_type.__name__)
        if total > 500:
            message = MIGRATION_INSTRUCTIONS.format(slug=slug, total=total)
            raise MigrationNotComplete(message)

        # just do the migration if the number of documents is small
        migrated, skipped = migrator.migrate()
        if skipped:
            raise MigrationNotComplete(MIGRATIONS_SKIPPED_WARNING.format(skipped))

    def reverse(apps, schema_editor):
        # NOTE: this does not move blobs back into couch. It only
        # un-sets the blob migration state so it can be run again.
        # It is safe to run the forward migration more than once.
        try:
            model = BlobMigrationState.objects.get(slug=slug)
            model.delete()
        except BlobMigrationState.DoesNotExist:
            pass

    return forwards, reverse

class MigrationError(Exception):
    pass

class MigrationNotComplete(MigrationError):
    pass
<commit_msg>Refactor attachments migration<commit_after>"""Migrate attachments from couchdb to an external blob store

## Running migrations

Use the `run_blob_migration` management command to transfer attachments
on selected models from couchdb to the external blob database. Run the
command without arguments to get a list of possible migrations that
can be run:

    ./manage.py run_blob_migration

The migration may be done automatically when running `./manage.py migrate`
if there are only a few documents to be migrated. However, if there are
many documents to be migrated then the normal migration process will
stop and propmpt you to run the blob migration manually before
continuing with the normal migration process.

## Writing new migrations

Complete each of the following steps when migrating a new set of couch
models' attachments to the blob database:

1. Add `BlobMixin` as a base class of each couch model with attachments
   to be migrated to the blob database. For each new `BlobMixin` model:

   a. The mixin must come before `Document` in the list of base classes.
   b. Add `migrating_blobs_from_couch = True` to the class.
   c. Adapt any uses of the `_attachments` property to use `blobs`
      instead (this is more than a simple find and replace; see
      `corehq.blobs.mixin` for details).

2. Add a `Migrator` instance to the list of `MIGRATIONS` in
   `corehq/blobs/migrate.py`. It will look something like
   `Migrator("<your_slug>", [<list of new BlobMixin couch models>])`.
   Don't forget to add a test to verify that your migration actually
   works.

3. Deploy.

4. Run the management command to migrate attachments out of couch:
   `./manage.py run_blob_migration <your_slug>`

5. Remove `migrating_blobs_from_couch = True` from each of your
   `BlobMixin` models.

6. Create a new django migration with the `makemigrations` management
   command:
   ```
   ./manage.py makemigrations --empty blobs
   ```
   Then modify the new migration, adding an operation:
   ```
   operations = [
       HqRunPython(*assert_migration_complete("<your_slug>"))
   ]
   ```
   Don't forget to put
   ```
   from corehq.blobs.migrate import assert_migration_complete
   from corehq.sql_db.operations import HqRunPython
   ```
   at the top of the file.

7. Deploy.

That's it, you're done!
"""
import json
import os
import traceback
from tempfile import mkdtemp

from django.conf import settings
from couchexport.models import SavedBasicExport
from corehq.blobs import get_blob_db
from corehq.blobs.exceptions import NotFound
from corehq.blobs.migratingdb import MigratingBlobDB
from corehq.blobs.mixin import BlobMixin
from corehq.blobs.models import BlobMigrationState
from corehq.dbaccessors.couchapps.all_docs import (
    get_all_docs_with_doc_types,
    get_doc_count_by_type,
)
from couchdbkit import ResourceConflict, ResourceNotFound

MIGRATION_INSTRUCTIONS = """
There are {total} documents that may have attachments, and they must be
migrated to a new blob database.

Run these commands to proceed with migrations:

./manage.py run_blob_migration {slug} --file=FILE
./manage.py migrate

Note: --file=FILE is optional and can be omitted if you do not want to
keep a copy of the couch documents, including attachments, that were
migrated.

See also:
https://github.com/dimagi/commcare-hq/blob/master/corehq/blobs/migrate.py
"""

BLOB_DB_NOT_CONFIGURED = """
Cannot get the blob database.

This often means that settings.SHARED_DRIVE_ROOT is not configured.
It should be set to a real directory. Update localsettings.py and
retry the migration.
"""

MIGRATIONS_SKIPPED_WARNING = """
WARNING {} documents were not migrated due to concurrent modification
during migration. Run the migration again until you do not see this
message.
"""

def migrate_from_couch_to_blobdb(filename, type_map, total):
    """Migrate attachments from couchdb to blob storage
    """
    skips = 0
    with open(filename, 'r') as f:
        for n, line in enumerate(f):
            if n % 100 == 0:
                print_status(n + 1, total)
            doc = json.loads(line)
            attachments = doc.pop("_attachments")
            doc["_attachments"] = {}
            obj = type_map[doc["doc_type"]](doc)
            try:
                with obj.atomic_blobs():
                    for name, data in list(attachments.iteritems()):
                        obj.put_attachment(name=name, **data)
            except ResourceConflict:
                # Do not migrate document if `atomic_blobs()` fails.
                # This is an unlikely state, but could happen if the
                # document is (externally) modified between when the
                # migration fetches and processes the document.
                skips += 1
    return total, skips

migrate_from_couch_to_blobdb.load_attachments = True
migrate_from_couch_to_blobdb.blobs_key = "_attachments"

def migrate_blob_db_backend(filename, type_map, total):
    """Copy blobs from old blob db to new blob db

    This is an idempotent operation. It is safe to interrupt and/or run
    it more than once for a given pair of blob dbs. Running it more than
    once may be very innefficient (it may copy the full content of every
    blob each time), depending on the implementation of the databases
    being migrated.
    """
    db = get_blob_db()
    if not isinstance(db, MigratingBlobDB):
        raise MigrationError(
            "Expected to find migrating blob db backend (got %r)" % db)

    not_found = 0
    with open(filename, 'r') as f:
        for n, line in enumerate(f):
            if n % 100 == 0:
                print_status(n + 1, total)
            doc = json.loads(line)
            obj = type_map[doc["doc_type"]](doc)
            bucket = obj._blobdb_bucket()
            assert obj.external_blobs == obj.blobs, (obj.external_blobs, obj.blobs)
            for name, meta in obj.blobs.iteritems():
                try:
                    content = db.old_db.get(meta.id, bucket)
                except NotFound:
                    not_found += 1
                else:
                    with content:
                        db.copy_blob(content, meta.info, bucket)
    if not_found:
        print("{} documents were not found in the old blob database. It is "
              "possible that some blobs were deleted as part of normal "
              "operation during the migration if the migration took a long "
              "time. However, it may be cause for concern if a majority of "
              "the total number of migrated blobs were not found."
              .format(not_found))
    return total - not_found, 0

migrate_blob_db_backend.blobs_key = "external_blobs"

class Migrator(object):

    def __init__(self, slug, doc_types, migrate_func):
        self.slug = slug
        self.doc_types = doc_types
        self.migrate_func = migrate_func

    def migrate(self, filename=None):
        return migrate(self.slug, self.doc_types, self.migrate_func, filename)

MIGRATIONS = {m.slug: m for m in [
    Migrator("saved_exports", [SavedBasicExport], migrate_from_couch_to_blobdb),
    Migrator("migrate_backend", [SavedBasicExport], migrate_blob_db_backend),
]}

def migrate(slug, doc_types, migrate_func, filename=None):
    """Migrate blobs

    :param doc_types: List of couch model classes to be migrated.
    :param filename: File path for intermediate storage of migration data.
    :param migrate_func: A function `func(filename, type_map, total)`
    returning a tuple `(<num migrated>, <num skipped>)`. If `<num skipped>`
    is non-zero the migration will be considered failed (a migration state
    record will not be saved). `<num migrated>` need not match the original
    `total` passed in. This could happen, for example, if a document is
    deleted during the migration (and should not cause migration failure).
    :returns: A tuple `(<num migrated>, <num skipped>)`
    """
    couchdb = doc_types[0].get_db()
    assert all(t.get_db() is couchdb for t in doc_types[1:]), repr(doc_types)
    type_map = {cls.__name__: cls for cls in doc_types}

    dirpath = None
    if filename is None:
        dirpath = mkdtemp()
        filename = os.path.join(dirpath, "export.txt")

    print("Loading documents: {}...".format(", ".join(type_map)))
    total = 0
    load_attachments = getattr(migrate_func, "load_attachments", False)
    with open(filename, 'w') as f:
        for doc in get_all_docs_with_doc_types(couchdb, list(type_map)):
            if doc.get(migrate_func.blobs_key):
                if load_attachments:
                    obj = type_map[doc["doc_type"]](doc)
                    fetch_attachment = super(BlobMixin, obj).fetch_attachment
                    doc["_attachments"] = {
                        name: {
                            "content_type": meta["content_type"],
                            "content": fetch_attachment(name),
                        }
                        for name, meta in doc["_attachments"].items()
                    }
                f.write('{}\n'.format(json.dumps(doc)))
                total += 1

    migrated, skips = migrate_func(filename, type_map, total)

    if dirpath is not None:
        os.remove(filename)
        os.rmdir(dirpath)

    print("Migrated {} documents.".format(migrated - skips))
    if skips:
        print(MIGRATIONS_SKIPPED_WARNING.format(skips))
    else:
        BlobMigrationState.objects.get_or_create(slug=slug)[0].save()
    return migrated - skips, skips

def print_status(num, total):
    print("Migrating {} of {} documents".format(num, total))

def assert_migration_complete(slug):

    def forwards(apps, schema_editor):
        if settings.UNIT_TESTING:
            return

        try:
            get_blob_db()
        except Exception:
            raise MigrationError(
                "Cannot get blob db:\n{error}{message}".format(
                    error=traceback.format_exc(),
                    message=BLOB_DB_NOT_CONFIGURED,
                ))

        try:
            BlobMigrationState.objects.get(slug=slug)
            return  # already migrated
        except BlobMigrationState.DoesNotExist:
            pass

        migrator = MIGRATIONS[slug]
        total = 0
        for doc_type in migrator.doc_types:
            total += get_doc_count_by_type(doc_type.get_db(), doc_type.__name__)
        if total > 500:
            message = MIGRATION_INSTRUCTIONS.format(slug=slug, total=total)
            raise MigrationNotComplete(message)

        # just do the migration if the number of documents is small
        migrated, skipped = migrator.migrate()
        if skipped:
            raise MigrationNotComplete(MIGRATIONS_SKIPPED_WARNING.format(skipped))

    def reverse(apps, schema_editor):
        # NOTE: this does not move blobs back into couch. It only
        # un-sets the blob migration state so it can be run again.
        # It is safe to run the forward migration more than once.
        try:
            model = BlobMigrationState.objects.get(slug=slug)
            model.delete()
        except BlobMigrationState.DoesNotExist:
            pass

    return forwards, reverse

class MigrationError(Exception):
    pass

class MigrationNotComplete(MigrationError):
    pass
<|endoftext|>


<commit_before>from logbook import Logger

from catalyst.api import (
    record,
    order,
    symbol,
    get_open_orders
)
from catalyst.exchange.stats_utils import get_pretty_stats
from catalyst.utils.run_algo import run_algorithm

algo_namespace = 'arbitrage_eth_btc'
log = Logger(algo_namespace)

def initialize(context):
    log.info('initializing arbitrage algorithm')

    # The context contains a new "exchanges" attribute which is a dictionary
    # of exchange objects by exchange name. This allow easy access to the
    # exchanges.
    context.buying_exchange = context.exchanges['poloniex']
    context.selling_exchange = context.exchanges['bitfinex']

    context.trading_pair_symbol = 'eth_btc'
    context.trading_pairs = dict()

    # Note the second parameter of the symbol() method
    # Passing the exchange name here returns a TradingPair object including
    # the exchange information. This allow all other operations using
    # the TradingPair to target the correct exchange.
    context.trading_pairs[context.buying_exchange] = \
        symbol('eth_btc', context.buying_exchange.name)

    context.trading_pairs[context.selling_exchange] = \
        symbol(context.trading_pair_symbol, context.selling_exchange.name)

    context.entry_points = [
        dict(gap=0.03, amount=0.05),
        dict(gap=0.04, amount=0.1),
        dict(gap=0.05, amount=0.5),
    ]
    context.exit_points = [
        dict(gap=-0.02, amount=0.5),
    ]

    context.SLIPPAGE_ALLOWED = 0.02
    pass

def place_orders(context, amount, buying_price, selling_price, action):
    """
    This method will always place two orders of the same amount to keep
    the currency position the same as it moves between the two exchanges.

    :param context: TradingAlgorithm
    :param amount: float
        The trading pair amount to trade on both exchanges.
    :param buying_price: float
        The current trading pair price on the buying exchange.
    :param selling_price: float
        The current trading pair price on the selling exchange.
    :param action: string
        "enter": buys on the buying exchange and sells on the selling exchange
        "exit": buys on the selling exchange and sells on the buying exchange

    :return:
    """
    if action == 'enter':
        enter_exchange = context.buying_exchange
        entry_price = buying_price

        exit_exchange = context.selling_exchange
        exit_price = selling_price

    elif action == 'exit':
        enter_exchange = context.selling_exchange
        entry_price = selling_price

        exit_exchange = context.buying_exchange
        exit_price = buying_price

    else:
        raise ValueError('invalid order action')

    quote_currency = enter_exchange.quote_currency
    quote_currency_amount = enter_exchange.portfolio.cash

    exit_balances = exit_exchange.get_balances()
    exit_currency = context.trading_pairs[
        context.selling_exchange].quote_currency

    if exit_currency in exit_balances:
        quote_currency_amount = exit_balances[exit_currency]
    else:
        log.warn(
            'the selling exchange {exchange_name} does not hold '
            'currency {currency}'.format(
                exchange_name=exit_exchange.name,
                currency=exit_currency
            )
        )
        return

    if quote_currency_amount < (amount * entry_price):
        adj_amount = quote_currency_amount / entry_price
        log.warn(
            'not enough {quote_currency} ({quote_currency_amount}) to buy '
            '{amount}, adjusting the amount to {adj_amount}'.format(
                quote_currency=quote_currency,
                quote_currency_amount=quote_currency_amount,
                amount=amount,
                adj_amount=adj_amount
            )
        )
        amount = adj_amount

    elif quote_currency_amount < amount:
        log.warn(
            'not enough {currency} ({currency_amount}) to sell '
            '{amount}, aborting'.format(
                currency=exit_currency,
                currency_amount=quote_currency_amount,
                amount=amount
            )
        )
        return

    adj_buy_price = entry_price * (1 + context.SLIPPAGE_ALLOWED)
    log.info(
        'buying {amount} {trading_pair} on {exchange_name} with price '
        'limit {limit_price}'.format(
            amount=amount,
            trading_pair=context.trading_pair_symbol,
            exchange_name=enter_exchange.name,
            limit_price=adj_buy_price
        )
    )
    order(
        asset=context.trading_pairs[enter_exchange],
        amount=amount,
        limit_price=adj_buy_price
    )

    adj_sell_price = exit_price * (1 - context.SLIPPAGE_ALLOWED)
    log.info(
        'selling {amount} {trading_pair} on {exchange_name} with price '
        'limit {limit_price}'.format(
            amount=-amount,
            trading_pair=context.trading_pair_symbol,
            exchange_name=exit_exchange.name,
            limit_price=adj_sell_price
        )
    )
    order(
        asset=context.trading_pairs[exit_exchange],
        amount=-amount,
        limit_price=adj_sell_price
    )
    pass

def handle_data(context, data):
    log.info('handling bar {}'.format(data.current_dt))

    buying_price = data.current(
        context.trading_pairs[context.buying_exchange], 'price')

    log.info('price on buying exchange {exchange}: {price}'.format(
        exchange=context.buying_exchange.name.upper(),
        price=buying_price,
    ))

    selling_price = data.current(
        context.trading_pairs[context.selling_exchange], 'price')

    log.info('price on selling exchange {exchange}: {price}'.format(
        exchange=context.selling_exchange.name.upper(),
        price=selling_price,
    ))

    # If for example,
    #   selling price = 50
    #   buying price = 25
    #   expected gap = 1

    # If follows that,
    #   selling price - buying price / buying price
    #   50 - 25 / 25 = 1
    gap = (selling_price - buying_price) / buying_price
    log.info(
        'the price gap: {gap} ({gap_percent}%)'.format(
            gap=gap,
            gap_percent=gap * 100
        )
    )
    record(buying_price=buying_price, selling_price=selling_price, gap=gap)

    # Waiting for orders to close before initiating new ones
    for exchange in context.trading_pairs:
        asset = context.trading_pairs[exchange]

        orders = get_open_orders(asset)
        if orders:
            log.info(
                'found {order_count} open orders on {exchange_name} '
                'skipping bar until all open orders execute'.format(
                    order_count=len(orders),
                    exchange_name=exchange.name
                )
            )
            return

    # Consider the least ambitious entry point first
    # Override of wider gap is found
    entry_points = sorted(
        context.entry_points,
        key=lambda point: point['gap'],
    )

    buy_amount = None
    for entry_point in entry_points:
        if gap > entry_point['gap']:
            buy_amount = entry_point['amount']

    if buy_amount:
        log.info('found buy trigger for amount: {}'.format(buy_amount))
        place_orders(
            context=context,
            amount=buy_amount,
            buying_price=buying_price,
            selling_price=selling_price,
            action='enter'
        )

    else:
        # Consider the narrowest exit gap first
        # Override of wider gap is found
        exit_points = sorted(
            context.exit_points,
            key=lambda point: point['gap'],
            reverse=True
        )

        sell_amount = None
        for exit_point in exit_points:
            if gap < exit_point['gap']:
                sell_amount = exit_point['amount']

        if sell_amount:
            log.info('found sell trigger for amount: {}'.format(sell_amount))
            place_orders(
                context=context,
                amount=sell_amount,
                buying_price=buying_price,
                selling_price=selling_price,
                action='exit'
            )

def analyze(context, stats):
    log.info('the daily stats:\n{}'.format(get_pretty_stats(stats)))
    pass

run_algorithm(
    initialize=initialize,
    handle_data=handle_data,
    analyze=analyze,
    exchange_name='poloniex,bitfinex',
    live=True,
    algo_namespace=algo_namespace,
    quote_currency='btc',
    live_graph=False
)
<commit_msg>BUG: adding capital_base to the interface<commit_after>from logbook import Logger

from catalyst.api import (
    record,
    order,
    symbol,
    get_open_orders
)
from catalyst.exchange.stats_utils import get_pretty_stats
from catalyst.utils.run_algo import run_algorithm

algo_namespace = 'arbitrage_eth_btc'
log = Logger(algo_namespace)

def initialize(context):
    log.info('initializing arbitrage algorithm')

    # The context contains a new "exchanges" attribute which is a dictionary
    # of exchange objects by exchange name. This allow easy access to the
    # exchanges.
    context.buying_exchange = context.exchanges['poloniex']
    context.selling_exchange = context.exchanges['bitfinex']

    context.trading_pair_symbol = 'eth_btc'
    context.trading_pairs = dict()

    # Note the second parameter of the symbol() method
    # Passing the exchange name here returns a TradingPair object including
    # the exchange information. This allow all other operations using
    # the TradingPair to target the correct exchange.
    context.trading_pairs[context.buying_exchange] = \
        symbol('eth_btc', context.buying_exchange.name)

    context.trading_pairs[context.selling_exchange] = \
        symbol(context.trading_pair_symbol, context.selling_exchange.name)

    context.entry_points = [
        dict(gap=0.03, amount=0.05),
        dict(gap=0.04, amount=0.1),
        dict(gap=0.05, amount=0.5),
    ]
    context.exit_points = [
        dict(gap=-0.02, amount=0.5),
    ]

    context.SLIPPAGE_ALLOWED = 0.02
    pass

def place_orders(context, amount, buying_price, selling_price, action):
    """
    This method will always place two orders of the same amount to keep
    the currency position the same as it moves between the two exchanges.

    :param context: TradingAlgorithm
    :param amount: float
        The trading pair amount to trade on both exchanges.
    :param buying_price: float
        The current trading pair price on the buying exchange.
    :param selling_price: float
        The current trading pair price on the selling exchange.
    :param action: string
        "enter": buys on the buying exchange and sells on the selling exchange
        "exit": buys on the selling exchange and sells on the buying exchange

    :return:
    """
    if action == 'enter':
        enter_exchange = context.buying_exchange
        entry_price = buying_price

        exit_exchange = context.selling_exchange
        exit_price = selling_price

    elif action == 'exit':
        enter_exchange = context.selling_exchange
        entry_price = selling_price

        exit_exchange = context.buying_exchange
        exit_price = buying_price

    else:
        raise ValueError('invalid order action')

    quote_currency = enter_exchange.quote_currency
    quote_currency_amount = enter_exchange.portfolio.cash

    exit_balances = exit_exchange.get_balances()
    exit_currency = context.trading_pairs[
        context.selling_exchange].quote_currency

    if exit_currency in exit_balances:
        quote_currency_amount = exit_balances[exit_currency]
    else:
        log.warn(
            'the selling exchange {exchange_name} does not hold '
            'currency {currency}'.format(
                exchange_name=exit_exchange.name,
                currency=exit_currency
            )
        )
        return

    if quote_currency_amount < (amount * entry_price):
        adj_amount = quote_currency_amount / entry_price
        log.warn(
            'not enough {quote_currency} ({quote_currency_amount}) to buy '
            '{amount}, adjusting the amount to {adj_amount}'.format(
                quote_currency=quote_currency,
                quote_currency_amount=quote_currency_amount,
                amount=amount,
                adj_amount=adj_amount
            )
        )
        amount = adj_amount

    elif quote_currency_amount < amount:
        log.warn(
            'not enough {currency} ({currency_amount}) to sell '
            '{amount}, aborting'.format(
                currency=exit_currency,
                currency_amount=quote_currency_amount,
                amount=amount
            )
        )
        return

    adj_buy_price = entry_price * (1 + context.SLIPPAGE_ALLOWED)
    log.info(
        'buying {amount} {trading_pair} on {exchange_name} with price '
        'limit {limit_price}'.format(
            amount=amount,
            trading_pair=context.trading_pair_symbol,
            exchange_name=enter_exchange.name,
            limit_price=adj_buy_price
        )
    )
    order(
        asset=context.trading_pairs[enter_exchange],
        amount=amount,
        limit_price=adj_buy_price
    )

    adj_sell_price = exit_price * (1 - context.SLIPPAGE_ALLOWED)
    log.info(
        'selling {amount} {trading_pair} on {exchange_name} with price '
        'limit {limit_price}'.format(
            amount=-amount,
            trading_pair=context.trading_pair_symbol,
            exchange_name=exit_exchange.name,
            limit_price=adj_sell_price
        )
    )
    order(
        asset=context.trading_pairs[exit_exchange],
        amount=-amount,
        limit_price=adj_sell_price
    )
    pass

def handle_data(context, data):
    log.info('handling bar {}'.format(data.current_dt))

    buying_price = data.current(
        context.trading_pairs[context.buying_exchange], 'price')

    log.info('price on buying exchange {exchange}: {price}'.format(
        exchange=context.buying_exchange.name.upper(),
        price=buying_price,
    ))

    selling_price = data.current(
        context.trading_pairs[context.selling_exchange], 'price')

    log.info('price on selling exchange {exchange}: {price}'.format(
        exchange=context.selling_exchange.name.upper(),
        price=selling_price,
    ))

    # If for example,
    #   selling price = 50
    #   buying price = 25
    #   expected gap = 1

    # If follows that,
    #   selling price - buying price / buying price
    #   50 - 25 / 25 = 1
    gap = (selling_price - buying_price) / buying_price
    log.info(
        'the price gap: {gap} ({gap_percent}%)'.format(
            gap=gap,
            gap_percent=gap * 100
        )
    )
    record(buying_price=buying_price, selling_price=selling_price, gap=gap)

    # Waiting for orders to close before initiating new ones
    for exchange in context.trading_pairs:
        asset = context.trading_pairs[exchange]

        orders = get_open_orders(asset)
        if orders:
            log.info(
                'found {order_count} open orders on {exchange_name} '
                'skipping bar until all open orders execute'.format(
                    order_count=len(orders),
                    exchange_name=exchange.name
                )
            )
            return

    # Consider the least ambitious entry point first
    # Override of wider gap is found
    entry_points = sorted(
        context.entry_points,
        key=lambda point: point['gap'],
    )

    buy_amount = None
    for entry_point in entry_points:
        if gap > entry_point['gap']:
            buy_amount = entry_point['amount']

    if buy_amount:
        log.info('found buy trigger for amount: {}'.format(buy_amount))
        place_orders(
            context=context,
            amount=buy_amount,
            buying_price=buying_price,
            selling_price=selling_price,
            action='enter'
        )

    else:
        # Consider the narrowest exit gap first
        # Override of wider gap is found
        exit_points = sorted(
            context.exit_points,
            key=lambda point: point['gap'],
            reverse=True
        )

        sell_amount = None
        for exit_point in exit_points:
            if gap < exit_point['gap']:
                sell_amount = exit_point['amount']

        if sell_amount:
            log.info('found sell trigger for amount: {}'.format(sell_amount))
            place_orders(
                context=context,
                amount=sell_amount,
                buying_price=buying_price,
                selling_price=selling_price,
                action='exit'
            )

def analyze(context, stats):
    log.info('the daily stats:\n{}'.format(get_pretty_stats(stats)))
    pass

if __name__ == '__main__':
    # The execution mode: backtest or live
    MODE = 'live'
    if MODE == 'live':
        run_algorithm(
            capital_base=0.1,
            initialize=initialize,
            handle_data=handle_data,
            analyze=analyze,
            exchange_name='poloniex,bitfinex',
            live=True,
            algo_namespace=algo_namespace,
            base_currency='btc',
            live_graph=False,
            simulate_orders=True,
            stats_output=None,
        )
<|endoftext|>


<commit_before>""" DB wrapper for PyPSA...wip

Attributes
----------

packagename: str
    ...
"""

__copyright__ = ""
__license__ = ""
__author__ = ""

import pypsa
from sqlalchemy.orm import sessionmaker
from oemof.db import engine
from importlib import import_module
import pandas as pd
from sqlalchemy import or_, and_, exists, inspect
from sqlalchemy.orm.exc import NoResultFound
from collections import OrderedDict
import re
import os
import json

# for debugging
conn = engine('v0.2.10')
Session = sessionmaker(bind=conn)
session = Session()

if pypsa.__version__ not in ['0.6.2', '0.8.0']:
    print('Pypsa version %s not supported.' % pypsa.__version__)

packagename = 'egoio.db_tables'
configuration = json.load(open('config.json'), object_pairs_hook=OrderedDict)
temp_ormclass = 'TempResolution'
carr_ormclass = 'Source'

class ScenarioBase():
    """ Hide package/db stuff...
    """

    def __init__(self, session, method, version=None, *args, **kwargs):

        global configuration
        global temp_ormclass
        global carr_ormclass

        schema = 'model_draft' if version is None else 'grid'

        config = kwargs.get('config', configuration.copy())
        self.config = config[method]
        self.session = session
        self.version = version
        self._prefix = kwargs.get('prefix', 'EgoGridPfHv')
        self._pkg = import_module(packagename + '.' + schema)
        self._mapped = {}

        # map static and timevarying classes
        for k, v in self.config.items():
            self.map_ormclass(k)
            if isinstance(v, dict):
                for kk in v.keys():
                    self.map_ormclass(kk)

        # map temporal resolution table
        self.map_ormclass(temp_ormclass)

        # map carrier id to carrier table
        self.map_ormclass(carr_ormclass)

    def map_ormclass(self, name):

        global packagename

        try:
            self._mapped[name] = getattr(self._pkg, self._prefix + name)

        except AttributeError:
            print('Warning: Relation %s does not exist.' % name)

class NetworkScenario(ScenarioBase):
    """
    """

    def __init__(self, session, *args, **kwargs):
        super().__init__(session, *args, **kwargs)

        self.scn_name = kwargs.get('scn_name', 'Status Quo')
        self.method   = kwargs.get('method', 'lopf')
        self.start_h  = kwargs.get('start_h', 1)
        self.end_h    = kwargs.get('end_h', 20)
        self.temp_id  = kwargs.get('temp_id', 1)
        self.network  = None

        self.configure_timeindex()

    def __repr__(self):
        r = ('NetworkScenario: %s' % self.scn_name)

        if self.network:
            r += "\nPyPSA network ready."

        return r

    def configure_timeindex(self):
        """
        """

        try:

            ormclass = self._mapped['TempResolution']
            tr = self.session.query(ormclass).filter(
                ormclass.temp_id == self.temp_id).one()

        except (KeyError, NoResultFound):
            print('temp_id %s does not exist.' % self.temp_id)

        timeindex = pd.DatetimeIndex(start=tr.start_time,
                                     periods=tr.timesteps,
                                     freq=tr.resolution)

        self.timeindex = timeindex[self.start_h - 1: self.end_h]

    def id_to_source(self):

        ormclass = self._mapped['Source']
        query = session.query(ormclass)

        # TODO column naming in database
        return {k.source_id:k.name for k in query.all()}

    def by_scenario(self, name):
        """
        """

        ormclass = self._mapped[name]
        query = session.query(ormclass).filter(ormclass.scn_name == self.scn_name)

        if self.version:
            query = query.filter(ormclass.version == self.version)

        # TODO: Better handled in db
        if name == 'Transformer':
            name = 'Trafo'

        df = pd.read_sql(query.statement,
                           session.bind,
                           index_col=name.lower() + '_id')

        if 'source' in df:
            df.source = df.source.map(self.id_to_source())

        return df

    def series_by_scenario(self, name, column):
        """
        """

        ormclass = self._mapped[name]

        # TODO: pls make more robust
        id_column = re.findall(r'[A-Z][^A-Z]*', name)[0] + '_' + 'id'
        id_column = id_column.lower()

        query = session.query(
            getattr(ormclass,id_column),
            getattr(ormclass,column)[self.start_h:self.end_h].\
            label(column)).filter(and_(
            ormclass.scn_name == self.scn_name,
            ormclass.temp_id == self.temp_id))

        if self.version:
            query = query.filter(ormclass.version == self.version)

        try:
            df =  pd.io.sql.read_sql(query.statement,
                            session.bind,
                            columns=[column],
                            index_col=id_column)

            df.index = df.index.astype(str)

            # change of format to fit pypsa
            df = df[column].apply(pd.Series).transpose()

            df.index = self.timeindex

            assert not df.empty
            return df

        except (ValueError, AssertionError):
            print("No data for %s in column %s." % (name, column))

    def build_network(self, *args, **kwargs):
        """
        """

        network = pypsa.Network()
        network.set_snapshots(self.timeindex)

        for c, v in self.config.items():

            # TODO: This is confusing, should be fixed in db
            name = 'StorageUnit' if c == 'Storage' else c

            network.import_components_from_dataframe(self.by_scenario(c), name)

            if isinstance(v, dict):
                for cc, vv in v.items():
                    for col in vv:
                        df = self.series_by_scenario(cc, col)
                        try:
                            pypsa.io.import_series_from_dataframe(network, df, name, col)
                        except (ValueError, AttributeError):
                            print("Series %s of component %s could not be "
                                  "imported" % (col, name))

        self.network = network

        return network

# for debugging
from egopowerflow.tools.plot import plot_line_loading, add_coordinates

md = NetworkScenario(session, method='lopf', end_h=2, start_h=1,
                          scn_name='Status Quo')
mdnw = md.build_network()
mdnw.lopf(snapshots=md.timeindex, solver_name='gurobi')
mdnw = add_coordinates(mdnw)
plot_line_loading(mdnw)

mdpf = NetworkScenario(session, method='pf', end_h=2, start_h=1,
                          scn_name='Status Quo')
mdpfnw = mdpf.build_network()
mdpfnw.pf(snapshots=mdpf.timeindex)

gr = NetworkScenario(session, method='lopf', end_h=2, start_h=1, version='v0.2.10',
                     prefix='EgoPfHv', scn_name='Status Quo')
grnw = gr.build_network()
grnw.lopf(snapshots=gr.timeindex, solver_name='gurobi')
grnw = add_coordinates(grnw)
plot_line_loading(grnw)

def results_to_oedb(session, network, grid='mv'):
    """Return results obtained from PyPSA to oedb"""
    # moved this here to prevent error when not using the mv-schema
    if grid.lower() == 'mv':
        from egoio.db_tables.calc_ego_mv_powerflow import ResBus, ResLine, ResTransformer
    elif grid.lower() == 'hv':
        print('Not implemented: Result schema for HV missing')
    else:
        print('Please enter mv or hv!')
    # from oemof import db
    # engine = db.engine(section='oedb')
    # from egoio.db_tables import calc_ego_mv_powerflow
    # calc_ego_mv_powerflow.Base.metadata.create_all(engine)

    #TODO: make this more safe, at least inform the user about the deleting results
    # empty all results table
    session.query(ResBus).delete()
    session.query(ResLine).delete()
    session.query(ResTransformer).delete()
    session.commit()

    # insert voltage at buses to database
    for col in network.buses_t.v_mag_pu:
        res_bus = ResBus(
            bus_id=col,
            v_mag_pu=network.buses_t.v_mag_pu[col].tolist()
        )
        session.add(res_bus)
    session.commit()

    # insert active and reactive power of lines to database
    for col in network.lines_t.p0:
        res_line = ResLine(
            line_id=col,
            p0=network.lines_t.p0[col].tolist(),
            q0=network.lines_t.q0[col].tolist(),
            p1=network.lines_t.p1[col].tolist(),
            q1=network.lines_t.q1[col].tolist()
        )
        session.add(res_line)
    session.commit()

    # insert active and reactive power of lines to database
    for col in network.transformers_t.p0:
        res_transformer = ResLine(
            trafo_id=col,
            p0=network.transformers_t.p0[col].tolist(),
            q0=network.transformers_t.q0[col].tolist(),
            p1=network.transformers_t.p1[col].tolist(),
            q1=network.transformers_t.q1[col].tolist()
        )
        session.add(res_transformer)
    session.commit()

def fix_storages(network):
    """
    Workaround to deal with the new name for storages
    used by PyPSA.
    Old: Storage
    New: StorageUnit

    Parameters
    ----------
    network : PyPSA network container

    Returns
    -------
    None
    """
    network.storage_units = network.storage_units.drop('state_of_charge_initial',1).\
                     rename(columns={'soc_initial':'state_of_charge_initial'})
    network.storage_units = network.storage_units.drop('cyclic_state_of_charge',1).\
                     rename(columns={'soc_cyclic':'cyclic_state_of_charge'})

if __name__ == '__main__':
    pass
<commit_msg>Change error handling in by_scneario func<commit_after>""" DB wrapper for PyPSA...wip

Attributes
----------

packagename: str
    ...
"""

__copyright__ = ""
__license__ = ""
__author__ = ""

import pypsa
from sqlalchemy.orm import sessionmaker
from oemof.db import engine
from importlib import import_module
import pandas as pd
from sqlalchemy import or_, and_, exists, inspect
from sqlalchemy.orm.exc import NoResultFound
from collections import OrderedDict
import re
import os
import json

# for debugging
conn = engine('v0.2.10')
Session = sessionmaker(bind=conn)
session = Session()

if pypsa.__version__ not in ['0.6.2', '0.8.0']:
    print('Pypsa version %s not supported.' % pypsa.__version__)

packagename = 'egoio.db_tables'
configuration = json.load(open('config.json'), object_pairs_hook=OrderedDict)
temp_ormclass = 'TempResolution'
carr_ormclass = 'Source'

class ScenarioBase():
    """ Hide package/db stuff...
    """

    def __init__(self, session, method, version=None, *args, **kwargs):

        global configuration
        global temp_ormclass
        global carr_ormclass

        schema = 'model_draft' if version is None else 'grid'

        config = kwargs.get('config', configuration.copy())
        self.config = config[method]
        self.session = session
        self.version = version
        self._prefix = kwargs.get('prefix', 'EgoGridPfHv')
        self._pkg = import_module(packagename + '.' + schema)
        self._mapped = {}

        # map static and timevarying classes
        for k, v in self.config.items():
            self.map_ormclass(k)
            if isinstance(v, dict):
                for kk in v.keys():
                    self.map_ormclass(kk)

        # map temporal resolution table
        self.map_ormclass(temp_ormclass)

        # map carrier id to carrier table
        self.map_ormclass(carr_ormclass)

    def map_ormclass(self, name):

        global packagename

        try:
            self._mapped[name] = getattr(self._pkg, self._prefix + name)

        except AttributeError:
            print('Warning: Relation %s does not exist.' % name)

class NetworkScenario(ScenarioBase):
    """
    """

    def __init__(self, session, *args, **kwargs):
        super().__init__(session, *args, **kwargs)

        self.scn_name = kwargs.get('scn_name', 'Status Quo')
        self.method   = kwargs.get('method', 'lopf')
        self.start_h  = kwargs.get('start_h', 1)
        self.end_h    = kwargs.get('end_h', 20)
        self.temp_id  = kwargs.get('temp_id', 1)
        self.network  = None

        self.configure_timeindex()

    def __repr__(self):
        r = ('NetworkScenario: %s' % self.scn_name)

        if self.network:
            r += "\nPyPSA network ready."

        return r

    def configure_timeindex(self):
        """
        """

        try:

            ormclass = self._mapped['TempResolution']
            tr = self.session.query(ormclass).filter(
                ormclass.temp_id == self.temp_id).one()

        except (KeyError, NoResultFound):
            print('temp_id %s does not exist.' % self.temp_id)

        timeindex = pd.DatetimeIndex(start=tr.start_time,
                                     periods=tr.timesteps,
                                     freq=tr.resolution)

        self.timeindex = timeindex[self.start_h - 1: self.end_h]

    def id_to_source(self):

        ormclass = self._mapped['Source']
        query = session.query(ormclass)

        # TODO column naming in database
        return {k.source_id:k.name for k in query.all()}

    def by_scenario(self, name):
        """
        """

        ormclass = self._mapped[name]
        query = session.query(ormclass).filter(ormclass.scn_name == self.scn_name)

        if self.version:
            query = query.filter(ormclass.version == self.version)

        # TODO: Better handled in db
        if name == 'Transformer':
            name = 'Trafo'

        df = pd.read_sql(query.statement,
                           session.bind,
                           index_col=name.lower() + '_id')

        if 'source' in df:
            df.source = df.source.map(self.id_to_source())

        return df

    def series_by_scenario(self, name, column):
        """
        """

        ormclass = self._mapped[name]

        # TODO: pls make more robust
        id_column = re.findall(r'[A-Z][^A-Z]*', name)[0] + '_' + 'id'
        id_column = id_column.lower()

        query = session.query(
            getattr(ormclass,id_column),
            getattr(ormclass,column)[self.start_h:self.end_h].\
            label(column)).filter(and_(
            ormclass.scn_name == self.scn_name,
            ormclass.temp_id == self.temp_id))

        if self.version:
            query = query.filter(ormclass.version == self.version)

        df =  pd.io.sql.read_sql(query.statement,
                        session.bind,
                        columns=[column],
                        index_col=id_column)

        df.index = df.index.astype(str)

        # change of format to fit pypsa
        df = df[column].apply(pd.Series).transpose()

        try:
            assert not df.empty
            df.index = self.timeindex
        except AssertionError:
            print("No data for %s in column %s." % (name, column))

        return df

    def build_network(self, *args, **kwargs):
        """
        """

        network = pypsa.Network()
        network.set_snapshots(self.timeindex)

        for c, v in self.config.items():

            # TODO: This is confusing, should be fixed in db
            name = 'StorageUnit' if c == 'Storage' else c

            network.import_components_from_dataframe(self.by_scenario(c), name)

            if isinstance(v, dict):
                for cc, vv in v.items():
                    for col in vv:
                        df = self.series_by_scenario(cc, col)
                        try:
                            pypsa.io.import_series_from_dataframe(network, df, name, col)
                        except (ValueError, AttributeError):
                            print("Series %s of component %s could not be "
                                  "imported" % (col, name))

        self.network = network

        return network

# for debugging
from egopowerflow.tools.plot import plot_line_loading, add_coordinates

md = NetworkScenario(session, method='lopf', end_h=2, start_h=1,
                          scn_name='Status Quo')
mdnw = md.build_network()
mdnw.lopf(snapshots=md.timeindex, solver_name='gurobi')
mdnw = add_coordinates(mdnw)
plot_line_loading(mdnw)

mdpf = NetworkScenario(session, method='pf', end_h=2, start_h=1,
                          scn_name='Status Quo')
mdpfnw = mdpf.build_network()
mdpfnw.pf(snapshots=mdpf.timeindex)

gr = NetworkScenario(session, method='lopf', end_h=2, start_h=1, version='v0.2.10',
                     prefix='EgoPfHv', scn_name='Status Quo')
grnw = gr.build_network()
grnw.lopf(snapshots=gr.timeindex, solver_name='gurobi')
grnw = add_coordinates(grnw)
plot_line_loading(grnw)

def results_to_oedb(session, network, grid='mv'):
    """Return results obtained from PyPSA to oedb"""
    # moved this here to prevent error when not using the mv-schema
    if grid.lower() == 'mv':
        from egoio.db_tables.calc_ego_mv_powerflow import ResBus, ResLine, ResTransformer
    elif grid.lower() == 'hv':
        print('Not implemented: Result schema for HV missing')
    else:
        print('Please enter mv or hv!')
    # from oemof import db
    # engine = db.engine(section='oedb')
    # from egoio.db_tables import calc_ego_mv_powerflow
    # calc_ego_mv_powerflow.Base.metadata.create_all(engine)

    #TODO: make this more safe, at least inform the user about the deleting results
    # empty all results table
    session.query(ResBus).delete()
    session.query(ResLine).delete()
    session.query(ResTransformer).delete()
    session.commit()

    # insert voltage at buses to database
    for col in network.buses_t.v_mag_pu:
        res_bus = ResBus(
            bus_id=col,
            v_mag_pu=network.buses_t.v_mag_pu[col].tolist()
        )
        session.add(res_bus)
    session.commit()

    # insert active and reactive power of lines to database
    for col in network.lines_t.p0:
        res_line = ResLine(
            line_id=col,
            p0=network.lines_t.p0[col].tolist(),
            q0=network.lines_t.q0[col].tolist(),
            p1=network.lines_t.p1[col].tolist(),
            q1=network.lines_t.q1[col].tolist()
        )
        session.add(res_line)
    session.commit()

    # insert active and reactive power of lines to database
    for col in network.transformers_t.p0:
        res_transformer = ResLine(
            trafo_id=col,
            p0=network.transformers_t.p0[col].tolist(),
            q0=network.transformers_t.q0[col].tolist(),
            p1=network.transformers_t.p1[col].tolist(),
            q1=network.transformers_t.q1[col].tolist()
        )
        session.add(res_transformer)
    session.commit()

def fix_storages(network):
    """
    Workaround to deal with the new name for storages
    used by PyPSA.
    Old: Storage
    New: StorageUnit

    Parameters
    ----------
    network : PyPSA network container

    Returns
    -------
    None
    """
    network.storage_units = network.storage_units.drop('state_of_charge_initial',1).\
                     rename(columns={'soc_initial':'state_of_charge_initial'})
    network.storage_units = network.storage_units.drop('cyclic_state_of_charge',1).\
                     rename(columns={'soc_cyclic':'cyclic_state_of_charge'})

if __name__ == '__main__':
    pass
<|endoftext|>


<commit_before>import json
import os
import re

# These will have to be updated manually over time, there's not an
# easy way to determine the latest version.
COMPILE_SDK_VERSION = 27
TARGET_SDK_VERSION = 27
BUILD_TOOLS_VERSION = '27.0.2'

COMPILE_SDK_RE = r'compileSdkVersion[\s][\w]+'
TARGET_SDK_RE = r'targetSdkVersion[\s][\w]+'
BUILD_TOOLS_RE = r'buildToolsVersion[\s][\'\"][\w\.]+[\'\"]'

GROUPS_ALLOWED_MAJOR_UPDATE = [
  'com.google.android.gms',
  'com.android.support'
]

# Depends on https://github.com/ben-manes/gradle-versions-plugin
#
# Must run this command:
# $ ./gradlew dependencyUpdates -Drevision=release -DoutputFormatter=json
INPUT_JSON = 'build/dependencyUpdates/report.json'

def findGradleFiles():
    """Finds all build.gradle files, recursively."""
    gradle_files = []
    for root, dirs, files in os.walk('.'):
        for file in files:
            if file.endswith('build.gradle'):
                gradle_files.append(os.path.join(root, file))

    return gradle_files

def getAndroidReplacements():
    """Gets a dictionary of all android-specific replacements to be made."""
    replacements = {}

    compileSdk = 'compileSdkVersion {}'.format(COMPILE_SDK_VERSION)
    targetSdk = 'targetSdkVersion {}'.format(TARGET_SDK_VERSION)
    buildToolsVersion = 'buildToolsVersion \'{}\''.format(BUILD_TOOLS_VERSION)

    replacements[COMPILE_SDK_RE] = compileSdk
    replacements[TARGET_SDK_RE] = targetSdk
    replacements[BUILD_TOOLS_RE] = buildToolsVersion

    return replacements

def isMajorUpdate(old_version, new_version):
  """Compares version strings to see if it's a major update."""
  old_major = old_version.split('.')[0]
  new_major = new_version.split('.')[0]

  return old_major != new_major

def getDepReplacements():
    """Gets a dictionary of all dependency replacements to be made."""
    replacements = {}
    with open(INPUT_JSON, 'r') as f:
        json_data = json.loads(f.read())

        outdated_deps = json_data['outdated']['dependencies']
        for dep in outdated_deps:
            group = dep['group']
            name = dep['name']

            curr_version = dep['version']
            new_version = dep['available']['release']

            if (isMajorUpdate(curr_version, new_version) and
                group not in GROUPS_ALLOWED_MAJOR_UPDATE):
              print 'Skipping major update to {}:{}'.format(group, name)
              continue

            curr_dep = '{}:{}:{}'.format(group, name, curr_version)
            new_dep = '{}:{}:{}'.format(group, name, new_version)
            replacements[curr_dep] = new_dep

    return replacements

def updateAll():
    """Runs through all build.gradle files and performs replacements."""
    replacements = {}
    replacements.update(getAndroidReplacements())
    replacements.update(getDepReplacements())

    # Print all updates found
    print 'Dependency updates:'
    for (k, v) in replacements.iteritems():
        print '{} --> {}'.format(k, v)

    # Iterate through each file and replace it
    for file in findGradleFiles():
        print 'Updating dependencies for: {}'.format(file)

        new_data = ''
        with open(file, 'r') as f:
            # Perform each replacement
            new_data = f.read()
            for (k, v) in replacements.iteritems():
                new_data = re.sub(k, v, new_data)

        # Write the file
        with open(file, 'w') as f:
            f.write(new_data)

if __name__ == '__main__':
  updateAll()
<commit_msg>Style updates<commit_after>import json
import os
import re

# These will have to be updated manually over time, there's not an
# easy way to determine the latest version.
COMPILE_SDK_VERSION = 27
TARGET_SDK_VERSION = 27
BUILD_TOOLS_VERSION = '27.0.2'

COMPILE_SDK_RE = r'compileSdkVersion[\s][\w]+'
TARGET_SDK_RE = r'targetSdkVersion[\s][\w]+'
BUILD_TOOLS_RE = r'buildToolsVersion[\s][\'\"][\w\.]+[\'\"]'

GROUPS_ALLOWED_MAJOR_UPDATE = [
  'com.google.android.gms',
  'com.android.support'
]

# Depends on https://github.com/ben-manes/gradle-versions-plugin
#
# Must run this command:
# $ ./gradlew dependencyUpdates -Drevision=release -DoutputFormatter=json
INPUT_JSON = 'build/dependencyUpdates/report.json'

def find_gradle_files():
    """Finds all build.gradle files, recursively."""
    gradle_files = []
    for root, dirs, files in os.walk('.'):
        for fn in files:
            if fn.endswith('build.gradle'):
                gradle_files.append(os.path.join(root, fn))

    return gradle_files

def get_android_replacements():
    """Gets a dictionary of all android-specific replacements to be made."""
    replacements = {}

    compileSdk = 'compileSdkVersion {}'.format(COMPILE_SDK_VERSION)
    targetSdk = 'targetSdkVersion {}'.format(TARGET_SDK_VERSION)
    buildToolsVersion = 'buildToolsVersion \'{}\''.format(BUILD_TOOLS_VERSION)

    replacements[COMPILE_SDK_RE] = compileSdk
    replacements[TARGET_SDK_RE] = targetSdk
    replacements[BUILD_TOOLS_RE] = buildToolsVersion

    return replacements

def is_major_update(old_version, new_version):
  """Compares version strings to see if it's a major update."""
  old_major = old_version.split('.')[0]
  new_major = new_version.split('.')[0]

  return old_major != new_major

def get_dep_replacements():
    """Gets a dictionary of all dependency replacements to be made."""
    replacements = {}
    with open(INPUT_JSON, 'r') as f:
        json_data = json.loads(f.read())

        outdated_deps = json_data['outdated']['dependencies']
        for dep in outdated_deps:
            group = dep['group']
            name = dep['name']

            curr_version = dep['version']
            new_version = dep['available']['release']

            if (is_major_update(curr_version, new_version) and
                group not in GROUPS_ALLOWED_MAJOR_UPDATE):
              print 'Skipping major update to {}:{}'.format(group, name)
              continue

            curr_dep = '{}:{}:{}'.format(group, name, curr_version)
            new_dep = '{}:{}:{}'.format(group, name, new_version)
            replacements[curr_dep] = new_dep

    return replacements

def update_all():
    """Runs through all build.gradle files and performs replacements."""
    replacements = {}
    replacements.update(get_android_replacements())
    replacements.update(get_dep_replacements())

    # Print all updates found
    print 'Dependency updates:'
    for (k, v) in replacements.iteritems():
        print '{} --> {}'.format(k, v)

    # Iterate through each file and replace it
    for fn in find_gradle_files():
        print 'Updating dependencies for: {}'.format(fn)

        new_data = ''
        with open(fn, 'r') as f:
            # Perform each replacement
            new_data = f.read()
            for (k, v) in replacements.iteritems():
                new_data = re.sub(k, v, new_data)

        # Write the file
        with open(fn, 'w') as f:
            f.write(new_data)

if __name__ == '__main__':
  update_all()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    Copyright (C) 2004-2012 Pexego Sistemas Informáticos All Rights Reserved
#    $Marta Vázquez Rodríguez$ <marta@pexego.es>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published
#    by the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
from openerp.osv import orm, fields

class purchase_order(orm.Model):
    _inherit = 'purchase.order'

    def _get_products_names(self, cr, uid, ids, field_name, args, context=None):
        if context is None:
            context = {}
        res = {}

        lang = ('lang' in context and context['lang'])

        for cur_obj in self.browse(cr, uid, ids, context={'lang':lang}):
            stream = []
            res[cur_obj.id] = "[]"
            if cur_obj.order_line:
                for line in cur_obj.order_line:
                    if line.product_id and line.product_id.name:
                        stream.append(line.product_id.name)
                res[cur_obj.id] = u" ###, ".join(stream)

        return res

    _columns = {
        'lines_product_name_str': fields.function(_get_products_names, method=True, string='Lines', type='char', size=255, readonly=True),
        'container_numbers': fields.char('Container numbers', size=32, help="Container numbers assigned to the order.", readonly=False, states={'done': [('readonly', True)],'cancel': [('readonly', True)]}),
    }
    _defaults = {
        'invoice_method': 'picking',
    }
<commit_msg>Añadida compañía en contexto cuando se llama a price_get<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    Copyright (C) 2004-2012 Pexego Sistemas Informáticos All Rights Reserved
#    $Marta Vázquez Rodríguez$ <marta@pexego.es>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published
#    by the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
from openerp.osv import orm, fields

class purchase_order(orm.Model):
    _inherit = 'purchase.order'

    def _get_products_names(self, cr, uid, ids, field_name, args, context=None):
        if context is None:
            context = {}
        res = {}

        lang = ('lang' in context and context['lang'])

        for cur_obj in self.browse(cr, uid, ids, context={'lang':lang}):
            stream = []
            res[cur_obj.id] = "[]"
            if cur_obj.order_line:
                for line in cur_obj.order_line:
                    if line.product_id and line.product_id.name:
                        stream.append(line.product_id.name)
                res[cur_obj.id] = u" ###, ".join(stream)

        return res

    _columns = {
        'lines_product_name_str': fields.function(_get_products_names, method=True, string='Lines', type='char', size=255, readonly=True),
        'container_numbers': fields.char('Container numbers', size=32, help="Container numbers assigned to the order.", readonly=False, states={'done': [('readonly', True)],'cancel': [('readonly', True)]}),
    }
    _defaults = {
        'invoice_method': 'picking',
    }

class procurement_order(orm.Model):
    _inherit = 'procurement.order'

    def _calc_new_qty_price(self, cr, uid, procurement, po_line=None, cancel=False, context=None):
        res = super(procurement_order, self)._calc_new_qty_price(cr, uid, procurement, po_line=po_line, cancel=cancel, context=context)
        # En la función original cuando se llama a price_get no se tiene en cuenta la compañía del abastecimiento,
        # y cuando es llamada desde planificadores usa la del usuario (generalmente admin) lo cual es un error en caso de multicompañía
        if res:
            qty = res[0]
            price = po_line.price_unit
            if qty != po_line.product_qty:
                pricelist_obj = self.pool.get('product.pricelist')
                pricelist_id = po_line.order_id.partner_id.property_product_pricelist_purchase.id
                ctx_company = dict(context or {}, force_company=procurement.company_id.id)
                uom_id = procurement.product_id.uom_po_id.id
                price = pricelist_obj.price_get(cr, uid, [pricelist_id], procurement.product_id.id, qty, po_line.order_id.partner_id.id, dict(ctx_company, uom=uom_id))[pricelist_id]
            res = qty, price
        return res
<|endoftext|>