<commit_before># -*- coding: utf-8 -*-

"""
Matchers for use in tests.

This module contains objects which can be used to write tests which are easier
to read, especially when using assertions against :py:class:`mock.Mock`
objects. Each matcher defines an __eq__ method, allowing instances of these
matchers to be used wherever you would usually pass a plain value.

For example, imagine you wanted to ensure that the `set_value` method of the
`foo` mock object was called exactly once with an integer. Previously you
would have to do something like:

    assert foo.set_value.call_count == 1
    assert isinstance(foo.set_value.call_args[0][0], int)

By using the `InstanceOf` matcher you can simply write:

    foo.set_value.assert_called_once_with(matchers.InstanceOf(int))

As a bonus, the second test will print substantially more useful debugging
output if it fails, e.g.

    E       AssertionError: Expected call: set_value(<instance of <type 'int'>>)
    E       Actual call: set_value('a string')

"""
from __future__ import unicode_literals

import re

from pyramid import httpexceptions

class Matcher(object):
    def __eq__(self, other):
        raise NotImplementedError("subclasses should provide an __eq__ method")

    def __ne__(self, other):
        return not self.__eq__(other)

class AnyCallable(Matcher):
    """An object __eq__ to any callable object."""

    def __eq__(self, other):
        """Return ``True`` if ``other`` is callable, ``False`` otherwise."""
        return callable(other)

class NativeString(Matcher):
    """
    Matches any native string with the given characters.

    "Native string" means the ``str`` type which is a byte string in Python 2
    and a unicode string in Python 3. Does not match unicode strings (type
    ``unicode``) in Python 2, or byte strings (type ``bytes``) in Python 3,
    even if they contain the same characters.

    In Python 3 a ``bytes`` is never ``==`` to a ``str`` anyway, even if they
    contain the same characters. But in Python 2 a ``str`` is equal to a
    ``unicode`` if they contain the same characters, and that's why this
    matcher is needed.

    TODO: Delete this matcher once we no longer support Python 2.

    """

    def __init__(self, string):
        self.string = str(string)

    def __eq__(self, other):
        if not isinstance(other, str):
            return False
        return other == self.string

    def __repr__(self):
        return '<native string matching "{string}">'.format(string=self.string)

    def lower(self):
        return NativeString(self.string.lower())

class InstanceOf(Matcher):
    """An object __eq__ to any object which is an instance of `type_`."""

    def __init__(self, type_):
        self.type = type_

    def __eq__(self, other):
        return isinstance(other, self.type)

    def __repr__(self):
        return "<instance of {!r}>".format(self.type)

class IterableWith(Matcher):
    """An object __eq__ to any iterable which yields `items`."""

    def __init__(self, items):
        self.items = items

    def __eq__(self, other):
        return list(other) == self.items

    def __repr__(self):
        return "<iterable with {!r}>".format(self.items)

class MappingContaining(Matcher):
    """An object __eq__ to any mapping with the passed `key`."""

    def __init__(self, key):
        self.key = key

    def __eq__(self, other):
        try:
            other[self.key]
        except (TypeError, KeyError):
            return False
        else:
            return True

    def __repr__(self):
        return "<mapping containing {!r}>".format(self.key)

class Redirect302To(Matcher):
    """Matches any HTTPFound redirect to the given URL."""

    def __init__(self, location):
        self.location = location

    def __eq__(self, other):
        if not isinstance(other, httpexceptions.HTTPFound):
            return False
        return other.location == self.location

class Redirect303To(Matcher):
    """Matches any HTTPSeeOther redirect to the given URL."""

    def __init__(self, location):
        self.location = location

    def __eq__(self, other):
        if not isinstance(other, httpexceptions.HTTPSeeOther):
            return False
        return other.location == self.location

class Regex(Matcher):
    """Matches any string matching the passed regex."""

    def __init__(self, patt):
        self.patt = re.compile(patt)

    def __eq__(self, other):
        return bool(self.patt.match(other))

    def __repr__(self):
        return "<string matching re {!r}>".format(self.patt.pattern)

class UnorderedList(Matcher):
    """
    Matches a list with the same items in any order.

    Matches any list that contains the same items as the given list
    (and no more), regardless of order.

    """

    def __init__(self, items):
        self.items = items

    def __eq__(self, other):
        if len(self.items) != len(other):
            return False
        for item in self.items:
            if item not in other:
                return False
        return True

    def __repr__(self):
        return "<unordered list containing {items}".format(items=self.items)
<commit_msg>matchers remove pyramid related<commit_after># -*- coding: utf-8 -*-

"""
Matchers for use in tests.

This module contains objects which can be used to write tests which are easier
to read, especially when using assertions against :py:class:`mock.Mock`
objects. Each matcher defines an __eq__ method, allowing instances of these
matchers to be used wherever you would usually pass a plain value.

For example, imagine you wanted to ensure that the `set_value` method of the
`foo` mock object was called exactly once with an integer. Previously you
would have to do something like:

    assert foo.set_value.call_count == 1
    assert isinstance(foo.set_value.call_args[0][0], int)

By using the `InstanceOf` matcher you can simply write:

    foo.set_value.assert_called_once_with(matchers.InstanceOf(int))

As a bonus, the second test will print substantially more useful debugging
output if it fails, e.g.

    E       AssertionError: Expected call: set_value(<instance of <type 'int'>>)
    E       Actual call: set_value('a string')

"""
from __future__ import unicode_literals

import re

class Matcher(object):
    def __eq__(self, other):
        raise NotImplementedError("subclasses should provide an __eq__ method")

    def __ne__(self, other):
        return not self.__eq__(other)

class AnyCallable(Matcher):
    """An object __eq__ to any callable object."""

    def __eq__(self, other):
        """Return ``True`` if ``other`` is callable, ``False`` otherwise."""
        return callable(other)

class NativeString(Matcher):
    """
    Matches any native string with the given characters.

    "Native string" means the ``str`` type which is a byte string in Python 2
    and a unicode string in Python 3. Does not match unicode strings (type
    ``unicode``) in Python 2, or byte strings (type ``bytes``) in Python 3,
    even if they contain the same characters.

    In Python 3 a ``bytes`` is never ``==`` to a ``str`` anyway, even if they
    contain the same characters. But in Python 2 a ``str`` is equal to a
    ``unicode`` if they contain the same characters, and that's why this
    matcher is needed.

    TODO: Delete this matcher once we no longer support Python 2.

    """

    def __init__(self, string):
        self.string = str(string)

    def __eq__(self, other):
        if not isinstance(other, str):
            return False
        return other == self.string

    def __repr__(self):
        return '<native string matching "{string}">'.format(string=self.string)

    def lower(self):
        return NativeString(self.string.lower())

class InstanceOf(Matcher):
    """An object __eq__ to any object which is an instance of `type_`."""

    def __init__(self, type_):
        self.type = type_

    def __eq__(self, other):
        return isinstance(other, self.type)

    def __repr__(self):
        return "<instance of {!r}>".format(self.type)

class IterableWith(Matcher):
    """An object __eq__ to any iterable which yields `items`."""

    def __init__(self, items):
        self.items = items

    def __eq__(self, other):
        return list(other) == self.items

    def __repr__(self):
        return "<iterable with {!r}>".format(self.items)

class MappingContaining(Matcher):
    """An object __eq__ to any mapping with the passed `key`."""

    def __init__(self, key):
        self.key = key

    def __eq__(self, other):
        try:
            other[self.key]
        except (TypeError, KeyError):
            return False
        else:
            return True

    def __repr__(self):
        return "<mapping containing {!r}>".format(self.key)

class Regex(Matcher):
    """Matches any string matching the passed regex."""

    def __init__(self, patt):
        self.patt = re.compile(patt)

    def __eq__(self, other):
        return bool(self.patt.match(other))

    def __repr__(self):
        return "<string matching re {!r}>".format(self.patt.pattern)

class UnorderedList(Matcher):
    """
    Matches a list with the same items in any order.

    Matches any list that contains the same items as the given list
    (and no more), regardless of order.

    """

    def __init__(self, items):
        self.items = items

    def __eq__(self, other):
        if len(self.items) != len(other):
            return False
        for item in self.items:
            if item not in other:
                return False
        return True

    def __repr__(self):
        return "<unordered list containing {items}".format(items=self.items)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# @License EPL-1.0 <http://spdx.org/licenses/EPL-1.0>
##############################################################################
# Copyright (c) 2016 The Linux Foundation and others.
#
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Eclipse Public License v1.0
# which accompanies this distribution, and is available at
# http://www.eclipse.org/legal/epl-v10.html
##############################################################################

import click

from spectrometer import create_app
from spectrometer import mirror_repos
from spectrometer.reporttool.git import GitReport

@click.group()
@click.pass_context
@click.version_option()
def cli(ctx):
    pass

###############################################################################
# Server
###############################################################################

@cli.group()
@click.option('-c', '--conf', default='/etc/spectrometer/config.py')
@click.pass_context
def server(ctx, conf):
    ctx.obj['conf'] = conf

@click.command()
@click.pass_context
def sync_repos(ctx):
    app = create_app(ctx.obj['conf'])
    gerrit_url = app.config['GERRIT_URL']
    mirror_dir = app.config['MIRROR_DIR']
    mirror_repos(mirror_dir, gerrit_url)

@click.command()
@click.pass_context
def start(ctx):
    """Runs the spectrometer server

    This function is effectively the spectrometer main() function and is the
    entry point for spectrometer server.
    """
    click.echo('Starting spectrometer...')

    app = create_app(ctx.obj['conf'])
    app.run(
        threaded=True,
        host=app.config['LISTEN_HOST'],
        port=app.config['LISTEN_PORT'],
    )

@click.command()
@click.pass_context
def profile(ctx):
    from werkzeug.contrib.profiler import ProfilerMiddleware
    from spectrometer import create_app

    app = create_app(ctx.obj['conf'])
    app.config['PROFILE'] = True
    app.wsgi_app = ProfilerMiddleware(app.wsgi_app, restrictions=[30])
    app.run(
        debug=True,
        host=app.config['LISTEN_HOST'],
        port=app.config['LISTEN_PORT'],
    )

server.add_command(sync_repos)
server.add_command(start)
server.add_command(profile)

###############################################################################
# Report Tool
###############################################################################

@cli.group()
@click.option('--server-url', default='https://spectrometer.opendaylight.org/api/',
              help='URL to Spectrometer API server.')
@click.pass_context
def reporttool(ctx, server_url):
    """Entry point for the report tool command"""
    ctx.obj['SERVER_URL'] = server_url

@click.command()
@click.argument('ref1')
@click.argument('ref2')
@click.pass_context
def full(ctx, ref1, ref2):
    """Generate a full report"""
    git_data = GitReport(ctx.obj['SERVER_URL'], ref1, ref2)
    git_data.print_report()

reporttool.add_command(full)

cli(obj={})
<commit_msg>Simplify sync_repos to just sync<commit_after># -*- coding: utf-8 -*-

# @License EPL-1.0 <http://spdx.org/licenses/EPL-1.0>
##############################################################################
# Copyright (c) 2016 The Linux Foundation and others.
#
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Eclipse Public License v1.0
# which accompanies this distribution, and is available at
# http://www.eclipse.org/legal/epl-v10.html
##############################################################################

import click

from spectrometer import create_app
from spectrometer import mirror_repos
from spectrometer.reporttool.git import GitReport

@click.group()
@click.pass_context
@click.version_option()
def cli(ctx):
    pass

###############################################################################
# Server
###############################################################################

@cli.group()
@click.option('-c', '--conf', default='/etc/spectrometer/config.py')
@click.pass_context
def server(ctx, conf):
    ctx.obj['conf'] = conf

@click.command()
@click.pass_context
def sync(ctx):
    """Syncs Spectrometer data

    Force syncs the git repositories to pull down the latest data.
    """
    app = create_app(ctx.obj['conf'])
    gerrit_url = app.config['GERRIT_URL']
    mirror_dir = app.config['MIRROR_DIR']
    mirror_repos(mirror_dir, gerrit_url)

@click.command()
@click.pass_context
def start(ctx):
    """Runs the spectrometer server

    This function is effectively the spectrometer main() function and is the
    entry point for spectrometer server.
    """
    click.echo('Starting spectrometer...')

    app = create_app(ctx.obj['conf'])
    app.run(
        threaded=True,
        host=app.config['LISTEN_HOST'],
        port=app.config['LISTEN_PORT'],
    )

@click.command()
@click.pass_context
def profile(ctx):
    from werkzeug.contrib.profiler import ProfilerMiddleware
    from spectrometer import create_app

    app = create_app(ctx.obj['conf'])
    app.config['PROFILE'] = True
    app.wsgi_app = ProfilerMiddleware(app.wsgi_app, restrictions=[30])
    app.run(
        debug=True,
        host=app.config['LISTEN_HOST'],
        port=app.config['LISTEN_PORT'],
    )

server.add_command(sync)
server.add_command(start)
server.add_command(profile)

###############################################################################
# Report Tool
###############################################################################

@cli.group()
@click.option('--server-url', default='https://spectrometer.opendaylight.org/api/',
              help='URL to Spectrometer API server.')
@click.pass_context
def reporttool(ctx, server_url):
    """Entry point for the report tool command"""
    ctx.obj['SERVER_URL'] = server_url

@click.command()
@click.argument('ref1')
@click.argument('ref2')
@click.pass_context
def full(ctx, ref1, ref2):
    """Generate a full report"""
    git_data = GitReport(ctx.obj['SERVER_URL'], ref1, ref2)
    git_data.print_report()

reporttool.add_command(full)

cli(obj={})
<|endoftext|>


<commit_before><commit_msg>options<commit_after><|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import uuid

from django.conf import settings
from django.contrib.contenttypes.fields import GenericForeignKey
from django.contrib.contenttypes.models import ContentType
from django.core.mail import send_mail
from django.db import models
from django.template.loader import render_to_string
from django.utils import timezone
from django.utils.translation import ugettext as _
from rest_framework.exceptions import NotAcceptable

from apps.events.models import AttendanceEvent, Attendee
from apps.marks.models import Suspension

User = settings.AUTH_USER_MODEL

class Payment(models.Model):

    TYPE_CHOICES = (
        (1, _('Umiddelbar')),
        (2, _('Frist')),
        (3, _('Utsettelse')),
    )

    # Make sure these exist in settings if they are to be used.
    STRIPE_KEY_CHOICES = (
        ('arrkom', 'arrkom'),
        ('prokom', 'prokom'),
        ('trikom', 'trikom'),
        ('fagkom', 'fagkom'),
    )

    content_type = models.ForeignKey(ContentType)
    """Which model the payment is created for. For attendance events this should be attendance_event(påmelding)."""
    object_id = models.PositiveIntegerField()
    """Object id for the model chosen in content_type."""
    content_object = GenericForeignKey()
    """Helper attribute which points to the object select in object_id"""
    stripe_key = models.CharField(
        _('stripe key'),
        max_length=10,
        choices=STRIPE_KEY_CHOICES,
        default="arrkom"
    )
    """Which Stripe key to use for payments"""

    payment_type = models.SmallIntegerField(_('type'), choices=TYPE_CHOICES)
    """
        Which payment type to use for payments.
        Can be one of the following:

        - Immediate
        - Deadline. See :attr:`deadline`
        - Delay. See :attr:`delay`
    """

    deadline = models.DateTimeField(_("frist"), blank=True, null=True)
    """Shared deadline for all payments"""
    active = models.BooleanField(default=True)
    """Is payment activated"""
    delay = models.SmallIntegerField(_('utsettelse'), blank=True, null=True, default=2)
    """Number of days after user attended which they have to pay."""

    # For logging and history
    added_date = models.DateTimeField(_("opprettet dato"), auto_now=True)
    """Day of payment creation. Automatically set"""
    changed_date = models.DateTimeField(auto_now=True, editable=False)
    """Last changed. Automatically set"""
    last_changed_by = models.ForeignKey(User, editable=False, null=True)  # Blank and null is temperarly
    """User who last changed payment. Automatically set"""

    def payment_delays(self):
        return self.paymentdelay_set.filter(active=True)

    def payment_delay_users(self):
        return [payment_delay.user for payment_delay in self.payment_delays()]

    def create_payment_delay(self, user, deadline):
        """
        Method for creating a payment delay for user with a set deadline.
        If a PaymentDelay already exsists it is updated with new deadline.

        :param OnlineUser user: User to create payment delay for
        :param datetime.datetime deadline: Payment delay deadline
        """
        payment_delays = self.paymentdelay_set.filter(payment=self, user=user)

        if payment_delays:
            for payment_delay in payment_delays:
                payment_delay.valid_to = deadline
                payment_delay.save()
        else:
            PaymentDelay.objects.create(payment=self, user=user, valid_to=deadline)

    def description(self):
        if self._is_type(AttendanceEvent):
            return self.content_object.event.title

    def get_receipt_description(self):
        receipt_description = ""
        description = [' '] * 30
        temp = self.description()[0:25]
        description[0:len(temp)+1] = list(temp)
        for c in description:
            receipt_description += c
        return receipt_description

    def responsible_mail(self):
        if self._is_type(AttendanceEvent):
            event_type = self.content_object.event.event_type
            if event_type == 1 or event_type == 4:  # Sosialt & Utflukt
                return settings.EMAIL_ARRKOM
            elif event_type == 2:  # Bedpres
                return settings.EMAIL_BEDKOM
            elif event_type == 3:  # Kurs
                return settings.EMAIL_FAGKOM
            elif event_type == 5:  # Ekskursjon
                return settings.EMAIL_EKSKOM
            else:
                return settings.DEFAULT_FROM_EMAIL
        else:
            return settings.DEFAULT_FROM_EMAIL

    def handle_payment(self, user):
        """
        Method for handling payments from user.
        Deletes any relevant payment delays, suspensions and marks user's attendee object as paid.

        :param OnlineUser user: User who paid

        """
        if self._is_type(AttendanceEvent):
            attendee = Attendee.objects.filter(event=self.content_object, user=user)

            # Delete payment delay objects for the user if there are any
            delays = PaymentDelay.objects.filter(payment=self, user=user)
            for delay in delays:
                delay.delete()

            # If the user is suspended because of a lack of payment the suspension is deactivated.
            suspensions = Suspension.objects.filter(payment_id=self.id, user=user)
            for suspension in suspensions:
                suspension.active = False
                suspension.save()

            if attendee:
                attendee[0].paid = True
                attendee[0].save()
            else:
                Attendee.objects.create(event=self.content_object, user=user, paid=True)

    def handle_refund(self, payment_relation):
        """
        Method for handling refunds.
        For events it deletes the Attendee object.

        :param str host: hostname to include in email
        :param PaymentRelation payment_relation: user payment to refund
        """
        payment_relation.refunded = True
        payment_relation.save()

        if self._is_type(AttendanceEvent):
            Attendee.objects.get(event=self.content_object,
                                 user=payment_relation.user).delete()

    def check_refund(self, payment_relation):
        """Method for checking if the payment can be refunded"""
        if self._is_type(AttendanceEvent):
            attendance_event = self.content_object
            if attendance_event.unattend_deadline < timezone.now():
                return False, _("Fristen for og melde seg av har utgått")
            if len(Attendee.objects.filter(event=attendance_event, user=payment_relation.user)) == 0:
                return False, _("Du er ikke påmeldt dette arrangementet.")
            if attendance_event.event.event_start < timezone.now():
                return False, _("Dette arrangementet har allerede startet.")

            return True, ''

        return False, 'Refund checks not implemented'

    def prices(self):
        return self.paymentprice_set.all()

    def price(self):
        # TODO implement group based pricing
        if self.paymentprice_set.count() > 0:
            return self.paymentprice_set.all()[0]
        return None

    def _is_type(self, model_type):
        return ContentType.objects.get_for_model(model_type) == self.content_type

    def __str__(self):
        return self.description()

    class Meta(object):
        unique_together = ('content_type', 'object_id')

        verbose_name = _("betaling")
        verbose_name_plural = _("betalinger")

class PaymentPrice(models.Model):
    payment = models.ForeignKey(Payment)
    """Payment object"""
    price = models.IntegerField(_("pris"))
    """Price in NOK"""
    description = models.CharField(max_length=128)
    """Description of price"""

    def __str__(self):
        return self.description + " (" + str(self.price) + "kr)"

    class Meta(object):
        verbose_name = _("pris")
        verbose_name_plural = _("priser")

class PaymentRelation(models.Model):
    """Payment metadata for user"""

    payment = models.ForeignKey(Payment)
    """Payment object"""
    payment_price = models.ForeignKey(PaymentPrice)
    """Price object"""
    user = models.ForeignKey(User)
    """User who paid"""
    datetime = models.DateTimeField(auto_now=True)
    """Datetime when payment was created"""
    refunded = models.BooleanField(default=False)
    """Has payment been refunded"""

    unique_id = models.CharField(max_length=128, null=True, blank=True)
    """Unique ID for payment"""
    stripe_id = models.CharField(max_length=128)
    """ID from Stripe payment"""

    def get_timestamp(self):
        return self.datetime

    def get_subject(self):
        return "[Kvittering] " + self.payment.description()

    def get_description(self):
        return self.payment.description()

    def get_items(self):
        items = [{
                    'name': self.payment.description(),
                    'price': self.payment_price.price,
                    'quantity': 1
                 }]
        return items

    def get_from_mail(self):
        return settings.EMAIL_ARRKOM

    def get_to_mail(self):
        return self.user.email

    def save(self, *args, **kwargs):
        if not self.unique_id:
            self.unique_id = str(uuid.uuid4())
        super(PaymentRelation, self).save(*args, **kwargs)
        receipt = PaymentReceipt(object_id=self.id,
                                 content_type=ContentType.objects.get_for_model(self))
        receipt.save()

    def __str__(self):
        return self.payment.description() + " - " + str(self.user)

    class Meta(object):
        verbose_name = _("betalingsrelasjon")
        verbose_name_plural = _("betalingsrelasjoner")

class PaymentDelay(models.Model):
    """User specific payment deadline"""
    payment = models.ForeignKey(Payment)
    """Payment object"""
    user = models.ForeignKey(User)
    """User object"""
    valid_to = models.DateTimeField()
    """Payment deadline"""

    active = models.BooleanField(default=True)
    """Is payment delay active"""

    def __str__(self):
        return self.payment.description() + " - " + str(self.user)

    class Meta(object):
        unique_together = ('payment', 'user')

        verbose_name = _('betalingsutsettelse')
        verbose_name_plural = _('betalingsutsettelser')

class PaymentTransaction(models.Model):
    """Transaction for a user"""
    user = models.ForeignKey(User)
    """User object"""
    amount = models.IntegerField(null=True, blank=True)
    """Amount in NOK"""
    used_stripe = models.BooleanField(default=False)
    """Was transaction paid for using Stripe"""
    datetime = models.DateTimeField(auto_now=True)
    """Transaction creation datetime. Automatically generated."""

    def get_timestamp(self):
        return self.datetime

    def get_subject(self):
        return "[Kvittering] Saldoinnskudd på online.ntnu.no"

    def get_description(self):
        return "saldoinnskudd på online.ntnu.no"

    def get_items(self):
        items = [{
                'name': "Påfyll av saldo",
                'price': self.amount,
                'quantity': 1
            }]
        return items

    def get_from_mail(self):
        return settings.EMAIL_TRIKOM

    def get_to_mail(self):
        return self.user.email

    def __str__(self):
        return str(self.user) + " - " + str(self.amount) + "(" + str(self.datetime) + ")"

    def save(self, *args, **kwargs):
        if not self.pk:
            self.user.saldo = self.user.saldo + self.amount

            if self.user.saldo < 0:
                raise NotAcceptable("Insufficient funds")

            self.user.save()
        super().save(*args, **kwargs)
        receipt = PaymentReceipt(object_id=self.id,
                                 content_type=ContentType.objects.get_for_model(self))
        receipt.save()

    class Meta:
        ordering = ['-datetime']
        verbose_name = _('transaksjon')
        verbose_name_plural = _('transaksjoner')

class PaymentReceipt(models.Model):
    """Transaction receipt"""
    receipt_id = models.UUIDField(default=uuid.uuid4)
    content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE)
    """Which model the receipt is created for"""
    object_id = models.PositiveIntegerField(null=True)
    """Object id for the model chosen in content_type"""
    content_object = GenericForeignKey()
    """Helper attribute which points to the object select in object_id"""

    def save(self, *args, **kwargs):
        transaction = self.content_object
        self.timestamp = transaction.get_timestamp()
        self.subject = transaction.get_subject()
        self.description = transaction.get_description()
        self.items = transaction.get_items()
        self.from_mail = transaction.get_from_mail()
        self.to_mail = [transaction.get_to_mail()]

        self._send_receipt(self.timestamp, self.subject, self.description, self.receipt_id,
                           self.items, self.to_mail, self.from_mail)
        super().save(*args, **kwargs)

    def _is_type(self, model_type):
        """Function for comparing content types to differentiate payment types"""
        return ContentType.objects.get_for_model(model_type) == self.content_type

    def _send_receipt(self, payment_date, subject, description, payment_id, items, to_mail, from_mail):
        """Send confirmation email with receipt
        param receipt: object
        """

        # Calculate total item price and ensure correct input to template.
        receipt_items = []
        total_amount = 0
        for item in items:
            receipt_items.append({
                    'amount': item['price'],
                    'description': item['name'],
                    'quantity': item['quantity']
                    })
            total_amount += item['price'] * item['quantity']

        context = {
            'payment_date': payment_date,
            'description': description,
            'payment_id': payment_id,
            'items': receipt_items,
            'total_amount': total_amount,
            'to_mail': to_mail,
            'from_mail': from_mail
        }

        email_message = render_to_string('payment/email/confirmation_mail.txt', context)
        send_mail(subject, email_message, from_mail, to_mail)
<commit_msg>Log occurences where Payment is not connected to an attendance event.<commit_after># -*- coding: utf-8 -*-

import logging
import uuid

from django.conf import settings
from django.contrib.contenttypes.fields import GenericForeignKey
from django.contrib.contenttypes.models import ContentType
from django.core.mail import send_mail
from django.db import models
from django.template.loader import render_to_string
from django.utils import timezone
from django.utils.translation import ugettext as _
from rest_framework.exceptions import NotAcceptable

from apps.events.models import AttendanceEvent, Attendee
from apps.marks.models import Suspension

User = settings.AUTH_USER_MODEL

class Payment(models.Model):

    TYPE_CHOICES = (
        (1, _('Umiddelbar')),
        (2, _('Frist')),
        (3, _('Utsettelse')),
    )

    # Make sure these exist in settings if they are to be used.
    STRIPE_KEY_CHOICES = (
        ('arrkom', 'arrkom'),
        ('prokom', 'prokom'),
        ('trikom', 'trikom'),
        ('fagkom', 'fagkom'),
    )

    content_type = models.ForeignKey(ContentType)
    """Which model the payment is created for. For attendance events this should be attendance_event(påmelding)."""
    object_id = models.PositiveIntegerField()
    """Object id for the model chosen in content_type."""
    content_object = GenericForeignKey()
    """Helper attribute which points to the object select in object_id"""
    stripe_key = models.CharField(
        _('stripe key'),
        max_length=10,
        choices=STRIPE_KEY_CHOICES,
        default="arrkom"
    )
    """Which Stripe key to use for payments"""

    payment_type = models.SmallIntegerField(_('type'), choices=TYPE_CHOICES)
    """
        Which payment type to use for payments.
        Can be one of the following:

        - Immediate
        - Deadline. See :attr:`deadline`
        - Delay. See :attr:`delay`
    """

    deadline = models.DateTimeField(_("frist"), blank=True, null=True)
    """Shared deadline for all payments"""
    active = models.BooleanField(default=True)
    """Is payment activated"""
    delay = models.SmallIntegerField(_('utsettelse'), blank=True, null=True, default=2)
    """Number of days after user attended which they have to pay."""

    # For logging and history
    added_date = models.DateTimeField(_("opprettet dato"), auto_now=True)
    """Day of payment creation. Automatically set"""
    changed_date = models.DateTimeField(auto_now=True, editable=False)
    """Last changed. Automatically set"""
    last_changed_by = models.ForeignKey(User, editable=False, null=True)  # Blank and null is temperarly
    """User who last changed payment. Automatically set"""

    def payment_delays(self):
        return self.paymentdelay_set.filter(active=True)

    def payment_delay_users(self):
        return [payment_delay.user for payment_delay in self.payment_delays()]

    def create_payment_delay(self, user, deadline):
        """
        Method for creating a payment delay for user with a set deadline.
        If a PaymentDelay already exsists it is updated with new deadline.

        :param OnlineUser user: User to create payment delay for
        :param datetime.datetime deadline: Payment delay deadline
        """
        payment_delays = self.paymentdelay_set.filter(payment=self, user=user)

        if payment_delays:
            for payment_delay in payment_delays:
                payment_delay.valid_to = deadline
                payment_delay.save()
        else:
            PaymentDelay.objects.create(payment=self, user=user, valid_to=deadline)

    def description(self):
        if self._is_type(AttendanceEvent):
            return self.content_object.event.title
        logging.getLogger(__name__).error(
            "Trying to get description for payment #{}, but it's not AttendanceEvent.".format(self.pk)
        )
        return 'No description'

    def get_receipt_description(self):
        receipt_description = ""
        description = [' '] * 30
        temp = self.description()[0:25]
        description[0:len(temp)+1] = list(temp)
        for c in description:
            receipt_description += c
        return receipt_description

    def responsible_mail(self):
        if self._is_type(AttendanceEvent):
            event_type = self.content_object.event.event_type
            if event_type == 1 or event_type == 4:  # Sosialt & Utflukt
                return settings.EMAIL_ARRKOM
            elif event_type == 2:  # Bedpres
                return settings.EMAIL_BEDKOM
            elif event_type == 3:  # Kurs
                return settings.EMAIL_FAGKOM
            elif event_type == 5:  # Ekskursjon
                return settings.EMAIL_EKSKOM
            else:
                return settings.DEFAULT_FROM_EMAIL
        else:
            return settings.DEFAULT_FROM_EMAIL

    def handle_payment(self, user):
        """
        Method for handling payments from user.
        Deletes any relevant payment delays, suspensions and marks user's attendee object as paid.

        :param OnlineUser user: User who paid

        """
        if self._is_type(AttendanceEvent):
            attendee = Attendee.objects.filter(event=self.content_object, user=user)

            # Delete payment delay objects for the user if there are any
            delays = PaymentDelay.objects.filter(payment=self, user=user)
            for delay in delays:
                delay.delete()

            # If the user is suspended because of a lack of payment the suspension is deactivated.
            suspensions = Suspension.objects.filter(payment_id=self.id, user=user)
            for suspension in suspensions:
                suspension.active = False
                suspension.save()

            if attendee:
                attendee[0].paid = True
                attendee[0].save()
            else:
                Attendee.objects.create(event=self.content_object, user=user, paid=True)

    def handle_refund(self, payment_relation):
        """
        Method for handling refunds.
        For events it deletes the Attendee object.

        :param str host: hostname to include in email
        :param PaymentRelation payment_relation: user payment to refund
        """
        payment_relation.refunded = True
        payment_relation.save()

        if self._is_type(AttendanceEvent):
            Attendee.objects.get(event=self.content_object,
                                 user=payment_relation.user).delete()

    def check_refund(self, payment_relation):
        """Method for checking if the payment can be refunded"""
        if self._is_type(AttendanceEvent):
            attendance_event = self.content_object
            if attendance_event.unattend_deadline < timezone.now():
                return False, _("Fristen for og melde seg av har utgått")
            if len(Attendee.objects.filter(event=attendance_event, user=payment_relation.user)) == 0:
                return False, _("Du er ikke påmeldt dette arrangementet.")
            if attendance_event.event.event_start < timezone.now():
                return False, _("Dette arrangementet har allerede startet.")

            return True, ''

        return False, 'Refund checks not implemented'

    def prices(self):
        return self.paymentprice_set.all()

    def price(self):
        # TODO implement group based pricing
        if self.paymentprice_set.count() > 0:
            return self.paymentprice_set.all()[0]
        return None

    def _is_type(self, model_type):
        return ContentType.objects.get_for_model(model_type) == self.content_type

    def __str__(self):
        return self.description()

    class Meta(object):
        unique_together = ('content_type', 'object_id')

        verbose_name = _("betaling")
        verbose_name_plural = _("betalinger")

class PaymentPrice(models.Model):
    payment = models.ForeignKey(Payment)
    """Payment object"""
    price = models.IntegerField(_("pris"))
    """Price in NOK"""
    description = models.CharField(max_length=128)
    """Description of price"""

    def __str__(self):
        return self.description + " (" + str(self.price) + "kr)"

    class Meta(object):
        verbose_name = _("pris")
        verbose_name_plural = _("priser")

class PaymentRelation(models.Model):
    """Payment metadata for user"""

    payment = models.ForeignKey(Payment)
    """Payment object"""
    payment_price = models.ForeignKey(PaymentPrice)
    """Price object"""
    user = models.ForeignKey(User)
    """User who paid"""
    datetime = models.DateTimeField(auto_now=True)
    """Datetime when payment was created"""
    refunded = models.BooleanField(default=False)
    """Has payment been refunded"""

    unique_id = models.CharField(max_length=128, null=True, blank=True)
    """Unique ID for payment"""
    stripe_id = models.CharField(max_length=128)
    """ID from Stripe payment"""

    def get_timestamp(self):
        return self.datetime

    def get_subject(self):
        return "[Kvittering] " + self.payment.description()

    def get_description(self):
        return self.payment.description()

    def get_items(self):
        items = [{
                    'name': self.payment.description(),
                    'price': self.payment_price.price,
                    'quantity': 1
                 }]
        return items

    def get_from_mail(self):
        return settings.EMAIL_ARRKOM

    def get_to_mail(self):
        return self.user.email

    def save(self, *args, **kwargs):
        if not self.unique_id:
            self.unique_id = str(uuid.uuid4())
        super(PaymentRelation, self).save(*args, **kwargs)
        receipt = PaymentReceipt(object_id=self.id,
                                 content_type=ContentType.objects.get_for_model(self))
        receipt.save()

    def __str__(self):
        return self.payment.description() + " - " + str(self.user)

    class Meta(object):
        verbose_name = _("betalingsrelasjon")
        verbose_name_plural = _("betalingsrelasjoner")

class PaymentDelay(models.Model):
    """User specific payment deadline"""
    payment = models.ForeignKey(Payment)
    """Payment object"""
    user = models.ForeignKey(User)
    """User object"""
    valid_to = models.DateTimeField()
    """Payment deadline"""

    active = models.BooleanField(default=True)
    """Is payment delay active"""

    def __str__(self):
        return self.payment.description() + " - " + str(self.user)

    class Meta(object):
        unique_together = ('payment', 'user')

        verbose_name = _('betalingsutsettelse')
        verbose_name_plural = _('betalingsutsettelser')

class PaymentTransaction(models.Model):
    """Transaction for a user"""
    user = models.ForeignKey(User)
    """User object"""
    amount = models.IntegerField(null=True, blank=True)
    """Amount in NOK"""
    used_stripe = models.BooleanField(default=False)
    """Was transaction paid for using Stripe"""
    datetime = models.DateTimeField(auto_now=True)
    """Transaction creation datetime. Automatically generated."""

    def get_timestamp(self):
        return self.datetime

    def get_subject(self):
        return "[Kvittering] Saldoinnskudd på online.ntnu.no"

    def get_description(self):
        return "saldoinnskudd på online.ntnu.no"

    def get_items(self):
        items = [{
                'name': "Påfyll av saldo",
                'price': self.amount,
                'quantity': 1
            }]
        return items

    def get_from_mail(self):
        return settings.EMAIL_TRIKOM

    def get_to_mail(self):
        return self.user.email

    def __str__(self):
        return str(self.user) + " - " + str(self.amount) + "(" + str(self.datetime) + ")"

    def save(self, *args, **kwargs):
        if not self.pk:
            self.user.saldo = self.user.saldo + self.amount

            if self.user.saldo < 0:
                raise NotAcceptable("Insufficient funds")

            self.user.save()
        super().save(*args, **kwargs)
        receipt = PaymentReceipt(object_id=self.id,
                                 content_type=ContentType.objects.get_for_model(self))
        receipt.save()

    class Meta:
        ordering = ['-datetime']
        verbose_name = _('transaksjon')
        verbose_name_plural = _('transaksjoner')

class PaymentReceipt(models.Model):
    """Transaction receipt"""
    receipt_id = models.UUIDField(default=uuid.uuid4)
    content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE)
    """Which model the receipt is created for"""
    object_id = models.PositiveIntegerField(null=True)
    """Object id for the model chosen in content_type"""
    content_object = GenericForeignKey()
    """Helper attribute which points to the object select in object_id"""

    def save(self, *args, **kwargs):
        transaction = self.content_object
        self.timestamp = transaction.get_timestamp()
        self.subject = transaction.get_subject()
        self.description = transaction.get_description()
        self.items = transaction.get_items()
        self.from_mail = transaction.get_from_mail()
        self.to_mail = [transaction.get_to_mail()]

        self._send_receipt(self.timestamp, self.subject, self.description, self.receipt_id,
                           self.items, self.to_mail, self.from_mail)
        super().save(*args, **kwargs)

    def _is_type(self, model_type):
        """Function for comparing content types to differentiate payment types"""
        return ContentType.objects.get_for_model(model_type) == self.content_type

    def _send_receipt(self, payment_date, subject, description, payment_id, items, to_mail, from_mail):
        """Send confirmation email with receipt
        param receipt: object
        """

        # Calculate total item price and ensure correct input to template.
        receipt_items = []
        total_amount = 0
        for item in items:
            receipt_items.append({
                    'amount': item['price'],
                    'description': item['name'],
                    'quantity': item['quantity']
                    })
            total_amount += item['price'] * item['quantity']

        context = {
            'payment_date': payment_date,
            'description': description,
            'payment_id': payment_id,
            'items': receipt_items,
            'total_amount': total_amount,
            'to_mail': to_mail,
            'from_mail': from_mail
        }

        email_message = render_to_string('payment/email/confirmation_mail.txt', context)
        send_mail(subject, email_message, from_mail, to_mail)
<|endoftext|>


<commit_before>__version__ = '0.9.27'
VERSION = __version__
<commit_msg>Release 0.9.28<commit_after>__version__ = '0.9.28'
VERSION = __version__
<|endoftext|>


<commit_before>import logging
import urllib

import requests
from requests.packages.urllib3.util.retry import Retry
from requests.adapters import HTTPAdapter

from gitlabform.configuration.core import ConfigurationCore

s = requests.Session()

retries = Retry(total=3,
                backoff_factor=0.25,
                status_forcelist=[500, 502, 503, 504])

s.mount('http://', HTTPAdapter(max_retries=retries))
s.mount('https://', HTTPAdapter(max_retries=retries))

class GitLabCore:

    url = None
    __token = None

    def __init__(self, config_path=None):
        configuration = ConfigurationCore(config_path)
        self.url = configuration.get("gitlab|url")
        self.__token = configuration.get("gitlab|token")
        try:
            version = self._make_requests_to_api("version")
            logging.info("Connected to GitLab version: %s (%s)" % (version['version'], version['revision']))
        except Exception as e:
            raise TestRequestFailedException(e)

    def get_project(self, project_and_group_or_id):
        return self._make_requests_to_api("projects/%s", project_and_group_or_id)

    def _get_user_id(self, username):
        users = self._make_requests_to_api("users?username=%s", username, 'GET')

        # this API endpoint is for lookup, not search, so 'username' has to be full and exact username
        # also it's not possible to get more than 1 user as a result

        if len(users) == 0:
            raise NotFoundException("No users found when searching for username '%s'" % username)

        return users[0]['id']

    def _get_group_id(self, path):
        groups = self._make_requests_to_api("groups?search=%s", path, 'GET')

        # this API endpoint is for search, not lookup, so: 1. we may find more than 1 group if the 'group' is part of
        # more than 1 groups name or path, for example: search for 'foo' will return 'foo' and 'foobar' groups.
        # 2. we may find non-exact match, for example: search for 'foo' will return 'foobar' group.
        # we only want the exact matches here.

        if len(groups) == 0:
            raise NotFoundException("No groups found when searching for group path '%s'" % path)

        for group in groups:
            if group['path'] == path:
                return group['id']

        raise NotFoundException("None of the found group(s) when searching for group path '%s'"
                                " has an exactly matching path: %s" % (path, groups))

    def _get_project_id(self, project_and_group):
        # This is a NEW workaround for https://github.com/gitlabhq/gitlabhq/issues/8290
        result = self.get_project(project_and_group)
        return str(result['id'])

    def _make_requests_to_api(self, path_as_format_string, args=None, method='GET', data=None, expected_codes=200,
                              paginated=False, json=None):
        if not paginated:
            response = self._make_request_to_api(path_as_format_string, args, method, data, expected_codes, json)
            return response.json()
        else:
            if '?' in path_as_format_string:
                path_as_format_string += '&per_page=100'
            else:
                path_as_format_string += '?per_page=100'

            first_response = self._make_request_to_api(path_as_format_string, args, method, data, expected_codes, json)
            results = first_response.json()
            total_pages = int(first_response.headers['X-Total-Pages'])
            for page in range(2, total_pages + 1):
                response = self._make_request_to_api(path_as_format_string + "&page=" + str(page), args, method, data,
                                                     expected_codes, json)
                results += response.json()

        return results

    def _make_request_to_api(self, path_as_format_string, args, method, data, expected_codes, json):
        if data and json:
            raise Exception("You need to pass either data or json, not both!")

        url = self.url + "/api/v3/" + self._format_with_url_encoding(path_as_format_string, args)
        logging.debug("URL-encoded url=%s" % url)
        headers = {'PRIVATE-TOKEN': self.__token}
        if data:
            response = s.request(method, url, headers=headers, data=data, timeout=10)
        elif json:
            response = s.request(method, url, headers=headers, json=json, timeout=10)
        else:
            response = s.request(method, url, headers=headers, timeout=10)
        logging.debug("response code=%s" % response.status_code)
        if response.status_code == 404:
            raise NotFoundException("Resource path='%s' not found!" % url)
        elif not self._is_expected_code(response.status_code, expected_codes):
            e = UnexpectedResponseException(
                "Request url='%s', method=%s, data='%s' failed "
                "- expected code(s) %s, got code %s & body: '%s'" %
                (url, method, data,
                 self._expected_code_to_str(expected_codes), response.status_code, response.content))
            e.status_code = response.status_code
            raise e
        else:
            return response

    @staticmethod
    def _format_with_url_encoding(format_string, single_arg_or_args_tuple):

        # we want to URL-encode all the args, but not the path itself which looks like "/foo/%s/bar"
        # because / here are NOT to be URL-encoded

        if not single_arg_or_args_tuple:
            # there are no params, so the format_string is the URL
            return format_string
        else:
            if type(single_arg_or_args_tuple) == tuple:
                # URL-encode each arg in the tuple and return it as tuple too
                url_encoded_args = ()
                for arg in single_arg_or_args_tuple:
                    url_encoded_args += (urllib.parse.quote_plus(str(arg)), )
            else:
                # URL-encode single arg
                url_encoded_args = urllib.parse.quote_plus(single_arg_or_args_tuple)

            return format_string % url_encoded_args

    @staticmethod
    def _is_expected_code(code, expected_codes):
        if isinstance(expected_codes, int):
            return code == expected_codes
        elif isinstance(expected_codes, list):
            return code in expected_codes

    @staticmethod
    def _expected_code_to_str(expected_codes):
        if isinstance(expected_codes, int):
            return str(expected_codes)
        elif isinstance(expected_codes, list):
            return ', '.join(map(lambda x: str(x), expected_codes))

class TestRequestFailedException(Exception):
    pass

class NotFoundException(Exception):
    pass

class UnexpectedResponseException(Exception):
    pass
<commit_msg>Switch to API v4<commit_after>import logging
import urllib

import requests
from requests.packages.urllib3.util.retry import Retry
from requests.adapters import HTTPAdapter

from gitlabform.configuration.core import ConfigurationCore

s = requests.Session()

retries = Retry(total=3,
                backoff_factor=0.25,
                status_forcelist=[500, 502, 503, 504])

s.mount('http://', HTTPAdapter(max_retries=retries))
s.mount('https://', HTTPAdapter(max_retries=retries))

class GitLabCore:

    url = None
    __token = None

    def __init__(self, config_path=None):
        configuration = ConfigurationCore(config_path)
        self.url = configuration.get("gitlab|url")
        self.__token = configuration.get("gitlab|token")
        try:
            version = self._make_requests_to_api("version")
            logging.info("Connected to GitLab version: %s (%s)" % (version['version'], version['revision']))
        except Exception as e:
            raise TestRequestFailedException(e)

    def get_project(self, project_and_group_or_id):
        return self._make_requests_to_api("projects/%s", project_and_group_or_id)

    def _get_user_id(self, username):
        users = self._make_requests_to_api("users?username=%s", username, 'GET')

        # this API endpoint is for lookup, not search, so 'username' has to be full and exact username
        # also it's not possible to get more than 1 user as a result

        if len(users) == 0:
            raise NotFoundException("No users found when searching for username '%s'" % username)

        return users[0]['id']

    def _get_group_id(self, path):
        groups = self._make_requests_to_api("groups?search=%s", path, 'GET')

        # this API endpoint is for search, not lookup, so: 1. we may find more than 1 group if the 'group' is part of
        # more than 1 groups name or path, for example: search for 'foo' will return 'foo' and 'foobar' groups.
        # 2. we may find non-exact match, for example: search for 'foo' will return 'foobar' group.
        # we only want the exact matches here.

        if len(groups) == 0:
            raise NotFoundException("No groups found when searching for group path '%s'" % path)

        for group in groups:
            if group['path'] == path:
                return group['id']

        raise NotFoundException("None of the found group(s) when searching for group path '%s'"
                                " has an exactly matching path: %s" % (path, groups))

    def _get_project_id(self, project_and_group):
        # This is a NEW workaround for https://github.com/gitlabhq/gitlabhq/issues/8290
        result = self.get_project(project_and_group)
        return str(result['id'])

    def _make_requests_to_api(self, path_as_format_string, args=None, method='GET', data=None, expected_codes=200,
                              paginated=False, json=None):
        if not paginated:
            response = self._make_request_to_api(path_as_format_string, args, method, data, expected_codes, json)
            return response.json()
        else:
            if '?' in path_as_format_string:
                path_as_format_string += '&per_page=100'
            else:
                path_as_format_string += '?per_page=100'

            first_response = self._make_request_to_api(path_as_format_string, args, method, data, expected_codes, json)
            results = first_response.json()
            total_pages = int(first_response.headers['X-Total-Pages'])
            for page in range(2, total_pages + 1):
                response = self._make_request_to_api(path_as_format_string + "&page=" + str(page), args, method, data,
                                                     expected_codes, json)
                results += response.json()

        return results

    def _make_request_to_api(self, path_as_format_string, args, method, data, expected_codes, json):
        if data and json:
            raise Exception("You need to pass either data or json, not both!")

        url = self.url + "/api/v4/" + self._format_with_url_encoding(path_as_format_string, args)
        logging.debug("URL-encoded url=%s" % url)
        headers = {'PRIVATE-TOKEN': self.__token}
        if data:
            response = s.request(method, url, headers=headers, data=data, timeout=10)
        elif json:
            response = s.request(method, url, headers=headers, json=json, timeout=10)
        else:
            response = s.request(method, url, headers=headers, timeout=10)
        logging.debug("response code=%s" % response.status_code)
        if response.status_code == 404:
            raise NotFoundException("Resource path='%s' not found!" % url)
        elif not self._is_expected_code(response.status_code, expected_codes):
            e = UnexpectedResponseException(
                "Request url='%s', method=%s, data='%s' failed "
                "- expected code(s) %s, got code %s & body: '%s'" %
                (url, method, data,
                 self._expected_code_to_str(expected_codes), response.status_code, response.content))
            e.status_code = response.status_code
            raise e
        else:
            return response

    @staticmethod
    def _format_with_url_encoding(format_string, single_arg_or_args_tuple):

        # we want to URL-encode all the args, but not the path itself which looks like "/foo/%s/bar"
        # because / here are NOT to be URL-encoded

        if not single_arg_or_args_tuple:
            # there are no params, so the format_string is the URL
            return format_string
        else:
            if type(single_arg_or_args_tuple) == tuple:
                # URL-encode each arg in the tuple and return it as tuple too
                url_encoded_args = ()
                for arg in single_arg_or_args_tuple:
                    url_encoded_args += (urllib.parse.quote_plus(str(arg)), )
            else:
                # URL-encode single arg
                url_encoded_args = urllib.parse.quote_plus(single_arg_or_args_tuple)

            return format_string % url_encoded_args

    @staticmethod
    def _is_expected_code(code, expected_codes):
        if isinstance(expected_codes, int):
            return code == expected_codes
        elif isinstance(expected_codes, list):
            return code in expected_codes

    @staticmethod
    def _expected_code_to_str(expected_codes):
        if isinstance(expected_codes, int):
            return str(expected_codes)
        elif isinstance(expected_codes, list):
            return ', '.join(map(lambda x: str(x), expected_codes))

class TestRequestFailedException(Exception):
    pass

class NotFoundException(Exception):
    pass

class UnexpectedResponseException(Exception):
    pass
<|endoftext|>


<commit_before>"""
Django settings for seed_stage_based_messaging project.

For more information on this file, see
https://docs.djangoproject.com/en/1.9/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.9/ref/settings/
"""

import os
import djcelery
import dj_database_url

from kombu import Exchange, Queue

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(__file__))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.9/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = os.environ.get('SECRET_KEY', 'REPLACEME')

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = os.environ.get('DEBUG', False)

TEMPLATE_DEBUG = DEBUG

ALLOWED_HOSTS = ['*']

# Application definition

INSTALLED_APPS = (
    # admin
    'django.contrib.admin',
    # core
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    # 3rd party
    'djcelery',
    'raven.contrib.django.raven_compat',
    'rest_framework',
    'rest_framework.authtoken',
    'django_filters',
    'contentstore',
    # us
    'subscriptions',

)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'seed_stage_based_messaging.urls'

WSGI_APPLICATION = 'seed_stage_based_messaging.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.9/ref/settings/#databases

DATABASES = {
    'default': dj_database_url.config(
        default=os.environ.get(
            'STAGE_BASED_MESSAGING_DATABASE',
            'postgres://postgres:@localhost/seed_stage_based_messaging')),
}

# Internationalization
# https://docs.djangoproject.com/en/1.9/topics/i18n/

LANGUAGE_CODE = 'en-gb'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.9/howto/static-files/

STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
    'django.contrib.staticfiles.finders.FileSystemFinder',
)

STATIC_ROOT = 'staticfiles'
STATIC_URL = '/static/'

# TEMPLATE_CONTEXT_PROCESSORS = (
#     "django.core.context_processors.request",
# )

# Sentry configuration
RAVEN_CONFIG = {
    # DevOps will supply you with this.
    'dsn': os.environ.get('STAGE_BASED_MESSAGING_SENTRY_DSN', None),
}

# REST Framework conf defaults
REST_FRAMEWORK = {
    'DEFAULT_PERMISSION_CLASSES': ('rest_framework.permissions.IsAdminUser',),
    'PAGINATE_BY': 1000,
    'DEFAULT_AUTHENTICATION_CLASSES': (
        'rest_framework.authentication.BasicAuthentication',
        'rest_framework.authentication.TokenAuthentication',
    ),
    'DEFAULT_PERMISSION_CLASSES': (
        'rest_framework.permissions.IsAuthenticated',
    ),
    'DEFAULT_FILTER_BACKENDS': ('rest_framework.filters.DjangoFilterBackend',)
}

# Celery configuration options
CELERY_RESULT_BACKEND = 'djcelery.backends.database:DatabaseBackend'
CELERYBEAT_SCHEDULER = 'djcelery.schedulers.DatabaseScheduler'

BROKER_URL = os.environ.get('RABBITMQ_URL', 'redis://localhost:6379/0')

CELERY_DEFAULT_QUEUE = 'seed_stage_based_messaging'
CELERY_QUEUES = (
    Queue('seed_stage_based_messaging',
          Exchange('seed_stage_based_messaging'),
          routing_key='seed_stage_based_messaging'),
)

CELERY_ALWAYS_EAGER = False

# Tell Celery where to find the tasks
CELERY_IMPORTS = (
    'subscriptions.tasks',
)

CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json'
CELERY_ACCEPT_CONTENT = ['json']

djcelery.setup_loader()

BASE_URL = 'http://seed-project/api/v1'

SUBSCRIPTIONS_URL = '%s/subscriptions' % BASE_URL

SCHEDULER_URL = '%s/scheduler' % BASE_URL
SCHEDULER_API_TOKEN = 'scheduler_api_token'
<commit_msg>Tweak static_root<commit_after>"""
Django settings for seed_stage_based_messaging project.

For more information on this file, see
https://docs.djangoproject.com/en/1.9/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.9/ref/settings/
"""

import os
import djcelery
import dj_database_url

from kombu import Exchange, Queue

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(__file__))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.9/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = os.environ.get('SECRET_KEY', 'REPLACEME')

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = os.environ.get('DEBUG', False)

TEMPLATE_DEBUG = DEBUG

ALLOWED_HOSTS = ['*']

# Application definition

INSTALLED_APPS = (
    # admin
    'django.contrib.admin',
    # core
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    # 3rd party
    'djcelery',
    'raven.contrib.django.raven_compat',
    'rest_framework',
    'rest_framework.authtoken',
    'django_filters',
    'contentstore',
    # us
    'subscriptions',

)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'seed_stage_based_messaging.urls'

WSGI_APPLICATION = 'seed_stage_based_messaging.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.9/ref/settings/#databases

DATABASES = {
    'default': dj_database_url.config(
        default=os.environ.get(
            'STAGE_BASED_MESSAGING_DATABASE',
            'postgres://postgres:@localhost/seed_stage_based_messaging')),
}

# Internationalization
# https://docs.djangoproject.com/en/1.9/topics/i18n/

LANGUAGE_CODE = 'en-gb'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.9/howto/static-files/

STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
    'django.contrib.staticfiles.finders.FileSystemFinder',
)

STATIC_ROOT = 'static'
STATIC_URL = '/static/'

# TEMPLATE_CONTEXT_PROCESSORS = (
#     "django.core.context_processors.request",
# )

# Sentry configuration
RAVEN_CONFIG = {
    # DevOps will supply you with this.
    'dsn': os.environ.get('STAGE_BASED_MESSAGING_SENTRY_DSN', None),
}

# REST Framework conf defaults
REST_FRAMEWORK = {
    'DEFAULT_PERMISSION_CLASSES': ('rest_framework.permissions.IsAdminUser',),
    'PAGINATE_BY': 1000,
    'DEFAULT_AUTHENTICATION_CLASSES': (
        'rest_framework.authentication.BasicAuthentication',
        'rest_framework.authentication.TokenAuthentication',
    ),
    'DEFAULT_PERMISSION_CLASSES': (
        'rest_framework.permissions.IsAuthenticated',
    ),
    'DEFAULT_FILTER_BACKENDS': ('rest_framework.filters.DjangoFilterBackend',)
}

# Celery configuration options
CELERY_RESULT_BACKEND = 'djcelery.backends.database:DatabaseBackend'
CELERYBEAT_SCHEDULER = 'djcelery.schedulers.DatabaseScheduler'

BROKER_URL = os.environ.get('RABBITMQ_URL', 'redis://localhost:6379/0')

CELERY_DEFAULT_QUEUE = 'seed_stage_based_messaging'
CELERY_QUEUES = (
    Queue('seed_stage_based_messaging',
          Exchange('seed_stage_based_messaging'),
          routing_key='seed_stage_based_messaging'),
)

CELERY_ALWAYS_EAGER = False

# Tell Celery where to find the tasks
CELERY_IMPORTS = (
    'subscriptions.tasks',
)

CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json'
CELERY_ACCEPT_CONTENT = ['json']

djcelery.setup_loader()

BASE_URL = 'http://seed-project/api/v1'

SUBSCRIPTIONS_URL = '%s/subscriptions' % BASE_URL

SCHEDULER_URL = '%s/scheduler' % BASE_URL
SCHEDULER_API_TOKEN = 'scheduler_api_token'
<|endoftext|>


<commit_before>import os
import os.path
import subprocess

class DefaultRepositoryHandler():
    def __init__(self, cwd=None):
        self.cwd = cwd or os.getcwd()

    def repository_exists(self, path):
        return os.path.exists(os.path.join(self.cwd, path, '.git'))

    def update_repository(self, path, remote_url, remote_branch):
        print 'Fetching repository %s ...' % path

        remote_name = 'origin'
        repository_path = os.path.join(self.cwd, path)

        url = subprocess.check_output(
            ['git', 'config', 'remote.%s.url' % remote_name],
            cwd=repository_path
        )

        if url != remote_url:
            subprocess.check_call(
                ['git', 'config', 'remote.%s.url' % remote_name, remote_url],
                cwd=repository_path
            )

        subprocess.check_call(
            ['git', 'fetch', remote_name],
            cwd=repository_path
        )

        output = subprocess.check_output(
            ['git', 'cherry', '%s/%s' % (remote_name, remote_branch), 'HEAD'],
            cwd=repository_path
        ).strip()
        local_change_count = len(output.split('\n')) - 1

        output = subprocess.check_output(
            ['git', 'cherry', 'HEAD', '%s/%s' % (remote_name, remote_branch)],
            cwd=repository_path
        ).strip()
        remote_change_count = len(output.split('\n')) - 1

        if remote_change_count or local_change_count:
            print (
                'Branch has %s more commits and %s less commits '
                'than %s/%s' % (
                    local_change_count, remote_change_count, remote_name,
                    remote_branch
                )
            )

    def clone_repository(self, path, remote_url, remote_branch):
        print 'Cloning repository %s ...' % path
        subprocess.check_call(
            ['git', 'clone', remote_url, '-b',  remote_branch, path],
            cwd=self.cwd
        )

    def update_or_clone(self, path, remote_url, remote_branch):
        if self.repository_exists(path):
            self.update_repository(path, remote_url, remote_branch)
        else:
            self.clone_repository(path, remote_url, remote_branch)
<commit_msg>If there are remote changes and no local changes, try to fast forward.<commit_after>import os
import os.path
import subprocess

class DefaultRepositoryHandler():
    def __init__(self, cwd=None):
        self.cwd = cwd or os.getcwd()

    def repository_exists(self, path):
        return os.path.exists(os.path.join(self.cwd, path, '.git'))

    def update_repository(self, path, remote_url, remote_branch):
        print 'Fetching repository %s ...' % path

        remote_name = 'origin'
        repository_path = os.path.join(self.cwd, path)

        url = subprocess.check_output(
            ['git', 'config', 'remote.%s.url' % remote_name],
            cwd=repository_path
        )

        if url != remote_url:
            subprocess.check_call(
                ['git', 'config', 'remote.%s.url' % remote_name, remote_url],
                cwd=repository_path
            )

        subprocess.check_call(
            ['git', 'fetch', remote_name],
            cwd=repository_path
        )

        output = subprocess.check_output(
            ['git', 'cherry', '%s/%s' % (remote_name, remote_branch), 'HEAD'],
            cwd=repository_path
        ).strip()
        local_change_count = len(output.split('\n')) - 1

        output = subprocess.check_output(
            ['git', 'cherry', 'HEAD', '%s/%s' % (remote_name, remote_branch)],
            cwd=repository_path
        ).strip()
        remote_change_count = len(output.split('\n')) - 1

        if remote_change_count or local_change_count:
            print (
                'Branch has %s more commits and %s less commits '
                'than %s/%s' % (
                    local_change_count, remote_change_count, remote_name,
                    remote_branch
                )
            )

        if remote_change_count and not local_change_count:
            print "Fast forwarding repository..."
            subprocess.check_call(
                ['git', 'merge', '%s/%s' % (remote_name, remote_branch), '--ff-only'],
                cwd=repository_path
            )

    def clone_repository(self, path, remote_url, remote_branch):
        print 'Cloning repository %s ...' % path
        subprocess.check_call(
            ['git', 'clone', remote_url, '-b',  remote_branch, path],
            cwd=self.cwd
        )

    def update_or_clone(self, path, remote_url, remote_branch):
        if self.repository_exists(path):
            self.update_repository(path, remote_url, remote_branch)
        else:
            self.clone_repository(path, remote_url, remote_branch)
<|endoftext|>


<commit_before><commit_msg>add little docstring, avoid errors if line costs and hvmv transformers are already in pypsa network in preprocess_...<commit_after><|endoftext|>


<commit_before>"""
This plugin provides analysis of RTT reported by
the sender side. It basicly takes the average observed
rtt by `ss` command on the server.

If the traffic from a server goes to two different queues,
so the flows have different RTTs, the average value would
be wrong.
"""

import numpy as np
from functools import partial

from plumbum.cmd import bash
import operator
import os

from aqmt import processes
from aqmt.plot import collectionutil, treeutil
from aqmt.plot.common import add_plot, add_scale

def pre_hook(testcase):
    """
    Run ss and collect server rtt stats in the background
    of the test. Calculates average rtt between all flows
    on each node.
    """
    for node in ['A', 'B']:
        cmd = bash['-c', """
            # We need to run the next command it in the background and wait
            # for it so that we can actually SIGTERM the bash process
            # causing the ssh connection to terminate and bash script
            # to end.
            ssh -tt """ + os.environ['IP_SERVER%s_MGMT' % node] + """ '
                getdata() {
                    ss -ni "( src %s or dst %s or src %s or dst %s )" | \\
                        grep -B1 " rtt:" | \\
                        awk "!(NR%%2){print $1\\" \\"p\\" XX \\"\\$0}{p=\\$5\\" \\"\\$6}" | \\
                        sed "s/XX.* rtt:\\([^\\/]\\+\\).*/\\1/"
                }
                start=$(($(date +%%s%%N)/1000000))
                while true; do
                    now=$(($(date +%%s%%N)/1000000))
                    getdata $((now-start))
                    sleep .1
                done
                ' >'%s' &
            sshpid=$!
            trap killssh TERM
            killssh() {
                kill $sshpid
                exit
            }
            wait
            """ % (
                os.environ['IP_CLIENTA'],
                os.environ['IP_CLIENTA'],
                os.environ['IP_CLIENTB'],
                os.environ['IP_CLIENTB'],
                testcase.test_folder + '/rtt-%s.log' % node
            )
        ]

        pid = testcase.testenv.run(cmd, bg=True)
        processes.add_known_pid(pid)

def _generate_stats(numbers):
    if len(numbers) == 0:
        res = ['-', '-', '-', '-', '-', '-', '-', '-', '-']
    else:
        res = [
            np.average(numbers).astype('str'),
            '-',  # not used: np.std(numbers).astype('str'),
            '-',  # not used: np.min(numbers).astype('str'),
            np.percentile(numbers, 1, interpolation='lower').astype('str'),
            '-',  # not used: np.percentile(numbers, 25, interpolation='lower').astype('str'),
            '-',  # not used: np.percentile(numbers, 50, interpolation='lower').astype('str'),
            '-',  # not used: np.percentile(numbers, 75, interpolation='lower').astype('str'),
            np.percentile(numbers, 99, interpolation='lower').astype('str'),
            '-',  # not used: np.max(numbers).astype('str'),
        ]

    return ' '.join(res)

def _parse_rtt_of_test(testfolder, node):
    # make tables with sums
    table_n = {}
    table_rtt = {}

    with open(testfolder + '/rtt-%s.log' %  node) as f:
        for line in f:
            i, src, dst, rtt = line.split()
            i = int(i)
            rtt = float(rtt)

            if i not in table_n:
                table_n[i] = 0
                table_rtt[i] = 0

            table_n[i] += 1
            table_rtt[i] += rtt

    # make average within each sample
    ret = []
    for i, n in sorted(table_n.items(), key=operator.itemgetter(0)):
        ret.append((i, table_rtt[i] / n))
    return ret

def plot_flow_rtt(initial_delay=0):
    """
    Initial delay is number of seconds the recording started
    before the actual test started.
    """

    def get_list(testfolder, node):
        ret = ''
        for i, rtt in _parse_rtt_of_test(testfolder, node):
            ret += '%s %f\n' % (i, rtt)
        return ret

    def plot(testfolder, plotdef):
        label_y_pos = -0.06 * (1/plotdef.y_scale)
        xpos = "($1/1000-" + str(initial_delay) + ")"

        gpi = """
            set format y "%g"
            set format x "%g s"
            set ylabel 'RTT reported at sender [ms]'
            set style fill transparent solid 0.5 noborder
            set key above
            set yrange [0<*:*]

            set label "Time:" at graph -0.01, graph """ + str(label_y_pos) + """ font 'Times-Roman,11pt' tc rgb 'black' right

            $data_node_a << EOD
            """ + get_list(testfolder, 'A') + """
            EOD
            $data_node_b << EOD
            """ + get_list(testfolder, 'B') + """
            EOD

            stats $data_node_a using ($1/1000) nooutput
            set xrange [-""" + str(initial_delay) + """:STATS_max]

            plot \\
                1 lc rgb '#FFFF0000' notitle, \\
                $data_node_a using """ + xpos + """:2               with lines   lc 1 lw 1.5 title 'Avg. RTT node A', \\
                $data_node_a using """ + xpos + """:2 smooth bezier with lines   lc 1 lw 3 notitle, \\
                $data_node_b using """ + xpos + """:2               with lines   lc 2 lw 1.5 title 'Avg. RTT node B', \\
                $data_node_b using """ + xpos + """:2 smooth bezier with lines   lc 2 lw 3 notitle

            set format y "%h"
            unset label
            """
        return {
            'gpi': gpi,
            'skip_sample_line': True,
        }
    return plot

def plot_comparison_rtt(y_logarithmic=False, keys=True):
    """
    Plot graph of interrupts and context switches
    """

    def plot(tree, plotdef, leaf_hook):
        gap = collectionutil.get_gap(tree)

        gpi = """
            set ylabel "RTT reported at sender\\n{/Times:Italic=10 [ms] (p_1, mean, p_{99})}"
            """ + add_scale(y_logarithmic)

        # add hidden line to force autoscaling if using logarithimic plot without any points
        plot_gpi = " 1 lc rgb '#FFFF0000' notitle, \\\n"

        def leaf(subtree, is_first_set, x):
            nonlocal gpi, plot_gpi
            leaf_hook(subtree, is_first_set, x)
            add_title = keys and is_first_set

            def parse_rtt(node, testcase_folder):
                rtts = _parse_rtt_of_test(testcase_folder, node)

                # take last half, so it is stable
                rtts = rtts[int(len(rtts)/2):]

                # get second element of list which is the rtt value
                rtts = [x[1] for x in rtts]

                return _generate_stats(rtts)

            gpi += """
                $data_rtt_a""" + str(x) + """ << EOD
                """ + collectionutil.merge_testcase_data(subtree, partial(parse_rtt, 'A'), plotdef.x_axis) + """
                EOD
                $data_rtt_b""" + str(x) + """ << EOD
                """ + collectionutil.merge_testcase_data(subtree, partial(parse_rtt, 'B'), plotdef.x_axis) + """
                EOD
                """

            xpos = '($1+%s)' % str(x)
            xtic = "" if plotdef.custom_xtics else ":xtic(2)"

            plot_gpi += ("""\\
                %s   using """ + xpos + """:3:10:6%s  with yerrorbars ls 3 pt 7 ps 0.4 lw 1.5 lc 1 %s, \\
                ''   using """ + xpos + """:3         with lines      lc rgb 'gray' notitle, \\
                %s   using """ + xpos + """:3:10:6    with yerrorbars ls 3 pt 7 ps 0.4 lw 1.5 lc 2 %s, \\
                ''   using """ + xpos + """:3         with lines      lc rgb 'gray' notitle, \\
                """) % (
                    "$data_rtt_a" + str(x),
                    xtic,
                    "title 'Server A' " if add_title else 'notitle',
                    "$data_rtt_b" + str(x),
                    "title 'Server B' " if add_title else 'notitle'
                )

        treeutil.walk_leaf(tree, leaf)
        gpi += """
            plot \\
            """ + add_plot(plot_gpi) + """

            unset logscale y
            """

        return {
            'y_logarithmic': y_logarithmic,
            'gpi': gpi,
            'key_rows': 1 if keys else 0,
        }

    return plot
<commit_msg>plugin/ss_rtt: Add option to subtract base rtt<commit_after>"""
This plugin provides analysis of RTT reported by
the sender side. It basicly takes the average observed
rtt by `ss` command on the server.

If the traffic from a server goes to two different queues,
so the flows have different RTTs, the average value would
be wrong.
"""

import numpy as np
from functools import partial

from plumbum.cmd import bash
import operator
import os

from aqmt import processes
from aqmt.plot import collectionutil, treeutil
from aqmt.plot.common import add_plot, add_scale
from ..testenv import read_metadata

def pre_hook(testcase):
    """
    Run ss and collect server rtt stats in the background
    of the test. Calculates average rtt between all flows
    on each node.
    """
    for node in ['A', 'B']:
        cmd = bash['-c', """
            # We need to run the next command it in the background and wait
            # for it so that we can actually SIGTERM the bash process
            # causing the ssh connection to terminate and bash script
            # to end.
            ssh -tt """ + os.environ['IP_SERVER%s_MGMT' % node] + """ '
                getdata() {
                    ss -ni "( src %s or dst %s or src %s or dst %s )" | \\
                        grep -B1 " rtt:" | \\
                        awk "!(NR%%2){print $1\\" \\"p\\" XX \\"\\$0}{p=\\$5\\" \\"\\$6}" | \\
                        sed "s/XX.* rtt:\\([^\\/]\\+\\).*/\\1/"
                }
                start=$(($(date +%%s%%N)/1000000))
                while true; do
                    now=$(($(date +%%s%%N)/1000000))
                    getdata $((now-start))
                    sleep .1
                done
                ' >'%s' &
            sshpid=$!
            trap killssh TERM
            killssh() {
                kill $sshpid
                exit
            }
            wait
            """ % (
                os.environ['IP_CLIENTA'],
                os.environ['IP_CLIENTA'],
                os.environ['IP_CLIENTB'],
                os.environ['IP_CLIENTB'],
                testcase.test_folder + '/rtt-%s.log' % node
            )
        ]

        pid = testcase.testenv.run(cmd, bg=True)
        processes.add_known_pid(pid)

def _generate_stats(numbers):
    if len(numbers) == 0:
        res = ['-', '-', '-', '-', '-', '-', '-', '-', '-']
    else:
        res = [
            np.average(numbers).astype('str'),
            '-',  # not used: np.std(numbers).astype('str'),
            '-',  # not used: np.min(numbers).astype('str'),
            np.percentile(numbers, 1, interpolation='lower').astype('str'),
            '-',  # not used: np.percentile(numbers, 25, interpolation='lower').astype('str'),
            '-',  # not used: np.percentile(numbers, 50, interpolation='lower').astype('str'),
            '-',  # not used: np.percentile(numbers, 75, interpolation='lower').astype('str'),
            np.percentile(numbers, 99, interpolation='lower').astype('str'),
            '-',  # not used: np.max(numbers).astype('str'),
        ]

    return ' '.join(res)

def _parse_rtt_of_test(testfolder, node, subtract_base_rtt):
    # make tables with sums
    table_n = {}
    table_rtt = {}

    subtract = 0
    if subtract_base_rtt:
        metadata_kv, metadata_lines = read_metadata(testfolder + '/details')
        subtract = float(metadata_kv['testbed_rtt_clients'])
        subtract += float(metadata_kv['testbed_rtt_server%s' % node.lower()])

    with open(testfolder + '/rtt-%s.log' %  node) as f:
        for line in f:
            i, src, dst, rtt = line.split()
            i = int(i)

            if i not in table_n:
                table_n[i] = 0
                table_rtt[i] = 0

            table_n[i] += 1
            table_rtt[i] += float(rtt) - subtract

    # make average within each sample
    ret = []
    for i, n in sorted(table_n.items(), key=operator.itemgetter(0)):
        ret.append((i, table_rtt[i] / n))
    return ret

def plot_flow_rtt(initial_delay=0, subtract_base_rtt=False):
    """
    Initial delay is number of seconds the recording started
    before the actual test started.
    """

    def get_list(testfolder, node):
        ret = ''
        for i, rtt in _parse_rtt_of_test(testfolder, node, subtract_base_rtt):
            ret += '%s %f\n' % (i, rtt)
        return ret

    def plot(testfolder, plotdef):
        label_y_pos = -0.06 * (1/plotdef.y_scale)
        xpos = "($1/1000-" + str(initial_delay) + ")"

        ylabel_extra = ''
        if subtract_base_rtt:
            ylabel_extra = '\\n{/Times:Italic=10 Excluding base RTT}'

        gpi = """
            set format y "%g"
            set format x "%g s"
            set ylabel 'RTT reported at sender [ms]""" + ylabel_extra + """'
            set style fill transparent solid 0.5 noborder
            set key above
            set yrange [0<*:*]

            set label "Time:" at graph -0.01, graph """ + str(label_y_pos) + """ font 'Times-Roman,11pt' tc rgb 'black' right

            $data_node_a << EOD
            """ + get_list(testfolder, 'A') + """
            EOD
            $data_node_b << EOD
            """ + get_list(testfolder, 'B') + """
            EOD

            stats $data_node_a using ($1/1000) nooutput
            set xrange [-""" + str(initial_delay) + """:STATS_max]

            plot \\
                1 lc rgb '#FFFF0000' notitle, \\
                $data_node_a using """ + xpos + """:2               with lines   lc 1 lw 1.5 title 'Avg. RTT node A', \\
                $data_node_a using """ + xpos + """:2 smooth bezier with lines   lc 1 lw 3 notitle, \\
                $data_node_b using """ + xpos + """:2               with lines   lc 2 lw 1.5 title 'Avg. RTT node B', \\
                $data_node_b using """ + xpos + """:2 smooth bezier with lines   lc 2 lw 3 notitle

            set format y "%h"
            unset label
            """
        return {
            'gpi': gpi,
            'skip_sample_line': True,
        }
    return plot

def plot_comparison_rtt(y_logarithmic=False, keys=True, subtract_base_rtt=False):
    """
    Plot graph of interrupts and context switches
    """

    def plot(tree, plotdef, leaf_hook):
        gap = collectionutil.get_gap(tree)

        ylabel_extra = ''
        if subtract_base_rtt:
            ylabel_extra = '\\n{/Times:Italic=10 Excluding base RTT}'

        gpi = """
            set ylabel "RTT reported at sender""" + ylabel_extra + """\\n{/Times:Italic=10 [ms] (p_1, mean, p_{99})}"
            """ + add_scale(y_logarithmic)

        # add hidden line to force autoscaling if using logarithimic plot without any points
        plot_gpi = " 1 lc rgb '#FFFF0000' notitle, \\\n"

        def leaf(subtree, is_first_set, x):
            nonlocal gpi, plot_gpi
            leaf_hook(subtree, is_first_set, x)
            add_title = keys and is_first_set

            def parse_rtt(node, testcase_folder):
                rtts = _parse_rtt_of_test(testcase_folder, node, subtract_base_rtt)

                # take last half, so it is stable
                rtts = rtts[int(len(rtts)/2):]

                # get second element of list which is the rtt value
                rtts = [x[1] for x in rtts]

                return _generate_stats(rtts)

            gpi += """
                $data_rtt_a""" + str(x) + """ << EOD
                """ + collectionutil.merge_testcase_data(subtree, partial(parse_rtt, 'A'), plotdef.x_axis) + """
                EOD
                $data_rtt_b""" + str(x) + """ << EOD
                """ + collectionutil.merge_testcase_data(subtree, partial(parse_rtt, 'B'), plotdef.x_axis) + """
                EOD
                """

            xpos = '($1+%s)' % str(x)
            xtic = "" if plotdef.custom_xtics else ":xtic(2)"

            plot_gpi += ("""\\
                %s   using """ + xpos + """:3:10:6%s  with yerrorbars ls 3 pt 7 ps 0.4 lw 1.5 lc 1 %s, \\
                ''   using """ + xpos + """:3         with lines      lc rgb 'gray' notitle, \\
                %s   using """ + xpos + """:3:10:6    with yerrorbars ls 3 pt 7 ps 0.4 lw 1.5 lc 2 %s, \\
                ''   using """ + xpos + """:3         with lines      lc rgb 'gray' notitle, \\
                """) % (
                    "$data_rtt_a" + str(x),
                    xtic,
                    "title 'Server A' " if add_title else 'notitle',
                    "$data_rtt_b" + str(x),
                    "title 'Server B' " if add_title else 'notitle'
                )

        treeutil.walk_leaf(tree, leaf)
        gpi += """
            plot \\
            """ + add_plot(plot_gpi) + """

            unset logscale y
            """

        return {
            'y_logarithmic': y_logarithmic,
            'gpi': gpi,
            'key_rows': 1 if keys else 0,
        }

    return plot
<|endoftext|>