<commit_before>import os
import mimetypes
from datetime import timedelta
from functools import reduce

from django import forms
from django.db import IntegrityError
from django.core.exceptions import ObjectDoesNotExist
from django.http import Http404, HttpResponseBadRequest, HttpResponseServerError, FileResponse, JsonResponse
from django.shortcuts import render
from django.utils.http import urlquote
from django.views import View
from django.contrib.auth import get_user_model

from website.views import PlusMemberCheck
from website.models import Session, Category
from .models import ProblemList, ProblemInstance, ProblemAttachment, ProblemAuthLog, ProblemQuestion
from .helpers.score import AuthReplay
from .helpers.problem_info import get_problem_list_user_info, get_user_problem_info, get_problem_list_total_score, get_problem_list_user_score

User = get_user_model()

class ProblemListForm(forms.Form):
    q = forms.CharField(required=False)
    search_by = forms.ChoiceField(required=False, choices=[
        ('list_title', 'list title'),
        ('session', 'session')
    ])

class ProblemListView(PlusMemberCheck, View):
    def get(self, request):
        form = ProblemListForm(request.GET)
        if not form.is_valid():
            return HttpResponseBadRequest()

        all_sessions = Session.objects.order_by('title')
        all_problem_lists = ProblemList.objects.order_by('title')
        sessions = all_sessions.filter(isActive=True)
        categories = Category.objects.order_by('title')
        problem_lists = all_problem_lists
        q = ''
        search_by = 'list_title'
        if form.cleaned_data['q']:
            if form.cleaned_data['search_by']:
                search_by = form.cleaned_data['search_by']
            q = form.cleaned_data['q']

            if search_by == "list_title":
                problem_lists = problem_lists.filter(title__search=q)
            elif search_by == "session":
                sessions = sessions.filter(title__search=q)
                problem_lists = problem_lists.filter(session__in=sessions)
        else:
            problem_lists = problem_lists.filter(session__in=sessions)

        def construct_response():
            for problem_list in problem_lists:
                problem_info, user_score = get_problem_list_user_info(problem_list, request.user)
                announcement_message = problem_list.announcement
                yield problem_list, problem_info, announcement_message, user_score

        queried_problem_lists = list(construct_response())

        return render(request, 'problem/list.html', {
            'sessions': all_sessions,
            'categories': categories,
            'problem_lists': all_problem_lists,
            'queried_problem_lists': queried_problem_lists,
            'search_by': search_by,
            'q': q
        })

class ProblemGetView(PlusMemberCheck, View):
    def get(self, request, pk):
        try:
            problem_instance = ProblemInstance.objects.get(pk=int(pk))
        except ObjectDoesNotExist:
            raise Http404 from ObjectDoesNotExist

        return render(request, 'problem/get.html', {
            'info': get_user_problem_info(request.user, problem_instance)
        })

class ProblemAuthForm(forms.Form):
    auth_key = forms.CharField(required=False)

class ProblemAuthView(PlusMemberCheck, View):
    def post(self, request, pk):
        form = ProblemAuthForm(request.POST)
        if not form.is_valid():
            return HttpResponseBadRequest()

        return_obj = {}
        prob_id = int(pk)
        auth_key = form.cleaned_data['auth_key']
        try:
            problem_instance = ProblemInstance.objects.get(pk=prob_id)
        except ObjectDoesNotExist:
            raise Http404 from ObjectDoesNotExist

        try:
            ProblemAuthLog.objects.create(
                user=request.user, problem_instance=problem_instance, auth_key=auth_key)
            is_correct = problem_instance.problem.auth_key == auth_key
            return_obj['result'] = is_correct

        except IntegrityError:
            return_obj['result'] = False

        return JsonResponse(return_obj)

class ProblemDownloadView(PlusMemberCheck, View):
    def get(self, request, pk):
        try:
            file_obj = ProblemAttachment.objects.get(pk=int(pk)).file
        except ObjectDoesNotExist:
            raise Http404 from ObjectDoesNotExist

        file_name = os.path.basename(file_obj.path)
        file_size = file_obj.size

        response = FileResponse(file_obj)
        content_type, encoding = mimetypes.guess_type(file_name)
        if content_type is None:
            content_type = 'application/octet-stream'
        if encoding is not None:
            response['Content-Encoding'] = encoding
        response['Content-Type'] = content_type
        response['Content-Disposition'] = 'attachment; filename*=UTF-8\'\'%s' % urlquote(file_name)
        response['Content-Length'] = str(file_size)
        return response

class ProblemRankView(PlusMemberCheck, View):
    def get(self, request, pk=None):
        problem_lists = ProblemList.objects
        if pk is None:
            sessions = Session.objects.filter(isActive=True).order_by('title')
            problem_lists = problem_lists.filter(session__in=sessions)
        else:
            problem_list_pk = int(pk)
            problem_lists = problem_lists.filter(pk=problem_list_pk)
            if not problem_lists.exists():
                raise Http404

        rank_info = []
        chart_info = []
        for problem_list in problem_lists:
            replay = AuthReplay(problem_list, timedelta(days=7))
            replay.prepare()
            top10_chart_data, top10_rank = replay.get_statistic_data()
            rank_info.append((problem_list, top10_rank))
            chart_info.append((problem_list, top10_chart_data))

        return render(request, 'problem/rank.html', {
            'rank_info': rank_info,
            'chart_info': chart_info
        })

class ProblemQuestionView(PlusMemberCheck, View):
    def get(self, request):
        questions = request.user.problemquestion_set.order_by('-datetime')
        if self.request.user.is_staff:
            answers = ProblemQuestion.objects.all().order_by('-datetime')
        else:
            answers = {}

        return render(request, 'problem/question.html', {
            'queried_questions': questions,
            'queried_answers': answers
        })

class ProblemQuestionAskForm(forms.Form):
    question = forms.CharField(required=False)

class ProblemQuestionAskView(PlusMemberCheck, View):
    def post(self, request, pk):
        form = ProblemQuestionAskForm(request.POST)
        if not form.is_valid():
            return HttpResponseBadRequest()

        question_text = form.cleaned_data['question']

        try:
            problem_instance = ProblemInstance.objects.get(pk=int(pk))
        except ObjectDoesNotExist:
            raise Http404 from ObjectDoesNotExist

        if not problem_instance.problem_list.allow_question:
            return HttpResponseBadRequest()

        question_response = {
            "name": problem_instance.problem.title,
            "list": problem_instance.problem_list.title,
            "question": question_text
        }

        if not question_text:
            question_response['ok'] = False
            return JsonResponse(question_response)

        else:
            question_response['ok'] = True

            try:
                ProblemQuestion.objects.create(
                    user=request.user, problem_instance=problem_instance, question=question_text)

            except BaseException:
                return HttpResponseServerError()

        return JsonResponse(question_response)

class ProblemUserView(PlusMemberCheck, View):
    def get(self, request):
        problem_lists = ProblemList.objects.filter(session__isActive=True)
        problem_lists_with_total = []
        for problem_list in problem_lists:
            total = get_problem_list_total_score(problem_list, fixed=True)
            problem_lists_with_total.append({"problem_list": problem_list, "total": total})

        solved_log_queries = []
        for problem_instance in ProblemInstance.objects.all():
            correct_auth_key = problem_instance.problem.auth_key
            solve_logs = ProblemAuthLog.objects.filter(problem_instance=problem_instance, auth_key=correct_auth_key)
            solved_log_queries.append(solve_logs)

        solve_logs = \
            reduce(lambda x, y: x | y, solved_log_queries, ProblemAuthLog.objects.none()) \
            .order_by('datetime')
        user_pks_with_logs = solve_logs.values_list('user', flat=True)
        users_with_logs = User.objects.filter(pk__in=user_pks_with_logs)

        user_data = []
        for user in users_with_logs:
            scores = []
            for problem_list_with_total in problem_lists_with_total:
                problem_list = problem_list_with_total["problem_list"]
                score = get_problem_list_user_score(problem_list, user, fixed=True)
                scores.append({"score": score, "total": problem_list_with_total["total"]})

            user_data.append({"user": user, "scores": scores})

        return render(request, 'problem/user.html', {
            "user_data": user_data,
            "problem_lists": problem_lists
        })
<commit_msg>R1705: Unnecessary "else" after "return"<commit_after>import os
import mimetypes
from datetime import timedelta
from functools import reduce

from django import forms
from django.db import IntegrityError
from django.core.exceptions import ObjectDoesNotExist
from django.http import Http404, HttpResponseBadRequest, HttpResponseServerError, FileResponse, JsonResponse
from django.shortcuts import render
from django.utils.http import urlquote
from django.views import View
from django.contrib.auth import get_user_model

from website.views import PlusMemberCheck
from website.models import Session, Category
from .models import ProblemList, ProblemInstance, ProblemAttachment, ProblemAuthLog, ProblemQuestion
from .helpers.score import AuthReplay
from .helpers.problem_info import get_problem_list_user_info, get_user_problem_info, get_problem_list_total_score, get_problem_list_user_score

User = get_user_model()

class ProblemListForm(forms.Form):
    q = forms.CharField(required=False)
    search_by = forms.ChoiceField(required=False, choices=[
        ('list_title', 'list title'),
        ('session', 'session')
    ])

class ProblemListView(PlusMemberCheck, View):
    def get(self, request):
        form = ProblemListForm(request.GET)
        if not form.is_valid():
            return HttpResponseBadRequest()

        all_sessions = Session.objects.order_by('title')
        all_problem_lists = ProblemList.objects.order_by('title')
        sessions = all_sessions.filter(isActive=True)
        categories = Category.objects.order_by('title')
        problem_lists = all_problem_lists
        q = ''
        search_by = 'list_title'
        if form.cleaned_data['q']:
            if form.cleaned_data['search_by']:
                search_by = form.cleaned_data['search_by']
            q = form.cleaned_data['q']

            if search_by == "list_title":
                problem_lists = problem_lists.filter(title__search=q)
            elif search_by == "session":
                sessions = sessions.filter(title__search=q)
                problem_lists = problem_lists.filter(session__in=sessions)
        else:
            problem_lists = problem_lists.filter(session__in=sessions)

        def construct_response():
            for problem_list in problem_lists:
                problem_info, user_score = get_problem_list_user_info(problem_list, request.user)
                announcement_message = problem_list.announcement
                yield problem_list, problem_info, announcement_message, user_score

        queried_problem_lists = list(construct_response())

        return render(request, 'problem/list.html', {
            'sessions': all_sessions,
            'categories': categories,
            'problem_lists': all_problem_lists,
            'queried_problem_lists': queried_problem_lists,
            'search_by': search_by,
            'q': q
        })

class ProblemGetView(PlusMemberCheck, View):
    def get(self, request, pk):
        try:
            problem_instance = ProblemInstance.objects.get(pk=int(pk))
        except ObjectDoesNotExist:
            raise Http404 from ObjectDoesNotExist

        return render(request, 'problem/get.html', {
            'info': get_user_problem_info(request.user, problem_instance)
        })

class ProblemAuthForm(forms.Form):
    auth_key = forms.CharField(required=False)

class ProblemAuthView(PlusMemberCheck, View):
    def post(self, request, pk):
        form = ProblemAuthForm(request.POST)
        if not form.is_valid():
            return HttpResponseBadRequest()

        return_obj = {}
        prob_id = int(pk)
        auth_key = form.cleaned_data['auth_key']
        try:
            problem_instance = ProblemInstance.objects.get(pk=prob_id)
        except ObjectDoesNotExist:
            raise Http404 from ObjectDoesNotExist

        try:
            ProblemAuthLog.objects.create(
                user=request.user, problem_instance=problem_instance, auth_key=auth_key)
            is_correct = problem_instance.problem.auth_key == auth_key
            return_obj['result'] = is_correct

        except IntegrityError:
            return_obj['result'] = False

        return JsonResponse(return_obj)

class ProblemDownloadView(PlusMemberCheck, View):
    def get(self, request, pk):
        try:
            file_obj = ProblemAttachment.objects.get(pk=int(pk)).file
        except ObjectDoesNotExist:
            raise Http404 from ObjectDoesNotExist

        file_name = os.path.basename(file_obj.path)
        file_size = file_obj.size

        response = FileResponse(file_obj)
        content_type, encoding = mimetypes.guess_type(file_name)
        if content_type is None:
            content_type = 'application/octet-stream'
        if encoding is not None:
            response['Content-Encoding'] = encoding
        response['Content-Type'] = content_type
        response['Content-Disposition'] = 'attachment; filename*=UTF-8\'\'%s' % urlquote(file_name)
        response['Content-Length'] = str(file_size)
        return response

class ProblemRankView(PlusMemberCheck, View):
    def get(self, request, pk=None):
        problem_lists = ProblemList.objects
        if pk is None:
            sessions = Session.objects.filter(isActive=True).order_by('title')
            problem_lists = problem_lists.filter(session__in=sessions)
        else:
            problem_list_pk = int(pk)
            problem_lists = problem_lists.filter(pk=problem_list_pk)
            if not problem_lists.exists():
                raise Http404

        rank_info = []
        chart_info = []
        for problem_list in problem_lists:
            replay = AuthReplay(problem_list, timedelta(days=7))
            replay.prepare()
            top10_chart_data, top10_rank = replay.get_statistic_data()
            rank_info.append((problem_list, top10_rank))
            chart_info.append((problem_list, top10_chart_data))

        return render(request, 'problem/rank.html', {
            'rank_info': rank_info,
            'chart_info': chart_info
        })

class ProblemQuestionView(PlusMemberCheck, View):
    def get(self, request):
        questions = request.user.problemquestion_set.order_by('-datetime')
        if self.request.user.is_staff:
            answers = ProblemQuestion.objects.all().order_by('-datetime')
        else:
            answers = {}

        return render(request, 'problem/question.html', {
            'queried_questions': questions,
            'queried_answers': answers
        })

class ProblemQuestionAskForm(forms.Form):
    question = forms.CharField(required=False)

class ProblemQuestionAskView(PlusMemberCheck, View):
    def post(self, request, pk):
        form = ProblemQuestionAskForm(request.POST)
        if not form.is_valid():
            return HttpResponseBadRequest()

        question_text = form.cleaned_data['question']

        try:
            problem_instance = ProblemInstance.objects.get(pk=int(pk))
        except ObjectDoesNotExist:
            raise Http404 from ObjectDoesNotExist

        if not problem_instance.problem_list.allow_question:
            return HttpResponseBadRequest()

        question_response = {
            "name": problem_instance.problem.title,
            "list": problem_instance.problem_list.title,
            "question": question_text
        }

        if not question_text:
            question_response['ok'] = False
            return JsonResponse(question_response)

        question_response['ok'] = True

        try:
            ProblemQuestion.objects.create(
                user=request.user, problem_instance=problem_instance, question=question_text)

        except BaseException:
            return HttpResponseServerError()

        return JsonResponse(question_response)

class ProblemUserView(PlusMemberCheck, View):
    def get(self, request):
        problem_lists = ProblemList.objects.filter(session__isActive=True)
        problem_lists_with_total = []
        for problem_list in problem_lists:
            total = get_problem_list_total_score(problem_list, fixed=True)
            problem_lists_with_total.append({"problem_list": problem_list, "total": total})

        solved_log_queries = []
        for problem_instance in ProblemInstance.objects.all():
            correct_auth_key = problem_instance.problem.auth_key
            solve_logs = ProblemAuthLog.objects.filter(problem_instance=problem_instance, auth_key=correct_auth_key)
            solved_log_queries.append(solve_logs)

        solve_logs = \
            reduce(lambda x, y: x | y, solved_log_queries, ProblemAuthLog.objects.none()) \
            .order_by('datetime')
        user_pks_with_logs = solve_logs.values_list('user', flat=True)
        users_with_logs = User.objects.filter(pk__in=user_pks_with_logs)

        user_data = []
        for user in users_with_logs:
            scores = []
            for problem_list_with_total in problem_lists_with_total:
                problem_list = problem_list_with_total["problem_list"]
                score = get_problem_list_user_score(problem_list, user, fixed=True)
                scores.append({"score": score, "total": problem_list_with_total["total"]})

            user_data.append({"user": user, "scores": scores})

        return render(request, 'problem/user.html', {
            "user_data": user_data,
            "problem_lists": problem_lists
        })
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import warnings
import theano
import pylearn2
import numpy as np

from sklearn.datasets import fetch_mldata
from sklearn.metrics import accuracy_score, classification_report

from data import Data
from pdb import set_trace as debug

from pylearn2.space import Conv2DSpace
from pylearn2 import termination_criteria
from pylearn2.models import mlp
from pylearn2.datasets.dense_design_matrix import DenseDesignMatrix
from pylearn2.training_algorithms import bgd

warnings.filterwarnings("ignore")

def convert_one_hot(data):
    return np.array([[1 if y == c else 0 for c in xrange(10)] for y in data])

def convert_categorical(data):
    return np.argmax(data, axis=1)

def train(d):
    print 'Creating dataset'
    # load mnist here
    # X = d.train_X
    # y = d.train_Y
    # test_X = d.test_X
    # test_Y = d.test_Y
    # nb_classes = len(np.unique(y))
    # train_y = convert_one_hot(y)
    # train_set = DenseDesignMatrix(X=X, y=y)
    train = DenseDesignMatrix(X=d.train_X, y=convert_one_hot(d.train_Y))
    valid = DenseDesignMatrix(X=d.valid_X, y=convert_one_hot(d.valid_Y))
    test = DenseDesignMatrix(X=d.test_X, y=convert_one_hot(d.test_Y))
    # train = mnist.MNIST(
    #     which_set='train',
    #     start=0,
    #     stop=50000,
    # )
    # valid = mnist.MNIST(
    #     which_set='train',
    #     start=50000,
    #     stop=60000,
    # )
    # test = mnist.MNIST(which_set='test')

    print 'Setting up'
    batch_size = 1000
    conv = mlp.ConvRectifiedLinear(
        layer_name='c0',
        output_channels=20,
        irange=.05,
        kernel_shape=[5, 5],
        pool_shape=[4, 4],
        pool_stride=[2, 2],
        # W_lr_scale=0.25,
        max_kernel_norm=1.9365
    )
    sigmoid = mlp.Sigmoid(
        layer_name='Sigmoid',
        dim=500,
        sparse_init=15,
    )
    smax = mlp.Softmax(
        layer_name='y',
        n_classes=10,
        irange=0.
    )
    in_space = Conv2DSpace(
        shape=[28, 28],
        num_channels=1,
    )
    net = mlp.MLP(
        layers=[conv, smax],
        input_space=in_space,
        # nvis=784,
    )
    trainer = bgd.BGD(
        batch_size=batch_size,
        line_search_mode='exhaustive',
        conjugate=1,
        updates_per_batch=10,
        monitoring_dataset={
            'train': train,
            'valid': valid,
            'test': test
        },
        termination_criterion=termination_criteria.MonitorBased(channel_name='valid_y_misclass')
    )
    trainer.setup(net, train)
    epoch = 0
    while True:
        print 'Training...', epoch
        trainer.train(dataset=train)
        net.monitor()
        epoch += 1

"""
    TODO: Get above .98 when classifying mnist from sklearn.
    Because it is possible to get that score on the PyLearn mnist, with this code.
"""

if __name__ == '__main__':
    mnist = fetch_mldata('MNIST original')
    debug()
    mnist.data = (mnist.data.astype(float) / 255)
    d = Data(dataset=mnist, train_perc=0.65, valid_perc=0.2, test_perc=0.15,
             shuffle=False)
    train(d=d)
    # train()
<commit_msg>Worked with Dropout and SGD. Error message with Maxout.<commit_after># -*- coding: utf-8 -*-

import warnings
import theano
import pylearn2
import numpy as np

from sklearn.datasets import fetch_mldata
from sklearn.metrics import accuracy_score, classification_report

from data import Data
from pdb import set_trace as debug

from pylearn2.space import Conv2DSpace
from pylearn2 import termination_criteria
from pylearn2.models import mlp
from pylearn2.models.maxout import MaxoutConvC01B
from pylearn2.datasets.dense_design_matrix import DenseDesignMatrix
from pylearn2.training_algorithms import bgd, sgd
from pylearn2.costs.mlp import dropout

warnings.filterwarnings("ignore")

def convert_one_hot(data):
    return np.array([[1 if y == c else 0 for c in xrange(10)] for y in data])

def convert_categorical(data):
    return np.argmax(data, axis=1)

def train(d):
    print 'Creating dataset'
    # load mnist here
    # X = d.train_X
    # y = d.train_Y
    # test_X = d.test_X
    # test_Y = d.test_Y
    # nb_classes = len(np.unique(y))
    # train_y = convert_one_hot(y)
    # train_set = DenseDesignMatrix(X=X, y=y)
    train = DenseDesignMatrix(X=d.train_X, y=convert_one_hot(d.train_Y))
    valid = DenseDesignMatrix(X=d.valid_X, y=convert_one_hot(d.valid_Y))
    test = DenseDesignMatrix(X=d.test_X, y=convert_one_hot(d.test_Y))
    # train = mnist.MNIST(
    #     which_set='train',
    #     start=0,
    #     stop=50000,
    # )
    # valid = mnist.MNIST(
    #     which_set='train',
    #     start=50000,
    #     stop=60000,
    # )
    # test = mnist.MNIST(which_set='test')

    print 'Setting up'
    batch_size = 1000
    conv = mlp.ConvRectifiedLinear(
        layer_name='c0',
        output_channels=20,
        irange=.05,
        kernel_shape=[5, 5],
        pool_shape=[4, 4],
        pool_stride=[2, 2],
        # W_lr_scale=0.25,
        max_kernel_norm=1.9365
    )
    mout = MaxoutConvC01B(
	layer_name='m0',
	num_pieces=4,
	num_channels=20,
	irange=.05,
	kernel_shape=[5, 5],
	pool_shape=[4, 4],
	pool_stride=[2, 2],
	max_kernel_norm=1.9365
    )
    sigmoid = mlp.Sigmoid(
        layer_name='Sigmoid',
        dim=500,
        sparse_init=15,
    )
    smax = mlp.Softmax(
        layer_name='y',
        n_classes=10,
        irange=0.
    )
    in_space = Conv2DSpace(
        shape=[28, 28],
        num_channels=1,
#	axes=['c', 0, 1, 'b']
    )
    net = mlp.MLP(
        layers=[conv, smax],
        input_space=in_space,
        # nvis=784,
    )
    trainer = bgd.BGD(
        batch_size=batch_size,
        line_search_mode='exhaustive',
        conjugate=1,
        updates_per_batch=10,
        monitoring_dataset={
            'train': train,
            'valid': valid,
            'test': test
        },
        termination_criterion=termination_criteria.MonitorBased(channel_name='valid_y_misclass')
    )
    trainer = sgd.SGD(
	learning_rate=0.1,
	cost=dropout.Dropout(),
	batch_size=batch_size,
	monitoring_dataset={
	    'train': train,
	    'valid': valid,
	    'test': test
	},
        termination_criterion=termination_criteria.MonitorBased(channel_name='valid_y_misclass')
    )
    trainer.setup(net, train)
    epoch = 0
    while True:
        print 'Training...', epoch
        trainer.train(dataset=train)
        net.monitor()
        epoch += 1

"""
    TODO: Get above .98 with one maxout conv layer. And then add several ones.
"""

if __name__ == '__main__':
    mnist = fetch_mldata('MNIST original')
    # debug()
    mnist.data = (mnist.data.astype(float) / 255)
    d = Data(dataset=mnist, train_perc=0.65, valid_perc=0.2, test_perc=0.15,
             shuffle=True)
    train(d=d)
    # train()
<|endoftext|>


<commit_before>from collections import OrderedDict
from django.db import models
from django.core.mail import EmailMultiAlternatives
from django.conf import settings
import django.template
from django.template.loader import render_to_string
from re import sub

from squad.core.models import Project, ProjectStatus, Build, KnownIssue, NotificationDelivery
from squad.core.comparison import TestComparison

jinja2 = django.template.engines['jinja2']

class Notification(object):
    """
    Represents a notification about a project status change, that may or may
    not need to be sent.
    """

    def __init__(self, status, previous=None):
        self.status = status
        self.build = status.build
        if previous is None:
            previous = status.get_previous()
        self.previous_build = previous and previous.build or None

    __comparison__ = None

    @property
    def comparison(self):
        if self.__comparison__ is None:
            self.__comparison__ = TestComparison.compare_builds(
                self.previous_build,
                self.build,
            )
        return self.__comparison__

    @property
    def diff(self):
        return self.comparison.diff

    @property
    def project(self):
        return self.build.project

    @property
    def metadata(self):
        if self.build.metadata is not None:
            return OrderedDict(sorted(self.build.metadata.items()))
        else:
            return {}

    @property
    def important_metadata(self):
        return self.build.important_metadata

    @property
    def summary(self):
        return self.build.test_summary

    @property
    def recipients(self):
        emails = []
        for subscription in self.project.subscriptions.all():
            email = subscription.get_email()
            if email:
                emails.append(email)
        return emails

    @property
    def known_issues(self):
        return KnownIssue.active_by_project_and_test(self.project)

    @property
    def subject(self):
        summary = self.summary
        subject_data = {
            'build': self.build.version,
            'important_metadata': self.important_metadata,
            'metadata': self.metadata,
            'project': self.project,
            'regressions': len(self.comparison.regressions),
            'tests_fail': summary.tests_fail,
            'tests_pass': summary.tests_pass,
            'tests_total': summary.tests_total,
        }
        custom_email_template = self.project.custom_email_template
        if custom_email_template and custom_email_template.subject:
            template = custom_email_template.subject
        else:
            template = '{{project}}: {{tests_total}} tests, {{tests_fail}} failed, {{tests_pass}} passed (build {{build}})'
        return jinja2.from_string(template).render(subject_data)

    def message(self, do_html=True, custom_email_template=None):
        """
        Returns a tuple with (text_message,html_message)
        """
        context = {
            'build': self.build,
            'important_metadata': self.important_metadata,
            'metadata': self.metadata,
            'notification': self,
            'previous_build': self.previous_build,
            'regressions_grouped_by_suite': self.comparison.regressions_grouped_by_suite,
            'fixes_grouped_by_suite': self.comparison.fixes_grouped_by_suite,
            'known_issues': self.known_issues,
            'regressions': self.comparison.regressions,
            'fixes': self.comparison.fixes,
            'settings': settings,
            'summary': self.summary,
        }

        html_message = ''
        if custom_email_template:
            text_template = jinja2.from_string(custom_email_template.plain_text)
            text_message = text_template.render(context)

            if do_html:
                html_template = jinja2.from_string(custom_email_template.html)
                html_message = html_template.render(context)
        else:
            text_message = render_to_string(
                'squad/notification/diff.txt',
                context=context,
            )
            if do_html:
                html_message = render_to_string(
                    'squad/notification/diff.html',
                    context=context,
                )

        return (text_message, html_message)

    def send(self):
        recipients = self.recipients
        if not recipients:
            return

        sender = "%s <%s>" % (settings.SITE_NAME, settings.EMAIL_FROM)
        subject = self.subject
        txt, html = self.message(self.project.html_mail, self.project.custom_email_template)

        if NotificationDelivery.exists(self.status, subject, txt, html):
            return

        message = EmailMultiAlternatives(subject, txt, sender, recipients)
        if self.project.html_mail:
            message.attach_alternative(html, "text/html")
        message.send()

        self.mark_as_notified()

    def mark_as_notified(self):
        self.status.notified = True
        self.status.save()

class PreviewNotification(Notification):

    @property
    def recipients(self):
        return [r.email for r in self.project.admin_subscriptions.all()]

    @property
    def subject(self):
        return '[PREVIEW] %s' % super(PreviewNotification, self).subject

    def message(self, do_html=True, custom_email_template=None):
        txt, html = super(PreviewNotification, self).message(do_html, custom_email_template)
        txt_banner = render_to_string(
            'squad/notification/moderation.txt',
            {
                "settings": settings,
                "status": self.status,
            }
        )
        html_banner = render_to_string(
            'squad/notification/moderation.html',
            {
                "settings": settings,
                "status": self.status,
            }
        )
        txt = txt_banner + txt
        html = sub("<body>", "<body>\n" + html_banner, html)
        return (txt, html)

def send_status_notification(status, project=None):
    project = project or status.build.project
    send_admin_notification(status, project)

    if project.moderate_notifications and not status.approved:
        notification = PreviewNotification(status)
    else:
        notification = Notification(status)

    if project.notification_strategy == Project.NOTIFY_ON_CHANGE:
        if not notification.diff or not notification.previous_build:
            return

    notification.send()

def send_admin_notification(status, project):
    build = status.build
    failed_test_jobs = build.test_jobs.filter(
        fetched=True,
        failure__isnull=False,
    )

    if not failed_test_jobs:
        return

    data = {
        'project': project,
        'build': build,
        'build_version': build.version,
        'project': project,
        'test_jobs': failed_test_jobs,
        'count': len(failed_test_jobs),
        'settings': settings,
    }
    subject = '%(build_version)s: FAILED TEST JOBS (%(count)d) -- %(project)s' % data
    sender = "%s <%s>" % (settings.SITE_NAME, settings.EMAIL_FROM)
    recipients = [r.email for r in project.admin_subscriptions.all()]
    txt = render_to_string('squad/notification/failed_test_jobs.txt', data)

    message = EmailMultiAlternatives(subject, txt, sender, recipients)
    if project.html_mail:
        html = render_to_string('squad/notification/failed_test_jobs.html', data)
        message.attach_alternative(html, "text/html")

    message.send()
<commit_msg>core/notification: avoid duplicated "X FAILED TEST JOBS" notifications<commit_after>from collections import OrderedDict
from django.db import models
from django.core.mail import EmailMultiAlternatives
from django.conf import settings
import django.template
from django.template.loader import render_to_string
from re import sub

from squad.core.models import Project, ProjectStatus, Build, KnownIssue, NotificationDelivery
from squad.core.comparison import TestComparison

jinja2 = django.template.engines['jinja2']

class Notification(object):
    """
    Represents a notification about a project status change, that may or may
    not need to be sent.
    """

    def __init__(self, status, previous=None):
        self.status = status
        self.build = status.build
        if previous is None:
            previous = status.get_previous()
        self.previous_build = previous and previous.build or None

    __comparison__ = None

    @property
    def comparison(self):
        if self.__comparison__ is None:
            self.__comparison__ = TestComparison.compare_builds(
                self.previous_build,
                self.build,
            )
        return self.__comparison__

    @property
    def diff(self):
        return self.comparison.diff

    @property
    def project(self):
        return self.build.project

    @property
    def metadata(self):
        if self.build.metadata is not None:
            return OrderedDict(sorted(self.build.metadata.items()))
        else:
            return {}

    @property
    def important_metadata(self):
        return self.build.important_metadata

    @property
    def summary(self):
        return self.build.test_summary

    @property
    def recipients(self):
        emails = []
        for subscription in self.project.subscriptions.all():
            email = subscription.get_email()
            if email:
                emails.append(email)
        return emails

    @property
    def known_issues(self):
        return KnownIssue.active_by_project_and_test(self.project)

    @property
    def subject(self):
        summary = self.summary
        subject_data = {
            'build': self.build.version,
            'important_metadata': self.important_metadata,
            'metadata': self.metadata,
            'project': self.project,
            'regressions': len(self.comparison.regressions),
            'tests_fail': summary.tests_fail,
            'tests_pass': summary.tests_pass,
            'tests_total': summary.tests_total,
        }
        custom_email_template = self.project.custom_email_template
        if custom_email_template and custom_email_template.subject:
            template = custom_email_template.subject
        else:
            template = '{{project}}: {{tests_total}} tests, {{tests_fail}} failed, {{tests_pass}} passed (build {{build}})'
        return jinja2.from_string(template).render(subject_data)

    def message(self, do_html=True, custom_email_template=None):
        """
        Returns a tuple with (text_message,html_message)
        """
        context = {
            'build': self.build,
            'important_metadata': self.important_metadata,
            'metadata': self.metadata,
            'notification': self,
            'previous_build': self.previous_build,
            'regressions_grouped_by_suite': self.comparison.regressions_grouped_by_suite,
            'fixes_grouped_by_suite': self.comparison.fixes_grouped_by_suite,
            'known_issues': self.known_issues,
            'regressions': self.comparison.regressions,
            'fixes': self.comparison.fixes,
            'settings': settings,
            'summary': self.summary,
        }

        html_message = ''
        if custom_email_template:
            text_template = jinja2.from_string(custom_email_template.plain_text)
            text_message = text_template.render(context)

            if do_html:
                html_template = jinja2.from_string(custom_email_template.html)
                html_message = html_template.render(context)
        else:
            text_message = render_to_string(
                'squad/notification/diff.txt',
                context=context,
            )
            if do_html:
                html_message = render_to_string(
                    'squad/notification/diff.html',
                    context=context,
                )

        return (text_message, html_message)

    def send(self):
        recipients = self.recipients
        if not recipients:
            return

        sender = "%s <%s>" % (settings.SITE_NAME, settings.EMAIL_FROM)
        subject = self.subject
        txt, html = self.message(self.project.html_mail, self.project.custom_email_template)

        if NotificationDelivery.exists(self.status, subject, txt, html):
            return

        message = EmailMultiAlternatives(subject, txt, sender, recipients)
        if self.project.html_mail:
            message.attach_alternative(html, "text/html")
        message.send()

        self.mark_as_notified()

    def mark_as_notified(self):
        self.status.notified = True
        self.status.save()

class PreviewNotification(Notification):

    @property
    def recipients(self):
        return [r.email for r in self.project.admin_subscriptions.all()]

    @property
    def subject(self):
        return '[PREVIEW] %s' % super(PreviewNotification, self).subject

    def message(self, do_html=True, custom_email_template=None):
        txt, html = super(PreviewNotification, self).message(do_html, custom_email_template)
        txt_banner = render_to_string(
            'squad/notification/moderation.txt',
            {
                "settings": settings,
                "status": self.status,
            }
        )
        html_banner = render_to_string(
            'squad/notification/moderation.html',
            {
                "settings": settings,
                "status": self.status,
            }
        )
        txt = txt_banner + txt
        html = sub("<body>", "<body>\n" + html_banner, html)
        return (txt, html)

def send_status_notification(status, project=None):
    project = project or status.build.project
    send_admin_notification(status, project)

    if project.moderate_notifications and not status.approved:
        notification = PreviewNotification(status)
    else:
        notification = Notification(status)

    if project.notification_strategy == Project.NOTIFY_ON_CHANGE:
        if not notification.diff or not notification.previous_build:
            return

    notification.send()

def send_admin_notification(status, project):
    build = status.build
    failed_test_jobs = build.test_jobs.filter(
        fetched=True,
        failure__isnull=False,
    )

    if not failed_test_jobs:
        return

    data = {
        'project': project,
        'build': build,
        'build_version': build.version,
        'project': project,
        'test_jobs': failed_test_jobs,
        'count': len(failed_test_jobs),
        'settings': settings,
    }
    subject = '%(build_version)s: FAILED TEST JOBS (%(count)d) -- %(project)s' % data
    sender = "%s <%s>" % (settings.SITE_NAME, settings.EMAIL_FROM)
    recipients = [r.email for r in project.admin_subscriptions.all()]
    txt = render_to_string('squad/notification/failed_test_jobs.txt', data)

    message = EmailMultiAlternatives(subject, txt, sender, recipients)
    html = ''
    if project.html_mail:
        html = render_to_string('squad/notification/failed_test_jobs.html', data)
        message.attach_alternative(html, "text/html")

    if NotificationDelivery.exists(status, subject, txt, html):
        return

    message.send()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from collections import OrderedDict

import pytest
import yaml

from apispec import APISpec, BasePlugin
from apispec.exceptions import APISpecError

from .utils import get_definitions, get_paths

description = 'This is a sample Petstore server.  You can find out more '
'about Swagger at http://swagger.wordnik.com '
'or on irc.freenode.net, #swagger.  For this sample, you can use the api '
'key \"special-key\" to test the authorization filters'

@pytest.fixture(params=('2.0', '3.0.0'))
def spec(request):
    openapi_version = request.param
    if openapi_version == '2.0':
        security_kwargs = {
            'security': [{'apiKey': []}],
        }
    else:
        security_kwargs = {
            'components': {
                'securitySchemes': {
                    'bearerAuth':
                        dict(type='http', scheme='bearer', bearerFormat='JWT'),
                },
                'schemas': {
                    'ErrorResponse': {
                        'type': 'object',
                        'properties': {
                            'ok': {
                                'type': 'boolean', 'description': 'status indicator', 'example': False,
                            },
                        },
                        'required': ['ok'],
                    },
                },
            },
        }
    return APISpec(
        title='Swagger Petstore',
        version='1.0.0',
        openapi_version=openapi_version,
        info={'description': description},
        **security_kwargs
    )

class TestAPISpecInit:

    def test_raises_wrong_apispec_version(self):
        with pytest.raises(APISpecError) as excinfo:
            APISpec(
                'Swagger Petstore',
                version='1.0.0',
                openapi_version='4.0',  # 4.0 is not supported
                info={'description': description},
                security=[{'apiKey': []}],
            )
        assert 'Not a valid OpenAPI version number:' in str(excinfo)

class TestMetadata:

    def test_openapi_metadata(self, spec):
        metadata = spec.to_dict()
        assert metadata['info']['title'] == 'Swagger Petstore'
        assert metadata['info']['version'] == '1.0.0'
        assert metadata['info']['description'] == description
        if spec.openapi_version.major < 3:
            assert metadata['swagger'] == spec.openapi_version.vstring
            assert metadata['security'] == [{'apiKey': []}]
        else:
            assert metadata['openapi'] == spec.openapi_version.vstring
            security_schemes = {'bearerAuth': dict(type='http', scheme='bearer', bearerFormat='JWT')}
            assert metadata['components']['securitySchemes'] == security_schemes
            assert metadata['components']['schemas'].get('ErrorResponse', False)
            assert metadata['info']['title'] == 'Swagger Petstore'
            assert metadata['info']['version'] == '1.0.0'
            assert metadata['info']['description'] == description

    @pytest.mark.parametrize('spec', ('3.0.0', ), indirect=True)
    def test_openapi_metadata_merge_v3(self, spec):
        properties = {
            'ok': {
                'type': 'boolean', 'description': 'property description', 'example': True,
            },
        }
        spec.definition('definition', properties=properties, description='definiton description')
        metadata = spec.to_dict()
        assert metadata['components']['schemas'].get('ErrorResponse', False)
        assert metadata['components']['schemas'].get('definition', False)

class TestTags:

    tag = {
        'name': 'MyTag',
        'description': 'This tag gathers all API endpoints which are mine.',
    }

    def test_tag(self, spec):
        spec.add_tag(self.tag)
        tags_json = spec.to_dict()['tags']
        assert self.tag in tags_json

class TestDefinitions:

    properties = {
        'id': {'type': 'integer', 'format': 'int64'},
        'name': {'type': 'string', 'example': 'doggie'},
    }

    def test_definition(self, spec):
        spec.definition('Pet', properties=self.properties)
        if spec.openapi_version.major < 3:
            defs = spec.to_dict()['definitions']
        else:
            defs = spec.to_dict()['components']['schemas']
        assert 'Pet' in defs
        assert defs['Pet']['properties'] == self.properties

    def test_definition_description(self, spec):
        model_description = 'An animal which lives with humans.'
        spec.definition('Pet', properties=self.properties, description=model_description)
        if spec.openapi_version.major < 3:
            defs = spec.to_dict()['definitions']
        else:
            defs = spec.to_dict()['components']['schemas']
        assert defs['Pet']['description'] == model_description

    def test_definition_stores_enum(self, spec):
        enum = ['name', 'photoUrls']
        spec.definition(
            'Pet',
            properties=self.properties,
            enum=enum,
        )
        if spec.openapi_version.major < 3:
            defs = spec.to_dict()['definitions']
        else:
            defs = spec.to_dict()['components']['schemas']
        assert defs['Pet']['enum'] == enum

    def test_definition_extra_fields(self, spec):
        extra_fields = {'discriminator': 'name'}
        spec.definition('Pet', properties=self.properties, extra_fields=extra_fields)
        if spec.openapi_version.major < 3:
            defs = spec.to_dict()['definitions']
        else:
            defs = spec.to_dict()['components']['schemas']
        assert defs['Pet']['discriminator'] == 'name'

    def test_to_yaml(self, spec):
        enum = ['name', 'photoUrls']
        spec.definition(
            'Pet',
            properties=self.properties,
            enum=enum,
        )
        assert spec.to_dict() == yaml.safe_load(spec.to_yaml())

class TestPath:
    paths = {
        '/pet/{petId}': {
            'get': {
                'parameters': [
                    {
                        'required': True,
                        'format': 'int64',
                        'name': 'petId',
                        'in': 'path',
                        'type': 'integer',
                        'description': 'ID of pet that needs to be fetched',
                    },
                ],
                'responses': {
                    '200': {
                        'schema': {'$ref': '#/definitions/Pet'},
                        'description': 'successful operation',
                    },
                    '400': {
                        'description': 'Invalid ID supplied',
                    },
                    '404': {
                        'description': 'Pet not found',
                    },
                },
                'produces': [
                    'application/json',
                    'application/xml',
                ],
                'operationId': 'getPetById',
                'summary': 'Find pet by ID',
                'description': (
                    'Returns a pet when ID < 10.  '
                    'ID > 10 or nonintegers will simulate API error conditions'
                ),
                'tags': ['pet'],
            },
        },
    }

    def test_add_path(self, spec):
        route_spec = self.paths['/pet/{petId}']['get']
        spec.add_path(
            path='/pet/{petId}',
            operations=dict(
                get=dict(
                    parameters=route_spec['parameters'],
                    responses=route_spec['responses'],
                    produces=route_spec['produces'],
                    operationId=route_spec['operationId'],
                    summary=route_spec['summary'],
                    description=route_spec['description'],
                    tags=route_spec['tags'],
                ),
            ),
        )

        p = get_paths(spec)['/pet/{petId}']['get']
        assert p['parameters'] == route_spec['parameters']
        assert p['responses'] == route_spec['responses']
        assert p['operationId'] == route_spec['operationId']
        assert p['summary'] == route_spec['summary']
        assert p['description'] == route_spec['description']
        assert p['tags'] == route_spec['tags']

    def test_paths_maintain_order(self, spec):
        spec.add_path(path='/path1')
        spec.add_path(path='/path2')
        spec.add_path(path='/path3')
        spec.add_path(path='/path4')
        assert list(spec.to_dict()['paths'].keys()) == ['/path1', '/path2', '/path3', '/path4']

    def test_methods_maintain_order(self, spec):
        methods = ['get', 'post', 'put', 'patch', 'delete', 'head', 'options']
        for method in methods:
            spec.add_path(path='/path', operations=OrderedDict({method: {}}))
        assert list(spec.to_dict()['paths']['/path']) == methods

    def test_add_path_merges_paths(self, spec):
        """Test that adding a second HTTP method to an existing path performs
        a merge operation instead of an overwrite"""
        path = '/pet/{petId}'
        route_spec = self.paths[path]['get']
        spec.add_path(
            path=path,
            operations=dict(
                get=route_spec,
            ),
        )
        spec.add_path(
            path=path,
            operations=dict(
                put=dict(
                    parameters=route_spec['parameters'],
                    responses=route_spec['responses'],
                    produces=route_spec['produces'],
                    operationId='updatePet',
                    summary='Updates an existing Pet',
                    description='Use this method to make changes to Pet `petId`',
                    tags=route_spec['tags'],
                ),
            ),
        )

        p = get_paths(spec)[path]
        assert 'get' in p
        assert 'put' in p

    def test_add_path_ensures_path_parameters_required(self, spec):
        path = '/pet/{petId}'
        spec.add_path(
            path=path,
            operations=dict(
                put=dict(
                    parameters=[{
                        'name': 'petId',
                        'in': 'path',
                    }],
                ),
            ),
        )
        assert get_paths(spec)[path]['put']['parameters'][0]['required'] is True

    def test_add_path_with_no_path_raises_error(self, spec):
        with pytest.raises(APISpecError) as excinfo:
            spec.add_path()
        assert 'Path template is not specified' in str(excinfo)

    def test_add_path_strips_base_path(self, spec):
        spec.options['basePath'] = '/v1'
        spec.add_path('/v1/pets')
        paths = get_paths(spec)
        assert '/pets' in paths
        assert '/v1/pets' not in paths

    def test_add_path_strips_path_base_path(self, spec):
        spec.options['basePath'] = '/v1'
        path = '/v1/pets'
        spec.add_path(path)
        paths = get_paths(spec)
        assert '/pets' in paths
        assert '/v1/pets' not in paths

    def test_add_parameters(self, spec):
        route_spec = self.paths['/pet/{petId}']['get']

        spec.add_parameter('test_parameter', 'path', **route_spec['parameters'][0])

        spec.add_path(
            path='/pet/{petId}',
            operations=dict(
                get=dict(
                    parameters=['test_parameter'],
                ),
            ),
        )

        metadata = spec.to_dict()
        p = get_paths(spec)['/pet/{petId}']['get']

        if spec.openapi_version.major < 3:
            assert p['parameters'][0] == {'$ref': '#/parameters/test_parameter'}
            assert route_spec['parameters'][0] == metadata['parameters']['test_parameter']
        else:
            assert p['parameters'][0] == {'$ref': '#/components/parameters/test_parameter'}
            assert route_spec['parameters'][0] == metadata['components']['parameters']['test_parameter']

    def test_add_path_check_invalid_http_method(self, spec):
        spec.add_path('/pet/{petId}', operations={'get': {}})
        spec.add_path('/pet/{petId}', operations={'x-dummy': {}})
        with pytest.raises(APISpecError) as excinfo:
            spec.add_path('/pet/{petId}', operations={'dummy': {}})
        assert 'One or more HTTP methods are invalid' in str(excinfo)

class TestPlugins:

    class TestPlugin(BasePlugin):
        def definition_helper(self, name, definition, **kwargs):
            return {'properties': {'name': {'type': 'string'}}}

        def path_helper(self, path, operations, **kwargs):
            if path == '/path_1':
                operations.update({'get': {'responses': {'200': {}}}})
                return '/path_1_modified'

        def operation_helper(self, path, operations, **kwargs):
            if path == '/path_2':
                operations['post'] = {'responses': {'201': {}}}

        def response_helper(self, method, status_code, **kwargs):
            if method == 'delete':
                return {'description': 'Clever description'}

    @pytest.mark.parametrize('openapi_version', ('2.0', '3.0.0', ))
    def test_plugin_definition_helper_is_used(self, openapi_version):
        spec = APISpec(
            title='Swagger Petstore',
            version='1.0.0',
            openapi_version=openapi_version,
            plugins=(self.TestPlugin(), ),
        )
        spec.definition('Pet', {})
        definitions = get_definitions(spec)
        assert definitions['Pet'] == {'properties': {'name': {'type': 'string'}}}

    @pytest.mark.parametrize('openapi_version', ('2.0', '3.0.0', ))
    def test_plugin_path_helper_is_used(self, openapi_version):
        spec = APISpec(
            title='Swagger Petstore',
            version='1.0.0',
            openapi_version=openapi_version,
            plugins=(self.TestPlugin(), ),
        )
        spec.add_path('/path_1')
        paths = get_paths(spec)
        assert len(paths) == 1
        assert paths['/path_1_modified'] == {'get': {'responses': {'200': {}}}}

    @pytest.mark.parametrize('openapi_version', ('2.0', '3.0.0', ))
    def test_plugin_operation_helper_is_used(self, openapi_version):
        spec = APISpec(
            title='Swagger Petstore',
            version='1.0.0',
            openapi_version=openapi_version,
            plugins=(self.TestPlugin(), ),
        )
        spec.add_path('/path_2', operations={'post': {'responses': {'200': {}}}})
        paths = get_paths(spec)
        assert len(paths) == 1
        assert paths['/path_2'] == {'post': {'responses': {'201': {}}}}

    @pytest.mark.parametrize('openapi_version', ('2.0', '3.0.0', ))
    def test_plugin_response_helper_is_used(self, spec, openapi_version):
        spec = APISpec(
            title='Swagger Petstore',
            version='1.0.0',
            openapi_version=openapi_version,
            plugins=(self.TestPlugin(), ),
        )
        spec.add_path('/path_3', operations={'delete': {'responses': {'204': {'content': {}}}}})
        paths = get_paths(spec)
        assert len(paths) == 1
        assert paths['/path_3'] == {'delete': {'responses': {'204': {
            'content': {}, 'description': 'Clever description',
        }}}}

class TestPluginsOrder:

    class OrderedPlugin(BasePlugin):

        def __init__(self, index, output):
            super(TestPluginsOrder.OrderedPlugin, self).__init__()
            self.index = index
            self.output = output

        def path_helper(self, path, operations, **kwargs):
            self.output.append('plugin_{}_path'.format(self.index))

        def operation_helper(self, path, operations, **kwargs):
            self.output.append('plugin_{}_operations'.format(self.index))

        def response_helper(self, method, status_code, **kwargs):
            self.output.append('plugin_{}_response'.format(self.index))

    def test_plugins_order(self):
        """Test plugins execution order in APISpec.add_path

        - All path helpers are called, then all operation helpers, then all response helpers.
        - At each step, helpers are executed in the order the plugins are passed to APISpec.
        """
        output = []
        spec = APISpec(
            title='Swagger Petstore',
            version='1.0.0',
            openapi_version='3.0.0',
            plugins=(self.OrderedPlugin(1, output), self.OrderedPlugin(2, output)),
        )
        spec.add_path('/path', operations={'get': {'responses': {200: {}}}})
        assert output == [
            'plugin_1_path', 'plugin_2_path',
            'plugin_1_operations', 'plugin_2_operations',
            'plugin_1_response', 'plugin_2_response',
        ]
<commit_msg>Remove duplicate test<commit_after># -*- coding: utf-8 -*-
from collections import OrderedDict

import pytest
import yaml

from apispec import APISpec, BasePlugin
from apispec.exceptions import APISpecError

from .utils import get_definitions, get_paths

description = 'This is a sample Petstore server.  You can find out more '
'about Swagger at http://swagger.wordnik.com '
'or on irc.freenode.net, #swagger.  For this sample, you can use the api '
'key \"special-key\" to test the authorization filters'

@pytest.fixture(params=('2.0', '3.0.0'))
def spec(request):
    openapi_version = request.param
    if openapi_version == '2.0':
        security_kwargs = {
            'security': [{'apiKey': []}],
        }
    else:
        security_kwargs = {
            'components': {
                'securitySchemes': {
                    'bearerAuth':
                        dict(type='http', scheme='bearer', bearerFormat='JWT'),
                },
                'schemas': {
                    'ErrorResponse': {
                        'type': 'object',
                        'properties': {
                            'ok': {
                                'type': 'boolean', 'description': 'status indicator', 'example': False,
                            },
                        },
                        'required': ['ok'],
                    },
                },
            },
        }
    return APISpec(
        title='Swagger Petstore',
        version='1.0.0',
        openapi_version=openapi_version,
        info={'description': description},
        **security_kwargs
    )

class TestAPISpecInit:

    def test_raises_wrong_apispec_version(self):
        with pytest.raises(APISpecError) as excinfo:
            APISpec(
                'Swagger Petstore',
                version='1.0.0',
                openapi_version='4.0',  # 4.0 is not supported
                info={'description': description},
                security=[{'apiKey': []}],
            )
        assert 'Not a valid OpenAPI version number:' in str(excinfo)

class TestMetadata:

    def test_openapi_metadata(self, spec):
        metadata = spec.to_dict()
        assert metadata['info']['title'] == 'Swagger Petstore'
        assert metadata['info']['version'] == '1.0.0'
        assert metadata['info']['description'] == description
        if spec.openapi_version.major < 3:
            assert metadata['swagger'] == spec.openapi_version.vstring
            assert metadata['security'] == [{'apiKey': []}]
        else:
            assert metadata['openapi'] == spec.openapi_version.vstring
            security_schemes = {'bearerAuth': dict(type='http', scheme='bearer', bearerFormat='JWT')}
            assert metadata['components']['securitySchemes'] == security_schemes
            assert metadata['components']['schemas'].get('ErrorResponse', False)
            assert metadata['info']['title'] == 'Swagger Petstore'
            assert metadata['info']['version'] == '1.0.0'
            assert metadata['info']['description'] == description

    @pytest.mark.parametrize('spec', ('3.0.0', ), indirect=True)
    def test_openapi_metadata_merge_v3(self, spec):
        properties = {
            'ok': {
                'type': 'boolean', 'description': 'property description', 'example': True,
            },
        }
        spec.definition('definition', properties=properties, description='definiton description')
        metadata = spec.to_dict()
        assert metadata['components']['schemas'].get('ErrorResponse', False)
        assert metadata['components']['schemas'].get('definition', False)

class TestTags:

    tag = {
        'name': 'MyTag',
        'description': 'This tag gathers all API endpoints which are mine.',
    }

    def test_tag(self, spec):
        spec.add_tag(self.tag)
        tags_json = spec.to_dict()['tags']
        assert self.tag in tags_json

class TestDefinitions:

    properties = {
        'id': {'type': 'integer', 'format': 'int64'},
        'name': {'type': 'string', 'example': 'doggie'},
    }

    def test_definition(self, spec):
        spec.definition('Pet', properties=self.properties)
        if spec.openapi_version.major < 3:
            defs = spec.to_dict()['definitions']
        else:
            defs = spec.to_dict()['components']['schemas']
        assert 'Pet' in defs
        assert defs['Pet']['properties'] == self.properties

    def test_definition_description(self, spec):
        model_description = 'An animal which lives with humans.'
        spec.definition('Pet', properties=self.properties, description=model_description)
        if spec.openapi_version.major < 3:
            defs = spec.to_dict()['definitions']
        else:
            defs = spec.to_dict()['components']['schemas']
        assert defs['Pet']['description'] == model_description

    def test_definition_stores_enum(self, spec):
        enum = ['name', 'photoUrls']
        spec.definition(
            'Pet',
            properties=self.properties,
            enum=enum,
        )
        if spec.openapi_version.major < 3:
            defs = spec.to_dict()['definitions']
        else:
            defs = spec.to_dict()['components']['schemas']
        assert defs['Pet']['enum'] == enum

    def test_definition_extra_fields(self, spec):
        extra_fields = {'discriminator': 'name'}
        spec.definition('Pet', properties=self.properties, extra_fields=extra_fields)
        if spec.openapi_version.major < 3:
            defs = spec.to_dict()['definitions']
        else:
            defs = spec.to_dict()['components']['schemas']
        assert defs['Pet']['discriminator'] == 'name'

    def test_to_yaml(self, spec):
        enum = ['name', 'photoUrls']
        spec.definition(
            'Pet',
            properties=self.properties,
            enum=enum,
        )
        assert spec.to_dict() == yaml.safe_load(spec.to_yaml())

class TestPath:
    paths = {
        '/pet/{petId}': {
            'get': {
                'parameters': [
                    {
                        'required': True,
                        'format': 'int64',
                        'name': 'petId',
                        'in': 'path',
                        'type': 'integer',
                        'description': 'ID of pet that needs to be fetched',
                    },
                ],
                'responses': {
                    '200': {
                        'schema': {'$ref': '#/definitions/Pet'},
                        'description': 'successful operation',
                    },
                    '400': {
                        'description': 'Invalid ID supplied',
                    },
                    '404': {
                        'description': 'Pet not found',
                    },
                },
                'produces': [
                    'application/json',
                    'application/xml',
                ],
                'operationId': 'getPetById',
                'summary': 'Find pet by ID',
                'description': (
                    'Returns a pet when ID < 10.  '
                    'ID > 10 or nonintegers will simulate API error conditions'
                ),
                'tags': ['pet'],
            },
        },
    }

    def test_add_path(self, spec):
        route_spec = self.paths['/pet/{petId}']['get']
        spec.add_path(
            path='/pet/{petId}',
            operations=dict(
                get=dict(
                    parameters=route_spec['parameters'],
                    responses=route_spec['responses'],
                    produces=route_spec['produces'],
                    operationId=route_spec['operationId'],
                    summary=route_spec['summary'],
                    description=route_spec['description'],
                    tags=route_spec['tags'],
                ),
            ),
        )

        p = get_paths(spec)['/pet/{petId}']['get']
        assert p['parameters'] == route_spec['parameters']
        assert p['responses'] == route_spec['responses']
        assert p['operationId'] == route_spec['operationId']
        assert p['summary'] == route_spec['summary']
        assert p['description'] == route_spec['description']
        assert p['tags'] == route_spec['tags']

    def test_paths_maintain_order(self, spec):
        spec.add_path(path='/path1')
        spec.add_path(path='/path2')
        spec.add_path(path='/path3')
        spec.add_path(path='/path4')
        assert list(spec.to_dict()['paths'].keys()) == ['/path1', '/path2', '/path3', '/path4']

    def test_methods_maintain_order(self, spec):
        methods = ['get', 'post', 'put', 'patch', 'delete', 'head', 'options']
        for method in methods:
            spec.add_path(path='/path', operations=OrderedDict({method: {}}))
        assert list(spec.to_dict()['paths']['/path']) == methods

    def test_add_path_merges_paths(self, spec):
        """Test that adding a second HTTP method to an existing path performs
        a merge operation instead of an overwrite"""
        path = '/pet/{petId}'
        route_spec = self.paths[path]['get']
        spec.add_path(
            path=path,
            operations=dict(
                get=route_spec,
            ),
        )
        spec.add_path(
            path=path,
            operations=dict(
                put=dict(
                    parameters=route_spec['parameters'],
                    responses=route_spec['responses'],
                    produces=route_spec['produces'],
                    operationId='updatePet',
                    summary='Updates an existing Pet',
                    description='Use this method to make changes to Pet `petId`',
                    tags=route_spec['tags'],
                ),
            ),
        )

        p = get_paths(spec)[path]
        assert 'get' in p
        assert 'put' in p

    def test_add_path_ensures_path_parameters_required(self, spec):
        path = '/pet/{petId}'
        spec.add_path(
            path=path,
            operations=dict(
                put=dict(
                    parameters=[{
                        'name': 'petId',
                        'in': 'path',
                    }],
                ),
            ),
        )
        assert get_paths(spec)[path]['put']['parameters'][0]['required'] is True

    def test_add_path_with_no_path_raises_error(self, spec):
        with pytest.raises(APISpecError) as excinfo:
            spec.add_path()
        assert 'Path template is not specified' in str(excinfo)

    def test_add_path_strips_base_path(self, spec):
        spec.options['basePath'] = '/v1'
        spec.add_path('/v1/pets')
        paths = get_paths(spec)
        assert '/pets' in paths
        assert '/v1/pets' not in paths

    def test_add_parameters(self, spec):
        route_spec = self.paths['/pet/{petId}']['get']

        spec.add_parameter('test_parameter', 'path', **route_spec['parameters'][0])

        spec.add_path(
            path='/pet/{petId}',
            operations=dict(
                get=dict(
                    parameters=['test_parameter'],
                ),
            ),
        )

        metadata = spec.to_dict()
        p = get_paths(spec)['/pet/{petId}']['get']

        if spec.openapi_version.major < 3:
            assert p['parameters'][0] == {'$ref': '#/parameters/test_parameter'}
            assert route_spec['parameters'][0] == metadata['parameters']['test_parameter']
        else:
            assert p['parameters'][0] == {'$ref': '#/components/parameters/test_parameter'}
            assert route_spec['parameters'][0] == metadata['components']['parameters']['test_parameter']

    def test_add_path_check_invalid_http_method(self, spec):
        spec.add_path('/pet/{petId}', operations={'get': {}})
        spec.add_path('/pet/{petId}', operations={'x-dummy': {}})
        with pytest.raises(APISpecError) as excinfo:
            spec.add_path('/pet/{petId}', operations={'dummy': {}})
        assert 'One or more HTTP methods are invalid' in str(excinfo)

class TestPlugins:

    class TestPlugin(BasePlugin):
        def definition_helper(self, name, definition, **kwargs):
            return {'properties': {'name': {'type': 'string'}}}

        def path_helper(self, path, operations, **kwargs):
            if path == '/path_1':
                operations.update({'get': {'responses': {'200': {}}}})
                return '/path_1_modified'

        def operation_helper(self, path, operations, **kwargs):
            if path == '/path_2':
                operations['post'] = {'responses': {'201': {}}}

        def response_helper(self, method, status_code, **kwargs):
            if method == 'delete':
                return {'description': 'Clever description'}

    @pytest.mark.parametrize('openapi_version', ('2.0', '3.0.0', ))
    def test_plugin_definition_helper_is_used(self, openapi_version):
        spec = APISpec(
            title='Swagger Petstore',
            version='1.0.0',
            openapi_version=openapi_version,
            plugins=(self.TestPlugin(), ),
        )
        spec.definition('Pet', {})
        definitions = get_definitions(spec)
        assert definitions['Pet'] == {'properties': {'name': {'type': 'string'}}}

    @pytest.mark.parametrize('openapi_version', ('2.0', '3.0.0', ))
    def test_plugin_path_helper_is_used(self, openapi_version):
        spec = APISpec(
            title='Swagger Petstore',
            version='1.0.0',
            openapi_version=openapi_version,
            plugins=(self.TestPlugin(), ),
        )
        spec.add_path('/path_1')
        paths = get_paths(spec)
        assert len(paths) == 1
        assert paths['/path_1_modified'] == {'get': {'responses': {'200': {}}}}

    @pytest.mark.parametrize('openapi_version', ('2.0', '3.0.0', ))
    def test_plugin_operation_helper_is_used(self, openapi_version):
        spec = APISpec(
            title='Swagger Petstore',
            version='1.0.0',
            openapi_version=openapi_version,
            plugins=(self.TestPlugin(), ),
        )
        spec.add_path('/path_2', operations={'post': {'responses': {'200': {}}}})
        paths = get_paths(spec)
        assert len(paths) == 1
        assert paths['/path_2'] == {'post': {'responses': {'201': {}}}}

    @pytest.mark.parametrize('openapi_version', ('2.0', '3.0.0', ))
    def test_plugin_response_helper_is_used(self, spec, openapi_version):
        spec = APISpec(
            title='Swagger Petstore',
            version='1.0.0',
            openapi_version=openapi_version,
            plugins=(self.TestPlugin(), ),
        )
        spec.add_path('/path_3', operations={'delete': {'responses': {'204': {'content': {}}}}})
        paths = get_paths(spec)
        assert len(paths) == 1
        assert paths['/path_3'] == {'delete': {'responses': {'204': {
            'content': {}, 'description': 'Clever description',
        }}}}

class TestPluginsOrder:

    class OrderedPlugin(BasePlugin):

        def __init__(self, index, output):
            super(TestPluginsOrder.OrderedPlugin, self).__init__()
            self.index = index
            self.output = output

        def path_helper(self, path, operations, **kwargs):
            self.output.append('plugin_{}_path'.format(self.index))

        def operation_helper(self, path, operations, **kwargs):
            self.output.append('plugin_{}_operations'.format(self.index))

        def response_helper(self, method, status_code, **kwargs):
            self.output.append('plugin_{}_response'.format(self.index))

    def test_plugins_order(self):
        """Test plugins execution order in APISpec.add_path

        - All path helpers are called, then all operation helpers, then all response helpers.
        - At each step, helpers are executed in the order the plugins are passed to APISpec.
        """
        output = []
        spec = APISpec(
            title='Swagger Petstore',
            version='1.0.0',
            openapi_version='3.0.0',
            plugins=(self.OrderedPlugin(1, output), self.OrderedPlugin(2, output)),
        )
        spec.add_path('/path', operations={'get': {'responses': {200: {}}}})
        assert output == [
            'plugin_1_path', 'plugin_2_path',
            'plugin_1_operations', 'plugin_2_operations',
            'plugin_1_response', 'plugin_2_response',
        ]
<|endoftext|>


<commit_before>from contextlib import contextmanager
from csv import DictWriter
import logging

from ktbs_bench.utils.decorators import bench as util_bench

class BenchManager:
    def __init__(self):
        self._contexts = []
        self._bench_funcs = []
        self._results = {}

    def bench(self, func):
        """Prepare a function to be benched and add it to the list to be run later."""
        func = util_bench(func)
        self._bench_funcs.append(func)

    def context(self, func):
        """Decorate a function to act as a context."""
        func = contextmanager(func)
        self._contexts.append(func)

    def run(self, output_filename, show_log=False):
        """Execute each collected function against each context."""
        # Run the bench for each function against each context
        if show_log:
            logging.getLogger().setLevel(logging.INFO)
        for func in self._bench_funcs:
            self._results[func.__name__] = {}
            if show_log:
                logging.info('Func: %s' % func.__name__)
            for context in self._contexts:
                if show_log:
                    logging.info('with context: %s' % context.__name__)
                with context() as arg:
                    if not isinstance(arg, tuple):
                        arg = tuple([arg])  # Make arg a tuple of one element
                    _, res_time = func(*arg)
                if show_log:
                    logging.info('res time: %s' % res_time)
                self._results[func.__name__][context.__name__] = res_time

        # Write output to a CSV file
        fieldnames = ['func_name']
        fieldnames += [context.__name__ for context in self._contexts]
        with open(output_filename, 'wb') as output_file:
            csv_file = DictWriter(output_file, fieldnames)
            csv_file.writeheader()

            for func_name, env_results in self._results.items():
                row_results = env_results
                row_results['func_name'] = func_name
                csv_file.writerow(row_results)
<commit_msg>Add docstings to bench_manager<commit_after>from contextlib import contextmanager
from csv import DictWriter
import logging

from ktbs_bench.utils.decorators import bench as util_bench

class BenchManager(object):
    """
    Manage benchmarks by timing function against different contexts.

    General concept
    ===============
    A BenchManager is instantiated in order to collect functions
    to benchmark, like so:
    >>> my_bench_manager = BenchManager()

    In order to add functions to bench, one flag them for bench
    by using the :meth:`bench` decorator. For example:
    >>> @my_bench_manager.bench
    ... def add_one(n):
    ...     return n + 1

    Each flagged function is then called against contexts.
    A context is a function with optional setup and teardown, and it
    must *yield* the parameter that benchmarked functions need.
    >>> @my_bench_manager.context
    ... def three():
    ...     # optional setup
    ...     try:
    ...        # yield the parameter
    ...        yield 3
    ...     finally:
    ...        # optional teardown
    ...        pass

    Finally, to perform the benchmarks, one must call:
    >>> my_bench_manager.run('/tmp/my_results.csv')

    The result of the two examples above is to time ``add_one(3)``.

    Technical details
    =================
    Each context is stored in the list :attr:`_contexts`.
    Each function to benchmark is stored in the list :attr:`_bench_funcs`.

    When :meth:`run` is called, it will iterate over functions and contexts
    to call each function against each context.
    """

    def __init__(self):
        self._contexts = []
        self._bench_funcs = []
        self._results = {}

    def bench(self, func):
        """Prepare a function to be benched and add it to the list to be run later.

        :param function func: the function to bench
        """
        func = util_bench(func)
        self._bench_funcs.append(func)

    def context(self, func):
        """Decorate a function to act as a context.

        :param function func: the function that describes the context
        """
        func = contextmanager(func)
        self._contexts.append(func)

    def run(self, output_filename, show_log=False):
        """Execute each collected function against each context.

        :param str output_filename: filename of the CSV output
        :param bool show_log: True to display log during the run, False otherwise
        """
        # Run the bench for each function against each context
        if show_log:
            logging.getLogger().setLevel(logging.INFO)
        for func in self._bench_funcs:
            self._results[func.__name__] = {}
            if show_log:
                logging.info('Func: %s' % func.__name__)
            for context in self._contexts:
                if show_log:
                    logging.info('with context: %s' % context.__name__)
                with context() as arg:
                    if not isinstance(arg, tuple):
                        arg = tuple([arg])  # Make arg a tuple of one element
                    _, res_time = func(*arg)
                if show_log:
                    logging.info('res time: %s' % res_time)
                self._results[func.__name__][context.__name__] = res_time

        # Write output to a CSV file
        fieldnames = ['func_name']
        fieldnames += [context.__name__ for context in self._contexts]
        with open(output_filename, 'wb') as output_file:
            csv_file = DictWriter(output_file, fieldnames)
            csv_file.writeheader()

            for func_name, env_results in self._results.items():
                row_results = env_results
                row_results['func_name'] = func_name
                csv_file.writerow(row_results)
<|endoftext|>