<commit_before>from django.conf import global_settings
from django.contrib.auth import get_user_model
from django.contrib.auth.models import Group, AnonymousUser
from django.core.exceptions import ObjectDoesNotExist, PermissionDenied
from django.db.models.base import ModelBase
from django.http import HttpRequest
from django.http import HttpResponse
from django.http import HttpResponseForbidden
from django.http import HttpResponseNotFound
from django.http import HttpResponseRedirect
from django.shortcuts import get_object_or_404
from django.template import TemplateDoesNotExist
from django.test import TestCase

from guardian.compat import get_user_model_path
from guardian.compat import get_user_permission_full_codename
from unittest import mock
from guardian.decorators import permission_required, permission_required_or_403, permission_required_or_404
from guardian.exceptions import GuardianError
from guardian.exceptions import WrongAppError
from guardian.shortcuts import assign_perm
from guardian.testapp.tests.conf import TestDataMixin
from guardian.testapp.tests.conf import override_settings
from guardian.testapp.tests.conf import skipUnlessTestApp

User = get_user_model()
user_model_path = get_user_model_path()

@skipUnlessTestApp
class PermissionRequiredTest(TestDataMixin, TestCase):

    def setUp(self):
        super().setUp()
        self.anon = AnonymousUser()
        self.user = User.objects.get_or_create(username='jack')[0]
        self.group = Group.objects.get_or_create(name='jackGroup')[0]

    def _get_request(self, user=None):
        if user is None:
            user = AnonymousUser()
        request = HttpRequest()
        request.user = user
        return request

    def test_no_args(self):

        try:
            @permission_required
            def dummy_view(request):
                return HttpResponse('dummy_view')
        except GuardianError:
            pass
        else:
            self.fail("Trying to decorate using permission_required without "
                      "permission as first argument should raise exception")

    def test_RENDER_403_is_false(self):
        request = self._get_request(self.anon)

        @permission_required_or_403('not_installed_app.change_user')
        def dummy_view(request):
            return HttpResponse('dummy_view')

        with mock.patch('guardian.conf.settings.RENDER_403', False):
            response = dummy_view(request)
            self.assertEqual(response.content, b'')
            self.assertTrue(isinstance(response, HttpResponseForbidden))

    def test_RENDER_404_is_false(self):
        request = self._get_request(self.anon)

        @permission_required_or_404('not_installed_app.change_user')
        def dummy_view(request):
            return HttpResponse('dummy_view')

        with mock.patch('guardian.conf.settings.RENDER_404', False):
            response = dummy_view(request)
            self.assertEqual(response.content, b'')
            self.assertTrue(isinstance(response, HttpResponseNotFound))

    @mock.patch('guardian.conf.settings.RENDER_403', True)
    def test_TEMPLATE_403_setting(self):
        request = self._get_request(self.anon)

        @permission_required_or_403('not_installed_app.change_user')
        def dummy_view(request):
            return HttpResponse('dummy_view')

        with mock.patch('guardian.conf.settings.TEMPLATE_403', 'dummy403.html'):
            response = dummy_view(request)
            self.assertEqual(response.content, b'foobar403\n')

    @mock.patch('guardian.conf.settings.RENDER_404', True)
    def test_TEMPLATE_404_setting(self):
        request = self._get_request(self.anon)

        @permission_required_or_404('not_installed_app.change_user')
        def dummy_view(request):
            return HttpResponse('dummy_view')

        with mock.patch('guardian.conf.settings.TEMPLATE_404', 'dummy404.html'):
            response = dummy_view(request)
            self.assertEqual(response.content, b'foobar404\n')

    @mock.patch('guardian.conf.settings.RENDER_403', True)
    def test_403_response_raises_error(self):
        request = self._get_request(self.anon)

        @permission_required_or_403('not_installed_app.change_user')
        def dummy_view(request):
            return HttpResponse('dummy_view')
        with mock.patch('guardian.conf.settings.TEMPLATE_403',
                        '_non-exisitng-403.html'):
            self.assertRaises(TemplateDoesNotExist, dummy_view, request)

    @mock.patch('guardian.conf.settings.RENDER_404', True)
    def test_404_response_raises_error(self):
        request = self._get_request(self.anon)

        @permission_required_or_404('not_installed_app.change_user')
        def dummy_view(request):
            return HttpResponse('dummy_view')
        with mock.patch('guardian.conf.settings.TEMPLATE_404',
                        '_non-exisitng-404.html'):
            self.assertRaises(TemplateDoesNotExist, dummy_view, request)

    @mock.patch('guardian.conf.settings.RENDER_403', False)
    @mock.patch('guardian.conf.settings.RAISE_403', True)
    def test_RAISE_403_setting_is_true(self):
        request = self._get_request(self.anon)

        @permission_required_or_403('not_installed_app.change_user')
        def dummy_view(request):
            return HttpResponse('dummy_view')

        self.assertRaises(PermissionDenied, dummy_view, request)

    @mock.patch('guardian.conf.settings.RENDER_404', False)
    @mock.patch('guardian.conf.settings.RAISE_404', True)
    def test_RAISE_404_setting_is_true(self):
        request = self._get_request(self.anon)

        @permission_required_or_404('not_installed_app.change_user')
        def dummy_view(request):
            return HttpResponse('dummy_view')

        self.assertRaises(ObjectDoesNotExist, dummy_view, request)

    def test_anonymous_user_wrong_app(self):

        request = self._get_request(self.anon)

        @permission_required_or_403('not_installed_app.change_user')
        def dummy_view(request):
            return HttpResponse('dummy_view')
        self.assertEqual(dummy_view(request).status_code, 403)

    def test_anonymous_user_wrong_codename(self):

        request = self._get_request()

        @permission_required_or_403('auth.wrong_codename')
        def dummy_view(request):
            return HttpResponse('dummy_view')
        self.assertEqual(dummy_view(request).status_code, 403)

    def test_anonymous_user(self):

        request = self._get_request()

        @permission_required_or_403('auth.change_user')
        def dummy_view(request):
            return HttpResponse('dummy_view')
        self.assertEqual(dummy_view(request).status_code, 403)

    def test_wrong_lookup_variables_number(self):

        request = self._get_request()

        try:
            @permission_required_or_403('auth.change_user', (User, 'username'))
            def dummy_view(request, username):
                pass
            dummy_view(request, username='jack')
        except GuardianError:
            pass
        else:
            self.fail("If lookup variables are passed they must be tuple of: "
                      "(ModelClass/app_label.ModelClass/queryset, "
                      "<pair of lookup_string and view_arg>)\n"
                      "Otherwise GuardianError should be raised")

    def test_wrong_lookup_variables(self):

        request = self._get_request()

        args = (
            (2010, 'username', 'username'),
            ('User', 'username', 'username'),
            (User, 'username', 'no_arg'),
        )
        for tup in args:
            try:
                @permission_required_or_403('auth.change_user', tup)
                def show_user(request, username):
                    user = get_object_or_404(User, username=username)
                    return HttpResponse("It's %s here!" % user.username)
                show_user(request, 'jack')
            except GuardianError:
                pass
            else:
                self.fail("Wrong arguments given but GuardianError not raised")

    def test_user_has_no_access(self):

        request = self._get_request()

        @permission_required_or_403('auth.change_user')
        def dummy_view(request):
            return HttpResponse('dummy_view')
        self.assertEqual(dummy_view(request).status_code, 403)

    def test_user_has_access(self):

        perm = get_user_permission_full_codename('change')
        joe, created = User.objects.get_or_create(username='joe')
        assign_perm(perm, self.user, obj=joe)

        request = self._get_request(self.user)

        @permission_required_or_403(perm, (
            user_model_path, 'username', 'username'))
        def dummy_view(request, username):
            return HttpResponse('dummy_view')
        response = dummy_view(request, username='joe')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.content, b'dummy_view')

    def test_user_has_access_on_model_with_metaclass(self):
        """
        Test to the fix issues of comparaison made via type()
        in the decorator. In the case of a `Model` implementing
        a custom metaclass, the decorator fail because type
        doesn't return `ModelBase`
        """
        perm = get_user_permission_full_codename('change')

        class TestMeta(ModelBase):
            pass

        class ProxyUser(User):

            class Meta:
                proxy = True
                app_label = User._meta.app_label
            __metaclass__ = TestMeta

        joe, created = ProxyUser.objects.get_or_create(username='joe')
        assign_perm(perm, self.user, obj=joe)

        request = self._get_request(self.user)

        @permission_required_or_403(perm, (
            ProxyUser, 'username', 'username'))
        def dummy_view(request, username):
            return HttpResponse('dummy_view')
        response = dummy_view(request, username='joe')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.content, b'dummy_view')

    def test_user_has_obj_access_even_if_we_also_check_for_global(self):

        perm = get_user_permission_full_codename('change')
        joe, created = User.objects.get_or_create(username='joe')
        assign_perm(perm, self.user, obj=joe)

        request = self._get_request(self.user)

        @permission_required_or_403(perm, (
            user_model_path, 'username', 'username'), accept_global_perms=True)
        def dummy_view(request, username):
            return HttpResponse('dummy_view')
        response = dummy_view(request, username='joe')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.content, b'dummy_view')

    def test_user_has_no_obj_perm_access(self):

        perm = get_user_permission_full_codename('change')
        joe, created = User.objects.get_or_create(username='joe')

        request = self._get_request(self.user)

        @permission_required_or_403(perm, (
            user_model_path, 'username', 'username'))
        def dummy_view(request, username):
            return HttpResponse('dummy_view')
        response = dummy_view(request, username='joe')
        self.assertEqual(response.status_code, 403)

    def test_user_has_global_perm_access_but_flag_not_set(self):

        perm = get_user_permission_full_codename('change')
        joe, created = User.objects.get_or_create(username='joe')
        assign_perm(perm, self.user)

        request = self._get_request(self.user)

        @permission_required_or_403(perm, (
            user_model_path, 'username', 'username'))
        def dummy_view(request, username):
            return HttpResponse('dummy_view')
        response = dummy_view(request, username='joe')
        self.assertEqual(response.status_code, 403)

    def test_user_has_global_perm_access(self):

        perm = get_user_permission_full_codename('change')
        joe, created = User.objects.get_or_create(username='joe')
        assign_perm(perm, self.user)

        request = self._get_request(self.user)

        @permission_required_or_403(perm, (
            user_model_path, 'username', 'username'), accept_global_perms=True)
        def dummy_view(request, username):
            return HttpResponse('dummy_view')
        response = dummy_view(request, username='joe')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.content, b'dummy_view')

    def test_model_lookup(self):

        request = self._get_request(self.user)

        perm = get_user_permission_full_codename('change')
        joe, created = User.objects.get_or_create(username='joe')
        assign_perm(perm, self.user, obj=joe)

        models = (
            user_model_path,
            User,
            User.objects.filter(is_active=True),
        )
        for model in models:
            @permission_required_or_403(perm, (model, 'username', 'username'))
            def dummy_view(request, username):
                get_object_or_404(User, username=username)
                return HttpResponse('hello')
            response = dummy_view(request, username=joe.username)
            self.assertEqual(response.content, b'hello')

    def test_redirection_raises_wrong_app_error(self):
        from guardian.testapp.models import Project
        request = self._get_request(self.user)

        User.objects.create(username='foo')
        Project.objects.create(name='foobar')

        @permission_required('auth.change_group',
                             (Project, 'name', 'group_name'),
                             login_url='/foobar/')
        def dummy_view(request, project_name):
            pass
        # 'auth.change_group' is wrong permission codename (should be one
        # related with User
        self.assertRaises(WrongAppError, dummy_view,
                          request, group_name='foobar')

    def test_redirection(self):
        from guardian.testapp.models import Project

        request = self._get_request(self.user)

        User.objects.create(username='foo')
        Project.objects.create(name='foobar')

        @permission_required('testapp.change_project',
                             (Project, 'name', 'project_name'),
                             login_url='/foobar/')
        def dummy_view(request, project_name):
            pass
        response = dummy_view(request, project_name='foobar')
        self.assertTrue(isinstance(response, HttpResponseRedirect))
        self.assertTrue(response._headers['location'][1].startswith(
            '/foobar/'))

    @override_settings(LOGIN_URL='django.contrib.auth.views.login')
    def test_redirection_class(self):
        view_url = '/permission_required/'

        response = self.client.get(view_url)
        # this should be '/account/login'
        self.assertRedirects(
            response, global_settings.LOGIN_URL + "?next=" + view_url)
<commit_msg>Fixed headers test for Django 3.2 The response.headers interface is added in Django 3.2 https://github.com/django/django/commit/dcb69043d0de45bb55998fc418d93c28bc7689ae<commit_after>import django
from django.conf import global_settings
from django.contrib.auth import get_user_model
from django.contrib.auth.models import Group, AnonymousUser
from django.core.exceptions import ObjectDoesNotExist, PermissionDenied
from django.db.models.base import ModelBase
from django.http import HttpRequest
from django.http import HttpResponse
from django.http import HttpResponseForbidden
from django.http import HttpResponseNotFound
from django.http import HttpResponseRedirect
from django.shortcuts import get_object_or_404
from django.template import TemplateDoesNotExist
from django.test import TestCase

from guardian.compat import get_user_model_path
from guardian.compat import get_user_permission_full_codename
from unittest import mock
from guardian.decorators import permission_required, permission_required_or_403, permission_required_or_404
from guardian.exceptions import GuardianError
from guardian.exceptions import WrongAppError
from guardian.shortcuts import assign_perm
from guardian.testapp.tests.conf import TestDataMixin
from guardian.testapp.tests.conf import override_settings
from guardian.testapp.tests.conf import skipUnlessTestApp

User = get_user_model()
user_model_path = get_user_model_path()

@skipUnlessTestApp
class PermissionRequiredTest(TestDataMixin, TestCase):

    def setUp(self):
        super().setUp()
        self.anon = AnonymousUser()
        self.user = User.objects.get_or_create(username='jack')[0]
        self.group = Group.objects.get_or_create(name='jackGroup')[0]

    def _get_request(self, user=None):
        if user is None:
            user = AnonymousUser()
        request = HttpRequest()
        request.user = user
        return request

    def test_no_args(self):

        try:
            @permission_required
            def dummy_view(request):
                return HttpResponse('dummy_view')
        except GuardianError:
            pass
        else:
            self.fail("Trying to decorate using permission_required without "
                      "permission as first argument should raise exception")

    def test_RENDER_403_is_false(self):
        request = self._get_request(self.anon)

        @permission_required_or_403('not_installed_app.change_user')
        def dummy_view(request):
            return HttpResponse('dummy_view')

        with mock.patch('guardian.conf.settings.RENDER_403', False):
            response = dummy_view(request)
            self.assertEqual(response.content, b'')
            self.assertTrue(isinstance(response, HttpResponseForbidden))

    def test_RENDER_404_is_false(self):
        request = self._get_request(self.anon)

        @permission_required_or_404('not_installed_app.change_user')
        def dummy_view(request):
            return HttpResponse('dummy_view')

        with mock.patch('guardian.conf.settings.RENDER_404', False):
            response = dummy_view(request)
            self.assertEqual(response.content, b'')
            self.assertTrue(isinstance(response, HttpResponseNotFound))

    @mock.patch('guardian.conf.settings.RENDER_403', True)
    def test_TEMPLATE_403_setting(self):
        request = self._get_request(self.anon)

        @permission_required_or_403('not_installed_app.change_user')
        def dummy_view(request):
            return HttpResponse('dummy_view')

        with mock.patch('guardian.conf.settings.TEMPLATE_403', 'dummy403.html'):
            response = dummy_view(request)
            self.assertEqual(response.content, b'foobar403\n')

    @mock.patch('guardian.conf.settings.RENDER_404', True)
    def test_TEMPLATE_404_setting(self):
        request = self._get_request(self.anon)

        @permission_required_or_404('not_installed_app.change_user')
        def dummy_view(request):
            return HttpResponse('dummy_view')

        with mock.patch('guardian.conf.settings.TEMPLATE_404', 'dummy404.html'):
            response = dummy_view(request)
            self.assertEqual(response.content, b'foobar404\n')

    @mock.patch('guardian.conf.settings.RENDER_403', True)
    def test_403_response_raises_error(self):
        request = self._get_request(self.anon)

        @permission_required_or_403('not_installed_app.change_user')
        def dummy_view(request):
            return HttpResponse('dummy_view')
        with mock.patch('guardian.conf.settings.TEMPLATE_403',
                        '_non-exisitng-403.html'):
            self.assertRaises(TemplateDoesNotExist, dummy_view, request)

    @mock.patch('guardian.conf.settings.RENDER_404', True)
    def test_404_response_raises_error(self):
        request = self._get_request(self.anon)

        @permission_required_or_404('not_installed_app.change_user')
        def dummy_view(request):
            return HttpResponse('dummy_view')
        with mock.patch('guardian.conf.settings.TEMPLATE_404',
                        '_non-exisitng-404.html'):
            self.assertRaises(TemplateDoesNotExist, dummy_view, request)

    @mock.patch('guardian.conf.settings.RENDER_403', False)
    @mock.patch('guardian.conf.settings.RAISE_403', True)
    def test_RAISE_403_setting_is_true(self):
        request = self._get_request(self.anon)

        @permission_required_or_403('not_installed_app.change_user')
        def dummy_view(request):
            return HttpResponse('dummy_view')

        self.assertRaises(PermissionDenied, dummy_view, request)

    @mock.patch('guardian.conf.settings.RENDER_404', False)
    @mock.patch('guardian.conf.settings.RAISE_404', True)
    def test_RAISE_404_setting_is_true(self):
        request = self._get_request(self.anon)

        @permission_required_or_404('not_installed_app.change_user')
        def dummy_view(request):
            return HttpResponse('dummy_view')

        self.assertRaises(ObjectDoesNotExist, dummy_view, request)

    def test_anonymous_user_wrong_app(self):

        request = self._get_request(self.anon)

        @permission_required_or_403('not_installed_app.change_user')
        def dummy_view(request):
            return HttpResponse('dummy_view')
        self.assertEqual(dummy_view(request).status_code, 403)

    def test_anonymous_user_wrong_codename(self):

        request = self._get_request()

        @permission_required_or_403('auth.wrong_codename')
        def dummy_view(request):
            return HttpResponse('dummy_view')
        self.assertEqual(dummy_view(request).status_code, 403)

    def test_anonymous_user(self):

        request = self._get_request()

        @permission_required_or_403('auth.change_user')
        def dummy_view(request):
            return HttpResponse('dummy_view')
        self.assertEqual(dummy_view(request).status_code, 403)

    def test_wrong_lookup_variables_number(self):

        request = self._get_request()

        try:
            @permission_required_or_403('auth.change_user', (User, 'username'))
            def dummy_view(request, username):
                pass
            dummy_view(request, username='jack')
        except GuardianError:
            pass
        else:
            self.fail("If lookup variables are passed they must be tuple of: "
                      "(ModelClass/app_label.ModelClass/queryset, "
                      "<pair of lookup_string and view_arg>)\n"
                      "Otherwise GuardianError should be raised")

    def test_wrong_lookup_variables(self):

        request = self._get_request()

        args = (
            (2010, 'username', 'username'),
            ('User', 'username', 'username'),
            (User, 'username', 'no_arg'),
        )
        for tup in args:
            try:
                @permission_required_or_403('auth.change_user', tup)
                def show_user(request, username):
                    user = get_object_or_404(User, username=username)
                    return HttpResponse("It's %s here!" % user.username)
                show_user(request, 'jack')
            except GuardianError:
                pass
            else:
                self.fail("Wrong arguments given but GuardianError not raised")

    def test_user_has_no_access(self):

        request = self._get_request()

        @permission_required_or_403('auth.change_user')
        def dummy_view(request):
            return HttpResponse('dummy_view')
        self.assertEqual(dummy_view(request).status_code, 403)

    def test_user_has_access(self):

        perm = get_user_permission_full_codename('change')
        joe, created = User.objects.get_or_create(username='joe')
        assign_perm(perm, self.user, obj=joe)

        request = self._get_request(self.user)

        @permission_required_or_403(perm, (
            user_model_path, 'username', 'username'))
        def dummy_view(request, username):
            return HttpResponse('dummy_view')
        response = dummy_view(request, username='joe')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.content, b'dummy_view')

    def test_user_has_access_on_model_with_metaclass(self):
        """
        Test to the fix issues of comparaison made via type()
        in the decorator. In the case of a `Model` implementing
        a custom metaclass, the decorator fail because type
        doesn't return `ModelBase`
        """
        perm = get_user_permission_full_codename('change')

        class TestMeta(ModelBase):
            pass

        class ProxyUser(User):

            class Meta:
                proxy = True
                app_label = User._meta.app_label
            __metaclass__ = TestMeta

        joe, created = ProxyUser.objects.get_or_create(username='joe')
        assign_perm(perm, self.user, obj=joe)

        request = self._get_request(self.user)

        @permission_required_or_403(perm, (
            ProxyUser, 'username', 'username'))
        def dummy_view(request, username):
            return HttpResponse('dummy_view')
        response = dummy_view(request, username='joe')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.content, b'dummy_view')

    def test_user_has_obj_access_even_if_we_also_check_for_global(self):

        perm = get_user_permission_full_codename('change')
        joe, created = User.objects.get_or_create(username='joe')
        assign_perm(perm, self.user, obj=joe)

        request = self._get_request(self.user)

        @permission_required_or_403(perm, (
            user_model_path, 'username', 'username'), accept_global_perms=True)
        def dummy_view(request, username):
            return HttpResponse('dummy_view')
        response = dummy_view(request, username='joe')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.content, b'dummy_view')

    def test_user_has_no_obj_perm_access(self):

        perm = get_user_permission_full_codename('change')
        joe, created = User.objects.get_or_create(username='joe')

        request = self._get_request(self.user)

        @permission_required_or_403(perm, (
            user_model_path, 'username', 'username'))
        def dummy_view(request, username):
            return HttpResponse('dummy_view')
        response = dummy_view(request, username='joe')
        self.assertEqual(response.status_code, 403)

    def test_user_has_global_perm_access_but_flag_not_set(self):

        perm = get_user_permission_full_codename('change')
        joe, created = User.objects.get_or_create(username='joe')
        assign_perm(perm, self.user)

        request = self._get_request(self.user)

        @permission_required_or_403(perm, (
            user_model_path, 'username', 'username'))
        def dummy_view(request, username):
            return HttpResponse('dummy_view')
        response = dummy_view(request, username='joe')
        self.assertEqual(response.status_code, 403)

    def test_user_has_global_perm_access(self):

        perm = get_user_permission_full_codename('change')
        joe, created = User.objects.get_or_create(username='joe')
        assign_perm(perm, self.user)

        request = self._get_request(self.user)

        @permission_required_or_403(perm, (
            user_model_path, 'username', 'username'), accept_global_perms=True)
        def dummy_view(request, username):
            return HttpResponse('dummy_view')
        response = dummy_view(request, username='joe')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.content, b'dummy_view')

    def test_model_lookup(self):

        request = self._get_request(self.user)

        perm = get_user_permission_full_codename('change')
        joe, created = User.objects.get_or_create(username='joe')
        assign_perm(perm, self.user, obj=joe)

        models = (
            user_model_path,
            User,
            User.objects.filter(is_active=True),
        )
        for model in models:
            @permission_required_or_403(perm, (model, 'username', 'username'))
            def dummy_view(request, username):
                get_object_or_404(User, username=username)
                return HttpResponse('hello')
            response = dummy_view(request, username=joe.username)
            self.assertEqual(response.content, b'hello')

    def test_redirection_raises_wrong_app_error(self):
        from guardian.testapp.models import Project
        request = self._get_request(self.user)

        User.objects.create(username='foo')
        Project.objects.create(name='foobar')

        @permission_required('auth.change_group',
                             (Project, 'name', 'group_name'),
                             login_url='/foobar/')
        def dummy_view(request, project_name):
            pass
        # 'auth.change_group' is wrong permission codename (should be one
        # related with User
        self.assertRaises(WrongAppError, dummy_view,
                          request, group_name='foobar')

    def test_redirection(self):
        from guardian.testapp.models import Project

        request = self._get_request(self.user)

        User.objects.create(username='foo')
        Project.objects.create(name='foobar')

        @permission_required('testapp.change_project',
                             (Project, 'name', 'project_name'),
                             login_url='/foobar/')
        def dummy_view(request, project_name):
            pass
        response = dummy_view(request, project_name='foobar')
        self.assertTrue(isinstance(response, HttpResponseRedirect))
        if django.VERSION >= (3, 2):
            self.assertTrue(response.headers['location'].startswith(
                '/foobar/'))
        else:
            self.assertTrue(response._headers['location'][1].startswith(
                '/foobar/'))

    @override_settings(LOGIN_URL='django.contrib.auth.views.login')
    def test_redirection_class(self):
        view_url = '/permission_required/'

        response = self.client.get(view_url)
        # this should be '/account/login'
        self.assertRedirects(
            response, global_settings.LOGIN_URL + "?next=" + view_url)
<|endoftext|>


<commit_before>"""A module containing a core representation of an IATI Dataset."""
import sys
from lxml import etree
import iati.core.exceptions
import iati.core.utilities

class Dataset(object):
    """Representation of an IATI XML file that may be validated against a schema.

    Attributes:
        xml_str (str): A string representation of the XML being represented.
        xml_tree (ElementTree): A tree representation of the XML being represented.

    Note:
        The current content of the dataset is deemed to be that which was last asigned to either `self.xml_str` or `self.xml_tree`.

    Warning:
        The behaviour of simultaneous assignment to both `self.xml_str` and `self.xml_tree` is undefined.

        `xml_str` and `xml_tree` are not great names. They are also too tied together. It should be determined whether this close relationship is really desired.

        Does not fully hide the lxml internal workings.

    Todo:
        Implement getters and setters for attributes.

        Implement an addition override to allow for combation of datasets.

    """

    def __init__(self, xml):
        """Initialise a dataset.

        Args:
            xml (str/ElementTree): A representation of the XML to encapsulate.
                May be either a string or an ElementTree.

        Raises:
            TypeError: If an attempt to pass something that is not a string or ElementTree is made.
            ValueError: If a provided XML string is not valid XML.
            iati.core.exceptions.ValidationError:
                If the provided XML should conform to the IATI standard, but does not.

        Warning:
            It should be possible to create a dataset from a file. In this situation, having `xml` as a required parameter does not seem sensible. Need to better consider this situation.

        Todo:
            Undertake validation.
            Add a way to determine whether a dataset fully conforms to the IATI standard and / or modify the dataset so that it does.

        """
        self._xml_str = None
        self._xml_tree = None

        if isinstance(xml, etree._Element):  # pylint: disable=W0212
            self.xml_tree = xml
        else:
            self.xml_str = xml

    @property
    def xml_str(self):
        """Return a string representation of the XML being represented.

        Raises:
            ValueError: If a value that is being assigned is not a valid XML string.
            TypeError: If a value that is being assigned is not a string at all.

        Todo:
            Clarify error messages, for example when a mismatched encoding is used.
            Perhaps pass on the original lxml error message instead of trying to intrepret what might have gone wrong when running etree.fromstring.

        """
        return self._xml_str

    @xml_str.setter
    def xml_str(self, value):
        if isinstance(value, etree._Element):  # pylint: disable=W0212
            msg = "If setting a dataset with an ElementTree, use the xml_tree property, not the xml_str property."
            iati.core.utilities.log_error(msg)
            raise TypeError(msg)
        else:
            try:
                value_stripped = value.strip()

                # Convert the input to bytes, as etree.fromstring works most consistently with bytes objects, especially if an XML encoding declaration has been used.
                if (isinstance(value_stripped, str) and
                        sys.version_info.major > 2):  # Python v2 treats strings as byte objects by default
                    value_stripped_bytes = value_stripped.encode()
                else:
                    value_stripped_bytes = value_stripped

                self.xml_tree = etree.fromstring(value_stripped_bytes)
                self._xml_str = value_stripped
            except etree.XMLSyntaxError:
                msg = "The string provided to create a Dataset from is not valid XML."
                iati.core.utilities.log_error(msg)
                raise ValueError(msg)
            except (AttributeError, TypeError, ValueError):
                msg = "Datasets can only be ElementTrees or strings containing valid XML, using the xml_tree and xml_str attributes respectively. Actual type: {0}".format(type(value))
                iati.core.utilities.log_error(msg)
                raise TypeError(msg)

    @property
    def xml_tree(self):
        """Return a tree representation of the XML being represented.

        Raises:
            TypeError: If a value that is being assigned is not an ElementTree.

        Warning:
            Does not fully hide the lxml internal workings.

        Todo:
            Check use of ElementTree in setter.

        """
        return self._xml_tree.getroottree()

    @xml_tree.setter
    def xml_tree(self, value):
        if isinstance(value, etree._Element):  # pylint: disable=W0212
            self._xml_tree = value
            self._xml_str = etree.tostring(value, pretty_print=True)
        else:
            msg = "If setting a dataset with the xml_property, an ElementTree should be provided, not a {0}.".format(type(value))
            iati.core.utilities.log_error(msg)
            raise TypeError(msg)

    def _raw_source_at_line(self, line_number):
        """Return the raw value of the XML source at the specified line.

        Args:
            line_number (int): A zero-indexed line number.

        Returns:
            str: The source of the XML at the specified line.

        Raises:
            TypeError: When the line_number is not an integer.
            ValueError: When the line_number is negative or more than the number of lines in the file.

        """
        if not isinstance(line_number, int) or isinstance(line_number, bool):
            raise TypeError

        if line_number < 0:
            raise ValueError

        try:
            return self.xml_str.split('\n')[line_number]
        except IndexError:
            raise ValueError

    def source_at_line(self, line_number):
        """Return the value of the XML source at the specified line.

        Args:
            line_number (int): A zero-indexed line number.

        Returns:
            str: The source of the XML at the specified line. Leading and trailing whitespace is trimmed.

        Raises:
            TypeError: When the line_number is not an integer.
            ValueError: When the line_number is negative or more than the number of lines in the file.

        """
        return self._raw_source_at_line(line_number).strip()

    def source_around_line(self, line_number, surrounding_lines=1):
        """Return the value of the XML source at the specified line, plus its surrounding context.

        Args:
            line_number (int): A zero-indexed line number.
            surrounding_lines (int): The number of lines of context to provide either side of the specified line number. Default 1.

        Returns:
            str: The source of the XML at the specified line, plus its surrounding whitespace.

        Raises:
            TypeError: When the line_number is not an integer.
            ValueError: When the line_number is negative or more than the number of lines in the file.

        """
        lines_arr = []
        lower_line_number =  max(line_number - surrounding_lines, 0)
        upper_line_number = min(line_number + surrounding_lines + 1, len(self.xml_str.split('\n')))

        for idx in range(lower_line_number, upper_line_number):
            lines_arr.append(self._raw_source_at_line(idx))

        return '\n'.join(lines_arr)
<commit_msg>idx -> line_num<commit_after>"""A module containing a core representation of an IATI Dataset."""
import sys
from lxml import etree
import iati.core.exceptions
import iati.core.utilities

class Dataset(object):
    """Representation of an IATI XML file that may be validated against a schema.

    Attributes:
        xml_str (str): A string representation of the XML being represented.
        xml_tree (ElementTree): A tree representation of the XML being represented.

    Note:
        The current content of the dataset is deemed to be that which was last asigned to either `self.xml_str` or `self.xml_tree`.

    Warning:
        The behaviour of simultaneous assignment to both `self.xml_str` and `self.xml_tree` is undefined.

        `xml_str` and `xml_tree` are not great names. They are also too tied together. It should be determined whether this close relationship is really desired.

        Does not fully hide the lxml internal workings.

    Todo:
        Implement getters and setters for attributes.

        Implement an addition override to allow for combation of datasets.

    """

    def __init__(self, xml):
        """Initialise a dataset.

        Args:
            xml (str/ElementTree): A representation of the XML to encapsulate.
                May be either a string or an ElementTree.

        Raises:
            TypeError: If an attempt to pass something that is not a string or ElementTree is made.
            ValueError: If a provided XML string is not valid XML.
            iati.core.exceptions.ValidationError:
                If the provided XML should conform to the IATI standard, but does not.

        Warning:
            It should be possible to create a dataset from a file. In this situation, having `xml` as a required parameter does not seem sensible. Need to better consider this situation.

        Todo:
            Undertake validation.
            Add a way to determine whether a dataset fully conforms to the IATI standard and / or modify the dataset so that it does.

        """
        self._xml_str = None
        self._xml_tree = None

        if isinstance(xml, etree._Element):  # pylint: disable=W0212
            self.xml_tree = xml
        else:
            self.xml_str = xml

    @property
    def xml_str(self):
        """Return a string representation of the XML being represented.

        Raises:
            ValueError: If a value that is being assigned is not a valid XML string.
            TypeError: If a value that is being assigned is not a string at all.

        Todo:
            Clarify error messages, for example when a mismatched encoding is used.
            Perhaps pass on the original lxml error message instead of trying to intrepret what might have gone wrong when running etree.fromstring.

        """
        return self._xml_str

    @xml_str.setter
    def xml_str(self, value):
        if isinstance(value, etree._Element):  # pylint: disable=W0212
            msg = "If setting a dataset with an ElementTree, use the xml_tree property, not the xml_str property."
            iati.core.utilities.log_error(msg)
            raise TypeError(msg)
        else:
            try:
                value_stripped = value.strip()

                # Convert the input to bytes, as etree.fromstring works most consistently with bytes objects, especially if an XML encoding declaration has been used.
                if (isinstance(value_stripped, str) and
                        sys.version_info.major > 2):  # Python v2 treats strings as byte objects by default
                    value_stripped_bytes = value_stripped.encode()
                else:
                    value_stripped_bytes = value_stripped

                self.xml_tree = etree.fromstring(value_stripped_bytes)
                self._xml_str = value_stripped
            except etree.XMLSyntaxError:
                msg = "The string provided to create a Dataset from is not valid XML."
                iati.core.utilities.log_error(msg)
                raise ValueError(msg)
            except (AttributeError, TypeError, ValueError):
                msg = "Datasets can only be ElementTrees or strings containing valid XML, using the xml_tree and xml_str attributes respectively. Actual type: {0}".format(type(value))
                iati.core.utilities.log_error(msg)
                raise TypeError(msg)

    @property
    def xml_tree(self):
        """Return a tree representation of the XML being represented.

        Raises:
            TypeError: If a value that is being assigned is not an ElementTree.

        Warning:
            Does not fully hide the lxml internal workings.

        Todo:
            Check use of ElementTree in setter.

        """
        return self._xml_tree.getroottree()

    @xml_tree.setter
    def xml_tree(self, value):
        if isinstance(value, etree._Element):  # pylint: disable=W0212
            self._xml_tree = value
            self._xml_str = etree.tostring(value, pretty_print=True)
        else:
            msg = "If setting a dataset with the xml_property, an ElementTree should be provided, not a {0}.".format(type(value))
            iati.core.utilities.log_error(msg)
            raise TypeError(msg)

    def _raw_source_at_line(self, line_number):
        """Return the raw value of the XML source at the specified line.

        Args:
            line_number (int): A zero-indexed line number.

        Returns:
            str: The source of the XML at the specified line.

        Raises:
            TypeError: When the line_number is not an integer.
            ValueError: When the line_number is negative or more than the number of lines in the file.

        """
        if not isinstance(line_number, int) or isinstance(line_number, bool):
            raise TypeError

        if line_number < 0:
            raise ValueError

        try:
            return self.xml_str.split('\n')[line_number]
        except IndexError:
            raise ValueError

    def source_at_line(self, line_number):
        """Return the value of the XML source at the specified line.

        Args:
            line_number (int): A zero-indexed line number.

        Returns:
            str: The source of the XML at the specified line. Leading and trailing whitespace is trimmed.

        Raises:
            TypeError: When the line_number is not an integer.
            ValueError: When the line_number is negative or more than the number of lines in the file.

        """
        return self._raw_source_at_line(line_number).strip()

    def source_around_line(self, line_number, surrounding_lines=1):
        """Return the value of the XML source at the specified line, plus its surrounding context.

        Args:
            line_number (int): A zero-indexed line number.
            surrounding_lines (int): The number of lines of context to provide either side of the specified line number. Default 1.

        Returns:
            str: The source of the XML at the specified line, plus its surrounding whitespace.

        Raises:
            TypeError: When the line_number is not an integer.
            ValueError: When the line_number is negative or more than the number of lines in the file.

        """
        lines_arr = []
        lower_line_number =  max(line_number - surrounding_lines, 0)
        upper_line_number = min(line_number + surrounding_lines + 1, len(self.xml_str.split('\n')))

        for line_num in range(lower_line_number, upper_line_number):
            lines_arr.append(self._raw_source_at_line(line_num))

        return '\n'.join(lines_arr)
<|endoftext|>


<commit_before>
#  IATI Data Quality, tools for Data QA on IATI-formatted  publications
#  by Mark Brough, Martin Keegan, Ben Webb and Jennifer Smith
#
#  Copyright (C) 2013  Publish What You Fund
#
#  This programme is free software; you may redistribute and/or modify
#  it under the terms of the GNU Affero General Public License v3.0

import operator
import itertools
import models # damn!
import pprint
import sys
from iatidq import db

COL_INDICATOR = 0
COL_TEST = 1
COL_RESULTS_DATA = 2
COL_RESULTS_NUM = 3
COL_HIERARCHY = 4
COL_PACKAGE = 5

class NoRelevantResults(Exception): pass

def reform_dict(d):
    def inner(k1):
        matches_first = lambda i: i[0] == k1
        return dict([ (k2, d[(k1, k2)]) for k2 in
                      map(lambda i: i[1], filter(matches_first, d.keys())) ])

    return dict([ (k1, inner(k1))
                   for k1 in set( k1 for k1, k2 in d.keys() ) ])

def remove_empty_dicts(h):
    has_keys = lambda kvp: len(kvp[1])
    return dict([
            (K, dict(filter(has_keys, V.items())))
            for K, V in h.items()
            ])

class TestInfo(object):
    def __init__(self):
        self.tests = dict([ (t.id, t) for t in
                            models.Test.query.all() ])

    def as_dict(self, test_id, results_raw_score, results_num, sampling_ok):
        test = self.tests[test_id]

        sampling_factor = {
            True: 1.0,
            False: 0.0
            }[sampling_ok]

        results_pct = results_raw_score * sampling_factor

        return {
            "test": {
                "id": test_id,
                "name": test.name,
                "description": test.description,
                "test_group": test.test_group,
                "test_level": test.test_level
                },
            "results_pct": results_pct,
            "results_num": results_num,
            "sampling_ok": sampling_ok,
            "results_raw_score": results_raw_score
            }

class IndicatorInfo(object):
    def __init__(self):
        self.inds = dict([ (i.id, i.as_dict()) for i in
                           models.Indicator.query.all() ])

    def as_dict(self, indicator_id):
        return self.inds[indicator_id]

    def as_dict_minus_group(self, indicator_id):
        tmp = dict(self.inds[indicator_id]) # deep copy or you mutate >1 time
        del(tmp['indicatorgroup_id'])
        return tmp

def publisher_indicators(indicator_info, indicators, indicators_tests,
                         simple_out):
    # get all tests which belong to a specific indicator
    # average the results for all tests in that indicator
    def per_indicator(indicator):
        indicator_test_data = []
        results_pct = 0.0
        results_num = 0.0

        relevant = lambda test: (indicator, test) in indicators_tests
        tests = filter(relevant, simple_out.keys())

        for test in tests:
            indic_info = simple_out[test]
            results_pct += indic_info["results_pct"]
            results_num += indic_info["results_num"]
            indicator_test_data.append(indic_info)

        num_tests = len(tests)

        return {
            "indicator": indicator_info.as_dict_minus_group(indicator),
            "tests": indicator_test_data,
            "results_pct": (results_pct/num_tests),
            "results_num": results_num
            }

    return dict([ (i, per_indicator(i)) for i in indicators ])

# The value of this function is immediately consumed by the one above it;
# it's only used in that one place
def publisher_simple(all_test_info, out, cdtns, indicator_lookup, indicators,
                     sampling_data):
    hierarchies = set(out)
    tests = set()

    for h in hierarchies:
        tests.update(set(out[h]))

    def per_test(t):
        results_pct = 0.0 # why does this not get used?
        results_num = 0.0
        results_weighted_pct_average_numerator = 0.0

        def relevant(hierarchy):
            if t not in out[hierarchy]:
                return False

            return cdtns.is_relevant(t, hierarchy)

        for hierarchy in filter(relevant, hierarchies):
            test_info = out[hierarchy][t]

            results_pct += test_info["results_pct"]
            results_num += test_info["results_num"]
            results_weighted_pct_average_numerator += (
                test_info["results_pct"] *
                test_info["results_num"]
                )

        if results_num == 0:
            raise NoRelevantResults("Results_num == 0 for test: %d" % t)

        sampling_ok = sampling_data[t]

        tmp = all_test_info.as_dict(
            t,
            (results_weighted_pct_average_numerator / results_num),
            results_num,
            sampling_ok
            )
        indicator_id = indicator_lookup[t]
        tmp["indicator"] = indicators.as_dict(indicator_id)
        return tmp

    return dict([ (t, per_test(t)) for t in tests ])

class Summary(object):
    def summary(self):
        return self._summary

    def generate_summaries(self, hierarchies, tests, summary_f):
        for h, t in itertools.product(hierarchies, tests):
            yield h, t, summary_f(h, t)

    def summarise_results(self, hierarchies,
                      tests, indicators,
                      indicators_tests, indicator_lookup,
                      summary_f):

        def add_condition(i):
            h, t, tdata = i
            if self.conditions.has_condition(t, h):
                tdata["condition"] = self.conditions.get_condition(t, h)
            return h, t, tdata

        summaries = (add_condition(i) for i in self.generate_summaries(
                hierarchies, tests, summary_f))

        tmp_out = dict([ ((h, t), tdata) for h, t, tdata in summaries ])

        out = reform_dict(tmp_out)

        out = remove_empty_dicts(out)
        return self.add_indicator_info(out, indicators,
                                       indicators_tests, indicator_lookup)

    def add_indicator_info(self, out, indicators,
                           indicators_tests, indicator_lookup):
        return out

class PublisherSummary(Summary):
    pass

class NewPublisherSummary(PublisherSummary):
    def __init__(self, conditions, organisation_id, aggregation_type):
        self.conditions = conditions
        self.indicators = IndicatorInfo()
        self.tests = TestInfo()
        self.sampling_data = self.get_sampling_data(organisation_id)

        join_clause = '''
            JOIN organisationpackage USING (package_id, organisation_id)
        '''

        where_clause = '''WHERE aggregateresult.organisation_id = %d AND
                            aggregateresulttype_id = %d''' % (organisation_id,
                                                              aggregation_type)

        self._summary = self.calculate(join_clause, where_clause)

    def calculate(self, join_clause, where_clause):
        # make list of data; hand over to
        # summarise_results(hierarchies, tests, indicators,
        #                   indicators_tests, summary_f):
        conn = db.session.connection()

        sql = '''SELECT DISTINCT result_hierarchy
                   FROM aggregateresult
                   %s %s;'''
        stmt = sql % (join_clause, where_clause)
        hierarchies = [ h[0] for h in conn.execute(stmt) ]

        sql = '''SELECT DISTINCT indicator_id, test_id
                   FROM aggregateresult
                   JOIN indicatortest USING (test_id)
                   %s %s;'''
        stmt = sql % (join_clause, where_clause)
        indicators_tests = [ it for it in conn.execute(stmt) ]
        tests = [ it[1] for it in indicators_tests ]
        indicators = [ it[0] for it in indicators_tests ]

        indicator_lookup = dict([ (it[1], it[0]) for it in indicators_tests ])

        sql = '''SELECT result_hierarchy, test_id, AVG(results_data) AS pct,
                        SUM(results_num) AS total_activities
                   FROM aggregateresult
                   %s %s
                   GROUP BY test_id, result_hierarchy;'''
        stmt = sql % (join_clause, where_clause)
        data = dict([ ((ar[0], ar[1]), ar) for ar in (conn.execute(stmt)) ])
        conn.close()
        del(conn)

        def summary_f(hierarchy, test_id):
            key = (hierarchy, test_id)
            if key not in data:
                return {}

            aresult = data[key]
            sampling_ok = self.sampling_data[test_id]
            indicator_id = indicator_lookup[test_id]

            tmp = self.tests.as_dict(test_id, aresult[2], aresult[3],
                                     sampling_ok)
            tmp["indicator"] = self.indicators.as_dict(indicator_id)
            return tmp

        return self.summarise_results(hierarchies, tests, indicators,
                                      indicators_tests, indicator_lookup,
                                      summary_f)

    def get_sampling_data(self, organisation_id):
        sql = '''SELECT test_id FROM sampling_failure
                   WHERE organisation_id = %s;'''
        failed_test_ids = [ row[0] for row in
                            db.engine.execute(sql, (organisation_id,)) ]

        def ok(t):
            return t not in failed_test_ids

        return dict([ (t, ok(t)) for t in self.tests.tests.keys() ])

class NewPackageSummary(NewPublisherSummary):
    def __init__(self, conditions, package_id, aggregation_type):
        self.conditions = conditions
        self.indicators = IndicatorInfo()
        self.tests = TestInfo()
        self.sampling_data = self.get_sampling_data(None)

        join_clause = ""

        where_clause = '''WHERE package_id = %d AND
                            aggregateresulttype_id = %d''' % (package_id,
                                                              aggregation_type)

        self._summary = self.calculate(join_clause, where_clause)

class PublisherIndicatorsSummary(NewPublisherSummary):
    def add_indicator_info(self, out, indicators,
                           indicators_tests, indicator_lookup):

        simple_out = publisher_simple(self.tests, out, self.conditions,
                                      indicator_lookup, self.indicators,
                                      self.sampling_data)
        return publisher_indicators(self.indicators, indicators,
                                    indicators_tests, simple_out)

from models import *

class OrgConditions(object):
    def __init__(self, organisation_id):
        # None is passed as organisation_id for case of no conditions wanted
        if organisation_id is None:
            self._conditions = {}
            return

        cc = OrganisationCondition.query.filter_by(
            organisation_id=organisation_id
            ).all()
        self._conditions = dict(map(lambda x: (
                    (x.test_id, x.condition, x.condition_value),
                    (x.operation, x.description)
                    ), cc))

    def _key(self, test_id, hierarchy):
        return (test_id, 'activity hierarchy', str(hierarchy))

    def is_relevant(self, test_id, hierarchy):
        key = self._key(test_id, hierarchy)

        if not self._conditions:
            return True

        if key not in self._conditions:
            return True

        if self._conditions[key][0] == 0:
            return False

        return True

    def has_condition(self, test_id, hierarchy):
        key = self._key(test_id, hierarchy)
        return key in self._conditions

    def get_condition(self, test_id, hierarchy):
        key = self._key(test_id, hierarchy)
        return self._conditions[key]

class SummaryCreator(object):
    @property
    def summary(self):
        return self._summary

    @property
    def aggregate_results(self):
        return self._aggregate_results

class PublisherSummaryCreator(SummaryCreator):
    def __init__(self, organisation, aggregation_type):
        organisation_id = organisation.id
        pconditions = OrgConditions(organisation_id)

        self._summary = NewPublisherSummary(pconditions,
                                            organisation_id,
                                            aggregation_type)

class PackageSummaryCreator(SummaryCreator):
    def __init__(self, package_id, latest_runtime, aggregation_type):

        self._summary = NewPackageSummary(OrgConditions(None),
                                          package_id,
                                          aggregation_type).summary()

class PublisherIndicatorsSummaryCreator(SummaryCreator):
    def __init__(self, organisation, aggregation_type):
        organisation_id = organisation.id
        pconditions = OrgConditions(organisation_id)

        self._summary = PublisherIndicatorsSummary(pconditions,
                                                   organisation_id,
                                                   aggregation_type)

# select domain, sum(results_data * results_num::float/t1.total)
# from example_summary join
#      (select domain, sum(results_num) as total
#         from example_summary
#         group by domain) as t1
# using (domain)
# group by example_summary.domain;

# select org55_aggresult.* from
# (select result_hierarchy, test_id, results_data, results_num
#  from aggregateresult
#  join organisationpackage USING (package_id, organisation_id)
#  where aggregateresult.organisation_id = 55
#    and aggregateresulttype_id=2
# ) as org55_aggresult;

_sql = """with org55_aggresult as (
	 select result_hierarchy, test_id, results_data, results_num
	 from aggregateresult
	 join organisationpackage USING (package_id, organisation_id)
     where aggregateresult.organisation_id = 55
       and aggregateresulttype_id=2
)
select result_hierarchy, test_id,
       sum(results_data * results_num::float/t1.total) as pct,
       sum(results_num) as total_activities
from org55_aggresult join
  (select result_hierarchy, test_id, sum(results_num) as total
   from org55_aggresult
   group by result_hierarchy, test_id) as t1
using (result_hierarchy, test_id)
group by org55_aggresult.result_hierarchy, org55_aggresult.test_id
"""
<commit_msg>reindent<commit_after>
#  IATI Data Quality, tools for Data QA on IATI-formatted  publications
#  by Mark Brough, Martin Keegan, Ben Webb and Jennifer Smith
#
#  Copyright (C) 2013  Publish What You Fund
#
#  This programme is free software; you may redistribute and/or modify
#  it under the terms of the GNU Affero General Public License v3.0

import operator
import itertools
import models # damn!
import pprint
import sys
from iatidq import db

COL_INDICATOR = 0
COL_TEST = 1
COL_RESULTS_DATA = 2
COL_RESULTS_NUM = 3
COL_HIERARCHY = 4
COL_PACKAGE = 5

class NoRelevantResults(Exception): pass

def reform_dict(d):
    def inner(k1):
        matches_first = lambda i: i[0] == k1
        return dict([ (k2, d[(k1, k2)]) for k2 in
                      map(lambda i: i[1], filter(matches_first, d.keys())) ])

    return dict([ (k1, inner(k1))
                   for k1 in set( k1 for k1, k2 in d.keys() ) ])

def remove_empty_dicts(h):
    has_keys = lambda kvp: len(kvp[1])
    return dict([
            (K, dict(filter(has_keys, V.items())))
            for K, V in h.items()
            ])

class TestInfo(object):
    def __init__(self):
        self.tests = dict([ (t.id, t) for t in
                            models.Test.query.all() ])

    def as_dict(self, test_id, results_raw_score, results_num, sampling_ok):
        test = self.tests[test_id]

        sampling_factor = {
            True: 1.0,
            False: 0.0
            }[sampling_ok]

        results_pct = results_raw_score * sampling_factor

        return {
            "test": {
                "id": test_id,
                "name": test.name,
                "description": test.description,
                "test_group": test.test_group,
                "test_level": test.test_level
                },
            "results_pct": results_pct,
            "results_num": results_num,
            "sampling_ok": sampling_ok,
            "results_raw_score": results_raw_score
            }

class IndicatorInfo(object):
    def __init__(self):
        self.inds = dict([ (i.id, i.as_dict()) for i in
                           models.Indicator.query.all() ])

    def as_dict(self, indicator_id):
        return self.inds[indicator_id]

    def as_dict_minus_group(self, indicator_id):
        tmp = dict(self.inds[indicator_id]) # deep copy or you mutate >1 time
        del(tmp['indicatorgroup_id'])
        return tmp

def publisher_indicators(indicator_info, indicators, indicators_tests,
                         simple_out):
    # get all tests which belong to a specific indicator
    # average the results for all tests in that indicator
    def per_indicator(indicator):
        indicator_test_data = []
        results_pct = 0.0
        results_num = 0.0

        relevant = lambda test: (indicator, test) in indicators_tests
        tests = filter(relevant, simple_out.keys())

        for test in tests:
            indic_info = simple_out[test]
            results_pct += indic_info["results_pct"]
            results_num += indic_info["results_num"]
            indicator_test_data.append(indic_info)

        num_tests = len(tests)

        return {
            "indicator": indicator_info.as_dict_minus_group(indicator),
            "tests": indicator_test_data,
            "results_pct": (results_pct/num_tests),
            "results_num": results_num
            }

    return dict([ (i, per_indicator(i)) for i in indicators ])

# The value of this function is immediately consumed by the one above it;
# it's only used in that one place
def publisher_simple(all_test_info, out, cdtns, indicator_lookup, indicators,
                     sampling_data):
    hierarchies = set(out)
    tests = set()

    for h in hierarchies:
        tests.update(set(out[h]))

    def per_test(t):
        results_pct = 0.0 # why does this not get used?
        results_num = 0.0
        results_weighted_pct_average_numerator = 0.0

        def relevant(hierarchy):
            if t not in out[hierarchy]:
                return False

            return cdtns.is_relevant(t, hierarchy)

        for hierarchy in filter(relevant, hierarchies):
            test_info = out[hierarchy][t]

            results_pct += test_info["results_pct"]
            results_num += test_info["results_num"]
            results_weighted_pct_average_numerator += (
                test_info["results_pct"] *
                test_info["results_num"]
                )

        if results_num == 0:
            raise NoRelevantResults("Results_num == 0 for test: %d" % t)

        sampling_ok = sampling_data[t]

        tmp = all_test_info.as_dict(
            t,
            (results_weighted_pct_average_numerator / results_num),
            results_num,
            sampling_ok
            )
        indicator_id = indicator_lookup[t]
        tmp["indicator"] = indicators.as_dict(indicator_id)
        return tmp

    return dict([ (t, per_test(t)) for t in tests ])

class Summary(object):
    def summary(self):
        return self._summary

    def generate_summaries(self, hierarchies, tests, summary_f):
        for h, t in itertools.product(hierarchies, tests):
            yield h, t, summary_f(h, t)

    def summarise_results(self, hierarchies,
                      tests, indicators,
                      indicators_tests, indicator_lookup,
                      summary_f):

        def add_condition(i):
            h, t, tdata = i
            if self.conditions.has_condition(t, h):
                tdata["condition"] = self.conditions.get_condition(t, h)
            return h, t, tdata

        summaries = (add_condition(i) for i in self.generate_summaries(
                hierarchies, tests, summary_f))

        tmp_out = dict([ ((h, t), tdata) for h, t, tdata in summaries ])

        out = reform_dict(tmp_out)

        out = remove_empty_dicts(out)
        return self.add_indicator_info(out, indicators,
                                       indicators_tests, indicator_lookup)

    def add_indicator_info(self, out, indicators,
                           indicators_tests, indicator_lookup):
        return out

class PublisherSummary(Summary):
    pass

class NewPublisherSummary(PublisherSummary):
    def __init__(self, conditions, organisation_id, aggregation_type):
        self.conditions = conditions
        self.indicators = IndicatorInfo()
        self.tests = TestInfo()
        self.sampling_data = self.get_sampling_data(organisation_id)

        join_clause = '''
            JOIN organisationpackage USING (package_id, organisation_id)
        '''

        where_clause = '''WHERE aggregateresult.organisation_id = %d AND
                            aggregateresulttype_id = %d''' % (organisation_id,
                                                              aggregation_type)

        self._summary = self.calculate(join_clause, where_clause)

    def calculate(self, join_clause, where_clause):
        # make list of data; hand over to
        # summarise_results(hierarchies, tests, indicators,
        #                   indicators_tests, summary_f):
        conn = db.session.connection()

        sql = '''SELECT DISTINCT result_hierarchy
                   FROM aggregateresult
                   %s %s;'''
        stmt = sql % (join_clause, where_clause)
        hierarchies = [ h[0] for h in conn.execute(stmt) ]

        sql = '''SELECT DISTINCT indicator_id, test_id
                   FROM aggregateresult
                   JOIN indicatortest USING (test_id)
                   %s %s;'''
        stmt = sql % (join_clause, where_clause)
        indicators_tests = [ it for it in conn.execute(stmt) ]
        tests = [ it[1] for it in indicators_tests ]
        indicators = [ it[0] for it in indicators_tests ]

        indicator_lookup = dict([ (it[1], it[0]) for it in indicators_tests ])

        sql = '''SELECT result_hierarchy, test_id, AVG(results_data) AS pct,
                        SUM(results_num) AS total_activities
                   FROM aggregateresult
                   %s %s
                   GROUP BY test_id, result_hierarchy;'''
        stmt = sql % (join_clause, where_clause)
        data = dict([ ((ar[0], ar[1]), ar) for ar in (conn.execute(stmt)) ])
        conn.close()
        del(conn)

        def summary_f(hierarchy, test_id):
            key = (hierarchy, test_id)
            if key not in data:
                return {}

            aresult = data[key]
            sampling_ok = self.sampling_data[test_id]
            indicator_id = indicator_lookup[test_id]

            tmp = self.tests.as_dict(test_id, aresult[2], aresult[3],
                                     sampling_ok)
            tmp["indicator"] = self.indicators.as_dict(indicator_id)
            return tmp

        return self.summarise_results(hierarchies, tests, indicators,
                                      indicators_tests, indicator_lookup,
                                      summary_f)

    def get_sampling_data(self, organisation_id):
        sql = '''SELECT test_id FROM sampling_failure
                   WHERE organisation_id = %s;'''
        failed_test_ids = [ row[0] for row in
                            db.engine.execute(sql, (organisation_id,)) ]

        def ok(t):
            return t not in failed_test_ids

        return dict([ (t, ok(t)) for t in self.tests.tests.keys() ])

class NewPackageSummary(NewPublisherSummary):
    def __init__(self, conditions, package_id, aggregation_type):
        self.conditions = conditions
        self.indicators = IndicatorInfo()
        self.tests = TestInfo()
        self.sampling_data = self.get_sampling_data(None)

        join_clause = ""

        where_clause = '''WHERE package_id = %d AND
                            aggregateresulttype_id = %d''' % (package_id,
                                                              aggregation_type)

        self._summary = self.calculate(join_clause, where_clause)

class PublisherIndicatorsSummary(NewPublisherSummary):
    def add_indicator_info(self, out, indicators,
                           indicators_tests, indicator_lookup):

        simple_out = publisher_simple(self.tests, out, self.conditions,
                                      indicator_lookup, self.indicators,
                                      self.sampling_data)
        return publisher_indicators(self.indicators, indicators,
                                    indicators_tests, simple_out)

from models import *

class OrgConditions(object):
    def __init__(self, organisation_id):
        # None is passed as organisation_id for case of no conditions wanted
        if organisation_id is None:
            self._conditions = {}
            return

        cc = OrganisationCondition.query.filter_by(
            organisation_id=organisation_id
            ).all()
        self._conditions = dict(map(lambda x: (
                    (x.test_id, x.condition, x.condition_value),
                    (x.operation, x.description)
                    ), cc))

    def _key(self, test_id, hierarchy):
        return (test_id, 'activity hierarchy', str(hierarchy))

    def is_relevant(self, test_id, hierarchy):
        key = self._key(test_id, hierarchy)

        if not self._conditions:
            return True

        if key not in self._conditions:
            return True

        if self._conditions[key][0] == 0:
            return False

        return True

    def has_condition(self, test_id, hierarchy):
        key = self._key(test_id, hierarchy)
        return key in self._conditions

    def get_condition(self, test_id, hierarchy):
        key = self._key(test_id, hierarchy)
        return self._conditions[key]

class SummaryCreator(object):
    @property
    def summary(self):
        return self._summary

    @property
    def aggregate_results(self):
        return self._aggregate_results

class PublisherSummaryCreator(SummaryCreator):
    def __init__(self, organisation, aggregation_type):
        organisation_id = organisation.id
        pconditions = OrgConditions(organisation_id)

        self._summary = NewPublisherSummary(pconditions,
                                            organisation_id,
                                            aggregation_type)

class PackageSummaryCreator(SummaryCreator):
    def __init__(self, package_id, latest_runtime, aggregation_type):

        self._summary = NewPackageSummary(OrgConditions(None),
                                          package_id,
                                          aggregation_type).summary()

class PublisherIndicatorsSummaryCreator(SummaryCreator):
    def __init__(self, organisation, aggregation_type):
        organisation_id = organisation.id
        pconditions = OrgConditions(organisation_id)

        self._summary = PublisherIndicatorsSummary(pconditions,
                                                   organisation_id,
                                                   aggregation_type)

# select domain, sum(results_data * results_num::float/t1.total)
# from example_summary join
#      (select domain, sum(results_num) as total
#         from example_summary
#         group by domain) as t1
# using (domain)
# group by example_summary.domain;

# select org55_aggresult.* from
# (select result_hierarchy, test_id, results_data, results_num
#  from aggregateresult
#  join organisationpackage USING (package_id, organisation_id)
#  where aggregateresult.organisation_id = 55
#    and aggregateresulttype_id=2
# ) as org55_aggresult;

_sql = """
with org55_aggresult as (
  select result_hierarchy, test_id, results_data, results_num
  from aggregateresult
  join organisationpackage USING (package_id, organisation_id)
  where aggregateresult.organisation_id = 55
    and aggregateresulttype_id=2
)
select result_hierarchy, test_id,
       sum(results_data * results_num::float/t1.total) as pct,
       sum(results_num) as total_activities
from org55_aggresult
join (select result_hierarchy, test_id, sum(results_num) as total
      from org55_aggresult
      group by result_hierarchy, test_id) as t1
using (result_hierarchy, test_id)
group by org55_aggresult.result_hierarchy, org55_aggresult.test_id
"""
<|endoftext|>


<commit_before>__all__ = ('AuthHandler', 'ImageHandler')

import logging
from threading import Thread

l = logging.getLogger(__name__)

class BaseHandler(Thread):
    thread_num = 0

    def __init__(self, *args, **kwargs):
        kwargs.setdefault(
            'name',
            "{0.__name__}-{0.thread_num}".format(self.__class__)
        )
        super().__init__(*args, **kwargs)

    def run(self):
        try:
            self.run_()
        except:
            l.exception("error in {}", self.__class__.__name__)

    # @abstractmethod
    def run_(self):
        pass

# I though Python could handly cyclic imports,
# but it seems like that is not the case.
from .auth import AuthHandler
from .image import ImageHandler
<commit_msg>Increase thread counter<commit_after>__all__ = ('AuthHandler', 'ImageHandler')

import logging
from threading import Thread

l = logging.getLogger(__name__)

class BaseHandler(Thread):
    thread_num = 0

    def __init__(self, *args, **kwargs):
        kwargs.setdefault(
            'name',
            "{0.__name__}-{0.thread_num}".format(self.__class__)
        )
        BaseHandler.thread_num += 1
        super().__init__(*args, **kwargs)

    def run(self):
        try:
            self.run_()
        except:
            l.exception("error in {}", self.__class__.__name__)

    # @abstractmethod
    def run_(self):
        pass

# I though Python could handly cyclic imports,
# but it seems like that is not the case.
from .auth import AuthHandler
from .image import ImageHandler
<|endoftext|>


<commit_before>import datetime

from sqlalchemy import (
    Column,
    DateTime,
    Index,
    LargeBinary,
    SmallInteger,
    String,
    UniqueConstraint,
)
from sqlalchemy.dialects.mysql import INTEGER as Integer
from sqlalchemy.dialects.mysql import BIGINT as BigInteger

from ichnaea.db import _Model

RADIO_TYPE = {
    '': -1,
    'gsm': 0,
    'cdma': 1,
    'umts': 2,
    'lte': 3,
}
RADIO_TYPE_KEYS = list(RADIO_TYPE.keys())

def normalize_wifi_key(key):
    if ":" in key or "-" in key or "." in key:
        key = key.replace(":", "").replace("-", "").replace(".", "")
    return key.lower()

class Cell(_Model):
    __tablename__ = 'cell'
    __table_args__ = (
        UniqueConstraint(
            'radio', 'mcc', 'mnc', 'lac', 'cid', name='cell_idx_unique'),
        Index('cell_created_idx', 'created'),
        Index('cell_new_measures_idx', 'new_measures'),
        {
            'mysql_engine': 'InnoDB',
            'mysql_charset': 'utf8',
        }
    )

    id = Column(BigInteger(unsigned=True),
                primary_key=True, autoincrement=True)
    created = Column(DateTime)
    # lat/lon * decimaljson.FACTOR
    max_lat = Column(Integer)
    min_lat = Column(Integer)
    lat = Column(Integer)

    max_lon = Column(Integer)
    min_lon = Column(Integer)
    lon = Column(Integer)
    # mapped via RADIO_TYPE
    radio = Column(SmallInteger)
    # int in the range 0-1000
    mcc = Column(SmallInteger)
    # int in the range 0-1000 for gsm
    # int in the range 0-32767 for cdma (system id)
    mnc = Column(Integer)
    lac = Column(Integer)
    cid = Column(Integer)
    range = Column(Integer)
    new_measures = Column(Integer(unsigned=True))
    total_measures = Column(Integer(unsigned=True))
    psc = Column(Integer)

    def __init__(self, *args, **kw):
        if 'created' not in kw:
            kw['created'] = datetime.datetime.utcnow()
        if 'lac' not in kw:
            kw['lac'] = -1
        if 'cid' not in kw:
            kw['cid'] = -1
        if 'new_measures' not in kw:
            kw['new_measures'] = 0
        if 'total_measures' not in kw:
            kw['total_measures'] = 0
        super(Cell, self).__init__(*args, **kw)

cell_table = Cell.__table__

class CellMeasure(_Model):
    __tablename__ = 'cell_measure'
    __table_args__ = (
        Index('cell_measure_created_idx', 'created'),
        Index('cell_measure_key_idx', 'radio', 'mcc', 'mnc', 'lac', 'cid'),
        {
            'mysql_engine': 'InnoDB',
            'mysql_charset': 'utf8',
            'mysql_row_format': 'compressed',
            'mysql_key_block_size': '4',
        }
    )

    id = Column(BigInteger(unsigned=True),
                primary_key=True, autoincrement=True)
    measure_id = Column(BigInteger(unsigned=True))
    created = Column(DateTime)  # the insert time of the record into the DB
    # lat/lon * decimaljson.FACTOR
    lat = Column(Integer)
    lon = Column(Integer)
    time = Column(DateTime)  # the time of observation of this data
    accuracy = Column(Integer)
    altitude = Column(Integer)
    altitude_accuracy = Column(Integer)
    # mapped via RADIO_TYPE
    radio = Column(SmallInteger)
    mcc = Column(SmallInteger)
    mnc = Column(Integer)
    lac = Column(Integer)
    cid = Column(Integer)
    psc = Column(Integer)
    asu = Column(SmallInteger)
    signal = Column(SmallInteger)
    ta = Column(SmallInteger)

    def __init__(self, *args, **kw):
        if 'measure_id' not in kw:
            kw['measure_id'] = 0
        if 'created' not in kw:
            kw['created'] = datetime.datetime.utcnow()
        super(CellMeasure, self).__init__(*args, **kw)

cell_measure_table = CellMeasure.__table__

class Wifi(_Model):
    __tablename__ = 'wifi'
    __table_args__ = (
        UniqueConstraint('key', name='wifi_key_unique'),
        Index('wifi_created_idx', 'created'),
        Index('wifi_new_measures_idx', 'new_measures'),
        {
            'mysql_engine': 'InnoDB',
            'mysql_charset': 'utf8',
        }
    )

    id = Column(BigInteger(unsigned=True),
                primary_key=True, autoincrement=True)
    created = Column(DateTime)
    key = Column(String(12))
    # lat/lon * decimaljson.FACTOR
    lat = Column(Integer)
    lon = Column(Integer)
    range = Column(Integer)
    new_measures = Column(Integer(unsigned=True))
    total_measures = Column(Integer(unsigned=True))

    def __init__(self, *args, **kw):
        if 'created' not in kw:
            kw['created'] = datetime.datetime.utcnow()
        if 'new_measures' not in kw:
            kw['new_measures'] = 0
        if 'total_measures' not in kw:
            kw['total_measures'] = 0
        super(Wifi, self).__init__(*args, **kw)

wifi_table = Wifi.__table__

class WifiBlacklist(_Model):
    __tablename__ = 'wifi_blacklist'
    __table_args__ = (
        UniqueConstraint('key', name='wifi_blacklist_key_unique'),
        {
            'mysql_engine': 'InnoDB',
            'mysql_charset': 'utf8',
        }
    )
    id = Column(BigInteger(unsigned=True),
                primary_key=True, autoincrement=True)
    created = Column(DateTime)
    key = Column(String(12))

    def __init__(self, *args, **kw):
        if 'created' not in kw:
            kw['created'] = datetime.datetime.utcnow()
        super(WifiBlacklist, self).__init__(*args, **kw)

class WifiMeasure(_Model):
    __tablename__ = 'wifi_measure'
    __table_args__ = (
        Index('wifi_measure_created_idx', 'created'),
        Index('wifi_measure_key_idx', 'key'),
        {
            'mysql_engine': 'InnoDB',
            'mysql_charset': 'utf8',
            'mysql_row_format': 'compressed',
            'mysql_key_block_size': '4',
        }
    )

    id = Column(BigInteger(unsigned=True),
                primary_key=True, autoincrement=True)
    measure_id = Column(BigInteger(unsigned=True))
    created = Column(DateTime)  # the insert time of the record into the DB
    # lat/lon * decimaljson.FACTOR
    lat = Column(Integer)
    lon = Column(Integer)
    time = Column(DateTime)  # the time of observation of this data
    accuracy = Column(Integer)
    altitude = Column(Integer)
    altitude_accuracy = Column(Integer)
    key = Column(String(12))
    channel = Column(SmallInteger)
    signal = Column(SmallInteger)

    def __init__(self, *args, **kw):
        if 'measure_id' not in kw:
            kw['measure_id'] = 0
        if 'created' not in kw:
            kw['created'] = datetime.datetime.utcnow()
        super(WifiMeasure, self).__init__(*args, **kw)

wifi_measure_table = WifiMeasure.__table__

class Measure(_Model):
    __tablename__ = 'measure'
    __table_args__ = (
        Index('measure_created_idx', 'created'),
        {
            'mysql_engine': 'InnoDB',
            'mysql_charset': 'utf8',
            'mysql_row_format': 'compressed',
            'mysql_key_block_size': '4',
        }
    )

    id = Column(BigInteger(unsigned=True),
                primary_key=True, autoincrement=True)
    created = Column(DateTime)  # the insert time of the record into the DB
    # lat/lon * decimaljson.FACTOR
    lat = Column(Integer)
    lon = Column(Integer)
    time = Column(DateTime)  # the time of observation of this data
    accuracy = Column(Integer)
    altitude = Column(Integer)
    altitude_accuracy = Column(Integer)
    radio = Column(SmallInteger)  # mapped via RADIO_TYPE
    # json blobs
    cell = Column(LargeBinary)
    wifi = Column(LargeBinary)

    def __init__(self, *args, **kw):
        if 'created' not in kw:
            kw['created'] = datetime.datetime.utcnow()
        super(Measure, self).__init__(*args, **kw)

measure_table = Measure.__table__
<commit_msg>moved psc field ordering in the Cell model to match other models<commit_after>import datetime

from sqlalchemy import (
    Column,
    DateTime,
    Index,
    LargeBinary,
    SmallInteger,
    String,
    UniqueConstraint,
)
from sqlalchemy.dialects.mysql import INTEGER as Integer
from sqlalchemy.dialects.mysql import BIGINT as BigInteger

from ichnaea.db import _Model

RADIO_TYPE = {
    '': -1,
    'gsm': 0,
    'cdma': 1,
    'umts': 2,
    'lte': 3,
}
RADIO_TYPE_KEYS = list(RADIO_TYPE.keys())

def normalize_wifi_key(key):
    if ":" in key or "-" in key or "." in key:
        key = key.replace(":", "").replace("-", "").replace(".", "")
    return key.lower()

class Cell(_Model):
    __tablename__ = 'cell'
    __table_args__ = (
        UniqueConstraint(
            'radio', 'mcc', 'mnc', 'lac', 'cid', name='cell_idx_unique'),
        Index('cell_created_idx', 'created'),
        Index('cell_new_measures_idx', 'new_measures'),
        {
            'mysql_engine': 'InnoDB',
            'mysql_charset': 'utf8',
        }
    )

    id = Column(BigInteger(unsigned=True),
                primary_key=True, autoincrement=True)
    created = Column(DateTime)
    # lat/lon * decimaljson.FACTOR
    max_lat = Column(Integer)
    min_lat = Column(Integer)
    lat = Column(Integer)

    max_lon = Column(Integer)
    min_lon = Column(Integer)
    lon = Column(Integer)
    # mapped via RADIO_TYPE
    radio = Column(SmallInteger)
    # int in the range 0-1000
    mcc = Column(SmallInteger)
    # int in the range 0-1000 for gsm
    # int in the range 0-32767 for cdma (system id)
    mnc = Column(Integer)
    lac = Column(Integer)
    cid = Column(Integer)
    psc = Column(Integer)
    range = Column(Integer)
    new_measures = Column(Integer(unsigned=True))
    total_measures = Column(Integer(unsigned=True))

    def __init__(self, *args, **kw):
        if 'created' not in kw:
            kw['created'] = datetime.datetime.utcnow()
        if 'lac' not in kw:
            kw['lac'] = -1
        if 'cid' not in kw:
            kw['cid'] = -1
        if 'new_measures' not in kw:
            kw['new_measures'] = 0
        if 'total_measures' not in kw:
            kw['total_measures'] = 0
        super(Cell, self).__init__(*args, **kw)

cell_table = Cell.__table__

class CellMeasure(_Model):
    __tablename__ = 'cell_measure'
    __table_args__ = (
        Index('cell_measure_created_idx', 'created'),
        Index('cell_measure_key_idx', 'radio', 'mcc', 'mnc', 'lac', 'cid'),
        {
            'mysql_engine': 'InnoDB',
            'mysql_charset': 'utf8',
            'mysql_row_format': 'compressed',
            'mysql_key_block_size': '4',
        }
    )

    id = Column(BigInteger(unsigned=True),
                primary_key=True, autoincrement=True)
    measure_id = Column(BigInteger(unsigned=True))
    created = Column(DateTime)  # the insert time of the record into the DB
    # lat/lon * decimaljson.FACTOR
    lat = Column(Integer)
    lon = Column(Integer)
    time = Column(DateTime)  # the time of observation of this data
    accuracy = Column(Integer)
    altitude = Column(Integer)
    altitude_accuracy = Column(Integer)
    # mapped via RADIO_TYPE
    radio = Column(SmallInteger)
    mcc = Column(SmallInteger)
    mnc = Column(Integer)
    lac = Column(Integer)
    cid = Column(Integer)
    psc = Column(Integer)
    asu = Column(SmallInteger)
    signal = Column(SmallInteger)
    ta = Column(SmallInteger)

    def __init__(self, *args, **kw):
        if 'measure_id' not in kw:
            kw['measure_id'] = 0
        if 'created' not in kw:
            kw['created'] = datetime.datetime.utcnow()
        super(CellMeasure, self).__init__(*args, **kw)

cell_measure_table = CellMeasure.__table__

class Wifi(_Model):
    __tablename__ = 'wifi'
    __table_args__ = (
        UniqueConstraint('key', name='wifi_key_unique'),
        Index('wifi_created_idx', 'created'),
        Index('wifi_new_measures_idx', 'new_measures'),
        {
            'mysql_engine': 'InnoDB',
            'mysql_charset': 'utf8',
        }
    )

    id = Column(BigInteger(unsigned=True),
                primary_key=True, autoincrement=True)
    created = Column(DateTime)
    key = Column(String(12))
    # lat/lon * decimaljson.FACTOR
    lat = Column(Integer)
    lon = Column(Integer)
    range = Column(Integer)
    new_measures = Column(Integer(unsigned=True))
    total_measures = Column(Integer(unsigned=True))

    def __init__(self, *args, **kw):
        if 'created' not in kw:
            kw['created'] = datetime.datetime.utcnow()
        if 'new_measures' not in kw:
            kw['new_measures'] = 0
        if 'total_measures' not in kw:
            kw['total_measures'] = 0
        super(Wifi, self).__init__(*args, **kw)

wifi_table = Wifi.__table__

class WifiBlacklist(_Model):
    __tablename__ = 'wifi_blacklist'
    __table_args__ = (
        UniqueConstraint('key', name='wifi_blacklist_key_unique'),
        {
            'mysql_engine': 'InnoDB',
            'mysql_charset': 'utf8',
        }
    )
    id = Column(BigInteger(unsigned=True),
                primary_key=True, autoincrement=True)
    created = Column(DateTime)
    key = Column(String(12))

    def __init__(self, *args, **kw):
        if 'created' not in kw:
            kw['created'] = datetime.datetime.utcnow()
        super(WifiBlacklist, self).__init__(*args, **kw)

class WifiMeasure(_Model):
    __tablename__ = 'wifi_measure'
    __table_args__ = (
        Index('wifi_measure_created_idx', 'created'),
        Index('wifi_measure_key_idx', 'key'),
        {
            'mysql_engine': 'InnoDB',
            'mysql_charset': 'utf8',
            'mysql_row_format': 'compressed',
            'mysql_key_block_size': '4',
        }
    )

    id = Column(BigInteger(unsigned=True),
                primary_key=True, autoincrement=True)
    measure_id = Column(BigInteger(unsigned=True))
    created = Column(DateTime)  # the insert time of the record into the DB
    # lat/lon * decimaljson.FACTOR
    lat = Column(Integer)
    lon = Column(Integer)
    time = Column(DateTime)  # the time of observation of this data
    accuracy = Column(Integer)
    altitude = Column(Integer)
    altitude_accuracy = Column(Integer)
    key = Column(String(12))
    channel = Column(SmallInteger)
    signal = Column(SmallInteger)

    def __init__(self, *args, **kw):
        if 'measure_id' not in kw:
            kw['measure_id'] = 0
        if 'created' not in kw:
            kw['created'] = datetime.datetime.utcnow()
        super(WifiMeasure, self).__init__(*args, **kw)

wifi_measure_table = WifiMeasure.__table__

class Measure(_Model):
    __tablename__ = 'measure'
    __table_args__ = (
        Index('measure_created_idx', 'created'),
        {
            'mysql_engine': 'InnoDB',
            'mysql_charset': 'utf8',
            'mysql_row_format': 'compressed',
            'mysql_key_block_size': '4',
        }
    )

    id = Column(BigInteger(unsigned=True),
                primary_key=True, autoincrement=True)
    created = Column(DateTime)  # the insert time of the record into the DB
    # lat/lon * decimaljson.FACTOR
    lat = Column(Integer)
    lon = Column(Integer)
    time = Column(DateTime)  # the time of observation of this data
    accuracy = Column(Integer)
    altitude = Column(Integer)
    altitude_accuracy = Column(Integer)
    radio = Column(SmallInteger)  # mapped via RADIO_TYPE
    # json blobs
    cell = Column(LargeBinary)
    wifi = Column(LargeBinary)

    def __init__(self, *args, **kw):
        if 'created' not in kw:
            kw['created'] = datetime.datetime.utcnow()
        super(Measure, self).__init__(*args, **kw)

measure_table = Measure.__table__
<|endoftext|>