<commit_before># coding:utf8

import os
from time import sleep
import sys

from kivy import platform
from kivy.logger import Logger
from kivy.config import Config
from kivy.lib import osc

import logging
logging.captureWarnings(True)

parent_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
parent_path += '\\' if platform == 'win' else '/'
os.sys.path.append(parent_path)

import bot.utils
if platform != 'android':
    bot.utils.PATH = parent_path + bot.utils.PATH
bot.utils.DATA_PATH = parent_path + bot.utils.DATA_PATH

bot.utils.update_paths()

from bot.core import LongPollSession

def ping(*args):
    global session
    if session.running:
        osc.sendMsg('/pong', [], port=3002)

def send_text(text):
    osc.sendMsg('/text', [text, ], port=3002)

def send_status(status):
    osc.sendMsg('/status', [status, ], port=3002)

def send_error(error):
    osc.sendMsg('/error', [error, ], port=3002)
    Logger.info(unicode(error))

def send_answers_count():
    osc.sendMsg('/answers', [session.reply_count, ], port=3002)

def exit(*args):
    send_status('exiting')
    session.runtime_error = 1
    sys.exit()

if __name__ == '__main__':
    osc.init()
    oscid = osc.listen(ipAddr='0.0.0.0', port=3000)
    send_status('connected')

    osc.bind(oscid, ping, '/ping')
    osc.bind(oscid, exit, '/exit')

    session = LongPollSession()
    authorized, error = session.authorization()

    if error:
        send_error(error)
        exit()

    Config.read(bot.utils.SETTINGS_FILE_PATH)
    appials = Config.get('General', 'appeals')
    activated = Config.get('General', 'bot_activated')
    use_custom_commands = Config.get('General', 'use_custom_commands')
    protect_custom_commands = Config.get('General', 'protect_cc')

    session.load_params(
    	            appials,
    	            activated=activated == 'True',
                use_custom_commands=use_custom_commands == 'True',
                protect_custom_commands=protect_custom_commands == 'True'
                )
    session.launch_bot()

    send_status('launched')

    while True:
        osc.readQueue(oscid)
        send_answers_count()
        if session.activated != activated:
            activated = session.activated
            Config.read(bot.utils.SETTINGS_FILE_PATH)
            Config.set('General', 'bot_activated', str(activated))
            Config.write()
        if session.runtime_error:
            if session.runtime_error != 1:
                send_error(session.runtime_error)
            break
        if not session.running:
            break
        sleep(1)
    exit()
<commit_msg>Добавлена обработка ошибок при запуске бота, улучшена обработка ошибок <commit_after># coding:utf8

import os
from time import sleep
import sys

from kivy import platform
from kivy.logger import Logger
from kivy.config import Config
from kivy.lib import osc

import logging
logging.captureWarnings(True)

parent_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
parent_path += '\\' if platform == 'win' else '/'
os.sys.path.append(parent_path)

import bot.utils
if platform != 'android':
    bot.utils.PATH = parent_path + bot.utils.PATH
bot.utils.DATA_PATH = parent_path + bot.utils.DATA_PATH

bot.utils.update_paths()

from bot.core import LongPollSession

def ping(*args):
    global session
    if session.running:
        osc.sendMsg('/pong', [], port=3002)

def send_text(text):
    osc.sendMsg('/text', [text, ], port=3002)

def send_status(status):
    osc.sendMsg('/status', [status, ], port=3002)

def send_error(error):
    osc.sendMsg('/error', [error, ], port=3002)

def send_answers_count():
    osc.sendMsg('/answers', [session.reply_count, ], port=3002)

def exit():
    send_status('exiting')
    sys.exit()

if __name__ == '__main__':
    osc.init()
    oscid = osc.listen(ipAddr='0.0.0.0', port=3000)
    send_status('connected')

    osc.bind(oscid, ping, '/ping')
    osc.bind(oscid, exit, '/exit')

    try:
        session = LongPollSession()
        authorized, error = session.authorization()

        if error:
            send_error(error)
            exit()

        Config.read(bot.utils.SETTINGS_FILE_PATH)
        appials = Config.get('General', 'appeals')
        activated = Config.get('General', 'bot_activated')
        use_custom_commands = Config.get('General', 'use_custom_commands')
        protect_custom_commands = Config.get('General', 'protect_cc')

        session.load_params(
    	            appials,
    	            activated=activated == 'True',
                use_custom_commands=use_custom_commands == 'True',
                protect_custom_commands=protect_custom_commands == 'True'
                )
        session.launch_bot()

        send_status('launched')
    except SystemExit:
        raise
    except:
        import traceback
        error = traceback.format_exc()
        send_error(error)
        exit()

    while True:
        osc.readQueue(oscid)
        send_answers_count()
        if session.activated != activated:
            activated = session.activated
            Config.read(bot.utils.SETTINGS_FILE_PATH)
            Config.set('General', 'bot_activated', str(activated))
            Config.write()
        if session.runtime_error:
            if session.runtime_error != 1:
                send_error(session.runtime_error)
            break
        if not session.running:
            break
        sleep(1)
    exit()
<|endoftext|>


<commit_before>import json
import re

from django.conf import settings
from django.utils import translation
from django.db.models import Q
from django.contrib.gis.geos import Polygon, MultiPolygon, GeometryCollection
from django.contrib.gis.db.models.fields import GeometryField
from django.contrib.gis.gdal import CoordTransform
from modeltranslation.translator import translator, NotRegistered
from rest_framework import serializers, viewsets, generics
from rest_framework.response import Response
from rest_framework.exceptions import ParseError

from haystack.query import SearchQuerySet
from haystack.inputs import AutoQuery

from services.models import *
from munigeo.models import *
from munigeo.api import AdministrativeDivisionSerializer, GeoModelSerializer, \
    GeoModelAPIView

# This allows us to find a serializer for Haystack search results
serializers_by_model = {}

all_views = []
def register_view(klass, name, base_name=None):
    entry = {'class': klass, 'name': name}
    if base_name is not None:
        entry['base_name'] = base_name
    all_views.append(entry)

    if klass.serializer_class and hasattr(klass.serializer_class.Meta, 'model'):
        model = klass.serializer_class.Meta.model
        serializers_by_model[model] = klass.serializer_class

LANGUAGES = [x[0] for x in settings.LANGUAGES]

class MPTTModelSerializer(serializers.ModelSerializer):
    def __init__(self, *args, **kwargs):
        super(MPTTModelSerializer, self).__init__(*args, **kwargs)
        for field_name in 'lft', 'rght', 'tree_id':
            if field_name in self.fields:
                del self.fields[field_name]

class TranslatedModelSerializer(serializers.ModelSerializer):
    def __init__(self, *args, **kwargs):
        super(TranslatedModelSerializer, self).__init__(*args, **kwargs)
        model = self.opts.model
        try:
            trans_opts = translator.get_options_for_model(model)
        except NotRegistered:
            self.translated_fields = []
            return

        self.translated_fields = trans_opts.fields.keys()
        # Remove the pre-existing data in the bundle.
        for field_name in self.translated_fields:
            for lang in LANGUAGES:
                key = "%s_%s" % (field_name, lang)
                if key in self.fields:
                    del self.fields[key]

    def to_native(self, obj):
        ret = super(TranslatedModelSerializer, self).to_native(obj)
        if obj is None:
            return ret

        for field_name in self.translated_fields:
            if not field_name in self.fields:
                continue
            d = {}
            for lang in LANGUAGES:
                key = "%s_%s" % (field_name, lang)
                val = getattr(obj, key, None)
                if val == None:
                    continue
                d[lang] = val

            # If no text provided, leave the field as null
            for key, val in d.items():
                if val != None:
                    break
            else:
                d = None
            ret[field_name] = d

        return ret

class OrganizationSerializer(TranslatedModelSerializer):
    class Meta:
        model = Organization

class OrganizationViewSet(viewsets.ReadOnlyModelViewSet):
    queryset = Organization.objects.all()
    serializer_class = OrganizationSerializer

register_view(OrganizationViewSet, 'organization')

class DepartmentSerializer(TranslatedModelSerializer):
    class Meta:
        model = Department

class DepartmentViewSet(viewsets.ReadOnlyModelViewSet):
    queryset = Department.objects.all()
    serializer_class = DepartmentSerializer

register_view(DepartmentViewSet, 'department')

def root_services(services):
    tree_ids = set(s.tree_id for s in services)
    return map(lambda x: x.id,
               Service.objects.filter(level=0).filter(
                   tree_id__in=tree_ids))

class ServiceSerializer(TranslatedModelSerializer, MPTTModelSerializer):
    children = serializers.PrimaryKeyRelatedField(many=True)

    def __init__(self, *args, **kwargs):
        super(ServiceSerializer, self).__init__(*args, **kwargs)
        self.fields['root'] = serializers.SerializerMethodField('root_services')

    def root_services(self, obj):
        return next(root_services([obj]))

    class Meta:
        model = Service

class ServiceViewSet(viewsets.ReadOnlyModelViewSet):
    queryset = Service.objects.all()
    serializer_class = ServiceSerializer
    filter_fields = ['level', 'parent']

    def get_queryset(self):
        queryset = super(ServiceViewSet, self).get_queryset()
        args = self.request.QUERY_PARAMS
        if 'ancestor' in args:
            val = args['ancestor']
            queryset = queryset.by_ancestor(val)
        return queryset

register_view(ServiceViewSet, 'service')

class JSONAPISerializer(serializers.ModelSerializer):
    def __init__(self, *args, **kwargs):
        super(JSONAPISerializer, self).__init__(*args, **kwargs)
        context = kwargs.get('context', {})
        if 'only' in context:
            keep_fields = set(context['only'] + ['id'])
            for field_name in list(self.fields.keys()):
                if field_name in keep_fields:
                    continue
                del self.fields[field_name]

class JSONAPIViewSet(viewsets.ReadOnlyModelViewSet):
    def initial(self, request, *args, **kwargs):
        ret = super(JSONAPIViewSet, self).initial(request, *args, **kwargs)

        include = self.request.QUERY_PARAMS.get('include', '')
        self.include_fields = [x.strip() for x in include.split(',') if x]

        only = self.request.QUERY_PARAMS.get('only', '')
        if only:
            self.only_fields = [x.strip() for x in only.split(',') if x]
        else:
            self.only_fields = None

        return ret

    def get_queryset(self):
        queryset = super(JSONAPIViewSet, self).get_queryset()
        model = queryset.model
        if self.only_fields:
            model_fields = model._meta.fields
            # Verify all field names are valid
            for field_name in self.only_fields:
                for field in model_fields:
                    if field.name == field_name:
                        break
                else:
                    raise ParseError("field '%s' supplied in 'only' not found" % field_name)
            queryset = queryset.only(*self.only_fields)
        return queryset

    def get_serializer_context(self):
        context = super(JSONAPIViewSet, self).get_serializer_context()

        context['include'] = self.include_fields
        if self.only_fields:
            context['only'] = self.only_fields

        return context

class UnitConnectionSerializer(TranslatedModelSerializer):
    class Meta:
        model = UnitConnection

class UnitConnectionViewSet(viewsets.ReadOnlyModelViewSet):
    queryset = UnitConnection.objects.all()
    serializer_class = UnitConnectionSerializer

register_view(UnitConnectionViewSet, 'unit_connection')

class UnitSerializer(TranslatedModelSerializer, MPTTModelSerializer, GeoModelSerializer,
                     JSONAPISerializer):
    connections = UnitConnectionSerializer(many=True)

    def __init__(self, *args, **kwargs):
        super(UnitSerializer, self).__init__(*args, **kwargs)
        self.fields['root_services'] = serializers.SerializerMethodField('root_services')

    def root_services(self, obj):
        return root_services(obj.services.all())

    class Meta:
        model = Unit

def make_muni_ocd_id(name, rest=None):
    s = 'ocd-division/country:%s/%s:%s' % (settings.DEFAULT_COUNTRY, settings.DEFAULT_OCD_MUNICIPALITY, name)
    if rest:
        s += '/' + rest
    return s

class UnitViewSet(GeoModelAPIView, JSONAPIViewSet, viewsets.ReadOnlyModelViewSet):
    queryset = Unit.objects.all()
    serializer_class = UnitSerializer
    filter_fields = ['services']

    def get_serializer_context(self):
        ret = super(UnitViewSet, self).get_serializer_context()
        ret['srs'] = self.srs
        return ret

    def get_queryset(self):
        queryset = super(UnitViewSet, self).get_queryset()
        filters = self.request.QUERY_PARAMS
        if 'municipality' in filters:
            val = filters['municipality'].lower()
            if val.startswith('ocd-division'):
                ocd_id = val
            else:
                ocd_id = make_muni_ocd_id(val)
            try:
                muni = Municipality.objects.get(ocd_id=ocd_id)
            except Municipality.DoesNotExist:
                raise ParseError("municipality with ID '%s' not found" % ocd_id)

            queryset = queryset.filter(location__within=muni.geometry.boundary)

        if 'provider_type' in filters:
            val = filters.get('provider_type')
            pr_ids = val.split(',')
            queryset = queryset.filter(provider_type__in=pr_ids)

        val = filters.get('service', '').lower()
        if val:
            query = Q()
            for srv_id in val.split(','):
                srv_list = Service.objects.all().by_ancestor(srv_id)
                query |= Q(services__in=srv_list)
                query |= Q(services=srv_id)
            queryset = queryset.filter(query).distinct()

        if 'division' in filters:
            # Divisions can be specified with form:
            # division=helsinki/kaupunginosa:kallio,vantaa/äänestysalue:5
            d_list = filters['division'].lower().split(',')
            div_list = []
            for division_path in d_list:
                if division_path.startswith('ocd-division'):
                    muni_ocd_id = division_path
                else:
                    ocd_id_base = r'[\w0-9~_.-]+'
                    match_re = r'(%s)/([\w_-]+):(%s)' % (ocd_id_base, ocd_id_base)
                    m = re.match(match_re, division_path, re.U)
                    if not m:
                        raise ParseError("'division' must be of form 'muni/type:id'")

                    arr = division_path.split('/')
                    muni_ocd_id = make_muni_ocd_id(arr.pop(0), '/'.join(arr))
                try:
                    div = AdministrativeDivision.objects.select_related('geometry').get(ocd_id=muni_ocd_id)
                except AdministrativeDivision.DoesNotExist:
                    raise ParseError("administrative division with OCD ID '%s' not found" % muni_ocd_id)
                div_list.append(div)

            div_geom = [div.geometry.boundary for div in div_list]
            if div_list:
                mp = div_list.pop(0).geometry.boundary
                for div in div_list:
                    mp += div.geometry.boundary

            queryset = queryset.filter(location__within=mp)

        return queryset

register_view(UnitViewSet, 'unit')

class SearchSerializer(serializers.Serializer):
    def to_native(self, search_result):
        model = search_result.model
        assert model in serializers_by_model, "Serializer for %s not found" % model
        ser_class = serializers_by_model[model]
        data = ser_class(search_result.object, context=self.context).data
        data['object_type'] = model._meta.model_name
        data['score'] = search_result.score
        return data

class SearchViewSet(GeoModelAPIView, viewsets.ViewSetMixin, generics.ListAPIView):
    serializer_class = SearchSerializer

    def list(self, request, *args, **kwargs):
        # If the incoming language is not specified, go with the default.
        self.lang_code = request.QUERY_PARAMS.get('language', LANGUAGES[0])
        if self.lang_code not in LANGUAGES:
            raise ParseError("Invalid language supplied. Supported languages: %s" %
                             ','.join(LANGUAGES))

        input_val = request.QUERY_PARAMS.get('input', '').strip()
        q_val = request.QUERY_PARAMS.get('q', '').strip()
        if not input_val and not q_val:
            raise ParseError("Supply search terms with 'q=' or autocomplete entry with 'input='")
        if input_val and q_val:
            raise ParseError("Supply either 'q' or 'input', not both")

        old_language = translation.get_language()[:2]
        translation.activate(self.lang_code)

        queryset = SearchQuerySet()
        if input_val:
            queryset = queryset.filter(autosuggest=input_val)
        else:
            queryset = queryset.filter(text=AutoQuery(q_val))

        self.object_list = queryset.load_all()

        # Switch between paginated or standard style responses
        page = self.paginate_queryset(self.object_list)
        if page is not None:
            serializer = self.get_pagination_serializer(page)
        else:
            serializer = self.get_serializer(self.object_list, many=True)

        resp = Response(serializer.data)

        translation.activate(old_language)

        return resp

    """
    def list(self, request):
        resp = []
        context = self.get_serializer_context()

        filter_name = "name_%s__icontains" % lang_code
        obj_hits = Service.objects.filter(**{filter_name: val})
        klass = ServiceSerializer
        for obj in obj_hits[0:5]:
            ser_obj = klass(obj, context=context).data
            ser_obj['object_type'] = 'service'
            resp.append(ser_obj)

        filter_name = "name_%s__icontains" % lang_code
        obj_hits = Unit.objects.filter(**{filter_name: val})
        klass = UnitSerializer
        for obj in obj_hits[0:5]:
            ser_obj = klass(obj, context=context).data
            ser_obj['object_type'] = 'unit'
            resp.append(ser_obj)

        filter_name = "name_%s__icontains" % lang_code
        ad_types = ['neighborhood', 'district', 'sub-district']
        obj_hits = AdministrativeDivision.objects.filter(type__type__in=ad_types).\
            filter(**{filter_name: val})
        klass = AdministrativeDivisionSerializer
        for obj in obj_hits[0:5]:
            ser_obj = klass(obj, context=context).data
            ser_obj['object_type'] = 'administrative_division'
            resp.append(ser_obj)

        return Response(resp)
    """

register_view(SearchViewSet, 'search', base_name='search')
<commit_msg>Improve root_services performance<commit_after>import json
import re

from django.conf import settings
from django.utils import translation
from django.db.models import Q
from django.contrib.gis.geos import Polygon, MultiPolygon, GeometryCollection
from django.contrib.gis.db.models.fields import GeometryField
from django.contrib.gis.gdal import CoordTransform
from modeltranslation.translator import translator, NotRegistered
from rest_framework import serializers, viewsets, generics
from rest_framework.response import Response
from rest_framework.exceptions import ParseError

from haystack.query import SearchQuerySet
from haystack.inputs import AutoQuery

from services.models import *
from munigeo.models import *
from munigeo.api import AdministrativeDivisionSerializer, GeoModelSerializer, \
    GeoModelAPIView

# This allows us to find a serializer for Haystack search results
serializers_by_model = {}

all_views = []
def register_view(klass, name, base_name=None):
    entry = {'class': klass, 'name': name}
    if base_name is not None:
        entry['base_name'] = base_name
    all_views.append(entry)

    if klass.serializer_class and hasattr(klass.serializer_class.Meta, 'model'):
        model = klass.serializer_class.Meta.model
        serializers_by_model[model] = klass.serializer_class

LANGUAGES = [x[0] for x in settings.LANGUAGES]

class MPTTModelSerializer(serializers.ModelSerializer):
    def __init__(self, *args, **kwargs):
        super(MPTTModelSerializer, self).__init__(*args, **kwargs)
        for field_name in 'lft', 'rght', 'tree_id':
            if field_name in self.fields:
                del self.fields[field_name]

class TranslatedModelSerializer(serializers.ModelSerializer):
    def __init__(self, *args, **kwargs):
        super(TranslatedModelSerializer, self).__init__(*args, **kwargs)
        model = self.opts.model
        try:
            trans_opts = translator.get_options_for_model(model)
        except NotRegistered:
            self.translated_fields = []
            return

        self.translated_fields = trans_opts.fields.keys()
        # Remove the pre-existing data in the bundle.
        for field_name in self.translated_fields:
            for lang in LANGUAGES:
                key = "%s_%s" % (field_name, lang)
                if key in self.fields:
                    del self.fields[key]

    def to_native(self, obj):
        ret = super(TranslatedModelSerializer, self).to_native(obj)
        if obj is None:
            return ret

        for field_name in self.translated_fields:
            if not field_name in self.fields:
                continue
            d = {}
            for lang in LANGUAGES:
                key = "%s_%s" % (field_name, lang)
                val = getattr(obj, key, None)
                if val == None:
                    continue
                d[lang] = val

            # If no text provided, leave the field as null
            for key, val in d.items():
                if val != None:
                    break
            else:
                d = None
            ret[field_name] = d

        return ret

class OrganizationSerializer(TranslatedModelSerializer):
    class Meta:
        model = Organization

class OrganizationViewSet(viewsets.ReadOnlyModelViewSet):
    queryset = Organization.objects.all()
    serializer_class = OrganizationSerializer

register_view(OrganizationViewSet, 'organization')

class DepartmentSerializer(TranslatedModelSerializer):
    class Meta:
        model = Department

class DepartmentViewSet(viewsets.ReadOnlyModelViewSet):
    queryset = Department.objects.all()
    serializer_class = DepartmentSerializer

register_view(DepartmentViewSet, 'department')

def root_services(services):
    tree_ids = set(s.tree_id for s in services)
    return map(lambda x: x.id,
               Service.objects.filter(level=0).filter(
                   tree_id__in=tree_ids))

class ServiceSerializer(TranslatedModelSerializer, MPTTModelSerializer):
    children = serializers.PrimaryKeyRelatedField(many=True)

    def __init__(self, *args, **kwargs):
        super(ServiceSerializer, self).__init__(*args, **kwargs)
        self.fields['root'] = serializers.SerializerMethodField('root_services')

    def root_services(self, obj):
        return next(root_services([obj]))

    class Meta:
        model = Service

class ServiceViewSet(viewsets.ReadOnlyModelViewSet):
    queryset = Service.objects.all()
    serializer_class = ServiceSerializer
    filter_fields = ['level', 'parent']

    def get_queryset(self):
        queryset = super(ServiceViewSet, self).get_queryset()
        args = self.request.QUERY_PARAMS
        if 'ancestor' in args:
            val = args['ancestor']
            queryset = queryset.by_ancestor(val)
        return queryset

register_view(ServiceViewSet, 'service')

class JSONAPISerializer(serializers.ModelSerializer):
    def __init__(self, *args, **kwargs):
        super(JSONAPISerializer, self).__init__(*args, **kwargs)
        context = kwargs.get('context', {})
        if 'only' in context:
            keep_fields = set(context['only'] + ['id'])
            for field_name in list(self.fields.keys()):
                if field_name in keep_fields:
                    continue
                del self.fields[field_name]

class JSONAPIViewSet(viewsets.ReadOnlyModelViewSet):
    def initial(self, request, *args, **kwargs):
        ret = super(JSONAPIViewSet, self).initial(request, *args, **kwargs)

        include = self.request.QUERY_PARAMS.get('include', '')
        self.include_fields = [x.strip() for x in include.split(',') if x]

        only = self.request.QUERY_PARAMS.get('only', '')
        if only:
            self.only_fields = [x.strip() for x in only.split(',') if x]
        else:
            self.only_fields = None

        return ret

    def get_queryset(self):
        queryset = super(JSONAPIViewSet, self).get_queryset()
        model = queryset.model
        if self.only_fields:
            model_fields = model._meta.fields
            # Verify all field names are valid
            for field_name in self.only_fields:
                for field in model_fields:
                    if field.name == field_name:
                        break
                else:
                    raise ParseError("field '%s' supplied in 'only' not found" % field_name)
            queryset = queryset.only(*self.only_fields)
        return queryset

    def get_serializer_context(self):
        context = super(JSONAPIViewSet, self).get_serializer_context()

        context['include'] = self.include_fields
        if self.only_fields:
            context['only'] = self.only_fields

        return context

class UnitConnectionSerializer(TranslatedModelSerializer):
    class Meta:
        model = UnitConnection

class UnitConnectionViewSet(viewsets.ReadOnlyModelViewSet):
    queryset = UnitConnection.objects.all()
    serializer_class = UnitConnectionSerializer

register_view(UnitConnectionViewSet, 'unit_connection')

class UnitSerializer(TranslatedModelSerializer, MPTTModelSerializer, GeoModelSerializer,
                     JSONAPISerializer):
    connections = UnitConnectionSerializer(many=True)
    root_services = serializers.SerializerMethodField('root_services')

    def __init__(self, *args, **kwargs):
        super(UnitSerializer, self).__init__(*args, **kwargs)

    def root_services(self, obj):
        return root_services(obj.services.all())

    class Meta:
        model = Unit

def make_muni_ocd_id(name, rest=None):
    s = 'ocd-division/country:%s/%s:%s' % (settings.DEFAULT_COUNTRY, settings.DEFAULT_OCD_MUNICIPALITY, name)
    if rest:
        s += '/' + rest
    return s

class UnitViewSet(GeoModelAPIView, JSONAPIViewSet, viewsets.ReadOnlyModelViewSet):
    queryset = Unit.objects.all()
    serializer_class = UnitSerializer
    filter_fields = ['services']

    def get_serializer_context(self):
        ret = super(UnitViewSet, self).get_serializer_context()
        ret['srs'] = self.srs
        return ret

    def get_queryset(self):
        queryset = super(UnitViewSet, self).get_queryset()
        filters = self.request.QUERY_PARAMS
        if 'municipality' in filters:
            val = filters['municipality'].lower()
            if val.startswith('ocd-division'):
                ocd_id = val
            else:
                ocd_id = make_muni_ocd_id(val)
            try:
                muni = Municipality.objects.get(ocd_id=ocd_id)
            except Municipality.DoesNotExist:
                raise ParseError("municipality with ID '%s' not found" % ocd_id)

            queryset = queryset.filter(location__within=muni.geometry.boundary)

        if 'provider_type' in filters:
            val = filters.get('provider_type')
            pr_ids = val.split(',')
            queryset = queryset.filter(provider_type__in=pr_ids)

        val = filters.get('service', '').lower()
        if val:
            query = Q()
            for srv_id in val.split(','):
                srv_list = Service.objects.all().by_ancestor(srv_id)
                query |= Q(services__in=srv_list)
                query |= Q(services=srv_id)
            queryset = queryset.filter(query).distinct()

        if 'division' in filters:
            # Divisions can be specified with form:
            # division=helsinki/kaupunginosa:kallio,vantaa/äänestysalue:5
            d_list = filters['division'].lower().split(',')
            div_list = []
            for division_path in d_list:
                if division_path.startswith('ocd-division'):
                    muni_ocd_id = division_path
                else:
                    ocd_id_base = r'[\w0-9~_.-]+'
                    match_re = r'(%s)/([\w_-]+):(%s)' % (ocd_id_base, ocd_id_base)
                    m = re.match(match_re, division_path, re.U)
                    if not m:
                        raise ParseError("'division' must be of form 'muni/type:id'")

                    arr = division_path.split('/')
                    muni_ocd_id = make_muni_ocd_id(arr.pop(0), '/'.join(arr))
                try:
                    div = AdministrativeDivision.objects.select_related('geometry').get(ocd_id=muni_ocd_id)
                except AdministrativeDivision.DoesNotExist:
                    raise ParseError("administrative division with OCD ID '%s' not found" % muni_ocd_id)
                div_list.append(div)

            div_geom = [div.geometry.boundary for div in div_list]
            if div_list:
                mp = div_list.pop(0).geometry.boundary
                for div in div_list:
                    mp += div.geometry.boundary

            queryset = queryset.filter(location__within=mp)

        return queryset

register_view(UnitViewSet, 'unit')

class SearchSerializer(serializers.Serializer):
    def to_native(self, search_result):
        model = search_result.model
        assert model in serializers_by_model, "Serializer for %s not found" % model
        ser_class = serializers_by_model[model]
        data = ser_class(search_result.object, context=self.context).data
        data['object_type'] = model._meta.model_name
        data['score'] = search_result.score
        return data

class SearchViewSet(GeoModelAPIView, viewsets.ViewSetMixin, generics.ListAPIView):
    serializer_class = SearchSerializer

    def list(self, request, *args, **kwargs):
        # If the incoming language is not specified, go with the default.
        self.lang_code = request.QUERY_PARAMS.get('language', LANGUAGES[0])
        if self.lang_code not in LANGUAGES:
            raise ParseError("Invalid language supplied. Supported languages: %s" %
                             ','.join(LANGUAGES))

        input_val = request.QUERY_PARAMS.get('input', '').strip()
        q_val = request.QUERY_PARAMS.get('q', '').strip()
        if not input_val and not q_val:
            raise ParseError("Supply search terms with 'q=' or autocomplete entry with 'input='")
        if input_val and q_val:
            raise ParseError("Supply either 'q' or 'input', not both")

        old_language = translation.get_language()[:2]
        translation.activate(self.lang_code)

        queryset = SearchQuerySet()
        if input_val:
            queryset = queryset.filter(autosuggest=input_val)
        else:
            queryset = queryset.filter(text=AutoQuery(q_val))

        self.object_list = queryset.load_all()

        # Switch between paginated or standard style responses
        page = self.paginate_queryset(self.object_list)
        if page is not None:
            serializer = self.get_pagination_serializer(page)
        else:
            serializer = self.get_serializer(self.object_list, many=True)

        resp = Response(serializer.data)

        translation.activate(old_language)

        return resp

    """
    def list(self, request):
        resp = []
        context = self.get_serializer_context()

        filter_name = "name_%s__icontains" % lang_code
        obj_hits = Service.objects.filter(**{filter_name: val})
        klass = ServiceSerializer
        for obj in obj_hits[0:5]:
            ser_obj = klass(obj, context=context).data
            ser_obj['object_type'] = 'service'
            resp.append(ser_obj)

        filter_name = "name_%s__icontains" % lang_code
        obj_hits = Unit.objects.filter(**{filter_name: val})
        klass = UnitSerializer
        for obj in obj_hits[0:5]:
            ser_obj = klass(obj, context=context).data
            ser_obj['object_type'] = 'unit'
            resp.append(ser_obj)

        filter_name = "name_%s__icontains" % lang_code
        ad_types = ['neighborhood', 'district', 'sub-district']
        obj_hits = AdministrativeDivision.objects.filter(type__type__in=ad_types).\
            filter(**{filter_name: val})
        klass = AdministrativeDivisionSerializer
        for obj in obj_hits[0:5]:
            ser_obj = klass(obj, context=context).data
            ser_obj['object_type'] = 'administrative_division'
            resp.append(ser_obj)

        return Response(resp)
    """

register_view(SearchViewSet, 'search', base_name='search')
<|endoftext|>


<commit_before>"""
These settings are used by the ``manage.py`` command.

"""
DEBUG = True

SITE_ID = 1

USE_TZ = True
TIME_ZONE = 'UTC'

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': 'db.sqlite',
    }
}

EXTERNAL_APPS = [
    # Django core apps
    #'django_admin_bootstrapped',
    'django.contrib.admin',
    'django.contrib.admindocs',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.messages',
    'django.contrib.sessions',
    'django.contrib.staticfiles',

    # Required apps
    'international',
    'django_fsm',
    'rest_framework',
    'django_filters',
    'rest_hooks',

    # Dev tools
    'django_extensions',
]

INTERNAL_APPS = [
    'silver',
]

INSTALLED_APPS = EXTERNAL_APPS + INTERNAL_APPS

ROOT_URLCONF = 'silver.urls'
STATIC_URL = '/static/'

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

SECRET_KEY = 'secret'

REST_FRAMEWORK = {
    'PAGINATE_BY': 30,
    'PAGINATE_BY_PARAM': 'per_page',
    'MAX_PAGINATE_BY': 100,
}

from django.contrib import messages

MESSAGE_TAGS = {
    messages.SUCCESS: 'alert-success success',
    messages.WARNING: 'alert-warning warning',
    messages.ERROR: 'alert-danger error'
}

from silver import HOOK_EVENTS as _HOOK_EVENTS
HOOK_EVENTS = _HOOK_EVENTS

# DEFAULT_TARGET_URL = 'http://presslabs.com/api'
<commit_msg>Removed unused settings code<commit_after>"""
These settings are used by the ``manage.py`` command.

"""
DEBUG = True

SITE_ID = 1

USE_TZ = True
TIME_ZONE = 'UTC'

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': 'db.sqlite',
    }
}

EXTERNAL_APPS = [
    # Django core apps
    #'django_admin_bootstrapped',
    'django.contrib.admin',
    'django.contrib.admindocs',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.messages',
    'django.contrib.sessions',
    'django.contrib.staticfiles',

    # Required apps
    'international',
    'django_fsm',
    'rest_framework',
    'django_filters',
    'rest_hooks',

    # Dev tools
    'django_extensions',
]

INTERNAL_APPS = [
    'silver',
]

INSTALLED_APPS = EXTERNAL_APPS + INTERNAL_APPS

ROOT_URLCONF = 'silver.urls'
STATIC_URL = '/static/'

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

SECRET_KEY = 'secret'

REST_FRAMEWORK = {
    'PAGINATE_BY': 30,
    'PAGINATE_BY_PARAM': 'per_page',
    'MAX_PAGINATE_BY': 100,
}

from silver import HOOK_EVENTS as _HOOK_EVENTS
HOOK_EVENTS = _HOOK_EVENTS

# DEFAULT_TARGET_URL = 'http://presslabs.com/api'
<|endoftext|>


<commit_before>import logging
from datetime import date, datetime, timedelta

from agagd_core.json_response import JsonResponse
from agagd_core.models import (
    Chapters,
    Country,
    Game,
    Member,
    MostRatedGamesPastYear,
    MostTournamentsPastYear,
    TopDan,
    TopKyu,
    Tournament,
)
from agagd_core.tables.core import (
    AllPlayerRatingsTable,
    ChapterMemberTable,
    GameTable,
    MemberTable,
    MostRatedGamesPastYearTable,
    MostTournamentsPastYearTable,
    OpponentTable,
    SecondaryGameTable,
    TopDanTable,
    TopKyuTable,
    TournamentPlayedTable,
    TournamentTable,
)
from django.core import exceptions
from django.core.paginator import PageNotAnInteger
from django.db.models import Count, F, Q
from django.http import HttpResponseRedirect
from django.shortcuts import get_object_or_404, redirect, render
from django.urls import reverse
from django.views.decorators.http import require_GET, require_POST
from django.views.generic import TemplateView
from django_tables2 import RequestConfig

logger = logging.getLogger("agagd.core.views")

def index(request):
    game_list = Game.objects.filter(
        game_date__gte=datetime.now() - timedelta(days=180)
    ).order_by("-game_date")
    table = GameTable(game_list, prefix="games")
    topDanList = TopDan.objects.values()
    topDanTable = TopDanTable(topDanList)
    topKyuList = TopKyu.objects.values()
    topKyuTable = TopKyuTable(topKyuList)
    mostRatedGamesPastYearList = MostRatedGamesPastYear.objects.values()
    mostRatedGamesTable = MostRatedGamesPastYearTable(mostRatedGamesPastYearList)
    mostTournamentsPastYearList = MostTournamentsPastYear.objects.values()
    mostTournamentsPastYearTable = MostTournamentsPastYearTable(
        mostTournamentsPastYearList
    )
    RequestConfig(request).configure(table)
    tourneys = Tournament.objects.all().order_by("-tournament_date")
    t_table = TournamentTable(tourneys, prefix="tourneys")
    RequestConfig(request, paginate={"per_page": 10}).configure(t_table)

    return render(
        request,
        "agagd_core/index.html",
        {
            "table": table,
            "top_dan_table": topDanTable,
            "top_kyu_table": topKyuTable,
            "most_rated_games_table": mostRatedGamesTable,
            "most_tournaments_table": mostTournamentsPastYearTable,
            "tournaments": t_table,
        },
    )

@require_GET
def search(request):
    query = request.GET.get("q", "")

    if not query:
        return HttpResponseRedirect("/")

    try:
        member_id = int(query)
        return HttpResponseRedirect(reverse("member_detail", args=(member_id,)))
    except ValueError:
        members_query = (
            Member.objects.filter(Q(member_id=F("players__pin_player")))
            .filter(full_name__icontains=query)
            .values(
                "member_id",
                "chapter_id",
                "renewal_due",
                "state",
                "players__rating",
                "country",
                "full_name",
                "family_name",
            )
            .order_by("family_name")
        )

        member_table = MemberTable(members_query)

        try:
            RequestConfig(request, paginate={"per_page": 100}).configure(member_table)
        except PageNotAnInteger:
            RequestConfig(request, paginate=False).configure(member_table)

        return render(
            request,
            "agagd_core/search_player.html",
            {"member_table": member_table, "query": query},
        )

def member_ratings(request, member_id):
    # returns a members rating data as a json dict for graphing
    try:
        player = Member.objects.get(pk=member_id)
        ratings = player.ratings_set.all().order_by("elab_date")
        ratings_dict = [
            {"sigma": r.sigma, "elab_date": r.elab_date, "rating": r.rating}
            for r in ratings
            if r.elab_date != None
        ]
        if len(ratings_dict) <= 1:
            logger.debug("Ratings error: only one rating")
            return JsonResponse(
                {"result": "error", "message": "Only one rated game. Not enough data"}
            )
        return JsonResponse(ratings_dict)
    except:
        logger.debug("Ratings error", exc_info=1)
        return JsonResponse({"result": "error"})

def member_detail(request, member_id):
    """
    The member detail page.
    Fetches the most recent games played and puts them in a Games Table.
    Fetches the list of ratings and puts them in a format for graphing.
    Computes the tournament data and opponent data for respective tables.
    """
    game_list = Game.objects.filter(
        Q(pin_player_1__exact=member_id) | Q(pin_player_2__exact=member_id)
    ).order_by("-game_date", "round")
    table = GameTable(game_list, prefix="games")
    RequestConfig(request, paginate={"per_page": 20}).configure(table)

    player = Member.objects.get(member_id=member_id)

    try:
        chapter = Chapters.objects.get(member_id=player.chapter_id.member_id)
    except exceptions.ObjectDoesNotExist:
        chapter = None

    ratings = player.ratings_set.all().order_by("-elab_date")
    if len(ratings) > 0:
        max_rating = max([r.rating for r in ratings])
        last_rating = ratings[0]
    else:
        max_rating = last_rating = None

    # compute additional tables for opponents & tournament info. here
    # TODO: refactor this into something nicer.
    opponent_data = {}
    tourney_data = {}
    for game in game_list:
        try:
            t_dat = tourney_data.get(game.tournament_code.pk, {})
            t_dat["tournament"] = game.tournament_code
            t_dat["won"] = t_dat.get("won", 0)
            t_dat["lost"] = t_dat.get("lost", 0)

            # Set default game_date to None
            game_date = None

            # Check for 0000-00-00 dates
            if game.game_date != u"0000-00-00":
                game_date = game.game_date

            t_dat["date"] = t_dat.get("date", game_date)

            op = game.player_other_than(player)
            opp_dat = opponent_data.get(op, {})
            opp_dat["opponent"] = op
            opp_dat["total"] = opp_dat.get("total", 0) + 1
            opp_dat["won"] = opp_dat.get("won", 0)
            opp_dat["lost"] = opp_dat.get("lost", 0)
            if game.won_by(player):
                opp_dat["won"] += 1
                t_dat["won"] += 1
            else:
                opp_dat["lost"] += 1
                t_dat["lost"] += 1
            opponent_data[op] = opp_dat
            tourney_data[game.tournament_code.pk] = t_dat
        except exceptions.ObjectDoesNotExist:
            print("failing game_id: %s" % game.pk)

    opp_table = OpponentTable(opponent_data.values(), player, prefix="opp")
    opp_table.this_player = player
    RequestConfig(request, paginate={"per_page": 10}).configure(opp_table)

    t_table = TournamentPlayedTable(
        sorted(
            tourney_data.values(),
            key=lambda d: d.get("date", date.today()) or date.today(),
            reverse=True,
        ),
        prefix="ts_played",
    )
    RequestConfig(request, paginate={"per_page": 10}).configure(t_table)

    return render(
        request,
        "agagd_core/member.html",
        {
            "table": table,
            "player": player,
            "chapter": chapter,
            "rating": last_rating,
            "max_rating": max_rating,
            "num_games": len(game_list),
            "opponents": opp_table,
            "tourneys": t_table,
        },
    )

@require_POST
def find_member_vs(request, member_id):
    opponent = get_object_or_404(Member, member_id=request.POST.get("opponent_id", ""))
    return HttpResponseRedirect(
        reverse("member_vs", args=(member_id, opponent.member_id))
    )

def member_vs(request, member_id, other_id):
    player_1 = get_object_or_404(Member, member_id=member_id)
    player_2 = get_object_or_404(Member, member_id=other_id)
    game_list = Game.objects.filter(
        Q(pin_player_1=player_1, pin_player_2=player_2)
        | Q(pin_player_1=player_2, pin_player_2=player_1)
    ).order_by("-game_date")
    table = GameTable(game_list)
    RequestConfig(request, paginate={"per_page": 20}).configure(table)
    return render(
        request,
        "agagd_core/member_vs.html",
        {"table": table, "player_1": player_1, "player_2": player_2},
    )

def tournament_detail(request, tourn_code):
    tourney = Tournament.objects.get(pk=tourn_code)
    secondary_gametable = SecondaryGameTable(tourney.games_in_tourney.all())
    RequestConfig(request, paginate={"per_page": 20}).configure(secondary_gametable)
    return render(
        request,
        "agagd_core/tourney.html",
        {"secondary_gametable": secondary_gametable, "tournament": tourney},
    )

def chapter_detail(request, chapter_id):
    chapter = get_object_or_404(Chapters, member_id=chapter_id)
    chapter_member_table = ChapterMemberTable(
        Member.objects.filter(chapter_id=chapter_id)
        .values(
            "member_id",
            "chapter_id",
            "renewal_due",
            "state",
            "players__rating",
            "country",
            "full_name",
            "family_name",
        )
        .order_by("family_name")
    )
    return render(
        request,
        "agagd_core/chapter.html",
        {"member_table": chapter_member_table, "chapter": chapter},
    )

def chapter_code_redirect(request, chapter_code):
    # Try the lookup with the 4-letter chapter code. These are deprecated,
    # but we continue to support them in case users have chapter pages bookmarked.
    chapter = get_object_or_404(Chapters, code=chapter_code)
    return redirect("chapter_detail", chapter_id=chapter.pk, permanent=True)

def country_detail(request, country_name):
    member_table = MemberTable(
        Member.objects.filter(country=country_name)
        .values(
            "member_id",
            "chapter_id",
            "renewal_due",
            "state",
            "players__rating",
            "country",
            "full_name",
            "family_name",
        )
        .order_by("family_name")
    )

    RequestConfig(request, paginate={"per_page": 100}).configure(member_table)

    return render(
        request,
        "agagd_core/country.html",
        {"country_name": country_name, "member_table": member_table},
    )

def all_player_ratings(request):
    all_player_ratings_query = (
        Member.objects.filter(
            Q(chapter_id=F("chapters__member_id")) | Q(chapters__member_id__isnull=True)
        )
        .filter(Q(member_id=F("players__pin_player")))
        .filter(status="accepted")
        .exclude(players__rating__isnull=True)
        .exclude(type="chapter")
        .exclude(type="e-journal")
        .exclude(type="library")
        .exclude(type="institution")
        .values(
            "full_name",
            "member_id",
            "type",
            "players__rating",
            "chapter_id",
            "state",
            "players__sigma",
        )
        .order_by("-players__rating")
    )

    all_player_ratings_table = AllPlayerRatingsTable(all_player_ratings_query)
    RequestConfig(request, paginate={"per_page": 50}).configure(
        all_player_ratings_table
    )

    return render(
        request,
        "agagd_core/all_player_ratings.html",
        {"all_player_ratings_table": all_player_ratings_table},
    )

@require_GET
def tournament_list(request):
    tourneys = Tournament.objects.order_by("-tournament_date")
    details = request.GET.get("details", "")
    from_date = request.GET.get("from_date", "")
    to_date = request.GET.get("to_date", "")

    if details:
        for search_token in details.split():
            query = search_token.lower()
            tourneys = tourneys.filter(
                Q(description__icontains=query)
                | Q(city__icontains=query)
                | Q(state__icontains=query)
            )

    if from_date:
        tourneys = tourneys.filter(
            tournament_date__gte=datetime.strptime(from_date, "%m/%d/%Y")
        )

    if to_date:
        tourneys = tourneys.filter(
            tournament_date__lte=datetime.strptime(to_date, "%m/%d/%Y")
        )

    tournament_table = TournamentTable(tourneys)
    RequestConfig(request, paginate={"per_page": 50}).configure(tournament_table)

    return render(
        request,
        "agagd_core/tournament_list.html",
        {
            "details": details,
            "from_date": from_date,
            "to_date": to_date,
            "tournament_table": tournament_table,
        },
    )

def game_stats(request):
    games_by_date = []

    for game_obj in Game.objects.values("game_date").annotate(Count("game_date")):
        try:
            game_date = datetime.strptime(str(game_obj["game_date"]), "%Y-%m-%d")
            games_by_date.append(
                {
                    "date": game_date.strftime("%Y-%m-%d"),
                    "count": game_obj["game_date__count"],
                }
            )
        except ValueError:
            pass

    sorted_games_by_date = sorted(games_by_date, key=lambda d: d["date"])
    return JsonResponse(sorted_games_by_date)

# AGAGD Pages
class InformationPageView(TemplateView):
    template_name = "information.html"

class QualificationsPageView(TemplateView):
    template_name = "qualifications.html"
<commit_msg>Added Correct query for all player ratings to fix legacy.<commit_after>import logging
from datetime import date, datetime, timedelta

from agagd_core.json_response import JsonResponse
from agagd_core.models import (
    Chapters,
    Country,
    Game,
    Member,
    MostRatedGamesPastYear,
    MostTournamentsPastYear,
    TopDan,
    TopKyu,
    Tournament,
)
from agagd_core.tables.core import (
    AllPlayerRatingsTable,
    ChapterMemberTable,
    GameTable,
    MemberTable,
    MostRatedGamesPastYearTable,
    MostTournamentsPastYearTable,
    OpponentTable,
    SecondaryGameTable,
    TopDanTable,
    TopKyuTable,
    TournamentPlayedTable,
    TournamentTable,
)
from django.core import exceptions
from django.core.paginator import PageNotAnInteger
from django.db.models import Count, F, Q
from django.http import HttpResponseRedirect
from django.shortcuts import get_object_or_404, redirect, render
from django.urls import reverse
from django.views.decorators.http import require_GET, require_POST
from django.views.generic import TemplateView
from django_tables2 import RequestConfig

logger = logging.getLogger("agagd.core.views")

def index(request):
    game_list = Game.objects.filter(
        game_date__gte=datetime.now() - timedelta(days=180)
    ).order_by("-game_date")
    table = GameTable(game_list, prefix="games")
    topDanList = TopDan.objects.values()
    topDanTable = TopDanTable(topDanList)
    topKyuList = TopKyu.objects.values()
    topKyuTable = TopKyuTable(topKyuList)
    mostRatedGamesPastYearList = MostRatedGamesPastYear.objects.values()
    mostRatedGamesTable = MostRatedGamesPastYearTable(mostRatedGamesPastYearList)
    mostTournamentsPastYearList = MostTournamentsPastYear.objects.values()
    mostTournamentsPastYearTable = MostTournamentsPastYearTable(
        mostTournamentsPastYearList
    )
    RequestConfig(request).configure(table)
    tourneys = Tournament.objects.all().order_by("-tournament_date")
    t_table = TournamentTable(tourneys, prefix="tourneys")
    RequestConfig(request, paginate={"per_page": 10}).configure(t_table)

    return render(
        request,
        "agagd_core/index.html",
        {
            "table": table,
            "top_dan_table": topDanTable,
            "top_kyu_table": topKyuTable,
            "most_rated_games_table": mostRatedGamesTable,
            "most_tournaments_table": mostTournamentsPastYearTable,
            "tournaments": t_table,
        },
    )

@require_GET
def search(request):
    query = request.GET.get("q", "")

    if not query:
        return HttpResponseRedirect("/")

    try:
        member_id = int(query)
        return HttpResponseRedirect(reverse("member_detail", args=(member_id,)))
    except ValueError:
        members_query = (
            Member.objects.filter(Q(member_id=F("players__pin_player")))
            .filter(full_name__icontains=query)
            .values(
                "member_id",
                "chapter_id",
                "renewal_due",
                "state",
                "players__rating",
                "country",
                "full_name",
                "family_name",
            )
            .order_by("family_name")
        )

        member_table = MemberTable(members_query)

        try:
            RequestConfig(request, paginate={"per_page": 100}).configure(member_table)
        except PageNotAnInteger:
            RequestConfig(request, paginate=False).configure(member_table)

        return render(
            request,
            "agagd_core/search_player.html",
            {"member_table": member_table, "query": query},
        )

def member_ratings(request, member_id):
    # returns a members rating data as a json dict for graphing
    try:
        player = Member.objects.get(pk=member_id)
        ratings = player.ratings_set.all().order_by("elab_date")
        ratings_dict = [
            {"sigma": r.sigma, "elab_date": r.elab_date, "rating": r.rating}
            for r in ratings
            if r.elab_date != None
        ]
        if len(ratings_dict) <= 1:
            logger.debug("Ratings error: only one rating")
            return JsonResponse(
                {"result": "error", "message": "Only one rated game. Not enough data"}
            )
        return JsonResponse(ratings_dict)
    except:
        logger.debug("Ratings error", exc_info=1)
        return JsonResponse({"result": "error"})

def member_detail(request, member_id):
    """
    The member detail page.
    Fetches the most recent games played and puts them in a Games Table.
    Fetches the list of ratings and puts them in a format for graphing.
    Computes the tournament data and opponent data for respective tables.
    """
    game_list = Game.objects.filter(
        Q(pin_player_1__exact=member_id) | Q(pin_player_2__exact=member_id)
    ).order_by("-game_date", "round")
    table = GameTable(game_list, prefix="games")
    RequestConfig(request, paginate={"per_page": 20}).configure(table)

    player = Member.objects.get(member_id=member_id)

    try:
        chapter = Chapters.objects.get(member_id=player.chapter_id.member_id)
    except exceptions.ObjectDoesNotExist:
        chapter = None

    ratings = player.ratings_set.all().order_by("-elab_date")
    if len(ratings) > 0:
        max_rating = max([r.rating for r in ratings])
        last_rating = ratings[0]
    else:
        max_rating = last_rating = None

    # compute additional tables for opponents & tournament info. here
    # TODO: refactor this into something nicer.
    opponent_data = {}
    tourney_data = {}
    for game in game_list:
        try:
            t_dat = tourney_data.get(game.tournament_code.pk, {})
            t_dat["tournament"] = game.tournament_code
            t_dat["won"] = t_dat.get("won", 0)
            t_dat["lost"] = t_dat.get("lost", 0)

            # Set default game_date to None
            game_date = None

            # Check for 0000-00-00 dates
            if game.game_date != u"0000-00-00":
                game_date = game.game_date

            t_dat["date"] = t_dat.get("date", game_date)

            op = game.player_other_than(player)
            opp_dat = opponent_data.get(op, {})
            opp_dat["opponent"] = op
            opp_dat["total"] = opp_dat.get("total", 0) + 1
            opp_dat["won"] = opp_dat.get("won", 0)
            opp_dat["lost"] = opp_dat.get("lost", 0)
            if game.won_by(player):
                opp_dat["won"] += 1
                t_dat["won"] += 1
            else:
                opp_dat["lost"] += 1
                t_dat["lost"] += 1
            opponent_data[op] = opp_dat
            tourney_data[game.tournament_code.pk] = t_dat
        except exceptions.ObjectDoesNotExist:
            print("failing game_id: %s" % game.pk)

    opp_table = OpponentTable(opponent_data.values(), player, prefix="opp")
    opp_table.this_player = player
    RequestConfig(request, paginate={"per_page": 10}).configure(opp_table)

    t_table = TournamentPlayedTable(
        sorted(
            tourney_data.values(),
            key=lambda d: d.get("date", date.today()) or date.today(),
            reverse=True,
        ),
        prefix="ts_played",
    )
    RequestConfig(request, paginate={"per_page": 10}).configure(t_table)

    return render(
        request,
        "agagd_core/member.html",
        {
            "table": table,
            "player": player,
            "chapter": chapter,
            "rating": last_rating,
            "max_rating": max_rating,
            "num_games": len(game_list),
            "opponents": opp_table,
            "tourneys": t_table,
        },
    )

@require_POST
def find_member_vs(request, member_id):
    opponent = get_object_or_404(Member, member_id=request.POST.get("opponent_id", ""))
    return HttpResponseRedirect(
        reverse("member_vs", args=(member_id, opponent.member_id))
    )

def member_vs(request, member_id, other_id):
    player_1 = get_object_or_404(Member, member_id=member_id)
    player_2 = get_object_or_404(Member, member_id=other_id)
    game_list = Game.objects.filter(
        Q(pin_player_1=player_1, pin_player_2=player_2)
        | Q(pin_player_1=player_2, pin_player_2=player_1)
    ).order_by("-game_date")
    table = GameTable(game_list)
    RequestConfig(request, paginate={"per_page": 20}).configure(table)
    return render(
        request,
        "agagd_core/member_vs.html",
        {"table": table, "player_1": player_1, "player_2": player_2},
    )

def tournament_detail(request, tourn_code):
    tourney = Tournament.objects.get(pk=tourn_code)
    secondary_gametable = SecondaryGameTable(tourney.games_in_tourney.all())
    RequestConfig(request, paginate={"per_page": 20}).configure(secondary_gametable)
    return render(
        request,
        "agagd_core/tourney.html",
        {"secondary_gametable": secondary_gametable, "tournament": tourney},
    )

def chapter_detail(request, chapter_id):
    chapter = get_object_or_404(Chapters, member_id=chapter_id)
    chapter_member_table = ChapterMemberTable(
        Member.objects.filter(chapter_id=chapter_id)
        .values(
            "member_id",
            "chapter_id",
            "renewal_due",
            "state",
            "players__rating",
            "country",
            "full_name",
            "family_name",
        )
        .order_by("family_name")
    )
    return render(
        request,
        "agagd_core/chapter.html",
        {"member_table": chapter_member_table, "chapter": chapter},
    )

def chapter_code_redirect(request, chapter_code):
    # Try the lookup with the 4-letter chapter code. These are deprecated,
    # but we continue to support them in case users have chapter pages bookmarked.
    chapter = get_object_or_404(Chapters, code=chapter_code)
    return redirect("chapter_detail", chapter_id=chapter.pk, permanent=True)

def country_detail(request, country_name):
    member_table = MemberTable(
        Member.objects.filter(country=country_name)
        .values(
            "member_id",
            "chapter_id",
            "renewal_due",
            "state",
            "players__rating",
            "country",
            "full_name",
            "family_name",
        )
        .order_by("family_name")
    )

    RequestConfig(request, paginate={"per_page": 100}).configure(member_table)

    return render(
        request,
        "agagd_core/country.html",
        {"country_name": country_name, "member_table": member_table},
    )

def all_player_ratings(request):
    all_player_ratings_query = (
        Member.objects.select_related("chapter_id")
        .filter(status="accepted")
        .filter(players__rating__isnull=False)
        .exclude(type__iexact="e-journal")
        .exclude(type__iexact="chapter")
        .exclude(type__iexact="library")
        .exclude(type__iexact="institution")
        .values(
            "chapter_id",
            "member_id",
            "chapter_id__name",
            "full_name",
            "type",
            "players__rating",
            "state",
            "players__sigma",
        )
        .order_by("-players__rating")
    )

    all_player_ratings_table = AllPlayerRatingsTable(all_player_ratings_query)
    RequestConfig(request, paginate={"per_page": 50}).configure(
        all_player_ratings_table
    )

    return render(
        request,
        "agagd_core/all_player_ratings.html",
        {"all_player_ratings_table": all_player_ratings_table},
    )

@require_GET
def tournament_list(request):
    tourneys = Tournament.objects.order_by("-tournament_date")
    details = request.GET.get("details", "")
    from_date = request.GET.get("from_date", "")
    to_date = request.GET.get("to_date", "")

    if details:
        for search_token in details.split():
            query = search_token.lower()
            tourneys = tourneys.filter(
                Q(description__icontains=query)
                | Q(city__icontains=query)
                | Q(state__icontains=query)
            )

    if from_date:
        tourneys = tourneys.filter(
            tournament_date__gte=datetime.strptime(from_date, "%m/%d/%Y")
        )

    if to_date:
        tourneys = tourneys.filter(
            tournament_date__lte=datetime.strptime(to_date, "%m/%d/%Y")
        )

    tournament_table = TournamentTable(tourneys)
    RequestConfig(request, paginate={"per_page": 50}).configure(tournament_table)

    return render(
        request,
        "agagd_core/tournament_list.html",
        {
            "details": details,
            "from_date": from_date,
            "to_date": to_date,
            "tournament_table": tournament_table,
        },
    )

def game_stats(request):
    games_by_date = []

    for game_obj in Game.objects.values("game_date").annotate(Count("game_date")):
        try:
            game_date = datetime.strptime(str(game_obj["game_date"]), "%Y-%m-%d")
            games_by_date.append(
                {
                    "date": game_date.strftime("%Y-%m-%d"),
                    "count": game_obj["game_date__count"],
                }
            )
        except ValueError:
            pass

    sorted_games_by_date = sorted(games_by_date, key=lambda d: d["date"])
    return JsonResponse(sorted_games_by_date)

# AGAGD Pages
class InformationPageView(TemplateView):
    template_name = "information.html"

class QualificationsPageView(TemplateView):
    template_name = "qualifications.html"
<|endoftext|>


<commit_before>"""
#-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=#
  This file is part of the Smart Developer Hub Project:
    http://www.smartdeveloperhub.org

  Center for Open Middleware
        http://www.centeropenmiddleware.com/
#-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=#
  Copyright (C) 2015 Center for Open Middleware.
#-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=#
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

            http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
#-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=#
"""
import calendar
import json
import logging
import traceback
from datetime import datetime

import networkx as nx
from agora.scholar.actions import FragmentConsumerResponse
from agora.scholar.daemons.fragment import fragment_lock, fragment_graph, fragments_key, fragment_updated_on, \
    FragmentPlugin
from agora.stoa.actions.core import STOA
from agora.stoa.actions.core.fragment import FragmentRequest, FragmentAction, FragmentSink
from agora.stoa.actions.core.utils import chunks, tp_parts
from agora.stoa.store import r
from agora.stoa.store.tables import db

__author__ = 'Fernando Serena'

log = logging.getLogger('agora.scholar.actions.query')

def fragment_has_result_set(fid):
    return r.get('{}:{}:rs'.format(fragments_key, fid)) is not None

def _update_result_set(fid, gp):
    try:
        result_gen = _query(fid, gp)
        removed = db[fid].delete_many({}).deleted_count
        log.info('{} rows removed from fragment {} result set'.format(removed, fid))
        table = db[fid]
        rows = set(result_gen)
        if rows:
            table.insert_many([{label: row[row.labels[label]] for label in row.labels} for row in rows])
        log.info('{} rows inserted into fragment {} result set'.format(len(rows), fid))

        with r.pipeline(transaction=True) as p:
            p.multi()
            p.set('{}:{}:rs'.format(fragments_key, fid), True)
            p.execute()

    except Exception as e:
        traceback.print_exc()
        log.error(e.message)

def _query(fid, gp):
    """
    Query the fragment using the original request graph pattern
    :param gp:
    :param fid:
    :return: The query result
    """

    def __build_query_from(x, depth=0):
        def build_pattern_query((u, v, data)):
            return '\nOPTIONAL { %s %s %s %s }' % (u, data['predicate'], v, __build_query_from(v, depth + 1))

        out_edges = list(gp_graph.out_edges_iter(x, data=True))
        out_edges = reversed(sorted(out_edges, key=lambda x: gp_graph.out_degree))
        if out_edges:
            return ' '.join([build_pattern_query(x) for x in out_edges])
        return ''

    gp_parts = [tp_parts(tp) for tp in gp]

    blocks = []
    gp_graph = nx.DiGraph()
    for gp_part in gp_parts:
        gp_graph.add_edge(gp_part[0], gp_part[2], predicate=gp_part[1])

    roots = filter(lambda x: gp_graph.in_degree(x) == 0, gp_graph.nodes())

    blocks += ['%s a stoa:Root\nOPTIONAL { %s }' % (root, __build_query_from(root)) for root in roots]

    where_gp = ' .\n'.join(blocks)
    q = """SELECT DISTINCT * WHERE { %s }""" % where_gp

    result = []
    try:
        log.info('Querying fragment {}:\n{}'.format(fid, q))
        result = fragment_graph(fid).query(q)
    except Exception as e:  # ParseException from query
        traceback.print_exc()
        log.warning(e.message)
    return result

class QueryPlugin(FragmentPlugin):
    @property
    def sink_class(self):
        return QuerySink

    def consume(self, fid, quad, graph, *args):
        pass

    @property
    def sink_aware(self):
        return False

    def complete(self, fid, *args):
        fragment_gp = args[0]

        try:
            if fragment_has_result_set(fid):
                _update_result_set(fid, fragment_gp)
        except Exception, e:
            traceback.print_exc()
            log.error(e.message)

FragmentPlugin.register(QueryPlugin)

class QueryRequest(FragmentRequest):
    def __init__(self):
        super(QueryRequest, self).__init__()

    def _extract_content(self, request_type=STOA.QueryRequest):
        """
        Parse query request data. For this operation, there is no additional data to extract.
        """
        super(QueryRequest, self)._extract_content(request_type=request_type)

class QueryAction(FragmentAction):
    def __init__(self, message):
        """
        Prepare request and sink objects before starting initialization
        """
        self.__request = QueryRequest()
        self.__sink = QuerySink()
        super(QueryAction, self).__init__(message)

    @property
    def sink(self):
        return self.__sink

    @classmethod
    def response_class(cls):
        return QueryResponse

    @property
    def request(self):
        return self.__request

    def submit(self):
        """
        If the fragment is already synced at submission time, the delivery becomes ready
        """
        super(QueryAction, self).submit()
        if fragment_updated_on(self.sink.fragment_id) is not None:
            self.sink.delivery = 'ready'

class QuerySink(FragmentSink):
    """
    Query sink does not need any extra behaviour
    """

    def _remove(self, pipe):
        super(QuerySink, self)._remove(pipe)

    def __init__(self):
        super(QuerySink, self).__init__()

    def _save(self, action, general=True):
        super(QuerySink, self)._save(action, general)

    def _load(self):
        super(QuerySink, self)._load()

class QueryResponse(FragmentConsumerResponse):
    def __init__(self, rid):
        # The creation of a response always require to load its corresponding sink
        self.__sink = QuerySink()
        self.__sink.load(rid)
        super(QueryResponse, self).__init__(rid)
        self.__fragment_lock = fragment_lock(self.__sink.fragment_id)

    @property
    def sink(self):
        return self.__sink

    def _build(self):
        self.__fragment_lock.acquire()
        result = self.result_set()
        log.debug('Building a query result for request number {}'.format(self._request_id))

        try:
            # Query result chunking, yields JSON
            for ch in chunks(result, 1000):
                result_rows = []
                for t in ch:
                    if any(t):
                        result_row = {self.sink.map('?' + v).lstrip('?'): t[v] for v in t}
                        result_rows.append(result_row)
                if result_rows:
                    yield json.dumps(result_rows), {'state': 'streaming', 'source': 'store',
                                                    'response_to': self.sink.message_id,
                                                    'submitted_on': calendar.timegm(datetime.utcnow().timetuple()),
                                                    'submitted_by': self.sink.submitted_by,
                                                    'format': 'json'}
        except Exception, e:
            log.error(e.message)
            raise
        finally:
            self.__fragment_lock.release()
            yield [], {'state': 'end', 'format': 'json'}

        # Just after sending the state:end message, the request delivery state switches to sent
        self.sink.delivery = 'sent'

    def result_set(self):
        def extract_fields(result):
            for r in result:
                yield r['_id']

        if not r.exists('{}:{}:rs'.format(fragments_key, self.sink.fragment_id)):
            _update_result_set(self.sink.fragment_id, self.sink.fragment_gp)

        pattern = {}
        projection = {}
        mapping = filter(lambda x: x.startswith('?'), self.sink.mapping)
        for v in mapping:
            value = self.sink.map(v, fmap=True)
            if not value.startswith('?'):
                if value.startswith('"'):
                    value = value.strip('"')
                else:
                    value = value.lstrip('<').rstrip('>')
                pattern[v.lstrip('?')] = value
            elif not value.startswith('?_'):
                # All those variables that start with '_' won't be projected
                projection[v.lstrip('?')] = True

        table = db[self.sink.fragment_id]
        pipeline = [{"$match": {v: pattern[v] for v in pattern}},
                    {"$group": {'_id': {v: '$' + v for v in projection}}}]
        return extract_fields(table.aggregate(pipeline))
<commit_msg>Build query intermediate solutions<commit_after>"""
#-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=#
  This file is part of the Smart Developer Hub Project:
    http://www.smartdeveloperhub.org

  Center for Open Middleware
        http://www.centeropenmiddleware.com/
#-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=#
  Copyright (C) 2015 Center for Open Middleware.
#-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=#
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

            http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
#-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=#
"""
import calendar
import json
import logging
import traceback
from datetime import datetime

import networkx as nx
from agora.scholar.actions import FragmentConsumerResponse
from agora.scholar.daemons.fragment import fragment_lock, fragment_graph, fragments_key, fragment_updated_on, \
    FragmentPlugin
from agora.stoa.actions.core import STOA
from agora.stoa.actions.core.fragment import FragmentRequest, FragmentAction, FragmentSink
from agora.stoa.actions.core.utils import chunks, tp_parts
from agora.stoa.store import r
from agora.stoa.store.tables import db

__author__ = 'Fernando Serena'

log = logging.getLogger('agora.scholar.actions.query')

def fragment_has_result_set(fid):
    return r.get('{}:{}:rs'.format(fragments_key, fid)) is not None

def _update_result_set(fid, gp):
    try:
        result_gen = _query(fid, gp)
        removed = db[fid].delete_many({}).deleted_count
        log.info('{} rows removed from fragment {} result set'.format(removed, fid))
        table = db[fid]
        rows = set(result_gen)
        if rows:
            table.insert_many([{label: row[row.labels[label]] for label in row.labels} for row in rows])
        log.info('{} rows inserted into fragment {} result set'.format(len(rows), fid))

        with r.pipeline(transaction=True) as p:
            p.multi()
            p.set('{}:{}:rs'.format(fragments_key, fid), True)
            p.execute()

    except Exception as e:
        traceback.print_exc()
        log.error(e.message)

def _query(fid, gp):
    """
    Query the fragment using the original request graph pattern
    :param gp:
    :param fid:
    :return: The query result
    """

    def __build_query_from(x, depth=0):
        def build_pattern_query((u, v, data)):
            return '\nOPTIONAL { %s %s %s %s }' % (u, data['predicate'], v, __build_query_from(v, depth + 1))

        out_edges = list(gp_graph.out_edges_iter(x, data=True))
        out_edges = reversed(sorted(out_edges, key=lambda x: gp_graph.out_degree))
        if out_edges:
            return ' '.join([build_pattern_query(x) for x in out_edges])
        return ''

    gp_parts = [tp_parts(tp) for tp in gp]

    blocks = []
    gp_graph = nx.DiGraph()
    for gp_part in gp_parts:
        gp_graph.add_edge(gp_part[0], gp_part[2], predicate=gp_part[1])

    roots = filter(lambda x: gp_graph.in_degree(x) == 0, gp_graph.nodes())

    blocks += ['%s a stoa:Root\nOPTIONAL { %s }' % (root, __build_query_from(root)) for root in roots]

    where_gp = ' .\n'.join(blocks)
    q = """SELECT DISTINCT * WHERE { %s }""" % where_gp

    result = []
    try:
        log.info('Querying fragment {}:\n{}'.format(fid, q))
        result = fragment_graph(fid).query(q)
    except Exception as e:  # ParseException from query
        traceback.print_exc()
        log.warning(e.message)
    return result

class QueryPlugin(FragmentPlugin):
    @property
    def sink_class(self):
        return QuerySink

    def consume(self, fid, quad, graph, *args):
        pass
        intermediate_fid_key = '{}:{}:int'.format(fragments_key, fid)
        (subj, _, obj) = quad[0]
        r.sadd(intermediate_fid_key + ':{}:{}'.format(subj, obj), (quad[1], quad[3]))

    @property
    def sink_aware(self):
        return False

    def complete(self, fid, *args):
        fragment_gp = args[0]

        try:
            if fragment_has_result_set(fid):
                _update_result_set(fid, fragment_gp)
        except Exception, e:
            traceback.print_exc()
            log.error(e.message)

        intermediate_fid_key = '{}:{}:int'.format(fragments_key, fid)
        r.delete(intermediate_fid_key)

FragmentPlugin.register(QueryPlugin)

class QueryRequest(FragmentRequest):
    def __init__(self):
        super(QueryRequest, self).__init__()

    def _extract_content(self, request_type=STOA.QueryRequest):
        """
        Parse query request data. For this operation, there is no additional data to extract.
        """
        super(QueryRequest, self)._extract_content(request_type=request_type)

class QueryAction(FragmentAction):
    def __init__(self, message):
        """
        Prepare request and sink objects before starting initialization
        """
        self.__request = QueryRequest()
        self.__sink = QuerySink()
        super(QueryAction, self).__init__(message)

    @property
    def sink(self):
        return self.__sink

    @classmethod
    def response_class(cls):
        return QueryResponse

    @property
    def request(self):
        return self.__request

    def submit(self):
        """
        If the fragment is already synced at submission time, the delivery becomes ready
        """
        super(QueryAction, self).submit()
        if fragment_updated_on(self.sink.fragment_id) is not None:
            self.sink.delivery = 'ready'

class QuerySink(FragmentSink):
    """
    Query sink does not need any extra behaviour
    """

    def _remove(self, pipe):
        super(QuerySink, self)._remove(pipe)

    def __init__(self):
        super(QuerySink, self).__init__()

    def _save(self, action, general=True):
        super(QuerySink, self)._save(action, general)

    def _load(self):
        super(QuerySink, self)._load()

class QueryResponse(FragmentConsumerResponse):
    def __init__(self, rid):
        # The creation of a response always require to load its corresponding sink
        self.__sink = QuerySink()
        self.__sink.load(rid)
        super(QueryResponse, self).__init__(rid)
        self.__fragment_lock = fragment_lock(self.__sink.fragment_id)

    @property
    def sink(self):
        return self.__sink

    def _build(self):
        self.__fragment_lock.acquire()
        result = self.result_set()
        log.debug('Building a query result for request number {}'.format(self._request_id))

        try:
            # Query result chunking, yields JSON
            for ch in chunks(result, 1000):
                result_rows = []
                for t in ch:
                    if any(t):
                        result_row = {self.sink.map('?' + v).lstrip('?'): t[v] for v in t}
                        result_rows.append(result_row)
                if result_rows:
                    yield json.dumps(result_rows), {'state': 'streaming', 'source': 'store',
                                                    'response_to': self.sink.message_id,
                                                    'submitted_on': calendar.timegm(datetime.utcnow().timetuple()),
                                                    'submitted_by': self.sink.submitted_by,
                                                    'format': 'json'}
        except Exception, e:
            log.error(e.message)
            raise
        finally:
            self.__fragment_lock.release()
            yield [], {'state': 'end', 'format': 'json'}

        # Just after sending the state:end message, the request delivery state switches to sent
        self.sink.delivery = 'sent'

    def result_set(self):
        def extract_fields(result):
            for r in result:
                yield r['_id']

        if not r.exists('{}:{}:rs'.format(fragments_key, self.sink.fragment_id)):
            _update_result_set(self.sink.fragment_id, self.sink.fragment_gp)

        pattern = {}
        projection = {}
        mapping = filter(lambda x: x.startswith('?'), self.sink.mapping)
        for v in mapping:
            value = self.sink.map(v, fmap=True)
            if not value.startswith('?'):
                if value.startswith('"'):
                    value = value.strip('"')
                else:
                    value = value.lstrip('<').rstrip('>')
                pattern[v.lstrip('?')] = value
            elif not value.startswith('?_'):
                # All those variables that start with '_' won't be projected
                projection[v.lstrip('?')] = True

        table = db[self.sink.fragment_id]
        pipeline = [{"$match": {v: pattern[v] for v in pattern}},
                    {"$group": {'_id': {v: '$' + v for v in projection}}}]
        return extract_fields(table.aggregate(pipeline))
<|endoftext|>


<commit_before>__author__ = 'Jordi Vilaplana'

from pymongo import MongoClient
from geopy.geocoders import Nominatim
import json
import logging

logging.basicConfig(
    filename='emovix_twitter_geocoding.log',
    level=logging.WARNING,
    format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
    datefmt='%d-%m-%y %H:%M')

# Configuration parameters
database_host = ""
database_name = ""

client = None
db = None

if __name__ == '__main__':
    logging.debug('emovix_twitter_geocoding.py starting ...')

    # Load configuration
    with open('config.json', 'r') as f:
        config = json.load(f)
        database_host = config['database_host']
        database_name = config['database_name']

    client = MongoClient('mongodb://' + database_host + ':27017/')
    db = client[database_name]

    geolocator = Nominatim()

    # Let's run for a while ...
    while True:
        try:
            # Get a twitter user account from the database that has not been geocoded yet.
            # This is, where the "location_geocoding" field doesn't exist.
            user = db.twitterUser.find_one({ "location_geocoding": { "$exists": False } })

            # If we get a user ... (maybe we have already geocoded all the users?)
            if user:
                # Get the user location. This field is filled in the twitter profile page of the user, and it's an
                # optional string field, so expect to get almost anything here.
                user_location = user['location']

                # As mentioned, this could be empty ...
                if user_location:
                    user_location = user_location.encode('utf-8').lower()

                # So now we are interested in checking whether we have already checked that same location before
                # or not. Once a location has been checked, it is 'cached' in the 'twitterGeocoding' collection.
                cached_location = db.twitterGeocoding.find_one({ "location": user_location})

                # So, did we have it?
                if cached_location:
                    # Yay! We just need to add the existing coordinates to the 'location_geocoding' field of the user.
                    user['location_geocoding'] = { "latitude": cached_location['latitude'], "longitude": cached_location['longitude'] }

                    # Well, let's also sum the total amount of users of the cached location. Why?
                    # Maybe it will be useful later to get the most relevant locations? Who knows, not me.
                    if not cached_location['total_users']:
                        cached_location['total_users'] = 0
                    cached_location['total_users'] += 1
                    db.twitterGeocoding.update( { '_id': cached_location['_id']}, cached_location, upsert=True)
                else:
                    # Nay! Welp, let's get the coordinates from our geocoding service.
                    location = geolocator.geocode(user_location)

                    # Wait, if our twitter user had no location, let's just override whatever the geocoding service
                    # got for us. We do not want some random coordinates for the ones that do not have a location.
                    if user_location == None:
                        # Okay, maybe not
                        location = None

                    # So, did we get a location from our geocoding service or not?
                    if location:
                        # Yay! So let's set the 'location_geocoding' parameter of our user to the coordinates obtained
                        # from the geocoding service and cache these coordinates as well.
                        user['location_geocoding'] = { "latitude": location.latitude, "longitude": location.longitude }
                        db.twitterGeocoding.insert_one( { "location": user_location, "latitude": location[1][0], "longitude": location[1][1], "total_users": 1})
                    else:
                        # Okay, so the user provided some weird location and our geocoding service was unable to
                        # provide us a set of coordinates, uh? Let's just set the coordiantes to {0,0} and we will
                        # sort this out later.
                        user['location_geocoding'] = { "latitude": 0, "longitude": 0 }
                        db.twitterGeocoding.insert_one( { "location": user_location, "latitude": 0, "longitude": 0, "total_users": 1})

                # Let's just end this and update our twitter user with whatever the result was.
                db.twitterUser.update( { "_id": user['_id']}, user, upsert=True)

        except Exception as e:
            # Oh well, just keep going
            logging.error(e.__class__)
            logging.error(e)
            continue
        except KeyboardInterrupt:
            break
<commit_msg>The "location_geocoding" field has been reformatted to use the geoJSON format.<commit_after>__author__ = 'Jordi Vilaplana'

from pymongo import MongoClient
from geopy.geocoders import Nominatim
import json
import logging

logging.basicConfig(
    filename='emovix_twitter_geocoding.log',
    level=logging.WARNING,
    format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
    datefmt='%d-%m-%y %H:%M')

# Configuration parameters
database_host = ""
database_name = ""

client = None
db = None

if __name__ == '__main__':
    logging.debug('emovix_twitter_geocoding.py starting ...')

    # Load configuration
    with open('config.json', 'r') as f:
        config = json.load(f)
        database_host = config['database_host']
        database_name = config['database_name']

    client = MongoClient('mongodb://' + database_host + ':27017/')
    db = client[database_name]

    geolocator = Nominatim()

    # Let's run for a while ...
    while True:
        try:
            # Get a twitter user account from the database that has not been geocoded yet.
            # This is, where the "location_geocoding" field doesn't exist.
            user = db.twitterUser.find_one({ "location_geocoding": { "$exists": False } })

            # If we get a user ... (maybe we have already geocoded all the users?)
            if user:
                # Get the user location. This field is filled in the twitter profile page of the user, and it's an
                # optional string field, so expect to get almost anything here.
                user_location = user['location']

                # As mentioned, this could be empty ...
                if user_location:
                    user_location = user_location.encode('utf-8').lower()

                # So now we are interested in checking whether we have already checked that same location before
                # or not. Once a location has been checked, it is 'cached' in the 'twitterGeocoding' collection.
                cached_location = db.twitterGeocoding.find_one({ "location": user_location})

                # So, did we have it?
                if cached_location:
                    # Yay! We just need to add the existing coordinates to the 'location_geocoding' field of the user.
                    # This is in geoJSON format: http://geojson.org/
                    user['location_geocoding'] = { "type": "Point", "coordinates": [ cached_location['longitude'], cached_location['latitude'] ] }

                    # Well, let's also sum the total amount of users of the cached location. Why?
                    # Maybe it will be useful later to get the most relevant locations? Who knows, not me.
                    if not cached_location['total_users']:
                        cached_location['total_users'] = 0
                    cached_location['total_users'] += 1
                    db.twitterGeocoding.update( { '_id': cached_location['_id']}, cached_location, upsert=True)
                else:
                    # Nay! Welp, let's get the coordinates from our geocoding service.
                    location = geolocator.geocode(user_location)

                    # Wait, if our twitter user had no location, let's just override whatever the geocoding service
                    # got for us. We do not want some random coordinates for the ones that do not have a location.
                    if user_location == None:
                        # Okay, maybe not
                        location = None

                    # So, did we get a location from our geocoding service or not?
                    if location:
                        # Yay! So let's set the 'location_geocoding' parameter of our user to the coordinates obtained
                        # from the geocoding service and cache these coordinates as well.
                        user['location_geocoding'] = { "type": "Point", "coordinates": [ location.longitude, location.latitude ] }
                        db.twitterGeocoding.insert_one( { "location": user_location, "latitude": location[1][0], "longitude": location[1][1], "total_users": 1})
                    else:
                        # Okay, so the user provided some weird location and our geocoding service was unable to
                        # provide us a set of coordinates, uh? Let's just set the coordiantes to {0,0} and we will
                        # sort this out later.
                        user['location_geocoding'] = { "type": "Point", "coordinates": [ 0, 0 ] }
                        db.twitterGeocoding.insert_one( { "location": user_location, "latitude": 0, "longitude": 0, "total_users": 1})

                # Let's just end this and update our twitter user with whatever the result was.
                db.twitterUser.update( { "_id": user['_id']}, user, upsert=True)

        except Exception as e:
            # Oh well, just keep going
            logging.error(e.__class__)
            logging.error(e)
            continue
        except KeyboardInterrupt:
            break
<|endoftext|>


<commit_before>from django.db import models

from cultures.models import Culture

class Question(models.Model):

    culture = models.ForeignKey(Culture)
    question = models.CharField(max_length=500)

    # Metadata
    created = models.DateTimeField(auto_now_add=True)
    modified = models.DateTimeField(auto_now=True)

    def __str__(self):
        return "Question: {question}".format(self.question)

class Answer(models.Model):

    question = models.ForeignKey(Question)
    answer = models.CharField(max_length=250)

    is_correct = models.BooleanField(default=False)

    # Metadata
    created = models.DateTimeField(auto_now_add=True)
    modified = models.DateTimeField(auto_now=True)

    def __str__(self):
        return "{answer} from {question}".format(
            answer=self.answer,
            question=self.question
        )
<commit_msg>Fix str method<commit_after>from django.db import models

from cultures.models import Culture

class Question(models.Model):

    culture = models.ForeignKey(Culture)
    question = models.CharField(max_length=500)

    # Metadata
    created = models.DateTimeField(auto_now_add=True)
    modified = models.DateTimeField(auto_now=True)

    def __str__(self):
        return "Question: {question}".format(question=self.question)

class Answer(models.Model):

    question = models.ForeignKey(Question)
    answer = models.CharField(max_length=250)

    is_correct = models.BooleanField(default=False)

    # Metadata
    created = models.DateTimeField(auto_now_add=True)
    modified = models.DateTimeField(auto_now=True)

    def __str__(self):
        return "{answer} from {question}".format(
            answer=self.answer,
            question=self.question
        )
<|endoftext|>


<commit_before>"""
Yahtzeething.py
"""
from ggame import App, Color, LineStyle, Sprite, RectangleAsset, CircleAsset, EllipseAsset, LineAsset
from ggame import ImageAsset, PolygonAsset, Frame, Sound, SoundAsset, TextAsset
import time
import math
import random

# colors
red = Color(0xff0000, 1.0)
green = Color(0x00ff00, 1.0)
blue = Color(0x0000ff, 1.0)
black = Color(0x000000, 1.0)
orange = Color(0xffa500, 1.0)
skin =Color(0xFCD15B, 1.0)
wall=Color(0xE8E8E8, 1.0)
orange=Color(0xFFa500,1.0)
platc=Color(0xB9BDBB,1.0)
gooy=Color(0xCDF238,1.0)
white=Color(0xFFFFFF,1.0)
darkblue=Color(0x052099,1.0)

# lines
thinline= LineStyle(1, black)
thickline= LineStyle(5, black)
thickishline= LineStyle(2.5, black)
noline= LineStyle(0, black)
portalline= LineStyle(1, blue)
portalline2= LineStyle(1, orange)

dielist=[]
dieroll =list.append(dielist(random.randrange(1,6,1)))
print(dieroll)<commit_msg>Updated from Brython Server: 3/1/2017 9:51:12 AM<commit_after>"""
Yahtzeething.py
"""
from ggame import App, Color, LineStyle, Sprite, RectangleAsset, CircleAsset, EllipseAsset, LineAsset
from ggame import ImageAsset, PolygonAsset, Frame, Sound, SoundAsset, TextAsset
import time
import math
import random

# colors
red = Color(0xff0000, 1.0)
green = Color(0x00ff00, 1.0)
blue = Color(0x0000ff, 1.0)
black = Color(0x000000, 1.0)
orange = Color(0xffa500, 1.0)
skin =Color(0xFCD15B, 1.0)
wall=Color(0xE8E8E8, 1.0)
orange=Color(0xFFa500,1.0)
platc=Color(0xB9BDBB,1.0)
gooy=Color(0xCDF238,1.0)
white=Color(0xFFFFFF,1.0)
darkblue=Color(0x052099,1.0)

# lines
thinline= LineStyle(1, black)
thickline= LineStyle(5, black)
thickishline= LineStyle(2.5, black)
noline= LineStyle(0, black)
portalline= LineStyle(1, blue)
portalline2= LineStyle(1, orange)

dielist=[]
dielist.append(random.randrange(1,6,1))
print(dieroll)<|endoftext|>