<commit_before>from rest_framework import status
from rest_framework import viewsets
from rest_framework.decorators import detail_route, list_route
from rest_framework.response import Response

from cognitive_tests import models
from cognitive_tests import utils
from cognitive_tests.api import permissions
from cognitive_tests.api import serializers

class NestedModelViewSet(viewsets.ModelViewSet):
    """
    The actions provided by the ModelViewSet class are:
        .list(), .retrieve(), .create(), .update(), .partial_update(), and .destroy().
    """
    NESTED_LIST_MAPPING = {
        'get': 'list',
        'post': 'create'
    }
    filter_kwargs = None  # required by ModelViewSet

    def __init__(self, queryset=None, filter_kwargs=None, **kwargs):
        super(NestedModelViewSet, self).__init__(**kwargs)
        if queryset:
            self.queryset = queryset
        elif filter_kwargs:
            self.queryset = self.queryset.filter(**filter_kwargs)
            self.filter_kwargs = filter_kwargs

    @classmethod
    def as_nested_view(cls, queryset=None, **kwargs):
        return cls.as_view(cls.NESTED_LIST_MAPPING, queryset=queryset, filter_kwargs=kwargs)

class ParticipantFiltered(viewsets.ModelViewSet):
    def __check(self, request, *args, **kwargs):
        if not request.user or not request.user.is_staff:
            participant = utils.get_participant(request)
            self.queryset = self.queryset.filter(participant=participant)

    def list(self, request, *args, **kwargs):
        self.__check(request, *args, **kwargs)
        return super(ParticipantFiltered, self).list(request, *args, **kwargs)

    def retrieve(self, request, *args, **kwargs):
        self.__check(request, *args, **kwargs)
        return super(ParticipantFiltered, self).retrieve(request, *args, **kwargs)

    def destroy(self, request, *args, **kwargs):
        self.__check(request, *args, **kwargs)
        return super(ParticipantFiltered, self).destroy(request, *args, **kwargs)

    def update(self, request, *args, **kwargs):
        self.__check(request, *args, **kwargs)
        return super(ParticipantFiltered, self).update(request, *args, **kwargs)

    def partial_update(self, request, *args, **kwargs):
        self.__check(request, *args, **kwargs)
        return super(ParticipantFiltered, self).partial_update(request, *args, **kwargs)

class TestResultValueViewSet(NestedModelViewSet):
    queryset = models.TestResultValue.objects.all()
    serializer_class = serializers.TestResultValueSerializer
    permission_classes = [permissions.IsParticipantOrStaff, ]

class TestMarkViewSet(NestedModelViewSet):
    queryset = models.TestMark.objects.all()
    serializer_class = serializers.TestMarkSerializer
    permission_classes = [permissions.IsStaffOrReadOnly, ]

    @detail_route(methods=['get', 'options'])
    def values(self, request, pk=None):
        return TestResultValueViewSet.as_nested_view(mark=self.get_object())(request)

class TestResultViewSet(ParticipantFiltered, NestedModelViewSet):
    queryset = models.TestResult.objects.all()
    serializer_class = serializers.TestResultSerializer
    permission_classes = [permissions.IsParticipantOrStaff, ]

    def create(self, request, *args, **kwargs):
        participant = utils.get_participant(request)
        survey_result_pk = request.POST.get('survey_result', None)
        survey_result = None
        if survey_result_pk:
            survey_result = models.SurveyResult.objects.get(pk=survey_result_pk)
        if not participant:
            return Response({'detail': 'No participant'}, status=status.HTTP_400_BAD_REQUEST)

        result = models.TestResult.objects.create(participant=participant,
                                                  survey_result=survey_result,
                                                  **self.filter_kwargs)

        result_serializer = serializers.TestResultSerializer(result, partial=True, data={},
                                                                 context={'request': request})
        if result_serializer.is_valid():
            result = result_serializer.save()

            for file_arg in request.FILES:
                for raw_file in request.FILES.getlist(file_arg):
                    file_serializer = serializers.TestResultFileSerializer(models.TestResultFile(result=result),
                                                                               data={'name': raw_file.name,
                                                                                'file': raw_file},
                                                                               partial=True)
                    if file_serializer.is_valid():
                        file_serializer.save()
                    else:
                        return Response(file_serializer.errors, status=status.HTTP_400_BAD_REQUEST)

            for arg in request.data:
                # TODO: TO VALIDATE
                if arg in models.TestResultTextData.RESTRICTED_NAMES or arg in result_serializer.fields or arg in request.FILES:
                    continue
                text_data_serializer = serializers.TestResultTextDataSerializer(
                    models.TestResultTextData(result=result),
                    data={'name': arg, 'data': request.data[arg]},
                    partial=True)
                if text_data_serializer.is_valid():
                    text_data_serializer.save()
                else:
                    return Response(text_data_serializer.errors, status=status.HTTP_400_BAD_REQUEST)
            result.process()
            return Response(result_serializer.data, status=status.HTTP_201_CREATED)
        return Response(result_serializer.errors, status=status.HTTP_400_BAD_REQUEST)

class TestViewSet(NestedModelViewSet):
    queryset = models.Test.objects.all()
    serializer_class = serializers.TestSerializer
    permission_classes = [permissions.IsParticipantOrStaff, ]

    @detail_route(methods=['get', 'post', 'options'])
    def results(self, request, pk=None):
        return TestResultViewSet(filter_kwargs={'test': self.get_object()}).create(request)
        #return TestResultViewSet.as_nested_view(test=self.get_object())(request)

    @detail_route(methods=['get', 'options'])
    def marks(self, request, pk=None):
        return TestMarkViewSet.as_nested_view(test=self.get_object())(request)

class TestResultTextDataViewSet(viewsets.ModelViewSet):
    queryset = models.TestResultTextData.objects.all()
    serializer_class = serializers.TestResultTextDataSerializer
    permission_classes = [permissions.IsStaff, ]

class TestResultFileViewSet(viewsets.ModelViewSet):
    queryset = models.TestResultFile.objects.all()
    serializer_class = serializers.TestResultFileSerializer
    permission_classes = [permissions.IsStaff, ]

class SurveyResultValueViewSet(NestedModelViewSet):
    queryset = models.SurveyResultValue.objects.all()
    serializer_class = serializers.SurveyResultValueSerializer
    permission_classes = [permissions.IsParticipantOrStaff, ]

class SurveyResultViewSet(ParticipantFiltered, NestedModelViewSet):
    queryset = models.SurveyResult.objects.all()
    serializer_class = serializers.SurveyResultSerializer
    permission_classes = [permissions.IsParticipantOrStaff, ]

class SurveyMarkViewSet(NestedModelViewSet):
    queryset = models.SurveyMark.objects.all()
    serializer_class = serializers.SurveyMarkSerializer
    permission_classes = [permissions.IsStaffOrReadOnly, ]

    @detail_route(methods=['get', 'options'])
    def values(self, request, pk=None):
        return SurveyResultValueViewSet.as_nested_view(mark=self.get_object())(request)

class SurveyViewSet(NestedModelViewSet):
    queryset = models.Survey.objects.all()
    serializer_class = serializers.SurveySerializer
    permission_classes = [permissions.IsStaffOrReadOnly, ]

    @detail_route(methods=['get', 'options'])
    def results(self, request, pk=None):
        return SurveyResultViewSet.as_nested_view(survey=self.get_object())(request)

    @detail_route(methods=['get', 'options'])
    def tests(self, request, pk=None):
        return TestViewSet.as_nested_view(queryset=self.get_object().tests)(request)

    @detail_route(methods=['get', 'options'])
    def marks(self, request, pk=None):
        return SurveyMarkViewSet.as_nested_view(survey=self.get_object())(request)

class ModuleViewSet(viewsets.ModelViewSet):
    queryset = models.Module.objects.all()
    serializer_class = serializers.ModuleSerializer
    permission_classes = [permissions.IsStaff, ]

    @detail_route(methods=['get', 'options'])
    def tests(self, request, pk=None):
        return TestViewSet.as_nested_view(module=self.get_object())(request)

    @detail_route(methods=['get', 'options'])
    def surveys(self, request, pk=None):
        return SurveyViewSet.as_nested_view(module=self.get_object())(request)

class ParticipantViewSet(viewsets.ModelViewSet):
    queryset = models.Participant.objects.all()
    serializer_class = serializers.ParticipantSerializer
    permission_classes = [permissions.IsParticipantOrStaff, ]

    @detail_route(methods=['get', 'options'])
    def testresults(self, request, pk=None):
        return TestResultViewSet.as_nested_view(participant=self.get_object())(request)

    @detail_route(methods=['get', 'options'])
    def surveyresults(self, request, pk=None):
        return SurveyResultViewSet.as_nested_view(participant=self.get_object())(request)

    @list_route(methods=['get', 'post', 'delete'])
    def current(self, request):
        participant = utils.get_participant(request)
        if request.method == 'GET':
            if not participant:
                return Response({'detail': 'you must sign up as a participant using POST'},
                                status=status.HTTP_400_BAD_REQUEST)

            serializer = serializers.ParticipantSerializer(participant, context={'request': request})
            return Response(serializer.data)

        if request.method == 'POST':
            if participant:
                return Response({'detail': 'participant already set'}, status=status.HTTP_400_BAD_REQUEST)
            serializer = serializers.ParticipantSerializer(data=request.data, context={'request': request})
            if serializer.is_valid():
                serializer.save()
                request.session[models.Participant.PARTICIPANT_SESSION_KEY] = request.session.session_key
                return Response(serializer.data, status=status.HTTP_201_CREATED)
            else:
                return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

        if request.method == 'DELETE':
            if not participant:
                return Response({'detail': 'you must sign up as a participant using POST'},
                                status=status.HTTP_400_BAD_REQUEST)
            del request.session[models.Participant.PARTICIPANT_SESSION_KEY]
            return Response({'detail': 'participant unset successfully'}, status=status.HTTP_202_ACCEPTED)
<commit_msg>API: Updated filtered viesets<commit_after>from rest_framework import status
from rest_framework import viewsets
from rest_framework.decorators import detail_route, list_route
from rest_framework.response import Response

from cognitive_tests import models
from cognitive_tests import utils
from cognitive_tests.api import permissions
from cognitive_tests.api import serializers

class FilteredModelViewSet(viewsets.ModelViewSet):
    """
    The actions provided by the ModelViewSet class are:
        .list(), .retrieve(), .create(), .update(), .partial_update(), and .destroy().
    """
    NESTED_LIST_MAPPING = {
        'get': 'list',
        'post': 'create'
    }
    filter_kwargs = None  # required by ModelViewSet

    def __init__(self, queryset=None, filter_kwargs=None, **kwargs):
        super(FilteredModelViewSet, self).__init__(**kwargs)
        if queryset:
            self.queryset = queryset
        elif filter_kwargs:
            self.queryset = self.queryset.filter(**filter_kwargs)
            self.filter_kwargs = filter_kwargs

    @classmethod
    def as_filtered_view(cls, mapping=NESTED_LIST_MAPPING, queryset=None, **kwargs):
        return cls.as_view(mapping, queryset=queryset, filter_kwargs=kwargs)

    @classmethod
    def proceed_filtered(cls, request, mapping=NESTED_LIST_MAPPING, queryset=None, **kwargs):
        if request.method in ('POST', 'PUT', 'PATCH', 'DELETE'):
            """
            For some reason we can't directly redirect POST-like methods to cls.as_view because of
            csrf_excempt and other stuff so request body become touched.
            We need to construct the request handler ourselves.
            """
            viewset = cls(queryset=queryset, filter_kwargs=kwargs)
            viewset.request = request
            if str.lower(request.method) not in mapping:
                raise RuntimeError('Method %s can\'t be proceeded' % request.method)
            action = getattr(viewset, mapping[str.lower(request.method)])
            return action(request)
        return cls.as_view(mapping, queryset=queryset, filter_kwargs=kwargs)(request)

class ParticipantFiltered(viewsets.ModelViewSet):
    def __check(self, request, *args, **kwargs):
        if not request.user or not request.user.is_staff:
            participant = utils.get_participant(request)
            self.queryset = self.queryset.filter(participant=participant)

    def list(self, request, *args, **kwargs):
        self.__check(request, *args, **kwargs)
        return super(ParticipantFiltered, self).list(request, *args, **kwargs)

    def retrieve(self, request, *args, **kwargs):
        self.__check(request, *args, **kwargs)
        return super(ParticipantFiltered, self).retrieve(request, *args, **kwargs)

    def destroy(self, request, *args, **kwargs):
        self.__check(request, *args, **kwargs)
        return super(ParticipantFiltered, self).destroy(request, *args, **kwargs)

    def update(self, request, *args, **kwargs):
        self.__check(request, *args, **kwargs)
        return super(ParticipantFiltered, self).update(request, *args, **kwargs)

    def partial_update(self, request, *args, **kwargs):
        self.__check(request, *args, **kwargs)
        return super(ParticipantFiltered, self).partial_update(request, *args, **kwargs)

class TestResultValueViewSet(FilteredModelViewSet):
    queryset = models.TestResultValue.objects.all()
    serializer_class = serializers.TestResultValueSerializer
    permission_classes = [permissions.IsParticipantOrStaff, ]

class TestMarkViewSet(FilteredModelViewSet):
    queryset = models.TestMark.objects.all()
    serializer_class = serializers.TestMarkSerializer
    permission_classes = [permissions.IsStaffOrReadOnly, ]

    @detail_route(methods=['get', 'options'])
    def values(self, request, pk=None):
        return TestResultValueViewSet.proceed_filtered(request, mark=self.get_object())

class TestResultViewSet(ParticipantFiltered, FilteredModelViewSet):
    queryset = models.TestResult.objects.all()
    serializer_class = serializers.TestResultSerializer
    permission_classes = [permissions.IsParticipantOrStaff, ]

    def create(self, request, *args, **kwargs):
        participant = utils.get_participant(request)
        survey_result_pk = request.POST.get('survey_result', None)
        survey_result = None
        if survey_result_pk:
            survey_result = models.SurveyResult.objects.get(pk=survey_result_pk)
        if not participant:
            return Response({'detail': 'No participant'}, status=status.HTTP_400_BAD_REQUEST)

        result = models.TestResult.objects.create(participant=participant,
                                                  survey_result=survey_result,
                                                  **self.filter_kwargs)

        result_serializer = serializers.TestResultSerializer(result, partial=True, data={},
                                                                 context={'request': request})
        if result_serializer.is_valid():
            result = result_serializer.save()

            for file_arg in request.FILES:
                for raw_file in request.FILES.getlist(file_arg):
                    file_serializer = serializers.TestResultFileSerializer(models.TestResultFile(result=result),
                                                                               data={'name': raw_file.name,
                                                                                'file': raw_file},
                                                                               partial=True)
                    if file_serializer.is_valid():
                        file_serializer.save()
                    else:
                        return Response(file_serializer.errors, status=status.HTTP_400_BAD_REQUEST)

            for arg in request.data:
                # TODO: TO VALIDATE
                if arg in models.TestResultTextData.RESTRICTED_NAMES or arg in result_serializer.fields or arg in request.FILES:
                    continue
                text_data_serializer = serializers.TestResultTextDataSerializer(
                    models.TestResultTextData(result=result),
                    data={'name': arg, 'data': request.data[arg]},
                    partial=True)
                if text_data_serializer.is_valid():
                    text_data_serializer.save()
                else:
                    return Response(text_data_serializer.errors, status=status.HTTP_400_BAD_REQUEST)
            result.process()
            return Response(result_serializer.data, status=status.HTTP_201_CREATED)
        return Response(result_serializer.errors, status=status.HTTP_400_BAD_REQUEST)

class TestViewSet(FilteredModelViewSet):
    queryset = models.Test.objects.all()
    serializer_class = serializers.TestSerializer
    permission_classes = [permissions.IsParticipantOrStaff, ]

    @detail_route(methods=['get', 'post', 'options'])
    def results(self, request, pk=None):
        return TestResultViewSet.proceed_filtered(request, test=self.get_object())

    @detail_route(methods=['get', 'options'])
    def marks(self, request, pk=None):
        return TestMarkViewSet.proceed_filtered(request, test=self.get_object())

class TestResultTextDataViewSet(viewsets.ModelViewSet):
    queryset = models.TestResultTextData.objects.all()
    serializer_class = serializers.TestResultTextDataSerializer
    permission_classes = [permissions.IsStaff, ]

class TestResultFileViewSet(viewsets.ModelViewSet):
    queryset = models.TestResultFile.objects.all()
    serializer_class = serializers.TestResultFileSerializer
    permission_classes = [permissions.IsStaff, ]

class SurveyResultValueViewSet(FilteredModelViewSet):
    queryset = models.SurveyResultValue.objects.all()
    serializer_class = serializers.SurveyResultValueSerializer
    permission_classes = [permissions.IsParticipantOrStaff, ]

class SurveyResultViewSet(ParticipantFiltered, FilteredModelViewSet):
    queryset = models.SurveyResult.objects.all()
    serializer_class = serializers.SurveyResultSerializer
    permission_classes = [permissions.IsParticipantOrStaff, ]

class SurveyMarkViewSet(FilteredModelViewSet):
    queryset = models.SurveyMark.objects.all()
    serializer_class = serializers.SurveyMarkSerializer
    permission_classes = [permissions.IsStaffOrReadOnly, ]

    @detail_route(methods=['get', 'options'])
    def values(self, request, pk=None):
        return SurveyResultValueViewSet.proceed_filtered(request, mark=self.get_object())

class SurveyViewSet(FilteredModelViewSet):
    queryset = models.Survey.objects.all()
    serializer_class = serializers.SurveySerializer
    permission_classes = [permissions.IsStaffOrReadOnly, ]

    @detail_route(methods=['get', 'options'])
    def results(self, request, pk=None):
        return SurveyResultViewSet.proceed_filtered(request, survey=self.get_object())

    @detail_route(methods=['get', 'options'])
    def tests(self, request, pk=None):
        return TestViewSet.proceed_filtered(request, queryset=self.get_object().tests)

    @detail_route(methods=['get', 'options'])
    def marks(self, request, pk=None):
        return SurveyMarkViewSet.proceed_filtered(request, survey=self.get_object())

class ModuleViewSet(viewsets.ModelViewSet):
    queryset = models.Module.objects.all()
    serializer_class = serializers.ModuleSerializer
    permission_classes = [permissions.IsStaff, ]

    @detail_route(methods=['get', 'options'])
    def tests(self, request, pk=None):
        return TestViewSet.proceed_filtered(request, module=self.get_object())

    @detail_route(methods=['get', 'options'])
    def surveys(self, request, pk=None):
        return SurveyViewSet.proceed_filtered(request, module=self.get_object())

class ParticipantViewSet(viewsets.ModelViewSet):
    queryset = models.Participant.objects.all()
    serializer_class = serializers.ParticipantSerializer
    permission_classes = [permissions.IsParticipantOrStaff, ]

    @detail_route(methods=['get', 'options'])
    def testresults(self, request, pk=None):
        return TestResultViewSet.proceed_filtered(request, participant=self.get_object())

    @detail_route(methods=['get', 'options'])
    def surveyresults(self, request, pk=None):
        return SurveyResultViewSet.proceed_filtered(request, participant=self.get_object())

    @list_route(methods=['get', 'post', 'delete'])
    def current(self, request):
        participant = utils.get_participant(request)
        if request.method == 'GET':
            if not participant:
                return Response({'detail': 'you must sign up as a participant using POST'},
                                status=status.HTTP_400_BAD_REQUEST)

            serializer = serializers.ParticipantSerializer(participant, context={'request': request})
            return Response(serializer.data)

        if request.method == 'POST':
            if participant:
                return Response({'detail': 'participant already set'}, status=status.HTTP_400_BAD_REQUEST)
            serializer = serializers.ParticipantSerializer(data=request.data, context={'request': request})
            if serializer.is_valid():
                serializer.save()
                request.session[models.Participant.PARTICIPANT_SESSION_KEY] = request.session.session_key
                return Response(serializer.data, status=status.HTTP_201_CREATED)
            else:
                return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

        if request.method == 'DELETE':
            if not participant:
                return Response({'detail': 'you must sign up as a participant using POST'},
                                status=status.HTTP_400_BAD_REQUEST)
            del request.session[models.Participant.PARTICIPANT_SESSION_KEY]
            return Response({'detail': 'participant unset successfully'}, status=status.HTTP_202_ACCEPTED)
<|endoftext|>


<commit_before>#!/usr/bin/python -tt
# Copyright 2010 Google Inc.
# Licensed under the Apache License, Version 2.0
# http://www.apache.org/licenses/LICENSE-2.0

# Google's Python Class
# http://code.google.com/edu/languages/google-python-class/

# Additional basic list exercises

# D. Given a list of numbers, return a list where
# all adjacent == elements have been reduced to a single element,
# so [1, 2, 2, 3] returns [1, 2, 3]. You may create a new list or
# modify the passed in list.
def remove_adjacent(nums):

    return

# E. Given two lists sorted in increasing order, create and return a merged
# list of all the elements in sorted order. You may modify the passed in lists.
# Ideally, the solution should work in "linear" time, making a single
# pass of both lists.
def linear_merge(list1, list2):
    # +++your code here+++
    return

# Note: the solution above is kind of cute, but unforunately list.pop(0)
# is not constant time with the standard python list implementation, so
# the above is not strictly linear time.
# An alternate approach uses pop(-1) to remove the endmost elements
# from each list, building a solution list which is backwards.
# Then use reversed() to put the result back in the correct order. That
# solution works in linear time, but is more ugly.

# Simple provided test() function used in main() to print
# what each function returns vs. what it's supposed to return.
def test(got, expected):
    if got == expected:
        prefix = ' OK '
    else:
        prefix = '  X '
    print('%s got: %s expected: %s' % (prefix, repr(got), repr(expected)))

# Calls the above functions with interesting inputs.
def main():
    print('remove_adjacent')
    test(remove_adjacent([1, 2, 2, 3]), [1, 2, 3])
    test(remove_adjacent([2, 2, 3, 3, 3]), [2, 3])
    test(remove_adjacent([]), [])

    print()
    print('linear_merge')
    test(linear_merge(['aa', 'xx', 'zz'], ['bb', 'cc']),
         ['aa', 'bb', 'cc', 'xx', 'zz'])
    test(linear_merge(['aa', 'xx'], ['bb', 'cc', 'zz']),
         ['aa', 'bb', 'cc', 'xx', 'zz'])
    test(linear_merge(['aa', 'aa'], ['aa', 'bb', 'bb']),
         ['aa', 'aa', 'aa', 'bb', 'bb'])

if __name__ == '__main__':
    main()
<commit_msg>solution of list2.py<commit_after>#!/usr/bin/python -tt
# Copyright 2010 Google Inc.
# Licensed under the Apache License, Version 2.0
# http://www.apache.org/licenses/LICENSE-2.0

# Google's Python Class
# http://code.google.com/edu/languages/google-python-class/

# Additional basic list exercises

# D. Given a list of numbers, return a list where
# all adjacent == elements have been reduced to a single element,
# so [1, 2, 2, 3] returns [1, 2, 3]. You may create a new list or
# modify the passed in list.
def remove_adjacent(nums):
    lst = []

    for num in nums:
        if not(num in lst):
            lst.append(num)

    return lst

# E. Given two lists sorted in increasing order, create and return a merged
# list of all the elements in sorted order. You may modify the passed in lists.
# Ideally, the solution should work in "linear" time, making a single
# pass of both lists.
def linear_merge(list1, list2):
    l = sorted(list1 + list2)

    return l

# Note: the solution above is kind of cute, but unforunately list.pop(0)
# is not constant time with the standard python list implementation, so
# the above is not strictly linear time.
# An alternate approach uses pop(-1) to remove the endmost elements
# from each list, building a solution list which is backwards.
# Then use reversed() to put the result back in the correct order. That
# solution works in linear time, but is more ugly.

# Simple provided test() function used in main() to print
# what each function returns vs. what it's supposed to return.
def test(got, expected):
    if got == expected:
        prefix = ' OK '
    else:
        prefix = '  X '
    print('%s got: %s expected: %s' % (prefix, repr(got), repr(expected)))

# Calls the above functions with interesting inputs.
def main():
    print('remove_adjacent')
    test(remove_adjacent([1, 2, 2, 3]), [1, 2, 3])
    test(remove_adjacent([2, 2, 3, 3, 3]), [2, 3])
    test(remove_adjacent([]), [])

    print()
    print('linear_merge')
    test(linear_merge(['aa', 'xx', 'zz'], ['bb', 'cc']),
         ['aa', 'bb', 'cc', 'xx', 'zz'])
    test(linear_merge(['aa', 'xx'], ['bb', 'cc', 'zz']),
         ['aa', 'bb', 'cc', 'xx', 'zz'])
    test(linear_merge(['aa', 'aa'], ['aa', 'bb', 'bb']),
         ['aa', 'aa', 'aa', 'bb', 'bb'])

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
# Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
#
from django.core.management.base import BaseCommand

from glob import glob
from optparse import make_option
from os import path
from sys import stdout
from django.conf import settings

class Command(BaseCommand):
    help = 'Reindex everything.'
    args = ''

    option_list = BaseCommand.option_list + (
        make_option('-C', '--check-just-read', action='store_true', dest='check', default=False,
                    help='Check snippets utf-8'),
        make_option('-c', '--check', action='store_true', dest='check2', default=False,
                    help='Check snippets utf-8 by walking through index'),
        )

    def handle(self, *args, **opts):
        from search.index import Search, Snippets

        if opts['check']:
            sfn = glob(settings.SEARCH_INDEX+'snippets/*')
            print sfn
            for fn in sfn:
                print fn
                bkid = int(path.basename(fn))
                with open(fn) as f:
                    cont = f.read()
                    try:
                        uc = cont.decode('utf-8')
                    except UnicodeDecodeError, ude:
                        print "error in snippets %d" % bkid
        if opts['check2']:
            s = Search()
            reader = s.searcher.getIndexReader()
            numdocs = reader.numDocs()
            for did in range(numdocs):
                doc = reader.document(did)
                if doc and doc.get('book_id'):
                    bkid = int(doc.get('book_id'))
                    # import pdb; pdb.set_trace()
                    stdout.write("\r%d / %d" % (did, numdocs))
                    stdout.flush()
                    ss = doc.get('snippet_position')
                    sl = doc.get('snippet_length')
                    if ss and sl:
                        # WTF (nie było zaimportowane)
                        snips = Snippets(bkid)
                        try:
                            txt = snips.get((ss, sl))
                            assert len(txt) == sl
                        except UnicodeDecodeError, ude:
                            stdout.write("\nerror in snippets %d\n" % bkid)
                            raise ude

            stdout.write("\ndone.\n")
<commit_msg>fix snippets command<commit_after># -*- coding: utf-8 -*-
# This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
# Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
#
from django.core.management.base import BaseCommand

from glob import glob
from os import path
from django.conf import settings

class Command(BaseCommand):
    help = 'Check snippets.'
    args = ''

    def handle(self, *args, **opts):
        sfn = glob(settings.SEARCH_INDEX+'snippets/*')
        for fn in sfn:
            print fn
            bkid = path.basename(fn)
            with open(fn) as f:
                cont = f.read()
                try:
                    cont.decode('utf-8')
                except UnicodeDecodeError:
                    print "error in snippets %s" % bkid
<|endoftext|>


<commit_before>#!/usr/bin/python

import serial, sys
import multiprocessing
import xml.etree.ElementTree as ET
import logging
from message import Msg
import json

logger = logging.getLogger(__name__)

class ReaderMsg(Msg):

    def __init__(self, rfid_data):
        self.msg_type = 'rfid'
        self.value = rfid_data
        self.needs_ack = True

class RfidReader(multiprocessing.Process):

    rfidPort = "/dev/serial0"
    baudrate = 9600
    timeout = 0.1

    def __init__(self, pipe, msg_queue, cardmap_file = '../data/cardmap.xml'):
        multiprocessing.Process.__init__(self)
        self.pipe = pipe
        self.msg_queue = msg_queue
        self.cardmap_file = cardmap_file
        self.read_cards()

    def run(self):
        self.ser = serial.Serial()
        self.ser.baudrate = self.baudrate
        self.ser.port = self.rfidPort
        self.ser.timeout = self.timeout

        self.ser.open()
        if self.ser.isOpen():
            logger.warning("Open: " + self.ser.portstr)
        logger.warning("RFID reader reading")
        try:
            self.ser.flushInput()
            start_flag = "\x02"
            end_flag = "\x03"
            while True:
                c = self.ser.read()
                if c == start_flag:
                    c = ''
                    rfidData = ''
                    counter = 0
                    while c != end_flag and c != start_flag and counter < 13:
                        c = self.ser.read()
                        rfidData += str(c)
                        counter += 1
                    if counter != 13:
                        continue
                    rfidData = rfidData.replace(end_flag, '')
                    if not(self.verify_checksum(rfidData)):
                        continue;
                    logger.info("Card Scanned: %s" % rfidData)
                    if rfidData in self.cards:
                        self.msg_queue.put(ReaderMsg(self.cards[rfidData]))
                    else:
                        logger.info("Adding new card")
                        self.add_card(rfidData)
                if self.pipe.poll():
                    cmnd = self.pipe.recv()
                    if cmnd[0] == 'reread':
                        self.read_cards()
                    if cmnd[0] == 'get_cards':
                        cards_id = {rfid : dict(self.cards[rfid], hid=self.get_card_id(rfid)) for rfid in self.cards}
                        self.pipe.send(('cards', json.dump(['cards', cards_id])))
                    if cmnd[0] == 'quit':
                        break
        except:
            #logger.error("error: %s" % sys.exc_info())
            print sys.exc_info()

        finally:
            self.ser.close()
        logger.warning("RFID reader terminating")

    def clean_rfid(self, data):
        d = ""
        for c in data[1:-1]:
            if (c < '0') or (c > 'F'):
                logger.error("Removed invalid character from rfid data")
                continue
            d += c
        return d

    def verify_checksum(self, rfid):
        if len(rfid) != 12:
            return False
        checksum = 0
        for i in range(0, 9, 2):
            checksum = checksum ^ (((int(rfid[i], 16)) << 4) + int(rfid[i + 1], 16))
        return checksum == (int(rfid[10], 16) << 4) + int(rfid[11], 16)

    def get_card_id(self, rfid):
        return int(rfid[4:10], 16)

    def add_card(self, rfid):
        self.read_cards()
        newkey = max(self.cards.values()) + 1
        logger.warning("Adding card rfid: %s, id: %s" % (rfid, newkey))
        self.tree.getroot().append(ET.Element(tag='card', attrib={'rfid': rfid, 'key': "%s" % newkey}))
        self.tree.write(self.cardmap_file, encoding='UTF-8')
        self.cards[rfid] = newkey

    def read_cards(self):
        self.cards = {}
        self.tree = ET.parse(self.cardmap_file)
        cardmap = self.tree.getroot()
        for card in cardmap:
            self.cards[card.attrib['rfid']] = int(card.attrib['key'])
        logger.info('Card map loaded')
        logger.debug(self.cards)

    def check_cards(self):
        for card in self.cards:
            is_valid = self.verify_checksum(card)
            name = "invalid"
            if is_valid:
                name = self.get_card_id(card)
            print self.cards[card], name, is_valid

if __name__ == "__main__":
    print "RFID card reader class"
    logging.basicConfig(level=logging.INFO)
    msg_queue = multiprocessing.Queue()
    to_worker, from_worker = multiprocessing.Pipe()
    reader = RfidReader(from_worker, msg_queue)
    reader.check_cards()
    reader.start()
    while True:
        msg = msg_queue.get()
        print msg.value

<commit_msg>Fixed bugs in reader.py<commit_after>#!/usr/bin/python

import serial, sys
import multiprocessing
import xml.etree.ElementTree as ET
import logging
from message import Msg
import json

logger = logging.getLogger(__name__)

class ReaderMsg(Msg):

    def __init__(self, rfid_data):
        self.msg_type = 'rfid'
        self.value = rfid_data
        self.needs_ack = True

class RfidReader(multiprocessing.Process):

    rfidPort = "/dev/serial0"
    baudrate = 9600
    timeout = 0.1

    def __init__(self, pipe, msg_queue, cardmap_file = '../data/cardmap.xml'):
        multiprocessing.Process.__init__(self)
        self.pipe = pipe
        self.msg_queue = msg_queue
        self.cardmap_file = cardmap_file
        self.read_cards()

    def run(self):
        self.ser = serial.Serial()
        self.ser.baudrate = self.baudrate
        self.ser.port = self.rfidPort
        self.ser.timeout = self.timeout

        self.ser.open()
        if self.ser.isOpen():
            logger.warning("Open: " + self.ser.portstr)
        logger.warning("RFID reader reading")
        try:
            self.ser.flushInput()
            start_flag = "\x02"
            end_flag = "\x03"
            while True:
                c = self.ser.read()
                if c == start_flag:
                    c = ''
                    rfidData = ''
                    counter = 0
                    while c != end_flag and c != start_flag and counter < 13:
                        c = self.ser.read()
                        rfidData += str(c)
                        counter += 1
                    if counter != 13:
                        continue
                    rfidData = rfidData.replace(end_flag, '')
                    if not(self.verify_checksum(rfidData)):
                        continue;
                    logger.info("Card Scanned: %s" % rfidData)
                    if rfidData in self.cards:
                        self.msg_queue.put(ReaderMsg(self.cards[rfidData]))
                    else:
                        logger.info("Adding new card")
                        self.add_card(rfidData)
                if self.pipe.poll():
                    cmnd = self.pipe.recv()
                    logger.debug("Reader received message %s" % cmnd[0])
                    if cmnd[0] == 'reread':
                        self.read_cards()
                    if cmnd[0] == 'get_cards':
                        cards_id = {rfid : dict(item_id=self.cards[rfid], hid=self.get_card_id(rfid)) for rfid in self.cards}
                        self.pipe.send(('cards', json.dumps(['cards', cards_id])))
                    if cmnd[0] == 'quit':
                        break
        except:
            #logger.error("error: %s" % sys.exc_info())
            print sys.exc_info()

        finally:
            self.ser.close()
        logger.warning("RFID reader terminating")

    def clean_rfid(self, data):
        d = ""
        for c in data[1:-1]:
            if (c < '0') or (c > 'F'):
                logger.error("Removed invalid character from rfid data")
                continue
            d += c
        return d

    def verify_checksum(self, rfid):
        if len(rfid) != 12:
            return False
        checksum = 0
        for i in range(0, 9, 2):
            checksum = checksum ^ (((int(rfid[i], 16)) << 4) + int(rfid[i + 1], 16))
        return checksum == (int(rfid[10], 16) << 4) + int(rfid[11], 16)

    def get_card_id(self, rfid):
        return int(rfid[4:10], 16)

    def add_card(self, rfid):
        self.read_cards()
        newkey = max(self.cards.values()) + 1
        logger.warning("Adding card rfid: %s, id: %s" % (rfid, newkey))
        self.tree.getroot().append(ET.Element(tag='card', attrib={'rfid': rfid, 'key': "%s" % newkey}))
        self.tree.write(self.cardmap_file, encoding='UTF-8')
        self.cards[rfid] = newkey

    def read_cards(self):
        self.cards = {}
        self.tree = ET.parse(self.cardmap_file)
        cardmap = self.tree.getroot()
        for card in cardmap:
            self.cards[card.attrib['rfid']] = int(card.attrib['key'])
        logger.info('Card map loaded')
        logger.debug(self.cards)

    def check_cards(self):
        for card in self.cards:
            is_valid = self.verify_checksum(card)
            name = "invalid"
            if is_valid:
                name = self.get_card_id(card)

if __name__ == "__main__":
    print "RFID card reader class"
    logging.basicConfig(level=logging.INFO)
    msg_queue = multiprocessing.Queue()
    to_worker, from_worker = multiprocessing.Pipe()
    reader = RfidReader(from_worker, msg_queue)
    reader.start()

    import time
    reader.check_cards()
    to_worker.send(('get_cards', ''))
    i = 0
    while not(to_worker.poll()):
        time.sleep(0.1)
        i += 1
        if i > 50: # we have waited 5 seconds
            print "Reader did not return cards"
            sys.exit(1)
    cmnd = to_worker.recv()
    print cmnd
    while True:
        msg = msg_queue.get()
        print msg.value

<|endoftext|>


<commit_before>#!/usr/bin/python
# This file is part of Archivematica.
#
# Copyright 2010-2011 Artefactual Systems Inc. <http://artefactual.com>
#
# Archivematica is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 2 of the License, or
# (at your option) any later version.
#
# Archivematica is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Archivematica.  If not, see <http://www.gnu.org/licenses/>.

# @package Archivematica
# @subpackage transcoder
# @author Joseph Perry <joseph@artefactual.com>
# @version svn: $Id$

import sys
import os
import uuid
from transcoder import main
from executeOrRunSubProcess import executeOrRun
import transcoder
#from premisXMLlinker import xmlNormalize
sys.path.append("/usr/lib/archivematica/MCPClient/clientScripts")
from fileAddedToSIP import addFileToSIP

global extractedCount
extractedCount = 1
removeOnceExtracted = True

date = sys.argv[4].__str__().split(".", 1)[0]
replacementDic = { \
        "%inputFile%": transcoder.fileFullName, \
        "%outputDirectory%": transcoder.fileFullName + '-' + date \
        }

def onceExtracted(command):
    extractedFiles = []
    print "TODO - Metadata regarding removal of extracted archive"
    if removeOnceExtracted:
        os.remove(replacementDic["%inputFile%"])
    print "OUTPUT DIRECTORY: ", replacementDic["%outputDirectory%"]
    for w in os.walk(replacementDic["%outputDirectory%"].replace("*", "asterisk*")):
        path, directories, files = w
        for p in files:
            p = os.path.join(path, p)
            #print "path: ", p
            if os.path.isfile(p):
                extractedFiles.append(p)
    for ef in extractedFiles:
        fileUUID = uuid.uuid4().__str__()
        #print "File Extracted:", ef
        if True: #Add the file to the SIP
            #<arguments>"%relativeLocation%" "%SIPObjectsDirectory%" "%SIPLogsDirectory%" "%date%" "%taskUUID%" "%fileUUID%"</arguments>
            objectsDirectory = sys.argv[2].__str__()
            logsDirectory = sys.argv[3].__str__()
            date = sys.argv[4].__str__()
            taskUUID = sys.argv[5].__str__()
            packageFileUUID = sys.argv[6].__str__()

            objects = "objects/"
            print "File Extracted:: {" + fileUUID + "} ", ef.replace(objectsDirectory, objects, 1)

            relativeFilePath = ef.replace(objectsDirectory, objects, 1)
            addFileToSIP( objectsDirectory, logsDirectory, ef, fileUUID, "unpacking", date, date, eventDetailText=command.eventDetailCommand.stdOut.__str__(), eventOutcomeDetailNote="extracted " + relativeFilePath)

        run = sys.argv[0].__str__() + \
        " \"" + transcoder.escapeForCommand(ef) + "\""
        if True: #Add the file to the SIP
            run = run + " \"" + transcoder.escapeForCommand(sys.argv[2].__str__()) + "\"" + \
            " \"" + transcoder.escapeForCommand(sys.argv[3].__str__()) + "\"" + \
            " \"" + transcoder.escapeForCommand(sys.argv[4].__str__()) + "\"" + \
            " \"" + transcoder.escapeForCommand(sys.argv[5].__str__()) + "\"" + \
            " \"" + fileUUID + "\""

        exitCode, stdOut, stdError = executeOrRun("command", run)
        print stdOut
        print >>sys.stderr, stdError
        if exitCode != 0 and command.exitCode == 0:
            command.exitCode = exitCode

    global extractedCount
    date = sys.argv[4].__str__()
    extractedCount = extractedCount + 1
    replacementDic["%outputDirectory%"] = transcoder.fileFullName + '-' + extractedCount.__str__() + '-' + date

def identifyCommands(fileName):
    """Identify file type(s)"""
    ret = []
    removeOnceExtractedSkip = ['.part01.rar', '.r01', '.pst']

    RarExtensions = ['.part01.rar', '.r01', '.rar']
    for extension in RarExtensions:
        if fileName.lower().endswith(extension.lower()):
            #sql find the file type,
            c=transcoder.database.cursor()
            sql = """SELECT CR.pk, CR.command, CR.GroupMember
            FROM CommandRelationships AS CR
            JOIN FileIDs ON CR.fileID=FileIDs.pk
            JOIN CommandClassifications ON CR.commandClassification = CommandClassifications.pk
            WHERE FileIDs.description='unrar-nonfreeCompatable'
            AND CommandClassifications.classification = 'extract';"""
            c.execute(sql)
            row = c.fetchone()
            while row != None:
                ret.append(row)
                row = c.fetchone()
            break

    SevenZipExtensions = ['.ARJ', '.CAB', '.CHM', '.CPIO',
                  '.DMG', '.HFS', '.LZH', '.LZMA',
                  '.NSIS', '.UDF', '.WIM', '.XAR',
                  '.Z', '.ZIP', '.GZIP', '.TAR',]
    for extension in SevenZipExtensions:
        if fileName.lower().endswith(extension.lower()):
            c=transcoder.database.cursor()
            sql = """SELECT CR.pk, CR.command, CR.GroupMember
            FROM CommandRelationships AS CR
            JOIN FileIDs ON CR.fileID=FileIDs.pk
            JOIN CommandClassifications ON CR.commandClassification = CommandClassifications.pk
            WHERE FileIDs.description='7ZipCompatable'
            AND CommandClassifications.classification = 'extract';"""
            c.execute(sql)
            row = c.fetchone()
            while row != None:
                ret.append(row)
                row = c.fetchone()
            break
    if fileName.lower().endswith('.pst'):
        global removeOnceExtracted
        c=transcoder.database.cursor()
        sql = """SELECT CR.pk, CR.command, CR.GroupMember
        FROM CommandRelationships AS CR
        JOIN FileIDs ON CR.fileID=FileIDs.pk
        JOIN CommandClassifications ON CR.commandClassification = CommandClassifications.pk
        WHERE FileIDs.description='A .pst file'
        AND CommandClassifications.classification = 'extract';"""
        c.execute(sql)
        row = c.fetchone()
        while row != None:
            ret.append(row)
            row = c.fetchone()

    #check if not to remove
    for extension in removeOnceExtractedSkip:
        if fileName.lower().endswith(extension.lower()):
            removeOnceExtracted = False
            break
    return ret

if __name__ == '__main__':
    transcoder.onSuccess = onceExtracted
    transcoder.identifyCommands = identifyCommands
    transcoder.replacementDic = replacementDic
    filename = sys.argv[1].__str__()
    print filename
    main(filename)

<commit_msg>Fix Issue 627: 	Inconsistent folder name formatting in PST AIP<commit_after>#!/usr/bin/python
# This file is part of Archivematica.
#
# Copyright 2010-2011 Artefactual Systems Inc. <http://artefactual.com>
#
# Archivematica is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 2 of the License, or
# (at your option) any later version.
#
# Archivematica is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Archivematica.  If not, see <http://www.gnu.org/licenses/>.

# @package Archivematica
# @subpackage transcoder
# @author Joseph Perry <joseph@artefactual.com>
# @version svn: $Id$

import sys
import os
import uuid
from transcoder import main
from executeOrRunSubProcess import executeOrRun
import transcoder
#from premisXMLlinker import xmlNormalize
sys.path.append("/usr/lib/archivematica/MCPClient/clientScripts")
from fileAddedToSIP import addFileToSIP

global extractedCount
extractedCount = 1
removeOnceExtracted = True

date = sys.argv[4].__str__().split(".", 1)[0]
replacementDic = { \
        "%inputFile%": transcoder.fileFullName, \
        "%outputDirectory%": transcoder.fileFullName + '-' + date \
        }

def onceExtracted(command):
    extractedFiles = []
    print "TODO - Metadata regarding removal of extracted archive"
    if removeOnceExtracted:
        os.remove(replacementDic["%inputFile%"])
    print "OUTPUT DIRECTORY: ", replacementDic["%outputDirectory%"]
    for w in os.walk(replacementDic["%outputDirectory%"].replace("*", "asterisk*")):
        path, directories, files = w
        for p in files:
            p = os.path.join(path, p)
            #print "path: ", p
            if os.path.isfile(p):
                extractedFiles.append(p)
    for ef in extractedFiles:
        fileUUID = uuid.uuid4().__str__()
        #print "File Extracted:", ef
        if True: #Add the file to the SIP
            #<arguments>"%relativeLocation%" "%SIPObjectsDirectory%" "%SIPLogsDirectory%" "%date%" "%taskUUID%" "%fileUUID%"</arguments>
            objectsDirectory = sys.argv[2].__str__()
            logsDirectory = sys.argv[3].__str__()
            date = sys.argv[4].__str__()
            taskUUID = sys.argv[5].__str__()
            packageFileUUID = sys.argv[6].__str__()

            objects = "objects/"
            print "File Extracted:: {" + fileUUID + "} ", ef.replace(objectsDirectory, objects, 1)

            relativeFilePath = ef.replace(objectsDirectory, objects, 1)
            addFileToSIP( objectsDirectory, logsDirectory, ef, fileUUID, "unpacking", date, date, eventDetailText=command.eventDetailCommand.stdOut.__str__(), eventOutcomeDetailNote="extracted " + relativeFilePath)

        run = sys.argv[0].__str__() + \
        " \"" + transcoder.escapeForCommand(ef) + "\""
        if True: #Add the file to the SIP
            run = run + " \"" + transcoder.escapeForCommand(sys.argv[2].__str__()) + "\"" + \
            " \"" + transcoder.escapeForCommand(sys.argv[3].__str__()) + "\"" + \
            " \"" + transcoder.escapeForCommand(sys.argv[4].__str__()) + "\"" + \
            " \"" + transcoder.escapeForCommand(sys.argv[5].__str__()) + "\"" + \
            " \"" + fileUUID + "\""

        exitCode, stdOut, stdError = executeOrRun("command", run)
        print stdOut
        print >>sys.stderr, stdError
        if exitCode != 0 and command.exitCode == 0:
            command.exitCode = exitCode

    global extractedCount
    date = sys.argv[4].__str__().split(".", 1)[0]
    extractedCount = extractedCount + 1
    replacementDic["%outputDirectory%"] = transcoder.fileFullName + '-' + extractedCount.__str__() + '-' + date

def identifyCommands(fileName):
    """Identify file type(s)"""
    ret = []
    removeOnceExtractedSkip = ['.part01.rar', '.r01', '.pst']

    RarExtensions = ['.part01.rar', '.r01', '.rar']
    for extension in RarExtensions:
        if fileName.lower().endswith(extension.lower()):
            #sql find the file type,
            c=transcoder.database.cursor()
            sql = """SELECT CR.pk, CR.command, CR.GroupMember
            FROM CommandRelationships AS CR
            JOIN FileIDs ON CR.fileID=FileIDs.pk
            JOIN CommandClassifications ON CR.commandClassification = CommandClassifications.pk
            WHERE FileIDs.description='unrar-nonfreeCompatable'
            AND CommandClassifications.classification = 'extract';"""
            c.execute(sql)
            row = c.fetchone()
            while row != None:
                ret.append(row)
                row = c.fetchone()
            break

    SevenZipExtensions = ['.ARJ', '.CAB', '.CHM', '.CPIO',
                  '.DMG', '.HFS', '.LZH', '.LZMA',
                  '.NSIS', '.UDF', '.WIM', '.XAR',
                  '.Z', '.ZIP', '.GZIP', '.TAR',]
    for extension in SevenZipExtensions:
        if fileName.lower().endswith(extension.lower()):
            c=transcoder.database.cursor()
            sql = """SELECT CR.pk, CR.command, CR.GroupMember
            FROM CommandRelationships AS CR
            JOIN FileIDs ON CR.fileID=FileIDs.pk
            JOIN CommandClassifications ON CR.commandClassification = CommandClassifications.pk
            WHERE FileIDs.description='7ZipCompatable'
            AND CommandClassifications.classification = 'extract';"""
            c.execute(sql)
            row = c.fetchone()
            while row != None:
                ret.append(row)
                row = c.fetchone()
            break
    if fileName.lower().endswith('.pst'):
        global removeOnceExtracted
        c=transcoder.database.cursor()
        sql = """SELECT CR.pk, CR.command, CR.GroupMember
        FROM CommandRelationships AS CR
        JOIN FileIDs ON CR.fileID=FileIDs.pk
        JOIN CommandClassifications ON CR.commandClassification = CommandClassifications.pk
        WHERE FileIDs.description='A .pst file'
        AND CommandClassifications.classification = 'extract';"""
        c.execute(sql)
        row = c.fetchone()
        while row != None:
            ret.append(row)
            row = c.fetchone()

    #check if not to remove
    for extension in removeOnceExtractedSkip:
        if fileName.lower().endswith(extension.lower()):
            removeOnceExtracted = False
            break
    return ret

if __name__ == '__main__':
    transcoder.onSuccess = onceExtracted
    transcoder.identifyCommands = identifyCommands
    transcoder.replacementDic = replacementDic
    filename = sys.argv[1].__str__()
    print filename
    main(filename)

<|endoftext|>


<commit_before>import libxml2, sys, time, os, weakref, logging
from types import ListType

from kaa.db import *
from kaa import ipc
from kaa.notifier import Signal

__all__ = ['DEFAULT_EPG_PORT', 'GuideServer']

DEFAULT_EPG_PORT = 4132

log = logging.getLogger('epg')

# TODO: merge updates when processing instead of wipe.

class GuideServer(object):
    def __init__(self, socket, address = None, dbfile = "/tmp/GuideServer.db",
                 auth_secret = None, log_file = "/tmp/GuideServer.log",
                 log_level = logging.INFO):

        # setup logger
        # TODO: get a better format!  half of my screen is taken up before getting
        #       to the log message.  A better time format would help, ie:
        #       '%Y%m%d %H:%M:%S'
        f = logging.Formatter('%(asctime)s %(levelname)-8s [%(name)6s] '+\
                              '%(filename)s %(lineno)s: '+\
                              '%(message)s')
        handler = logging.FileHandler(log_file)
        handler.setFormatter(f)
        logging.getLogger().addHandler(handler)

        # setup log
        logging.getLogger().setLevel(log_level)
        log.info('start EPG server')
        log.info('using database in %s', dbfile)

        db = Database(dbfile)
        db.register_object_type_attrs("channel",
            tuner_id   = (list, ATTR_SIMPLE),
            name = (unicode, ATTR_SEARCHABLE),
            long_name  = (unicode, ATTR_SEARCHABLE),
        )
        db.register_object_type_attrs("program",
            [ ("start", "stop") ],
            title = (unicode, ATTR_KEYWORDS),
            desc = (unicode, ATTR_KEYWORDS),
            date = (int, ATTR_SEARCHABLE),
            start = (int, ATTR_SEARCHABLE),
            stop = (int, ATTR_SEARCHABLE),
            ratings = (dict, ATTR_SIMPLE)
        )

        self.signals = {
            "updated": Signal(),
            "update_progress": Signal()
        }

        self._clients = []
        self._db = db
        self._load()

        self._ipc = ipc.IPCServer(socket, auth_secret = auth_secret)
        self._ipc.signals["client_connected"].connect_weak(self._client_connected)
        self._ipc.signals["client_closed"].connect_weak(self._client_closed)
        self._ipc.register_object(self, "guide")

        self._ipc_net = None

        if address and \
           address.split(':')[0] not in ['127.0.0.1', '0.0.0.0']:
            # listen on tcp port too
            if address.find(':') >= 0:
                host, port = address.split(':', 1)
            else:
                host = address
                port = DEFAULT_EPG_PORT

            self._ipc_net = ipc.IPCServer((host, int(port)), auth_secret = auth_secret)
            log.info('listening on address %s:%s', host, port)
            self._ipc_net.signals["client_connected"].connect_weak(self._client_connected)
            self._ipc_net.signals["client_closed"].connect_weak(self._client_closed)
            self._ipc_net.register_object(self, "guide")

    def _load(self):
        self._max_program_length = self._num_programs = 0
        q = "SELECT stop-start AS length FROM objects_program ORDER BY length DESC LIMIT 1"
        res = self.get_db()._db_query(q)
        if len(res):
            self._max_program_length = res[0][0]

        res = self.get_db()._db_query("SELECT count(*) FROM objects_program")
        if len(res):
            self._num_programs = res[0][0]

        self._tuner_ids = []
        channels = self._db.query(type = "channel")
        for c in channels:
            for t in c["tuner_id"]:
                if t in self._tuner_ids:
                    log.warning('loading channel %s with tuner_id %s '+\
                                'allready claimed by another channel',
                                c["name"], t)
                else:
                    self._tuner_ids.append(t)

    def _client_connected(self, client):
        """
        Connect a new client to the server.
        """
        self._clients.append(client)

    def _client_closed(self, client):
        for signal in self.signals.values():
            for callback in signal:
                if ipc.get_ipc_from_proxy(callback) == client:
                    signal.disconnect(callback)

        for c in self._clients:
            if c == client:
                log.warning('disconnect client')
                self._clients.remove(c)

    def update(self, backend, *args, **kwargs):
        try:
            exec('import source_%s as backend' % backend)
        except ImportError:
            raise ValueError, "No such update backend '%s'" % backend

        # TODO: delte old programs
        # self._wipe()
        self.signals["update_progress"].connect_weak(self._update_progress, time.time())
        backend.update(self, *args, **kwargs)

    def _update_progress(self, cur, total, update_start_time):
        if total <= 0:
            # Processing something, but don't yet know how much
            n = 0
        else:
            n = int((cur / float(total)) * 50)

        # Temporary: output progress status to stdout.
        # sys.stdout.write("|%51s| %d / %d\r" % (("="*n + ">").ljust(51), cur, total))
        # sys.stdout.flush()

        if cur == total:
            self._db.commit()
            self._load()
            self.signals["updated"].emit()
            self.signals["update_progress"].disconnect(self._update_progress)
            print "\nProcessed in %.02f seconds." % (time.time()-update_start_time)

    def _wipe(self):
        t0=time.time()
        self._db.delete_by_query()
        self._channel_id_to_db_id = {}

    def _add_channel_to_db(self, tuner_id, name, long_name):
        """
        This method requires at least one of tuner_id, name, long_name.
        Depending on the source (various XMLTV sources, Zap2it, etc.) not all
        of the information we would like is available.  Also, channels are
        perceived differently around the world and handled differently by
        differnent systems (DVB, analog TV).

        Following the KISS philosophy (Keep It Simple Stupid) we can follow some
        simple rules.

        The most important field here is name.  If there's no name
        we make it based on tuner_id or long_name.  If there's no long_name we
        base that on name or tuner_id.  If there's no tuner_id it does
        not matter because we will then always have a value for name.
        If there is a tuner_id then it will assist programs using kaa.epg to
        match real channels and EPG data.
        """

        if type(tuner_id) != ListType:
            tuner_id = [ tuner_id ]

        # require at least one field
        if not tuner_id and not name and not long_name:
            log.error('need at least one field to add a channel')
            return None

        if not name:
            # then there must be one of the others
            if tuner_id:
                name = tuner_id[0]
            else:
                name = long_name

        if not long_name:
            # then there must be one of the others
            if name:
                long_name = name
            elif tuner_id:
                long_name = tuner_id[0]

        if not tuner_id:
            tuner_id = [ name ]

        c2 = self._db.query(type = "channel", name = name)
        if len(c2):
            c2 = c2[0]

            for t in tuner_id:
                if t not in c2["tuner_id"]:
                    if t in self._tuner_ids:
                        log.warning('not adding tuner_id %s for channel %s - '+\
                            'it is claimed by another channel', t, name)
                    else:
                        # only add this id if it's not already there and not
                        # claimed by another channel
                        c2["tuner_id"].append(t)
                        self._tuner_ids.append(t)

            # TODO: if everything is the same do not update
            self._db.update_object(("channel", c2["id"]),
                                   tuner_id = c2["tuner_id"],
                                   long_name = long_name)
            return c2["id"]

        for t in tuner_id:
            if t in self._tuner_ids:
                log.warning('not adding tuner_id %s for channel %s - it is '+\
                            'claimed by another channel', t, name)
                tuner_id.remove(t)
            else:
                self._tuner_ids.append(t)

        o = self._db.add_object("channel",
                                tuner_id = tuner_id,
                                name = name,
                                long_name = long_name)
        return o["id"]

    def _add_program_to_db(self, channel_db_id, start, stop, title, desc):
        #log.debug('channel_db_id: "%s" start: "%s" title: "%s"',
        #          channel_db_id, start, title)

        # TODO: check time range
        p2 = self._db.query(parent = ("channel", channel_db_id),
                            type = "program", start = start)

        if len(p2):
            # we have a program at this time
            p2 = p2[0]

            #log.debug('updating program: %s', p2["title"])
            # TODO: if everything is the same do not update
            self._db.update_object(("program", p2["id"]),
                                   start = start,
                                   stop = stop,
                                   title = title,
                                   desc = desc)
            return p2["id"]

        # TODO: check title, see if it is a different program.  Also check
        #       if the program is the same but shifted times a bit
        else:
            #log.debug('adding program: %s', title)
            o = self._db.add_object("program",
                                    parent = ("channel", channel_db_id),
                                    start = start,
                                    stop = stop,
                                    title = title,
                                    desc = desc, ratings = 42)

            if stop - start > self._max_program_length:
                self._max_program_length = stop = start
            return o["id"]

    def query(self, **kwargs):
        if "channel" in kwargs:
            if type(kwargs["channel"]) in (list, tuple):
                kwargs["parent"] = [("channel", x) for x in kwargs["channel"]]
            else:
                kwargs["parent"] = "channel", kwargs["channel"]
            del kwargs["channel"]

        for key in kwargs.copy():
            if key.startswith("__ipc_"):
                del kwargs[key]

        res = self._db.query_raw(**kwargs)
        return res

    def get_db(self):
        return self._db

    def get_max_program_length(self):
        return self._max_program_length

    def get_num_programs(self):
        return self._num_programs

if __name__ == "__main__":
    # ARGS: log file, log level, db file

    # python imports
    import gc
    import sys

    # kaa imports
    from kaa.notifier import Timer, execute_in_timer, loop

    @execute_in_timer(Timer, 1)
    def garbage_collect():
        g = gc.collect()
        if g:
            log.debug('gc: deleted %s objects' % g)
        if gc.garbage:
            log.warning('gc: found %s garbage objects' % len(gc.garbage))
            for g in gc.garbage:
                log.warning(g)
        return True

    shutdown_timer = 5

    @execute_in_timer(Timer, 1)
    def autoshutdown():
        global shutdown_timer
        global _server

        #log.debug("clients: %s", len(_server._clients))
        if _server and len(_server._clients) > 0:
            shutdown_timer = 5
            return True
        shutdown_timer -= 1
        if shutdown_timer == 0:
            log.info('shutdown EPG server')
            sys.exit(0)
        return True

    try:
        # detach for parent using a new sesion
        os.setsid()
    except OSError:
        # looks like we are started from the shell
        # TODO: start some extra debug here and disable autoshutdown
        pass

    if len(sys.argv) < 5:
        address = None
    else:
        address=sys.argv[4]

    _server = GuideServer("epg", log_file=str(sys.argv[1]),
                          log_level=int(sys.argv[2]), dbfile=sys.argv[3],
                          address=sys.argv[4])

    garbage_collect()
    autoshutdown()
    loop()
<commit_msg>fix name handling<commit_after>import libxml2, sys, time, os, weakref, logging
from types import ListType

from kaa.db import *
from kaa import ipc
from kaa.notifier import Signal

__all__ = ['DEFAULT_EPG_PORT', 'GuideServer']

DEFAULT_EPG_PORT = 4132

log = logging.getLogger('epg')

# TODO: merge updates when processing instead of wipe.

class GuideServer(object):
    def __init__(self, socket, address = None, dbfile = "/tmp/GuideServer.db",
                 auth_secret = None, log_file = "/tmp/GuideServer.log",
                 log_level = logging.INFO):

        # setup logger
        # TODO: get a better format!  half of my screen is taken up before getting
        #       to the log message.  A better time format would help, ie:
        #       '%Y%m%d %H:%M:%S'
        f = logging.Formatter('%(asctime)s %(levelname)-8s [%(name)6s] '+\
                              '%(filename)s %(lineno)s: '+\
                              '%(message)s')
        handler = logging.FileHandler(log_file)
        handler.setFormatter(f)
        logging.getLogger().addHandler(handler)

        # setup log
        logging.getLogger().setLevel(log_level)
        log.info('start EPG server')
        log.info('using database in %s', dbfile)

        db = Database(dbfile)
        db.register_object_type_attrs("channel",
            tuner_id   = (list, ATTR_SIMPLE),
            name = (unicode, ATTR_SEARCHABLE),
            long_name  = (unicode, ATTR_SEARCHABLE),
        )
        db.register_object_type_attrs("program",
            [ ("start", "stop") ],
            title = (unicode, ATTR_KEYWORDS),
            desc = (unicode, ATTR_KEYWORDS),
            date = (int, ATTR_SEARCHABLE),
            start = (int, ATTR_SEARCHABLE),
            stop = (int, ATTR_SEARCHABLE),
            ratings = (dict, ATTR_SIMPLE)
        )

        self.signals = {
            "updated": Signal(),
            "update_progress": Signal()
        }

        self._clients = []
        self._db = db
        self._load()

        self._ipc = ipc.IPCServer(socket, auth_secret = auth_secret)
        self._ipc.signals["client_connected"].connect_weak(self._client_connected)
        self._ipc.signals["client_closed"].connect_weak(self._client_closed)
        self._ipc.register_object(self, "guide")

        self._ipc_net = None

        if address and \
           address.split(':')[0] not in ['127.0.0.1', '0.0.0.0']:
            # listen on tcp port too
            if address.find(':') >= 0:
                host, port = address.split(':', 1)
            else:
                host = address
                port = DEFAULT_EPG_PORT

            self._ipc_net = ipc.IPCServer((host, int(port)), auth_secret = auth_secret)
            log.info('listening on address %s:%s', host, port)
            self._ipc_net.signals["client_connected"].connect_weak(self._client_connected)
            self._ipc_net.signals["client_closed"].connect_weak(self._client_closed)
            self._ipc_net.register_object(self, "guide")

    def _load(self):
        self._max_program_length = self._num_programs = 0
        q = "SELECT stop-start AS length FROM objects_program ORDER BY length DESC LIMIT 1"
        res = self.get_db()._db_query(q)
        if len(res):
            self._max_program_length = res[0][0]

        res = self.get_db()._db_query("SELECT count(*) FROM objects_program")
        if len(res):
            self._num_programs = res[0][0]

        self._tuner_ids = []
        channels = self._db.query(type = "channel")
        for c in channels:
            for t in c["tuner_id"]:
                if t in self._tuner_ids:
                    log.warning('loading channel %s with tuner_id %s '+\
                                'allready claimed by another channel',
                                c["name"], t)
                else:
                    self._tuner_ids.append(t)

    def _client_connected(self, client):
        """
        Connect a new client to the server.
        """
        self._clients.append(client)

    def _client_closed(self, client):
        for signal in self.signals.values():
            for callback in signal:
                if ipc.get_ipc_from_proxy(callback) == client:
                    signal.disconnect(callback)

        for c in self._clients:
            if c == client:
                log.warning('disconnect client')
                self._clients.remove(c)

    def update(self, backend, *args, **kwargs):
        log.error('update')
        try:
            exec('import source_%s as backend' % backend)
        except ImportError:
            raise ValueError, "No such update backend '%s'" % backend

        # TODO: delte old programs
        # self._wipe()
        self.signals["update_progress"].connect_weak(self._update_progress, time.time())
        backend.update(self, *args, **kwargs)

    def _update_progress(self, cur, total, update_start_time):
        if total <= 0:
            # Processing something, but don't yet know how much
            n = 0
        else:
            n = int((cur / float(total)) * 50)

        # Temporary: output progress status to stdout.
        # sys.stdout.write("|%51s| %d / %d\r" % (("="*n + ">").ljust(51), cur, total))
        # sys.stdout.flush()

        if cur == total:
            self._db.commit()
            self._load()
            self.signals["updated"].emit()
            self.signals["update_progress"].disconnect(self._update_progress)
            log.info("Processed in %.02f seconds." % (time.time()-update_start_time))

    def _wipe(self):
        t0=time.time()
        self._db.delete_by_query()
        self._channel_id_to_db_id = {}

    def _add_channel_to_db(self, tuner_id, name, long_name):
        """
        This method requires at least one of tuner_id, name, long_name.
        Depending on the source (various XMLTV sources, Zap2it, etc.) not all
        of the information we would like is available.  Also, channels are
        perceived differently around the world and handled differently by
        differnent systems (DVB, analog TV).

        Following the KISS philosophy (Keep It Simple Stupid) we can follow some
        simple rules.

        The most important field here is name.  If there's no name
        we make it based on tuner_id or long_name.  If there's no long_name we
        base that on name or tuner_id.  If there's no tuner_id it does
        not matter because we will then always have a value for name.
        If there is a tuner_id then it will assist programs using kaa.epg to
        match real channels and EPG data.
        """

        log.info('add channel %s %s %s', tuner_id, name, long_name)
        if type(tuner_id) != ListType and tuner_id:
            tuner_id = [ tuner_id ]

        # require at least one field
        if not tuner_id and not name and not long_name:
            log.error('need at least one field to add a channel')
            return None

        if not name:
            # then there must be one of the others
            if tuner_id:
                name = tuner_id[0]
            else:
                name = long_name

        if not long_name:
            # then there must be one of the others
            if name:
                long_name = name
            elif tuner_id:
                long_name = tuner_id[0]

        if not tuner_id:
            tuner_id = [ name ]

        c2 = self._db.query(type = "channel", name = name)
        if len(c2):
            c2 = c2[0]

            for t in tuner_id:
                if t not in c2["tuner_id"]:
                    if t in self._tuner_ids:
                        log.warning('not adding tuner_id %s for channel %s - '+\
                            'it is claimed by another channel', t, name)
                    else:
                        # only add this id if it's not already there and not
                        # claimed by another channel
                        c2["tuner_id"].append(t)
                        self._tuner_ids.append(t)

            # TODO: if everything is the same do not update
            self._db.update_object(("channel", c2["id"]),
                                   tuner_id = c2["tuner_id"],
                                   long_name = long_name)
            return c2["id"]

        for t in tuner_id:
            if t in self._tuner_ids:
                log.warning('not adding tuner_id %s for channel %s - it is '+\
                            'claimed by another channel', t, name)
                tuner_id.remove(t)
            else:
                self._tuner_ids.append(t)

        o = self._db.add_object("channel",
                                tuner_id = tuner_id,
                                name = name,
                                long_name = long_name)
        return o["id"]

    def _add_program_to_db(self, channel_db_id, start, stop, title, desc):
        #log.debug('channel_db_id: "%s" start: "%s" title: "%s"',
        #          channel_db_id, start, title)

        # TODO: check time range
        p2 = self._db.query(parent = ("channel", channel_db_id),
                            type = "program", start = start)

        if len(p2):
            # we have a program at this time
            p2 = p2[0]

            #log.debug('updating program: %s', p2["title"])
            # TODO: if everything is the same do not update
            self._db.update_object(("program", p2["id"]),
                                   start = start,
                                   stop = stop,
                                   title = title,
                                   desc = desc)
            return p2["id"]

        # TODO: check title, see if it is a different program.  Also check
        #       if the program is the same but shifted times a bit
        else:
            #log.debug('adding program: %s', title)
            o = self._db.add_object("program",
                                    parent = ("channel", channel_db_id),
                                    start = start,
                                    stop = stop,
                                    title = title,
                                    desc = desc, ratings = 42)

            if stop - start > self._max_program_length:
                self._max_program_length = stop = start
            return o["id"]

    def query(self, **kwargs):
        if "channel" in kwargs:
            if type(kwargs["channel"]) in (list, tuple):
                kwargs["parent"] = [("channel", x) for x in kwargs["channel"]]
            else:
                kwargs["parent"] = "channel", kwargs["channel"]
            del kwargs["channel"]

        for key in kwargs.copy():
            if key.startswith("__ipc_"):
                del kwargs[key]

        res = self._db.query_raw(**kwargs)
        return res

    def get_db(self):
        return self._db

    def get_max_program_length(self):
        return self._max_program_length

    def get_num_programs(self):
        return self._num_programs

if __name__ == "__main__":
    # ARGS: log file, log level, db file

    # python imports
    import gc
    import sys

    # kaa imports
    from kaa.notifier import Timer, execute_in_timer, loop

    @execute_in_timer(Timer, 1)
    def garbage_collect():
        g = gc.collect()
        if g:
            log.debug('gc: deleted %s objects' % g)
        if gc.garbage:
            log.warning('gc: found %s garbage objects' % len(gc.garbage))
            for g in gc.garbage:
                log.warning(g)
        return True

    shutdown_timer = 5

    @execute_in_timer(Timer, 1)
    def autoshutdown():
        global shutdown_timer
        global _server

        #log.debug("clients: %s", len(_server._clients))
        if _server and len(_server._clients) > 0:
            shutdown_timer = 5
            return True
        shutdown_timer -= 1
        if shutdown_timer == 0:
            log.info('shutdown EPG server')
            sys.exit(0)
        return True

    try:
        # detach for parent using a new sesion
        os.setsid()
    except OSError:
        # looks like we are started from the shell
        # TODO: start some extra debug here and disable autoshutdown
        pass

    if len(sys.argv) < 5:
        address = None
    else:
        address=sys.argv[4]

    _server = GuideServer("epg", log_file=str(sys.argv[1]),
                          log_level=int(sys.argv[2]), dbfile=sys.argv[3],
                          address=sys.argv[4])

    garbage_collect()
    autoshutdown()
    loop()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
 * Copyright © 2015 Uncharted Software Inc.
 *
 * Property of Uncharted™, formerly Oculus Info Inc.
 * http://uncharted.software/
 *
 * Released under the MIT License.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy of
 * this software and associated documentation files (the "Software"), to deal in
 * the Software without restriction, including without limitation the rights to
 * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
 * of the Software, and to permit persons to whom the Software is furnished to do
 * so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
"""

import sys, os, json, web

parent = os.path.dirname(os.path.realpath(__file__))
sys.path.append(parent + '/../MITIE/mitielib')
from mitie import *

urls = { '/ner', 'NerHandler'}

print "\nloading NER model..."
ner = named_entity_extractor('../MITIE/MITIE-models/english/ner_model.dat')
print "\nTags output by this NER model:", ner.get_possible_ner_tags()

class MyApplication(web.application):
    def run(self, argv):
        func = self.wsgifunc()
        return web.httpserver.runsimple(func, (sys.argv[1], int(sys.argv[2])))

app = MyApplication(urls, globals())

def extractEntities(text):
    response = {}
    response['text'] = text
    response['tokens'] = tokenize(text)
    print "\nTokenized input:", response['tokens']

    global ner
    entities = ner.extract_entities(response['tokens'])
    print "\nEntities found:", entities
    print "\nNumber of entities detected:", len(entities)

    response['entities'] = []
    for e in entities:
        range = e[0]
        tag = e[1]
        score = e[2]
        score_text = "{:0.3f}".format(score)
        entity_text = " ".join(response['tokens'][i] for i in range)
        print "   Score: " + score_text + ": " + tag + ": " + entity_text
        result = {"score":score, "tag": tag, "label": entity_text}
        response['entities'].append(result)
    return response

class NerHandler:
    def POST(self):
        data = web.data()
        print "\nData:", data

        text = json.loads(data)['text']
        print "\nText:", text

        result = []
        if isinstance(text, list):
            for item in text:
                result.append(extractEntities(item))
        else:
            result = extractEntities(text)

        return json.dumps(result)

if __name__ == "__main__":
    app.run(argv=sys.argv)
<commit_msg>remove debug info, add comments<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
 * Copyright © 2015 Uncharted Software Inc.
 *
 * Property of Uncharted™, formerly Oculus Info Inc.
 * http://uncharted.software/
 *
 * Released under the MIT License.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy of
 * this software and associated documentation files (the "Software"), to deal in
 * the Software without restriction, including without limitation the rights to
 * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
 * of the Software, and to permit persons to whom the Software is furnished to do
 * so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
"""

import sys, os, json, web

# specify path to MITIE and import
parent = os.path.dirname(os.path.realpath(__file__))
sys.path.append(parent + '/../MITIE/mitielib')
from mitie import *

# load the MITIE models
print "\nloading NER model..."
ner = named_entity_extractor('../MITIE/MITIE-models/english/ner_model.dat')
print "\nTags output by this NER model:", ner.get_possible_ner_tags()

# define the web application
class MyApplication(web.application):
    def run(self, argv):
        func = self.wsgifunc()
        return web.httpserver.runsimple(func, (sys.argv[1], int(sys.argv[2])))

# define the REST endpoints
urls = { '/ner', 'NerHandler'}
app = MyApplication(urls, globals())

# NER
def extractEntities(text):
    response = {}
    response['text'] = text
    response['tokens'] = tokenize(text)

    global ner
    entities = ner.extract_entities(response['tokens'])

    response['entities'] = []
    for e in entities:
        range = e[0]
        tag = e[1]
        score = e[2]
        entity_text = " ".join(response['tokens'][i] for i in range)
        result = {"score":score, "tag": tag, "label": entity_text}
        response['entities'].append(result)
    return response

# REST handler
class NerHandler:
    def POST(self):
        data = web.data()
        text = json.loads(data)['text']
        result = []
        if isinstance(text, list):
            for item in text:
                result.append(extractEntities(item))
        else:
            result = extractEntities(text)

        return json.dumps(result)

# main() entry point
if __name__ == "__main__":
    app.run(argv=sys.argv)
<|endoftext|>