<commit_before>from __future__ import absolute_import, unicode_literals, print_function, division

from collections import OrderedDict

from django.contrib.auth import get_user_model
from django.shortcuts import get_object_or_404
from dry_rest_permissions.generics import DRYPermissions
from rest_framework import viewsets, filters, generics, status
from rest_framework.parsers import MultiPartParser, FormParser
from rest_framework.permissions import IsAuthenticated, BasePermission, SAFE_METHODS
from rest_framework.views import APIView, Response

from main import models
from main.api import serializers
from main.api.uploaders import SiteUploader
from main.models import Project, Site, Dataset, GenericRecord, Observation, SpeciesObservation
from main.utils_auth import is_admin
from main.utils_species import HerbieFacade

class UserPermission(BasePermission):
    """
    Rules:
    Get: authenticated
    Update: admin or user itself
    Create: admin
    Delete: forbidden through API
    """

    def has_permission(self, request, view):
        """
        Global level.
        Reject Delete and Create for non admin.
        The rest will be checked at object level (below)
        """
        method = request.method
        if method == 'DELETE':
            return False
        if method == 'POST':
            return is_admin(request.user)
        return True

    def has_object_permission(self, request, view, obj):
        """
        Object level. Will be called only if the global level passed (see above).
        Note: it won't be called for a Create (POST) method
        """
        is_owner = (request.user == obj)
        return request.method in SAFE_METHODS or is_admin(request.user) or is_owner

class UserViewSet(viewsets.ModelViewSet):
    permission_classes = (IsAuthenticated, UserPermission,)
    queryset = get_user_model().objects.all()
    serializer_class = serializers.UserSerializer
    filter_backends = (filters.DjangoFilterBackend,)
    filter_fields = ('username', 'first_name', 'last_name', 'email')

class ProjectViewSet(viewsets.ModelViewSet):
    permission_classes = (IsAuthenticated, DRYPermissions)
    queryset = models.Project.objects.all()
    serializer_class = serializers.ProjectSerializer
    filter_backends = (filters.DjangoFilterBackend,)
    filter_fields = ('id', 'title',)

class ProjectPermission(BasePermission):
    def has_permission(self, request, view):
        user = request.user
        return \
            request.method in SAFE_METHODS \
            or is_admin(user) \
            or (hasattr(view, 'project') and view.project and view.project.is_custodian(user))

class ProjectSitesView(generics.ListCreateAPIView):
    permission_classes = (IsAuthenticated, ProjectPermission)
    serializer_class = serializers.SiteSerializer

    def __init__(self, **kwargs):
        super(ProjectSitesView, self).__init__(**kwargs)
        self.project = None

    def dispatch(self, request, *args, **kwargs):
        """
        Intercept any request to set the project from the pk
        :param request:
        """
        self.project = get_object_or_404(Project, pk=self.kwargs.get('pk'))
        return super(ProjectSitesView, self).dispatch(request, *args, **kwargs)

    def get_queryset(self):
        project = self.project if hasattr(self, 'project') else get_object_or_404(Project, pk=self.kwargs.get('pk'))
        return Site.objects.filter(project=project)

    def get_serializer(self, *args, **kwargs):
        kwargs["many"] = True
        ser = super(ProjectSitesView, self).get_serializer(*args, **kwargs)
        if hasattr(ser, 'initial_data') and self.project:
            for r in ser.initial_data:
                r['project'] = self.project.pk
        return ser

class ProjectSitesUploadView(APIView):
    # TODO: unit test for this view + Implement xlsx version.
    permission_classes = (IsAuthenticated, ProjectPermission)
    parser_classes = (FormParser, MultiPartParser)

    def post(self, request, *args, **kwargs):
        project = get_object_or_404(Project, pk=self.kwargs.get('pk'))
        file_obj = request.data['file']
        if file_obj.content_type not in SiteUploader.SUPPORTED_TYPES:
            msg = "Wrong file type {}. Should be one of: {}".format(file_obj.content_type, SiteUploader.SUPPORTED_TYPES)
            return Response(msg, status=status.HTTP_501_NOT_IMPLEMENTED)

        uploader = SiteUploader(file_obj, project)
        data = {}
        # return an items by row
        # {1: { site: pk|None, error: msg|None}, 2:...., 3:... }

        has_error = False
        row = 0
        for site, error in uploader:
            row += 1
            result = {
                'site': None,
                'error': None
            }
            if site:
                result['site'] = site.pk
            if error:
                print('error at row #{}: {}'.format(row, error))
                has_error = True
                result['error'] = str(error)
            data[row] = result
        uploader.close()
        status_code = status.HTTP_200_OK if not has_error else status.HTTP_400_BAD_REQUEST
        return Response(data, status=status_code)

class SiteViewSet(viewsets.ModelViewSet):
    permission_classes = (IsAuthenticated, DRYPermissions)
    queryset = models.Site.objects.all()
    serializer_class = serializers.SiteSerializer
    filter_backends = (filters.DjangoFilterBackend,)
    filter_fields = ('id', 'name', 'code')

class DatasetViewSet(viewsets.ModelViewSet):
    permission_classes = (IsAuthenticated, DRYPermissions)
    queryset = models.Dataset.objects.all()
    serializer_class = serializers.DatasetSerializer
    filter_backends = (filters.DjangoFilterBackend,)
    filter_fields = ('name', 'project', 'type')

class DatasetDataPermission(BasePermission):
    def has_permission(self, request, view):
        user = request.user
        return \
            request.method in SAFE_METHODS \
            or is_admin(user) \
            or (hasattr(view, 'dataset') and view.dataset and view.dataset.is_custodian(user))

class SpeciesMixin:
    species_facade_class = HerbieFacade

class DatasetDataView(generics.ListCreateAPIView, SpeciesMixin):
    permission_classes = (IsAuthenticated, DatasetDataPermission)

    def __init__(self, **kwargs):
        super(DatasetDataView, self).__init__(**kwargs)
        self.dataset = None

    def dispatch(self, request, *args, **kwargs):
        """
        Intercept any request to set the dataset
        :param request:
        """
        self.dataset = get_object_or_404(models.Dataset, pk=kwargs.get('pk'))
        return super(DatasetDataView, self).dispatch(request, *args, **kwargs)

    def get_serializer_class(self):
        if self.dataset:  # test needed for the swagger
            if self.dataset.type == models.Dataset.TYPE_SPECIES_OBSERVATION:
                return serializers.SpeciesObservationSerializer
            elif self.dataset.type == models.Dataset.TYPE_OBSERVATION:
                return serializers.ObservationSerializer
            else:
                return serializers.GenericRecordSerializer
        else:
            # for the swagger
            return serializers.GenericRecordSerializer

    def get_serializer(self, *args, **kwargs):
        kwargs["many"] = True
        ser = super(DatasetDataView, self).get_serializer(*args, **kwargs)
        # TODO: find a better way to initialize the dataset field of the serializer
        if hasattr(ser, 'initial_data'):
            for r in ser.initial_data:
                r['dataset'] = self.dataset.pk
        return ser

    def get_serializer_context(self):
        ctx = super(DatasetDataView, self).get_serializer_context()
        if self.dataset:
            ctx['dataset'] = self.dataset
            if self.dataset.type == Dataset.TYPE_SPECIES_OBSERVATION and 'species_mapping' not in ctx:
                ctx['species_mapping'] = self.species_facade_class().name_id_by_species_name()
        return ctx

    def get_queryset(self):
        return self.dataset.record_queryset if self.dataset else Dataset.objects.none()

    def get(self, request, *args, **kwargs):
        """
        Forward the get to the list of ListModelMixin
        :param request:
        :param args:
        :param kwargs:
        :return:
        """
        if not self.dataset:
            self.dataset = get_object_or_404(models.Dataset, pk=kwargs.get('pk'))
        return super(DatasetDataView, self).list(request, *args, **kwargs)

    def create(self, request, *args, **kwargs):
        result = super(DatasetDataView, self).create(request, *args, **kwargs)
        return result

    def post(self, request, *args, **kwargs):
        if not self.dataset:
            self.dataset = get_object_or_404(models.Dataset, pk=kwargs.get('pk'))
        return super(DatasetDataView, self).post(request, *args, **kwargs)

class GenericRecordViewSet(viewsets.ModelViewSet):
    permission_classes = (IsAuthenticated, DRYPermissions)
    queryset = models.GenericRecord.objects.all()
    serializer_class = serializers.GenericRecordSerializer
    filter_backends = (filters.DjangoFilterBackend,)
    filter_fields = ('id', 'site', 'dataset__id', 'dataset__name')

class ObservationViewSet(GenericRecordViewSet):
    queryset = models.Observation.objects.all()
    serializer_class = serializers.ObservationSerializer
    filter_fields = GenericRecordViewSet.filter_fields + ('datetime',)

class SpeciesObservationViewSet(ObservationViewSet, SpeciesMixin):
    queryset = models.SpeciesObservation.objects.all()
    serializer_class = serializers.SpeciesObservationSerializer
    filter_fields = ObservationViewSet.filter_fields + ('species_name', 'name_id',)

    def get_serializer_context(self):
        ctx = super(SpeciesObservationViewSet, self).get_serializer_context()
        if 'species_mapping' not in ctx:
            ctx['species_mapping'] = self.species_facade_class().name_id_by_species_name()
        return ctx

class StatisticsView(APIView):
    permission_classes = (IsAuthenticated,)

    def get(self, request, **kwargs):
        data = OrderedDict()
        qs = Project.objects.all()
        data['projects'] = {
            'total': qs.count()
        }
        qs = Dataset.objects.all()
        data['datasets'] = OrderedDict([
            ('total', qs.count()),
            ('generic', {
                'total': qs.filter(type=Dataset.TYPE_GENERIC).count()
            }),
            ('observation', {
                'total': qs.filter(type=Dataset.TYPE_OBSERVATION).count()
            }),
            ('speciesObservation', {
                'total': qs.filter(type=Dataset.TYPE_SPECIES_OBSERVATION).count()
            }),
        ])
        # records
        generic_records_count = GenericRecord.objects.count()
        observation_record_count = Observation.objects.count()
        species_observation_count = SpeciesObservation.objects.count()
        data['records'] = OrderedDict([
            ('total', generic_records_count + observation_record_count + species_observation_count),
            ('generic', {
                'total': generic_records_count
            }),
            ('observation', {
                'total': observation_record_count
            }),
            ('speciesObservation', {
                'total': species_observation_count
            }),
        ])
        qs = Site.objects.all()
        data['sites'] = {
            'total': qs.count()
        }
        return Response(data)

class WhoamiView(APIView):
    serializers = serializers.SimpleUserSerializer

    def get(self, request, **kwargs):
        data = {}
        if request.user.is_authenticated():
            data = self.serializers(request.user).data
        return Response(data)
<commit_msg>Removed a done TODO<commit_after>from __future__ import absolute_import, unicode_literals, print_function, division

from collections import OrderedDict

from django.contrib.auth import get_user_model
from django.shortcuts import get_object_or_404
from dry_rest_permissions.generics import DRYPermissions
from rest_framework import viewsets, filters, generics, status
from rest_framework.parsers import MultiPartParser, FormParser
from rest_framework.permissions import IsAuthenticated, BasePermission, SAFE_METHODS
from rest_framework.views import APIView, Response

from main import models
from main.api import serializers
from main.api.uploaders import SiteUploader
from main.models import Project, Site, Dataset, GenericRecord, Observation, SpeciesObservation
from main.utils_auth import is_admin
from main.utils_species import HerbieFacade

class UserPermission(BasePermission):
    """
    Rules:
    Get: authenticated
    Update: admin or user itself
    Create: admin
    Delete: forbidden through API
    """

    def has_permission(self, request, view):
        """
        Global level.
        Reject Delete and Create for non admin.
        The rest will be checked at object level (below)
        """
        method = request.method
        if method == 'DELETE':
            return False
        if method == 'POST':
            return is_admin(request.user)
        return True

    def has_object_permission(self, request, view, obj):
        """
        Object level. Will be called only if the global level passed (see above).
        Note: it won't be called for a Create (POST) method
        """
        is_owner = (request.user == obj)
        return request.method in SAFE_METHODS or is_admin(request.user) or is_owner

class UserViewSet(viewsets.ModelViewSet):
    permission_classes = (IsAuthenticated, UserPermission,)
    queryset = get_user_model().objects.all()
    serializer_class = serializers.UserSerializer
    filter_backends = (filters.DjangoFilterBackend,)
    filter_fields = ('username', 'first_name', 'last_name', 'email')

class ProjectViewSet(viewsets.ModelViewSet):
    permission_classes = (IsAuthenticated, DRYPermissions)
    queryset = models.Project.objects.all()
    serializer_class = serializers.ProjectSerializer
    filter_backends = (filters.DjangoFilterBackend,)
    filter_fields = ('id', 'title',)

class ProjectPermission(BasePermission):
    def has_permission(self, request, view):
        user = request.user
        return \
            request.method in SAFE_METHODS \
            or is_admin(user) \
            or (hasattr(view, 'project') and view.project and view.project.is_custodian(user))

class ProjectSitesView(generics.ListCreateAPIView):
    permission_classes = (IsAuthenticated, ProjectPermission)
    serializer_class = serializers.SiteSerializer

    def __init__(self, **kwargs):
        super(ProjectSitesView, self).__init__(**kwargs)
        self.project = None

    def dispatch(self, request, *args, **kwargs):
        """
        Intercept any request to set the project from the pk
        :param request:
        """
        self.project = get_object_or_404(Project, pk=self.kwargs.get('pk'))
        return super(ProjectSitesView, self).dispatch(request, *args, **kwargs)

    def get_queryset(self):
        project = self.project if hasattr(self, 'project') else get_object_or_404(Project, pk=self.kwargs.get('pk'))
        return Site.objects.filter(project=project)

    def get_serializer(self, *args, **kwargs):
        kwargs["many"] = True
        ser = super(ProjectSitesView, self).get_serializer(*args, **kwargs)
        if hasattr(ser, 'initial_data') and self.project:
            for r in ser.initial_data:
                r['project'] = self.project.pk
        return ser

class ProjectSitesUploadView(APIView):
    # TODO: unit test for this view
    permission_classes = (IsAuthenticated, ProjectPermission)
    parser_classes = (FormParser, MultiPartParser)

    def post(self, request, *args, **kwargs):
        project = get_object_or_404(Project, pk=self.kwargs.get('pk'))
        file_obj = request.data['file']
        if file_obj.content_type not in SiteUploader.SUPPORTED_TYPES:
            msg = "Wrong file type {}. Should be one of: {}".format(file_obj.content_type, SiteUploader.SUPPORTED_TYPES)
            return Response(msg, status=status.HTTP_501_NOT_IMPLEMENTED)

        uploader = SiteUploader(file_obj, project)
        data = {}
        # return an items by row
        # {1: { site: pk|None, error: msg|None}, 2:...., 3:... }

        has_error = False
        row = 0
        for site, error in uploader:
            row += 1
            result = {
                'site': None,
                'error': None
            }
            if site:
                result['site'] = site.pk
            if error:
                print('error at row #{}: {}'.format(row, error))
                has_error = True
                result['error'] = str(error)
            data[row] = result
        uploader.close()
        status_code = status.HTTP_200_OK if not has_error else status.HTTP_400_BAD_REQUEST
        return Response(data, status=status_code)

class SiteViewSet(viewsets.ModelViewSet):
    permission_classes = (IsAuthenticated, DRYPermissions)
    queryset = models.Site.objects.all()
    serializer_class = serializers.SiteSerializer
    filter_backends = (filters.DjangoFilterBackend,)
    filter_fields = ('id', 'name', 'code')

class DatasetViewSet(viewsets.ModelViewSet):
    permission_classes = (IsAuthenticated, DRYPermissions)
    queryset = models.Dataset.objects.all()
    serializer_class = serializers.DatasetSerializer
    filter_backends = (filters.DjangoFilterBackend,)
    filter_fields = ('name', 'project', 'type')

class DatasetDataPermission(BasePermission):
    def has_permission(self, request, view):
        user = request.user
        return \
            request.method in SAFE_METHODS \
            or is_admin(user) \
            or (hasattr(view, 'dataset') and view.dataset and view.dataset.is_custodian(user))

class SpeciesMixin:
    species_facade_class = HerbieFacade

class DatasetDataView(generics.ListCreateAPIView, SpeciesMixin):
    permission_classes = (IsAuthenticated, DatasetDataPermission)

    def __init__(self, **kwargs):
        super(DatasetDataView, self).__init__(**kwargs)
        self.dataset = None

    def dispatch(self, request, *args, **kwargs):
        """
        Intercept any request to set the dataset
        :param request:
        """
        self.dataset = get_object_or_404(models.Dataset, pk=kwargs.get('pk'))
        return super(DatasetDataView, self).dispatch(request, *args, **kwargs)

    def get_serializer_class(self):
        if self.dataset:  # test needed for the swagger
            if self.dataset.type == models.Dataset.TYPE_SPECIES_OBSERVATION:
                return serializers.SpeciesObservationSerializer
            elif self.dataset.type == models.Dataset.TYPE_OBSERVATION:
                return serializers.ObservationSerializer
            else:
                return serializers.GenericRecordSerializer
        else:
            # for the swagger
            return serializers.GenericRecordSerializer

    def get_serializer(self, *args, **kwargs):
        kwargs["many"] = True
        ser = super(DatasetDataView, self).get_serializer(*args, **kwargs)
        # TODO: find a better way to initialize the dataset field of the serializer
        if hasattr(ser, 'initial_data'):
            for r in ser.initial_data:
                r['dataset'] = self.dataset.pk
        return ser

    def get_serializer_context(self):
        ctx = super(DatasetDataView, self).get_serializer_context()
        if self.dataset:
            ctx['dataset'] = self.dataset
            if self.dataset.type == Dataset.TYPE_SPECIES_OBSERVATION and 'species_mapping' not in ctx:
                ctx['species_mapping'] = self.species_facade_class().name_id_by_species_name()
        return ctx

    def get_queryset(self):
        return self.dataset.record_queryset if self.dataset else Dataset.objects.none()

    def get(self, request, *args, **kwargs):
        """
        Forward the get to the list of ListModelMixin
        :param request:
        :param args:
        :param kwargs:
        :return:
        """
        if not self.dataset:
            self.dataset = get_object_or_404(models.Dataset, pk=kwargs.get('pk'))
        return super(DatasetDataView, self).list(request, *args, **kwargs)

    def create(self, request, *args, **kwargs):
        result = super(DatasetDataView, self).create(request, *args, **kwargs)
        return result

    def post(self, request, *args, **kwargs):
        if not self.dataset:
            self.dataset = get_object_or_404(models.Dataset, pk=kwargs.get('pk'))
        return super(DatasetDataView, self).post(request, *args, **kwargs)

class GenericRecordViewSet(viewsets.ModelViewSet):
    permission_classes = (IsAuthenticated, DRYPermissions)
    queryset = models.GenericRecord.objects.all()
    serializer_class = serializers.GenericRecordSerializer
    filter_backends = (filters.DjangoFilterBackend,)
    filter_fields = ('id', 'site', 'dataset__id', 'dataset__name')

class ObservationViewSet(GenericRecordViewSet):
    queryset = models.Observation.objects.all()
    serializer_class = serializers.ObservationSerializer
    filter_fields = GenericRecordViewSet.filter_fields + ('datetime',)

class SpeciesObservationViewSet(ObservationViewSet, SpeciesMixin):
    queryset = models.SpeciesObservation.objects.all()
    serializer_class = serializers.SpeciesObservationSerializer
    filter_fields = ObservationViewSet.filter_fields + ('species_name', 'name_id',)

    def get_serializer_context(self):
        ctx = super(SpeciesObservationViewSet, self).get_serializer_context()
        if 'species_mapping' not in ctx:
            ctx['species_mapping'] = self.species_facade_class().name_id_by_species_name()
        return ctx

class StatisticsView(APIView):
    permission_classes = (IsAuthenticated,)

    def get(self, request, **kwargs):
        data = OrderedDict()
        qs = Project.objects.all()
        data['projects'] = {
            'total': qs.count()
        }
        qs = Dataset.objects.all()
        data['datasets'] = OrderedDict([
            ('total', qs.count()),
            ('generic', {
                'total': qs.filter(type=Dataset.TYPE_GENERIC).count()
            }),
            ('observation', {
                'total': qs.filter(type=Dataset.TYPE_OBSERVATION).count()
            }),
            ('speciesObservation', {
                'total': qs.filter(type=Dataset.TYPE_SPECIES_OBSERVATION).count()
            }),
        ])
        # records
        generic_records_count = GenericRecord.objects.count()
        observation_record_count = Observation.objects.count()
        species_observation_count = SpeciesObservation.objects.count()
        data['records'] = OrderedDict([
            ('total', generic_records_count + observation_record_count + species_observation_count),
            ('generic', {
                'total': generic_records_count
            }),
            ('observation', {
                'total': observation_record_count
            }),
            ('speciesObservation', {
                'total': species_observation_count
            }),
        ])
        qs = Site.objects.all()
        data['sites'] = {
            'total': qs.count()
        }
        return Response(data)

class WhoamiView(APIView):
    serializers = serializers.SimpleUserSerializer

    def get(self, request, **kwargs):
        data = {}
        if request.user.is_authenticated():
            data = self.serializers(request.user).data
        return Response(data)
<|endoftext|>


<commit_before>import re
import json
import logging
from tornado.web import HTTPError
from biothings.www.helper import BaseHandler
from biothings.utils.common import split_ids
from biothings.settings import BiothingSettings

biothing_settings = BiothingSettings()

class BiothingHandler(BaseHandler):

    def _ga_event_object(self, action, data={}):
        ''' Returns the google analytics object for requests on this endpoint (annotation handler).'''
        return biothing_settings.ga_event_object(endpoint=biothing_settings._annotation_endpoint, action=action, data=data)

    def _examine_kwargs(self, action, kwargs):
        ''' A function for sub-classing.  This will be run after the get_query_params but before the actual
            elasticsearch querying. '''
        if action == 'GET':
            pass
        elif action == 'POST':
            pass
        pass

    def get(self, bid=None):
        '''
        '''
        if bid:
            kwargs = self.get_query_params()
            self._examine_kwargs('GET', kwargs)
            biothing_object = self.esq.get_biothing(bid, **kwargs)
            if biothing_object:
                self.return_json(biothing_object)
                self.ga_track(event=self._ga_event_object('GET'))
            else:
                raise HTTPError(404)
        else:
            raise HTTPError(404)

    def post(self, ids=None):
        '''
           This is essentially the same as post request in QueryHandler, with different defaults.

           parameters:
            ids
            fields
            email
        '''
        kwargs = self.get_query_params()
        self._examine_kwargs('POST', kwargs)
        ids = kwargs.pop('ids', None)
        if ids:
            ids = re.split('[\s\r\n+|,]+', ids)
            res = self.esq.mget_biothings(ids, **kwargs)
        else:
            res = {'success': False, 'error': "Missing required parameters."}
        encode = not isinstance(res, str)    # when res is a string, e.g. when rawquery is true, do not encode it as json
        self.return_json(res, encode=encode)
        self.ga_track(event=self._ga_event_object('POST', {'qsize': len(ids) if ids else 0}))

class QueryHandler(BaseHandler):

    def _ga_event_object(self, action, data={}):
        ''' Returns the google analytics object for requests on this endpoint (query handler).'''
        return biothing_settings.ga_event_object(endpoint=biothing_settings._query_endpoint, action=action, data=data)

    def _examine_kwargs(self, action, kwargs):
        ''' A function for sub-classing.  This will be run after the get_query_params but before the actual
            elasticsearch querying. '''
        if action == 'GET':
            pass
        elif action == 'POST':
            pass
        pass

    def get(self):
        '''
        parameters:
            q
            fields
            from
            size
            sort
            facets
            callback
            email
            fetch_all

            explain
            raw
        '''
        kwargs = self.get_query_params()
        self._examine_kwargs('GET', kwargs)
        q = kwargs.pop('q', None)
        scroll_id = kwargs.pop('scroll_id', None)
        _has_error = False
        if scroll_id:
            res = self.esq.scroll(scroll_id, **kwargs)
        elif q:
            for arg in ['from', 'size']:
                value = kwargs.get(arg, None)
                if value:
                    try:
                        kwargs[arg] = int(value)
                    except ValueError:
                        res = {'success': False, 'error': 'Parameter "{}" must be an integer.'.format(arg)}
                        _has_error = True
            if not _has_error:
                res = self.esq.query(q, **kwargs)
                if kwargs.get('fetch_all', False):
                    self.ga_track(event=self._ga_event_object('fetch_all', {'total': res.get('total', None)}))
        else:
            res = {'success': False, 'error': "Missing required parameters."}

        self.return_json(res)
        self.ga_track(event=self._ga_event_object('GET', {'qsize': len(q) if q else 0}))

    def post(self):
        '''
        parameters:
            q
            scopes
            fields
            email

            jsoninput   if true, input "q" is a json string, must be decoded as a list.
        '''
        kwargs = self.get_query_params()
        self._examine_kwargs('POST', kwargs)
        q = kwargs.pop('q', None)
        jsoninput = kwargs.pop('jsoninput', None) in ('1', 'true')
        if q:
            # ids = re.split('[\s\r\n+|,]+', q)
            try:
                ids = json.loads(q) if jsoninput else split_ids(q)
                if not isinstance(ids, list):
                    raise ValueError
            except ValueError:
                ids = None
                res = {'success': False, 'error': 'Invalid input for "q" parameter.'}
            if ids:
                scopes = kwargs.pop('scopes', None)
                fields = kwargs.pop('fields', None)
                res = self.esq.mget_biothings(ids, fields=fields, scopes=scopes, **kwargs)
        else:
            res = {'success': False, 'error': "Missing required parameters."}

        encode = not isinstance(res, str)    # when res is a string, e.g. when rawquery is true, do not encode it as json
        self.return_json(res, encode=encode)
        self.ga_track(event=self._ga_event_object('POST', {'qsize': len(q) if q else 0}))

class MetaDataHandler(BaseHandler):

    def get(self):
        self.return_json({})

class FieldsHandler(BaseHandler):

    def get(self):
        if biothing_settings.field_notes_path:
            notes = json.load(open(biothing_settings.field_notes_path, 'r'))
        else:
            notes = {}
        es_mapping = self.esq.query_fields()
        kwargs = self.get_query_params()

        def get_indexed_properties_in_dict(d, prefix):
            r = {}
            for (k, v) in d.items():
                r[prefix + '.' + k] = {}
                r[prefix + '.' + k]['indexed'] = False
                if 'properties' not in v:
                    r[prefix + '.' + k]['type'] = v['type']
                    if ('index' not in v) or ('index' in v and v['index'] != 'no'):
                        # indexed field
                        r[prefix + '.' + k]['indexed'] = True
                else:
                    r[prefix + '.' + k]['type'] = 'object'
                    r.update(get_indexed_properties_in_dict(v['properties'], prefix + '.' + k))
                if ('include_in_all' in v) and v['include_in_all']:
                    r[prefix + '.' + k]['include_in_all'] = True
                else:
                    r[prefix + '.' + k]['include_in_all'] = False
            return r

        r = {}
        search = kwargs.pop('search', None)
        prefix = kwargs.pop('prefix', None)
        for (k, v) in get_indexed_properties_in_dict(es_mapping, '').items():
            k1 = k.lstrip('.')
            if (search and search in k1) or (prefix and k1.startswith(prefix)) or (not search and not prefix):
                r[k1] = v
                if k1 in notes:
                    r[k1]['notes'] = notes[k1]
        self.return_json(r)

class StatusHandler(BaseHandler):
    ''' Handles requests to check the status of the server. '''

    def head(self):
        logging.error("biothing_settings.status_check_id: %s" % biothing_settings.status_check_id)
        r = self.esq.status_check(biothing_settings.status_check_id)
        if r is None:
            # we failed to retrieve ref/test doc, something is wrong -> service unavailable
            raise HTTPError(503)

    def get(self):
        self.head()
        self.write('OK')
<commit_msg>remove debug msg...<commit_after>import re
import json
from tornado.web import HTTPError
from biothings.www.helper import BaseHandler
from biothings.utils.common import split_ids
from biothings.settings import BiothingSettings

biothing_settings = BiothingSettings()

class BiothingHandler(BaseHandler):

    def _ga_event_object(self, action, data={}):
        ''' Returns the google analytics object for requests on this endpoint (annotation handler).'''
        return biothing_settings.ga_event_object(endpoint=biothing_settings._annotation_endpoint, action=action, data=data)

    def _examine_kwargs(self, action, kwargs):
        ''' A function for sub-classing.  This will be run after the get_query_params but before the actual
            elasticsearch querying. '''
        if action == 'GET':
            pass
        elif action == 'POST':
            pass
        pass

    def get(self, bid=None):
        '''
        '''
        if bid:
            kwargs = self.get_query_params()
            self._examine_kwargs('GET', kwargs)
            biothing_object = self.esq.get_biothing(bid, **kwargs)
            if biothing_object:
                self.return_json(biothing_object)
                self.ga_track(event=self._ga_event_object('GET'))
            else:
                raise HTTPError(404)
        else:
            raise HTTPError(404)

    def post(self, ids=None):
        '''
           This is essentially the same as post request in QueryHandler, with different defaults.

           parameters:
            ids
            fields
            email
        '''
        kwargs = self.get_query_params()
        self._examine_kwargs('POST', kwargs)
        ids = kwargs.pop('ids', None)
        if ids:
            ids = re.split('[\s\r\n+|,]+', ids)
            res = self.esq.mget_biothings(ids, **kwargs)
        else:
            res = {'success': False, 'error': "Missing required parameters."}
        encode = not isinstance(res, str)    # when res is a string, e.g. when rawquery is true, do not encode it as json
        self.return_json(res, encode=encode)
        self.ga_track(event=self._ga_event_object('POST', {'qsize': len(ids) if ids else 0}))

class QueryHandler(BaseHandler):

    def _ga_event_object(self, action, data={}):
        ''' Returns the google analytics object for requests on this endpoint (query handler).'''
        return biothing_settings.ga_event_object(endpoint=biothing_settings._query_endpoint, action=action, data=data)

    def _examine_kwargs(self, action, kwargs):
        ''' A function for sub-classing.  This will be run after the get_query_params but before the actual
            elasticsearch querying. '''
        if action == 'GET':
            pass
        elif action == 'POST':
            pass
        pass

    def get(self):
        '''
        parameters:
            q
            fields
            from
            size
            sort
            facets
            callback
            email
            fetch_all

            explain
            raw
        '''
        kwargs = self.get_query_params()
        self._examine_kwargs('GET', kwargs)
        q = kwargs.pop('q', None)
        scroll_id = kwargs.pop('scroll_id', None)
        _has_error = False
        if scroll_id:
            res = self.esq.scroll(scroll_id, **kwargs)
        elif q:
            for arg in ['from', 'size']:
                value = kwargs.get(arg, None)
                if value:
                    try:
                        kwargs[arg] = int(value)
                    except ValueError:
                        res = {'success': False, 'error': 'Parameter "{}" must be an integer.'.format(arg)}
                        _has_error = True
            if not _has_error:
                res = self.esq.query(q, **kwargs)
                if kwargs.get('fetch_all', False):
                    self.ga_track(event=self._ga_event_object('fetch_all', {'total': res.get('total', None)}))
        else:
            res = {'success': False, 'error': "Missing required parameters."}

        self.return_json(res)
        self.ga_track(event=self._ga_event_object('GET', {'qsize': len(q) if q else 0}))

    def post(self):
        '''
        parameters:
            q
            scopes
            fields
            email

            jsoninput   if true, input "q" is a json string, must be decoded as a list.
        '''
        kwargs = self.get_query_params()
        self._examine_kwargs('POST', kwargs)
        q = kwargs.pop('q', None)
        jsoninput = kwargs.pop('jsoninput', None) in ('1', 'true')
        if q:
            # ids = re.split('[\s\r\n+|,]+', q)
            try:
                ids = json.loads(q) if jsoninput else split_ids(q)
                if not isinstance(ids, list):
                    raise ValueError
            except ValueError:
                ids = None
                res = {'success': False, 'error': 'Invalid input for "q" parameter.'}
            if ids:
                scopes = kwargs.pop('scopes', None)
                fields = kwargs.pop('fields', None)
                res = self.esq.mget_biothings(ids, fields=fields, scopes=scopes, **kwargs)
        else:
            res = {'success': False, 'error': "Missing required parameters."}

        encode = not isinstance(res, str)    # when res is a string, e.g. when rawquery is true, do not encode it as json
        self.return_json(res, encode=encode)
        self.ga_track(event=self._ga_event_object('POST', {'qsize': len(q) if q else 0}))

class MetaDataHandler(BaseHandler):

    def get(self):
        self.return_json({})

class FieldsHandler(BaseHandler):

    def get(self):
        if biothing_settings.field_notes_path:
            notes = json.load(open(biothing_settings.field_notes_path, 'r'))
        else:
            notes = {}
        es_mapping = self.esq.query_fields()
        kwargs = self.get_query_params()

        def get_indexed_properties_in_dict(d, prefix):
            r = {}
            for (k, v) in d.items():
                r[prefix + '.' + k] = {}
                r[prefix + '.' + k]['indexed'] = False
                if 'properties' not in v:
                    r[prefix + '.' + k]['type'] = v['type']
                    if ('index' not in v) or ('index' in v and v['index'] != 'no'):
                        # indexed field
                        r[prefix + '.' + k]['indexed'] = True
                else:
                    r[prefix + '.' + k]['type'] = 'object'
                    r.update(get_indexed_properties_in_dict(v['properties'], prefix + '.' + k))
                if ('include_in_all' in v) and v['include_in_all']:
                    r[prefix + '.' + k]['include_in_all'] = True
                else:
                    r[prefix + '.' + k]['include_in_all'] = False
            return r

        r = {}
        search = kwargs.pop('search', None)
        prefix = kwargs.pop('prefix', None)
        for (k, v) in get_indexed_properties_in_dict(es_mapping, '').items():
            k1 = k.lstrip('.')
            if (search and search in k1) or (prefix and k1.startswith(prefix)) or (not search and not prefix):
                r[k1] = v
                if k1 in notes:
                    r[k1]['notes'] = notes[k1]
        self.return_json(r)

class StatusHandler(BaseHandler):
    ''' Handles requests to check the status of the server. '''

    def head(self):
        r = self.esq.status_check(biothing_settings.status_check_id)
        if r is None:
            # we failed to retrieve ref/test doc, something is wrong -> service unavailable
            raise HTTPError(503)

    def get(self):
        self.head()
        self.write('OK')
<|endoftext|>


<commit_before># coding=utf-8
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/.

"""Allows the funfuzz harness to run continuously.
"""

from __future__ import absolute_import, print_function, unicode_literals  # isort:skip

import io
import json
from optparse import OptionParser  # pylint: disable=deprecated-module
import os
import sys
import time

from . import compare_jit
from . import js_interesting
from . import link_fuzzer
from . import shell_flags
from ..util import create_collector
from ..util import file_manipulation
from ..util import lithium_helpers
from ..util import os_ops

if sys.version_info.major == 2:
    if os.name == "posix":
        import subprocess32 as subprocess  # pylint: disable=import-error
    from pathlib2 import Path  # pylint: disable=import-error
else:
    from pathlib import Path  # pylint: disable=import-error
    import subprocess

def parseOpts(args):  # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc
    parser = OptionParser()
    parser.disable_interspersed_args()
    parser.add_option("--compare-jit",
                      action="store_true", dest="use_compare_jit",
                      default=False,
                      help="After running the fuzzer, run the FCM lines against the engine "
                           "in two configurations and compare the output.")
    parser.add_option("--random-flags",
                      action="store_true", dest="randomFlags",
                      default=False,
                      help="Pass a random set of flags (e.g. --ion-eager) to the js engine")
    parser.add_option("--repo",
                      action="store",
                      dest="repo",
                      help="The hg repository (e.g. ~/trees/mozilla-central/), for bisection")
    parser.add_option("--build",
                      action="store", dest="build_options_str",
                      help="The build options, for bisection",
                      default=None)  # if you run loop directly w/o --build, lithium_helpers.pinpoint will try to guess
    parser.add_option("--valgrind",
                      action="store_true", dest="valgrind",
                      default=False,
                      help="use valgrind with a reasonable set of options")
    options, args = parser.parse_args(args)

    # optparse does not recognize pathlib - we will need to move to argparse
    if options.repo:
        options.repo = Path(options.repo)
    else:
        options.repo = Path.home() / "trees" / "mozilla-central"

    if options.valgrind and options.use_compare_jit:
        print("Note: When running compare_jit, the --valgrind option will be ignored")

    # kill js shell if it runs this long.
    # jsfunfuzz will quit after half this time if it's not ilooping.
    # higher = more complex mixing, especially with regression tests.
    # lower = less time wasted in timeouts and in compare_jit testcases that are thrown away due to OOMs.
    options.timeout = int(args[0])

    # FIXME: We can probably remove args[1]  # pylint: disable=fixme
    options.knownPath = "mozilla-central"
    options.jsEngine = Path(args[2])
    options.engineFlags = args[3:]

    return options

def showtail(filename):  # pylint: disable=missing-docstring
    # pylint: disable=fixme
    # FIXME: Get jsfunfuzz to output start & end of interesting result boundaries instead of this.
    cmd = []
    cmd.extend(["tail", "-n", "20"])
    cmd.append(str(filename))
    print(" ".join(cmd))
    print()
    subprocess.run(cmd, check=True)
    print()
    print()

def makeRegressionTestPrologue(repo):  # pylint: disable=invalid-name,missing-param-doc
    # pylint: disable=missing-return-doc,missing-return-type-doc,missing-type-doc
    """Generate a JS string to tell jsfunfuzz where to find SpiderMonkey's regression tests."""
    return """
const regressionTestsRoot = %s;
const libdir = regressionTestsRoot + %s; // needed by jit-tests
const regressionTestList = %s;
""" % (json.dumps(str(repo) + os.sep),
       json.dumps(str(Path("js") / "src" / "jit-test" / "lib") + os.sep),
       json.dumps(inTreeRegressionTests(repo)))

def inTreeRegressionTests(repo):  # pylint: disable=invalid-name,missing-docstring,missing-return-doc
    # pylint: disable=missing-return-type-doc
    jit_tests = jsFilesIn(len(str(repo)), repo / "js" / "src" / "jit-test" / "tests")
    js_tests = jsFilesIn(len(str(repo)), repo / "js" / "src" / "tests")
    return jit_tests + js_tests

def jsFilesIn(repoPathLength, root):  # pylint: disable=invalid-name,missing-docstring,missing-return-doc
    # pylint: disable=missing-return-type-doc
    return [os.path.join(path, filename)[repoPathLength:]
            for path, _dirs, files in os.walk(str(root))
            for filename in files
            if filename.endswith(".js")]

def many_timed_runs(targetTime, wtmpDir, args, collector, ccoverage):  # pylint: disable=invalid-name,missing-docstring
    # pylint: disable=too-many-branches,too-complex,too-many-locals,too-many-statements
    options = parseOpts(args)
    # engineFlags is overwritten later if --random-flags is set.
    engineFlags = options.engineFlags  # pylint: disable=invalid-name
    startTime = time.time()  # pylint: disable=invalid-name

    if options.repo.is_dir():
        regressionTestPrologue = makeRegressionTestPrologue(options.repo)  # pylint: disable=invalid-name
    else:
        regressionTestPrologue = ""  # pylint: disable=invalid-name

    fuzzjs = wtmpDir / "jsfunfuzz.js"

    link_fuzzer.link_fuzzer(fuzzjs, regressionTestPrologue)
    assert fuzzjs.is_file()

    iteration = 0
    while True:
        if targetTime and time.time() > startTime + targetTime:
            print("Out of time!")
            fuzzjs.unlink()
            if not os.listdir(str(wtmpDir)):
                wtmpDir.rmdir()
            break

        # Construct command needed to loop jsfunfuzz fuzzing.
        js_interesting_args = []
        js_interesting_args.append("--timeout=" + str(options.timeout))
        if options.valgrind:
            js_interesting_args.append("--valgrind")
        js_interesting_args.append(str(options.knownPath))
        js_interesting_args.append(str(options.jsEngine))
        if options.randomFlags:
            engineFlags = shell_flags.random_flag_set(options.jsEngine)  # pylint: disable=invalid-name
            js_interesting_args.extend(engineFlags)
        js_interesting_args.extend(["-e", "maxRunTime=" + str(options.timeout * (1000 // 2))])
        js_interesting_args.extend(["-f", fuzzjs])
        js_interesting_options = js_interesting.parseOptions(js_interesting_args)

        iteration += 1
        logPrefix = wtmpDir / ("w" + str(iteration))  # pylint: disable=invalid-name

        env = {}  # default environment will be used
        if ccoverage:
            env["GCOV_PREFIX_STRIP"] = "13"  # Assumes ccoverage build from b.f.m.o
            cov_build_path = Path(args[-2]).parent.parent.parent
            assert "cov-build" in str(cov_build_path)
            env["GCOV_PREFIX"] = str(cov_build_path)

        res = js_interesting.ShellResult(js_interesting_options,
                                         # pylint: disable=no-member
                                         js_interesting_options.jsengineWithArgs, logPrefix, False, env=env)

        if res.lev != js_interesting.JS_FINE:
            out_log = (logPrefix.parent / (logPrefix.stem + "-out")).with_suffix(".txt")
            showtail(out_log)
            err_log = (logPrefix.parent / (logPrefix.stem + "-err")).with_suffix(".txt")
            showtail(err_log)

            # splice jsfunfuzz.js with `grep "/*FRC-" wN-out`
            reduced_log = (logPrefix.parent / (logPrefix.stem + "-reduced")).with_suffix(".js")
            [before, after] = file_manipulation.fuzzSplice(fuzzjs)

            with io.open(str(out_log), "r", encoding="utf-8", errors="replace") as f:
                newfileLines = before + [  # pylint: disable=invalid-name
                    l.replace("/*FRC-", "/*") for l in file_manipulation.linesStartingWith(f, "/*FRC-")] + after
            orig_log = (logPrefix.parent / (logPrefix.stem + "-orig")).with_suffix(".js")
            with io.open(str(orig_log), "w", encoding="utf-8", errors="replace") as f:
                f.writelines(newfileLines)
            with io.open(str(reduced_log), "w", encoding="utf-8", errors="replace") as f:
                f.writelines(newfileLines)

            if not ccoverage:
                # Run Lithium and autobisectjs (make a reduced testcase and find a regression window)
                interestingpy = "funfuzz.js.js_interesting"
                itest = [interestingpy]
                if options.valgrind:
                    itest.append("--valgrind")
                itest.append("--minlevel=" + str(res.lev))
                itest.append("--timeout=" + str(options.timeout))
                itest.append(options.knownPath)
                (lithResult, _lithDetails, autoBisectLog) = lithium_helpers.pinpoint(  # pylint: disable=invalid-name
                    itest, logPrefix, options.jsEngine, engineFlags, reduced_log, options.repo,
                    options.build_options_str, targetTime, res.lev)

                # Upload with final output
                if lithResult == lithium_helpers.LITH_FINISHED:
                    # pylint: disable=no-member
                    fargs = js_interesting_options.jsengineWithArgs[:-1] + [reduced_log]
                    # pylint: disable=invalid-name
                    retestResult = js_interesting.ShellResult(js_interesting_options,
                                                              fargs,
                                                              logPrefix.parent / (logPrefix.stem + "-final"),
                                                              False)
                    if retestResult.lev > js_interesting.JS_FINE:
                        res = retestResult
                        quality = 0
                    else:
                        quality = 6
                else:
                    quality = 10

                print("Submitting %s (quality=%s) at %s" % (reduced_log, quality, time.asctime()))

                metadata = {}
                if autoBisectLog:
                    metadata = {"autoBisectLog": "".join(autoBisectLog)}
                collector.submit(res.crashInfo, str(reduced_log), quality, metaData=metadata)
                print("Submitted %s" % reduced_log)

        else:
            are_flags_deterministic = "--dump-bytecode" not in engineFlags and "-D" not in engineFlags
            # pylint: disable=no-member
            if options.use_compare_jit and res.lev == js_interesting.JS_FINE and \
                    js_interesting_options.shellIsDeterministic and are_flags_deterministic:
                out_log = (logPrefix.parent / (logPrefix.stem + "-out")).with_suffix(".txt")
                linesToCompare = jitCompareLines(out_log, "/*FCM*/")  # pylint: disable=invalid-name
                jitcomparefilename = (logPrefix.parent / (logPrefix.stem + "-cj-in")).with_suffix(".js")
                with io.open(str(jitcomparefilename), "w", encoding="utf-8", errors="replace") as f:
                    f.writelines(linesToCompare)
                if not ccoverage:
                    compare_jit.compare_jit(options.jsEngine, engineFlags, jitcomparefilename,
                                            logPrefix.parent / (logPrefix.stem + "-cj"), options.repo,
                                            options.build_options_str, targetTime, js_interesting_options)
                if jitcomparefilename.is_file():
                    jitcomparefilename.unlink()

            js_interesting.deleteLogs(logPrefix)

def jitCompareLines(jsfunfuzzOutputFilename, marker):  # pylint: disable=invalid-name,missing-param-doc
    # pylint: disable=missing-return-doc,missing-return-type-doc,missing-type-doc
    """Create a compare_jit file, using the lines marked by jsfunfuzz as valid for comparison."""
    lines = [
        "backtrace = function() { };\n",
        "dumpHeap = function() { };\n",
        "dumpObject = function() { };\n",
        "dumpStringRepresentation = function() { };\n",
        "evalInCooperativeThread = function() { };\n",
        "evalInWorker = function() { };\n",
        "getBacktrace = function() { };\n",
        "getLcovInfo = function() { };\n",
        "isAsmJSCompilationAvailable = function() { };\n",
        "offThreadCompileScript = function() { };\n",
        "oomTest = function() { };\n",
        "printProfilerEvents = function() { };\n",
        "saveStack = function() { };\n",
        "wasmIsSupported = function() { return true; };\n",
        "// DDBEGIN\n"
    ]
    with io.open(str(jsfunfuzzOutputFilename), "r", encoding="utf-8", errors="replace") as f:
        for line in f:
            if line.startswith(marker):
                sline = line[len(marker):]
                # We only override wasmIsSupported above for the main global.
                # Hopefully, any imported tests that try to use wasmIsSupported within a newGlobal
                # will do so in a straightforward way where everything is on one line.
                if not ("newGlobal" in sline and "wasmIsSupported" in sline):
                    lines.append(sline)
    lines += [
        "\ntry{print(uneval(this));}catch(e){}\n",
        "// DDEND\n"
    ]
    return lines

if __name__ == "__main__":
    # pylint: disable=no-member
    many_timed_runs(None, os_ops.make_wtmp_dir(Path(os.getcwdu() if sys.version_info.major == 2 else os.getcwd())),
                    sys.argv[1:], create_collector.make_collector(), False)
<commit_msg>Fix #206 - Include .js files present in non262 and test262 folders by default.<commit_after># coding=utf-8
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/.

"""Allows the funfuzz harness to run continuously.
"""

from __future__ import absolute_import, print_function, unicode_literals  # isort:skip

import io
import json
from optparse import OptionParser  # pylint: disable=deprecated-module
import os
import sys
import time

from . import compare_jit
from . import js_interesting
from . import link_fuzzer
from . import shell_flags
from ..util import create_collector
from ..util import file_manipulation
from ..util import lithium_helpers
from ..util import os_ops

if sys.version_info.major == 2:
    if os.name == "posix":
        import subprocess32 as subprocess  # pylint: disable=import-error
    from pathlib2 import Path  # pylint: disable=import-error
else:
    from pathlib import Path  # pylint: disable=import-error
    import subprocess

def parseOpts(args):  # pylint: disable=invalid-name,missing-docstring,missing-return-doc,missing-return-type-doc
    parser = OptionParser()
    parser.disable_interspersed_args()
    parser.add_option("--compare-jit",
                      action="store_true", dest="use_compare_jit",
                      default=False,
                      help="After running the fuzzer, run the FCM lines against the engine "
                           "in two configurations and compare the output.")
    parser.add_option("--random-flags",
                      action="store_true", dest="randomFlags",
                      default=False,
                      help="Pass a random set of flags (e.g. --ion-eager) to the js engine")
    parser.add_option("--repo",
                      action="store",
                      dest="repo",
                      help="The hg repository (e.g. ~/trees/mozilla-central/), for bisection")
    parser.add_option("--build",
                      action="store", dest="build_options_str",
                      help="The build options, for bisection",
                      default=None)  # if you run loop directly w/o --build, lithium_helpers.pinpoint will try to guess
    parser.add_option("--valgrind",
                      action="store_true", dest="valgrind",
                      default=False,
                      help="use valgrind with a reasonable set of options")
    options, args = parser.parse_args(args)

    # optparse does not recognize pathlib - we will need to move to argparse
    if options.repo:
        options.repo = Path(options.repo)
    else:
        options.repo = Path.home() / "trees" / "mozilla-central"

    if options.valgrind and options.use_compare_jit:
        print("Note: When running compare_jit, the --valgrind option will be ignored")

    # kill js shell if it runs this long.
    # jsfunfuzz will quit after half this time if it's not ilooping.
    # higher = more complex mixing, especially with regression tests.
    # lower = less time wasted in timeouts and in compare_jit testcases that are thrown away due to OOMs.
    options.timeout = int(args[0])

    # FIXME: We can probably remove args[1]  # pylint: disable=fixme
    options.knownPath = "mozilla-central"
    options.jsEngine = Path(args[2])
    options.engineFlags = args[3:]

    return options

def showtail(filename):  # pylint: disable=missing-docstring
    # pylint: disable=fixme
    # FIXME: Get jsfunfuzz to output start & end of interesting result boundaries instead of this.
    cmd = []
    cmd.extend(["tail", "-n", "20"])
    cmd.append(str(filename))
    print(" ".join(cmd))
    print()
    subprocess.run(cmd, check=True)
    print()
    print()

def makeRegressionTestPrologue(repo):  # pylint: disable=invalid-name,missing-param-doc
    # pylint: disable=missing-return-doc,missing-return-type-doc,missing-type-doc
    """Generate a JS string to tell jsfunfuzz where to find SpiderMonkey's regression tests."""
    return """
const regressionTestsRoot = %s;
const libdir = regressionTestsRoot + %s; // needed by jit-tests
const regressionTestList = %s;
""" % (json.dumps(str(repo) + os.sep),
       json.dumps(str(Path("js") / "src" / "jit-test" / "lib") + os.sep),
       json.dumps(inTreeRegressionTests(repo)))

def inTreeRegressionTests(repo):  # pylint: disable=invalid-name,missing-docstring,missing-return-doc
    # pylint: disable=missing-return-type-doc
    jit_tests = jsFilesIn(len(str(repo)), repo / "js" / "src" / "jit-test" / "tests")
    js_tests = jsFilesIn(len(str(repo)), repo / "js" / "src" / "tests")
    non262_tests = jsFilesIn(len(str(repo)), repo / "js" / "src" / "tests" / "non262")
    test262_tests = jsFilesIn(len(str(repo)), repo / "js" / "src" / "tests" / "test262")
    return jit_tests + js_tests + non262_tests + test262_tests

def jsFilesIn(repoPathLength, root):  # pylint: disable=invalid-name,missing-docstring,missing-return-doc
    # pylint: disable=missing-return-type-doc
    return [os.path.join(path, filename)[repoPathLength:]
            for path, _dirs, files in os.walk(str(root))
            for filename in files
            if filename.endswith(".js")]

def many_timed_runs(targetTime, wtmpDir, args, collector, ccoverage):  # pylint: disable=invalid-name,missing-docstring
    # pylint: disable=too-many-branches,too-complex,too-many-locals,too-many-statements
    options = parseOpts(args)
    # engineFlags is overwritten later if --random-flags is set.
    engineFlags = options.engineFlags  # pylint: disable=invalid-name
    startTime = time.time()  # pylint: disable=invalid-name

    if options.repo.is_dir():
        regressionTestPrologue = makeRegressionTestPrologue(options.repo)  # pylint: disable=invalid-name
    else:
        regressionTestPrologue = ""  # pylint: disable=invalid-name

    fuzzjs = wtmpDir / "jsfunfuzz.js"

    link_fuzzer.link_fuzzer(fuzzjs, regressionTestPrologue)
    assert fuzzjs.is_file()

    iteration = 0
    while True:
        if targetTime and time.time() > startTime + targetTime:
            print("Out of time!")
            fuzzjs.unlink()
            if not os.listdir(str(wtmpDir)):
                wtmpDir.rmdir()
            break

        # Construct command needed to loop jsfunfuzz fuzzing.
        js_interesting_args = []
        js_interesting_args.append("--timeout=" + str(options.timeout))
        if options.valgrind:
            js_interesting_args.append("--valgrind")
        js_interesting_args.append(str(options.knownPath))
        js_interesting_args.append(str(options.jsEngine))
        if options.randomFlags:
            engineFlags = shell_flags.random_flag_set(options.jsEngine)  # pylint: disable=invalid-name
            js_interesting_args.extend(engineFlags)
        js_interesting_args.extend(["-e", "maxRunTime=" + str(options.timeout * (1000 // 2))])
        js_interesting_args.extend(["-f", fuzzjs])
        js_interesting_options = js_interesting.parseOptions(js_interesting_args)

        iteration += 1
        logPrefix = wtmpDir / ("w" + str(iteration))  # pylint: disable=invalid-name

        env = {}  # default environment will be used
        if ccoverage:
            env["GCOV_PREFIX_STRIP"] = "13"  # Assumes ccoverage build from b.f.m.o
            cov_build_path = Path(args[-2]).parent.parent.parent
            assert "cov-build" in str(cov_build_path)
            env["GCOV_PREFIX"] = str(cov_build_path)

        res = js_interesting.ShellResult(js_interesting_options,
                                         # pylint: disable=no-member
                                         js_interesting_options.jsengineWithArgs, logPrefix, False, env=env)

        if res.lev != js_interesting.JS_FINE:
            out_log = (logPrefix.parent / (logPrefix.stem + "-out")).with_suffix(".txt")
            showtail(out_log)
            err_log = (logPrefix.parent / (logPrefix.stem + "-err")).with_suffix(".txt")
            showtail(err_log)

            # splice jsfunfuzz.js with `grep "/*FRC-" wN-out`
            reduced_log = (logPrefix.parent / (logPrefix.stem + "-reduced")).with_suffix(".js")
            [before, after] = file_manipulation.fuzzSplice(fuzzjs)

            with io.open(str(out_log), "r", encoding="utf-8", errors="replace") as f:
                newfileLines = before + [  # pylint: disable=invalid-name
                    l.replace("/*FRC-", "/*") for l in file_manipulation.linesStartingWith(f, "/*FRC-")] + after
            orig_log = (logPrefix.parent / (logPrefix.stem + "-orig")).with_suffix(".js")
            with io.open(str(orig_log), "w", encoding="utf-8", errors="replace") as f:
                f.writelines(newfileLines)
            with io.open(str(reduced_log), "w", encoding="utf-8", errors="replace") as f:
                f.writelines(newfileLines)

            if not ccoverage:
                # Run Lithium and autobisectjs (make a reduced testcase and find a regression window)
                interestingpy = "funfuzz.js.js_interesting"
                itest = [interestingpy]
                if options.valgrind:
                    itest.append("--valgrind")
                itest.append("--minlevel=" + str(res.lev))
                itest.append("--timeout=" + str(options.timeout))
                itest.append(options.knownPath)
                (lithResult, _lithDetails, autoBisectLog) = lithium_helpers.pinpoint(  # pylint: disable=invalid-name
                    itest, logPrefix, options.jsEngine, engineFlags, reduced_log, options.repo,
                    options.build_options_str, targetTime, res.lev)

                # Upload with final output
                if lithResult == lithium_helpers.LITH_FINISHED:
                    # pylint: disable=no-member
                    fargs = js_interesting_options.jsengineWithArgs[:-1] + [reduced_log]
                    # pylint: disable=invalid-name
                    retestResult = js_interesting.ShellResult(js_interesting_options,
                                                              fargs,
                                                              logPrefix.parent / (logPrefix.stem + "-final"),
                                                              False)
                    if retestResult.lev > js_interesting.JS_FINE:
                        res = retestResult
                        quality = 0
                    else:
                        quality = 6
                else:
                    quality = 10

                print("Submitting %s (quality=%s) at %s" % (reduced_log, quality, time.asctime()))

                metadata = {}
                if autoBisectLog:
                    metadata = {"autoBisectLog": "".join(autoBisectLog)}
                collector.submit(res.crashInfo, str(reduced_log), quality, metaData=metadata)
                print("Submitted %s" % reduced_log)

        else:
            are_flags_deterministic = "--dump-bytecode" not in engineFlags and "-D" not in engineFlags
            # pylint: disable=no-member
            if options.use_compare_jit and res.lev == js_interesting.JS_FINE and \
                    js_interesting_options.shellIsDeterministic and are_flags_deterministic:
                out_log = (logPrefix.parent / (logPrefix.stem + "-out")).with_suffix(".txt")
                linesToCompare = jitCompareLines(out_log, "/*FCM*/")  # pylint: disable=invalid-name
                jitcomparefilename = (logPrefix.parent / (logPrefix.stem + "-cj-in")).with_suffix(".js")
                with io.open(str(jitcomparefilename), "w", encoding="utf-8", errors="replace") as f:
                    f.writelines(linesToCompare)
                if not ccoverage:
                    compare_jit.compare_jit(options.jsEngine, engineFlags, jitcomparefilename,
                                            logPrefix.parent / (logPrefix.stem + "-cj"), options.repo,
                                            options.build_options_str, targetTime, js_interesting_options)
                if jitcomparefilename.is_file():
                    jitcomparefilename.unlink()

            js_interesting.deleteLogs(logPrefix)

def jitCompareLines(jsfunfuzzOutputFilename, marker):  # pylint: disable=invalid-name,missing-param-doc
    # pylint: disable=missing-return-doc,missing-return-type-doc,missing-type-doc
    """Create a compare_jit file, using the lines marked by jsfunfuzz as valid for comparison."""
    lines = [
        "backtrace = function() { };\n",
        "dumpHeap = function() { };\n",
        "dumpObject = function() { };\n",
        "dumpStringRepresentation = function() { };\n",
        "evalInCooperativeThread = function() { };\n",
        "evalInWorker = function() { };\n",
        "getBacktrace = function() { };\n",
        "getLcovInfo = function() { };\n",
        "isAsmJSCompilationAvailable = function() { };\n",
        "offThreadCompileScript = function() { };\n",
        "oomTest = function() { };\n",
        "printProfilerEvents = function() { };\n",
        "saveStack = function() { };\n",
        "wasmIsSupported = function() { return true; };\n",
        "// DDBEGIN\n"
    ]
    with io.open(str(jsfunfuzzOutputFilename), "r", encoding="utf-8", errors="replace") as f:
        for line in f:
            if line.startswith(marker):
                sline = line[len(marker):]
                # We only override wasmIsSupported above for the main global.
                # Hopefully, any imported tests that try to use wasmIsSupported within a newGlobal
                # will do so in a straightforward way where everything is on one line.
                if not ("newGlobal" in sline and "wasmIsSupported" in sline):
                    lines.append(sline)
    lines += [
        "\ntry{print(uneval(this));}catch(e){}\n",
        "// DDEND\n"
    ]
    return lines

if __name__ == "__main__":
    # pylint: disable=no-member
    many_timed_runs(None, os_ops.make_wtmp_dir(Path(os.getcwdu() if sys.version_info.major == 2 else os.getcwd())),
                    sys.argv[1:], create_collector.make_collector(), False)
<|endoftext|>


<commit_before>import re
import json
from tornado.web import HTTPError
from biothings.www.helper import BaseHandler
from biothings.utils.common import split_ids
from biothings.utils.version import get_python_version
from biothings.utils.version import get_repository_information
from biothings.settings import BiothingSettings

biothing_settings = BiothingSettings()

class BiothingHandler(BaseHandler):

    def _ga_event_object(self, action, data={}):
        ''' Returns the google analytics object for requests on this endpoint (annotation handler).'''
        return biothing_settings.ga_event_object(endpoint=biothing_settings._annotation_endpoint, action=action, data=data)

    def _examine_kwargs(self, action, kwargs):
        ''' A function for sub-classing.  This will be run after the get_query_params but before the actual
            elasticsearch querying. '''
        if action == 'GET':
            pass
        elif action == 'POST':
            pass
        pass

    def get(self, bid=None):
        '''
        '''
        if bid:
            kwargs = self.get_query_params()
            self._examine_kwargs('GET', kwargs)
            biothing_object = self.esq.get_biothing(bid, **kwargs)
            if biothing_object:
                self.return_json(biothing_object)
                self.ga_track(event=self._ga_event_object('GET'))
            else:
                raise HTTPError(404)
        else:
            raise HTTPError(404)

    def post(self, ids=None):
        '''
           This is essentially the same as post request in QueryHandler, with different defaults.

           parameters:
            ids
            fields
            email
            jsonld
        '''
        kwargs = self.get_query_params()
        self._examine_kwargs('POST', kwargs)
        ids = kwargs.pop('ids', None)
        if ids:
            ids = re.split('[\s\r\n+|,]+', ids)
            res = self.esq.mget_biothings(ids, **kwargs)
        else:
            res = {'success': False, 'error': "Missing required parameters."}
        encode = not isinstance(res, str)    # when res is a string, e.g. when rawquery is true, do not encode it as json
        self.return_json(res, encode=encode)
        self.ga_track(event=self._ga_event_object('POST', {'qsize': len(ids) if ids else 0}))

class QueryHandler(BaseHandler):

    def _ga_event_object(self, action, data={}):
        ''' Returns the google analytics object for requests on this endpoint (query handler).'''
        return biothing_settings.ga_event_object(endpoint=biothing_settings._query_endpoint, action=action, data=data)

    def _examine_kwargs(self, action, kwargs):
        ''' A function for sub-classing.  This will be run after the get_query_params but before the actual
            elasticsearch querying. '''
        if action == 'GET':
            pass
        elif action == 'POST':
            pass
        pass

    def get(self):
        '''
        parameters:
            q
            fields
            from
            size
            sort
            facets
            callback
            email
            fetch_all
            jsonld
            explain
            raw
        '''
        kwargs = self.get_query_params()
        self._examine_kwargs('GET', kwargs)
        q = kwargs.pop('q', None)
        scroll_id = kwargs.pop('scroll_id', None)
        _has_error = False
        if scroll_id:
            res = self.esq.scroll(scroll_id, **kwargs)
        elif q:
            for arg in ['from', 'size']:
                value = kwargs.get(arg, None)
                if value:
                    try:
                        kwargs[arg] = int(value)
                    except ValueError:
                        res = {'success': False, 'error': 'Parameter "{}" must be an integer.'.format(arg)}
                        _has_error = True
            if not _has_error:
                res = self.esq.query(q, **kwargs)
                if kwargs.get('fetch_all', False):
                    self.ga_track(event=self._ga_event_object('fetch_all', {'total': res.get('total', None)}))
        else:
            res = {'success': False, 'error': "Missing required parameters."}

        self.return_json(res)
        self.ga_track(event=self._ga_event_object('GET', {'qsize': len(q) if q else 0}))

    def post(self):
        '''
        parameters:
            q
            scopes
            fields
            email
            jsonld
            jsoninput   if true, input "q" is a json string, must be decoded as a list.
        '''
        kwargs = self.get_query_params()
        self._examine_kwargs('POST', kwargs)
        q = kwargs.pop('q', None)
        jsoninput = kwargs.pop('jsoninput', None) in ('1', 'true')
        if q:
            # ids = re.split('[\s\r\n+|,]+', q)
            try:
                ids = json.loads(q) if jsoninput else split_ids(q)
                if not isinstance(ids, list):
                    raise ValueError
            except ValueError:
                ids = None
                res = {'success': False, 'error': 'Invalid input for "q" parameter.'}
            if ids:
                scopes = kwargs.pop('scopes', None)
                fields = kwargs.pop('fields', None)
                res = self.esq.mget_biothings(ids, fields=fields, scopes=scopes, **kwargs)
        else:
            res = {'success': False, 'error': "Missing required parameters."}

        encode = not isinstance(res, str)    # when res is a string, e.g. when rawquery is true, do not encode it as json
        self.return_json(res, encode=encode)
        self.ga_track(event=self._ga_event_object('POST', {'qsize': len(q) if q else 0}))

class Neo4jQueryHandler(BaseHandler):
    ''' Implements a graph query endpoint for HTML GET. '''

    def _ga_event_object(self, action, data={}):
        ''' Returns the google analytics object for requests on this endpoint (query handler).'''
        return biothing_settings.ga_event_object(endpoint=biothing_settings._graph_query_endpoint, action=action, data=data)

    def _examine_kwargs(self, action, kwargs):
        ''' A function for sub-classing.  This will be run after the get_query_params but before the actual
            elasticsearch querying. '''
        pass

    def get(self):
        kwargs = self.get_query_params()
        self._examine_kwargs('GET', kwargs)
        q = kwargs.pop('q', None)
        if q:
            res = self.neo4jq.query(q, **kwargs)
        else:
            res = {'success': False, 'error': "Missing required parameters."}

        self.return_json(res)
        self.ga_track(event=self._ga_event_object('GET'))

class MetaDataHandler(BaseHandler):

    def get(self):
        _meta = self.esq.get_mapping_meta()
        _meta['software'] = {
            'python-package-info': get_python_version(),
            'codebase': get_repository_information()
        }
        self.return_json(_meta)

    def get_python_package_info(self):

class FieldsHandler(BaseHandler):

    def get(self):
        es_mapping = self.esq.query_fields()
        if biothing_settings.field_notes_path:
            notes = json.load(open(biothing_settings.field_notes_path, 'r'))
        else:
            notes = {}
        kwargs = self.get_query_params()

        def get_indexed_properties_in_dict(d, prefix):
            r = {}
            for (k, v) in d.items():
                r[prefix + '.' + k] = {}
                r[prefix + '.' + k]['indexed'] = False
                if 'properties' not in v:
                    r[prefix + '.' + k]['type'] = v['type']
                    if ('index' not in v) or ('index' in v and v['index'] != 'no'):
                        # indexed field
                        r[prefix + '.' + k]['indexed'] = True
                else:
                    r[prefix + '.' + k]['type'] = 'object'
                    r.update(get_indexed_properties_in_dict(v['properties'], prefix + '.' + k))
                if ('include_in_all' in v) and v['include_in_all']:
                    r[prefix + '.' + k]['include_in_all'] = True
                else:
                    r[prefix + '.' + k]['include_in_all'] = False
            return r

        r = {}
        search = kwargs.pop('search', None)
        prefix = kwargs.pop('prefix', None)
        for (k, v) in get_indexed_properties_in_dict(es_mapping, '').items():
            k1 = k.lstrip('.')
            if (search and search in k1) or (prefix and k1.startswith(prefix)) or (not search and not prefix):
                r[k1] = v
                if k1 in notes:
                    r[k1]['notes'] = notes[k1]
        self.return_json(r)

class StatusHandler(BaseHandler):
    ''' Handles requests to check the status of the server. '''

    def head(self):
        r = self.esq.status_check(biothing_settings.status_check_id)
        if r is None:
            # we failed to retrieve ref/test doc, something is wrong -> service unavailable
            raise HTTPError(503)

    def get(self):
        self.head()
        self.write('OK')
<commit_msg>Update handlers.py<commit_after>import re
import json
from tornado.web import HTTPError
from biothings.www.helper import BaseHandler
from biothings.utils.common import split_ids
from biothings.utils.version import get_python_version
from biothings.utils.version import get_repository_information
from biothings.settings import BiothingSettings

biothing_settings = BiothingSettings()

class BiothingHandler(BaseHandler):

    def _ga_event_object(self, action, data={}):
        ''' Returns the google analytics object for requests on this endpoint (annotation handler).'''
        return biothing_settings.ga_event_object(endpoint=biothing_settings._annotation_endpoint, action=action, data=data)

    def _examine_kwargs(self, action, kwargs):
        ''' A function for sub-classing.  This will be run after the get_query_params but before the actual
            elasticsearch querying. '''
        if action == 'GET':
            pass
        elif action == 'POST':
            pass
        pass

    def get(self, bid=None):
        '''
        '''
        if bid:
            kwargs = self.get_query_params()
            self._examine_kwargs('GET', kwargs)
            biothing_object = self.esq.get_biothing(bid, **kwargs)
            if biothing_object:
                self.return_json(biothing_object)
                self.ga_track(event=self._ga_event_object('GET'))
            else:
                raise HTTPError(404)
        else:
            raise HTTPError(404)

    def post(self, ids=None):
        '''
           This is essentially the same as post request in QueryHandler, with different defaults.

           parameters:
            ids
            fields
            email
            jsonld
        '''
        kwargs = self.get_query_params()
        self._examine_kwargs('POST', kwargs)
        ids = kwargs.pop('ids', None)
        if ids:
            ids = re.split('[\s\r\n+|,]+', ids)
            res = self.esq.mget_biothings(ids, **kwargs)
        else:
            res = {'success': False, 'error': "Missing required parameters."}
        encode = not isinstance(res, str)    # when res is a string, e.g. when rawquery is true, do not encode it as json
        self.return_json(res, encode=encode)
        self.ga_track(event=self._ga_event_object('POST', {'qsize': len(ids) if ids else 0}))

class QueryHandler(BaseHandler):

    def _ga_event_object(self, action, data={}):
        ''' Returns the google analytics object for requests on this endpoint (query handler).'''
        return biothing_settings.ga_event_object(endpoint=biothing_settings._query_endpoint, action=action, data=data)

    def _examine_kwargs(self, action, kwargs):
        ''' A function for sub-classing.  This will be run after the get_query_params but before the actual
            elasticsearch querying. '''
        if action == 'GET':
            pass
        elif action == 'POST':
            pass
        pass

    def get(self):
        '''
        parameters:
            q
            fields
            from
            size
            sort
            facets
            callback
            email
            fetch_all
            jsonld
            explain
            raw
        '''
        kwargs = self.get_query_params()
        self._examine_kwargs('GET', kwargs)
        q = kwargs.pop('q', None)
        scroll_id = kwargs.pop('scroll_id', None)
        _has_error = False
        if scroll_id:
            res = self.esq.scroll(scroll_id, **kwargs)
        elif q:
            for arg in ['from', 'size']:
                value = kwargs.get(arg, None)
                if value:
                    try:
                        kwargs[arg] = int(value)
                    except ValueError:
                        res = {'success': False, 'error': 'Parameter "{}" must be an integer.'.format(arg)}
                        _has_error = True
            if not _has_error:
                res = self.esq.query(q, **kwargs)
                if kwargs.get('fetch_all', False):
                    self.ga_track(event=self._ga_event_object('fetch_all', {'total': res.get('total', None)}))
        else:
            res = {'success': False, 'error': "Missing required parameters."}

        self.return_json(res)
        self.ga_track(event=self._ga_event_object('GET', {'qsize': len(q) if q else 0}))

    def post(self):
        '''
        parameters:
            q
            scopes
            fields
            email
            jsonld
            jsoninput   if true, input "q" is a json string, must be decoded as a list.
        '''
        kwargs = self.get_query_params()
        self._examine_kwargs('POST', kwargs)
        q = kwargs.pop('q', None)
        jsoninput = kwargs.pop('jsoninput', None) in ('1', 'true')
        if q:
            # ids = re.split('[\s\r\n+|,]+', q)
            try:
                ids = json.loads(q) if jsoninput else split_ids(q)
                if not isinstance(ids, list):
                    raise ValueError
            except ValueError:
                ids = None
                res = {'success': False, 'error': 'Invalid input for "q" parameter.'}
            if ids:
                scopes = kwargs.pop('scopes', None)
                fields = kwargs.pop('fields', None)
                res = self.esq.mget_biothings(ids, fields=fields, scopes=scopes, **kwargs)
        else:
            res = {'success': False, 'error': "Missing required parameters."}

        encode = not isinstance(res, str)    # when res is a string, e.g. when rawquery is true, do not encode it as json
        self.return_json(res, encode=encode)
        self.ga_track(event=self._ga_event_object('POST', {'qsize': len(q) if q else 0}))

class Neo4jQueryHandler(BaseHandler):
    ''' Implements a graph query endpoint for HTML GET. '''

    def _ga_event_object(self, action, data={}):
        ''' Returns the google analytics object for requests on this endpoint (query handler).'''
        return biothing_settings.ga_event_object(endpoint=biothing_settings._graph_query_endpoint, action=action, data=data)

    def _examine_kwargs(self, action, kwargs):
        ''' A function for sub-classing.  This will be run after the get_query_params but before the actual
            elasticsearch querying. '''
        pass

    def get(self):
        kwargs = self.get_query_params()
        self._examine_kwargs('GET', kwargs)
        q = kwargs.pop('q', None)
        if q:
            res = self.neo4jq.query(q, **kwargs)
        else:
            res = {'success': False, 'error': "Missing required parameters."}

        self.return_json(res)
        self.ga_track(event=self._ga_event_object('GET'))

class MetaDataHandler(BaseHandler):

    def get(self):
        _meta = self.esq.get_mapping_meta()
        _meta['software'] = {
            'python-package-info': get_python_version(),
            'codebase': get_repository_information()
        }
        self.return_json(_meta)

class FieldsHandler(BaseHandler):

    def get(self):
        es_mapping = self.esq.query_fields()
        if biothing_settings.field_notes_path:
            notes = json.load(open(biothing_settings.field_notes_path, 'r'))
        else:
            notes = {}
        kwargs = self.get_query_params()

        def get_indexed_properties_in_dict(d, prefix):
            r = {}
            for (k, v) in d.items():
                r[prefix + '.' + k] = {}
                r[prefix + '.' + k]['indexed'] = False
                if 'properties' not in v:
                    r[prefix + '.' + k]['type'] = v['type']
                    if ('index' not in v) or ('index' in v and v['index'] != 'no'):
                        # indexed field
                        r[prefix + '.' + k]['indexed'] = True
                else:
                    r[prefix + '.' + k]['type'] = 'object'
                    r.update(get_indexed_properties_in_dict(v['properties'], prefix + '.' + k))
                if ('include_in_all' in v) and v['include_in_all']:
                    r[prefix + '.' + k]['include_in_all'] = True
                else:
                    r[prefix + '.' + k]['include_in_all'] = False
            return r

        r = {}
        search = kwargs.pop('search', None)
        prefix = kwargs.pop('prefix', None)
        for (k, v) in get_indexed_properties_in_dict(es_mapping, '').items():
            k1 = k.lstrip('.')
            if (search and search in k1) or (prefix and k1.startswith(prefix)) or (not search and not prefix):
                r[k1] = v
                if k1 in notes:
                    r[k1]['notes'] = notes[k1]
        self.return_json(r)

class StatusHandler(BaseHandler):
    ''' Handles requests to check the status of the server. '''

    def head(self):
        r = self.esq.status_check(biothing_settings.status_check_id)
        if r is None:
            # we failed to retrieve ref/test doc, something is wrong -> service unavailable
            raise HTTPError(503)

    def get(self):
        self.head()
        self.write('OK')
<|endoftext|>