<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

###############################################################################
#  Copyright Kitware Inc.
#
#  Licensed under the Apache License, Version 2.0 ( the "License" );
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
###############################################################################

import csv
from cStringIO import StringIO
import functools
import itertools

import six

from girder.api import access
from girder.api.rest import RestException, loadmodel, setResponseHeader
from girder.api.describe import Description, describeRoute
from girder.constants import AccessType, SortDir
from girder.models.model_base import ValidationException

from .base import IsicResource

class StudyResource(IsicResource):
    def __init__(self,):
        super(StudyResource, self).__init__()
        self.resourceName = 'study'

        self.route('GET', (), self.find)
        self.route('GET', (':id',), self.getStudy)
        self.route('POST', (), self.createStudy)
        self.route('POST', (':id', 'users'), self.addAnnotators)
        self.route('POST', (':id', 'images'), self.addImages)

    @describeRoute(
        Description('Return a list of annotation studies.')
        .pagingParams(defaultSort='lowerName')
        .param('state', 'Filter studies to those at a given state',
               paramType='query', required=False, enum=('active', 'complete'))
        .param('userId',
               'Filter studies to those containing a user ID, or "me".',
               paramType='query', required=False)
        .errorResponse()
    )
    @access.public
    def find(self, params):
        Study = self.model('study', 'isic_archive')
        User = self.model('user', 'isic_archive')

        limit, offset, sort = self.getPagingParameters(params, 'lowerName')

        annotatorUser = None
        if params.get('userId'):
            if params['userId'] == 'me':
                annotatorUser = self.getCurrentUser()
            else:
                annotatorUser = User.load(
                    id=params['userId'],
                    force=True, exc=True)

        state = None
        if params.get('state'):
            state = params['state']
            if state not in {Study.State.ACTIVE, Study.State.COMPLETE}:
                raise RestException('Query parameter "state" may only '
                                    'be "active" or "complete".')

        return [
            {
                field: study[field]
                for field in
                Study.summaryFields
            }
            for study in
            Study.filterResultsByPermission(
                Study.find(query=None, annotatorUser=annotatorUser,
                           state=state, sort=sort),
                user=self.getCurrentUser(),
                level=AccessType.READ, limit=limit, offset=offset
            )
        ]

    @describeRoute(
        Description('Get a study by ID.')
        .param('id', 'The ID of the study.', paramType='path')
        .param('format', 'The output format.',
               paramType='query', required=False, enum=('csv', 'json'),
               default='json')
        .errorResponse()
    )
    @access.public
    @loadmodel(model='study', plugin='isic_archive', level=AccessType.READ)
    def getStudy(self, study, params):
        Study = self.model('study', 'isic_archive')
        Featureset = self.model('featureset', 'isic_archive')
        Image = self.model('image', 'isic_archive')
        User = self.model('user', 'isic_archive')

        if params.get('format') == 'csv':
            setResponseHeader('Content-Type', 'text/csv')
            setResponseHeader(
                'Content-Disposition',
                'attachment; filename="%s.csv"' % study['name'])
            return functools.partial(self._getStudyCSVStream, study)

        else:
            currentUser = self.getCurrentUser()
            output = Study.filter(study, currentUser)
            del output['_accessLevel']
            output.update({
                '_modelType': 'study',
                'featureset': Featureset.load(
                    id=study['meta']['featuresetId'],
                    fields=Featureset.summaryFields, exc=True),
                'creator': User.filteredSummary(
                    User.load(
                        output.pop('creatorId'),
                        force=True, exc=True),
                    currentUser),
                'users': sorted(
                    (
                        User.filteredSummary(annotatorUser, currentUser)
                        for annotatorUser in
                        Study.getAnnotators(study)
                    ),
                    key=lambda annotatorUser: User.obfuscatedName(annotatorUser)
                ),
                'images': list(
                    Study.getImages(
                        study, Image.summaryFields
                    ).sort('lowerName', SortDir.ASCENDING))
            })

            return output

    def _getStudyCSVStream(self, study):
        Study = self.model('study', 'isic_archive')
        Featureset = self.model('featureset', 'isic_archive')
        User = self.model('user', 'isic_archive')

        currentUser = self.getCurrentUser()

        featureset = Featureset.load(study['meta']['featuresetId'], exc=True)
        csvFields = tuple(itertools.chain(
            ('study_name', 'study_id',
             'user_name', 'user_id',
             'image_name', 'image_id',
             'flag_status', 'elapsed_seconds'),
            (feature['id'] for feature in featureset['globalFeatures']),
            ('superpixel_id',),
            (feature['id'] for feature in featureset['localFeatures'])
        ))

        responseBody = StringIO()
        csvWriter = csv.DictWriter(responseBody, csvFields, restval='')

        csvWriter.writeheader()
        yield responseBody.getvalue()
        responseBody.seek(0)
        responseBody.truncate()

        images = list(
            Study.getImages(study).sort('lowerName', SortDir.ASCENDING))

        for annotatorUser, image in itertools.product(
            Study.getAnnotators(study).sort('login', SortDir.ASCENDING),
                images
        ):
            # this will iterate either 0 or 1 times
            for annotation in Study.childAnnotations(
                study=study,
                annotatorUser=annotatorUser,
                image=image,
                state=Study.State.COMPLETE
            ):
                elapsedSeconds = \
                    int((annotation['meta']['stopTime'] -
                         annotation['meta']['startTime']).total_seconds())

                filteredAnnotatorUser = User.filteredSummary(
                    annotatorUser, currentUser)
                annotatorUserName = filteredAnnotatorUser.get(
                    'login', filteredAnnotatorUser['name'])

                outDictBase = {
                    'study_name': study['name'],
                    'study_id': str(study['_id']),
                    'user_name': annotatorUserName,
                    'user_id': str(annotatorUser['_id']),
                    'image_name': image['name'],
                    'image_id': str(image['_id']),
                    'flag_status': annotation['meta']['status'],
                    'elapsed_seconds': elapsedSeconds
                }

                outDict = outDictBase.copy()
                for globalFeature in featureset['globalFeatures']:
                    if globalFeature['id'] in annotation['meta']['annotations']:
                        outDict[globalFeature['id']] = \
                            annotation['meta']['annotations'][globalFeature['id']]  # noqa: E501
                csvWriter.writerow(outDict)
                yield responseBody.getvalue()
                responseBody.seek(0)
                responseBody.truncate()

                # TODO: move this into the query
                if 'localFeatures' in annotation['meta']['annotations']:
                    superpixelCount = len(next(six.viewvalues(
                        annotation['meta']['annotations']['localFeatures'])))
                    for superpixelMum in xrange(superpixelCount):

                        outDict = outDictBase.copy()
                        outDict['superpixel_id'] = superpixelMum
                        for featureName, featureValue in six.viewitems(
                                annotation['meta']['annotations']['localFeatures']):  # noqa: E501
                            outDict[featureName] = featureValue[superpixelMum]

                        csvWriter.writerow(outDict)
                        yield responseBody.getvalue()
                        responseBody.seek(0)
                        responseBody.truncate()

    @describeRoute(
        Description('Create an annotation study.')
        .param('name', 'The name of the study.', paramType='form')
        .param('featuresetId', 'The featureset ID of the study.',
               paramType='form')
        .param('userIds',
               'The annotators user IDs of the study, as a JSON array.',
               paramType='form')
        .param('imageIds',
               'The image IDs of the study, as a JSON array.',
               paramType='form')
        .errorResponse('Write access was denied on the parent folder.', 403)
    )
    @access.user
    def createStudy(self, params):
        Study = self.model('study', 'isic_archive')
        Featureset = self.model('featureset', 'isic_archive')
        Image = self.model('image', 'isic_archive')
        User = self.model('user', 'isic_archive')

        creatorUser = self.getCurrentUser()
        User.requireAdminStudy(creatorUser)

        params = self._decodeParams(params)
        self.requireParams(
            ['name', 'featuresetId', 'userIds', 'imageIds'],
            params)

        studyName = params['name'].strip()
        if not studyName:
            raise ValidationException('Name must not be empty.', 'name')

        featuresetId = params['featuresetId']
        if not featuresetId:
            raise ValidationException('Invalid featureset ID.', 'featuresetId')
        featureset = Featureset.load(featuresetId, exc=True)

        if len(set(params['userIds'])) != len(params['userIds']):
            raise RestException('Duplicate user IDs.')
        annotatorUsers = [
            User.load(
                annotatorUserId, user=creatorUser, level=AccessType.READ,
                exc=True)
            for annotatorUserId in params['userIds']
        ]

        if len(set(params['imageIds'])) != len(params['imageIds']):
            raise RestException('Duplicate image IDs.')
        images = [
            Image.load(
                imageId, user=creatorUser, level=AccessType.READ, exc=True)
            for imageId in params['imageIds']
        ]

        study = Study.createStudy(
            name=studyName,
            creatorUser=creatorUser,
            featureset=featureset,
            annotatorUsers=annotatorUsers,
            images=images)

        return self.getStudy(id=study['_id'], params={})

    @describeRoute(
        Description('Add annotator users to a study.')
        .param('id', 'The ID of the study.', paramType='path')
        .param('userIds', 'The user IDs to add, as a JSON array.',
               paramType='form')
        .errorResponse('ID was invalid.')
        .errorResponse('You don\'t have permission to add a study annotator.',
                       403)
    )
    @access.user
    @loadmodel(model='study', plugin='isic_archive', level=AccessType.READ)
    def addAnnotators(self, study, params):
        Folder = self.model('folder')
        Study = self.model('study', 'isic_archive')
        User = self.model('user', 'isic_archive')

        # TODO: make the loadmodel decorator use AccessType.WRITE,
        # once permissions work
        params = self._decodeParams(params)
        self.requireParams('userIds', params)

        creatorUser = self.getCurrentUser()
        User.requireAdminStudy(creatorUser)

        # Load all users before adding any, to ensure all are valid
        if len(set(params['userIds'])) != len(params['userIds']):
            raise RestException('Duplicate user IDs.')
        annotatorUsers = [
            User.load(userId, user=creatorUser, level=AccessType.READ, exc=True)
            for userId in params['userIds']
        ]
        # Existing duplicate Annotators are tricky to check for, because it's
        # possible to have a Study with multiple annotator Users (each with a
        # sub-Folder), but with no Images yet, and thus no Annotation (Items)
        # inside yet
        duplicateAnnotatorFolders = Folder.find({
            'parentId': study['_id'],
            'meta.userId': {'$in': [
                annotatorUser['_id'] for annotatorUser in annotatorUsers]}
        })
        if duplicateAnnotatorFolders.count():
            # Just list the first duplicate
            duplicateAnnotatorFolder = next(iter(duplicateAnnotatorFolders))
            raise ValidationException(
                'Annotator user "%s" is already part of the study.' %
                duplicateAnnotatorFolder['meta']['userId'])
        # Look up images only once for efficiency
        images = Study.getImages(study)
        for annotatorUser in annotatorUsers:
            Study.addAnnotator(study, annotatorUser, creatorUser, images)

        return self.getStudy(id=study['_id'], params={})

    @describeRoute(
        Description('Add images to a study.')
        .param('id', 'The ID of the study.', paramType='path')
        .param('imageIds', 'The image IDs to add, as a JSON array.',
               paramType='form')
        .errorResponse('ID was invalid.')
        .errorResponse('You don\'t have permission to add a study image.', 403)
    )
    @access.user
    @loadmodel(model='study', plugin='isic_archive', level=AccessType.READ)
    def addImages(self, study, params):
        Annotation = self.model('annotation', 'isic_archive')
        Image = self.model('image', 'isic_archive')
        Study = self.model('study', 'isic_archive')
        User = self.model('user', 'isic_archive')

        params = self._decodeParams(params)
        self.requireParams('imageIds', params)

        creatorUser = self.getCurrentUser()
        User.requireAdminStudy(creatorUser)

        # Load all images before adding any, to ensure all are valid and
        # accessible
        if len(set(params['imageIds'])) != len(params['imageIds']):
            raise RestException('Duplicate image IDs.')
        images = [
            Image.load(
                imageId, user=creatorUser, level=AccessType.READ, exc=True)
            for imageId in params['imageIds']
        ]
        duplicateAnnotations = Annotation.find({
            'meta.studyId': study['_id'],
            'meta.imageId': {'$in': [image['_id'] for image in images]}
        })
        if duplicateAnnotations.count():
            # Just list the first duplicate
            duplicateAnnotation = next(iter(duplicateAnnotations))
            raise ValidationException(
                'Image "%s" is already part of the study.' %
                duplicateAnnotation['meta']['imageId'])
        for image in images:
            Study.addImage(study, image, creatorUser)

        return self.getStudy(id=study['_id'], params={})
<commit_msg>Output obfuscated user name in Study CSV download API<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

###############################################################################
#  Copyright Kitware Inc.
#
#  Licensed under the Apache License, Version 2.0 ( the "License" );
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
###############################################################################

import csv
from cStringIO import StringIO
import functools
import itertools

import six

from girder.api import access
from girder.api.rest import RestException, loadmodel, setResponseHeader
from girder.api.describe import Description, describeRoute
from girder.constants import AccessType, SortDir
from girder.models.model_base import ValidationException

from .base import IsicResource

class StudyResource(IsicResource):
    def __init__(self,):
        super(StudyResource, self).__init__()
        self.resourceName = 'study'

        self.route('GET', (), self.find)
        self.route('GET', (':id',), self.getStudy)
        self.route('POST', (), self.createStudy)
        self.route('POST', (':id', 'users'), self.addAnnotators)
        self.route('POST', (':id', 'images'), self.addImages)

    @describeRoute(
        Description('Return a list of annotation studies.')
        .pagingParams(defaultSort='lowerName')
        .param('state', 'Filter studies to those at a given state',
               paramType='query', required=False, enum=('active', 'complete'))
        .param('userId',
               'Filter studies to those containing a user ID, or "me".',
               paramType='query', required=False)
        .errorResponse()
    )
    @access.public
    def find(self, params):
        Study = self.model('study', 'isic_archive')
        User = self.model('user', 'isic_archive')

        limit, offset, sort = self.getPagingParameters(params, 'lowerName')

        annotatorUser = None
        if params.get('userId'):
            if params['userId'] == 'me':
                annotatorUser = self.getCurrentUser()
            else:
                annotatorUser = User.load(
                    id=params['userId'],
                    force=True, exc=True)

        state = None
        if params.get('state'):
            state = params['state']
            if state not in {Study.State.ACTIVE, Study.State.COMPLETE}:
                raise RestException('Query parameter "state" may only '
                                    'be "active" or "complete".')

        return [
            {
                field: study[field]
                for field in
                Study.summaryFields
            }
            for study in
            Study.filterResultsByPermission(
                Study.find(query=None, annotatorUser=annotatorUser,
                           state=state, sort=sort),
                user=self.getCurrentUser(),
                level=AccessType.READ, limit=limit, offset=offset
            )
        ]

    @describeRoute(
        Description('Get a study by ID.')
        .param('id', 'The ID of the study.', paramType='path')
        .param('format', 'The output format.',
               paramType='query', required=False, enum=('csv', 'json'),
               default='json')
        .errorResponse()
    )
    @access.public
    @loadmodel(model='study', plugin='isic_archive', level=AccessType.READ)
    def getStudy(self, study, params):
        Study = self.model('study', 'isic_archive')
        Featureset = self.model('featureset', 'isic_archive')
        Image = self.model('image', 'isic_archive')
        User = self.model('user', 'isic_archive')

        if params.get('format') == 'csv':
            setResponseHeader('Content-Type', 'text/csv')
            setResponseHeader(
                'Content-Disposition',
                'attachment; filename="%s.csv"' % study['name'])
            return functools.partial(self._getStudyCSVStream, study)

        else:
            currentUser = self.getCurrentUser()
            output = Study.filter(study, currentUser)
            del output['_accessLevel']
            output.update({
                '_modelType': 'study',
                'featureset': Featureset.load(
                    id=study['meta']['featuresetId'],
                    fields=Featureset.summaryFields, exc=True),
                'creator': User.filteredSummary(
                    User.load(
                        output.pop('creatorId'),
                        force=True, exc=True),
                    currentUser),
                'users': sorted(
                    (
                        User.filteredSummary(annotatorUser, currentUser)
                        for annotatorUser in
                        Study.getAnnotators(study)
                    ),
                    key=lambda annotatorUser: User.obfuscatedName(annotatorUser)
                ),
                'images': list(
                    Study.getImages(
                        study, Image.summaryFields
                    ).sort('lowerName', SortDir.ASCENDING))
            })

            return output

    def _getStudyCSVStream(self, study):
        Study = self.model('study', 'isic_archive')
        Featureset = self.model('featureset', 'isic_archive')
        User = self.model('user', 'isic_archive')

        currentUser = self.getCurrentUser()

        featureset = Featureset.load(study['meta']['featuresetId'], exc=True)
        csvFields = tuple(itertools.chain(
            ('study_name', 'study_id',
             'user_name', 'user_id',
             'image_name', 'image_id',
             'flag_status', 'elapsed_seconds'),
            (feature['id'] for feature in featureset['globalFeatures']),
            ('superpixel_id',),
            (feature['id'] for feature in featureset['localFeatures'])
        ))

        responseBody = StringIO()
        csvWriter = csv.DictWriter(responseBody, csvFields, restval='')

        csvWriter.writeheader()
        yield responseBody.getvalue()
        responseBody.seek(0)
        responseBody.truncate()

        images = list(
            Study.getImages(study).sort('lowerName', SortDir.ASCENDING))

        for annotatorUser, image in itertools.product(
            sorted(
                Study.getAnnotators(study),
                key=lambda annotatorUser: User.obfuscatedName(annotatorUser)),
            images
        ):
            # this will iterate either 0 or 1 times
            for annotation in Study.childAnnotations(
                study=study,
                annotatorUser=annotatorUser,
                image=image,
                state=Study.State.COMPLETE
            ):
                elapsedSeconds = \
                    int((annotation['meta']['stopTime'] -
                         annotation['meta']['startTime']).total_seconds())

                filteredAnnotatorUser = User.filteredSummary(
                    annotatorUser, currentUser)
                annotatorUserName = filteredAnnotatorUser['name']
                if 'login' in filteredAnnotatorUser:
                    annotatorUserName += ' [%s %s (%s)]' % (
                        filteredAnnotatorUser['firstName'],
                        filteredAnnotatorUser['lastName'],
                        filteredAnnotatorUser['login'])

                outDictBase = {
                    'study_name': study['name'],
                    'study_id': str(study['_id']),
                    'user_name': annotatorUserName,
                    'user_id': str(annotatorUser['_id']),
                    'image_name': image['name'],
                    'image_id': str(image['_id']),
                    'flag_status': annotation['meta']['status'],
                    'elapsed_seconds': elapsedSeconds
                }

                outDict = outDictBase.copy()
                for globalFeature in featureset['globalFeatures']:
                    if globalFeature['id'] in annotation['meta']['annotations']:
                        outDict[globalFeature['id']] = \
                            annotation['meta']['annotations'][globalFeature['id']]  # noqa: E501
                csvWriter.writerow(outDict)
                yield responseBody.getvalue()
                responseBody.seek(0)
                responseBody.truncate()

                # TODO: move this into the query
                if 'localFeatures' in annotation['meta']['annotations']:
                    superpixelCount = len(next(six.viewvalues(
                        annotation['meta']['annotations']['localFeatures'])))
                    for superpixelMum in xrange(superpixelCount):

                        outDict = outDictBase.copy()
                        outDict['superpixel_id'] = superpixelMum
                        for featureName, featureValue in six.viewitems(
                                annotation['meta']['annotations']['localFeatures']):  # noqa: E501
                            outDict[featureName] = featureValue[superpixelMum]

                        csvWriter.writerow(outDict)
                        yield responseBody.getvalue()
                        responseBody.seek(0)
                        responseBody.truncate()

    @describeRoute(
        Description('Create an annotation study.')
        .param('name', 'The name of the study.', paramType='form')
        .param('featuresetId', 'The featureset ID of the study.',
               paramType='form')
        .param('userIds',
               'The annotators user IDs of the study, as a JSON array.',
               paramType='form')
        .param('imageIds',
               'The image IDs of the study, as a JSON array.',
               paramType='form')
        .errorResponse('Write access was denied on the parent folder.', 403)
    )
    @access.user
    def createStudy(self, params):
        Study = self.model('study', 'isic_archive')
        Featureset = self.model('featureset', 'isic_archive')
        Image = self.model('image', 'isic_archive')
        User = self.model('user', 'isic_archive')

        creatorUser = self.getCurrentUser()
        User.requireAdminStudy(creatorUser)

        params = self._decodeParams(params)
        self.requireParams(
            ['name', 'featuresetId', 'userIds', 'imageIds'],
            params)

        studyName = params['name'].strip()
        if not studyName:
            raise ValidationException('Name must not be empty.', 'name')

        featuresetId = params['featuresetId']
        if not featuresetId:
            raise ValidationException('Invalid featureset ID.', 'featuresetId')
        featureset = Featureset.load(featuresetId, exc=True)

        if len(set(params['userIds'])) != len(params['userIds']):
            raise RestException('Duplicate user IDs.')
        annotatorUsers = [
            User.load(
                annotatorUserId, user=creatorUser, level=AccessType.READ,
                exc=True)
            for annotatorUserId in params['userIds']
        ]

        if len(set(params['imageIds'])) != len(params['imageIds']):
            raise RestException('Duplicate image IDs.')
        images = [
            Image.load(
                imageId, user=creatorUser, level=AccessType.READ, exc=True)
            for imageId in params['imageIds']
        ]

        study = Study.createStudy(
            name=studyName,
            creatorUser=creatorUser,
            featureset=featureset,
            annotatorUsers=annotatorUsers,
            images=images)

        return self.getStudy(id=study['_id'], params={})

    @describeRoute(
        Description('Add annotator users to a study.')
        .param('id', 'The ID of the study.', paramType='path')
        .param('userIds', 'The user IDs to add, as a JSON array.',
               paramType='form')
        .errorResponse('ID was invalid.')
        .errorResponse('You don\'t have permission to add a study annotator.',
                       403)
    )
    @access.user
    @loadmodel(model='study', plugin='isic_archive', level=AccessType.READ)
    def addAnnotators(self, study, params):
        Folder = self.model('folder')
        Study = self.model('study', 'isic_archive')
        User = self.model('user', 'isic_archive')

        # TODO: make the loadmodel decorator use AccessType.WRITE,
        # once permissions work
        params = self._decodeParams(params)
        self.requireParams('userIds', params)

        creatorUser = self.getCurrentUser()
        User.requireAdminStudy(creatorUser)

        # Load all users before adding any, to ensure all are valid
        if len(set(params['userIds'])) != len(params['userIds']):
            raise RestException('Duplicate user IDs.')
        annotatorUsers = [
            User.load(userId, user=creatorUser, level=AccessType.READ, exc=True)
            for userId in params['userIds']
        ]
        # Existing duplicate Annotators are tricky to check for, because it's
        # possible to have a Study with multiple annotator Users (each with a
        # sub-Folder), but with no Images yet, and thus no Annotation (Items)
        # inside yet
        duplicateAnnotatorFolders = Folder.find({
            'parentId': study['_id'],
            'meta.userId': {'$in': [
                annotatorUser['_id'] for annotatorUser in annotatorUsers]}
        })
        if duplicateAnnotatorFolders.count():
            # Just list the first duplicate
            duplicateAnnotatorFolder = next(iter(duplicateAnnotatorFolders))
            raise ValidationException(
                'Annotator user "%s" is already part of the study.' %
                duplicateAnnotatorFolder['meta']['userId'])
        # Look up images only once for efficiency
        images = Study.getImages(study)
        for annotatorUser in annotatorUsers:
            Study.addAnnotator(study, annotatorUser, creatorUser, images)

        return self.getStudy(id=study['_id'], params={})

    @describeRoute(
        Description('Add images to a study.')
        .param('id', 'The ID of the study.', paramType='path')
        .param('imageIds', 'The image IDs to add, as a JSON array.',
               paramType='form')
        .errorResponse('ID was invalid.')
        .errorResponse('You don\'t have permission to add a study image.', 403)
    )
    @access.user
    @loadmodel(model='study', plugin='isic_archive', level=AccessType.READ)
    def addImages(self, study, params):
        Annotation = self.model('annotation', 'isic_archive')
        Image = self.model('image', 'isic_archive')
        Study = self.model('study', 'isic_archive')
        User = self.model('user', 'isic_archive')

        params = self._decodeParams(params)
        self.requireParams('imageIds', params)

        creatorUser = self.getCurrentUser()
        User.requireAdminStudy(creatorUser)

        # Load all images before adding any, to ensure all are valid and
        # accessible
        if len(set(params['imageIds'])) != len(params['imageIds']):
            raise RestException('Duplicate image IDs.')
        images = [
            Image.load(
                imageId, user=creatorUser, level=AccessType.READ, exc=True)
            for imageId in params['imageIds']
        ]
        duplicateAnnotations = Annotation.find({
            'meta.studyId': study['_id'],
            'meta.imageId': {'$in': [image['_id'] for image in images]}
        })
        if duplicateAnnotations.count():
            # Just list the first duplicate
            duplicateAnnotation = next(iter(duplicateAnnotations))
            raise ValidationException(
                'Image "%s" is already part of the study.' %
                duplicateAnnotation['meta']['imageId'])
        for image in images:
            Study.addImage(study, image, creatorUser)

        return self.getStudy(id=study['_id'], params={})
<|endoftext|>


<commit_before>class TerminalColorMapException(Exception):
  pass

def _rgb(color):
  return ((color >> 16) & 0xff, (color >> 8) & 0xff, color & 0xff)

def _diff(color1, color2):
  (r1, g1, b1) = _rgb(color1)
  (r2, g2, b2) = _rgb(color2)
  return abs(r1-r2) + abs(g1-g2) + abs(b1-b2)

class TerminalColorMap:
  def getColors(self, order='rgb'):
    return self.colors

  def convert(self, hexcolor):
    diffs = {}
    for xterm, rgb in self.colors.items():
      diffs[_diff(rgb, hexcolor)] = xterm
    minDiffAnsi = diffs[min(diffs.keys())]
    return (minDiffAnsi, self.colors[minDiffAnsi])

  def colorize(self, string, rgb=None, ansi=None,bg=None,ansi_bg=None):
    if rgb is None and ansi is None:
      raise TerminalColorMapException('colorize: must specify one named parameter: rgb or ansi')
    if rgb is not None and ansi is not None:
      raise TerminalColorMapException('colorize: must specify only one named parameter: rgb or ansi')
    if bg is not None and ansi_bg is not None:
      raise TerminalColorMapException('colorize: must specify only one named parameter: bg or ansi_bg')

    if rgb != None:
      (closestAnsi, closestRgb) = self.convert(rgb)
    elif ansi != None:
      (closestAnsi, closestRgb) = (ansi, self.colors[ansi])

    if bg == None and ansi_bg == None:
        return "\033[38;5;{ansiCode:d}m{string:s}\033[0m".format(ansiCode=closestAnsi, string=string)

    if bg != None:
        (closestBgAnsi,unused) = self.convert(bg)
    elif ansi_bg != None:
        (closestBgAnsi,unused) = (ansi_bg, self.colors[ansi_bg])

    return u"\033[38;5;{ansiCode:d}m\033[48;5;{bf:d}m{string:s}\033[0m".format(ansiCode=closestAnsi,bf=closestBgAnsi, string=string)

class VT100ColorMap(TerminalColorMap):
  primary = [
    0x000000, 0x800000, 0x008000, 0x808000, 0x000080, 0x800080, 0x008080, 0xc0c0c0
  ]

  bright = [
    0x808080, 0xff0000, 0x00ff00, 0xffff00, 0x0000ff, 0xff00ff, 0x00ffff, 0xffffff
  ]

  def __init__(self):
    self.colors = dict()
    self._compute()

  def _compute(self):
    for index, color in enumerate(self.primary + self.bright):
      self.colors[index] = color

class XTermColorMap(VT100ColorMap):
  grayscale_start = 0x08;
  grayscale_end = 0xf8;
  grayscale_step = 10;
  intensities = [
    0x00, 0x5F, 0x87, 0xAF, 0xD7, 0xFF
  ]

  def _compute(self):
    for index, color in enumerate(self.primary + self.bright):
      self.colors[index] = color

    c = 16
    for i in self.intensities:
      color = i << 16;
      for j in self.intensities:
        color &= ~(0xff << 8);
        color |= j << 8;
        for k in self.intensities:
          color &= ~0xff;
          color |= k;
          self.colors[c] = color
          c += 1

    c = 232
    for hex in list(range(self.grayscale_start, self.grayscale_end, self.grayscale_step)):
      color = (hex << 16) | (hex << 8) | hex;
      self.colors[c] = color
      c += 1
<commit_msg>Python3 fix<commit_after>class TerminalColorMapException(Exception):
  pass

def _rgb(color):
  return ((color >> 16) & 0xff, (color >> 8) & 0xff, color & 0xff)

def _diff(color1, color2):
  (r1, g1, b1) = _rgb(color1)
  (r2, g2, b2) = _rgb(color2)
  return abs(r1-r2) + abs(g1-g2) + abs(b1-b2)

class TerminalColorMap:
  def getColors(self, order='rgb'):
    return self.colors

  def convert(self, hexcolor):
    diffs = {}
    for xterm, rgb in self.colors.items():
      diffs[_diff(rgb, hexcolor)] = xterm
    minDiffAnsi = diffs[min(diffs.keys())]
    return (minDiffAnsi, self.colors[minDiffAnsi])

  def colorize(self, string, rgb=None, ansi=None,bg=None,ansi_bg=None):
    if rgb is None and ansi is None:
      raise TerminalColorMapException('colorize: must specify one named parameter: rgb or ansi')
    if rgb is not None and ansi is not None:
      raise TerminalColorMapException('colorize: must specify only one named parameter: rgb or ansi')
    if bg is not None and ansi_bg is not None:
      raise TerminalColorMapException('colorize: must specify only one named parameter: bg or ansi_bg')

    if rgb != None:
      (closestAnsi, closestRgb) = self.convert(rgb)
    elif ansi != None:
      (closestAnsi, closestRgb) = (ansi, self.colors[ansi])

    if bg == None and ansi_bg == None:
        return "\033[38;5;{ansiCode:d}m{string:s}\033[0m".format(ansiCode=closestAnsi, string=string)

    if bg != None:
        (closestBgAnsi,unused) = self.convert(bg)
    elif ansi_bg != None:
        (closestBgAnsi,unused) = (ansi_bg, self.colors[ansi_bg])

    return "\033[38;5;{ansiCode:d}m\033[48;5;{bf:d}m{string:s}\033[0m".format(ansiCode=closestAnsi,bf=closestBgAnsi, string=string)

class VT100ColorMap(TerminalColorMap):
  primary = [
    0x000000, 0x800000, 0x008000, 0x808000, 0x000080, 0x800080, 0x008080, 0xc0c0c0
  ]

  bright = [
    0x808080, 0xff0000, 0x00ff00, 0xffff00, 0x0000ff, 0xff00ff, 0x00ffff, 0xffffff
  ]

  def __init__(self):
    self.colors = dict()
    self._compute()

  def _compute(self):
    for index, color in enumerate(self.primary + self.bright):
      self.colors[index] = color

class XTermColorMap(VT100ColorMap):
  grayscale_start = 0x08;
  grayscale_end = 0xf8;
  grayscale_step = 10;
  intensities = [
    0x00, 0x5F, 0x87, 0xAF, 0xD7, 0xFF
  ]

  def _compute(self):
    for index, color in enumerate(self.primary + self.bright):
      self.colors[index] = color

    c = 16
    for i in self.intensities:
      color = i << 16;
      for j in self.intensities:
        color &= ~(0xff << 8);
        color |= j << 8;
        for k in self.intensities:
          color &= ~0xff;
          color |= k;
          self.colors[c] = color
          c += 1

    c = 232
    for hex in list(range(self.grayscale_start, self.grayscale_end, self.grayscale_step)):
      color = (hex << 16) | (hex << 8) | hex;
      self.colors[c] = color
      c += 1
<|endoftext|>


<commit_before># No shebang line, this module is meant to be imported
#
# Copyright 2015 Oliver Palmer
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Configuration
============

A small wrapper around :class:`pyfarm.core.config.Configuration`
that loads in the configuration files and provides backwards
compatibility for some environment variables.
"""

import os
from functools import partial

from pyfarm.core.config import (
    Configuration, read_env_int, read_env, read_env_bool)

def load_environment():
    """
    Provides a mapping of environment variable values to their
    configuration counterparts.
    """
    read_env_no_log = partial(read_env, log_result=False)
    output = {}
    environment = {
        "secret_key": ("PYFARM_SECRET_KEY", read_env_no_log),
        "autocreate_users": ("PYFARM_AUTOCREATE_USERS", read_env_bool),
        "autocreate_user_domain": (
            "PYFARM_AUTO_USERS_DEFAULT_DOMAIN", read_env_bool),
        "default_job_delete_time": (
            "PYFARM_DEFAULT_JOB_DELETE_TIME", read_env_int),
        "login_disabled": ("PYFARM_LOGIN_DISABLED", read_env_bool),
        "pretty_json": ("PYFARM_JSON_PRETTY", read_env_bool),
        "echo_sql": ("PYFARM_SQL_ECHO", read_env_bool),
        "database": ("PYFARM_DATABASE_URI", read_env_no_log),
        "timestamp_format": ("PYFARM_TIMESTAMP_FORMAT", read_env),
        "allow_agents_from_loopback": (
            "PYFARM_DEV_ALLOW_AGENT_LOOPBACK_ADDRESSES", read_env_bool)
    }

    for config_key, (env_key, get_env) in environment.items():
        if env_key in os.environ:
            output[config_key] = get_env()

    return environment

try:
    config
except NameError:  # pragma: no cover
    config = Configuration("pyfarm.master")
    config.load(environment=load_environment())
<commit_msg>need to actually lookup the key....<commit_after># No shebang line, this module is meant to be imported
#
# Copyright 2015 Oliver Palmer
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Configuration
============

A small wrapper around :class:`pyfarm.core.config.Configuration`
that loads in the configuration files and provides backwards
compatibility for some environment variables.
"""

import os
from functools import partial

from pyfarm.core.config import (
    Configuration, read_env_int, read_env, read_env_bool)

def load_environment():
    """
    Provides a mapping of environment variable values to their
    configuration counterparts.
    """
    read_env_no_log = partial(read_env, log_result=False)
    output = {}
    environment = {
        "secret_key": ("PYFARM_SECRET_KEY", read_env_no_log),
        "autocreate_users": ("PYFARM_AUTOCREATE_USERS", read_env_bool),
        "autocreate_user_domain": (
            "PYFARM_AUTO_USERS_DEFAULT_DOMAIN", read_env_bool),
        "default_job_delete_time": (
            "PYFARM_DEFAULT_JOB_DELETE_TIME", read_env_int),
        "login_disabled": ("PYFARM_LOGIN_DISABLED", read_env_bool),
        "pretty_json": ("PYFARM_JSON_PRETTY", read_env_bool),
        "echo_sql": ("PYFARM_SQL_ECHO", read_env_bool),
        "database": ("PYFARM_DATABASE_URI", read_env_no_log),
        "timestamp_format": ("PYFARM_TIMESTAMP_FORMAT", read_env),
        "allow_agents_from_loopback": (
            "PYFARM_DEV_ALLOW_AGENT_LOOPBACK_ADDRESSES", read_env_bool)
    }

    for config_key, (env_key, get_env) in environment.items():
        if env_key in os.environ:
            output[config_key] = get_env(env_key)

    return environment

try:
    config
except NameError:  # pragma: no cover
    config = Configuration("pyfarm.master")
    config.load(environment=load_environment())
<|endoftext|>


<commit_before>class TerminalColorMapException(Exception):
  pass

class TerminalColorMap:
  def getColors(self, order='rgb'):
    return self.colors

  def rgb(self, color):
    return ((color >> 16) & 0xff, (color >> 8) & 0xff, color & 0xff)

  def diff(self, color1, color2):
    (r1, g1, b1) = self.rgb(color1)
    (r2, g2, b2) = self.rgb(color2)
    return abs(r1-r2) + abs(g1-g2) + abs(b1-b2)

  def convert(self, hexcolor):
    diffs = {}
    for xterm, rgb in self.colors.items():
      diffs[self.diff(rgb, hexcolor)] = xterm
    minDiffAnsi = diffs[min(diffs.keys())]
    return (minDiffAnsi, self.colors[minDiffAnsi])

  def colorize(self, string, rgb=None, ansi=None,bg=None,ansi_bg=None):
    if rgb is None and ansi is None:
      raise TerminalColorMapException('colorize: must specify one named parameter: rgb or ansi')
    if rgb is not None and ansi is not None:
      raise TerminalColorMapException('colorize: must specify only one named parameter: rgb or ansi')
    if bg is not None and ansi_bg is not None:
      raise TerminalColorMapException('colorize: must specify only one named parameter: bg or ansi_bg')

    if rgb:
      (closestAnsi, closestRgb) = self.convert(rgb)
    elif ansi:
      (closestAnsi, closestRgb) = (ansi, self.colors[ansi])

    if bg == None and ansi_bg == None:
        return "\033[38;5;{ansiCode:d}m{string:s}\033[0m".format(ansiCode=closestAnsi, string=string)

    if bg:
        (closestBgAnsi,unused) = self.convert(bg)
    elif ansi_bg:
        (closestBgAnsi,unused) = (ansi_bg, self.colors[ansi_bg])

    return u"\033[38;5;{ansiCode:d}m\033[48;5;{bf:d}m{string:s}\033[0m".format(ansiCode=closestAnsi,bf=closestBgAnsi, string=string)

class VT100ColorMap(TerminalColorMap):
  primary = [
    0x000000, 0x800000, 0x008000, 0x808000, 0x000080, 0x800080, 0x008080, 0xc0c0c0
  ]

  bright = [
    0x808080, 0xff0000, 0x00ff00, 0xffff00, 0x0000ff, 0xff00ff, 0x00ffff, 0xffffff
  ]

  def __init__(self):
    self.colors = dict()
    self._compute()

  def _compute(self):
    for index, color in enumerate(self.primary + self.bright):
      self.colors[index] = color

class XTermColorMap(VT100ColorMap):
  grayscale_start = 0x08;
  grayscale_end = 0xf8;
  grayscale_step = 10;
  intensities = [
    0x00, 0x5F, 0x87, 0xAF, 0xD7, 0xFF
  ]

  def _compute(self):
    for index, color in enumerate(self.primary + self.bright):
      self.colors[index] = color

    c = 16
    for i in self.intensities:
      color = i << 16;
      for j in self.intensities:
        color &= ~(0xff << 8);
        color |= j << 8;
        for k in self.intensities:
          color &= ~0xff;
          color |= k;
          self.colors[c] = color
          c += 1

    c = 232
    for hex in list(range(self.grayscale_start, self.grayscale_end, self.grayscale_step)):
      color = (hex << 16) | (hex << 8) | hex;
      self.colors[c] = color
      c += 1
<commit_msg>De-nested 2 practically static functions<commit_after>class TerminalColorMapException(Exception):
  pass

def rgb(color):
  return ((color >> 16) & 0xff, (color >> 8) & 0xff, color & 0xff)

def diff(color1, color2):
  (r1, g1, b1) = rgb(color1)
  (r2, g2, b2) = rgb(color2)
  return abs(r1-r2) + abs(g1-g2) + abs(b1-b2)

class TerminalColorMap:
  def getColors(self, order='rgb'):
    return self.colors

  def convert(self, hexcolor):
    diffs = {}
    for xterm, rgb in self.colors.items():
      diffs[diff(rgb, hexcolor)] = xterm
    minDiffAnsi = diffs[min(diffs.keys())]
    return (minDiffAnsi, self.colors[minDiffAnsi])

  def colorize(self, string, rgb=None, ansi=None,bg=None,ansi_bg=None):
    if rgb is None and ansi is None:
      raise TerminalColorMapException('colorize: must specify one named parameter: rgb or ansi')
    if rgb is not None and ansi is not None:
      raise TerminalColorMapException('colorize: must specify only one named parameter: rgb or ansi')
    if bg is not None and ansi_bg is not None:
      raise TerminalColorMapException('colorize: must specify only one named parameter: bg or ansi_bg')

    if rgb:
      (closestAnsi, closestRgb) = self.convert(rgb)
    elif ansi:
      (closestAnsi, closestRgb) = (ansi, self.colors[ansi])

    if bg == None and ansi_bg == None:
        return "\033[38;5;{ansiCode:d}m{string:s}\033[0m".format(ansiCode=closestAnsi, string=string)

    if bg:
        (closestBgAnsi,unused) = self.convert(bg)
    elif ansi_bg:
        (closestBgAnsi,unused) = (ansi_bg, self.colors[ansi_bg])

    return u"\033[38;5;{ansiCode:d}m\033[48;5;{bf:d}m{string:s}\033[0m".format(ansiCode=closestAnsi,bf=closestBgAnsi, string=string)

class VT100ColorMap(TerminalColorMap):
  primary = [
    0x000000, 0x800000, 0x008000, 0x808000, 0x000080, 0x800080, 0x008080, 0xc0c0c0
  ]

  bright = [
    0x808080, 0xff0000, 0x00ff00, 0xffff00, 0x0000ff, 0xff00ff, 0x00ffff, 0xffffff
  ]

  def __init__(self):
    self.colors = dict()
    self._compute()

  def _compute(self):
    for index, color in enumerate(self.primary + self.bright):
      self.colors[index] = color

class XTermColorMap(VT100ColorMap):
  grayscale_start = 0x08;
  grayscale_end = 0xf8;
  grayscale_step = 10;
  intensities = [
    0x00, 0x5F, 0x87, 0xAF, 0xD7, 0xFF
  ]

  def _compute(self):
    for index, color in enumerate(self.primary + self.bright):
      self.colors[index] = color

    c = 16
    for i in self.intensities:
      color = i << 16;
      for j in self.intensities:
        color &= ~(0xff << 8);
        color |= j << 8;
        for k in self.intensities:
          color &= ~0xff;
          color |= k;
          self.colors[c] = color
          c += 1

    c = 232
    for hex in list(range(self.grayscale_start, self.grayscale_end, self.grayscale_step)):
      color = (hex << 16) | (hex << 8) | hex;
      self.colors[c] = color
      c += 1
<|endoftext|>


<commit_before>#!/usr/bin/env python

from __future__ import division
from __future__ import print_function
import cv2
import pygecko.lib.ZmqClass as zmq
import pygecko.lib.Messages as Msg
import multiprocessing as mp
from time import sleep

class BallTracker(object):
	# hsv colors to threshold on
	greenLower = (29, 86, 6)
	greenUpper = (64, 255, 255)
	diameter = 6.7  # diameter of average tennis ball in cm

	def __init__(self, lower=(29, 86, 6), upper=(64, 255, 255), diameter=6.7):
		"""
		Tracks a ball in an image.
		"""
		self.greenLower = lower
		self.greenUpper = upper
		self.diameter = diameter

	def distance(self, radius):
		return self.diameter

	def find(self, frame):
		hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)

		# threshold and find the tennis ball
		mask = cv2.inRange(hsv, self.greenLower, self.greenUpper)

		# do some morphological operators to fill in mask gaps and remove
		# outliers (false positives)
		mask = cv2.erode(mask, None, iterations=2)
		mask = cv2.dilate(mask, None, iterations=2)

		# find contours in the mask
		cnts = cv2.findContours(mask.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)[-2]
		center = None
		radius = 0

		# did we find something?
		if len(cnts) > 0:
			# find the largest contour in the mask, then use
			# it to compute the minimum enclosing circle and
			# centroid
			c = max(cnts, key=cv2.contourArea)
			((x, y), radius) = cv2.minEnclosingCircle(c)

			# only proceed if the radius meets a minimum size
			if radius > 10:
				# find moments -----------
				# M = cv2.moments(c)
				# center = (int(M["m10"] / M["m00"]), int(M["m01"] / M["m00"]))
				# cv2.circle(frame, center, 5, (0, 0, 255), -1)

				# set center and radius of ball
				center = (int(x), int(y))
				radius = int(radius)

		return center, radius

class Command_BT(mp.Process):
	pubport = None
	subport = None

	def __init__(self):
		"""
		"""
		mp.Process.__init__(self)

	def init(self, pport, sport):
		self.pubport = pport
		self.subport = sport

	def run(self):
		bt = BallTracker()

		sub = zmq.Sub(topics='image_color', connect_to=('0.0.0.0', self.subport))
		pub = zmq.Pub(bind_to=('0.0.0.0', self.pubport))

		print('Started {} on ports: pub {} sub {}'.format('Command_BT', self.pubport, self.subport))

		while True:
			_, msg = sub.recvB64()
			if msg:
				im = msg['image']
				width, height = im.shape[:2]
				center, radius = bt.find(im)
				if center and radius > 10:
					x, y = center
					xx = x-width/2
					yy = y-height/2
					# print('adjust:', xx, yy)

					t = Msg.Twist()
					pub.pub('command', t)
					# print im.shape
			sleep(0.01)

if __name__ == '__main__':
	bt = Command_BT()
	bt.init(9000, 9001)
	bt.start()
<commit_msg>update<commit_after>#!/usr/bin/env python

from __future__ import division
from __future__ import print_function
import cv2
import pygecko.lib.ZmqClass as zmq
import pygecko.lib.Messages as Msg
import multiprocessing as mp
from time import sleep

class BallTracker(object):
	# hsv colors to threshold on
	greenLower = (29, 86, 6)
	greenUpper = (64, 255, 255)
	diameter = 6.7  # diameter of average tennis ball in cm

	def __init__(self, lower=(29, 86, 6), upper=(64, 255, 255), diameter=6.7):
		"""
		Tracks a ball in an image.
		"""
		self.greenLower = lower
		self.greenUpper = upper
		self.diameter = diameter

	def distance(self, radius):
		return self.diameter

	def find(self, frame):
		hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)

		# threshold and find the tennis ball
		mask = cv2.inRange(hsv, self.greenLower, self.greenUpper)

		# do some morphological operators to fill in mask gaps and remove
		# outliers (false positives)
		mask = cv2.erode(mask, None, iterations=2)
		mask = cv2.dilate(mask, None, iterations=2)

		# find contours in the mask
		cnts = cv2.findContours(mask.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)[-2]
		center = None
		radius = 0

		# did we find something?
		if len(cnts) > 0:
			# find the largest contour in the mask, then use
			# it to compute the minimum enclosing circle and
			# centroid
			c = max(cnts, key=cv2.contourArea)
			((x, y), radius) = cv2.minEnclosingCircle(c)

			# only proceed if the radius meets a minimum size
			if radius > 10:
				# find moments -----------
				# M = cv2.moments(c)
				# center = (int(M["m10"] / M["m00"]), int(M["m01"] / M["m00"]))
				# cv2.circle(frame, center, 5, (0, 0, 255), -1)

				# set center and radius of ball
				center = (int(x), int(y))
				radius = int(radius)

		return center, radius

class Command_BT(mp.Process):
	pubport = None
	subport = None

	def __init__(self):
		"""
		"""
		mp.Process.__init__(self)

	def init(self, pport, sport):
		self.pubport = pport
		self.subport = sport

	def run(self):
		bt = BallTracker()

		sub = zmq.Sub(topics='image_color', connect_to=('0.0.0.0', self.subport))
		pub = zmq.Pub(bind_to=('0.0.0.0', self.pubport))

		print('Started {} on ports: pub {} sub {}'.format('Command_BT', self.pubport, self.subport))

		while True:
			_, msg = sub.recv()
			if msg:
				im = msg['image']
				width, height = im.shape[:2]
				center, radius = bt.find(im)
				if center and radius > 10:
					x, y = center
					xx = x-width/2
					yy = y-height/2
					print('adjust:', xx, yy)

					# t = Msg.Twist()
					# pub.pub('command', t)
					# print im.shape

			sleep(0.01)

if __name__ == '__main__':
	bt = Command_BT()
	bt.init(9000, 9001)
	bt.start()
<|endoftext|>


<commit_before># Prod env
TEST_CONFIG = 'prod'

DEBUG = False

# home image_poem fetch time gap
HOME_IMAGE_POEM_FETCH_TIME_GAP = 10*60
<commit_msg>image poem change by 1 min<commit_after># Prod env
TEST_CONFIG = 'prod'

DEBUG = False

# home image_poem fetch time gap
HOME_IMAGE_POEM_FETCH_TIME_GAP = 1*60
<|endoftext|>


<commit_before>#! /usr/bin/env python

from osgeo import gdal, osr
import numpy as np

from . import malib

#Functions for IO, mostly wrapped around GDAL
#Written before RasterIO existed, which should probably be used instead of these

gdal_opt = ['COMPRESS=LZW', 'TILED=YES', 'BIGTIFF=IF_SAFER']
#gdal_opt += ['BLOCKXSIZE=1024', 'BLOCKYSIZE=1024']

#Add methods to load ma from OpenCV, PIL, etc.
#These formats should be directly readable as np arrays

#Note: want to modify to import all bands as separate arrays in ndarray
#Unless the user requests a single band, or range of bands

#Wrapper around gdal.Open
def fn_getds(fn):
    ds = None
    if fn_check(fn):
        ds = gdal.Open(fn, gdal.GA_ReadOnly)
    else:
        print("Unable to find %s" % fn)
    return ds

#Given input filename, return a masked array for specified band
def fn_getma(fn, bnum=1):
    #Add check for filename existence
    ds = fn_getds(fn)
    return ds_getma(ds, bnum=bnum)

#Given input dataset, return a masked array for the input band
def ds_getma(ds, bnum=1):
    b = ds.GetRasterBand(bnum)
    return b_getma(b)

#Given input band, return a masked array
def b_getma(b):
    b_ndv = get_ndv_b(b)
    bma = np.ma.masked_equal(b.ReadAsArray(), b_ndv)
    bma = malib.checkma(bma)
    return bma

def get_sub_dim(src_ds, scale=None, maxdim=1024.):
    ns = src_ds.RasterXSize
    nl = src_ds.RasterYSize
    if scale is None:
        scale_ns = ns/maxdim
        scale_nl = nl/maxdim
        scale = max(scale_ns, scale_nl)
    #Need to check to make sure scale is positive real
    if scale > 1:
        ns = int(round(ns/scale))
        nl = int(round(nl/scale))
    return ns, nl

#Load a subsampled array
#Can specify scale factor or max dimension
#No need to load the entire dataset for stats computation
def gdal_getma_sub(src_ds, bnum=1, scale=None, maxdim=1024.):
    #print src_ds.GetFileList()[0]
    b = src_ds.GetRasterBand(bnum)
    b_ndv = get_ndv_b(b)
    ns, nl = get_sub_dim(src_ds, scale, maxdim)
    #The buf_size parameters determine the final array dimensions
    b_array = b.ReadAsArray(buf_xsize=ns, buf_ysize=nl)
    bma = np.ma.masked_equal(b_array, b_ndv)
    bma = malib.checkma(bma)
    return bma

#Note: need to consolidate with warplib.writeout (takes ds, not ma)
#Add option to build overviews when writing GTiff
#Input proj must be WKT
def writeGTiff(a, dst_fn, src_ds=None, bnum=1, ndv=None, gt=None, proj=None, create=False, sparse=False):
    #If input is not np.ma, this creates a new ma, which has default filL_value of 1E20
    #Must manually override with ndv
    a = malib.checkma(a, fix=False)
    #Want to preserve fill_value if already specified
    if ndv is not None:
        a.set_fill_value(ndv)
    format = "GTiff"
    driver = gdal.GetDriverByName(format)
    #Currently only support writing singleband rasters
    #if a.ndim > 2:
    #   np_nbands = a.shape[2]
    #   if src_ds.RasterCount np_nbands:
    #      for bnum in np_nbands:
    nbands = 1
    np_dt = a.dtype.name
    if src_ds is not None:
        src_dt = gdal.GetDataTypeName(src_ds.GetRasterBand(bnum).DataType)
        src_gt = src_ds.GetGeoTransform()
        #This is WKT
        src_proj = src_ds.GetProjection()
        src_srs = osr.SpatialReference()
        src_srs.ImportFromWkt(src_ds.GetProjectionRef())

    #Probably a cleaner way to handle this
    if gt is None:
        gt = src_gt
    if proj is None:
        proj = src_proj

    #Need to create a new copy of the default options
    opt = list(gdal_opt)

    #Note: packbits is better for sparse data
    if sparse:
        opt.remove('COMPRESS=LZW')
        opt.append('COMPRESS=PACKBITS')
        #Not sure if VW can handle sparse tif
        #opt.append('SPARSE_OK=TRUE')

    #Use predictor=3 for floating point data
    if 'float' in np_dt.lower() and 'COMPRESS=LZW' in opt:
        opt.append('PREDICTOR=3')

    #If input ma is same as src_ds, write out array using CreateCopy from existing dataset
    #if not create and (src_ds is not None) and ((a.shape[0] == src_ds.RasterYSize) and (a.shape[1] == src_ds.RasterXSize) and (np_dt.lower() == src_dt.lower())):
    #Should compare srs.IsSame(src_srs)
    if not create and (src_ds is not None) and ((a.shape[0] == src_ds.RasterYSize) and (a.shape[1] == src_ds.RasterXSize) and (np_dt.lower() == src_dt.lower())) and (src_gt == gt) and (src_proj == proj):
        #Note: third option is strict flag, set to false
        dst_ds = driver.CreateCopy(dst_fn, src_ds, 0, options=opt)
    #Otherwise, use Create
    else:
        a_dtype = a.dtype
        gdal_dtype = np_gdal_dtype(a_dtype)
        if a_dtype.name == 'bool':
            #Set ndv to 0
            a.fill_value = False
            opt.remove('COMPRESS=LZW')
            opt.append('COMPRESS=DEFLATE')
            #opt.append('NBITS=1')
        #Create(fn, nx, ny, nbands, dtype, opt)
        dst_ds = driver.Create(dst_fn, a.shape[1], a.shape[0], nbands, gdal_dtype, options=opt)
        #Note: Need GeoMA here to make this work, or accept gt as argument
        #Could also do ds creation in calling script
        if gt is not None:
            dst_ds.SetGeoTransform(gt)
        if proj is not None:
            dst_ds.SetProjection(proj)

    dst_ds.GetRasterBand(bnum).WriteArray(a.filled())
    dst_ds.GetRasterBand(bnum).SetNoDataValue(float(a.fill_value))
    dst_ds = None

#Move to geolib?
#Look up equivalent GDAL data type
def np_gdal_dtype(d):
    from osgeo import gdal_array
    dt_dict = gdal_array.codes
    if not isinstance(d, np.dtype):
        d = np.dtype(d)
    if d.name == 'int8':
        gdal_dt = 1
    elif d.name == 'bool':
        #Write out as Byte
        gdal_dt = 1
    else:
        gdal_dt = list(dt_dict.keys())[list(dt_dict.values()).index(d)]
    return gdal_dt

#Replace nodata value in GDAL band
def replace_ndv(b, new_ndv):
    b_ndv = get_ndv_b(b)
    bma = np.ma.masked_equal(b.ReadAsArray(), b_ndv)
    bma.set_fill_value(new_ndv)
    b.WriteArray(bma.filled())
    b.SetNoDataValue(new_ndv)
    return b

def set_ndv(dst_fn, ndv):
    dst_ds = gdal.Open(dst_fn, gdal.GA_Update)
    for n in range(1, dst_ds.RasterCount+1):
        b = dst_ds.GetRasterBand(1)
        b.SetNoDataValue(ndv)
    dst_ds = None

#Should overload these functions to handle fn, ds, or b
#Perhaps abstract, as many functions will need this functionality
def get_ndv_fn(fn):
    ds = gdal.Open(fn, gdal.GA_ReadOnly)
    return get_ndv_ds(ds)

#Want to modify to handle multi-band images and return list of ndv
def get_ndv_ds(ds, bnum=1):
    b = ds.GetRasterBand(bnum)
    return get_ndv_b(b)

#Return nodata value for GDAL band
def get_ndv_b(b):
    b_ndv = b.GetNoDataValue()
    if b_ndv is None:
        #Check ul pixel for ndv
        ns = b.XSize
        nl = b.YSize
        ul = float(b.ReadAsArray(0, 0, 1, 1))
        ur = float(b.ReadAsArray(ns-1, 0, 1, 1))
        lr = float(b.ReadAsArray(ns-1, nl-1, 1, 1))
        ll = float(b.ReadAsArray(0, nl-1, 1, 1))
        #Probably better to use 3/4 corner criterion
        #if ul == ur == lr == ll:
        if np.isnan(ul) or ul == lr:
            b_ndv = ul
        else:
            #Assume ndv is 0
            b_ndv = 0
    elif np.isnan(b_ndv):
        b_dt = gdal.GetDataTypeName(b.DataType)
        if 'Float' in b_dt:
            b_ndv = np.nan
        else:
            b_ndv = 0
    return b_ndv

#Check for file existence
def fn_check(fn):
    import os
    return os.path.exists(fn)

#This method avoids race condition, but is slower than simple os.path.exists
def fn_check_full(fn):
    import os
    status = True
    if not os.path.isfile(fn):
        status = False
    else:
        try:
            open(fn)
        except IOError:
            status = False
    return status

def fn_list_check(fn_list):
    status = True
    for fn in fn_list:
        if not fn_check(fn):
            print('Unable to find: %s' % fn)
            status = False
    return status

#Write out a recarray as a csv
def write_recarray(outfn, ra):
    with open(outfn,'w') as f:
        f.write(','.join([str(item) for item in ra.dtype.names])+'\n')
        for row in ra:
            f.write(','.join([str(item) for item in row])+'\n')

#Check to make sure image doesn't contain errors
def image_check(fn):
    import sys
    from osgeo import gdal
    ds = gdal.Open(fn)
    status = True
    for i in range(ds.RasterCount):
        ds.GetRasterBand(i).Checksum()
        if gdal.GetLastErrorType() != 0:
            status = False
    return status

def cpu_count():
    from multiprocessing import cpu_count
    return cpu_count()
<commit_msg>iolib: just a couple changes<commit_after>#! /usr/bin/env python

from osgeo import gdal, osr
import numpy as np

from . import malib

#Functions for IO, mostly wrapped around GDAL
#Written before RasterIO existed, which should probably be used instead of these

gdal_opt = ['COMPRESS=LZW', 'TILED=YES', 'BIGTIFF=IF_SAFER']
#gdal_opt += ['BLOCKXSIZE=1024', 'BLOCKYSIZE=1024']

#Add methods to load ma from OpenCV, PIL, etc.
#These formats should be directly readable as np arrays

#Note: want to modify to import all bands as separate arrays in ndarray
#Unless the user requests a single band, or range of bands

#Wrapper around gdal.Open
def fn_getds(fn):
    ds = None
    if fn_check(fn):
        ds = gdal.Open(fn, gdal.GA_ReadOnly)
    else:
        print("Unable to find %s" % fn)
    return ds

#Given input filename, return a masked array for specified band
def fn_getma(fn, bnum=1):
    #Add check for filename existence
    ds = fn_getds(fn)
    return ds_getma(ds, bnum=bnum)

#Given input dataset, return a masked array for the input band
def ds_getma(ds, bnum=1):
    b = ds.GetRasterBand(bnum)
    return b_getma(b)

#Given input band, return a masked array
def b_getma(b):
    b_ndv = get_ndv_b(b)
    bma = np.ma.masked_equal(b.ReadAsArray(), b_ndv)
    bma = malib.checkma(bma)
    return bma

def get_sub_dim(src_ds, scale=None, maxdim=1024.):
    ns = src_ds.RasterXSize
    nl = src_ds.RasterYSize
    if scale is None:
        scale_ns = ns/maxdim
        scale_nl = nl/maxdim
        scale = max(scale_ns, scale_nl)
    #Need to check to make sure scale is positive real
    if scale > 1:
        ns = int(round(ns/scale))
        nl = int(round(nl/scale))
    return ns, nl

#Load a subsampled array
#Can specify scale factor or max dimension
#No need to load the entire dataset for stats computation
def gdal_getma_sub(src_ds, bnum=1, scale=None, maxdim=1024.):
    #print src_ds.GetFileList()[0]
    b = src_ds.GetRasterBand(bnum)
    b_ndv = get_ndv_b(b)
    ns, nl = get_sub_dim(src_ds, scale, maxdim)
    #The buf_size parameters determine the final array dimensions
    b_array = b.ReadAsArray(buf_xsize=ns, buf_ysize=nl)
    bma = np.ma.masked_equal(b_array, b_ndv)
    bma = malib.checkma(bma)
    return bma

#Note: need to consolidate with warplib.writeout (takes ds, not ma)
#Add option to build overviews when writing GTiff
#Input proj must be WKT
def writeGTiff(a, dst_fn, src_ds=None, bnum=1, ndv=None, gt=None, proj=None, create=False, sparse=False):
    #If input is not np.ma, this creates a new ma, which has default filL_value of 1E20
    #Must manually override with ndv
    a = malib.checkma(a, fix=False)
    #Want to preserve fill_value if already specified
    if ndv is not None:
        a.set_fill_value(ndv)
    format = "GTiff"
    driver = gdal.GetDriverByName(format)
    #Currently only support writing singleband rasters
    #if a.ndim > 2:
    #   np_nbands = a.shape[2]
    #   if src_ds.RasterCount np_nbands:
    #      for bnum in np_nbands:
    nbands = 1
    np_dt = a.dtype.name
    if src_ds is not None:
        src_dt = gdal.GetDataTypeName(src_ds.GetRasterBand(bnum).DataType)
        src_gt = src_ds.GetGeoTransform()
        #This is WKT
        src_proj = src_ds.GetProjection()
        src_srs = osr.SpatialReference()
        src_srs.ImportFromWkt(src_ds.GetProjectionRef())

    #Probably a cleaner way to handle this
    if gt is None:
        gt = src_gt
    if proj is None:
        proj = src_proj

    #Need to create a new copy of the default options
    opt = list(gdal_opt)

    #Note: packbits is better for sparse data
    if sparse:
        opt.remove('COMPRESS=LZW')
        opt.append('COMPRESS=PACKBITS')
        #Not sure if VW can handle sparse tif
        #opt.append('SPARSE_OK=TRUE')

    #Use predictor=3 for floating point data
    if 'float' in np_dt.lower() and 'COMPRESS=LZW' in opt:
        opt.append('PREDICTOR=3')

    #If input ma is same as src_ds, write out array using CreateCopy from existing dataset
    #if not create and (src_ds is not None) and ((a.shape[0] == src_ds.RasterYSize) and (a.shape[1] == src_ds.RasterXSize) and (np_dt.lower() == src_dt.lower())):
    #Should compare srs.IsSame(src_srs)
    if not create and (src_ds is not None) and ((a.shape[0] == src_ds.RasterYSize) and (a.shape[1] == src_ds.RasterXSize) and (np_dt.lower() == src_dt.lower())) and (src_gt == gt) and (src_proj == proj):
        #Note: third option is strict flag, set to false
        dst_ds = driver.CreateCopy(dst_fn, src_ds, 0, options=opt)
    #Otherwise, use Create
    else:
        a_dtype = a.dtype
        gdal_dtype = np_gdal_dtype(a_dtype)
        if a_dtype.name == 'bool':
            #Set ndv to 0
            a.fill_value = False
            opt.remove('COMPRESS=LZW')
            opt.append('COMPRESS=DEFLATE')
            #opt.append('NBITS=1')
        #Create(fn, nx, ny, nbands, dtype, opt)
        dst_ds = driver.Create(dst_fn, a.shape[1], a.shape[0], nbands, gdal_dtype, options=opt)
        #Note: Need GeoMA here to make this work, or accept gt as argument
        #Could also do ds creation in calling script
        if gt is not None:
            dst_ds.SetGeoTransform(gt)
        if proj is not None:
            dst_ds.SetProjection(proj)

    dst_ds.GetRasterBand(bnum).WriteArray(a.filled())
    dst_ds.GetRasterBand(bnum).SetNoDataValue(float(a.fill_value))
    dst_ds = None

#Move to geolib?
#Look up equivalent GDAL data type
def np_gdal_dtype(d):
    from osgeo import gdal_array
    dt_dict = gdal_array.codes
    if not isinstance(d, np.dtype):
        d = np.dtype(d)
    if d.name == 'int8':
        gdal_dt = 1
    elif d.name == 'bool':
        #Write out as Byte
        gdal_dt = 1
    else:
        gdal_dt = list(dt_dict.keys())[list(dt_dict.values()).index(d)]
    return gdal_dt

#Replace nodata value in GDAL band
def replace_ndv(b, new_ndv):
    b_ndv = get_ndv_b(b)
    bma = np.ma.masked_equal(b.ReadAsArray(), b_ndv)
    bma.set_fill_value(new_ndv)
    b.WriteArray(bma.filled())
    b.SetNoDataValue(new_ndv)
    return b

def set_ndv(dst_fn, ndv):
    dst_ds = gdal.Open(dst_fn, gdal.GA_Update)
    for n in range(1, dst_ds.RasterCount+1):
        b = dst_ds.GetRasterBand(1)
        b.SetNoDataValue(ndv)
    dst_ds = None

#Should overload these functions to handle fn, ds, or b
#Perhaps abstract, as many functions will need this functionality
def get_ndv_fn(fn):
    ds = gdal.Open(fn, gdal.GA_ReadOnly)
    return get_ndv_ds(ds)

#Want to modify to handle multi-band images and return list of ndv
def get_ndv_ds(ds, bnum=1):
    b = ds.GetRasterBand(bnum)
    return get_ndv_b(b)

#Return nodata value for GDAL band
def get_ndv_b(b):
    b_ndv = b.GetNoDataValue()
    if b_ndv is None:
        #Check ul pixel for ndv
        ns = b.XSize
        nl = b.YSize
        ul = float(b.ReadAsArray(0, 0, 1, 1))
        #ur = float(b.ReadAsArray(ns-1, 0, 1, 1))
        lr = float(b.ReadAsArray(ns-1, nl-1, 1, 1))
        #ll = float(b.ReadAsArray(0, nl-1, 1, 1))
        #Probably better to use 3/4 corner criterion
        #if ul == ur == lr == ll:
        if np.isnan(ul) or ul == lr:
            b_ndv = ul
        else:
            #Assume ndv is 0
            b_ndv = 0
    elif np.isnan(b_ndv):
        b_dt = gdal.GetDataTypeName(b.DataType)
        if 'Float' in b_dt:
            b_ndv = np.nan
        else:
            b_ndv = 0
    return b_ndv

#Check for file existence
def fn_check(fn):
    import os
    return os.path.exists(fn)

#This method avoids race condition, but is slower than simple os.path.exists
def fn_check_full(fn):
    import os
    status = True
    if not os.path.isfile(fn):
        status = False
    else:
        try:
            open(fn)
        except IOError:
            status = False
    return status

def fn_list_check(fn_list):
    status = True
    for fn in fn_list:
        if not fn_check(fn):
            print('Unable to find: %s' % fn)
            status = False
    return status

#Write out a recarray as a csv
def write_recarray(outfn, ra):
    with open(outfn,'w') as f:
        f.write(','.join([str(item) for item in ra.dtype.names])+'\n')
        for row in ra:
            f.write(','.join([str(item) for item in row])+'\n')

#Check to make sure image doesn't contain errors
def image_check(fn):
    from osgeo import gdal
    ds = gdal.Open(fn)
    status = True
    for i in range(ds.RasterCount):
        ds.GetRasterBand(i).Checksum()
        if gdal.GetLastErrorType() != 0:
            status = False
    return status

def cpu_count():
    from multiprocessing import cpu_count
    return cpu_count()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    pygments.lexers.golo
    ~~~~~~~~~~~~~~~~~~~~~~

    Lexer for Golo source code.

    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""
#TODO: single quoted string
#TODO: backquote escaped litterals

from pygments.lexer import RegexLexer, bygroups, include, combined
from pygments.token import Text, Comment, Operator, Punctuation, Name,\
                           Keyword, Number, String, Generic

__all__ = ['GoloLexer']

class GoloLexer(RegexLexer):
    """
    For `Golo <http://golo-lang.org/>`_ source code.
    """

    name = 'Golo'
    filenames = ['*.golo']
    aliases = ['golo']

    tokens = {
        'root': [
            (r'[^\S\n]+', Text),

            (r'#.*$', Comment),

            (r'(\^|\.\.\.|:|\?:|->|==|!=|=|\+|-|\*|%|/|<|<=|>|>=|=|\.)',
                Operator),
            (r'(is|isnt|and|or|not|oftype|in|orIfNull)\b', Operator.Word),
            (r'[]{}|(),[]', Punctuation),

            (r'(module|import)(\s+)',
                bygroups(Keyword.Namespace, Text),
                'modname'),
            (r'(let|var)(\s+)',
                bygroups(Keyword.Declaration, Text),
                'varname'),
            (r'(struct)(\s+)',
                bygroups(Keyword.Declaration, Text),
                'structname'),
            (r'(function)(\s+)',
                bygroups(Keyword.Declaration, Text),
                'funcname'),

            (r'(null|true|false)\b', Keyword.Constant),
            (r'(augment|pimp'
             r'|if|else|case|match|return'
             r'|case|when|then|otherwise'
             r'|while|for|foreach'
             r'|try|catch|finally|throw'
             r'|local'
             r'|continue|break)\b', Keyword),

            (r'(map|array|list|set|vector|tuple)(\[)',
                bygroups(Name.Builtin, Punctuation)),
            (r'(print|println|readln|raise|fun'
             r'|asInterfaceInstance)\b', Name.Builtin),
            (r'\b([a-zA-Z_][a-z$A-Z0-9_]*)(\()',
                bygroups(Name.Function, Punctuation)),

            (r'-?[\d_]*\.[\d_]*([eE][+-]?\d[\d_]*)?F?', Number.Float),
            (r'0[0-7]+j?', Number.Oct),
            (r'0[xX][a-fA-F0-9]+', Number.Hex),
            (r'-?\d[\d_]*L', Number.Integer.Long),
            (r'-?\d[\d_]*', Number.Integer),

            ('[a-zA-Z_][a-z$A-Z0-9_]*', Name),

            (r'"""', String, combined('stringescape', 'triplestring')),
            (r'"', String, combined('stringescape', 'doublestring')),
            (r'----', String.Doc, 'doc'),

        ],

        'funcname': [
            (r'[a-zA-Z_][a-z$A-Z0-9_]*', Name.Function, '#pop'),
        ],
        'modname': [
            (r'[a-zA-Z_][a-z$A-Z0-9._]*\*?', Name.Namespace, '#pop')
        ],
        'structname': [
            (r'[a-zA-Z0-9_.]+\*?', Name.Class, '#pop')
        ],
        'varname': [
            (r'[a-zA-Z_][a-z$A-Z0-9_]*', Name.Variable, '#pop'),
        ],
        'string': [
            (r'[^\\\"\n]+', String),
            (r'[\'"\\]', String)
        ],
        'stringescape': [
            (r'\\([\\abfnrtv"\']|\n|N{.*?}|u[a-fA-F0-9]{4}|'
             r'U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape)
        ],
        'triplestring': [
            (r'"""', String, '#pop'),
            include('string'),
            (r'\n', String),
        ],
        'doublestring': [
            (r'"', String.Double, '#pop'),
            (r'\\\\|\\"|\\n', String.Escape),
            include('string')
        ],
        'doc': [
            (r'----', String.Doc, '#pop'),
            (r'#\s+.*#?$', Generic.Heading),
            (r'##+\s+.*#?$', Generic.Subheading),
            (r'\*[^*]+\*', Generic.Emph),
            (r'_[^_]+_', Generic.Emph),
            (r'\*\*[^*]+\*\*', Generic.Strong),
            (r'__[^_]+__', Generic.Strong),
            #(r'`(.*?)`', using(GoloLexer)),
        ],

    }
<commit_msg>add single quoted string: still bugged<commit_after># -*- coding: utf-8 -*-
"""
    pygments.lexers.golo
    ~~~~~~~~~~~~~~~~~~~~~~

    Lexer for Golo source code.

    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""
#TODO: single quoted string
#TODO: backquote escaped litterals

from pygments.lexer import RegexLexer, bygroups, include, combined
from pygments.token import Text, Comment, Operator, Punctuation, Name,\
                           Keyword, Number, String, Generic

__all__ = ['GoloLexer']

class GoloLexer(RegexLexer):
    """
    For `Golo <http://golo-lang.org/>`_ source code.
    """

    name = 'Golo'
    filenames = ['*.golo']
    aliases = ['golo']

    tokens = {
        'root': [
            (r'[^\S\n]+', Text),

            (r'#.*$', Comment),

            (r'(\^|\.\.\.|:|\?:|->|==|!=|=|\+|-|\*|%|/|<|<=|>|>=|=|\.)',
                Operator),
            (r'(is|isnt|and|or|not|oftype|in|orIfNull)\b', Operator.Word),
            (r'[]{}|(),[]', Punctuation),

            (r'(module|import)(\s+)',
                bygroups(Keyword.Namespace, Text),
                'modname'),
            (r'(let|var)(\s+)',
                bygroups(Keyword.Declaration, Text),
                'varname'),
            (r'(struct)(\s+)',
                bygroups(Keyword.Declaration, Text),
                'structname'),
            (r'(function)(\s+)',
                bygroups(Keyword.Declaration, Text),
                'funcname'),

            (r'(null|true|false)\b', Keyword.Constant),
            (r'(augment|pimp'
             r'|if|else|case|match|return'
             r'|case|when|then|otherwise'
             r'|while|for|foreach'
             r'|try|catch|finally|throw'
             r'|local'
             r'|continue|break)\b', Keyword),

            (r'(map|array|list|set|vector|tuple)(\[)',
                bygroups(Name.Builtin, Punctuation)),
            (r'(print|println|readln|raise|fun'
             r'|asInterfaceInstance)\b', Name.Builtin),
            (r'\b([a-zA-Z_][a-z$A-Z0-9_]*)(\()',
                bygroups(Name.Function, Punctuation)),

            (r'-?[\d_]*\.[\d_]*([eE][+-]?\d[\d_]*)?F?', Number.Float),
            (r'0[0-7]+j?', Number.Oct),
            (r'0[xX][a-fA-F0-9]+', Number.Hex),
            (r'-?\d[\d_]*L', Number.Integer.Long),
            (r'-?\d[\d_]*', Number.Integer),

            ('[a-zA-Z_][a-z$A-Z0-9_]*', Name),

            (r'"""', String, combined('stringescape', 'triplestring')),
            (r'"', String, combined('stringescape', 'doublestring')),
            (r"'", String, 'singlestring'),
            (r'----', String.Doc, 'doc'),

        ],

        'funcname': [
            (r'[a-zA-Z_][a-z$A-Z0-9_]*', Name.Function, '#pop'),
        ],
        'modname': [
            (r'[a-zA-Z_][a-z$A-Z0-9._]*\*?', Name.Namespace, '#pop')
        ],
        'structname': [
            (r'[a-zA-Z0-9_.]+\*?', Name.Class, '#pop')
        ],
        'varname': [
            (r'[a-zA-Z_][a-z$A-Z0-9_]*', Name.Variable, '#pop'),
        ],
        'string': [
            (r'[^\\\"\n]+', String),
            (r'[\'"\\]', String)
        ],
        'stringescape': [
            (r'\\([\\abfnrtv"\']|\n|N{.*?}|u[a-fA-F0-9]{4}|'
             r'U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape)
        ],
        'triplestring': [
            (r'"""', String, '#pop'),
            include('string'),
            (r'\n', String),
        ],
        'doublestring': [
            (r'"', String.Double, '#pop'),
            (r'\\\\|\\"|\\n', String.Escape),
            include('string'),
        ],
        'singlestring': [
            (r"'", String, '#pop'),
            (r"\\\\|\\'|\\n", String.Escape),
            include('string'),
        ],
        'doc': [
            (r'----', String.Doc, '#pop'),
            (r'#\s+.*#?$', Generic.Heading),
            (r'##+\s+.*#?$', Generic.Subheading),
            (r'\*[^*]+\*', Generic.Emph),
            (r'_[^_]+_', Generic.Emph),
            (r'\*\*[^*]+\*\*', Generic.Strong),
            (r'__[^_]+__', Generic.Strong),
            #(r'`(.*?)`', using(GoloLexer)),
        ],

    }
<|endoftext|>


<commit_before>"""
    pygments.lexers.mime
    ~~~~~~~~~~~~~~~~~~~~

    Lexer for Multipurpose Internet Mail Extensions (MIME) data.

    :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""

import re

from pygments.lexer import RegexLexer, include
from pygments.lexers import get_lexer_for_mimetype
from pygments.token import Text, Name, String, Operator, Comment, Other
from pygments.util import get_int_opt, ClassNotFound

__all__ = ["MIMELexer"]

class MIMELexer(RegexLexer):
    """
    Lexer for Multipurpose Internet Mail Extensions (MIME) data. This lexer is
    designed to process the nested mulitpart data.

    It assumes that the given data contains both header and body (and is
    splitted by empty line). If no valid header is found, then the entire data
    would be treated as body.

    Additional options accepted:

    `MIME-max-level`
        Max recurssion level for nested MIME structure. Any negative number
        would treated as unlimited. (default: -1)

    `Content-Type`
        Treat the data as specific content type. Useful when header is
        missing, or this lexer would try to parse from header. (default:
        `text/plain`)

    `Multipart-Boundary`
        Set the default multipart boundary delimiter. This option is only used
        when `Content-Type` is `multipart` and header is missing. This lexer
        would try to parse from header by default. (default: None)

    `Content-Transfer-Encoding`
        Treat the data as specific encoding. Or this lexer would try to parse
        from header by default. (default: None)

    .. versionadded:: 2.5
    """

    name = "MIME"
    aliases = ["mime"]
    mimetypes = ["multipart/mixed",
                 "multipart/related",
                 "multipart/alternative"]

    def __init__(self, **options):
        super().__init__(**options)
        self.boundary = options.get("Multipart-Boundary")
        self.content_transfer_encoding = options.get("Content_Transfer_Encoding")
        self.content_type = options.get("Content_Type", "text/plain")
        self.max_nested_level = get_int_opt(options, "MIME-max-level", -1)

    def analyse_text(text):
        try:
            header, body = text.strip().split("\n\n", 1)
            if not body.strip():
                return 0.1

            invalid_headers = MIMELexer.tokens["header"].sub("", header)
            if invalid_headers.strip():
                return 0.1
            else:
                return 1

        except ValueError:
            return 0

    def get_header_tokens(self, match):
        field = match.group(1)

        if field.lower() in self.attention_headers:
            yield match.start(1), Name.Tag, field + ":"
            yield match.start(2), Text.Whitespace, match.group(2)

            pos = match.end(2)
            body = match.group(3)
            for i, t, v in self.get_tokens_unprocessed(body, ("root", field.lower())):
                yield pos + i, t, v

        else:
            yield match.start(), Comment, match.group()

    def get_body_tokens(self, match):
        pos_body_start = match.start()
        entire_body = match.group()

        # skip first newline
        if entire_body[0] == '\n':
            yield pos_body_start, Text.Whitespace, '\n'
            pos_body_start = pos_body_start + 1
            entire_body = entire_body[1:]

        # if it is not a mulitpart
        if not self.content_type.startswith("multipart") or not self.boundary:
            for i, t, v in self.get_bodypart_tokens(entire_body):
                yield pos_body_start + i, t, v
            return

        # find boundary
        bdry_pattern = r"^--%s(--)?\n" % re.escape(self.boundary)
        bdry_matcher = re.compile(bdry_pattern, re.MULTILINE)

        # some data has prefix text before first boundary
        m = bdry_matcher.search(entire_body)
        if m:
            pos_part_start = pos_body_start + m.end()
            pos_iter_start = lpos_end = m.end()
            yield pos_body_start, Text, entire_body[:m.start()]
            yield pos_body_start + lpos_end, String.Delimiter, m.group()
        else:
            pos_part_start = pos_body_start
            pos_iter_start = 0

        # process tokens of each body part
        for m in bdry_matcher.finditer(entire_body, pos_iter_start):
            # bodypart
            lpos_start = pos_part_start - pos_body_start
            lpos_end = m.start()
            part = entire_body[lpos_start:lpos_end]
            for i, t, v in self.get_bodypart_tokens(part):
                yield pos_part_start + i, t, v

            # boundary
            yield pos_body_start + lpos_end, String.Delimiter, m.group()
            pos_part_start = pos_body_start + m.end()

        # some data has suffix text after last boundary
        lpos_start = pos_part_start - pos_body_start
        if lpos_start != len(entire_body):
            yield pos_part_start, Text, entire_body[lpos_start:]

    def get_bodypart_tokens(self, text):
        # return if:
        #  * no content
        #  * no content type specific
        #  * content encoding is not readable
        #  * max recurrsion exceed
        if not text.strip() or not self.content_type:
            return [(0, Other, text)]

        cte = self.content_transfer_encoding
        if cte and cte not in {"8bit", "7bit", "quoted-printable"}:
            return [(0, Other, text)]

        if self.max_nested_level == 0:
            return [(0, Other, text)]

        # get lexer
        try:
            lexer = get_lexer_for_mimetype(self.content_type)
        except ClassNotFound:
            return [(0, Other, text)]

        if isinstance(lexer, type(self)):
            lexer.max_nested_level = self.max_nested_level - 1

        return lexer.get_tokens_unprocessed(text)

    def store_content_type(self, match):
        self.content_type = match.group(1)

        prefix_len = match.start(1) - match.start(0)
        yield match.start(0), Text.Whitespace, match.group(0)[:prefix_len]
        yield match.start(1), Name.Label, match.group(2)
        yield match.end(2), String.Delimiter, '/'
        yield match.start(3), Name.Label, match.group(3)

    def get_content_type_subtokens(self, match):
        yield match.start(1), Text, match.group(1)
        yield match.start(2), Text.Whitespace, match.group(2)
        yield match.start(3), Name.Attribute, match.group(3)
        yield match.start(4), Operator, match.group(4)
        yield match.start(5), String, match.group(5)

        if match.group(3).lower() == "boundary":
            boundary = match.group(5).strip()
            if boundary[0] == '"' and boundary[-1] == '"':
                boundary = boundary[1:-1]
            self.boundary = boundary

    def store_content_transfer_encoding(self, match):
        self.content_transfer_encoding = match.group(0).lower()
        yield match.start(0), Name.Constant, match.group(0)

    attention_headers = {"content-type", "content-transfer-encoding"}

    tokens = {
        "root": [
            (r"^([\w-]+):( *)([\s\S]*?\n)(?![ \t])", get_header_tokens),
            (r"^$[\s\S]+", get_body_tokens),
        ],
        "header": [
            # folding
            (r"\n[ \t]", Text.Whitespace),
            (r"\n(?![ \t])", Text.Whitespace, "#pop"),
        ],
        "content-type": [
            include("header"),
            (
                r"^\s*((multipart|application|audio|font|image|model|text|video"
                r"|message)/([\w-]+))",
                store_content_type,
            ),
            (r'(;)((?:[ \t]|\n[ \t])*)([\w:-]+)(=)([\s\S]*?)(?=;|\n(?![ \t]))',
             get_content_type_subtokens),
            (r';[ \t]*\n(?![ \t])', Text, '#pop'),
        ],
        "content-transfer-encoding": [
            include("header"),
            (r"([\w-]+)", store_content_transfer_encoding),
        ],
    }
<commit_msg>MIME lexer: remove analyse_text<commit_after>"""
    pygments.lexers.mime
    ~~~~~~~~~~~~~~~~~~~~

    Lexer for Multipurpose Internet Mail Extensions (MIME) data.

    :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""

import re

from pygments.lexer import RegexLexer, include
from pygments.lexers import get_lexer_for_mimetype
from pygments.token import Text, Name, String, Operator, Comment, Other
from pygments.util import get_int_opt, ClassNotFound

__all__ = ["MIMELexer"]

class MIMELexer(RegexLexer):
    """
    Lexer for Multipurpose Internet Mail Extensions (MIME) data. This lexer is
    designed to process nested multipart data.

    It assumes that the given data contains both header and body (and is
    split at an empty line). If no valid header is found, then the entire data
    will be treated as body.

    Additional options accepted:

    `MIME-max-level`
        Max recursion level for nested MIME structure. Any negative number
        would treated as unlimited. (default: -1)

    `Content-Type`
        Treat the data as a specific content type. Useful when header is
        missing, or this lexer would try to parse from header. (default:
        `text/plain`)

    `Multipart-Boundary`
        Set the default multipart boundary delimiter. This option is only used
        when `Content-Type` is `multipart` and header is missing. This lexer
        would try to parse from header by default. (default: None)

    `Content-Transfer-Encoding`
        Treat the data as a specific encoding. Or this lexer would try to parse
        from header by default. (default: None)

    .. versionadded:: 2.5
    """

    name = "MIME"
    aliases = ["mime"]
    mimetypes = ["multipart/mixed",
                 "multipart/related",
                 "multipart/alternative"]

    def __init__(self, **options):
        super().__init__(**options)
        self.boundary = options.get("Multipart-Boundary")
        self.content_transfer_encoding = options.get("Content_Transfer_Encoding")
        self.content_type = options.get("Content_Type", "text/plain")
        self.max_nested_level = get_int_opt(options, "MIME-max-level", -1)

    def get_header_tokens(self, match):
        field = match.group(1)

        if field.lower() in self.attention_headers:
            yield match.start(1), Name.Tag, field + ":"
            yield match.start(2), Text.Whitespace, match.group(2)

            pos = match.end(2)
            body = match.group(3)
            for i, t, v in self.get_tokens_unprocessed(body, ("root", field.lower())):
                yield pos + i, t, v

        else:
            yield match.start(), Comment, match.group()

    def get_body_tokens(self, match):
        pos_body_start = match.start()
        entire_body = match.group()

        # skip first newline
        if entire_body[0] == '\n':
            yield pos_body_start, Text.Whitespace, '\n'
            pos_body_start = pos_body_start + 1
            entire_body = entire_body[1:]

        # if it is not a mulitpart
        if not self.content_type.startswith("multipart") or not self.boundary:
            for i, t, v in self.get_bodypart_tokens(entire_body):
                yield pos_body_start + i, t, v
            return

        # find boundary
        bdry_pattern = r"^--%s(--)?\n" % re.escape(self.boundary)
        bdry_matcher = re.compile(bdry_pattern, re.MULTILINE)

        # some data has prefix text before first boundary
        m = bdry_matcher.search(entire_body)
        if m:
            pos_part_start = pos_body_start + m.end()
            pos_iter_start = lpos_end = m.end()
            yield pos_body_start, Text, entire_body[:m.start()]
            yield pos_body_start + lpos_end, String.Delimiter, m.group()
        else:
            pos_part_start = pos_body_start
            pos_iter_start = 0

        # process tokens of each body part
        for m in bdry_matcher.finditer(entire_body, pos_iter_start):
            # bodypart
            lpos_start = pos_part_start - pos_body_start
            lpos_end = m.start()
            part = entire_body[lpos_start:lpos_end]
            for i, t, v in self.get_bodypart_tokens(part):
                yield pos_part_start + i, t, v

            # boundary
            yield pos_body_start + lpos_end, String.Delimiter, m.group()
            pos_part_start = pos_body_start + m.end()

        # some data has suffix text after last boundary
        lpos_start = pos_part_start - pos_body_start
        if lpos_start != len(entire_body):
            yield pos_part_start, Text, entire_body[lpos_start:]

    def get_bodypart_tokens(self, text):
        # return if:
        #  * no content
        #  * no content type specific
        #  * content encoding is not readable
        #  * max recurrsion exceed
        if not text.strip() or not self.content_type:
            return [(0, Other, text)]

        cte = self.content_transfer_encoding
        if cte and cte not in {"8bit", "7bit", "quoted-printable"}:
            return [(0, Other, text)]

        if self.max_nested_level == 0:
            return [(0, Other, text)]

        # get lexer
        try:
            lexer = get_lexer_for_mimetype(self.content_type)
        except ClassNotFound:
            return [(0, Other, text)]

        if isinstance(lexer, type(self)):
            lexer.max_nested_level = self.max_nested_level - 1

        return lexer.get_tokens_unprocessed(text)

    def store_content_type(self, match):
        self.content_type = match.group(1)

        prefix_len = match.start(1) - match.start(0)
        yield match.start(0), Text.Whitespace, match.group(0)[:prefix_len]
        yield match.start(1), Name.Label, match.group(2)
        yield match.end(2), String.Delimiter, '/'
        yield match.start(3), Name.Label, match.group(3)

    def get_content_type_subtokens(self, match):
        yield match.start(1), Text, match.group(1)
        yield match.start(2), Text.Whitespace, match.group(2)
        yield match.start(3), Name.Attribute, match.group(3)
        yield match.start(4), Operator, match.group(4)
        yield match.start(5), String, match.group(5)

        if match.group(3).lower() == "boundary":
            boundary = match.group(5).strip()
            if boundary[0] == '"' and boundary[-1] == '"':
                boundary = boundary[1:-1]
            self.boundary = boundary

    def store_content_transfer_encoding(self, match):
        self.content_transfer_encoding = match.group(0).lower()
        yield match.start(0), Name.Constant, match.group(0)

    attention_headers = {"content-type", "content-transfer-encoding"}

    tokens = {
        "root": [
            (r"^([\w-]+):( *)([\s\S]*?\n)(?![ \t])", get_header_tokens),
            (r"^$[\s\S]+", get_body_tokens),
        ],
        "header": [
            # folding
            (r"\n[ \t]", Text.Whitespace),
            (r"\n(?![ \t])", Text.Whitespace, "#pop"),
        ],
        "content-type": [
            include("header"),
            (
                r"^\s*((multipart|application|audio|font|image|model|text|video"
                r"|message)/([\w-]+))",
                store_content_type,
            ),
            (r'(;)((?:[ \t]|\n[ \t])*)([\w:-]+)(=)([\s\S]*?)(?=;|\n(?![ \t]))',
             get_content_type_subtokens),
            (r';[ \t]*\n(?![ \t])', Text, '#pop'),
        ],
        "content-transfer-encoding": [
            include("header"),
            (r"([\w-]+)", store_content_transfer_encoding),
        ],
    }
<|endoftext|>