<commit_before>from __future__ import division

import numpy as np
import os
from six.moves.urllib import request
import unittest

from chainer import testing

from chainercv.evaluations import calc_detection_voc_ap
from chainercv.evaluations import calc_detection_voc_prec_rec
from chainercv.evaluations import eval_detection_voc_ap

@testing.parameterize(*(
    testing.product_dict(
        [{
            'pred_bboxes': [
                [[0, 0, 1, 1], [0, 0, 2, 2], [0.3, 0.3, 0.5, 0.5]],
            ],
            'pred_labels': [
                [0, 0, 0],
            ],
            'pred_scores': [
                [0.8, 0.9, 1],
            ],
            'gt_bboxes': [
                [[0, 0, 1, 0.9]],
            ],
            'gt_labels': [
                [0],
            ],
        }],
        [
            {
                'iou_thresh': 0.5,
                'prec': [
                    [0, 0, 1 / 3],
                ],
                'rec': [
                    [0, 0, 1],
                ],
            },
            {
                'iou_thresh': 0.97,
                'prec': [
                    [0, 0, 0],
                ],
                'rec': [
                    [0, 0, 0],
                ],
            },
        ]
    ) +
    [
        {
            'pred_bboxes': [
                [[0, 4, 1, 5], [0, 0, 1, 1]],
                [[0, 0, 2, 2], [2, 2, 3, 3], [5, 5, 7, 7]],
            ],
            'pred_labels': [
                [0, 0],
                [0, 2, 2],
            ],
            'pred_scores': [
                [1, 0.9],
                [0.7, 0.6, 0.8],
            ],
            'gt_bboxes': [
                [[0, 0, 1, 1], [1, 0, 4, 4]],
                [[2, 2, 3, 3]],
            ],
            'gt_labels': [
                [0, 0],
                [2],
            ],
            'iou_thresh': 0.4,
            'prec': [
                [0, 0.5, 1 / 3],
                None,
                [0, 0.5],
            ],
            'rec': [
                [0, 0.5, 0.5],
                None,
                [0, 1],
            ],
        },
        {
            'pred_bboxes': [
                [[0, 0, 1, 1], [0, 0, 2, 2], [0.3, 0.3, 0.5, 0.5]],
            ],
            'pred_labels': [
                [0, 0, 0],
            ],
            'pred_scores': [
                [0.8, 0.9, 1],
            ],
            'gt_bboxes': [
                [[0, 0, 1, 0.9], [1., 1., 2., 2.]],
            ],
            'gt_labels': [
                [0, 0],
            ],
            'gt_difficults': [
                [False, True],
            ],
            'iou_thresh': 0.5,
            'prec': [
                [0, 0, 1 / 3],
            ],
            'rec': [
                [0, 0, 1],
            ],
        },

    ]
))
class TestCalcDetectionVOCPrecRec(unittest.TestCase):

    def setUp(self):
        self.pred_bboxes = (np.array(bbox) for bbox in self.pred_bboxes)
        self.pred_labels = (np.array(label) for label in self.pred_labels)
        self.pred_scores = (np.array(score) for score in self.pred_scores)
        self.gt_bboxes = (np.array(bbox) for bbox in self.gt_bboxes)
        self.gt_labels = (np.array(label) for label in self.gt_labels)

        if hasattr(self, 'gt_difficults'):
            self.gt_difficults = (
                np.array(difficult) for difficult in self.gt_difficults)
        else:
            self.gt_difficults = None

    def test_calc_detection_voc_prec_rec(self):
        prec, rec = calc_detection_voc_prec_rec(
            self.pred_bboxes, self.pred_labels, self.pred_scores,
            self.gt_bboxes, self.gt_labels, self.gt_difficults,
            iou_thresh=self.iou_thresh)

        self.assertEqual(len(prec), len(self.prec))
        for prec_l, expected_prec_l in zip(prec, self.prec):
            if prec_l is None and expected_prec_l is None:
                continue
            np.testing.assert_equal(prec_l, expected_prec_l)

        self.assertEqual(len(rec), len(self.rec))
        for rec_l, expected_rec_l in zip(rec, self.rec):
            if rec_l is None and expected_rec_l is None:
                continue
            np.testing.assert_equal(rec_l, expected_rec_l)

@testing.parameterize(
    {'use_07_metric': False,
     'ap': [0.25, None, 0.5]},
    {'use_07_metric': True,
     'ap': [0.5 / 11 * 6, None, 0.5]},
)
class TestCalcDetectionVOCAP(unittest.TestCase):

    prec = [[0, 0.5, 1 / 3], None, [0, 0.5]]
    rec = [[0, 0.5, 0.5], None, [0, 1]]

    def setUp(self):
        self.prec = [
            np.array(prec_l) if prec_l is not None else None
            for prec_l in self.prec]
        self.rec = [
            np.array(rec_l) if rec_l is not None else None
            for rec_l in self.rec]

    def test_calc_detection_voc_ap(self):
        ap = calc_detection_voc_ap(
            self.prec, self.rec, use_07_metric=self.use_07_metric)

        self.assertEqual(len(ap), len(self.ap))
        for ap_l, expected_ap_l in zip(ap, self.ap):
            if ap_l is None and expected_ap_l is None:
                continue
            self.assertAlmostEqual(ap_l, expected_ap_l)

class TestEvalDetectionVOCAP(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        base_url = 'https://github.com/yuyu2172/' \
            'share-weights/releases/download/0.0.2'

        cls.dataset = np.load(request.urlretrieve(os.path.join(
            base_url,
            'voc_detection_dataset_2007_test_truncated_2017_06_02.npz'))[0])
        cls.result = np.load(request.urlretrieve(os.path.join(
            base_url,
            'voc_detection_result_2007_test_truncated_2017_06_02.npz'))[0])

    def test_eval_detection_voc_ap(self):
        pred_bboxes = self.result['bboxes']
        pred_labels = self.result['labels']
        pred_scores = self.result['scores']

        gt_bboxes = self.dataset['bboxes']
        gt_labels = self.dataset['labels']
        gt_difficults = self.dataset['difficults']

        ap = eval_detection_voc_ap(
            pred_bboxes, pred_labels, pred_scores,
            gt_bboxes, gt_labels, gt_difficults,
            use_07_metric=True)

        # these scores were calculated by MATLAB code
        expected = [
            0.772727,
            0.738780,
            0.957576,
            0.640153,
            0.579473,
            1.000000,
            0.970030,
            1.000000,
            0.705931,
            0.678719,
            0.863636,
            1.000000,
            1.000000,
            0.561364,
            0.798813,
            0.712121,
            0.939394,
            0.563636,
            0.927273,
            0.654545,
        ]

        np.testing.assert_almost_equal(ap, expected, decimal=5)
<commit_msg>fix test<commit_after>from __future__ import division

import numpy as np
import os
from six.moves.urllib import request
import unittest

from chainer import testing

from chainercv.evaluations import calc_detection_voc_ap
from chainercv.evaluations import calc_detection_voc_prec_rec
from chainercv.evaluations import eval_detection_voc_ap

@testing.parameterize(*(
    testing.product_dict(
        [{
            'pred_bboxes': [
                [[0, 0, 1, 1], [0, 0, 2, 2], [0.3, 0.3, 0.5, 0.5]],
            ],
            'pred_labels': [
                [0, 0, 0],
            ],
            'pred_scores': [
                [0.8, 0.9, 1],
            ],
            'gt_bboxes': [
                [[0, 0, 1, 0.9]],
            ],
            'gt_labels': [
                [0],
            ],
        }],
        [
            {
                'iou_thresh': 0.5,
                'prec': [
                    [0, 0, 1 / 3],
                ],
                'rec': [
                    [0, 0, 1],
                ],
            },
            {
                'iou_thresh': 0.97,
                'prec': [
                    [0, 0, 0],
                ],
                'rec': [
                    [0, 0, 0],
                ],
            },
        ]
    ) +
    [
        {
            'pred_bboxes': [
                [[0, 4, 1, 5], [0, 0, 1, 1]],
                [[0, 0, 2, 2], [2, 2, 3, 3], [5, 5, 7, 7]],
            ],
            'pred_labels': [
                [0, 0],
                [0, 2, 2],
            ],
            'pred_scores': [
                [1, 0.9],
                [0.7, 0.6, 0.8],
            ],
            'gt_bboxes': [
                [[0, 0, 1, 1], [1, 0, 4, 4]],
                [[2, 2, 3, 3]],
            ],
            'gt_labels': [
                [0, 0],
                [2],
            ],
            'iou_thresh': 0.4,
            'prec': [
                [0, 0.5, 1 / 3],
                None,
                [0, 0.5],
            ],
            'rec': [
                [0, 0.5, 0.5],
                None,
                [0, 1],
            ],
        },
        {
            'pred_bboxes': [
                [[0, 0, 1, 1], [0, 0, 2, 2], [0.3, 0.3, 0.5, 0.5]],
            ],
            'pred_labels': [
                [0, 0, 0],
            ],
            'pred_scores': [
                [0.8, 0.9, 1],
            ],
            'gt_bboxes': [
                [[0, 0, 1, 0.9], [1., 1., 2., 2.]],
            ],
            'gt_labels': [
                [0, 0],
            ],
            'gt_difficults': [
                [False, True],
            ],
            'iou_thresh': 0.5,
            'prec': [
                [0, 0, 1 / 3],
            ],
            'rec': [
                [0, 0, 1],
            ],
        },

    ]
))
class TestCalcDetectionVOCPrecRec(unittest.TestCase):

    def setUp(self):
        self.pred_bboxes = (np.array(bbox) for bbox in self.pred_bboxes)
        self.pred_labels = (np.array(label) for label in self.pred_labels)
        self.pred_scores = (np.array(score) for score in self.pred_scores)
        self.gt_bboxes = (np.array(bbox) for bbox in self.gt_bboxes)
        self.gt_labels = (np.array(label) for label in self.gt_labels)

        if hasattr(self, 'gt_difficults'):
            self.gt_difficults = (
                np.array(difficult) for difficult in self.gt_difficults)
        else:
            self.gt_difficults = None

    def test_calc_detection_voc_prec_rec(self):
        prec, rec = calc_detection_voc_prec_rec(
            self.pred_bboxes, self.pred_labels, self.pred_scores,
            self.gt_bboxes, self.gt_labels, self.gt_difficults,
            iou_thresh=self.iou_thresh)

        self.assertEqual(len(prec), len(self.prec))
        for prec_l, expected_prec_l in zip(prec, self.prec):
            if prec_l is None and expected_prec_l is None:
                continue
            np.testing.assert_equal(prec_l, expected_prec_l)

        self.assertEqual(len(rec), len(self.rec))
        for rec_l, expected_rec_l in zip(rec, self.rec):
            if rec_l is None and expected_rec_l is None:
                continue
            np.testing.assert_equal(rec_l, expected_rec_l)

@testing.parameterize(
    {'use_07_metric': False,
     'ap': [0.25, np.nan, 0.5]},
    {'use_07_metric': True,
     'ap': [0.5 / 11 * 6, np.nan, 0.5]},
)
class TestCalcDetectionVOCAP(unittest.TestCase):

    prec = [[0, 0.5, 1 / 3], None, [0, 0.5]]
    rec = [[0, 0.5, 0.5], None, [0, 1]]

    def setUp(self):
        self.prec = [
            np.array(prec_l) if prec_l is not None else None
            for prec_l in self.prec]
        self.rec = [
            np.array(rec_l) if rec_l is not None else None
            for rec_l in self.rec]

    def test_calc_detection_voc_ap(self):
        ap = calc_detection_voc_ap(
            self.prec, self.rec, use_07_metric=self.use_07_metric)

        self.assertEqual(len(ap), len(self.ap))
        for ap_l, expected_ap_l in zip(ap, self.ap):
            if ap_l is None and expected_ap_l is None:
                continue
            self.assertAlmostEqual(ap_l, expected_ap_l)

class TestEvalDetectionVOCAP(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        base_url = 'https://github.com/yuyu2172/' \
            'share-weights/releases/download/0.0.2'

        cls.dataset = np.load(request.urlretrieve(os.path.join(
            base_url,
            'voc_detection_dataset_2007_test_truncated_2017_06_02.npz'))[0])
        cls.result = np.load(request.urlretrieve(os.path.join(
            base_url,
            'voc_detection_result_2007_test_truncated_2017_06_02.npz'))[0])

    def test_eval_detection_voc_ap(self):
        pred_bboxes = self.result['bboxes']
        pred_labels = self.result['labels']
        pred_scores = self.result['scores']

        gt_bboxes = self.dataset['bboxes']
        gt_labels = self.dataset['labels']
        gt_difficults = self.dataset['difficults']

        ap = eval_detection_voc_ap(
            pred_bboxes, pred_labels, pred_scores,
            gt_bboxes, gt_labels, gt_difficults,
            use_07_metric=True)

        # these scores were calculated by MATLAB code
        expected = [
            0.772727,
            0.738780,
            0.957576,
            0.640153,
            0.579473,
            1.000000,
            0.970030,
            1.000000,
            0.705931,
            0.678719,
            0.863636,
            1.000000,
            1.000000,
            0.561364,
            0.798813,
            0.712121,
            0.939394,
            0.563636,
            0.927273,
            0.654545,
        ]

        np.testing.assert_almost_equal(ap, expected, decimal=5)
<|endoftext|>


<commit_before># !/usr/bin/env python
# encoding: utf-8
#
# This file is part of ckanext-nhm
# Created by the Natural History Museum in London, UK

'''Routes for handling stable objects - those with GUIDs in KE EMu

If someone accesses URL:
    object/73f450db-46b3-45a0-ac18-f00547be5af1
It will 302 redirect to the specimen, artefact or index lot page (though currently we
only
support specimens).

If a user requests the object in RDF format:
    object/73f450db-46b3-45a0-ac18-f00547be5af1.ttl
Returns RDF

In both cases, if a version is appended to the end of the URL then that version of the
record is
returned. For example:
    object/73f450db-46b3-45a0-ac18-f00547be5af1/1551692486000
otherwise the current version is returned.

'''

import logging

from ckanext.dcat.controllers import check_access_header
from ckanext.dcat.utils import CONTENT_TYPES
from ckanext.nhm.lib.record import get_record_by_uuid
from flask import Blueprint, Response, redirect, url_for

from ckan.plugins import toolkit

log = logging.getLogger(__name__)

ABYSSLINE_UUIDS = [u'bc03fc1a-3613-41a2-b1f1-bf905e0fa6d0',
                   u'de4bd6ce-07fe-496e-bffc-67a4c6b9782c',
                   u'16599946-2aba-4710-98e6-43c522061878',
                   u'b7ffe7a2-7be1-4d4f-b784-7aaecf0ee743',
                   u'76acc5a2-6e0e-4599-8104-b8e243af10c4',
                   u'7e8ca2d8-aea1-45bd-b7e0-d0575cadd82d',
                   u'95d0bd7f-0df9-47e4-8003-cd12007d54b4',
                   u'c57f1bd3-1b32-41e6-8e1d-0ad6472e4327',
                   u'd15a68e0-b2b3-40b4-8cab-0563609cc80d',
                   u'4ae2430e-549e-47f2-ba5d-0e9a08443d31',
                   u'b2a871bf-46d5-4639-a839-427a3efa848c',
                   u'280c758b-5287-4a13-9f45-f6a6150b37d0',
                   u'92825c07-a16d-4c5e-a8e9-4fbcdc8cf44a',
                   u'2866f91e-b99e-4703-a9d3-fe1876df1da1',
                   u'd0062182-89dc-4deb-b746-688289783b5f',
                   u'38c16bec-7bf9-4c2b-b862-5da460ba6c0c',
                   u'15e6ddc7-3ca7-453c-bba5-f84888716505',
                   u'72db478a-ea4f-4f3e-be08-95ec9fb4d20e',
                   u'11948cb9-654f-4519-a654-f134380093ea',
                   u'292bd655-83d6-440f-9668-82dfa4185b04',
                   u'2ed865af-1605-4d78-8fd8-9c7659781854',
                   u'4d6f6aaf-93fd-4629-b224-2ce8dd3320f6',
                   u'5ad996fe-134a-4625-a404-9d0cdae435d4',
                   u'68072fc9-3e84-4202-8e97-6c9c0c5fc83d',
                   u'c1c4d8f3-6cd5-439f-a546-943b5e2e8d8f',
                   u'479218ae-813b-4736-b3f2-7eec63640ffd',
                   u'90e22ace-ef5d-4cb5-a4a5-29fcd55ed660',
                   u'97d40306-fe6c-4911-8e68-1f9efc3d838f',
                   u'bd6fe2ce-b4ae-470e-8bdc-cf28a94c6208',
                   u'608349ff-5adf-4e1e-8cd7-7e0e41aee222',
                   u'241d094a-568f-4194-997c-fd08f67dcdac',
                   u'93b0a70d-c74e-4735-b70e-0c6e4c6a36ff',
                   u'e9f38ce3-5ed5-49f3-8713-c26de2eefd2b',
                   u'f263bc90-6307-462c-9e02-7b87d20e2840']

# this is the main record citation blueprint, use this in url_fors etc
blueprint = Blueprint(name=u'object', import_name=__name__, url_prefix=u'/object')
# this blueprint is here for the old citation urls, don't use it in url_fors etc
specimen_blueprint = Blueprint(name=u'specimen', import_name=__name__, url_prefix=u'/specimen')

def _context():
    return {
        u'user': toolkit.c.user or toolkit.c.author,
        u'auth_user_obj': toolkit.c.userobj
        }

@specimen_blueprint.route(u'/<uuid>.<_format>', defaults={u'version': None})
@specimen_blueprint.route(u'/<uuid>/<int:version>.<_format>')
@blueprint.route(u'/<uuid>.<_format>', defaults={u'version': None})
@blueprint.route(u'/<uuid>/<int:version>.<_format>')
def rdf(uuid, _format, version):
    '''
    Return RDF view of object.

    :param uuid: the object's uuid
    :param _format: the format requested
    :param version: the version of the record to retrieve, or None if the current
                    version is desired
    :return: the data to display
    '''
    data_dict = {
        u'uuid': uuid,
        u'format': _format,
        u'version': version,
    }
    try:
        result = toolkit.get_action(u'object_rdf')(_context(), data_dict)
        return Response(result, mimetype=CONTENT_TYPES[_format])
    except toolkit.ValidationError, e:
        toolkit.abort(409, str(e))

@specimen_blueprint.route(u'/<uuid>', defaults={u'version': None})
@specimen_blueprint.route(u'/<uuid>/<int:version>')
@blueprint.route(u'/<uuid>', defaults={u'version': None})
@blueprint.route(u'/<uuid>/<int:version>')
def view(uuid, version):
    '''View object. If this is normal HTTP request, this will redirect to the record,
    otherwise if
    the request is for RDF (content negotiation) return the rdf.
    :param uuid: the uuid of the object
    :param version: the version of the object, or None for current version

    '''
    if uuid in ABYSSLINE_UUIDS:
        abyssline_object_redirect(uuid, version)

    # is the request for a particular format?
    _format = check_access_header()

    if _format:
        # cetaf standards require us to return a 303 redirect to the rdf doc
        url = url_for(u'object.rdf', uuid=uuid, _format=_format, version=version)
        return redirect(url, code=303)
    else:
        try:
            # get the record at the given version
            record, resource = get_record_by_uuid(uuid, version)
        except TypeError:
            pass
        else:
            if record:
                package_id = resource.get_package_id()
                package = toolkit.get_action(u'package_show')(_context(), {u'id': package_id})
                # cetaf standards require us to return a 303 redirect to the html record page
                url = url_for(u'record.view', package_name=package[u'name'],
                              resource_id=resource.id, record_id=record[u'_id'], version=version)
                return redirect(url, code=303)

    toolkit.abort(404, toolkit._(u'Record not found'))

def abyssline_object_redirect(uuid, version):
    '''Temporary fix to allow abyssline object references
    to resolve to the temp dataset.

    :param uuid: the object's uuid
    :param version: the version to get

    '''
    resource_id = toolkit.config.get(u'ckanext.nhm.abyssline_resource_id')

    # figure out the rounded version

    data_dict = {
        u'resource_id': resource_id,
        u'version': version,
        }
    version = toolkit.get_action(u'datastore_get_rounded_version')(_context(), data_dict)

    # search for the record
    search_data_dict = {
        u'resource_id': resource_id,
        u'filters': {
            u'catalogNumber': uuid
            }
        }

    search_result = toolkit.get_action(u'datastore_search')(_context(), search_data_dict)
    try:
        record = search_result[u'records'][0]
    except KeyError:
        pass
    else:
        toolkit.redirect_to(u'record.view', package_name=u'abyssline',
                            resource_id=resource_id,
                            record_id=record[u'_id'], version=version)

    toolkit.abort(404, toolkit._(u'Record not found'))
<commit_msg>Reformat object module<commit_after># !/usr/bin/env python
# encoding: utf-8
#
# This file is part of ckanext-nhm
# Created by the Natural History Museum in London, UK

'''
Routes for handling stable objects - those with GUIDs in KE EMu

If someone accesses URL:
    object/73f450db-46b3-45a0-ac18-f00547be5af1
It will 302 redirect to the specimen, artefact or index lot page (though currently we only support
specimens).

If a user requests the object in RDF format:
    object/73f450db-46b3-45a0-ac18-f00547be5af1.ttl
Returns RDF

In both cases, if a version is appended to the end of the URL then that version of the record is
returned. For example:
    object/73f450db-46b3-45a0-ac18-f00547be5af1/1551692486000
otherwise the current version is returned.

Old style /specimen urls are also supported to ensure backwards compatibility.
'''

import logging
from ckan.plugins import toolkit
from ckanext.dcat.controllers import check_access_header
from ckanext.dcat.utils import CONTENT_TYPES
from ckanext.nhm.lib.record import get_record_by_uuid
from flask import Blueprint, Response, redirect, url_for

log = logging.getLogger(__name__)

ABYSSLINE_UUIDS = [
    u'bc03fc1a-3613-41a2-b1f1-bf905e0fa6d0',
    u'de4bd6ce-07fe-496e-bffc-67a4c6b9782c',
    u'16599946-2aba-4710-98e6-43c522061878',
    u'b7ffe7a2-7be1-4d4f-b784-7aaecf0ee743',
    u'76acc5a2-6e0e-4599-8104-b8e243af10c4',
    u'7e8ca2d8-aea1-45bd-b7e0-d0575cadd82d',
    u'95d0bd7f-0df9-47e4-8003-cd12007d54b4',
    u'c57f1bd3-1b32-41e6-8e1d-0ad6472e4327',
    u'd15a68e0-b2b3-40b4-8cab-0563609cc80d',
    u'4ae2430e-549e-47f2-ba5d-0e9a08443d31',
    u'b2a871bf-46d5-4639-a839-427a3efa848c',
    u'280c758b-5287-4a13-9f45-f6a6150b37d0',
    u'92825c07-a16d-4c5e-a8e9-4fbcdc8cf44a',
    u'2866f91e-b99e-4703-a9d3-fe1876df1da1',
    u'd0062182-89dc-4deb-b746-688289783b5f',
    u'38c16bec-7bf9-4c2b-b862-5da460ba6c0c',
    u'15e6ddc7-3ca7-453c-bba5-f84888716505',
    u'72db478a-ea4f-4f3e-be08-95ec9fb4d20e',
    u'11948cb9-654f-4519-a654-f134380093ea',
    u'292bd655-83d6-440f-9668-82dfa4185b04',
    u'2ed865af-1605-4d78-8fd8-9c7659781854',
    u'4d6f6aaf-93fd-4629-b224-2ce8dd3320f6',
    u'5ad996fe-134a-4625-a404-9d0cdae435d4',
    u'68072fc9-3e84-4202-8e97-6c9c0c5fc83d',
    u'c1c4d8f3-6cd5-439f-a546-943b5e2e8d8f',
    u'479218ae-813b-4736-b3f2-7eec63640ffd',
    u'90e22ace-ef5d-4cb5-a4a5-29fcd55ed660',
    u'97d40306-fe6c-4911-8e68-1f9efc3d838f',
    u'bd6fe2ce-b4ae-470e-8bdc-cf28a94c6208',
    u'608349ff-5adf-4e1e-8cd7-7e0e41aee222',
    u'241d094a-568f-4194-997c-fd08f67dcdac',
    u'93b0a70d-c74e-4735-b70e-0c6e4c6a36ff',
    u'e9f38ce3-5ed5-49f3-8713-c26de2eefd2b',
    u'f263bc90-6307-462c-9e02-7b87d20e2840'
]

# this is the main record citation blueprint, use this in url_fors etc
blueprint = Blueprint(name=u'object', import_name=__name__, url_prefix=u'/object')
# this blueprint is here for the old citation urls, don't use it in url_fors etc
specimen_blueprint = Blueprint(name=u'specimen', import_name=__name__, url_prefix=u'/specimen')

def _context():
    return {
        u'user': toolkit.c.user or toolkit.c.author,
        u'auth_user_obj': toolkit.c.userobj
    }

@specimen_blueprint.route(u'/<uuid>.<_format>', defaults={u'version': None})
@specimen_blueprint.route(u'/<uuid>/<int:version>.<_format>')
@blueprint.route(u'/<uuid>.<_format>', defaults={u'version': None})
@blueprint.route(u'/<uuid>/<int:version>.<_format>')
def rdf(uuid, _format, version):
    '''
    Return RDF view of object.

    :param uuid: the object's uuid
    :param _format: the format requested
    :param version: the version of the record to retrieve, or None if the current version is desired
    :return: the data to display
    '''
    data_dict = {
        u'uuid': uuid,
        u'format': _format,
        u'version': version,
    }
    try:
        result = toolkit.get_action(u'object_rdf')(_context(), data_dict)
        return Response(result, mimetype=CONTENT_TYPES[_format])
    except toolkit.ValidationError, e:
        toolkit.abort(409, str(e))

@specimen_blueprint.route(u'/<uuid>', defaults={u'version': None})
@specimen_blueprint.route(u'/<uuid>/<int:version>')
@blueprint.route(u'/<uuid>', defaults={u'version': None})
@blueprint.route(u'/<uuid>/<int:version>')
def view(uuid, version):
    '''
    View object. If this is normal HTTP request, this will redirect to the record, otherwise if
    the request is for RDF (content negotiation) return the rdf.

    :param uuid: the uuid of the object
    :param version: the version of the object, or None for current version
    '''
    if uuid in ABYSSLINE_UUIDS:
        abyssline_object_redirect(uuid, version)

    # is the request for a particular format?
    _format = check_access_header()

    if _format:
        # cetaf standards require us to return a 303 redirect to the rdf doc
        url = url_for(u'object.rdf', uuid=uuid, _format=_format, version=version)
        return redirect(url, code=303)
    else:
        try:
            # get the record at the given version
            record, resource = get_record_by_uuid(uuid, version)
        except TypeError:
            pass
        else:
            if record:
                package_id = resource.get_package_id()
                package = toolkit.get_action(u'package_show')(_context(), {u'id': package_id})
                # cetaf standards require us to return a 303 redirect to the html record page
                url = url_for(u'record.view', package_name=package[u'name'],
                              resource_id=resource.id, record_id=record[u'_id'], version=version)
                return redirect(url, code=303)

    toolkit.abort(404, toolkit._(u'Record not found'))

def abyssline_object_redirect(uuid, version):
    '''
    Temporary fix to allow abyssline object references to resolve to the temp dataset.

    :param uuid: the object's uuid
    :param version: the version to get
    '''
    resource_id = toolkit.config.get(u'ckanext.nhm.abyssline_resource_id')

    # figure out the rounded version
    data_dict = {
        u'resource_id': resource_id,
        u'version': version,
    }
    version = toolkit.get_action(u'datastore_get_rounded_version')(_context(), data_dict)

    # search for the record
    search_data_dict = {
        u'resource_id': resource_id,
        u'filters': {
            u'catalogNumber': uuid
        }
    }

    search_result = toolkit.get_action(u'datastore_search')(_context(), search_data_dict)
    try:
        record = search_result[u'records'][0]
    except KeyError:
        pass
    else:
        toolkit.redirect_to(u'record.view', package_name=u'abyssline',
                            resource_id=resource_id,
                            record_id=record[u'_id'], version=version)

    toolkit.abort(404, toolkit._(u'Record not found'))
<|endoftext|>


<commit_before>####################################################################################################
#
# Bleau Database - A database of the bouldering area of Fontainebleau
# Copyright (C) 2015 Fabrice Salvaire
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
####################################################################################################

####################################################################################################

import hashlib

# from bokeh.plotting import figure
from bokeh.embed import components
from bokeh._legacy_charts import Bar

####################################################################################################

from BleauDataBase.Statistics import CircuitStatistics

####################################################################################################

class BokehPlot:

    ##############################################

    def __init__(self, *args, **kwargs):

        script, div = components(*args, **kwargs)
        self.script = script
        self.div = div

####################################################################################################

class CircuitStatisticsData:

    ##############################################

    def __init__(self, circuits):

        self._circuit_statistics = CircuitStatistics(circuits)
        self._circuit_grade_barchart = self._make_barchart(self._circuit_statistics.circuit_grade_histogram,
                                                           'Cotation des Circuits') # Circuit Grade
        self._global_boulder_grade_barchart = self._make_barchart(self._circuit_statistics.global_boulder_grade_histogram,
                                                                  'Cotations des Blocs') # Boulder Grade
        self._boulder_grade_barchart_map = {grade:self._make_barchart(self._circuit_statistics.boulder_grade_histogram(grade),
                                                                      'Cotations des Blocs pour les circuits {}'.format(grade))
                                            for grade in self._circuit_statistics.circuit_grades}

    ##############################################

    def _make_barchart(self, histogram, title):

        grade_counters = histogram.domain()
        y_data = [grade_counter.count for grade_counter in grade_counters]
        x_data = [str(grade_counter) for grade_counter in grade_counters]
        bar = Bar(y_data, x_data, title=title, stacked=True, tools='', width=300, height=200, responsive=True)
        bar.toolbar_location = None

        return BokehPlot(bar)

    ##############################################

    @property
    def circuit_statistics(self):
        return self._circuit_statistics

    @property
    def circuit_grade_barchart(self):
        return self._circuit_grade_barchart

    @property
    def global_boulder_grade_barchart(self):
        return self._global_boulder_grade_barchart

    @property
    def circuit_grades(self):
        return list(self._boulder_grade_barchart_map.keys())

    def boulder_grade_barchart(self, grade):
        return self._boulder_grade_barchart_map[grade]

####################################################################################################

class CircuitStatisticsCache:

    ##############################################

    def __init__(self):

        self._cache = {}

    ##############################################

    def __getitem__(self, circuits):

        ids = [id(circuit) for circuit in circuits]
        id_string = ''.join([str(x) for x in sorted(ids)])
        key = hashlib.sha256(id_string.encode('ascii'))
        if key not in self._cache:
            self._cache[key] = CircuitStatisticsData(circuits)
        return self._cache[key]

####################################################################################################
#
# End
#
####################################################################################################
<commit_msg>make responsive bokeh plot<commit_after>####################################################################################################
#
# Bleau Database - A database of the bouldering area of Fontainebleau
# Copyright (C) 2015 Fabrice Salvaire
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
####################################################################################################

####################################################################################################

import hashlib

# from bokeh.plotting import figure
from bokeh.embed import components
from bokeh._legacy_charts import Bar

####################################################################################################

from BleauDataBase.Statistics import CircuitStatistics

####################################################################################################

class BokehPlot:

    ##############################################

    def __init__(self, *args, **kwargs):

        script, div = components(*args, **kwargs)
        self.script = script
        self.div = div

####################################################################################################

class CircuitStatisticsData:

    ##############################################

    def __init__(self, circuits):

        self._circuit_statistics = CircuitStatistics(circuits)
        self._circuit_grade_barchart = self._make_barchart(self._circuit_statistics.circuit_grade_histogram,
                                                           'Cotation des Circuits') # Circuit Grade
        self._global_boulder_grade_barchart = self._make_barchart(self._circuit_statistics.global_boulder_grade_histogram,
                                                                  'Cotations des Blocs') # Boulder Grade
        self._boulder_grade_barchart_map = {grade:self._make_barchart(self._circuit_statistics.boulder_grade_histogram(grade),
                                                                      'Cotations des Blocs pour les circuits {}'.format(grade))
                                            for grade in self._circuit_statistics.circuit_grades}

    ##############################################

    def _make_barchart(self, histogram, title):

        grade_counters = histogram.domain()
        y_data = [grade_counter.count for grade_counter in grade_counters]
        x_data = [str(grade_counter) for grade_counter in grade_counters]
        bar = Bar(y_data, x_data, title=title, stacked=True, tools='', width=300, height=150, responsive=True)
        bar.toolbar_location = None

        return BokehPlot(bar)

    ##############################################

    @property
    def circuit_statistics(self):
        return self._circuit_statistics

    @property
    def circuit_grade_barchart(self):
        return self._circuit_grade_barchart

    @property
    def global_boulder_grade_barchart(self):
        return self._global_boulder_grade_barchart

    @property
    def circuit_grades(self):
        return list(self._boulder_grade_barchart_map.keys())

    def boulder_grade_barchart(self, grade):
        return self._boulder_grade_barchart_map[grade]

####################################################################################################

class CircuitStatisticsCache:

    ##############################################

    def __init__(self):

        self._cache = {}

    ##############################################

    def __getitem__(self, circuits):

        ids = [id(circuit) for circuit in circuits]
        id_string = ''.join([str(x) for x in sorted(ids)])
        key = hashlib.sha256(id_string.encode('ascii'))
        if key not in self._cache:
            self._cache[key] = CircuitStatisticsData(circuits)
        return self._cache[key]

####################################################################################################
#
# End
#
####################################################################################################
<|endoftext|>


<commit_before>import collections
import itertools
import json

from tabulate import tabulate

from cook.format import format_instance_run_time, format_instance_status, format_job_memory, format_list, format_dict, \
    format_job_attempts, format_job_status
from cook.querying import print_no_data, parse_entity_refs, query_with_stdin_support
from cook.util import millis_to_timedelta, millis_to_date_string, guard_no_cluster

DEFAULT_MAX_RUNTIME = 2 ** 63 - 1

def tabulate_job_instances(instances):
    """Returns a table displaying the instance info"""
    if len(instances) > 0:
        rows = [collections.OrderedDict([('Job Instance', i['task_id']),
                                         ('Run Time', format_instance_run_time(i)),
                                         ('Host', i['hostname']),
                                         ('Instance Status', format_instance_status(i))])
                for i in instances]
        instance_table = tabulate(rows, headers='keys', tablefmt='plain')
        return '\n\n%s' % instance_table
    else:
        return ''

def juxtapose_text(text_a, text_b, buffer_len=15):
    """Places text_a to the left of text_b with a buffer of spaces in between"""
    lines_a = text_a.splitlines()
    lines_b = text_b.splitlines()
    longest_line_length_a = max(map(len, lines_a))
    paired_lines = itertools.zip_longest(lines_a, lines_b, fillvalue="")
    a_columns = longest_line_length_a + buffer_len
    return "\n".join("{0:<{1}}{2}".format(a, a_columns, b) for a, b in paired_lines)

def tabulate_job(cluster_name, job):
    """Given a job, returns a string containing tables for the job and instance fields"""
    job_definition = [['Cluster', cluster_name],
                      ['Pool', job.get('pool', '-')],
                      ['Memory', format_job_memory(job)],
                      ['CPUs', job['cpus']],
                      ['User', job['user']],
                      ['Priority', job['priority']]]
    if job['max_runtime'] != DEFAULT_MAX_RUNTIME:
        job_definition.append(['Max Runtime', millis_to_timedelta(job['max_runtime'])])
    if job['gpus'] > 0:
        job_definition.append(['GPUs', job['gpus']])
    if job['ports'] > 0:
        job_definition.append(['Ports Requested', job['ports']])
    if len(job['constraints']) > 0:
        job_definition.append(['Constraints', format_list(job['constraints'])])
    if len(job['labels']) > 0:
        job_definition.append(['Labels', format_dict(job['labels'])])
    if job['uris'] and len(job['uris']) > 0:
        job_definition.append(['URI(s)', format_list(job['uris'])])
    if 'groups' in job:
        job_definition.append(['Job Group(s)', format_list(job['groups'])])
    if 'application' in job:
        job_definition.append(['Application', '%s (%s)' % (job['application']['name'], job['application']['version'])])
    if 'executor' in job:
        job_definition.append(['Executor', job['executor']])

    job_state = [['Attempts', format_job_attempts(job)],
                 ['Job Status', format_job_status(job)],
                 ['Submitted', millis_to_date_string(job['submit_time'])]]

    job_command = 'Command:\n%s' % job['command']

    if len(job['env']) > 0:
        environment = '\n\nEnvironment:\n%s' % '\n'.join(['%s=%s' % (k, v) for k, v in job['env'].items()])
    else:
        environment = ''

    instances = job['instances']

    job_definition_table = tabulate(job_definition, tablefmt='plain')
    job_state_table = tabulate(job_state, tablefmt='plain')
    job_tables = juxtapose_text(job_definition_table, job_state_table)
    instance_table = tabulate_job_instances(instances)
    return '\n=== Job: %s (%s) ===\n\n%s\n\n%s%s%s' % \
           (job['uuid'], job['name'], job_tables, job_command, environment, instance_table)

def tabulate_instance(cluster_name, instance_job_pair):
    """Given an instance, returns a string containing a table for the instance fields"""
    instance = instance_job_pair[0]
    job = instance_job_pair[1]

    left = [['Cluster', cluster_name],
            ['Host', instance['hostname']],
            ['Slave', instance['slave_id']],
            ['Job', '%s (%s)' % (job['name'], job['uuid'])]]
    if len(instance['ports']) > 0:
        left.append(['Ports Allocated', format_list(instance['ports'])])

    right = [['Run Time', format_instance_run_time(instance)],
             ['Instance Status', format_instance_status(instance)],
             ['Job Status', format_job_status(job)]]
    if 'exit_code' in instance:
        right.append(['Exit Code', instance['exit_code']])

    left_table = tabulate(left, tablefmt='plain')
    right_table = tabulate(right, tablefmt='plain')
    instance_tables = juxtapose_text(left_table, right_table)
    return '\n=== Job Instance: %s ===\n\n%s' % (instance['task_id'], instance_tables)

def tabulate_group(cluster_name, group):
    """Given a group, returns a string containing a table for the group fields"""
    left = [['Cluster', cluster_name]]
    if group['host_placement']['type'] == 'all':
        left.append(['Host Placement', 'all hosts'])
    else:
        left.append(['Host Placement', format_dict(group['host_placement'])])
    if group['straggler_handling']['type'] == 'none':
        left.append(['Straggler Handling', 'none'])
    else:
        left.append(['Straggler Handling', format_dict(group['straggler_handling'])])

    right = [['# Completed', group['completed']],
             ['# Running', group['running']],
             ['# Waiting', group['waiting']]]

    num_jobs = len(group['jobs'])
    jobs = 'Job group contains %s job%s:\n%s' % (num_jobs, '' if num_jobs == 1 else 's', '\n'.join(group['jobs']))

    left_table = tabulate(left, tablefmt='plain')
    right_table = tabulate(right, tablefmt='plain')
    group_tables = juxtapose_text(left_table, right_table)
    return '\n=== Job Group: %s (%s) ===\n\n%s\n\n%s' % (group['uuid'], group['name'], group_tables, jobs)

def show_data(cluster_name, data, format_fn):
    """Iterates over the data collection and formats and prints each element"""
    count = len(data)
    if count > 0:
        tables = [format_fn(cluster_name, datum) for datum in data]
        output = '\n\n'.join(tables)
        print(output)
        print()
    return count

def show(clusters, args, _):
    """Prints info for the jobs / instances / groups with the given UUIDs."""
    guard_no_cluster(clusters)
    as_json = args.get('json')
    entity_refs, _ = parse_entity_refs(clusters, args.get('uuid'))
    query_result, clusters_of_interest = query_with_stdin_support(clusters, entity_refs)
    if as_json:
        print(json.dumps(query_result))
    else:
        for cluster_name, entities in query_result['clusters'].items():
            if 'jobs' in entities:
                show_data(cluster_name, entities['jobs'], tabulate_job)

            if 'instances' in entities:
                show_data(cluster_name, entities['instances'], tabulate_instance)

            if 'groups' in entities:
                show_data(cluster_name, entities['groups'], tabulate_group)

    if query_result['count'] > 0:
        return 0
    else:
        if not as_json:
            print_no_data(clusters_of_interest)
        return 1

def register(add_parser, _):
    """Adds this sub-command's parser and returns the action function"""
    show_parser = add_parser('show', help='show jobs / instances / groups by uuid')
    show_parser.add_argument('uuid', nargs='*')
    show_parser.add_argument('--json', help='show the data in JSON format', dest='json', action='store_true')
    return show
<commit_msg>Replaces "Slave" with "Agent" in cs show output (#1613)<commit_after>import collections
import itertools
import json

from tabulate import tabulate

from cook.format import format_instance_run_time, format_instance_status, format_job_memory, format_list, format_dict, \
    format_job_attempts, format_job_status
from cook.querying import print_no_data, parse_entity_refs, query_with_stdin_support
from cook.util import millis_to_timedelta, millis_to_date_string, guard_no_cluster

DEFAULT_MAX_RUNTIME = 2 ** 63 - 1

def tabulate_job_instances(instances):
    """Returns a table displaying the instance info"""
    if len(instances) > 0:
        rows = [collections.OrderedDict([('Job Instance', i['task_id']),
                                         ('Run Time', format_instance_run_time(i)),
                                         ('Host', i['hostname']),
                                         ('Instance Status', format_instance_status(i))])
                for i in instances]
        instance_table = tabulate(rows, headers='keys', tablefmt='plain')
        return '\n\n%s' % instance_table
    else:
        return ''

def juxtapose_text(text_a, text_b, buffer_len=15):
    """Places text_a to the left of text_b with a buffer of spaces in between"""
    lines_a = text_a.splitlines()
    lines_b = text_b.splitlines()
    longest_line_length_a = max(map(len, lines_a))
    paired_lines = itertools.zip_longest(lines_a, lines_b, fillvalue="")
    a_columns = longest_line_length_a + buffer_len
    return "\n".join("{0:<{1}}{2}".format(a, a_columns, b) for a, b in paired_lines)

def tabulate_job(cluster_name, job):
    """Given a job, returns a string containing tables for the job and instance fields"""
    job_definition = [['Cluster', cluster_name],
                      ['Pool', job.get('pool', '-')],
                      ['Memory', format_job_memory(job)],
                      ['CPUs', job['cpus']],
                      ['User', job['user']],
                      ['Priority', job['priority']]]
    if job['max_runtime'] != DEFAULT_MAX_RUNTIME:
        job_definition.append(['Max Runtime', millis_to_timedelta(job['max_runtime'])])
    if job['gpus'] > 0:
        job_definition.append(['GPUs', job['gpus']])
    if job['ports'] > 0:
        job_definition.append(['Ports Requested', job['ports']])
    if len(job['constraints']) > 0:
        job_definition.append(['Constraints', format_list(job['constraints'])])
    if len(job['labels']) > 0:
        job_definition.append(['Labels', format_dict(job['labels'])])
    if job['uris'] and len(job['uris']) > 0:
        job_definition.append(['URI(s)', format_list(job['uris'])])
    if 'groups' in job:
        job_definition.append(['Job Group(s)', format_list(job['groups'])])
    if 'application' in job:
        job_definition.append(['Application', '%s (%s)' % (job['application']['name'], job['application']['version'])])
    if 'executor' in job:
        job_definition.append(['Executor', job['executor']])

    job_state = [['Attempts', format_job_attempts(job)],
                 ['Job Status', format_job_status(job)],
                 ['Submitted', millis_to_date_string(job['submit_time'])]]

    job_command = 'Command:\n%s' % job['command']

    if len(job['env']) > 0:
        environment = '\n\nEnvironment:\n%s' % '\n'.join(['%s=%s' % (k, v) for k, v in job['env'].items()])
    else:
        environment = ''

    instances = job['instances']

    job_definition_table = tabulate(job_definition, tablefmt='plain')
    job_state_table = tabulate(job_state, tablefmt='plain')
    job_tables = juxtapose_text(job_definition_table, job_state_table)
    instance_table = tabulate_job_instances(instances)
    return '\n=== Job: %s (%s) ===\n\n%s\n\n%s%s%s' % \
           (job['uuid'], job['name'], job_tables, job_command, environment, instance_table)

def tabulate_instance(cluster_name, instance_job_pair):
    """Given an instance, returns a string containing a table for the instance fields"""
    instance = instance_job_pair[0]
    job = instance_job_pair[1]

    left = [['Cluster', cluster_name],
            ['Host', instance['hostname']],
            ['Agent', instance['slave_id']],
            ['Job', '%s (%s)' % (job['name'], job['uuid'])]]
    if len(instance['ports']) > 0:
        left.append(['Ports Allocated', format_list(instance['ports'])])

    right = [['Run Time', format_instance_run_time(instance)],
             ['Instance Status', format_instance_status(instance)],
             ['Job Status', format_job_status(job)]]
    if 'exit_code' in instance:
        right.append(['Exit Code', instance['exit_code']])

    left_table = tabulate(left, tablefmt='plain')
    right_table = tabulate(right, tablefmt='plain')
    instance_tables = juxtapose_text(left_table, right_table)
    return '\n=== Job Instance: %s ===\n\n%s' % (instance['task_id'], instance_tables)

def tabulate_group(cluster_name, group):
    """Given a group, returns a string containing a table for the group fields"""
    left = [['Cluster', cluster_name]]
    if group['host_placement']['type'] == 'all':
        left.append(['Host Placement', 'all hosts'])
    else:
        left.append(['Host Placement', format_dict(group['host_placement'])])
    if group['straggler_handling']['type'] == 'none':
        left.append(['Straggler Handling', 'none'])
    else:
        left.append(['Straggler Handling', format_dict(group['straggler_handling'])])

    right = [['# Completed', group['completed']],
             ['# Running', group['running']],
             ['# Waiting', group['waiting']]]

    num_jobs = len(group['jobs'])
    jobs = 'Job group contains %s job%s:\n%s' % (num_jobs, '' if num_jobs == 1 else 's', '\n'.join(group['jobs']))

    left_table = tabulate(left, tablefmt='plain')
    right_table = tabulate(right, tablefmt='plain')
    group_tables = juxtapose_text(left_table, right_table)
    return '\n=== Job Group: %s (%s) ===\n\n%s\n\n%s' % (group['uuid'], group['name'], group_tables, jobs)

def show_data(cluster_name, data, format_fn):
    """Iterates over the data collection and formats and prints each element"""
    count = len(data)
    if count > 0:
        tables = [format_fn(cluster_name, datum) for datum in data]
        output = '\n\n'.join(tables)
        print(output)
        print()
    return count

def show(clusters, args, _):
    """Prints info for the jobs / instances / groups with the given UUIDs."""
    guard_no_cluster(clusters)
    as_json = args.get('json')
    entity_refs, _ = parse_entity_refs(clusters, args.get('uuid'))
    query_result, clusters_of_interest = query_with_stdin_support(clusters, entity_refs)
    if as_json:
        print(json.dumps(query_result))
    else:
        for cluster_name, entities in query_result['clusters'].items():
            if 'jobs' in entities:
                show_data(cluster_name, entities['jobs'], tabulate_job)

            if 'instances' in entities:
                show_data(cluster_name, entities['instances'], tabulate_instance)

            if 'groups' in entities:
                show_data(cluster_name, entities['groups'], tabulate_group)

    if query_result['count'] > 0:
        return 0
    else:
        if not as_json:
            print_no_data(clusters_of_interest)
        return 1

def register(add_parser, _):
    """Adds this sub-command's parser and returns the action function"""
    show_parser = add_parser('show', help='show jobs / instances / groups by uuid')
    show_parser.add_argument('uuid', nargs='*')
    show_parser.add_argument('--json', help='show the data in JSON format', dest='json', action='store_true')
    return show
<|endoftext|>


<commit_before><commit_msg>doubtful video interface<commit_after>from myhdl import Signal, intbv, always

class VideoIntf:

    color_depth = (8, 8, 8)

    def __init__(self, color_depth = (8, 8, 8)):

        self.color_depth = color_depth

        # RGB data from video source
        self.red = Signal(intbv(0)[color_depth[0]:])
        self.green = Signal(intbv(0)[color_depth[1]:])
        self.blue = Signal(intbv(0)[color_depth[2]:])

        # Video timing controls
        self.vsync = Signal(bool(0))
        self.hsync = Signal(bool(0))
        self.vde = Signal(bool(0))

    # Transactors for passing signals to video interface

    COLOR_BARS = (
            # r, g, b
            (1, 1, 1,),  # white
            (1, 1, 0,),  # yellow
            (0, 1, 1,),  # cyan
            (0, 1, 0,),  # green
            (1, 0, 1,),  # magenta
            (1, 0, 0,),  # red
            (0, 0, 1,),  # blue
            (0, 0, 0,),  # black
        )

    hpxl = 0
    vpxl = 0

    def write_video(self, clk, resolution = (640, 480)):

        # Generate color bars and sends it through the video interface
        @always(clk.posedge)
        def write():
            bar_index = self.hpxl // (resolution[0] // len(self.COLOR_BARS))
            color_max_value = map(lambda x : 2**x - 1, self.color_depth)
            pixel_value = [a*b for a,b in zip(color_max_value, self.COLOR_BARS[bar_index])]
            print(pixel_value)
            # setting the pixel xolor values
            self.red.next = Signal(intbv(pixel_value[0])[self.color_depth[0]:])
            self.green.next = Signal(intbv(pixel_value[1])[self.color_depth[1]:])
            self.blue.next = Signal(intbv(pixel_value[2])[self.color_depth[2]:])

            self.hpxl += 1
            if self.hpxl >= resolution[0]:
                self.hsync.next = Signal(bool(1))
                self.hpxl = 0
                self.vpxl += 1
            else:
                self.hsync.next = Signal(bool(0))
            if self.vpxl >= resolution[1]:
                self.vsync.next = Signal(bool(1))
                self.vpxl = 0
            else:
                self.vsync.next = Signal(bool(0))

        return write

    def reset_counter(self):

        self.hpxl = 0
        self.vpxl = 0
<|endoftext|>


<commit_before>import re
import pprint

OP = 'OP'       # operator
VAR = 'VAR'     # variable
L_PAR = "L_PAR" # left parenthesis
R_PAR = "R_PAR" # right parenthesis

def tokenizer(raw_str):
    pos = 0
    while pos < len(raw_str):
        if re.match(r'[+*/]', raw_str[pos]):
            yield (OP, raw_str[pos])
        elif re.match(r'\w', raw_str[pos]):
            variable = raw_str[pos]
            var_pos = pos + 1
            while var_pos < len(raw_str) and re.match(r'[\w]', raw_str[var_pos]):
                variable += raw_str[var_pos]
                var_pos += 1
            pos = var_pos - 1
            yield (VAR, variable)
        elif raw_str[pos] == ')':
            yield (R_PAR, raw_str[pos])
        elif raw_str[pos] == '(':
            yield (L_PAR, raw_str[pos])
        else:
            print("Error: invalid token detected: ",raw_str[pos])
            exit()
        pos += 1

def populateVectors(variable_names):
    dict = {}
    length = len(variable_names)
    for i in range(length):
        vect = []
        for j in range(2**i):
            vect += [0 for num in range(2**(length-(i+1)))]
            vect += [1 for num in range(2**(length-(i+1)))]
        dict[variable_names[i]] = vect
    return dict

def perform_operation(op, *args):
    result = []
    if len(args) == 2: # binary operation
        vect1, vect2 = args
        assert len(vect1) == len(vect2)
        for index in range(len(vect1)):
            result.append(op(vect1[index],vect2[index]))
    elif len(args) == 1: # unary operation
        vect = args[0]
        for index in range(len(vect)):
            result.append(op(vect[index]))
    else: #invalid operation
        raise RuntimeError
    return result

tokens = [token for token in tokenizer(input("> "))]
variable_names = {token[1] for token in tokens if token[0] == VAR}
variable_vectors = populateVectors(list(variable_names))
print(tokens)
print(variable_names)
pprint.pprint(variable_vectors)
<commit_msg>added shunting yard algorithm<commit_after>import re
import collections

def tokenizer(raw_str):
    pos = 0
    while pos < len(raw_str):
        if re.match(r'[+*/]', raw_str[pos]):
            yield ("OP", raw_str[pos])
        elif re.match(r'\w', raw_str[pos]):
            variable = raw_str[pos]
            var_pos = pos + 1
            while var_pos < len(raw_str) and re.match(r'[\w]', raw_str[var_pos]):
                variable += raw_str[var_pos]
                var_pos += 1
            pos = var_pos - 1
            yield ("VAR", variable)
        elif raw_str[pos] == ')':
            yield (')', raw_str[pos])
        elif raw_str[pos] == '(':
            yield ('(', raw_str[pos])
        else:
            print("Error: invalid character detected:", raw_str[pos])
            exit()
        pos += 1

def populateVectors(variable_names):
    dict = collections.OrderedDict()
    length = len(variable_names)
    for i in range(length):
        vect = []
        for j in range(2**i):
            vect += [0 for num in range(2**(length-(i+1)))]
            vect += [1 for num in range(2**(length-(i+1)))]
        dict[variable_names[i]] = vect
    return dict

def perform_operation(op, *args):
    result = []
    if len(args) == 2: # binary operation
        vect1, vect2 = args
        assert len(vect1) == len(vect2)
        for index in range(len(vect1)):
            result.append(op(vect1[index],vect2[index]))
    elif len(args) == 1: # unary operation
        vect = args[0]
        for index in range(len(vect)):
            result.append(op(vect[index]))
    else: #invalid operation
        raise RuntimeError
    return result

def _cmpPrecedence(lhs,rhs):
    precedence = {'+' : 0 , '*' : 1, '/': 2}
    return precedence[lhs] - precedence[rhs]

# TODO:: fix the shunting yard
def to_RPN(tokens):
    RPN_tokens = []
    opStack =[]
    for tokenType, tokenVal in tokens:
        if tokenType == "VAR":
            RPN_tokens.append((tokenType, tokenVal))
        elif tokenType == "OP":
            while len(opStack) > 0:
                topType, topVal = opStack[-1]
                if topType == '(' or topType == ')': break
                if topVal != '/' and _cmpPrecedence(tokenVal,topVal) <= 0:
                    RPN_tokens.append((topType,topVal))
                    opStack.pop()
                else: break
            opStack.append((tokenType, tokenVal))
        elif tokenType == '(':
            RPN_tokens.append((tokenType, tokenVal))
        elif tokenType == ')':
            while len(opStack) > 0:
                topType, topVal = opStack[-1]
                if tokenType == '(': break
                RPN_tokens.append((topType,topVal))
                opStack.pop()
            try:
                topType, topVal = opStack[-1]
                if topType == '(': raise RuntimeError
            except:
                print("mismatch in parenthesis1")
                exit()
            opStack.pop()
    while len(opStack) > 0:
        topType, topVal = opStack[-1]
        if topType == ')' or topType == '(':
            print("mismatch in parenthesis2")
            exit()
        RPN_tokens.append((topType, topVal))
        opStack.pop()
    return RPN_tokens

tokens = [token for token in tokenizer(input("> "))]
seen = set()
variable_names = [x for x in tokens if not (x in seen or seen.add(x) or x[0] != 'VAR')]
variable_vectors = populateVectors(variable_names)
print(tokens)
print(to_RPN(tokens))
<|endoftext|>


<commit_before>source("../../shared/qtcreator.py")
import __builtin__

cppEditorStr = ":Qt Creator_CppEditor::Internal::CPPEditorWidget"
originalSources = os.path.abspath(os.path.join(os.getcwd(), "..", "shared", "simplePlainCPP"))

def init():
    global homeShortCut, endShortCut
    if platform.system() == "Darwin":
        homeShortCut = "<Ctrl+Left>"
        endShortCut = "<End>"
    else:
        homeShortCut = "<Home>"
        endShortCut = "<Ctrl+End>"

def main():
    global cppEditorStr
    folder = prepareTemplate(originalSources)
    if folder == None:
        test.fatal("Could not prepare test files - leaving test")
        return
    proFile = os.path.join(folder, "testfiles.pro")
    startApplication("qtcreator" + SettingsPath)
    if not startedWithoutPluginError():
        return
    openQmakeProject(proFile)
    fileModifications = {"testfiles.testfiles\\.pro":__modifyProFile__,
                         "testfiles.Headers.testfile\\.h":__modifyHeader__,
                         "testfiles.Sources.testfile\\.cpp":__modifySource__,
                         "testfiles.Sources.main\\.cpp":None}
    for fileName, modification in fileModifications.iteritems():
        __modifyFile__(fileName, modification)
    test.log("Reverting all files...")
    fileModifications = dict(zip(fileModifications.keys(),
                                 (__builtin__.bool(v) for v in fileModifications.values())))
    revertChanges(fileModifications)
    invokeMenuItem("File", "Exit")

def __modifyFile__(fileName, modificationFunc):
    simpleFName = simpleFileName(fileName)
    test.log("Opening file '%s'" % simpleFName)
    openDocument(fileName)
    if modificationFunc:
        test.log("Modifying file '%s'" % simpleFName)
        modificationFunc()
    else:
        test.log("Leaving file '%s' unmodified." % simpleFName)

# add some stuff to pro file
def __modifyProFile__():
    proEditorStr = ":Qt Creator_ProFileEditorWidget"
    addConfig = ["", "CONFIG += thread", "",
                 "lessThan(QT_VER_MAJ, 4) | lessThan(QT_VER_MIN, 7) {",
                 "    error(Qt 4.7 or newer is required but version $$[QT_VERSION] was detected.)",
                 "}"]
    addFile = [" \\", "    not_existing.cpp"]
    if placeCursorToLine(proEditorStr, "CONFIG -= qt"):
        typeLines(proEditorStr, addConfig)
    if placeCursorToLine(proEditorStr, "testfile.cpp"):
        typeLines(proEditorStr, addFile)

# re-order some stuff inside header
def __modifyHeader__():
    global cppEditorStr, homeShortCut, endShortCut
    if placeCursorToLine(cppEditorStr, "class.+", True):
        type(cppEditorStr, homeShortCut)
        markText(cppEditorStr, "Down", 5)
        invokeMenuItem("Edit", "Cut")
        type(cppEditorStr, endShortCut)
        type(cppEditorStr, "<Return>")
        invokeMenuItem("Edit", "Paste")

# remove some stuff from source
def __modifySource__():
    global cppEditorStr, homeShortCut
    if placeCursorToLine(cppEditorStr, "void function1(int a);"):
        type(cppEditorStr, homeShortCut)
        markText(cppEditorStr, "Down")
        type(cppEditorStr, "<Delete>")
    if placeCursorToLine(cppEditorStr, "bool function1(int a) {"):
        type(cppEditorStr, homeShortCut)
        markText(cppEditorStr, "Down", 4)
        type(cppEditorStr, "<Delete>")

def revertChanges(files):
    for f,canRevert in files.iteritems():
        simpleName = simpleFileName(f)
        test.log("Trying to revert changes for '%s'" % simpleName)
        if openDocument(f):
            fileMenu = findObject("{name='QtCreator.Menu.File' title='File' type='QMenu' "
                                  "window=':Qt Creator_Core::Internal::MainWindow'}")
            for menuItem in object.children(fileMenu):
                if str(menuItem.text) == 'Revert "%s" to Saved' % simpleName:
                    if (test.compare(canRevert, menuItem.enabled, "Verifying whether MenuItem "
                                     "'Revert to Saved' has expected state (%s)"
                                     % str(canRevert)) and canRevert):
                        invokeMenuItem('File', 'Revert "%s" to Saved' % simpleName)
                        clickButton(waitForObject(":Revert to Saved.Proceed_QPushButton"))
                        compareFileToOriginal(simpleName)
                        test.log("Reverted changes inside %s" % simpleName)
        else:
            test.fail("Could not open %s for reverting changes" % simpleName)

def compareFileToOriginal(fileName):
    global originalSources
    currentContent = str(waitForObject(getEditorForFileSuffix(fileName)).plainText)
    origFile = open(os.path.join(originalSources, fileName), "r")
    originalContent = origFile.read()
    origFile.close()
    test.compare(originalContent, currentContent,
                 "Comparing original to reverted file content for '%s'" % fileName)

def simpleFileName(navigatorFileName):
    return ".".join(navigatorFileName.split(".")[-2:]).replace("\\","")
<commit_msg>Squish: Add Mac workaround to tst_revert_changes<commit_after>source("../../shared/qtcreator.py")
import __builtin__

cppEditorStr = ":Qt Creator_CppEditor::Internal::CPPEditorWidget"
originalSources = os.path.abspath(os.path.join(os.getcwd(), "..", "shared", "simplePlainCPP"))

def init():
    global homeShortCut, endShortCut
    if platform.system() == "Darwin":
        homeShortCut = "<Ctrl+Left>"
        endShortCut = "<End>"
    else:
        homeShortCut = "<Home>"
        endShortCut = "<Ctrl+End>"

def main():
    global cppEditorStr
    folder = prepareTemplate(originalSources)
    if folder == None:
        test.fatal("Could not prepare test files - leaving test")
        return
    proFile = os.path.join(folder, "testfiles.pro")
    startApplication("qtcreator" + SettingsPath)
    if not startedWithoutPluginError():
        return
    openQmakeProject(proFile)
    fileModifications = {"testfiles.testfiles\\.pro":__modifyProFile__,
                         "testfiles.Headers.testfile\\.h":__modifyHeader__,
                         "testfiles.Sources.testfile\\.cpp":__modifySource__,
                         "testfiles.Sources.main\\.cpp":None}
    for fileName, modification in fileModifications.iteritems():
        __modifyFile__(fileName, modification)
    test.log("Reverting all files...")
    fileModifications = dict(zip(fileModifications.keys(),
                                 (__builtin__.bool(v) for v in fileModifications.values())))
    revertChanges(fileModifications)
    invokeMenuItem("File", "Exit")

def __modifyFile__(fileName, modificationFunc):
    simpleFName = simpleFileName(fileName)
    test.log("Opening file '%s'" % simpleFName)
    openDocument(fileName)
    if modificationFunc:
        test.log("Modifying file '%s'" % simpleFName)
        modificationFunc()
    else:
        test.log("Leaving file '%s' unmodified." % simpleFName)

# add some stuff to pro file
def __modifyProFile__():
    proEditorStr = ":Qt Creator_ProFileEditorWidget"
    addConfig = ["", "CONFIG += thread", "",
                 "lessThan(QT_VER_MAJ, 4) | lessThan(QT_VER_MIN, 7) {",
                 "    error(Qt 4.7 or newer is required but version $$[QT_VERSION] was detected.)",
                 "}"]
    addFile = [" \\", "    not_existing.cpp"]
    if placeCursorToLine(proEditorStr, "CONFIG -= qt"):
        typeLines(proEditorStr, addConfig)
    if placeCursorToLine(proEditorStr, "testfile.cpp"):
        typeLines(proEditorStr, addFile)

# re-order some stuff inside header
def __modifyHeader__():
    global cppEditorStr, homeShortCut, endShortCut
    if platform.system() == "Darwin":
        JIRA.performWorkaroundIfStillOpen(6918, JIRA.Bug.CREATOR, waitForObject(cppEditorStr, 1000))
    if placeCursorToLine(cppEditorStr, "class.+", True):
        type(cppEditorStr, homeShortCut)
        markText(cppEditorStr, "Down", 5)
        invokeMenuItem("Edit", "Cut")
        type(cppEditorStr, endShortCut)
        type(cppEditorStr, "<Return>")
        invokeMenuItem("Edit", "Paste")

# remove some stuff from source
def __modifySource__():
    global cppEditorStr, homeShortCut
    if placeCursorToLine(cppEditorStr, "void function1(int a);"):
        type(cppEditorStr, homeShortCut)
        markText(cppEditorStr, "Down")
        type(cppEditorStr, "<Delete>")
    if placeCursorToLine(cppEditorStr, "bool function1(int a) {"):
        type(cppEditorStr, homeShortCut)
        markText(cppEditorStr, "Down", 4)
        type(cppEditorStr, "<Delete>")

def revertChanges(files):
    for f,canRevert in files.iteritems():
        simpleName = simpleFileName(f)
        test.log("Trying to revert changes for '%s'" % simpleName)
        if openDocument(f):
            fileMenu = findObject("{name='QtCreator.Menu.File' title='File' type='QMenu' "
                                  "window=':Qt Creator_Core::Internal::MainWindow'}")
            for menuItem in object.children(fileMenu):
                if str(menuItem.text) == 'Revert "%s" to Saved' % simpleName:
                    if (test.compare(canRevert, menuItem.enabled, "Verifying whether MenuItem "
                                     "'Revert to Saved' has expected state (%s)"
                                     % str(canRevert)) and canRevert):
                        invokeMenuItem('File', 'Revert "%s" to Saved' % simpleName)
                        clickButton(waitForObject(":Revert to Saved.Proceed_QPushButton"))
                        compareFileToOriginal(simpleName)
                        test.log("Reverted changes inside %s" % simpleName)
        else:
            test.fail("Could not open %s for reverting changes" % simpleName)

def compareFileToOriginal(fileName):
    global originalSources
    currentContent = str(waitForObject(getEditorForFileSuffix(fileName)).plainText)
    origFile = open(os.path.join(originalSources, fileName), "r")
    originalContent = origFile.read()
    origFile.close()
    test.compare(originalContent, currentContent,
                 "Comparing original to reverted file content for '%s'" % fileName)

def simpleFileName(navigatorFileName):
    return ".".join(navigatorFileName.split(".")[-2:]).replace("\\","")
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import itertools

from sqlparse import sql
from sqlparse import tokens as T
from sqlparse.utils import recurse, imt, find_matching

M_ROLE = (T.Keyword, ('null', 'role'))
M_SEMICOLON = (T.Punctuation, ';')

T_NUMERICAL = (T.Number, T.Number.Integer, T.Number.Float)
T_STRING = (T.String, T.String.Single, T.String.Symbol)
T_NAME = (T.Name, T.Name.Placeholder)

def _group_left_right(tlist, m, cls,
                      valid_left=lambda t: t is not None,
                      valid_right=lambda t: t is not None,
                      semicolon=False):
    """Groups together tokens that are joined by a middle token. ie. x < y"""
    [_group_left_right(sgroup, m, cls, valid_left, valid_right, semicolon)
     for sgroup in tlist.get_sublists() if not isinstance(sgroup, cls)]

    token = tlist.token_next_by(m=m)
    while token:
        left, right = tlist.token_prev(token), tlist.token_next(token)

        if valid_left(left) and valid_right(right):
            if semicolon:
                sright = tlist.token_next_by(m=M_SEMICOLON, idx=right)
                right = sright or right  # only overwrite if a semicolon present.
            tokens = tlist.tokens_between(left, right)
            token = tlist.group_tokens(cls, tokens, extend=True)
        token = tlist.token_next_by(m=m, idx=token)

def _group_matching(tlist, cls):
    """Groups Tokens that have beginning and end. ie. parenthesis, brackets.."""
    idx = 1 if imt(tlist, i=cls) else 0

    token = tlist.token_next_by(m=cls.M_OPEN, idx=idx)
    while token:
        end = find_matching(tlist, token, cls.M_OPEN, cls.M_CLOSE)
        if end is not None:
            token = tlist.group_tokens(cls, tlist.tokens_between(token, end))
            _group_matching(token, cls)
        token = tlist.token_next_by(m=cls.M_OPEN, idx=token)

def group_if(tlist):
    _group_matching(tlist, sql.If)

def group_for(tlist):
    _group_matching(tlist, sql.For)

def group_foreach(tlist):
    _group_matching(tlist, sql.For)

def group_begin(tlist):
    _group_matching(tlist, sql.Begin)

def group_as(tlist):
    lfunc = lambda tk: not imt(tk, t=T.Keyword) or tk.value == 'NULL'
    rfunc = lambda tk: not imt(tk, t=(T.DML, T.DDL))
    _group_left_right(tlist, (T.Keyword, 'AS'), sql.Identifier,
                      valid_left=lfunc, valid_right=rfunc)

def group_assignment(tlist):
    _group_left_right(tlist, (T.Assignment, ':='), sql.Assignment,
                      semicolon=True)

def group_comparison(tlist):
    I_COMPERABLE = (sql.Parenthesis, sql.Function, sql.Identifier)
    T_COMPERABLE = T_NUMERICAL + T_STRING + T_NAME

    func = lambda tk: imt(tk, t=T_COMPERABLE, i=I_COMPERABLE) or (
        imt(tk, t=T.Keyword) and tk.value.upper() == 'NULL')

    _group_left_right(tlist, (T.Operator.Comparison, None), sql.Comparison,
                      valid_left=func, valid_right=func)

def group_case(tlist):
    _group_matching(tlist, sql.Case)

def group_identifier(tlist):
    def _consume_cycle(tl, i):
        # TODO: Usage of Wildcard token is ambivalent here.
        x = itertools.cycle((
            lambda y: (y.match(T.Punctuation, '.')
                       or y.ttype in (T.Operator,
                                      T.Wildcard,
                                      T.Name)
                       or isinstance(y, sql.SquareBrackets)),
            lambda y: (y.ttype in (T.String.Symbol,
                                   T.Name,
                                   T.Wildcard,
                                   T.Literal.String.Single,
                                   T.Literal.Number.Integer,
                                   T.Literal.Number.Float)
                       or isinstance(y, (sql.Parenthesis,
                                         sql.SquareBrackets,
                                         sql.Function)))))
        for t in tl.tokens[i:]:
            # Don't take whitespaces into account.
            if t.ttype is T.Whitespace:
                yield t
                continue
            if next(x)(t):
                yield t
            else:
                if isinstance(t, sql.Comment) and t.is_multiline():
                    yield t
                if t.ttype is T.Keyword.Order:
                    yield t
                return

    def _next_token(tl, i):
        # chooses the next token. if two tokens are found then the
        # first is returned.
        t1 = tl.token_next_by_type(
            i, (T.String.Symbol, T.Name, T.Literal.Number.Integer,
                T.Literal.Number.Float))

        i1 = tl.token_index(t1, start=i) if t1 else None
        t2_end = None if i1 is None else i1 + 1
        t2 = tl.token_next_by_instance(i, (sql.Function, sql.Parenthesis),
                                       end=t2_end)

        if t1 and t2:
            i2 = tl.token_index(t2, start=i)
            if i1 > i2:
                return t2
            else:
                return t1
        elif t1:
            return t1
        else:
            return t2

    # bottom up approach: group subgroups first
    [group_identifier(sgroup) for sgroup in tlist.get_sublists()
     if not isinstance(sgroup, sql.Identifier)]

    # real processing
    idx = 0
    token = _next_token(tlist, idx)
    while token:
        identifier_tokens = [token] + list(
            _consume_cycle(tlist,
                           tlist.token_index(token, start=idx) + 1))
        # remove trailing whitespace
        if identifier_tokens and identifier_tokens[-1].ttype is T.Whitespace:
            identifier_tokens = identifier_tokens[:-1]
        if not (len(identifier_tokens) == 1
                and (isinstance(identifier_tokens[0], (sql.Function,
                                                       sql.Parenthesis))
                     or identifier_tokens[0].ttype in (
                    T.Literal.Number.Integer, T.Literal.Number.Float))):
            group = tlist.group_tokens(sql.Identifier, identifier_tokens)
            idx = tlist.token_index(group, start=idx) + 1
        else:
            idx += 1
        token = _next_token(tlist, idx)

@recurse(sql.IdentifierList)
def group_identifier_list(tlist):
    # Allowed list items
    fend1_funcs = [lambda t: isinstance(t, (sql.Identifier, sql.Function,
                                            sql.Case)),
                   lambda t: t.is_whitespace(),
                   lambda t: t.ttype == T.Name,
                   lambda t: t.ttype == T.Wildcard,
                   lambda t: t.match(T.Keyword, 'null'),
                   lambda t: t.match(T.Keyword, 'role'),
                   lambda t: t.ttype == T.Number.Integer,
                   lambda t: t.ttype == T.String.Single,
                   lambda t: t.ttype == T.Name.Placeholder,
                   lambda t: t.ttype == T.Keyword,
                   lambda t: isinstance(t, sql.Comparison),
                   lambda t: isinstance(t, sql.Comment),
                   lambda t: t.ttype == T.Comment.Multiline,
                   ]
    tcomma = tlist.token_next_match(0, T.Punctuation, ',')
    start = None
    while tcomma is not None:
        # Go back one idx to make sure to find the correct tcomma
        idx = tlist.token_index(tcomma)
        before = tlist.token_prev(idx)
        after = tlist.token_next(idx)
        # Check if the tokens around tcomma belong to a list
        bpassed = apassed = False
        for func in fend1_funcs:
            if before is not None and func(before):
                bpassed = True
            if after is not None and func(after):
                apassed = True
        if not bpassed or not apassed:
            # Something's wrong here, skip ahead to next ","
            start = None
            tcomma = tlist.token_next_match(idx + 1,
                                            T.Punctuation, ',')
        else:
            if start is None:
                start = before
            after_idx = tlist.token_index(after, start=idx)
            next_ = tlist.token_next(after_idx)
            if next_ is None or not next_.match(T.Punctuation, ','):
                # Reached the end of the list
                tokens = tlist.tokens_between(start, after)
                group = tlist.group_tokens(sql.IdentifierList, tokens)
                start = None
                tcomma = tlist.token_next_match(tlist.token_index(group) + 1,
                                                T.Punctuation, ',')
            else:
                tcomma = next_

def group_brackets(tlist):
    _group_matching(tlist, sql.SquareBrackets)

def group_parenthesis(tlist):
    _group_matching(tlist, sql.Parenthesis)

@recurse(sql.Comment)
def group_comments(tlist):
    idx = 0
    token = tlist.token_next_by_type(idx, T.Comment)
    while token:
        tidx = tlist.token_index(token)
        end = tlist.token_not_matching(tidx + 1,
                                       [lambda t: t.ttype in T.Comment,
                                        lambda t: t.is_whitespace()])
        if end is None:
            idx = tidx + 1
        else:
            eidx = tlist.token_index(end)
            grp_tokens = tlist.tokens_between(token,
                                              tlist.token_prev(eidx, False))
            group = tlist.group_tokens(sql.Comment, grp_tokens)
            idx = tlist.token_index(group)
        token = tlist.token_next_by_type(idx, T.Comment)

@recurse(sql.Where)
def group_where(tlist):
    token = tlist.token_next_by(m=sql.Where.M_OPEN)
    while token:
        end = tlist.token_next_by(m=sql.Where.M_CLOSE, idx=token)

        if end is None:
            tokens = tlist.tokens_between(token, tlist._groupable_tokens[-1])
        else:
            tokens = tlist.tokens_between(
                token, tlist.tokens[tlist.token_index(end) - 1])

        token = tlist.group_tokens(sql.Where, tokens)
        token = tlist.token_next_by(m=sql.Where.M_OPEN, idx=token)

@recurse(sql.Identifier, sql.Function, sql.Case)
def group_aliased(tlist):
    clss = (sql.Identifier, sql.Function, sql.Case)
    idx = 0
    token = tlist.token_next_by_instance(idx, clss)
    while token:
        next_ = tlist.token_next(tlist.token_index(token))
        if next_ is not None and isinstance(next_, clss):
            if not next_.value.upper().startswith('VARCHAR'):
                grp = tlist.tokens_between(token, next_)[1:]
                token.tokens.extend(grp)
                for t in grp:
                    tlist.tokens.remove(t)
        idx = tlist.token_index(token) + 1
        token = tlist.token_next_by_instance(idx, clss)

def group_typecasts(tlist):
    _group_left_right(tlist, (T.Punctuation, '::'), sql.Identifier)

@recurse(sql.Function)
def group_functions(tlist):
    token = tlist.token_next_by(t=T.Name)
    while token:
        next_ = tlist.token_next(token)
        if imt(next_, i=sql.Parenthesis):
            tokens = tlist.tokens_between(token, next_)
            token = tlist.group_tokens(sql.Function, tokens)
        token = tlist.token_next_by(t=T.Name, idx=token)

def group_order(tlist):
    """Group together Identifier and Asc/Desc token"""
    token = tlist.token_next_by(t=T.Keyword.Order)
    while token:
        prev = tlist.token_prev(token)
        if imt(prev, i=sql.Identifier, t=T.Number):
            tokens = tlist.tokens_between(prev, token)
            token = tlist.group_tokens(sql.Identifier, tokens)
        token = tlist.token_next_by(t=T.Keyword.Order, idx=token)

@recurse()
def align_comments(tlist):
    token = tlist.token_next_by(i=sql.Comment)
    while token:
        before = tlist.token_prev(token)
        if isinstance(before, sql.TokenList):
            tokens = tlist.tokens_between(before, token)
            token = tlist.group_tokens(sql.TokenList, tokens, extend=True)
        token = tlist.token_next_by(i=sql.Comment, idx=token)

def group(tlist):
    for func in [
        group_comments,
        group_brackets,
        group_parenthesis,
        group_functions,
        group_where,
        group_case,
        group_identifier,
        group_order,
        group_typecasts,
        group_as,
        group_aliased,
        group_assignment,
        group_comparison,
        align_comments,
        group_identifier_list,
        group_if,
        group_for,
        group_foreach,
        group_begin,
    ]:
        func(tlist)
<commit_msg>refactor identifier list and alias grouping<commit_after># -*- coding: utf-8 -*-

import itertools

from sqlparse import sql
from sqlparse import tokens as T
from sqlparse.utils import recurse, imt, find_matching

M_ROLE = (T.Keyword, ('null', 'role'))
M_SEMICOLON = (T.Punctuation, ';')
M_COMMA = (T.Punctuation, ',')

T_NUMERICAL = (T.Number, T.Number.Integer, T.Number.Float)
T_STRING = (T.String, T.String.Single, T.String.Symbol)
T_NAME = (T.Name, T.Name.Placeholder)

def _group_left_right(tlist, m, cls,
                      valid_left=lambda t: t is not None,
                      valid_right=lambda t: t is not None,
                      semicolon=False):
    """Groups together tokens that are joined by a middle token. ie. x < y"""
    [_group_left_right(sgroup, m, cls, valid_left, valid_right, semicolon)
     for sgroup in tlist.get_sublists() if not isinstance(sgroup, cls)]

    token = tlist.token_next_by(m=m)
    while token:
        left, right = tlist.token_prev(token), tlist.token_next(token)

        if valid_left(left) and valid_right(right):
            if semicolon:
                sright = tlist.token_next_by(m=M_SEMICOLON, idx=right)
                right = sright or right  # only overwrite if a semicolon present.
            tokens = tlist.tokens_between(left, right)
            token = tlist.group_tokens(cls, tokens, extend=True)
        token = tlist.token_next_by(m=m, idx=token)

def _group_matching(tlist, cls):
    """Groups Tokens that have beginning and end. ie. parenthesis, brackets.."""
    idx = 1 if imt(tlist, i=cls) else 0

    token = tlist.token_next_by(m=cls.M_OPEN, idx=idx)
    while token:
        end = find_matching(tlist, token, cls.M_OPEN, cls.M_CLOSE)
        if end is not None:
            token = tlist.group_tokens(cls, tlist.tokens_between(token, end))
            _group_matching(token, cls)
        token = tlist.token_next_by(m=cls.M_OPEN, idx=token)

def group_if(tlist):
    _group_matching(tlist, sql.If)

def group_for(tlist):
    _group_matching(tlist, sql.For)

def group_foreach(tlist):
    _group_matching(tlist, sql.For)

def group_begin(tlist):
    _group_matching(tlist, sql.Begin)

def group_as(tlist):
    lfunc = lambda tk: not imt(tk, t=T.Keyword) or tk.value == 'NULL'
    rfunc = lambda tk: not imt(tk, t=(T.DML, T.DDL))
    _group_left_right(tlist, (T.Keyword, 'AS'), sql.Identifier,
                      valid_left=lfunc, valid_right=rfunc)

def group_assignment(tlist):
    _group_left_right(tlist, (T.Assignment, ':='), sql.Assignment,
                      semicolon=True)

def group_comparison(tlist):
    I_COMPERABLE = (sql.Parenthesis, sql.Function, sql.Identifier)
    T_COMPERABLE = T_NUMERICAL + T_STRING + T_NAME

    func = lambda tk: imt(tk, t=T_COMPERABLE, i=I_COMPERABLE) or (
        imt(tk, t=T.Keyword) and tk.value.upper() == 'NULL')

    _group_left_right(tlist, (T.Operator.Comparison, None), sql.Comparison,
                      valid_left=func, valid_right=func)

def group_case(tlist):
    _group_matching(tlist, sql.Case)

def group_identifier(tlist):
    def _consume_cycle(tl, i):
        # TODO: Usage of Wildcard token is ambivalent here.
        x = itertools.cycle((
            lambda y: (y.match(T.Punctuation, '.')
                       or y.ttype in (T.Operator,
                                      T.Wildcard,
                                      T.Name)
                       or isinstance(y, sql.SquareBrackets)),
            lambda y: (y.ttype in (T.String.Symbol,
                                   T.Name,
                                   T.Wildcard,
                                   T.Literal.String.Single,
                                   T.Literal.Number.Integer,
                                   T.Literal.Number.Float)
                       or isinstance(y, (sql.Parenthesis,
                                         sql.SquareBrackets,
                                         sql.Function)))))
        for t in tl.tokens[i:]:
            # Don't take whitespaces into account.
            if t.ttype is T.Whitespace:
                yield t
                continue
            if next(x)(t):
                yield t
            else:
                if isinstance(t, sql.Comment) and t.is_multiline():
                    yield t
                if t.ttype is T.Keyword.Order:
                    yield t
                return

    def _next_token(tl, i):
        # chooses the next token. if two tokens are found then the
        # first is returned.
        t1 = tl.token_next_by_type(
            i, (T.String.Symbol, T.Name, T.Literal.Number.Integer,
                T.Literal.Number.Float))

        i1 = tl.token_index(t1, start=i) if t1 else None
        t2_end = None if i1 is None else i1 + 1
        t2 = tl.token_next_by_instance(i, (sql.Function, sql.Parenthesis),
                                       end=t2_end)

        if t1 and t2:
            i2 = tl.token_index(t2, start=i)
            if i1 > i2:
                return t2
            else:
                return t1
        elif t1:
            return t1
        else:
            return t2

    # bottom up approach: group subgroups first
    [group_identifier(sgroup) for sgroup in tlist.get_sublists()
     if not isinstance(sgroup, sql.Identifier)]

    # real processing
    idx = 0
    token = _next_token(tlist, idx)
    while token:
        identifier_tokens = [token] + list(
            _consume_cycle(tlist,
                           tlist.token_index(token, start=idx) + 1))
        # remove trailing whitespace
        if identifier_tokens and identifier_tokens[-1].ttype is T.Whitespace:
            identifier_tokens = identifier_tokens[:-1]
        if not (len(identifier_tokens) == 1
                and (isinstance(identifier_tokens[0], (sql.Function,
                                                       sql.Parenthesis))
                     or identifier_tokens[0].ttype in (
                    T.Literal.Number.Integer, T.Literal.Number.Float))):
            group = tlist.group_tokens(sql.Identifier, identifier_tokens)
            idx = tlist.token_index(group, start=idx) + 1
        else:
            idx += 1
        token = _next_token(tlist, idx)

@recurse(sql.IdentifierList)
def group_identifier_list(tlist):
    I_IDENT_LIST = (sql.Function, sql.Case, sql.Identifier, sql.Comparison,
                    sql.IdentifierList)  # sql.Operation
    T_IDENT_LIST = (T_NUMERICAL + T_STRING + T_NAME +
                    (T.Keyword, T.Comment, T.Wildcard))

    func = lambda t: imt(t, i=I_IDENT_LIST, m=M_ROLE, t=T_IDENT_LIST)
    token = tlist.token_next_by(m=M_COMMA)

    while token:
        before, after = tlist.token_prev(token), tlist.token_next(token)

        if func(before) and func(after):
            tokens = tlist.tokens_between(before, after)
            token = tlist.group_tokens(sql.IdentifierList, tokens, extend=True)
        token = tlist.token_next_by(m=M_COMMA, idx=token)

def group_brackets(tlist):
    _group_matching(tlist, sql.SquareBrackets)

def group_parenthesis(tlist):
    _group_matching(tlist, sql.Parenthesis)

@recurse(sql.Comment)
def group_comments(tlist):
    idx = 0
    token = tlist.token_next_by_type(idx, T.Comment)
    while token:
        tidx = tlist.token_index(token)
        end = tlist.token_not_matching(tidx + 1,
                                       [lambda t: t.ttype in T.Comment,
                                        lambda t: t.is_whitespace()])
        if end is None:
            idx = tidx + 1
        else:
            eidx = tlist.token_index(end)
            grp_tokens = tlist.tokens_between(token,
                                              tlist.token_prev(eidx, False))
            group = tlist.group_tokens(sql.Comment, grp_tokens)
            idx = tlist.token_index(group)
        token = tlist.token_next_by_type(idx, T.Comment)

@recurse(sql.Where)
def group_where(tlist):
    token = tlist.token_next_by(m=sql.Where.M_OPEN)
    while token:
        end = tlist.token_next_by(m=sql.Where.M_CLOSE, idx=token)

        if end is None:
            tokens = tlist.tokens_between(token, tlist._groupable_tokens[-1])
        else:
            tokens = tlist.tokens_between(
                token, tlist.tokens[tlist.token_index(end) - 1])

        token = tlist.group_tokens(sql.Where, tokens)
        token = tlist.token_next_by(m=sql.Where.M_OPEN, idx=token)

@recurse()
def group_aliased(tlist):
    I_ALIAS = (sql.Parenthesis, sql.Function, sql.Case, sql.Identifier,
               )  # sql.Operation)

    token = tlist.token_next_by(i=I_ALIAS, t=T.Number)
    while token:
        next_ = tlist.token_next(token)
        if imt(next_, i=sql.Identifier):
            tokens = tlist.tokens_between(token, next_)
            token = tlist.group_tokens(sql.Identifier, tokens, extend=True)
        token = tlist.token_next_by(i=I_ALIAS, t=T.Number, idx=token)

def group_typecasts(tlist):
    _group_left_right(tlist, (T.Punctuation, '::'), sql.Identifier)

@recurse(sql.Function)
def group_functions(tlist):
    token = tlist.token_next_by(t=T.Name)
    while token:
        next_ = tlist.token_next(token)
        if imt(next_, i=sql.Parenthesis):
            tokens = tlist.tokens_between(token, next_)
            token = tlist.group_tokens(sql.Function, tokens)
        token = tlist.token_next_by(t=T.Name, idx=token)

def group_order(tlist):
    """Group together Identifier and Asc/Desc token"""
    token = tlist.token_next_by(t=T.Keyword.Order)
    while token:
        prev = tlist.token_prev(token)
        if imt(prev, i=sql.Identifier, t=T.Number):
            tokens = tlist.tokens_between(prev, token)
            token = tlist.group_tokens(sql.Identifier, tokens)
        token = tlist.token_next_by(t=T.Keyword.Order, idx=token)

@recurse()
def align_comments(tlist):
    token = tlist.token_next_by(i=sql.Comment)
    while token:
        before = tlist.token_prev(token)
        if isinstance(before, sql.TokenList):
            tokens = tlist.tokens_between(before, token)
            token = tlist.group_tokens(sql.TokenList, tokens, extend=True)
        token = tlist.token_next_by(i=sql.Comment, idx=token)

def group(tlist):
    for func in [
        group_comments,
        group_brackets,
        group_parenthesis,
        group_functions,
        group_where,
        group_case,
        group_identifier,
        group_order,
        group_typecasts,
        group_as,
        group_aliased,
        group_assignment,
        group_comparison,
        align_comments,
        group_identifier_list,
        group_if,
        group_for,
        group_foreach,
        group_begin,
    ]:
        func(tlist)
<|endoftext|>