<commit_before>import six

from collections import defaultdict
import itertools
import json
import os
import posixpath
import re
import requests

import numpy as np
from tqdm import tqdm

from cloudfiles import CloudFiles

from ....lib import red, toiter, Bbox, Vec, first, nvl
from ....mesh import Mesh
from .... import paths

from ..sharding import GrapheneShardReader
from ...precomputed.sharding import ShardingSpecification
from ....mesh import is_draco_chunk_aligned
from .unsharded import GrapheneUnshardedMeshSource

class GrapheneShardedMeshSource(GrapheneUnshardedMeshSource):
  def __init__(self, mesh_meta, cache, config, readonly):
    super(GrapheneShardedMeshSource, self).__init__(mesh_meta, cache, config, readonly)

    self.readers = {}
    for level, sharding in self.meta.info['sharding'].items(): # { level: std sharding, ... }
      spec = ShardingSpecification.from_dict(sharding)
      self.readers[int(level)] = GrapheneShardReader(self.meta, self.cache, spec)

  def initial_path(self, level):
    return self.meta.join(self.meta.mesh_path, 'initial', str(level))

  def dynamic_path(self):
    return self.meta.join(self.meta.mesh_path, 'dynamic')

  # 1. determine if the segid is before or after the shard time point
  # 2. assuming it is sharded, fetch the draco encoded file from the
  #    correct level

  def dynamic_exists(self, labels, progress=None):
    """
    Checks for dynamic mesh existence.

    Returns: { label: path or None, ... }
    """
    labels = toiter(labels)

    checks = [ self.compute_filename(label) for label in labels ]

    cloudpath = self.meta.join(self.meta.meta.cloudpath, self.meta.mesh_path, 'dynamic')
    progress = progress if progress is not None else self.config.progress

    results = CloudFiles(
      cloudpath, progress=progress,
      green=self.config.green, secrets=self.config.secrets
    ).exists(checks)

    output = {}
    for filepath, exists in results.items():
      label = int(os.path.basename(filepath).split(':')[0]) # strip :0
      output[label] = filepath if exists else None

    return output

  def initial_exists(self, labels, return_byte_range=False, progress=None):
    """
    Checks for initial mesh existence.

    Returns:
      If return_byte_range:
        { label: [ path, byte offset, byte size ] or None, ... }
      Else:
        { label: path or None, ... }
    """
    labels = toiter(labels)
    progress = progress if progress is not None else self.config.progress

    layers = defaultdict(list)
    for label in labels:
      if label == 0:
        continue
      layer = self.meta.meta.decode_layer_id(label)
      layers[layer].append(label)

    all_results = {}
    for layer, layer_labels in layers.items():
      path = self.initial_path(layer)
      results = self.readers[int(layer)].exists(
        layer_labels,
        path=path,
        return_byte_range=return_byte_range,
        progress=progress
      )
      all_results.update(results)

    return all_results

  def exists(self, labels, progress=None):
    """
    Checks dynamic then initial meshes for existence.

    Returns: { label: path or None, ... }
    """
    labels = set(toiter(labels))

    dynamic_labels = self.dynamic_exists(labels, progress)
    remainder_labels = set([ label for label, path in dynamic_labels.items() if path ])

    initial_labels = self.initial_exists(remainder_labels, progress=progress, return_byte_range=False)

    dynamic_labels.update(initial_labels)
    return dynamic_labels

  def parse_manifest_filenames(self, manifest):
    lists = defaultdict(list)
    initial_regexp = re.compile(r'~(\d+)/([\d\-]+\.shard):(\d+):(\d+)')

    filenames, segids = manifest['fragments'], manifest['seg_ids']

    for filename, segid in zip(filenames, segids):
      if not filename:
        continue

      # eg. ~2/344239114-0.shard:224659:442
      # tilde means initial, missing tilde means dynamic
      initial = filename[0] == '~'

      if initial:
        (layer_id, parsed_filename, byte_start, size) = re.search(
          initial_regexp, filename
        ).groups()
        lists['initial'].append((layer_id, parsed_filename, int(byte_start), int(size), int(segid)))
      else:
        lists['dynamic'].append(filename)

    return lists

  def get_meshes_via_manifest_byte_offsets(self, seg_id, bounding_box):
    """
    The manifest for sharded is a bit strange in that exists(..., return_byte_offset=True)
    is being called on the server side. To avoid duplicative delay by recomputing the offset
    locations, the manifest breaks encapsulation by returning the shard filename and byte
    offsets. This breaks enapsulation of the shard fetching logic rather severely but
    it is probably worth it.
    """
    level = self.meta.meta.decode_layer_id(seg_id)
    dynamic_cloudpath = self.meta.join(self.meta.meta.cloudpath, self.dynamic_path())
    if bounding_box is not None:
      level=2
    manifest = self.fetch_manifest(seg_id, level=level, bbox=bounding_box, return_segids=True)
    lists = self.parse_manifest_filenames(manifest)

    files = []
    if lists['dynamic']:
      files = CloudFiles(
        dynamic_cloudpath,
        green=self.config.green,
        secrets=self.config.secrets,
        parallel=self.config.parallel,
      ).get(lists['dynamic'])

    dynamic_meshes = []
    while files:
      f = files.pop()
      mesh = Mesh.from_draco(f['content'])
      mesh.segid = int(os.path.basename(f['path']).split(':')[0])
      dynamic_meshes.append(mesh)

    fetches = []
    segid_map = {}
    for layer_id, filename, byte_start, size, segid in lists['initial']:
      path = self.meta.join(layer_id, filename)
      byte_end = byte_start + size
      fetches.append({
        'path': path,
        'start': byte_start,
        'end': byte_end,
      })
      segid_map[(path, byte_start, byte_end)] = segid

    cloudpath = self.meta.join(self.meta.meta.cloudpath, self.meta.mesh_path, 'initial')
    files = CloudFiles(
      cloudpath,
      green=self.config.green,
      secrets=self.config.secrets,
      parallel=self.config.parallel,
    ).get(fetches)
    initial_meshes = []
    while files:
      f = files.pop()
      mesh = Mesh.from_draco(f['content'])
      start, end = f['byte_range']
      key = (f['path'], start, end)
      mesh.segid = segid_map[key]
      initial_meshes.append(mesh)

    return dynamic_meshes + initial_meshes

  def get_meshes_via_manifest_labels(self, seg_id, bounding_box):
    level = None
    if bounding_box is not None:
      level=2
    labels = self.get_fragment_labels(seg_id, level=level, bbox=bounding_box)
    meshes = self.get_meshes_on_bypass(labels, allow_missing=True) # sometimes a tiny label won't get meshed
    return list(meshes.values())

  def get_meshes_via_manifest(self, seg_id, bounding_box, use_byte_offsets):
    if use_byte_offsets:
      return self.get_meshes_via_manifest_byte_offsets(seg_id, bounding_box)
    return self.get_meshes_via_manifest_labels(seg_id, bounding_box)

  def get_chunk_aligned_mask(self, meshes):
    meta = self.meta.meta
    first_mesh = first(meshes)
    if first_mesh is None:
      raise IndexError("No meshes found.")

    draco_grid_size = meta.get_draco_grid_size(
      meta.decode_layer_id(first_mesh.segid)
    )
    base_resolution = meta.resolution(self.config.mip)
    lvl2_resolution = meta.resolution(self.meta.mip)

    voxel_offset = Vec(0,0,0)
    if meta.chunks_start_at_voxel_offset:
      voxel_offset = meta.voxel_offset(self.meta.mip)

    offset = voxel_offset * lvl2_resolution
    lvl_2_size_nm = meta.chunk_size(self.meta.mip) * base_resolution

    chunk_aligned_masks = []
    for mesh in meshes:
      level = meta.decode_layer_id(mesh.segid)
      chunk_size = (lvl_2_size_nm * (2 ** (level-2))).astype(np.int32)
      verts = mesh.vertices - offset
      # find all vertices that are exactly on chunk_size boundaries
      is_chunk_aligned = is_draco_chunk_aligned(
        verts, chunk_size, draco_grid_size=draco_grid_size
      )
      chunk_aligned_masks.append(is_chunk_aligned)

    return np.concatenate(chunk_aligned_masks)

  def stitch_multi_level_draco_mesh_fragments(self, meshes, segid):
    chunk_aligned_mask = self.get_chunk_aligned_mask(meshes)
    mesh = Mesh.concatenate(*meshes)
    mesh.segid = segid
    return mesh.deduplicate_vertices(chunk_aligned_mask)

  def get_meshes_on_bypass(self, segids, allow_missing=False):
    """
    Attempt to fetch a mesh directly from storage without going through
    the chunk graph server. This capability should only be used in special
    circumstances.
    """
    segids = toiter(segids)

    dynamic_cloudpath = self.meta.join(self.meta.meta.cloudpath, self.dynamic_path())
    filenames = [ self.compute_filename(segid) for segid in segids ]

    cf = CloudFiles(
      dynamic_cloudpath,
      progress=self.config.progress,
      green=self.config.green,
      secrets=self.config.secrets,
      parallel=self.config.parallel,
    )
    raw_binaries = cf.get(filenames)

    # extract the label ID from the mesh manifest.
    # e.g. 387463568301300850:0:24576-25088_17920-18432_2048-3072
    label_regexp = re.compile(r'(\d+):\d:[\d_-]+$')

    output = {}
    remaining = []
    for res in raw_binaries:
      if res['error']:
        raise res['error']

      (label,) = re.search(label_regexp, res['path']).groups()
      label = int(label)

      if res['content'] is None:
        remaining.append(label)
      else:
        output[label] = res['content']

    layers = defaultdict(list)
    for segid in remaining:
      layer_id = self.meta.meta.decode_layer_id(segid)
      layers[layer_id].append(segid)

    for layer_id, labels in layers.items():
      subdirectory = self.meta.join(self.meta.mesh_path, 'initial', str(layer_id))
      initial_output = self.readers[layer_id].get_data(
        labels, path=subdirectory,
        progress=self.config.progress,
        parallel=self.config.parallel,
      )
      for label, raw_binary in initial_output.items():
        if raw_binary is None:
          if allow_missing:
            continue
          else:
            raise IndexError('No mesh found for segment {}'.format(label))
        else:
          output[label] = raw_binary

    return {
      label: Mesh.from_draco(raw_binary, segid=label)
      for label, raw_binary in output.items()
    }

  def download_segid(self, seg_id, bounding_box, bypass=False, use_byte_offsets=True):
    """See GrapheneUnshardedMeshSource.get for the user facing function."""
    level = self.meta.meta.decode_layer_id(seg_id)
    if level not in self.readers:
      raise KeyError("There is no shard configuration in the mesh info file for level {}.".format(level))

    if bypass:
      mesh = self.get_meshes_on_bypass(seg_id)[seg_id]
    else:
      meshes = self.get_meshes_via_manifest(seg_id, bounding_box, use_byte_offsets=use_byte_offsets)
      mesh = self.stitch_multi_level_draco_mesh_fragments(meshes, seg_id)

    if mesh is None:
      raise IndexError('No mesh found for segment {}'.format(seg_id))

    mesh.segid = seg_id
    return mesh, True
<commit_msg> unnecesary integer casting for floating chunksize (#538)<commit_after>import six

from collections import defaultdict
import itertools
import json
import os
import posixpath
import re
import requests

import numpy as np
from tqdm import tqdm

from cloudfiles import CloudFiles

from ....lib import red, toiter, Bbox, Vec, first, nvl
from ....mesh import Mesh
from .... import paths

from ..sharding import GrapheneShardReader
from ...precomputed.sharding import ShardingSpecification
from ....mesh import is_draco_chunk_aligned
from .unsharded import GrapheneUnshardedMeshSource

class GrapheneShardedMeshSource(GrapheneUnshardedMeshSource):
  def __init__(self, mesh_meta, cache, config, readonly):
    super(GrapheneShardedMeshSource, self).__init__(mesh_meta, cache, config, readonly)

    self.readers = {}
    for level, sharding in self.meta.info['sharding'].items(): # { level: std sharding, ... }
      spec = ShardingSpecification.from_dict(sharding)
      self.readers[int(level)] = GrapheneShardReader(self.meta, self.cache, spec)

  def initial_path(self, level):
    return self.meta.join(self.meta.mesh_path, 'initial', str(level))

  def dynamic_path(self):
    return self.meta.join(self.meta.mesh_path, 'dynamic')

  # 1. determine if the segid is before or after the shard time point
  # 2. assuming it is sharded, fetch the draco encoded file from the
  #    correct level

  def dynamic_exists(self, labels, progress=None):
    """
    Checks for dynamic mesh existence.

    Returns: { label: path or None, ... }
    """
    labels = toiter(labels)

    checks = [ self.compute_filename(label) for label in labels ]

    cloudpath = self.meta.join(self.meta.meta.cloudpath, self.meta.mesh_path, 'dynamic')
    progress = progress if progress is not None else self.config.progress

    results = CloudFiles(
      cloudpath, progress=progress,
      green=self.config.green, secrets=self.config.secrets
    ).exists(checks)

    output = {}
    for filepath, exists in results.items():
      label = int(os.path.basename(filepath).split(':')[0]) # strip :0
      output[label] = filepath if exists else None

    return output

  def initial_exists(self, labels, return_byte_range=False, progress=None):
    """
    Checks for initial mesh existence.

    Returns:
      If return_byte_range:
        { label: [ path, byte offset, byte size ] or None, ... }
      Else:
        { label: path or None, ... }
    """
    labels = toiter(labels)
    progress = progress if progress is not None else self.config.progress

    layers = defaultdict(list)
    for label in labels:
      if label == 0:
        continue
      layer = self.meta.meta.decode_layer_id(label)
      layers[layer].append(label)

    all_results = {}
    for layer, layer_labels in layers.items():
      path = self.initial_path(layer)
      results = self.readers[int(layer)].exists(
        layer_labels,
        path=path,
        return_byte_range=return_byte_range,
        progress=progress
      )
      all_results.update(results)

    return all_results

  def exists(self, labels, progress=None):
    """
    Checks dynamic then initial meshes for existence.

    Returns: { label: path or None, ... }
    """
    labels = set(toiter(labels))

    dynamic_labels = self.dynamic_exists(labels, progress)
    remainder_labels = set([ label for label, path in dynamic_labels.items() if path ])

    initial_labels = self.initial_exists(remainder_labels, progress=progress, return_byte_range=False)

    dynamic_labels.update(initial_labels)
    return dynamic_labels

  def parse_manifest_filenames(self, manifest):
    lists = defaultdict(list)
    initial_regexp = re.compile(r'~(\d+)/([\d\-]+\.shard):(\d+):(\d+)')

    filenames, segids = manifest['fragments'], manifest['seg_ids']

    for filename, segid in zip(filenames, segids):
      if not filename:
        continue

      # eg. ~2/344239114-0.shard:224659:442
      # tilde means initial, missing tilde means dynamic
      initial = filename[0] == '~'

      if initial:
        (layer_id, parsed_filename, byte_start, size) = re.search(
          initial_regexp, filename
        ).groups()
        lists['initial'].append((layer_id, parsed_filename, int(byte_start), int(size), int(segid)))
      else:
        lists['dynamic'].append(filename)

    return lists

  def get_meshes_via_manifest_byte_offsets(self, seg_id, bounding_box):
    """
    The manifest for sharded is a bit strange in that exists(..., return_byte_offset=True)
    is being called on the server side. To avoid duplicative delay by recomputing the offset
    locations, the manifest breaks encapsulation by returning the shard filename and byte
    offsets. This breaks enapsulation of the shard fetching logic rather severely but
    it is probably worth it.
    """
    level = self.meta.meta.decode_layer_id(seg_id)
    dynamic_cloudpath = self.meta.join(self.meta.meta.cloudpath, self.dynamic_path())
    if bounding_box is not None:
      level=2
    manifest = self.fetch_manifest(seg_id, level=level, bbox=bounding_box, return_segids=True)
    lists = self.parse_manifest_filenames(manifest)

    files = []
    if lists['dynamic']:
      files = CloudFiles(
        dynamic_cloudpath,
        green=self.config.green,
        secrets=self.config.secrets,
        parallel=self.config.parallel,
      ).get(lists['dynamic'])

    dynamic_meshes = []
    while files:
      f = files.pop()
      mesh = Mesh.from_draco(f['content'])
      mesh.segid = int(os.path.basename(f['path']).split(':')[0])
      dynamic_meshes.append(mesh)

    fetches = []
    segid_map = {}
    for layer_id, filename, byte_start, size, segid in lists['initial']:
      path = self.meta.join(layer_id, filename)
      byte_end = byte_start + size
      fetches.append({
        'path': path,
        'start': byte_start,
        'end': byte_end,
      })
      segid_map[(path, byte_start, byte_end)] = segid

    cloudpath = self.meta.join(self.meta.meta.cloudpath, self.meta.mesh_path, 'initial')
    files = CloudFiles(
      cloudpath,
      green=self.config.green,
      secrets=self.config.secrets,
      parallel=self.config.parallel,
    ).get(fetches)
    initial_meshes = []
    while files:
      f = files.pop()
      mesh = Mesh.from_draco(f['content'])
      start, end = f['byte_range']
      key = (f['path'], start, end)
      mesh.segid = segid_map[key]
      initial_meshes.append(mesh)

    return dynamic_meshes + initial_meshes

  def get_meshes_via_manifest_labels(self, seg_id, bounding_box):
    level = None
    if bounding_box is not None:
      level=2
    labels = self.get_fragment_labels(seg_id, level=level, bbox=bounding_box)
    meshes = self.get_meshes_on_bypass(labels, allow_missing=True) # sometimes a tiny label won't get meshed
    return list(meshes.values())

  def get_meshes_via_manifest(self, seg_id, bounding_box, use_byte_offsets):
    if use_byte_offsets:
      return self.get_meshes_via_manifest_byte_offsets(seg_id, bounding_box)
    return self.get_meshes_via_manifest_labels(seg_id, bounding_box)

  def get_chunk_aligned_mask(self, meshes):
    meta = self.meta.meta
    first_mesh = first(meshes)
    if first_mesh is None:
      raise IndexError("No meshes found.")

    draco_grid_size = meta.get_draco_grid_size(
      meta.decode_layer_id(first_mesh.segid)
    )
    base_resolution = meta.resolution(self.config.mip)
    lvl2_resolution = meta.resolution(self.meta.mip)

    voxel_offset = Vec(0,0,0)
    if meta.chunks_start_at_voxel_offset:
      voxel_offset = meta.voxel_offset(self.meta.mip)

    offset = voxel_offset * lvl2_resolution
    lvl_2_size_nm = meta.chunk_size(self.meta.mip) * base_resolution

    chunk_aligned_masks = []
    for mesh in meshes:
      level = meta.decode_layer_id(mesh.segid)
      chunk_size = lvl_2_size_nm * (2 ** (level-2))
      verts = mesh.vertices - offset
      # find all vertices that are exactly on chunk_size boundaries
      is_chunk_aligned = is_draco_chunk_aligned(
        verts, chunk_size, draco_grid_size=draco_grid_size
      )
      chunk_aligned_masks.append(is_chunk_aligned)

    return np.concatenate(chunk_aligned_masks)

  def stitch_multi_level_draco_mesh_fragments(self, meshes, segid):
    chunk_aligned_mask = self.get_chunk_aligned_mask(meshes)
    mesh = Mesh.concatenate(*meshes)
    mesh.segid = segid
    return mesh.deduplicate_vertices(chunk_aligned_mask)

  def get_meshes_on_bypass(self, segids, allow_missing=False):
    """
    Attempt to fetch a mesh directly from storage without going through
    the chunk graph server. This capability should only be used in special
    circumstances.
    """
    segids = toiter(segids)

    dynamic_cloudpath = self.meta.join(self.meta.meta.cloudpath, self.dynamic_path())
    filenames = [ self.compute_filename(segid) for segid in segids ]

    cf = CloudFiles(
      dynamic_cloudpath,
      progress=self.config.progress,
      green=self.config.green,
      secrets=self.config.secrets,
      parallel=self.config.parallel,
    )
    raw_binaries = cf.get(filenames)

    # extract the label ID from the mesh manifest.
    # e.g. 387463568301300850:0:24576-25088_17920-18432_2048-3072
    label_regexp = re.compile(r'(\d+):\d:[\d_-]+$')

    output = {}
    remaining = []
    for res in raw_binaries:
      if res['error']:
        raise res['error']

      (label,) = re.search(label_regexp, res['path']).groups()
      label = int(label)

      if res['content'] is None:
        remaining.append(label)
      else:
        output[label] = res['content']

    layers = defaultdict(list)
    for segid in remaining:
      layer_id = self.meta.meta.decode_layer_id(segid)
      layers[layer_id].append(segid)

    for layer_id, labels in layers.items():
      subdirectory = self.meta.join(self.meta.mesh_path, 'initial', str(layer_id))
      initial_output = self.readers[layer_id].get_data(
        labels, path=subdirectory,
        progress=self.config.progress,
        parallel=self.config.parallel,
      )
      for label, raw_binary in initial_output.items():
        if raw_binary is None:
          if allow_missing:
            continue
          else:
            raise IndexError('No mesh found for segment {}'.format(label))
        else:
          output[label] = raw_binary

    return {
      label: Mesh.from_draco(raw_binary, segid=label)
      for label, raw_binary in output.items()
    }

  def download_segid(self, seg_id, bounding_box, bypass=False, use_byte_offsets=True):
    """See GrapheneUnshardedMeshSource.get for the user facing function."""
    level = self.meta.meta.decode_layer_id(seg_id)
    if level not in self.readers:
      raise KeyError("There is no shard configuration in the mesh info file for level {}.".format(level))

    if bypass:
      mesh = self.get_meshes_on_bypass(seg_id)[seg_id]
    else:
      meshes = self.get_meshes_via_manifest(seg_id, bounding_box, use_byte_offsets=use_byte_offsets)
      mesh = self.stitch_multi_level_draco_mesh_fragments(meshes, seg_id)

    if mesh is None:
      raise IndexError('No mesh found for segment {}'.format(seg_id))

    mesh.segid = seg_id
    return mesh, True
<|endoftext|>


<commit_before># coding=utf-8

# Copyright (c) 2001-2016, Canal TP and/or its affiliates. All rights reserved.
#
# This file is part of Navitia,
#     the software to build cool stuff with public transport.
#
# Hope you'll enjoy and contribute to this project,
#     powered by Canal TP (www.canaltp.fr).
# Help us simplify mobility and open public transport:
#     a non ending quest to the responsive locomotion way of traveling!
#
# LICENCE: This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# Stay tuned using
# twitter @navitia
# IRC #navitia on freenode
# https://groups.google.com/d/forum/navitia
# www.navitia.io

from __future__ import absolute_import, print_function, unicode_literals, division
from jormungandr.autocomplete.abstract_autocomplete import AbstractAutocomplete
from jormungandr.scenarios.utils import build_pagination, pb_type
from jormungandr.interfaces.v1.fields import NonNullList, place, NonNullNested, PbField, error, feed_publisher,\
    disruption_marshaller
from flask.ext.restful import marshal_with, fields
import navitiacommon.request_pb2 as request_pb2
import navitiacommon.type_pb2 as type_pb2
from jormungandr.utils import date_to_timestamp

places = {
    "places": NonNullList(NonNullNested(place)),
    "error": PbField(error, attribute='error'),
    "disruptions": fields.List(NonNullNested(disruption_marshaller), attribute="impacts"),
    "feed_publishers": fields.List(NonNullNested(feed_publisher))
}

class Kraken(AbstractAutocomplete):

    @marshal_with(places)
    def get(self, request, instance):

        req = request_pb2.Request()
        req.requested_api = type_pb2.places
        req.places.q = request['q']
        req.places.depth = request['depth']
        req.places.count = request['count']
        req.places.search_type = request['search_type']
        req._current_datetime = date_to_timestamp(request['_current_datetime'])
        if request["type[]"]:
            for type in request["type[]"]:
                if type not in pb_type:
                    abort(422, message="{} is not an acceptable type".format(type))

                req.places.types.append(pb_type[type])

        if request["admin_uri[]"]:
            for admin_uri in request["admin_uri[]"]:
                req.places.admin_uris.append(admin_uri)

        resp = instance.send_and_receive(req)
        if (not hasattr(resp, 'places')) and request['search_type'] == 0:
            args["search_type"] = 1
            resp = instance.send_and_receive(req)
        build_pagination(request, resp)
        return resp
<commit_msg>jormungandr: delete empty line<commit_after># coding=utf-8

# Copyright (c) 2001-2016, Canal TP and/or its affiliates. All rights reserved.
#
# This file is part of Navitia,
#     the software to build cool stuff with public transport.
#
# Hope you'll enjoy and contribute to this project,
#     powered by Canal TP (www.canaltp.fr).
# Help us simplify mobility and open public transport:
#     a non ending quest to the responsive locomotion way of traveling!
#
# LICENCE: This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# Stay tuned using
# twitter @navitia
# IRC #navitia on freenode
# https://groups.google.com/d/forum/navitia
# www.navitia.io
from __future__ import absolute_import, print_function, unicode_literals, division
from jormungandr.autocomplete.abstract_autocomplete import AbstractAutocomplete
from jormungandr.scenarios.utils import build_pagination, pb_type
from jormungandr.interfaces.v1.fields import NonNullList, place, NonNullNested, PbField, error, feed_publisher,\
    disruption_marshaller
from flask.ext.restful import marshal_with, fields
import navitiacommon.request_pb2 as request_pb2
import navitiacommon.type_pb2 as type_pb2
from jormungandr.utils import date_to_timestamp

places = {
    "places": NonNullList(NonNullNested(place)),
    "error": PbField(error, attribute='error'),
    "disruptions": fields.List(NonNullNested(disruption_marshaller), attribute="impacts"),
    "feed_publishers": fields.List(NonNullNested(feed_publisher))
}

class Kraken(AbstractAutocomplete):

    @marshal_with(places)
    def get(self, request, instance):

        req = request_pb2.Request()
        req.requested_api = type_pb2.places
        req.places.q = request['q']
        req.places.depth = request['depth']
        req.places.count = request['count']
        req.places.search_type = request['search_type']
        req._current_datetime = date_to_timestamp(request['_current_datetime'])
        if request["type[]"]:
            for type in request["type[]"]:
                if type not in pb_type:
                    abort(422, message="{} is not an acceptable type".format(type))

                req.places.types.append(pb_type[type])

        if request["admin_uri[]"]:
            for admin_uri in request["admin_uri[]"]:
                req.places.admin_uris.append(admin_uri)

        resp = instance.send_and_receive(req)
        if (not hasattr(resp, 'places')) and request['search_type'] == 0:
            args["search_type"] = 1
            resp = instance.send_and_receive(req)
        build_pagination(request, resp)
        return resp
<|endoftext|>


<commit_before>#
# linter.py
# Linter for SublimeLinter3, a code checking framework for Sublime Text 3
#
# Written by Adam Altmejd
# Copyright (c) 2013 Adam Altmejd
# License: MIT
#

"""Uses chktex to lint LaTeX files."""

from SublimeLinter.lint import PythonLinter, util

class Chktex(PythonLinter):

    """ Provides an interface to use chktex in SublimeText with SublimeLinter3.

    Chktex uses stdin for input and stdout for output.
    Todo:
        - user settings
        - ignore some warnings
        - allow checking of latex scope in eg .Rnw (knitr) files
        - combine with lacheck?

    """

    syntax = ('latex', 'latexing', 'latexing (knitr)')
    selectors = {'latexing (knitr)': }

    cmd = 'chktex "-f%l:%c %k %m\n" '
    regex = (
        r'^(?P<line>\d+):(?P<col>\d+) '
        r'(?:(?P<error>Error)|(?P<warning>Warning)) '
        r'(?P<message>.+)'
    )
    error_stream = util.STREAM_STDOUT

    defaults = {

    }
<commit_msg>Added environments for Knitr and some default settings for ignoring warnings.<commit_after>#
# linter.py
# Linter for SublimeLinter3, a code checking framework for Sublime Text 3
#
# Written by Adam Altmejd
# Copyright (c) 2013 Adam Altmejd
# License: MIT
#

"""Uses chktex to lint LaTeX files."""

from SublimeLinter.lint import PythonLinter, util

class Chktex(PythonLinter):

    """ Provides an interface to use chktex in SublimeText with SublimeLinter3."""

    syntax = ('latex', 'latexing', 'latex (knitr)', 'knitr-rnw')
    selectors = {
        'latex (knitr)': 'text.tex.latex.knitr.ing - meta.block.parameters.knitr - source.r.embedded.knitr',
        'knitr-rnw': 'text.tex.latex.knitr - meta.block.parameters.knitr - source.r.embedded.knitr'
        }

    cmd = 'chktex "-f%l:%c %k %k %n: %m\n" *'
    regex = (
        r'^(?P<line>\d+):(?P<col>\d+) '
        r'(?:(?P<error>Error)|(?P<warning>Warning)) '
        r'(?P<message>.+)'
    )
    error_stream = util.STREAM_STDOUT
    config_file = ('--localrc', '.chktexrc')
    defaults = {
        '--nowarn,+': '3'
    }
<|endoftext|>


<commit_before>from SublimeLinter.lint import Linter, util
from os.path import basename

class Ghc(Linter):
    cmd = ('ghc', '-fno-code', '-Wall', '-Wwarn', '-fno-helpful-errors',
           '$temp_file')
    regex = (
        r'\s*(?P<filename>.+):'
        r'\s*(?P<line>\d+):(?P<col>\d+):'
        r'\s*(?:(?P<warning>[Ww]arning):|(?P<error>[Ee]rror):)?'
        r'\s*(?P<flag>\[-W[^\]]*\])?'
        r'\s*(?P<message>.+?):?$'
    )
    multiline = True
    defaults = {
        'selector': 'source.haskell, text.tex.latex.haskell'
    }

    tempfile_suffix = {
        'haskell': 'hs',
        'haskell-sublimehaskell': 'hs',
        'literate haskell': 'lhs',
        'literatehaskellbirdstyle': 'lhs'
    }

    # ghc writes errors to STDERR
    error_stream = util.STREAM_STDERR

    def split_match(self, match):
        """Override to ignore errors reported in imported files."""
        error = super().split_match(match)

        match_filename = basename(error['filename'])
        linted_filename = basename(self.context['temp_file'])
        if match_filename != linted_filename:
            return None

        if error['flag']:
            error['message'] += " " + error['flag']
        return error
<commit_msg>Also yield the errors from related files<commit_after>from SublimeLinter.lint import Linter, util

class Ghc(Linter):
    cmd = ('ghc', '-fno-code', '-Wall', '-Wwarn', '-fno-helpful-errors',
           '$temp_file')
    regex = (
        r'\s*(?P<filename>.+):'
        r'\s*(?P<line>\d+):(?P<col>\d+):'
        r'\s*(?:(?P<warning>[Ww]arning):|(?P<error>[Ee]rror):)?'
        r'\s*(?P<flag>\[-W[^\]]*\])?'
        r'\s*(?P<message>.+?):?$'
    )
    multiline = True
    defaults = {
        'selector': 'source.haskell, text.tex.latex.haskell'
    }

    tempfile_suffix = {
        'haskell': 'hs',
        'haskell-sublimehaskell': 'hs',
        'literate haskell': 'lhs',
        'literatehaskellbirdstyle': 'lhs'
    }

    # ghc writes errors to STDERR
    error_stream = util.STREAM_STDERR

    def split_match(self, match):
        """Override to ignore errors reported in imported files."""
        error = super().split_match(match)
        if error['flag']:
            error['message'] += " " + error['flag']
        return error
<|endoftext|>


<commit_before>#
# linter.py
# Linter r SublimeLinter3, a code checking framework for Sublime Text 3
#
# Written by
# Copyright (c) 2015
#
# License: MIT
#

"""This module exports the FileExists plugin class."""

from SublimeLinter.lint import Linter, util
import re
import os
import sublime

class FileExists(Linter):

    """Provides an interface to fileExists."""

    syntax = ('pbs','source.pbs')
    # syntax = ('source.pbs','pbs')
    # cmd = 'fileExists.py'
    cmd = None
    # executable = None
    # version_args = '--version'
    # version_re = r'(?P<version>\d+\.\d+\.\d+)'
    # version_requirement = '>= 1.0'
    # regex = r''
    regex = (
        r'^.+?:(?P<line>\d+):(?P<col>\d+):'
        r'(?:(?P<error>error)|(?P<warning>(warning|note))):'
        r'(?P<message>.+)$'
    )
    word_re = r'(^[-\w\.]+)'
    multiline = False
    line_col_base = (1, 1)
    # error_stream = util.STREAM_BOTH
    # selectors = {
    #     'source.pbs': 'schrodinger.input'
    # }
    # word_re = None
    defaults = {}
    inline_settings = None
    inline_overrides = None
    comment_re = None

    def linearToRowCol(self,pos,code,tablength=4):
    	untabbed = code.replace('\t',' '*tablength)
    	# print ('00000'+' '*tablength+'`````````')
    	row = 1
    	currentlen = 0
    	lastlen = 0
    	print ("###"+str(pos))
    	splitcode = untabbed.split('\n')
    	print (len(untabbed))
    	print (len(code))

    	for line in splitcode:
    		lastlen = currentlen
    		currentlen += len(line)+1
    		# print (currentlen)
    		if currentlen >= pos:
    			return((row,pos-lastlen+1))
    		row += 1

    def run(self,cmd,code):
        untabbed = code.replace('\t',' '*4)
        untabbed = code
        splitcode = code.split('\n')
        found_schrod = False

        schrod_regex = re.compile(
            '\$SCHRODINGER(\/\w+)+'
            '(.+(?P<config>-c\s+[a-zA-Z_][\w\.-_]+)?.+\\\\\n)+'
            '.+(?P<config2>-c\s+[a-zA-Z_][\w\.-_]+)?.+\n',
            re.M)

        config_regex = re.compile(
            r'\$SCHRODINGER(.+\\\n)*'
            r'.+(?P<config>-c\s+[a-zA-Z_][\w\._-]+).+\n'
            ,re.M)

        for f in config_regex.finditer(untabbed):
            print (f.group('config'))
            match = re.search("[a-zA-Z_][\w\._-]*$", f.group('config'))
            filenameStart = match.start(0)
            # matchInLine = re.search(filename, line)
            print (self.linearToRowCol(f.start('config')+filenameStart,untabbed))

        cms_regex = re.compile(
            r'\$SCHRODINGER(.+\\\n)*'
            r'.+(?P<config>-in\s+[a-zA-Z_][\w\._-]+).+\n'
            ,re.M)

        for f in cms_regex.finditer(untabbed):
            print (f.group('config'))
            match = re.search("[a-zA-Z_][\w\._-]*$", f.group('config'))
            filenameStart = match.start(0)
            print (self.linearToRowCol(f.start('config')+filenameStart,untabbed))

        alllints = ''

        lineNum = 1
        for line in splitcode:
            if re.search('\$SCHRODINGER', line):
                found_schrod = True

            fileArgRe = re.compile("-([dcm]|in)\s+[a-zA-Z_][\w_\.-]*")
            # fileArg = re.search("-[dcm(in)]\s+[a-zA-Z_][a-zA-Z0-9_\.-]*", line)
            # if fileArg and found_schrod:
            if found_schrod:
            	print (line)
            	for fileArg in fileArgRe.finditer(line):
	                match = re.search("[a-zA-Z_][\w\._-]*$", fileArg.group(0))
	                print (fileArg)
	                print (fileArg.group(0))
	                filename = match.group(0)
	                print (filename)
	                path = os.path.dirname(self.view.file_name())

	                if (os.path.isfile(path+"/"+filename)):
	                    matchInLine = re.search(filename, line)
	                    print (matchInLine.group(0))
	                    pos = matchInLine.start(0)
	                    linted = 'W:%s:%s:warning:File exists'%(lineNum,pos+1)
	                    alllints += '\n'+linted
            else:
                pass
            lineNum += 1

        print(alllints)
        return(alllints)

<commit_msg>working across multiple instances of a command<commit_after>#
# linter.py
# Linter r SublimeLinter3, a code checking framework for Sublime Text 3
#
# Written by
# Copyright (c) 2015
#
# License: MIT
#

"""This module exports the FileExists plugin class."""

from SublimeLinter.lint import Linter, util
import re
import os
import sublime

class FileExists(Linter):

    """Provides an interface to fileExists."""

    syntax = ('pbs','source.pbs')
    # syntax = ('source.pbs','pbs')
    # cmd = 'fileExists.py'
    cmd = None
    # executable = None
    # version_args = '--version'
    # version_re = r'(?P<version>\d+\.\d+\.\d+)'
    # version_requirement = '>= 1.0'
    # regex = r''
    regex = (
        r'^.+?:(?P<line>\d+):(?P<col>\d+):'
        r'(?:(?P<error>error)|(?P<warning>(warning|note))):'
        r'(?P<message>.+)$'
    )
    word_re = r'(^[-\w\.]+)'
    multiline = False
    line_col_base = (1, 1)
    # error_stream = util.STREAM_BOTH
    # selectors = {
    #     'source.pbs': 'schrodinger.input'
    # }
    # word_re = None
    defaults = {}
    inline_settings = None
    inline_overrides = None
    comment_re = None

    def linearToRowCol(self,pos,code,tablength=4):
    	row = 1
    	currentlen = 0
    	lastlen = 0
    	splitcode = code.split('\n')

    	for line in splitcode:
    		lastlen = currentlen
    		currentlen += len(line)+1
    		if currentlen >= pos:
    			return((row,pos-lastlen+1))
    		row += 1

    def scanRegex(self,prog,arg,code):

        regex = re.compile(
            r'%s(.+\\\n)*'
            r'.+(?P<arg>%s\s+[a-zA-Z_][\w\._-]+).+\n' %(prog,arg)
            ,re.M)

        alllints = ''
        path = os.path.dirname(self.view.file_name())

        for f in regex.finditer(code):
            match = re.search("[a-zA-Z_][\w\._-]*$", f.group('arg'))
            filenameStart = match.start(0)
            filename = match.group(0)
            pos = self.linearToRowCol(f.start('arg')+filenameStart,code)

            if (os.path.isfile(path+"/"+filename)):
                linted = 'W:%s:%s:warning:File exists'%(pos[0],pos[1])
                alllints += '\n'+linted

        return alllints

    def run(self,cmd,code):
        untabbed = code.replace('\t',' '*4)
        untabbed = code
        splitcode = code.split('\n')
        found_schrod = False

        schrod_regex = re.compile(
            '\$SCHRODINGER(\/\w+)+'
            '(.+(?P<config>-c\s+[a-zA-Z_][\w\.-_]+)?.+\\\\\n)+'
            '.+(?P<config2>-c\s+[a-zA-Z_][\w\.-_]+)?.+\n',
            re.M)

        alllints = ''

        alllints += self.scanRegex('\$SCHRODINGER', '-c',code)
        alllints += self.scanRegex('\$SCHRODINGER', '-in',code)

        print (alllints)

        return(alllints)

<|endoftext|>


<commit_before>from SublimeLinter.lint import PythonLinter, util, const

class Pydocstyle(PythonLinter):
    cmd = 'pydocstyle'
    regex = r'^.+?:(?P<line>\d+).*:\r?\n\s*(?P<warning>D\d{3}):\s(?P<message>.+)$'
    multiline = True
    default_type = const.WARNING
    error_stream = util.STREAM_BOTH
    line_col_base = (1, 0)  # uses one-based line and zero-based column numbers
    tempfile_suffix = 'py'
    defaults = {
        'selector': 'source.python',
        '--add-ignore=': '',
        '--add-select=': '',
        '--ignore=': '',
        '--select=': '',
        '--config=': '',
        '--convention=': '',
        '--ignore-decorators=': ''
    }
<commit_msg>Take emitted names as 'near' value and add it to the message<commit_after>from SublimeLinter.lint import PythonLinter, util, const

class Pydocstyle(PythonLinter):
    cmd = 'pydocstyle'
    regex = r'''(?x)
        ^(?P<filename>.+):(?P<line>\d+)[^`\n]*(`(?P<near>.+)`)?.*:\n
        \s*(?P<warning>D\d{3}):\s(?P<message>.+)
    '''
    multiline = True
    default_type = const.WARNING
    error_stream = util.STREAM_BOTH
    line_col_base = (1, 0)  # uses one-based line and zero-based column numbers
    tempfile_suffix = 'py'
    defaults = {
        'selector': 'source.python',
        '--add-ignore=': '',
        '--add-select=': '',
        '--ignore=': '',
        '--select=': '',
        '--config=': '',
        '--convention=': '',
        '--ignore-decorators=': ''
    }

    def split_match(self, match):
        match = super().split_match(match)
        if match.near and '__init__' not in match.message:
            return match._replace(
                message='{} `{}`'.format(match.message, match.near))

        return match
<|endoftext|>


<commit_before>#
# linter.py
# Linter for SublimeLinter3, a code checking framework for Sublime Text 3
#
# Written by ckaznocha
# Copyright (c) 2014 ckaznocha
#
# License: MIT
#

"""This module exports the CFLint plugin class."""
from SublimeLinter.lint import Linter, util

class CFLint(Linter):
    """Provides an interface to CFLint."""

    syntax = ('coldfusioncfc', 'html+cfml', 'cfml')
    executable = 'java'
    cmd = None
    regex = r'''(?xi)
        # Severity
        ^\s*Severity:(?:(?P<warning>(WARNING|CAUTION|INFO|COSMETIC))|(?P<error>(FATAL|CRITICAL|ERROR)))\s*$\r?\n

        # Message code
        ^.*$\r?\n

        # File name
        ^.*$\r?\n

        # Column number
        ^\s*Column:(?P<col>\d+)\s*$\r?\n

        # Line number
        ^\s*Line:(?P<line>\d+)\s*$\r?\n

        # Message
        ^\s*Message:(?P<message>.+)$\r?\n

        # Variable
        ^.*$\r?\n

        # Expression
        ^.*$\r?\n
    '''
    multiline = True
    error_stream = util.STREAM_STDOUT
    word_re = r'^<?(#?[-\w]+#?)'
    tempfile_suffix = '-'
    defaults = {
        'jar_file': '',
        'config_file_name': 'cflintrc.xml',
        'aux_config_dirs': []
    }

    def __init__(self, view, syntax):
        Linter.__init__(self, view, syntax)

        settings = self.get_view_settings()
        config_file_name = settings.get('config_file_name')
        aux_config_dirs = settings.get('aux_config_dirs')
        config_file_tuple = ('-configfile', config_file_name)

        for conf in aux_config_dirs:
            config_file_tuple += (conf,)

        self.config_file = config_file_tuple

    def cmd(self):
        """Return the command line to execute."""

        jar_file = self.get_jarfile_path()

        return [self.executable_path, '-jar', jar_file, '-file', '@', '-q', '-text']

    def get_jarfile_path(self):
        """Return the absolute path to the CFLint jar file."""

        settings = self.get_view_settings()
        jar_file = settings.get('jar_file')

        return jar_file
<commit_msg>Fix build failure due to pep257: D102<commit_after>#
# linter.py
# Linter for SublimeLinter3, a code checking framework for Sublime Text 3
#
# Written by ckaznocha
# Copyright (c) 2014 ckaznocha
#
# License: MIT
#

"""This module exports the CFLint plugin class."""
from SublimeLinter.lint import Linter, util

class CFLint(Linter):
    """Provides an interface to CFLint."""

    syntax = ('coldfusioncfc', 'html+cfml', 'cfml')
    executable = 'java'
    cmd = None
    regex = r'''(?xi)
        # Severity
        ^\s*Severity:(?:(?P<warning>(WARNING|CAUTION|INFO|COSMETIC))|(?P<error>(FATAL|CRITICAL|ERROR)))\s*$\r?\n

        # Message code
        ^.*$\r?\n

        # File name
        ^.*$\r?\n

        # Column number
        ^\s*Column:(?P<col>\d+)\s*$\r?\n

        # Line number
        ^\s*Line:(?P<line>\d+)\s*$\r?\n

        # Message
        ^\s*Message:(?P<message>.+)$\r?\n

        # Variable
        ^.*$\r?\n

        # Expression
        ^.*$\r?\n
    '''
    multiline = True
    error_stream = util.STREAM_STDOUT
    word_re = r'^<?(#?[-\w]+#?)'
    tempfile_suffix = '-'
    defaults = {
        'jar_file': '',
        'config_file_name': 'cflintrc.xml',
        'aux_config_dirs': []
    }

    def __init__(self, view, syntax):
        """Override init to dynamically set config_file."""

        Linter.__init__(self, view, syntax)

        settings = self.get_view_settings()
        config_file_name = settings.get('config_file_name')
        aux_config_dirs = settings.get('aux_config_dirs')
        config_file_tuple = ('-configfile', config_file_name)

        for conf in aux_config_dirs:
            config_file_tuple += (conf,)

        self.config_file = config_file_tuple

    def cmd(self):
        """Return the command line to execute."""

        jar_file = self.get_jarfile_path()

        return [self.executable_path, '-jar', jar_file, '-file', '@', '-q', '-text']

    def get_jarfile_path(self):
        """Return the absolute path to the CFLint jar file."""

        settings = self.get_view_settings()
        jar_file = settings.get('jar_file')

        return jar_file
<|endoftext|>


<commit_before>#
# linter.py
# Linter for SublimeLinter3, a code checking framework for Sublime Text 3
#
# Written by Aparajita Fishman
# Copyright (c) 2013 Aparajita Fishman
#
# License: MIT
#

"""This module exports the PHPLint plugin class."""

from SublimeLinter.lint import Linter

class PHPLint(Linter):

    """Provides an interface to the phplint executable."""

    syntax = ('php', 'html')
    cmd = 'phplint --print-path shortest --print-context --tab-size 4 --no-overall'
    regex = (
        r'^(?i)(?:'
        r'\t.*?\r?\n'
        r'\t(?P<col>.*)\\_ HERE\r?\n)?'
        r'(?P<filename>.+?):(?P<line>\d+): '
        r'(?:(?P<error>(?:fatal )?error)|(?P<warning>warning|notice)): '
        r'(?P<message>[^`\r\n]*(?:`(?P<near>[^\']+)\')?[^\r\n]*)$'
    )
    multiline = True
    tempfile_suffix = 'php'
    selectors = {
        'html': 'source.php.embedded.block.html'
    }

    def split_match(self, match):
        """Return the match with ` quotes transformed to '."""
        match, line, col, error, warning, message, near = super().split_match(match)

        message = message.replace('`', '\'')

        return match, line, col, error, warning, message, near
<commit_msg>Fixes and enhancements...<commit_after>#
# linter.py
# Linter for SublimeLinter3, a code checking framework for Sublime Text 3
#
# Written by Aparajita Fishman
# Copyright (c) 2013 Aparajita Fishman
#
# License: MIT
#

"""This module exports the PHPLint plugin class."""

from SublimeLinter.lint import Linter

class PHPLint(Linter):

    """Provides an interface to the phplint executable."""

    syntax = ('php', 'html', 'html 5')
    cmd = 'phplint --print-path shortest --print-context --tab-size 4 --no-overall'
    regex = (
        r'(?i)^(?:'
        r'\t.*?\r?\n'
        r'\t(?P<col>.*) \\_ HERE\r?\n)?'
        r'(?P<filename>.+?):(?P<line>\d+): '
        r'(?:(?P<error>(?:fatal )?error)|(?P<warning>warning|notice)): '
        r'(?P<message>[^`\r\n]*(?:`(?P<near>[^\']+)\')?[^\r\n]*)'
    )
    multiline = True
    tempfile_suffix = 'php'

    def split_match(self, match):
        """Return the match with ` quotes transformed to '."""
        match, line, col, error, warning, message, near = super().split_match(match)

        if message == 'no PHP code found at all':
            match = None
        else:
            message = message.replace('`', '\'')

            # If the message contains a complaint about a function
            # and near looks like a function reference, remove the trailing
            # () so it can be found.
            if 'function \'' in message and near and near.endswith('()'):
                near = near[:-2]

        return match, line, col, error, warning, message, near
<|endoftext|>


<commit_before>#
# linter.py
# Linter for SublimeLinter3, a code checking framework for Sublime Text 3
#
# Written by Aparajita Fishman
# Copyright (c) 2015-2016 The SublimeLinter Community
# Copyright (c) 2013-2014 Aparajita Fishman
#
# License: MIT
#

"""This module exports the Ruby plugin class."""

from SublimeLinter.lint import RubyLinter
import re

class Ruby(RubyLinter):
    """Provides an interface to ruby -wc."""

    syntax = ('ruby', 'ruby on rails', 'rspec')
    cmd = 'ruby -wc'
    regex = (
        r'^(?P<file>.+?):(?P<line>\d+): (?:(?P<error>.*?error)|(?P<warning>warning))[,:] (?P<message>[^\r\n]+)\r?\n'
        r'(?:^[^\r\n]+\r?\n^(?P<col>.*?)\^)?'
    )
    multiline = True
    comment_re = r'\s*#'

    def split_match(self, match):
        """
        Return the components of the match.

        We override this because unrelated library files can throw errors,
        and we only want errors from the linted file.

        """

        if match:
            if match.group('file') != '-':
                match = None

        match, line, col, error, warning, message, _ = super().split_match(match)
        near = self.search_token(message)

        return match, line, col, error, warning, message, near

    def search_token(self, message):
        """Search text token to be highlighted."""

        # First search for variable name enclosed in quotes
        m = re.search("(?<=`).*(?=')", message)

        # Then search for variable name following a dash
        if m is None:
            m = re.search('(?<= - )\S+', message)

        # Then search for mismatched indentation
        if m is None:
            m = re.search("(?<=mismatched indentations at ')end", message)

        # Then search for equal operator in conditional
        if m is None:
            m = re.search('(?<=found )=(?= in conditional)', message)

        # Then search for use of operator in void context
        if m is None:
            m = re.search('\S+(?= in void context)', message)

        # Then search for END in method
        if m is None:
            m = re.search('END(?= in method)', message)

        return m.group(0) if m else None
<commit_msg>Use `selector` instead of `syntax`<commit_after>#
# linter.py
# Linter for SublimeLinter3, a code checking framework for Sublime Text 3
#
# Written by Aparajita Fishman
# Copyright (c) 2015-2016 The SublimeLinter Community
# Copyright (c) 2013-2014 Aparajita Fishman
#
# License: MIT
#

"""This module exports the Ruby plugin class."""

from SublimeLinter.lint import RubyLinter
import re

class Ruby(RubyLinter):
    """Provides an interface to ruby -wc."""

    defaults = {
        'selector': 'source.ruby'
    }

    cmd = 'ruby -wc'
    regex = (
        r'^(?P<file>.+?):(?P<line>\d+): (?:(?P<error>.*?error)|(?P<warning>warning))[,:] (?P<message>[^\r\n]+)\r?\n'
        r'(?:^[^\r\n]+\r?\n^(?P<col>.*?)\^)?'
    )
    multiline = True
    comment_re = r'\s*#'

    def split_match(self, match):
        """
        Return the components of the match.

        We override this because unrelated library files can throw errors,
        and we only want errors from the linted file.

        """

        if match:
            if match.group('file') != '-':
                match = None

        match, line, col, error, warning, message, _ = super().split_match(match)
        near = self.search_token(message)

        return match, line, col, error, warning, message, near

    def search_token(self, message):
        """Search text token to be highlighted."""

        # First search for variable name enclosed in quotes
        m = re.search("(?<=`).*(?=')", message)

        # Then search for variable name following a dash
        if m is None:
            m = re.search('(?<= - )\S+', message)

        # Then search for mismatched indentation
        if m is None:
            m = re.search("(?<=mismatched indentations at ')end", message)

        # Then search for equal operator in conditional
        if m is None:
            m = re.search('(?<=found )=(?= in conditional)', message)

        # Then search for use of operator in void context
        if m is None:
            m = re.search('\S+(?= in void context)', message)

        # Then search for END in method
        if m is None:
            m = re.search('END(?= in method)', message)

        return m.group(0) if m else None
<|endoftext|>


<commit_before>#
# linter.py
# Markdown Linter for SublimeLinter, a code checking framework
# for Sublime Text 3
#
# Written by Jon LaBelle
# Copyright (c) 2018 Jon LaBelle
#
# License: MIT
#

"""This module exports the Markdownlint plugin class."""

from SublimeLinter.lint import NodeLinter, util

class MarkdownLint(NodeLinter):
    """Provides an interface to markdownlint."""

    defaults = {
        'selector': 'text.html.markdown,'
                    'text.html.markdown.multimarkdown,'
                    'text.html.markdown.extended,'
                    'text.html.markdown.gfm'
    }
    cmd = ('markdownlint', '${args}', '${file}')
    regex = r'.+?[:](?P<line>\d+)\s(?P<error>MD\d+)?[/]?(?P<message>.+)'
    multiline = False
    line_col_base = (1, 1)
    tempfile_suffix = '-'
    error_stream = util.STREAM_STDERR
    word_re = None
<commit_msg>fix: wrong line and col in markdownlint 0.22 (#10)<commit_after>#
# linter.py
# Markdown Linter for SublimeLinter, a code checking framework
# for Sublime Text 3
#
# Written by Jon LaBelle
# Copyright (c) 2018 Jon LaBelle
#
# License: MIT
#

"""This module exports the Markdownlint plugin class."""

from SublimeLinter.lint import NodeLinter, util

class MarkdownLint(NodeLinter):
    """Provides an interface to markdownlint."""

    defaults = {
        'selector': 'text.html.markdown,'
                    'text.html.markdown.multimarkdown,'
                    'text.html.markdown.extended,'
                    'text.html.markdown.gfm'
    }
    cmd = ('markdownlint', '${args}', '${file}')
    regex = r'.+?(?:[:](?P<line>\d+))(?:[:](?P<col>\d+))?\s+(?P<error>MD\d+)?[/]?(?P<message>.+)'
    multiline = False
    line_col_base = (1, 1)
    tempfile_suffix = '-'
    error_stream = util.STREAM_STDERR
    word_re = None
<|endoftext|>


<commit_before>from SublimeLinter.lint import NodeLinter

class JSHint(NodeLinter):
    npm_name = 'jshint'
    cmd = ['jshint', '--verbose', '${args}', '-']
    regex = (
        r'^.+?: line (?P<line>\d+), col (?P<col>\d+), '
        r'(?P<message>'
        # unexpected use of ++ etc
        r'|.+\'(?P<unexpected>.+)\'\.(?=.+W016)'
        # duplicate key
        r'|.+\'(?P<duplicate>.+)\'.+(?=.+W075)'
        # camel case
        r'|.+\'(?P<no_camel>.+)\'.+(?=.+W106)'
        # unexpected use, typically use of non strict operators
        r'|.+\'(?P<actual>.+)\'\.(?=.+W116)'
        # match all messages
        r'|.+)'
        # capture error, warning and code
        r' \((?:(?P<error>E\d+)|(?P<warning>W\d+))\)'
    )
    defaults = {
        'selector': 'source.js - meta.attribute-with-value',
        '--filename:': '${file:${folder}/unsaved.js}'
    }

    def split_match(self, match):
        """
        Return the components of the match.

        We override this to catch linter error messages and return more presise
        info used for highlighting.

        """
        if match:
            # now safe to proceed, no error occured with jshint
            error = match.group('error')
            warning = match.group('warning')
            message = match.group('message')
            # force line numbers to be at least 0
            # if not they appear at end of file
            line = max(int(match.group('line')) - 1, 0)
            col = int(match.group('col')) - 1
            near = None

            # Note: jshint usually produces a `col` value, but sometimes the
            # col points to the last char of the offending code. When a col is
            # given, we can simply adjust the size of the error using near bc
            # SublimeLinter will just take the length of near in that case.
            # So when you see `col -= ...` we just shift the beginning of the
            # error to the left.

            if warning:
                # unexpected use of ++ etc.
                if warning == 'W016':
                    near = match.group('unexpected')

                # mark the duplicate key
                elif warning == 'W075' and match.group('duplicate'):
                    near = match.group('duplicate')
                    col -= len(match.group('duplicate'))

                # mark the no camel case key
                elif warning == 'W106':
                    near = match.group('no_camel')
                    col -= len(match.group('no_camel'))

                # if we have a operator == or != manually change the column,
                # this also handles the warning when curly brackets are required
                elif warning == 'W116':
                    # match the actual result
                    near = match.group('actual')

                    # if a comparison then also change the column
                    if near == '!=' or near == '==':
                        col -= len(near)

            return match, line, col, error, warning, message, near

        return None
<commit_msg>Add comment about `--filename` hack<commit_after>from SublimeLinter.lint import NodeLinter

class JSHint(NodeLinter):
    npm_name = 'jshint'
    cmd = ['jshint', '--verbose', '${args}', '-']
    regex = (
        r'^.+?: line (?P<line>\d+), col (?P<col>\d+), '
        r'(?P<message>'
        # unexpected use of ++ etc
        r'|.+\'(?P<unexpected>.+)\'\.(?=.+W016)'
        # duplicate key
        r'|.+\'(?P<duplicate>.+)\'.+(?=.+W075)'
        # camel case
        r'|.+\'(?P<no_camel>.+)\'.+(?=.+W106)'
        # unexpected use, typically use of non strict operators
        r'|.+\'(?P<actual>.+)\'\.(?=.+W116)'
        # match all messages
        r'|.+)'
        # capture error, warning and code
        r' \((?:(?P<error>E\d+)|(?P<warning>W\d+))\)'
    )
    defaults = {
        'selector': 'source.js - meta.attribute-with-value',
        # `filename` will determine the config finding algo of jshint. We
        # fake a name for unsaved files bc we sadly cannot just point to a
        # folder.
        '--filename:': '${file:${folder}/unsaved.js}'
    }

    def split_match(self, match):
        """
        Return the components of the match.

        We override this to catch linter error messages and return more presise
        info used for highlighting.

        """
        if match:
            # now safe to proceed, no error occured with jshint
            error = match.group('error')
            warning = match.group('warning')
            message = match.group('message')
            # force line numbers to be at least 0
            # if not they appear at end of file
            line = max(int(match.group('line')) - 1, 0)
            col = int(match.group('col')) - 1
            near = None

            # Note: jshint usually produces a `col` value, but sometimes the
            # col points to the last char of the offending code. When a col is
            # given, we can simply adjust the size of the error using near bc
            # SublimeLinter will just take the length of near in that case.
            # So when you see `col -= ...` we just shift the beginning of the
            # error to the left.

            if warning:
                # unexpected use of ++ etc.
                if warning == 'W016':
                    near = match.group('unexpected')

                # mark the duplicate key
                elif warning == 'W075' and match.group('duplicate'):
                    near = match.group('duplicate')
                    col -= len(match.group('duplicate'))

                # mark the no camel case key
                elif warning == 'W106':
                    near = match.group('no_camel')
                    col -= len(match.group('no_camel'))

                # if we have a operator == or != manually change the column,
                # this also handles the warning when curly brackets are required
                elif warning == 'W116':
                    # match the actual result
                    near = match.group('actual')

                    # if a comparison then also change the column
                    if near == '!=' or near == '==':
                        col -= len(near)

            return match, line, col, error, warning, message, near

        return None
<|endoftext|>


<commit_before>__author__ = "Tsuyoshi Hombashi"
__copyright__ = "Copyright 2016, {}".format(__author__)
__license__ = "MIT License"
__version__ = "0.32.1"
__maintainer__ = __author__
__email__ = "tsuyoshi.hombashi@gmail.com"
<commit_msg>Bump version<commit_after>__author__ = "Tsuyoshi Hombashi"
__copyright__ = "Copyright 2016, {}".format(__author__)
__license__ = "MIT License"
__version__ = "0.33.0"
__maintainer__ = __author__
__email__ = "tsuyoshi.hombashi@gmail.com"
<|endoftext|>


<commit_before>import os
import argparse
import cppyy
import PIL.Image
from typing import NamedTuple, Tuple, List
import numpy as np
import time
import ctypes
import xml.etree.ElementTree as ET

def get_naoth_dir():
    script_path = os.path.abspath(__file__)
    return os.path.abspath(os.path.join(script_path, "../../../../../"))

def get_toolchain_dir():
    return os.path.abspath(os.environ["NAOTH_TOOLCHAIN_PATH"])

class Point2D(NamedTuple):
    x: float
    y: float

class Rectangle(NamedTuple):
    top_left: Point2D
    bottom_right: Point2D

    def intersection_over_union(self, xtl: float, ytl: float, xbr: float, ybr: float):

        # compute x and y coordinates of the intersection rectangle
        intersection_topleft = Point2D(
            max(self.top_left.x, xtl), max(self.top_left.y, ytl))
        intersection_bottomright = Point2D(
            min(self.bottom_right.x, xbr), min(self.bottom_right.y, ybr))
        intersection = Rectangle(intersection_topleft,
                                 intersection_bottomright)

        # compute the intersection, self and other area
        intersectionArea = max(0, intersection.bottom_right.x - intersection.top_left.x + 1) * \
            max(0, intersection.bottom_right.y - intersection.top_left.y + 1)

        selfArea = (self.bottom_right.x - self.top_left.x + 1) * \
            (self.bottom_right.y - self.top_left.y + 1)
        otherArea = (xbr - xtl + 1) * (ybr - ytl + 1)

        # return the intersecton over union
        return intersectionArea / float(selfArea + otherArea - intersectionArea)

class Frame(NamedTuple):
    file: str
    bottom: bool
    balls: List[Rectangle]
    cam_matrix_translation: Tuple[float, float, float]
    cam_matrix_rotation: np.array

def get_frames_for_dir(d):
    # parse the CVAT XML 1.1 file with the annotations, we assume it has the same name as the directory, but ends with ".xml"
    annos_file = d.rstrip("/\\") + ".xml"
    if os.path.isfile(annos_file):
        annos = ET.parse(annos_file)
    else:
        raise "annotation file " + annos_file + " was not found!"

    file_names = os.listdir(d)
    file_names.sort()
    imageFiles = [os.path.join(d, f) for f in file_names if os.path.isfile(
        os.path.join(d, f)) and f.endswith(".png")]

    result = list()
    for file in imageFiles:

        # open image to get the metadata
        img = PIL.Image.open(file)
        bottom = img.info["CameraID"] == "1"
        # parse camera matrix using metadata in the PNG file
        cam_matrix_translation = (float(img.info["t_x"]), float(
            img.info["t_y"]), float(img.info["t_z"]))

        cam_matrix_rotation = np.array(
            [
                [float(img.info["r_11"]), float(
                    img.info["r_12"]), float(img.info["r_13"])],
                [float(img.info["r_21"]), float(
                    img.info["r_22"]), float(img.info["r_23"])],
                [float(img.info["r_31"]), float(
                    img.info["r_32"]), float(img.info["r_33"])]
            ])
        # add ground truth (all actual balls) to the frame
        balls = list()
        # Search for XML tag which name attribute has the filename as value
        for m in annos.findall(".//image[@name='{}']/box[@label='ball']".format(os.path.basename(file))):
            top_left = Point2D(float(m.attrib["xtl"]), float(m.attrib["ytl"]))
            bottom_right = Point2D(
                float(m.attrib["xbr"]), float(m.attrib["ybr"]))
            balls.append(Rectangle(top_left, bottom_right))
        frame = Frame(file, bottom, balls, cam_matrix_translation,
                      cam_matrix_rotation)

        result.append(frame)

    return result

def load_image(f: Frame):
    # don't import cv globally, because the dummy simulator shared library might need to load a non-system library
    # and we need to make sure loading the dummy simulator shared library happens first
    import cv2
    cv_img = cv2.imread(f)

    # convert image for bottom to yuv422
    cv_img = cv2.cvtColor(cv_img, cv2.COLOR_BGR2YUV).tobytes()
    yuv422 = np.ndarray(480*640*2, np.uint8)
    for i in range(0, 480 * 640, 2):
        yuv422[i*2] = cv_img[i*3]
        yuv422[i*2 + 1] = (cv_img[i*3 + 1] + cv_img[i*3 + 4]) / 2.0
        yuv422[i*2 + 2] = cv_img[i*3 + 3]
        yuv422[i*2 + 3] = (cv_img[i*3 + 2] + cv_img[i*3 + 5]) / 2.0
    return yuv422

def create_debug_image(file, balls, patches):
    import cv2

    img = cv2.imread(file)
    # draw all actual balls
    for b in balls:
        cv2.rectangle(img, (int(b.top_left.x), int(b.top_left.y)), (int(
            b.bottom_right.x), int(b.bottom_right.y)), (0, 165, 255))

    # draw the generated patches with their respective IOU (with first ball)
    for p in patches:
        cv2.rectangle(img, (p.min.x, p.min.y), (p.max.x, p.max.y), (0, 0, 255))
        if len(balls) > 0:
            iou = balls[0].intersection_over_union(
                p.min.x, p.min.y, p.max.x, p.max.y)
            cv2.addText(img, str(iou), (p.min.x + 5, p.min.y),
                        "Serif", pointSize=12, color=(0, 0, 255))

    return img

class Evaluator:

    def __init__(self):
        naoth_dir = get_naoth_dir()
        toolchain_dir = get_toolchain_dir()

        # load shared library: all depending libraries should be found automatically
        cppyy.load_library(os.path.join(
            naoth_dir, "NaoTHSoccer/dist/Native/libdummysimulator.so"))

        # add relevant include paths to allow mapping our code
        cppyy.add_include_path(os.path.join(
            naoth_dir, "Framework/Commons/Source"))
        cppyy.add_include_path(os.path.join(naoth_dir, "NaoTHSoccer/Source"))
        cppyy.add_include_path(os.path.join(
            toolchain_dir, "toolchain_native/extern/include"))

        # include platform
        cppyy.include(os.path.join(
            naoth_dir, "Framework/Commons/Source/PlatformInterface/Platform.h"))
        cppyy.include(os.path.join(
            naoth_dir, "Framework/Platforms/Source/DummySimulator/DummySimulator.h"))

        # change working directory so that the configuration is found
        os.chdir(os.path.join(naoth_dir, "NaoTHSoccer"))

        # start dummy simulator
        self.sim = cppyy.gbl.DummySimulator(False, False, 5401)
        cppyy.gbl.naoth.Platform.getInstance().init(self.sim)

        # create the cognition and motion objects
        cog = cppyy.gbl.createCognition()
        mo = cppyy.gbl.createMotion()

        # cast to callable
        callable_cog = cppyy.bind_object(
            cppyy.addressof(cog), cppyy.gbl.naoth.Callable)
        callable_mo = cppyy.bind_object(
            cppyy.addressof(mo), cppyy.gbl.naoth.Callable)

        self.sim.registerCognition(callable_cog)
        self.sim.registerMotion(callable_mo)

        # make more representations available to cppyy
        cppyy.include(os.path.join(
            naoth_dir, "Framework/Commons/Source/ModuleFramework/ModuleManager.h"))
        cppyy.include(os.path.join(
            naoth_dir, "NaoTHSoccer/Source/Cognition/Cognition.h"))
        cppyy.include(os.path.join(
            naoth_dir, "NaoTHSoccer/Source/Representations/Perception/BallCandidates.h"))
        cppyy.include(os.path.join(
            naoth_dir, "NaoTHSoccer/Source/Representations/Perception/CameraMatrix.h"))

        # get access to the module manager and return it to the calling function
        self.moduleManager = cppyy.gbl.getModuleManager(cog)

        # get the ball detector module
        self.ball_detector = self.moduleManager.getModule(
            "CNNBallDetector").getModule()

        # initialize the score object
        self.scores = dict()

    def set_current_frame(self, frame):
        # get reference to the image input representation
        if frame.bottom:
            imageRepresentation = self.ball_detector.getRequire().at("Image")
        else:
            imageRepresentation = self.ball_detector.getRequire().at("ImageTop")

        print(frame.file)

        # load image in YUV422 format
        yuv422 = load_image(frame.file)
        p_data = yuv422.ctypes.data_as(ctypes.POINTER(ctypes.c_uint8))
        # move image into representation
        imageRepresentation.copyImageDataYUV422(p_data, yuv422.size)

        # set camera matrix
        if frame.bottom:
            camMatrix = self.ball_detector.getRequire().at("CameraMatrix")
        else:
            camMatrix = self.ball_detector.getRequire().at("CameraMatrixTop")

        # TODO: invalidate other camera matrix

        camMatrix.valid = True
        camMatrix.translation.x = frame.cam_matrix_translation[0]
        camMatrix.translation.y = frame.cam_matrix_translation[1]
        camMatrix.translation.z = frame.cam_matrix_translation[2]

        for c in range(0, 3):
            for r in range(0, 3):
                camMatrix.rotation.c[c][r] = frame.cam_matrix_rotation[r, c]

    def evaluate_detection(self, frame: Frame, eval_functions, show_debug_image=False):
        import cv2 as cv

        # get the ball candidates from the module
        if frame.bottom:
            detected_balls = self.ball_detector.getProvide().at("BallCandidates")
        else:
            detected_balls = self.ball_detector.getProvide().at("BallCandidatesTop")

        for score_name, f in eval_functions.items():
            score = f(frame, detected_balls.patchesYUVClassified)
            self.scores[score_name].append(score)

        if show_debug_image:
            img = create_debug_image(
                frame.file, frame.balls, detected_balls.patchesYUVClassified)
            cv.imshow("Ball Evaluator", img)
            cv.waitKey(1)

    def execute(self, directories, eval_functions):

        # init the score lists
        for score_name, f in eval_functions.items():
            self.scores[score_name] = list()

        for d in directories:
            frames = get_frames_for_dir(d)
            for f in frames:
                self.set_current_frame(f)
                self.sim.executeFrame()
                self.evaluate_detection(f, eval_functions)

    def show_report(self):
        for score_name, score_values in self.scores.items():
            print()
            print(score_name)
            print("=" * len(score_name))
            # get percentile, average and other basic informations
            scores = np.array(score_values)

            print("average: {}".format(np.average(scores)))
            print()

            for percentile in [1, 5, 10, 25, 30, 40, 50, 75, 90, 99]:
                if percentile == 50:
                    marker = "(median)"
                else:
                    marker = ""
                print("best {}% >= {} {}".format(100 - percentile,
                                              np.percentile(scores, percentile), marker))

        # TODO: collect items with the worst scores

def best_ball_patch_intersection(frame, patches):

    if len(frame.balls) == 0:
        # Don't penalize the patch detector for not finding a non-existent ball
        return 1.0

    best = 0.0
    for p in patches:
        for b in frame.balls:
            iuo = b.intersection_over_union(p.min.x, p.min.y, p.max.x, p.max.y)
            if iuo > best:
                best = iuo
    return best

if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description='Evaluate ball detection on logfile images annotated with CVAT.')
    parser.add_argument('directory', nargs='+',
                        help='A list of directories containing the images files (as png with the integrated camera matrix)')

    args = parser.parse_args()

    evaluator = Evaluator()
    evaluator.execute(
        args.directory, {"Best IOU per image": best_ball_patch_intersection})
    evaluator.show_report()
<commit_msg>Add parameter to show debug image when best IOU is below threshold<commit_after>import os
import argparse
import cppyy
import PIL.Image
from typing import NamedTuple, Tuple, List
import numpy as np
import time
import ctypes
import xml.etree.ElementTree as ET

def get_naoth_dir():
    script_path = os.path.abspath(__file__)
    return os.path.abspath(os.path.join(script_path, "../../../../../"))

def get_toolchain_dir():
    return os.path.abspath(os.environ["NAOTH_TOOLCHAIN_PATH"])

class Point2D(NamedTuple):
    x: float
    y: float

class Rectangle(NamedTuple):
    top_left: Point2D
    bottom_right: Point2D

    def intersection_over_union(self, xtl: float, ytl: float, xbr: float, ybr: float):

        # compute x and y coordinates of the intersection rectangle
        intersection_topleft = Point2D(
            max(self.top_left.x, xtl), max(self.top_left.y, ytl))
        intersection_bottomright = Point2D(
            min(self.bottom_right.x, xbr), min(self.bottom_right.y, ybr))
        intersection = Rectangle(intersection_topleft,
                                 intersection_bottomright)

        # compute the intersection, self and other area
        intersectionArea = max(0, intersection.bottom_right.x - intersection.top_left.x + 1) * \
            max(0, intersection.bottom_right.y - intersection.top_left.y + 1)

        selfArea = (self.bottom_right.x - self.top_left.x + 1) * \
            (self.bottom_right.y - self.top_left.y + 1)
        otherArea = (xbr - xtl + 1) * (ybr - ytl + 1)

        # return the intersecton over union
        return intersectionArea / float(selfArea + otherArea - intersectionArea)

class Frame(NamedTuple):
    file: str
    bottom: bool
    balls: List[Rectangle]
    cam_matrix_translation: Tuple[float, float, float]
    cam_matrix_rotation: np.array

def get_frames_for_dir(d):
    # parse the CVAT XML 1.1 file with the annotations, we assume it has the same name as the directory, but ends with ".xml"
    annos_file = d.rstrip("/\\") + ".xml"
    if os.path.isfile(annos_file):
        annos = ET.parse(annos_file)
    else:
        raise "annotation file " + annos_file + " was not found!"

    file_names = os.listdir(d)
    file_names.sort()
    imageFiles = [os.path.join(d, f) for f in file_names if os.path.isfile(
        os.path.join(d, f)) and f.endswith(".png")]

    result = list()
    for file in imageFiles:

        # open image to get the metadata
        img = PIL.Image.open(file)
        bottom = img.info["CameraID"] == "1"
        # parse camera matrix using metadata in the PNG file
        cam_matrix_translation = (float(img.info["t_x"]), float(
            img.info["t_y"]), float(img.info["t_z"]))

        cam_matrix_rotation = np.array(
            [
                [float(img.info["r_11"]), float(
                    img.info["r_12"]), float(img.info["r_13"])],
                [float(img.info["r_21"]), float(
                    img.info["r_22"]), float(img.info["r_23"])],
                [float(img.info["r_31"]), float(
                    img.info["r_32"]), float(img.info["r_33"])]
            ])
        # add ground truth (all actual balls) to the frame
        balls = list()
        # Search for XML tag which name attribute has the filename as value
        for m in annos.findall(".//image[@name='{}']/box[@label='ball']".format(os.path.basename(file))):
            top_left = Point2D(float(m.attrib["xtl"]), float(m.attrib["ytl"]))
            bottom_right = Point2D(
                float(m.attrib["xbr"]), float(m.attrib["ybr"]))
            balls.append(Rectangle(top_left, bottom_right))
        frame = Frame(file, bottom, balls, cam_matrix_translation,
                      cam_matrix_rotation)

        result.append(frame)

    return result

def load_image(f: Frame):
    # don't import cv globally, because the dummy simulator shared library might need to load a non-system library
    # and we need to make sure loading the dummy simulator shared library happens first
    import cv2
    cv_img = cv2.imread(f)

    # convert image for bottom to yuv422
    cv_img = cv2.cvtColor(cv_img, cv2.COLOR_BGR2YUV).tobytes()
    yuv422 = np.ndarray(480*640*2, np.uint8)
    for i in range(0, 480 * 640, 2):
        yuv422[i*2] = cv_img[i*3]
        yuv422[i*2 + 1] = (cv_img[i*3 + 1] + cv_img[i*3 + 4]) / 2.0
        yuv422[i*2 + 2] = cv_img[i*3 + 3]
        yuv422[i*2 + 3] = (cv_img[i*3 + 2] + cv_img[i*3 + 5]) / 2.0
    return yuv422

def create_debug_image(file, balls, patches):
    import cv2

    img = cv2.imread(file)
    # draw all actual balls
    for b in balls:
        cv2.rectangle(img, (int(b.top_left.x), int(b.top_left.y)), (int(
            b.bottom_right.x), int(b.bottom_right.y)), (0, 165, 255))

    # draw the generated patches with their respective IOU (with first ball)
    for p in patches:
        cv2.rectangle(img, (p.min.x, p.min.y), (p.max.x, p.max.y), (0, 0, 255))
        if len(balls) > 0:
            iou = balls[0].intersection_over_union(
                p.min.x, p.min.y, p.max.x, p.max.y)
            cv2.addText(img, str(iou), (p.min.x + 5, p.min.y),
                        "Serif", pointSize=12, color=(0, 0, 255))

    return img

class Evaluator:

    def __init__(self):
        naoth_dir = get_naoth_dir()
        toolchain_dir = get_toolchain_dir()

        # load shared library: all depending libraries should be found automatically
        cppyy.load_library(os.path.join(
            naoth_dir, "NaoTHSoccer/dist/Native/libdummysimulator.so"))

        # add relevant include paths to allow mapping our code
        cppyy.add_include_path(os.path.join(
            naoth_dir, "Framework/Commons/Source"))
        cppyy.add_include_path(os.path.join(naoth_dir, "NaoTHSoccer/Source"))
        cppyy.add_include_path(os.path.join(
            toolchain_dir, "toolchain_native/extern/include"))

        # include platform
        cppyy.include(os.path.join(
            naoth_dir, "Framework/Commons/Source/PlatformInterface/Platform.h"))
        cppyy.include(os.path.join(
            naoth_dir, "Framework/Platforms/Source/DummySimulator/DummySimulator.h"))

        # change working directory so that the configuration is found
        os.chdir(os.path.join(naoth_dir, "NaoTHSoccer"))

        # start dummy simulator
        self.sim = cppyy.gbl.DummySimulator(False, False, 5401)
        cppyy.gbl.naoth.Platform.getInstance().init(self.sim)

        # create the cognition and motion objects
        cog = cppyy.gbl.createCognition()
        mo = cppyy.gbl.createMotion()

        # cast to callable
        callable_cog = cppyy.bind_object(
            cppyy.addressof(cog), cppyy.gbl.naoth.Callable)
        callable_mo = cppyy.bind_object(
            cppyy.addressof(mo), cppyy.gbl.naoth.Callable)

        self.sim.registerCognition(callable_cog)
        self.sim.registerMotion(callable_mo)

        # make more representations available to cppyy
        cppyy.include(os.path.join(
            naoth_dir, "Framework/Commons/Source/ModuleFramework/ModuleManager.h"))
        cppyy.include(os.path.join(
            naoth_dir, "NaoTHSoccer/Source/Cognition/Cognition.h"))
        cppyy.include(os.path.join(
            naoth_dir, "NaoTHSoccer/Source/Representations/Perception/BallCandidates.h"))
        cppyy.include(os.path.join(
            naoth_dir, "NaoTHSoccer/Source/Representations/Perception/CameraMatrix.h"))

        # get access to the module manager and return it to the calling function
        self.moduleManager = cppyy.gbl.getModuleManager(cog)

        # get the ball detector module
        self.ball_detector = self.moduleManager.getModule(
            "CNNBallDetector").getModule()

        # initialize the score object
        self.scores = dict()

    def set_current_frame(self, frame):
        # get reference to the image input representation
        if frame.bottom:
            imageRepresentation = self.ball_detector.getRequire().at("Image")
        else:
            imageRepresentation = self.ball_detector.getRequire().at("ImageTop")

        print(frame.file)

        # load image in YUV422 format
        yuv422 = load_image(frame.file)
        p_data = yuv422.ctypes.data_as(ctypes.POINTER(ctypes.c_uint8))
        # move image into representation
        imageRepresentation.copyImageDataYUV422(p_data, yuv422.size)

        # set camera matrix
        if frame.bottom:
            camMatrix = self.ball_detector.getRequire().at("CameraMatrix")
        else:
            camMatrix = self.ball_detector.getRequire().at("CameraMatrixTop")

        # TODO: invalidate other camera matrix

        camMatrix.valid = True
        camMatrix.translation.x = frame.cam_matrix_translation[0]
        camMatrix.translation.y = frame.cam_matrix_translation[1]
        camMatrix.translation.z = frame.cam_matrix_translation[2]

        for c in range(0, 3):
            for r in range(0, 3):
                camMatrix.rotation.c[c][r] = frame.cam_matrix_rotation[r, c]

    def evaluate_detection(self, frame: Frame, eval_functions, debug_threshold=None):
        import cv2

        # get the ball candidates from the module
        if frame.bottom:
            detected_balls = self.ball_detector.getProvide().at("BallCandidates")
        else:
            detected_balls = self.ball_detector.getProvide().at("BallCandidatesTop")

        debug = False
        for score_name, f in eval_functions.items():
            score = f(frame, detected_balls.patchesYUVClassified)
            self.scores[score_name].append(score)

            if debug_threshold is not None and score <= debug_threshold:
                debug = True

        title = "Ball Evaluator: press any key to continue or Q to exit"
        if debug:
            img = create_debug_image(
                frame.file, frame.balls, detected_balls.patchesYUVClassified)
            cv2.imshow(title, img)
            key = cv2.waitKey()
        else:
            img = np.ones((240, 320))*255
            cv2.addText(img, "Image {} is over threshold".format(frame.file), (0,0),
                        "Serif", pointSize=12, color=(0, 0, 0))
            cv2.imshow(title, img)

            key = cv2.waitKey(1)

        if key == 113 or key == 27:
            exit(0)

    def execute(self, directories, eval_functions, debug_threshold=None):

        # init the score lists
        for score_name, f in eval_functions.items():
            self.scores[score_name] = list()

        for d in directories:
            frames = get_frames_for_dir(d)
            for f in frames:
                self.set_current_frame(f)
                self.sim.executeFrame()
                self.evaluate_detection(f, eval_functions, debug_threshold)

    def show_report(self):
        for score_name, score_values in self.scores.items():
            print()
            print(score_name)
            print("=" * len(score_name))
            # get percentile, average and other basic informations
            scores = np.array(score_values)

            print("average: {}".format(np.average(scores)))
            print()

            for percentile in [1, 5, 10, 25, 30, 40, 50, 75, 90, 99]:
                if percentile == 50:
                    marker = "(median)"
                else:
                    marker = ""
                print("best {}% >= {} {}".format(100 - percentile,
                                                 np.percentile(scores, percentile), marker))

        # TODO: collect items with the worst scores

def best_ball_patch_intersection(frame, patches):

    if len(frame.balls) == 0:
        # Don't penalize the patch detector for not finding a non-existent ball
        return 1.0

    best = 0.0
    for p in patches:
        for b in frame.balls:
            iuo = b.intersection_over_union(p.min.x, p.min.y, p.max.x, p.max.y)
            if iuo > best:
                best = iuo
    return best

if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description='Evaluate ball detection on logfile images annotated with CVAT.')
    parser.add_argument('directory', nargs='+',
                        help='A list of directories containing the images files (as png with the integrated camera matrix)')

    parser.add_argument('--debug-threshold=T', type=float, dest="debug_threshold",
                        help="If an image has a score with a worse than the given threshold, include this image in a debug view at the end.")

    args = parser.parse_args()

    evaluator = Evaluator()
    evaluator.execute(
        args.directory, {"Best IOU per image": best_ball_patch_intersection}, args.debug_threshold)
    evaluator.show_report()
<|endoftext|>


<commit_before>"""
Copyright (c) 2017, 2018 Genome Research Ltd.

Authors:
* Christopher Harrison <ch12@sanger.ac.uk>
* Simon Beal <sb48@sanger.ac.uk>

This program is free software: you can redistribute it and/or modify it
under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or (at
your option) any later version.

This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""

import atexit
from datetime import date, timedelta, datetime
from typing import ClassVar, List

from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore
from apscheduler.schedulers.asyncio import AsyncIOScheduler
from apscheduler.triggers.date import DateTrigger
from pytz import utc

from cogs.common import logging
from cogs.db.interface import Database
from cogs.db.models import ProjectGroup
from cogs.mail import Postman
from cogs.file_handler import FileHandler
from . import jobs
from .constants import GROUP_DEADLINES, USER_DEADLINES

class Scheduler(logging.LogWriter):
    """ AsyncIO scheduler interface """
    _scheduler:AsyncIOScheduler
    _db:Database
    _mail:Postman
    _file_handler:FileHandler
    proxy:ClassVar["Scheduler"]

    def __init__(self, database:Database, mail:Postman, file_handler:FileHandler) -> None:
        """
        Constructor

        :param database:
        :param mail:
        :param file_handler:
        :return:
        """
        Scheduler.proxy = self
        self._db = database
        self._mail = mail
        self._file_handler = file_handler

        job_defaults = {
            # APScheduler will only fire events that are up to 31 days out of date
            # This should only happen if the program is abandoned for significant periods
            "misfire_grace_time": int(timedelta(days=31).total_seconds())}

        jobstores = {
            "default": SQLAlchemyJobStore(engine=database.engine)}

        self._scheduler = AsyncIOScheduler(
            logger=self._logger,
            timezone=utc,
            job_defaults=job_defaults,
            jobstores=jobstores)

        self._scheduler.start()
        atexit.register(self._scheduler.shutdown)

    @staticmethod
    async def _job(deadline:str, *args, **kwargs) -> None:
        """ Wrapper for the scheduled job, injecting itself """
        print(f"Running job: {deadline}(*{args}, **{kwargs})")
        await getattr(jobs, deadline)(Scheduler.proxy, *args, **kwargs)

    def reset_all(self) -> None:
        """ Remove all jobs """
        self._scheduler.remove_all_jobs()

    def schedule_deadline(self, when:date, deadline:str, group:ProjectGroup, suffix:str="", recipients:List[str]=[], *args, **kwargs) -> None:
        """
        Schedule a deadline for the project group

        :param when:
        :param deadline:
        :param group:
        :param suffix:
        :param recipients:
        :return:
        """
        assert deadline in GROUP_DEADLINES

        schedule_time = self.fix_time(when)

        # Main deadline
        job_id = f"{group.series}_{group.part}_{deadline}_{suffix}"
        self.log(logging.DEBUG, f"Scheduling a deadline `{job_id}` to be ran at `{schedule_time}`")
        self._scheduler.add_job(self._job,
            trigger          = DateTrigger(run_date=schedule_time),
            id               = job_id,
            args             = (deadline, *args),
            kwargs           = kwargs,
            replace_existing = True)

        # Pester points
        # The pester job contains logic to ensure that tasks are only
        # completed if the appropriate conditions are met, otherwise
        # they're effectively no-ops
        for delta_day in GROUP_DEADLINES[deadline].pester_times:
            pester_job_id = f"pester_{delta_day}_{job_id}"
            pester_time = schedule_time - timedelta(days=delta_day)
            self._scheduler.add_job(self._job,
                trigger          = DateTrigger(run_date=pester_time),
                id               = pester_job_id,
                args             = ("pester", deadline, delta_day, group.series, group.part, *recipients),
                replace_existing = True)

    def schedule_user_deadline(self, when:date, deadline, suffix, *args, **kwargs):
        assert deadline in USER_DEADLINES
        schedule_time = self.fix_time(when)
        job_id = f"{deadline}_{suffix}"
        self.log(logging.DEBUG, f"Scheduling a user deadline `{job_id}` to be ran at `{schedule_time}`")
        self._scheduler.add_job(self._job,
                                trigger          = DateTrigger(run_date=schedule_time),
                                id               = job_id,
                                args             = (deadline, *args),
                                kwargs           = kwargs,
                                replace_existing = True)

    def fix_time(self, when: date) -> datetime:
        """
        Given a date, return the actual time the deadline should be scheduled for

        :param when:
        :return:
        """
        return datetime(
            year=when.year,
            month=when.month,
            day=when.day,
            hour=23,
            minute=59
        )

    def get_job(self, job_id):
        return self._scheduler.get_job(job_id)

<commit_msg>Debug output of scheduled jobs on startup<commit_after>"""
Copyright (c) 2017, 2018 Genome Research Ltd.

Authors:
* Christopher Harrison <ch12@sanger.ac.uk>
* Simon Beal <sb48@sanger.ac.uk>

This program is free software: you can redistribute it and/or modify it
under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or (at
your option) any later version.

This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""

import atexit
from datetime import date, timedelta, datetime
from typing import ClassVar, List

from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore
from apscheduler.schedulers.asyncio import AsyncIOScheduler
from apscheduler.triggers.date import DateTrigger
from pytz import utc

from cogs.common import logging
from cogs.db.interface import Database
from cogs.db.models import ProjectGroup
from cogs.mail import Postman
from cogs.file_handler import FileHandler
from . import jobs
from .constants import GROUP_DEADLINES, USER_DEADLINES

class Scheduler(logging.LogWriter):
    """ AsyncIO scheduler interface """
    _scheduler:AsyncIOScheduler
    _db:Database
    _mail:Postman
    _file_handler:FileHandler
    proxy:ClassVar["Scheduler"]

    def __init__(self, database:Database, mail:Postman, file_handler:FileHandler) -> None:
        """
        Constructor

        :param database:
        :param mail:
        :param file_handler:
        :return:
        """
        Scheduler.proxy = self
        self._db = database
        self._mail = mail
        self._file_handler = file_handler

        job_defaults = {
            # APScheduler will only fire events that are up to 31 days out of date
            # This should only happen if the program is abandoned for significant periods
            "misfire_grace_time": int(timedelta(days=31).total_seconds())}

        jobstores = {
            "default": SQLAlchemyJobStore(engine=database.engine)}

        self._scheduler = AsyncIOScheduler(
            logger=self._logger,
            timezone=utc,
            job_defaults=job_defaults,
            jobstores=jobstores)

        self._scheduler.start()

        for job in self._scheduler.get_jobs():
            self.log(logging.DEBUG, f"name: {job.name}; "
                                    f"trigger: {job.trigger}; "
                                    f"next run: {job.next_run_time}; "
                                    f"handler: {job.func}; "
                                    f"args: {job.args}; "
                                    f"kwargs: {job.kwargs}; "
                                    f"misfire: {job.misfire_grace_time}")

        atexit.register(self._scheduler.shutdown)

    @staticmethod
    async def _job(deadline:str, *args, **kwargs) -> None:
        """ Wrapper for the scheduled job, injecting itself """
        print(f"Running job: {deadline}(*{args}, **{kwargs})")
        await getattr(jobs, deadline)(Scheduler.proxy, *args, **kwargs)

    def reset_all(self) -> None:
        """ Remove all jobs """
        self._scheduler.remove_all_jobs()

    def schedule_deadline(self, when:date, deadline:str, group:ProjectGroup, suffix:str="", recipients:List[str]=[], *args, **kwargs) -> None:
        """
        Schedule a deadline for the project group

        :param when:
        :param deadline:
        :param group:
        :param suffix:
        :param recipients:
        :return:
        """
        assert deadline in GROUP_DEADLINES

        schedule_time = self.fix_time(when)

        # Main deadline
        job_id = f"{group.series}_{group.part}_{deadline}_{suffix}"
        self.log(logging.DEBUG, f"Scheduling a deadline `{job_id}` to be ran at `{schedule_time}`")
        self._scheduler.add_job(self._job,
            trigger          = DateTrigger(run_date=schedule_time),
            id               = job_id,
            args             = (deadline, *args),
            kwargs           = kwargs,
            replace_existing = True)

        # Pester points
        # The pester job contains logic to ensure that tasks are only
        # completed if the appropriate conditions are met, otherwise
        # they're effectively no-ops
        for delta_day in GROUP_DEADLINES[deadline].pester_times:
            pester_job_id = f"pester_{delta_day}_{job_id}"
            pester_time = schedule_time - timedelta(days=delta_day)
            self._scheduler.add_job(self._job,
                trigger          = DateTrigger(run_date=pester_time),
                id               = pester_job_id,
                args             = ("pester", deadline, delta_day, group.series, group.part, *recipients),
                replace_existing = True)

    def schedule_user_deadline(self, when:date, deadline, suffix, *args, **kwargs):
        assert deadline in USER_DEADLINES
        schedule_time = self.fix_time(when)
        job_id = f"{deadline}_{suffix}"
        self.log(logging.DEBUG, f"Scheduling a user deadline `{job_id}` to be ran at `{schedule_time}`")
        self._scheduler.add_job(self._job,
                                trigger          = DateTrigger(run_date=schedule_time),
                                id               = job_id,
                                args             = (deadline, *args),
                                kwargs           = kwargs,
                                replace_existing = True)

    def fix_time(self, when: date) -> datetime:
        """
        Given a date, return the actual time the deadline should be scheduled for

        :param when:
        :return:
        """
        return datetime(
            year=when.year,
            month=when.month,
            day=when.day,
            hour=23,
            minute=59
        )

    def get_job(self, job_id):
        return self._scheduler.get_job(job_id)

<|endoftext|>