<commit_before># ==================================================================================================
# Copyright 2011 Twitter, Inc.
# --------------------------------------------------------------------------------------------------
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this work except in compliance with the License.
# You may obtain a copy of the License in the LICENSE file, or at:
#
#  http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==================================================================================================

from .exportable_jvm_library import ExportableJvmLibrary
from .resources import WithLegacyResources

class AnnotationProcessor(ExportableJvmLibrary, WithLegacyResources):
  """Defines a target that produces a java library containing one or more annotation processors."""

  def __init__(self, name, sources, provides=None, dependencies=None, excludes=None, resources=None,
               processors=None, exclusives=None):

    """name: The name of this module target, addressable via pants via the portion of the spec
        following the colon
    sources: A list of paths containing the java source files this modules jar is compiled from
    provides: An optional Dependency object indicating the The ivy artifact to export
    dependencies: An optional list of Dependency objects specifying the binary (jar) dependencies of
        this module.
    excludes: An optional list of dependency exclude patterns to filter all of this module's
        transitive dependencies against.
    resources: An optional list of paths containing (filterable) text file resources to place in
        this module's jar
    processors: a list of the fully qualified class names of the annotation processors this library
        exports
    exclusives:   An optional map of exclusives tags. See CheckExclusives for details.
    """

    ExportableJvmLibrary.__init__(self, name, sources, provides, dependencies, excludes,
                                  exclusives=exclusives)
    WithLegacyResources.__init__(self, name, sources=sources, resources=resources)
    self.add_labels('java', 'apt')
    self.processors = processors
<commit_msg>AnnotationProcessor bug was already fixed...<commit_after># ==================================================================================================
# Copyright 2011 Twitter, Inc.
# --------------------------------------------------------------------------------------------------
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this work except in compliance with the License.
# You may obtain a copy of the License in the LICENSE file, or at:
#
#  http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==================================================================================================

from .exportable_jvm_library import ExportableJvmLibrary
from .resources import WithLegacyResources

class AnnotationProcessor(ExportableJvmLibrary, WithLegacyResources):
  """Defines a target that produces a java library containing one or more annotation processors."""

  def __init__(self, name, sources, provides=None, dependencies=None, excludes=None, resources=None,
               processors=None, exclusives=None):

    """name: The name of this module target, addressable via pants via the portion of the spec
        following the colon
    sources: A list of paths containing the java source files this modules jar is compiled from
    provides: An optional Dependency object indicating the The ivy artifact to export
    dependencies: An optional list of Dependency objects specifying the binary (jar) dependencies of
        this module.
    excludes: An optional list of dependency exclude patterns to filter all of this module's
        transitive dependencies against.
    resources: An optional list of paths containing (filterable) text file resources to place in
        this module's jar
    processors: a list of the fully qualified class names of the annotation processors this library
        exports
    exclusives:   An optional map of exclusives tags. See CheckExclusives for details.
    """

    ExportableJvmLibrary.__init__(self, name, sources, provides, dependencies, excludes,
                                  exclusives=exclusives)
    WithLegacyResources.__init__(self, name, sources=sources, resources=resources,
                                 exclusives=exclusives)
    self.add_labels('java', 'apt')
    self.processors = processors
<|endoftext|>


<commit_before>from django import forms
from django.core.exceptions import ValidationError

from taggit.forms import TagField

try:
    from django.apps.apps import get_model
except ImportError:
    from django.db.models import get_model

class TagForm(forms.Form):
    tags = TagField()

    app_label = forms.CharField(required=True, widget=forms.HiddenInput())
    model_name = forms.CharField(required=True, widget=forms.HiddenInput())
    object_id = forms.CharField(required=True, widget=forms.HiddenInput())

    def __init__(self, *args, **kwargs):
        target = kwargs.pop('target', None)
        if target is not None:
            initial = kwargs.get('initial', {})
            initial['app_label'] = target._meta.app_label
            initial['model_name'] = target._meta.model_name
            initial['object_id'] = target.pk
            kwargs['initial'] = initial
        super(TagForm, self).__init__(*args, **kwargs)

    _obj = None
    def clean(self):
        cleaned_data = super(TagForm, self).clean()

        try:
            Model = get_model(cleaned_data['app_label'], cleaned_data['model_name'])
        except LookupError as e:
            raise ValidationError(e.message)

        try:
            obj = Model._default_manager.get(pk=cleaned_data['object_id'])
        except Model.DoesNotExist:
            raise ValidationError('object does not exist')

        self._obj = obj

        return cleaned_data
<commit_msg>Fixed form validation issue for Python 3<commit_after>from django import forms
from django.core.exceptions import ValidationError

from taggit.forms import TagField

try:
    from django.apps.apps import get_model
except ImportError:
    from django.db.models import get_model

class TagForm(forms.Form):
    tags = TagField()

    app_label = forms.CharField(required=True, widget=forms.HiddenInput())
    model_name = forms.CharField(required=True, widget=forms.HiddenInput())
    object_id = forms.CharField(required=True, widget=forms.HiddenInput())

    def __init__(self, *args, **kwargs):
        target = kwargs.pop('target', None)
        if target is not None:
            initial = kwargs.get('initial', {})
            initial['app_label'] = target._meta.app_label
            initial['model_name'] = target._meta.model_name
            initial['object_id'] = target.pk
            kwargs['initial'] = initial
        super(TagForm, self).__init__(*args, **kwargs)

    _obj = None
    def clean(self):
        cleaned_data = super(TagForm, self).clean()

        app_label = cleaned_data.get('app_label')
        model_name = cleaned_data.get('model_name')
        object_id = cleaned_data.get('object_id')
        if not (app_label and model_name and object_id):
            raise ValidationError('unable to specify the object.')

        try:
            Model = get_model(app_label, model_name)
        except LookupError as e:
            raise ValidationError(e)

        try:
            obj = Model._default_manager.get(pk=object_id)
        except Model.DoesNotExist:
            raise ValidationError('object does not exist')

        self._obj = obj

        return cleaned_data
<|endoftext|>


<commit_before>#!/usr/bin/python

import time
import math

# ============================================================================
# Adafruit PCA9685 16-Channel PWM Servo Driver
# ============================================================================
from lightcontrol.ledcontrol import Adafruit_I2C

class PWM :
  # Registers/etc.
  __MODE1              = 0x00
  __MODE2              = 0x01
  __SUBADR1            = 0x02
  __SUBADR2            = 0x03
  __SUBADR3            = 0x04
  __PRESCALE           = 0xFE
  __LED0_ON_L          = 0x06
  __LED0_ON_H          = 0x07
  __LED0_OFF_L         = 0x08
  __LED0_OFF_H         = 0x09
  __ALL_LED_ON_L       = 0xFA
  __ALL_LED_ON_H       = 0xFB
  __ALL_LED_OFF_L      = 0xFC
  __ALL_LED_OFF_H      = 0xFD

  # Bits
  __RESTART            = 0x80
  __SLEEP              = 0x10
  __ALLCALL            = 0x01
  __INVRT              = 0x10
  __OUTDRV             = 0x04

  general_call_i2c = Adafruit_I2C(0x00)

  @classmethod
  def softwareReset(cls):
    "Sends a software reset (SWRST) command to all the servo drivers on the bus"
    cls.general_call_i2c.writeRaw8(0x06)        # SWRST

  def __init__(self, address=0x40, debug=False):
    self.i2c = Adafruit_I2C(address)
    self.i2c.debug = debug
    self.address = address
    self.debug = debug
    if (self.debug):
      print "Reseting PCA9685 MODE1 (without SLEEP) and MODE2"
    self.setAllPWM(0, 0)
    self.i2c.write8(self.__MODE2, self.__OUTDRV)
    self.i2c.write8(self.__MODE1, self.__ALLCALL)
    time.sleep(0.005)                                       # wait for oscillator

    mode1 = self.i2c.readU8(self.__MODE1)
    mode1 = mode1 & ~self.__SLEEP                 # wake up (reset sleep)
    self.i2c.write8(self.__MODE1, mode1)
    time.sleep(0.005)                             # wait for oscillator

  def setPWMFreq(self, freq):
    "Sets the PWM frequency"
    prescaleval = 25000000.0    # 25MHz
    prescaleval /= 4096.0       # 12-bit
    prescaleval /= float(freq)
    prescaleval -= 1.0
    if (self.debug):
      print "Setting PWM frequency to %d Hz" % freq
      print "Estimated pre-scale: %d" % prescaleval
    prescale = math.floor(prescaleval + 0.5)
    if (self.debug):
      print "Final pre-scale: %d" % prescale

    oldmode = self.i2c.readU8(self.__MODE1);
    newmode = (oldmode & 0x7F) | 0x10             # sleep
    self.i2c.write8(self.__MODE1, newmode)        # go to sleep
    self.i2c.write8(self.__PRESCALE, int(math.floor(prescale)))
    self.i2c.write8(self.__MODE1, oldmode)
    time.sleep(0.005)
    self.i2c.write8(self.__MODE1, oldmode | 0x80)

  def setPWM(self, channel, on, off):
    "Sets a single PWM channel"
    self.i2c.write8(self.__LED0_ON_L+4*channel, on & 0xFF)
    self.i2c.write8(self.__LED0_ON_H+4*channel, on >> 8)
    self.i2c.write8(self.__LED0_OFF_L+4*channel, off & 0xFF)
    self.i2c.write8(self.__LED0_OFF_H+4*channel, off >> 8)

  def setAllPWM(self, on, off):
    "Sets a all PWM channels"
    self.i2c.write8(self.__ALL_LED_ON_L, on & 0xFF)
    self.i2c.write8(self.__ALL_LED_ON_H, on >> 8)
    self.i2c.write8(self.__ALL_LED_OFF_L, off & 0xFF)
    self.i2c.write8(self.__ALL_LED_OFF_H, off >> 8)
<commit_msg>Corrected import Error<commit_after>#!/usr/bin/python

import time
import math

# ============================================================================
# Adafruit PCA9685 16-Channel PWM Servo Driver
# ============================================================================
from lightcontrol.ledcontrol.Adafruit_I2C import Adafruit_I2C

class PWM :
  # Registers/etc.
  __MODE1              = 0x00
  __MODE2              = 0x01
  __SUBADR1            = 0x02
  __SUBADR2            = 0x03
  __SUBADR3            = 0x04
  __PRESCALE           = 0xFE
  __LED0_ON_L          = 0x06
  __LED0_ON_H          = 0x07
  __LED0_OFF_L         = 0x08
  __LED0_OFF_H         = 0x09
  __ALL_LED_ON_L       = 0xFA
  __ALL_LED_ON_H       = 0xFB
  __ALL_LED_OFF_L      = 0xFC
  __ALL_LED_OFF_H      = 0xFD

  # Bits
  __RESTART            = 0x80
  __SLEEP              = 0x10
  __ALLCALL            = 0x01
  __INVRT              = 0x10
  __OUTDRV             = 0x04

  general_call_i2c = Adafruit_I2C(0x00)

  @classmethod
  def softwareReset(cls):
    "Sends a software reset (SWRST) command to all the servo drivers on the bus"
    cls.general_call_i2c.writeRaw8(0x06)        # SWRST

  def __init__(self, address=0x40, debug=False):
    self.i2c = Adafruit_I2C(address)
    self.i2c.debug = debug
    self.address = address
    self.debug = debug
    if (self.debug):
      print "Reseting PCA9685 MODE1 (without SLEEP) and MODE2"
    self.setAllPWM(0, 0)
    self.i2c.write8(self.__MODE2, self.__OUTDRV)
    self.i2c.write8(self.__MODE1, self.__ALLCALL)
    time.sleep(0.005)                                       # wait for oscillator

    mode1 = self.i2c.readU8(self.__MODE1)
    mode1 = mode1 & ~self.__SLEEP                 # wake up (reset sleep)
    self.i2c.write8(self.__MODE1, mode1)
    time.sleep(0.005)                             # wait for oscillator

  def setPWMFreq(self, freq):
    "Sets the PWM frequency"
    prescaleval = 25000000.0    # 25MHz
    prescaleval /= 4096.0       # 12-bit
    prescaleval /= float(freq)
    prescaleval -= 1.0
    if (self.debug):
      print "Setting PWM frequency to %d Hz" % freq
      print "Estimated pre-scale: %d" % prescaleval
    prescale = math.floor(prescaleval + 0.5)
    if (self.debug):
      print "Final pre-scale: %d" % prescale

    oldmode = self.i2c.readU8(self.__MODE1);
    newmode = (oldmode & 0x7F) | 0x10             # sleep
    self.i2c.write8(self.__MODE1, newmode)        # go to sleep
    self.i2c.write8(self.__PRESCALE, int(math.floor(prescale)))
    self.i2c.write8(self.__MODE1, oldmode)
    time.sleep(0.005)
    self.i2c.write8(self.__MODE1, oldmode | 0x80)

  def setPWM(self, channel, on, off):
    "Sets a single PWM channel"
    self.i2c.write8(self.__LED0_ON_L+4*channel, on & 0xFF)
    self.i2c.write8(self.__LED0_ON_H+4*channel, on >> 8)
    self.i2c.write8(self.__LED0_OFF_L+4*channel, off & 0xFF)
    self.i2c.write8(self.__LED0_OFF_H+4*channel, off >> 8)

  def setAllPWM(self, on, off):
    "Sets a all PWM channels"
    self.i2c.write8(self.__ALL_LED_ON_L, on & 0xFF)
    self.i2c.write8(self.__ALL_LED_ON_H, on >> 8)
    self.i2c.write8(self.__ALL_LED_OFF_L, off & 0xFF)
    self.i2c.write8(self.__ALL_LED_OFF_H, off >> 8)
<|endoftext|>


<commit_before>"""
  Copyright 2014 Google Inc. All rights reserved.

  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
"""

import array
from datetime import datetime
import logging
import sys
import struct
import json as JSON
from os import path
import cStringIO
from gzip import GzipFile

last_time = None

def elapsed_time(msg, new_start=False):
  global last_time
  this_time = datetime.now()
  if last_time == None:
    logging.getLogger().setLevel(logging.INFO)
    last_time = this_time
  if new_start:
    last_time = this_time
  if this_time > last_time:
    logging.info('{0} took {1} seconds'.format(msg, this_time - last_time))
  else:
    logging.info(msg)
  last_time = this_time

def _parse_json(data):
  return JSON.loads(data)

def _parse_array_from_str(data, type, byteorder):
  """Using array.fromstring function , parses given binary string and type Returns array of type
  """
  arr = array.array(type)
  arr.fromstring(data)
  if sys.byteorder != byteorder:
    arr.byteswap()
  return arr

def _parse_array_from_file(filename, type, byteorder):
  """Using _parse_array_from_str, open given filename and parses it Returns array of type
  """
  file = open(filename, 'rb')
  arr = _parse_array_from_str(file.read(), type, byteorder)
  file.close()
  return arr

def _parse_array_fmt(fmt, count, data):
  """Using struct.unpack_from, parses given binary string as array if given fmt
  """
  pos = 0
  fmt_size = struct.calcsize(fmt)
  rv = [None] * count
  for i in xrange(count):
    rv[i] = struct.unpack_from(fmt, data, pos)
    pos += fmt_size
  return rv

def _build_cmap(cp_file, gid_file):
  """Build cmap dictionary from codepoints to glyph ids using given files
  """
  keys = _parse_array_from_file(cp_file, 'H', 'big')
  gids = _parse_array_from_file(gid_file, 'H', 'big')
  cmap = dict.fromkeys(keys)

  for i, key in enumerate(keys):
    cmap[key] = gids[i]
  return cmap

def _parse_glyf_table(file):
  """Parses given file as glyf table, where each entry is such tuples (glyph id , [hmtx lsb] ,[vmtx tsb] , offset , size )
  """
  fmt_GlyphTable = '>HH'
  HAS_HMTX = (1 << 0)
  HAS_VMTX = (1 << 1)
  header_size = struct.calcsize(fmt_GlyphTable)
  file.seek(0)
  (flags, numGlyphs) = struct.unpack(fmt_GlyphTable, file.read(header_size))
  fmt_mtx = ''
  if flags & HAS_HMTX: fmt_mtx+='h'
  if flags & HAS_VMTX: fmt_mtx+='h'
  fmt_entry = '>H' + fmt_mtx + 'LH'
  return \
    _parse_array_fmt(fmt_entry, numGlyphs, file.read()), flags & HAS_HMTX, \
    flags & HAS_VMTX, header_size, struct.calcsize(fmt_entry)

def _read_region(file, offset, size):
  prev = file.tell()
  file.seek(offset)
  data = file.read(size)
  file.seek(prev)
  return data

def _gzip(input):
  buffer = cStringIO.StringIO()
  f = GzipFile(fileobj=buffer, mode='wb')
  f.write(input)
  f.close()
  compressed = buffer.getvalue()
  buffer.close()
  return compressed

def prepare_bundle(request):
  """Parse requests, then prepares response bundle for glyphs
  """
  glyph_request = _parse_json(request.body)
  font = glyph_request['font']
  codepoints = glyph_request['arr']
  elapsed_time('prepare_bundle for {0} characters'.format(len(codepoints)), True)
  base = path.dirname(path.abspath(__file__)) + '/data/' + font + '/'
  cmap = _build_cmap(base + 'codepoints', base + '/gids')
  closure_reader = ClosureReader(base + '/closure_idx', base + '/closure_data')
  gids = set()
  for code in codepoints:
    if code in cmap:
      gids.update(closure_reader.read(cmap[code]))
  closure_reader.close()
  elapsed_time('gather glyph info')

  table = open(base + '/glyph_table', 'rb')
  (glyf_table, has_hmtx, has_vmtx, header_size, entry_size ) = \
  _parse_glyf_table(table)
  mtx_count = has_hmtx + has_vmtx
  flag_mtx = has_hmtx | has_vmtx << 1
  bundle = bytearray()
  data = open(base + '/glyph_data', 'rb')
  bundle.extend(struct.pack('>HB', len(gids), flag_mtx))
  for id in gids:
    entry_offset = header_size + id * entry_size
    bundle.extend(_read_region(table, entry_offset, entry_size))
    data_offset = glyf_table[id][mtx_count + 1]
    data_size = glyf_table[id][mtx_count + 2]
    bundle.extend(_read_region(data, data_offset, data_size))
  table.close()
  data.close()
  elapsed_time('build request')
  result = _gzip(str(bundle))
  elapsed_time('compress request')
  return result

class ClosureReader(object):
  """Class to read closure list of a given glyph id Init with serialized closure files
  """
  fmt_idx = '>lH'
  fmt_idx_len = struct.calcsize(fmt_idx)
  fmt_one = 'H'
  fmt_one_len = struct.calcsize(fmt_one)

  def __init__(self, index_file, data_file):
    self.idx = open(index_file, 'rb')
    self.data_file = open(data_file, 'rb')

  def read(self, glyphID):
    """Return closure list of glyph ids for given glyphID
    """
    closure_set = set([glyphID])
    idx_offset = glyphID * ClosureReader.fmt_idx_len
    self.idx.seek(idx_offset)
    (offset, size) = struct.unpack(
        ClosureReader.fmt_idx,
        self.idx.read(ClosureReader.fmt_idx_len))
    if offset == -1: return closure_set
    self.data_file.seek(offset)
    arr = (
        _parse_array_from_str(
            self.data_file.read(size),
            ClosureReader.fmt_one, 'big'))
    closure_set.update(arr)
    return closure_set

  def close(self):
    self.idx.close()
    self.data_file.close()
<commit_msg>checkpoint: read whole file on gae (30 ms) rather than 100+ small reads (800 ms). This still needs clean up.<commit_after>"""
  Copyright 2014 Google Inc. All rights reserved.

  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
"""

import array
from datetime import datetime
import logging
import sys
import struct
import json as JSON
from os import path
import cStringIO
from gzip import GzipFile

last_time = None

def elapsed_time(msg, new_start=False):
  global last_time
  this_time = datetime.now()
  if last_time == None:
    logging.getLogger().setLevel(logging.INFO)
    last_time = this_time
  if new_start:
    last_time = this_time
  if this_time > last_time:
    logging.info('{0} took {1} seconds'.format(msg, this_time - last_time))
  else:
    logging.info(msg)
  last_time = this_time

def _parse_json(data):
  return JSON.loads(data)

def _parse_array_from_str(data, type, byteorder):
  """Using array.fromstring function , parses given binary string and type Returns array of type
  """
  arr = array.array(type)
  arr.fromstring(data)
  if sys.byteorder != byteorder:
    arr.byteswap()
  return arr

def _parse_array_from_file(filename, type, byteorder):
  """Using _parse_array_from_str, open given filename and parses it Returns array of type
  """
  file = open(filename, 'rb')
  arr = _parse_array_from_str(file.read(), type, byteorder)
  file.close()
  return arr

def _parse_array_fmt(fmt, count, data):
  """Using struct.unpack_from, parses given binary string as array if given fmt
  """
  pos = 0
  fmt_size = struct.calcsize(fmt)
  rv = [None] * count
  for i in xrange(count):
    rv[i] = struct.unpack_from(fmt, data, pos)
    pos += fmt_size
  return rv

def _build_cmap(cp_file, gid_file):
  """Build cmap dictionary from codepoints to glyph ids using given files
  """
  keys = _parse_array_from_file(cp_file, 'H', 'big')
  gids = _parse_array_from_file(gid_file, 'H', 'big')
  cmap = dict.fromkeys(keys)

  for i, key in enumerate(keys):
    cmap[key] = gids[i]
  return cmap

def _parse_glyf_table(file):
  """Parses given file as glyf table, where each entry is such tuples (glyph id , [hmtx lsb] ,[vmtx tsb] , offset , size )
  """
  fmt_GlyphTable = '>HH'
  HAS_HMTX = (1 << 0)
  HAS_VMTX = (1 << 1)
  header_size = struct.calcsize(fmt_GlyphTable)
  file.seek(0)
  (flags, numGlyphs) = struct.unpack(fmt_GlyphTable, file.read(header_size))
  fmt_mtx = ''
  if flags & HAS_HMTX: fmt_mtx+='h'
  if flags & HAS_VMTX: fmt_mtx+='h'
  fmt_entry = '>H' + fmt_mtx + 'LH'
  return \
    _parse_array_fmt(fmt_entry, numGlyphs, file.read()), flags & HAS_HMTX, \
    flags & HAS_VMTX, header_size, struct.calcsize(fmt_entry)

def _read_region(file, offset, size):
  prev = file.tell()
  file.seek(offset)
  data = file.read(size)
  file.seek(prev)
  return data

def _gzip(input):
  buffer = cStringIO.StringIO()
  f = GzipFile(fileobj=buffer, mode='wb')
  f.write(input)
  f.close()
  compressed = buffer.getvalue()
  buffer.close()
  return compressed

def prepare_bundle(request):
  """Parse requests, then prepares response bundle for glyphs
  """
  glyph_request = _parse_json(request.body)
  font = glyph_request['font']
  codepoints = glyph_request['arr']
  elapsed_time('prepare_bundle for {0} characters'.format(len(codepoints)), True)
  base = path.dirname(path.abspath(__file__)) + '/data/' + font + '/'
  cmap = _build_cmap(base + 'codepoints', base + '/gids')
  closure_reader = ClosureReader(base + '/closure_idx', base + '/closure_data')
  gids = set()
  for code in codepoints:
    if code in cmap:
      gids.update(closure_reader.read(cmap[code]))
  closure_reader.close()
  elapsed_time('gather glyph info')

  table_fh = open(base + '/glyph_table', 'rb')
  table_bytes = bytearray(table_fh.read())
  elapsed_time('read glyph table ({0} bytes)'.format(len(table_bytes)))

  data_fh = open(base + '/glyph_data', 'rb')
  data_bytes = bytearray(data_fh.read())
  elapsed_time('read glyph data ({0} bytes)'.format(len(data_bytes)))

  table = open(base + '/glyph_table', 'rb')
  (glyf_table, has_hmtx, has_vmtx, header_size, entry_size ) = \
  _parse_glyf_table(table)
  mtx_count = has_hmtx + has_vmtx
  flag_mtx = has_hmtx | has_vmtx << 1
  elapsed_time('open & parse glyph table')

  bundle_header = struct.pack('>HB', len(gids), flag_mtx)
  bundle_length = len(bundle_header)
  bundle_length += len(gids) * entry_size
  for id in gids:
    bundle_length += glyf_table[id][mtx_count + 2]
  bundle_bytes = bytearray(bundle_length)
  bundle_pos = 0
  length = len(bundle_header)
  bundle_bytes[bundle_pos:bundle_pos+length] = bundle_header
  bundle_pos += length
  elapsed_time('calc bundle length')

  # Copy in the data from table_bytes and data_bytes
  for id in gids:
    entry_offset = header_size + id * entry_size
    bundle_bytes[bundle_pos:bundle_pos + entry_size] = \
        table_bytes[entry_offset:entry_offset + entry_size]
    bundle_pos += entry_size

    data_offset = glyf_table[id][mtx_count + 1]
    data_size = glyf_table[id][mtx_count + 2]
    bundle_bytes[bundle_pos:bundle_pos + data_size] = \
        data_bytes[data_offset:data_offset + data_size]
    bundle_pos += data_size
  elapsed_time('build mem bundle')

  bundle = bytearray()
  data = open(base + '/glyph_data', 'rb')
  bundle.extend(bundle_header)
  for id in gids:
    # the following is slow
    entry_offset = header_size + id * entry_size
    bundle.extend(_read_region(table, entry_offset, entry_size))
    data_offset = glyf_table[id][mtx_count + 1]
    data_size = glyf_table[id][mtx_count + 2]
    bundle.extend(_read_region(data, data_offset, data_size))
  elapsed_time('build bundle')
  if len(bundle) != len(bundle_bytes):
    elapsed_time('len(bundle) = {0}, len(bundle_bytes) = {1}'
                 .format(len(bundle), len(bundle_bytes)))
  for i in range(len(bundle_bytes)):
    if bundle[i] != bundle_bytes[i]:
      elapsed_time('mismatch')
      break

  elapsed_time('compare bundles')
  table.close()
  data.close()
  table_fh.close()
  data_fh.close()
  elapsed_time('close files')
  result = _gzip(str(bundle))
  elapsed_time('compress request')
  return result

class ClosureReader(object):
  """Class to read closure list of a given glyph id Init with serialized closure files
  """
  fmt_idx = '>lH'
  fmt_idx_len = struct.calcsize(fmt_idx)
  fmt_one = 'H'
  fmt_one_len = struct.calcsize(fmt_one)

  def __init__(self, index_file, data_file):
    self.idx = open(index_file, 'rb')
    self.data_file = open(data_file, 'rb')

  def read(self, glyphID):
    """Return closure list of glyph ids for given glyphID
    """
    closure_set = set([glyphID])
    idx_offset = glyphID * ClosureReader.fmt_idx_len
    self.idx.seek(idx_offset)
    (offset, size) = struct.unpack(
        ClosureReader.fmt_idx,
        self.idx.read(ClosureReader.fmt_idx_len))
    if offset == -1: return closure_set
    self.data_file.seek(offset)
    arr = (
        _parse_array_from_str(
            self.data_file.read(size),
            ClosureReader.fmt_one, 'big'))
    closure_set.update(arr)
    return closure_set

  def close(self):
    self.idx.close()
    self.data_file.close()
<|endoftext|>


<commit_before>import logging
logger = logging.getLogger('compdb')

def info(args):
    from compdb.contrib import get_project
    project = get_project()
    if args.all:
        args.status = True
        args.jobs = True
        args.pulse = True
        args.more = True

    print(project)
    if args.more:
        print(project.config)
    if args.status:
        print("{} registered job(s)".format(len(list(project.find_job_ids()))))
        print("{} active job(s)".format(len(list(project.active_jobs()))))
    if args.jobs:
        legit_ids = project.find_job_ids()
        if args.jobs is True:
            known_ids = legit_ids
        else:
            job_ids = set(args.jobs.split(','))
            unknown_ids = job_ids.difference(legit_ids)
            known_ids = job_ids.intersection(legit_ids)
            if unknown_ids:
                print("Unknown ids: {}".format(','.join(unknown_ids)))
        if args.status:
            print("Job ID{}".format(' ' * 26), "Open Instances")
        for known in known_ids:
            job = project.get_job(known)
            if args.status:
                print(job, job.num_open_instances())
            else:
                print(job)
            if args.more:
                from json import dumps
                print(dumps(job.spec['parameters'], sort_keys = True))
    if args.pulse:
        from datetime import datetime
        from compdb.contrib.job import PULSE_PERIOD
        jobs = list(project.job_pulse())
        if jobs:
            print("Pulse period (expected): {}s.".format(PULSE_PERIOD))
            for uid, age in jobs:
                delta = datetime.utcnow() - age
                msg = "UID: {uid}, last signal: {age:.2f} seconds"
                print(msg.format(
                    uid = uid,
                    age = delta.total_seconds()))
        else:
            print("No active jobs found.")

def view(args):
    from compdb.contrib import get_project
    from os.path import join, exists
    from os import listdir
    project = get_project()
    if args.url:
        url = join(args.prefix, args.url)
    else:
        url = join(args.prefix, project.get_default_view_url())
    if args.copy:
        q = "Are you sure you want to create copy of the whole dataset? This might create extremely high network load!"
        if not(args.yes or query_yes_no(q, 'no')):
            return
    if args.script:
        for line in project.create_view_script(url = url, cmd = args.script):
            print(line)
    else:
        if exists(args.prefix) and listdir(args.prefix):
            print("Path '{}' is not empty.".format(args.prefix))
            return
        project.create_view(url = url, copy = args.copy)

def check(args):
    from . import check
    checks = [
        ('Checking database connection...',
        check.check_database_connection),
        ('Checking global configuration...',
        check.check_global_config),
        ('Checking project configuration...',
        check.check_project_config),
        ]
    for msg, check in checks:
        print(msg)
        try:
            check()
        except Exception as error:
            print("Error: {}".format(error))
            if args.verbosity > 0:
                raise
        else:
            print("OK")

def submit(args):
    from os.path import abspath
    from . job_submit import find_all_pools, submit_mpi
    for pool in find_all_pools(abspath(args.module)):
        submit_mpi(pool)

def store_snapshot(args):
    from . import get_project
    from . utility import query_yes_no
    from os.path import exists
    if not args.overwrite and exists(args.snapshot):
        q = "File with name '{}' already exists. Overwrite?"
        if args.yes or query_yes_no(q.format(args.snapshot), 'no'):
            pass
        else:
            return
    project = get_project()
    try:
        if args.database_only:
            print("Creating project database snapshot.")
        else:
            print("Creating project snapshot.")
        project.create_snapshot(args.snapshot, not args.database_only)
    except Exception as error:
        msg = "Failed to create snapshot."
        print(msg)
        raise
    else:
        print("Success.")

def restore_snapshot(args):
    from . import get_project
    from . utility import query_yes_no
    from . project import RollBackupExistsError
    project = get_project()
    print("Trying to restore from: {}".format(args.snapshot))
    try:
        project.restore_snapshot(args.snapshot)
    except FileNotFoundError as error:
        raise RuntimeError("File not found: {}".format(error))
    except RollBackupExistsError as dst:
        q = "A backup from a previous restore attempt exists. "
        q += "Do you want to try to recover from that?"
        if query_yes_no(q, 'no'):
            try:
                project._restore_rollbackup(str(dst))
            except Exception as error:
                print("The recovery failed. The corrupted recovery backup lies in '{}'. It is probably safe to delete it after inspection.".format(dst))
                raise
            else:
                print("Successfully recovered.")
                project._remove_rollbackup(str(dst))
        else:
            q = "Do you want to delete it?"
            if query_yes_no(q, 'no'):
                project._remove_rollbackup(str(dst))
                print("Removed.")
    else:
        print("Success.")

def clean_up(args):
    from . import get_project
    project = get_project()
    logger.info("Killing dead jobs...")
    project.kill_dead_jobs(seconds = args.tolerance_time)

def remove(args):
    from . import get_project
    from . utility import query_yes_no
    project = get_project()
    if args.project:
        question = "Are you sure you want to remove project '{}'?"
        if args.yes or query_yes_no(question.format(project.get_id()), default = 'no'):
            try:
                project.remove(force = args.force)
            except RuntimeError as error:
                print("Error during project removal.")
                if not args.force:
                    print("This can be caused by currently executed jobs.")
                    print("Try 'compdb clenaup'.")
                    if args.yes or query_yes_no("Ignore this warning and remove anywas?", default = 'no'):
                        project.remove(force = True)
            else:
                print("Project removed from database.")
    elif args.job:
        job_ids = set(args.job.split(','))
        legit_ids = project.find_job_ids()
        unknown_ids = job_ids.difference(legit_ids)
        known_ids = job_ids.intersection(legit_ids)
        print(known_ids, unknown_ids)
        return
        if len(unknown_ids):
            if not(args.yes or query_yes_no(q)):
                return
        for id_ in known_ids:
            job = project.get_job(id_)
            print(job)
    else:
        print("No selection.")

def main():
    import sys
    from . utility import add_verbosity_argument, set_verbosity_level, EmptyIsTrue
    from argparse import ArgumentParser
    parser = ArgumentParser()
    parser.add_argument(
        '-y', '--yes',
        action = 'store_true',
        help = "Assume yes to all questions.",)
    add_verbosity_argument(parser)

    subparsers = parser.add_subparsers()

    from compdb.contrib import init_project
    parser_init = subparsers.add_parser('init')
    init_project.setup_parser(parser_init)
    parser_init.set_defaults(func = init_project.init_project)

    from compdb.contrib import configure
    parser_config = subparsers.add_parser('config')
    configure.setup_parser(parser_config)
    parser_config.set_defaults(func = configure.configure)

    parser_remove = subparsers.add_parser('remove')
    parser_remove.add_argument(
        '-j', '--job',
        nargs = '*',
        help = "A list of jobs, that are to be removed as a comma separated list of job ids.",
        )
    parser_remove.add_argument(
        '-p', '--project',
        action = 'store_true',
        help = 'Remove the whole project.'
        )
    parser_remove.add_argument(
        '-f', '--force',
        action = 'store_true',
        help = "Ignore errors during removal. May lead to data loss!")
    parser_remove.set_defaults(func = remove)

    parser_snapshot = subparsers.add_parser('snapshot')
    parser_snapshot.add_argument(
        'snapshot',
        type = str,
        help = "Name of the file used to create the snapshot.",)
    parser_snapshot.add_argument(
        '--database-only',
        action = 'store_true',
        help = "Create only a snapshot of the database, without a copy of the value storage.",)
    parser_snapshot.add_argument(
        '--overwrite',
        action = 'store_true',
        help = "Overwrite existing snapshots with the same name without asking.",
        )
    parser_snapshot.set_defaults(func = store_snapshot)

    parser_restore = subparsers.add_parser('restore')
    parser_restore.add_argument(
        'snapshot',
        type = str,
        help = "Name of the snapshot file or directory, used for restoring.",
        )
    parser_restore.set_defaults(func = restore_snapshot)

    parser_cleanup = subparsers.add_parser('cleanup')
    from . job import PULSE_PERIOD
    parser_cleanup.add_argument(
        '-t', '--tolerance-time',
        type = int,
        help = "Tolerated time in seconds since last pulse before a job is declared dead.",
        default = int(5 * PULSE_PERIOD))
    parser_cleanup.set_defaults(func = clean_up)

    parser_info = subparsers.add_parser('info')
    parser_info.add_argument(
        '-j', '--jobs',
        nargs = '?',
        action = EmptyIsTrue,
        help = "Lists the jobs of this project. Provide a comma-separated list to show only a subset.",
        )
    parser_info.add_argument(
        '-s', '--status',
        action = 'store_true',
        help = "Print status information.")
    parser_info.add_argument(
        '-p', '--pulse',
        action = 'store_true',
        help = "Print job pulse status.")
    parser_info.add_argument(
        '-m', '--more',
        action = 'store_true',
        help = "Show more details.")
    parser_info.add_argument(
        '-a', '--all',
        action = 'store_true',
        help = "Show everything.")
    parser_info.set_defaults(func = info)

    parser_view = subparsers.add_parser('view')
    parser_view.add_argument(
        '--prefix',
        type = str,
        default = 'view/',
        help = "Prefix the given view url.")
    parser_view.add_argument(
        '-u', '--url',
        type = str,
        help = "Provide a url for the view in the form: abc/{a}/{b}, where each value in curly brackets denotes the name of a parameter."
        )
    parser_view.add_argument(
        '-c', '--copy',
        action = 'store_true',
        help = "Generate a copy of the whole dataset instead of linking to it. WARNING: This option may create very high network load!")
    parser_view.add_argument(
        '-s', '--script',
        type = str,
        nargs = '?',
        const = 'mkdir -p {head}\nln -s {src} {head}/{tail}',
        help = r"Output a line foreach job where {src} is replaced with the the job's storage directory path, {head} and {tail} combined represent your view path. Default: 'mkdir -p {head}\nln -s {src} {head}/{tail}'."
        )
    parser_view.set_defaults(func = view)

    parser_check = subparsers.add_parser('check')
    parser_check.set_defaults(func = check)

    parser_submit = subparsers.add_parser('submit')
    parser_submit.add_argument(
        'module',
        type = str,
        help = "The path to the python module containing job_pools.")
    parser_submit.set_defaults(func = submit)

    args = parser.parse_args()
    set_verbosity_level(args.verbosity)
    try:
        if 'func' in args:
            args.func(args)
        else:
            parser.print_usage()
    except Exception as error:
        if args.verbosity > 0:
            raise
        else:
            print("Error: {}".format(error))
            print("Use -v to increase verbosity of messages.")
    else:
        sys.exit(0)
<commit_msg>Renamed submit command for script to run.<commit_after>import logging
logger = logging.getLogger('compdb')

def info(args):
    from compdb.contrib import get_project
    project = get_project()
    if args.all:
        args.status = True
        args.jobs = True
        args.pulse = True
        args.more = True

    print(project)
    if args.more:
        print(project.config)
    if args.status:
        print("{} registered job(s)".format(len(list(project.find_job_ids()))))
        print("{} active job(s)".format(len(list(project.active_jobs()))))
    if args.jobs:
        legit_ids = project.find_job_ids()
        if args.jobs is True:
            known_ids = legit_ids
        else:
            job_ids = set(args.jobs.split(','))
            unknown_ids = job_ids.difference(legit_ids)
            known_ids = job_ids.intersection(legit_ids)
            if unknown_ids:
                print("Unknown ids: {}".format(','.join(unknown_ids)))
        if args.status:
            print("Job ID{}".format(' ' * 26), "Open Instances")
        for known in known_ids:
            job = project.get_job(known)
            if args.status:
                print(job, job.num_open_instances())
            else:
                print(job)
            if args.more:
                from json import dumps
                print(dumps(job.spec['parameters'], sort_keys = True))
    if args.pulse:
        from datetime import datetime
        from compdb.contrib.job import PULSE_PERIOD
        jobs = list(project.job_pulse())
        if jobs:
            print("Pulse period (expected): {}s.".format(PULSE_PERIOD))
            for uid, age in jobs:
                delta = datetime.utcnow() - age
                msg = "UID: {uid}, last signal: {age:.2f} seconds"
                print(msg.format(
                    uid = uid,
                    age = delta.total_seconds()))
        else:
            print("No active jobs found.")

def view(args):
    from compdb.contrib import get_project
    from os.path import join, exists
    from os import listdir
    project = get_project()
    if args.url:
        url = join(args.prefix, args.url)
    else:
        url = join(args.prefix, project.get_default_view_url())
    if args.copy:
        q = "Are you sure you want to create copy of the whole dataset? This might create extremely high network load!"
        if not(args.yes or query_yes_no(q, 'no')):
            return
    if args.script:
        for line in project.create_view_script(url = url, cmd = args.script):
            print(line)
    else:
        if exists(args.prefix) and listdir(args.prefix):
            print("Path '{}' is not empty.".format(args.prefix))
            return
        project.create_view(url = url, copy = args.copy)

def check(args):
    from . import check
    checks = [
        ('Checking database connection...',
        check.check_database_connection),
        ('Checking global configuration...',
        check.check_global_config),
        ('Checking project configuration...',
        check.check_project_config),
        ]
    for msg, check in checks:
        print(msg)
        try:
            check()
        except Exception as error:
            print("Error: {}".format(error))
            if args.verbosity > 0:
                raise
        else:
            print("OK")

def run_pools(args):
    from os.path import abspath
    from . job_submit import find_all_pools, submit_mpi
    for pool in find_all_pools(abspath(args.module)):
        submit_mpi(pool)

def store_snapshot(args):
    from . import get_project
    from . utility import query_yes_no
    from os.path import exists
    if not args.overwrite and exists(args.snapshot):
        q = "File with name '{}' already exists. Overwrite?"
        if args.yes or query_yes_no(q.format(args.snapshot), 'no'):
            pass
        else:
            return
    project = get_project()
    try:
        if args.database_only:
            print("Creating project database snapshot.")
        else:
            print("Creating project snapshot.")
        project.create_snapshot(args.snapshot, not args.database_only)
    except Exception as error:
        msg = "Failed to create snapshot."
        print(msg)
        raise
    else:
        print("Success.")

def restore_snapshot(args):
    from . import get_project
    from . utility import query_yes_no
    from . project import RollBackupExistsError
    project = get_project()
    print("Trying to restore from: {}".format(args.snapshot))
    try:
        project.restore_snapshot(args.snapshot)
    except FileNotFoundError as error:
        raise RuntimeError("File not found: {}".format(error))
    except RollBackupExistsError as dst:
        q = "A backup from a previous restore attempt exists. "
        q += "Do you want to try to recover from that?"
        if query_yes_no(q, 'no'):
            try:
                project._restore_rollbackup(str(dst))
            except Exception as error:
                print("The recovery failed. The corrupted recovery backup lies in '{}'. It is probably safe to delete it after inspection.".format(dst))
                raise
            else:
                print("Successfully recovered.")
                project._remove_rollbackup(str(dst))
        else:
            q = "Do you want to delete it?"
            if query_yes_no(q, 'no'):
                project._remove_rollbackup(str(dst))
                print("Removed.")
    else:
        print("Success.")

def clean_up(args):
    from . import get_project
    project = get_project()
    logger.info("Killing dead jobs...")
    project.kill_dead_jobs(seconds = args.tolerance_time)

def remove(args):
    from . import get_project
    from . utility import query_yes_no
    project = get_project()
    if args.project:
        question = "Are you sure you want to remove project '{}'?"
        if args.yes or query_yes_no(question.format(project.get_id()), default = 'no'):
            try:
                project.remove(force = args.force)
            except RuntimeError as error:
                print("Error during project removal.")
                if not args.force:
                    print("This can be caused by currently executed jobs.")
                    print("Try 'compdb clenaup'.")
                    if args.yes or query_yes_no("Ignore this warning and remove anywas?", default = 'no'):
                        project.remove(force = True)
            else:
                print("Project removed from database.")
    elif args.job:
        job_ids = set(args.job.split(','))
        legit_ids = project.find_job_ids()
        unknown_ids = job_ids.difference(legit_ids)
        known_ids = job_ids.intersection(legit_ids)
        print(known_ids, unknown_ids)
        return
        if len(unknown_ids):
            if not(args.yes or query_yes_no(q)):
                return
        for id_ in known_ids:
            job = project.get_job(id_)
            print(job)
    else:
        print("No selection.")

def main():
    import sys
    from . utility import add_verbosity_argument, set_verbosity_level, EmptyIsTrue
    from argparse import ArgumentParser
    parser = ArgumentParser()
    parser.add_argument(
        '-y', '--yes',
        action = 'store_true',
        help = "Assume yes to all questions.",)
    add_verbosity_argument(parser)

    subparsers = parser.add_subparsers()

    from compdb.contrib import init_project
    parser_init = subparsers.add_parser('init')
    init_project.setup_parser(parser_init)
    parser_init.set_defaults(func = init_project.init_project)

    from compdb.contrib import configure
    parser_config = subparsers.add_parser('config')
    configure.setup_parser(parser_config)
    parser_config.set_defaults(func = configure.configure)

    parser_remove = subparsers.add_parser('remove')
    parser_remove.add_argument(
        '-j', '--job',
        nargs = '*',
        help = "A list of jobs, that are to be removed as a comma separated list of job ids.",
        )
    parser_remove.add_argument(
        '-p', '--project',
        action = 'store_true',
        help = 'Remove the whole project.'
        )
    parser_remove.add_argument(
        '-f', '--force',
        action = 'store_true',
        help = "Ignore errors during removal. May lead to data loss!")
    parser_remove.set_defaults(func = remove)

    parser_snapshot = subparsers.add_parser('snapshot')
    parser_snapshot.add_argument(
        'snapshot',
        type = str,
        help = "Name of the file used to create the snapshot.",)
    parser_snapshot.add_argument(
        '--database-only',
        action = 'store_true',
        help = "Create only a snapshot of the database, without a copy of the value storage.",)
    parser_snapshot.add_argument(
        '--overwrite',
        action = 'store_true',
        help = "Overwrite existing snapshots with the same name without asking.",
        )
    parser_snapshot.set_defaults(func = store_snapshot)

    parser_restore = subparsers.add_parser('restore')
    parser_restore.add_argument(
        'snapshot',
        type = str,
        help = "Name of the snapshot file or directory, used for restoring.",
        )
    parser_restore.set_defaults(func = restore_snapshot)

    parser_cleanup = subparsers.add_parser('cleanup')
    from . job import PULSE_PERIOD
    parser_cleanup.add_argument(
        '-t', '--tolerance-time',
        type = int,
        help = "Tolerated time in seconds since last pulse before a job is declared dead.",
        default = int(5 * PULSE_PERIOD))
    parser_cleanup.set_defaults(func = clean_up)

    parser_info = subparsers.add_parser('info')
    parser_info.add_argument(
        '-j', '--jobs',
        nargs = '?',
        action = EmptyIsTrue,
        help = "Lists the jobs of this project. Provide a comma-separated list to show only a subset.",
        )
    parser_info.add_argument(
        '-s', '--status',
        action = 'store_true',
        help = "Print status information.")
    parser_info.add_argument(
        '-p', '--pulse',
        action = 'store_true',
        help = "Print job pulse status.")
    parser_info.add_argument(
        '-m', '--more',
        action = 'store_true',
        help = "Show more details.")
    parser_info.add_argument(
        '-a', '--all',
        action = 'store_true',
        help = "Show everything.")
    parser_info.set_defaults(func = info)

    parser_view = subparsers.add_parser('view')
    parser_view.add_argument(
        '--prefix',
        type = str,
        default = 'view/',
        help = "Prefix the given view url.")
    parser_view.add_argument(
        '-u', '--url',
        type = str,
        help = "Provide a url for the view in the form: abc/{a}/{b}, where each value in curly brackets denotes the name of a parameter."
        )
    parser_view.add_argument(
        '-c', '--copy',
        action = 'store_true',
        help = "Generate a copy of the whole dataset instead of linking to it. WARNING: This option may create very high network load!")
    parser_view.add_argument(
        '-s', '--script',
        type = str,
        nargs = '?',
        const = 'mkdir -p {head}\nln -s {src} {head}/{tail}',
        help = r"Output a line foreach job where {src} is replaced with the the job's storage directory path, {head} and {tail} combined represent your view path. Default: 'mkdir -p {head}\nln -s {src} {head}/{tail}'."
        )
    parser_view.set_defaults(func = view)

    parser_check = subparsers.add_parser('check')
    parser_check.set_defaults(func = check)

    parser_run = subparsers.add_parser('run')
    parser_run.add_argument(
        'module',
        type = str,
        help = "The path to the python module defining job_pools.")
    parser_run.set_defaults(func = run_pools)

    args = parser.parse_args()
    set_verbosity_level(args.verbosity)
    try:
        if 'func' in args:
            args.func(args)
        else:
            parser.print_usage()
    except Exception as error:
        if args.verbosity > 0:
            raise
        else:
            print("Error: {}".format(error))
            print("Use -v to increase verbosity of messages.")
    else:
        sys.exit(0)
<|endoftext|>


<commit_before>import json
import jsonpickle
import ifind.common.make_json_serializable

class Response(object):
    """
    Models a Response object for use with ifind's search interface.

    Response Attributes:
        query_terms:  string representation of original query terms
        results:      list representation of retrieved results (each result being a dict)
                      i.e {'title': Crow Rearing, 'url': http://etc.com, 'summary': How to rear crows?}
        result_total: integer representation of total results retrieved
    """

    def __init__(self, query_terms):
        """
        Response constructor.

        Args:
            query_terms (str): original query terms

        Attributes:
            results (list): list of retrieved results (each result being a Result object)
            result_total (int): total results retrieved

        Usage:
            response = Response(query.terms)

        """
        self.query_terms = query_terms
        self.results = []
        self.result_total = 0
        self.total_pages = 0
        self.results_on_page = 0
        self.actual_page = 0

    def add_result_object(self, result_object):
        """
        Adds a Result object to the Response's results list.
        """
        self.results.append(result_object)
        self.result_total += 1

    def add_result(self, title="", url="", summary="", imageurl='', rank=-1, **kwargs):
        """
        Adds a result to Response's results list.

        Kwargs:
            title (str): title of search result
            url (str): url of search result
            summary (str): summary of search result
            imageurl (str): the url of an image from the search result
            rank (int): the rank of the result
            **kwargs: further optional result attributes

        Usage:
            response.add_result(title="don's shop", url="www.dons.com",\
            summary="a very nice place", imageurl="http://stuff.com/img" rank=2)

        """
        self.results.append(Result(title, url, summary, imageurl , rank, **kwargs))
        self.result_total += 1

    def to_json(self):
        """
        Serialises Response and returns it as JSON.

        Returns:
            str: json string of Response

        Usage:
            response = engine.search(query)
            json_response = response.to_json()

        """
        response_dict = json.loads(jsonpickle.encode(self.__dict__))

        for result in response_dict[u'results']:
            del result[u'py/object']

        return json.dumps(response_dict)

    def __str__(self):
        """
        Returns human-readable string representation of response object.

        Returns:
            str: formatted new-lined list of results with info above

        Usage:
            print response

        """
        half_string = 'Result_total: {0}\nQuery_terms: {1}\nResults:\n\n'.format(self.result_total, self.query_terms)
        end_string = '\n\n'.join(['{0}'.format(result) for result in self.results])

        return half_string + end_string

    def __iter__(self):
        """
        Implements iterator for Response, returning a single result at a time.
        Uses generator function to lazy load results.

        Usage:
            for result in response:
                print result

        """
        for result in self.results:
            yield result

    def __len__(self):
        """
        Implements len() builtin for Response.

        Returns:
            int: number of results within response

        Usage:
            print len(response) --> 15

        """
        return self.result_total

    def __iadd__(self, other):
        """
        Overrides '+=' operator, adds other.results to self.results.

        Usage:
            print len(response1) --> 10
            print len(response2) --> 15
            response1 += response2
            print len(response1) --> 25

        """
        self.results += other.results
        self.result_total += other.result_total

        return self

    def __eq__(self, other):
        """
        Overrides '==' operator, returns True if both responses hash to the same value.

        Usage:
            response = Response("hello world")
            response2 = Response("hello world")
            print response == response2 --> False

        """
        return tuple(self.__dict__.items()) == tuple(other.__dict__.items())

class Result(object):
    """
    Models a Result object for use with ifind's Response class.

    """
    def __init__(self, title='', url='', summary='', imageurl='', rank=0, **kwargs):
        """
        Result constructor.

        Kwargs:
            title (str): title of search result
            url (str): url of search result
            summary (str): summary of search result
            imageurl (str): the url of an image from the search result
            rank (int): the rank of the search result
            **kwargs: further optional result attributes

        Usage:
            result = Result(title="pam's shop", url="www.pam.com", summary="a nice place")

        """
        self.title = title
        self.url = url
        self.summary = summary
        self.rank = rank
        self.imageurl = imageurl

        for key, value in kwargs.items():
            setattr(self, key, value)

        for key, value in self.__dict__.items():
            self.__dict__[key] = value

            if isinstance(value, str):
                self.__dict__[key] = value.encode('utf-8').rstrip()

    def __str__(self):
        """
        Returns human-readable string representation of result object.

        Returns:
            str: formatted new-lined list of result attributes

        Usage:
            print result

        """
        result = "\n"
        for key, value in self.__dict__.items():
            if isinstance(value,unicode):
                value = value.encode('ascii','ignore')

            result = result + "{0}: {1}\n".format(key, value)
        return result

    def __eq__(self, other):
        """
        Overrides '==' operator, returns True if both responses hash to the same value.

        Usage:
            response = Response("hello world")
            response2 = Response("hello world")
            print response == response2 --> False

        """
        return tuple(self.__dict__.items()) == tuple(other.__dict__.items())

    def to_json(self):
        """
        Returns object instance as a JSON string.
        """
        return vars(self)<commit_msg>Added some pagination support to the response object (next page (url), flag to set if no more results exist)<commit_after>import json
import jsonpickle
import ifind.common.make_json_serializable

class Response(object):
    """
    Models a Response object for use with ifind's search interface.

    Response Attributes:
        query_terms:  string representation of original query terms
        results:      list representation of retrieved results (each result being a dict)
                      i.e {'title': Crow Rearing, 'url': http://etc.com, 'summary': How to rear crows?}
        result_total: integer representation of total results retrieved
    """

    def __init__(self, query_terms):
        """
        Response constructor.

        Args:
            query_terms (str): original query terms

        Attributes:
            results (list): list of retrieved results (each result being a Result object)
            result_total (int): total results retrieved

        Usage:
            response = Response(query.terms)

        """
        self.query_terms = query_terms
        self.results = []
        self.result_total = 0
        self.total_pages = 0
        self.results_on_page = 0
        self.actual_page = 0
        # Flag to determine whether the engine has any more results for this query.
        self.no_more_results = False
        # The url to request the next page from
        self.next_page = None

    def add_result_object(self, result_object):
        """
        Adds a Result object to the Response's results list.
        """
        self.results.append(result_object)
        self.result_total += 1

    def add_result(self, title="", url="", summary="", imageurl='', rank=-1, **kwargs):
        """
        Adds a result to Response's results list.

        Kwargs:
            title (str): title of search result
            url (str): url of search result
            summary (str): summary of search result
            imageurl (str): the url of an image from the search result
            rank (int): the rank of the result
            **kwargs: further optional result attributes

        Usage:
            response.add_result(title="don's shop", url="www.dons.com",\
            summary="a very nice place", imageurl="http://stuff.com/img" rank=2)

        """
        self.results.append(Result(title, url, summary, imageurl , rank, **kwargs))
        self.result_total += 1

    def to_json(self):
        """
        Serialises Response and returns it as JSON.

        Returns:
            str: json string of Response

        Usage:
            response = engine.search(query)
            json_response = response.to_json()

        """
        response_dict = json.loads(jsonpickle.encode(self.__dict__))

        for result in response_dict[u'results']:
            del result[u'py/object']

        return json.dumps(response_dict)

    def __str__(self):
        """
        Returns human-readable string representation of response object.

        Returns:
            str: formatted new-lined list of results with info above

        Usage:
            print response

        """
        half_string = 'Result_total: {0}\nQuery_terms: {1}\nResults:\n\n'.format(self.result_total, self.query_terms)
        end_string = '\n\n'.join(['{0}'.format(result) for result in self.results])

        return half_string + end_string

    def __iter__(self):
        """
        Implements iterator for Response, returning a single result at a time.
        Uses generator function to lazy load results.

        Usage:
            for result in response:
                print result

        """
        for result in self.results:
            yield result

    def __len__(self):
        """
        Implements len() builtin for Response.

        Returns:
            int: number of results within response

        Usage:
            print len(response) --> 15

        """
        return self.result_total

    def __iadd__(self, other):
        """
        Overrides '+=' operator, adds other.results to self.results.

        Usage:
            print len(response1) --> 10
            print len(response2) --> 15
            response1 += response2
            print len(response1) --> 25

        """
        self.results += other.results
        self.result_total += other.result_total

        return self

    def __eq__(self, other):
        """
        Overrides '==' operator, returns True if both responses hash to the same value.

        Usage:
            response = Response("hello world")
            response2 = Response("hello world")
            print response == response2 --> False

        """
        return tuple(self.__dict__.items()) == tuple(other.__dict__.items())

class Result(object):
    """
    Models a Result object for use with ifind's Response class.

    """
    def __init__(self, title='', url='', summary='', imageurl='', rank=0, **kwargs):
        """
        Result constructor.

        Kwargs:
            title (str): title of search result
            url (str): url of search result
            summary (str): summary of search result
            imageurl (str): the url of an image from the search result
            rank (int): the rank of the search result
            **kwargs: further optional result attributes

        Usage:
            result = Result(title="pam's shop", url="www.pam.com", summary="a nice place")

        """
        self.title = title
        self.url = url
        self.summary = summary
        self.rank = rank
        self.imageurl = imageurl

        for key, value in kwargs.items():
            setattr(self, key, value)

        for key, value in self.__dict__.items():
            self.__dict__[key] = value

            if isinstance(value, str):
                self.__dict__[key] = value.encode('utf-8').rstrip()

    def __str__(self):
        """
        Returns human-readable string representation of result object.

        Returns:
            str: formatted new-lined list of result attributes

        Usage:
            print result

        """
        result = "\n"
        for key, value in self.__dict__.items():
            if isinstance(value,unicode):
                value = value.encode('ascii','ignore')

            result = result + "{0}: {1}\n".format(key, value)
        return result

    def __eq__(self, other):
        """
        Overrides '==' operator, returns True if both responses hash to the same value.

        Usage:
            response = Response("hello world")
            response2 = Response("hello world")
            print response == response2 --> False

        """
        return tuple(self.__dict__.items()) == tuple(other.__dict__.items())

    def to_json(self):
        """
        Returns object instance as a JSON string.
        """
        return vars(self)<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

def make_locations(apps, schema_editor):
    Location = apps.get_model('inventory', 'Location')
    default_locations = [
        ('Lab', 'L', 1,),
    ]
    for item in default_locations:
        l = Location(name=item[0], code=item[1], lft=0, rght=0, level=0, tree_id=item[2])
        l.save()

def make_measures(apps, schema_editor):
    AmountMeasure = apps.get_model('inventory', 'AmountMeasure')
    default_measures = [
        ('Litres', 'l'),
        ('Millilitres', 'ml'),
        ('Microlitres', 'ul'),
        ('Nanolitres', 'nl'),
        ('Metres', 'm'),
        ('Centimetres', 'cm'),
        ('Millimetres', 'mm'),
        ('Micrometres', 'um'),
        ('Nanometres', 'nm'),
        ('Millimolar', 'mM'),
        ('Micromolar', 'uM'),
        ('Nanomolar', 'nM'),
        ('Kilogram', 'kg'),
        ('Gram', 'g'),
        ('Milligram', 'mg'),
        ('Microgram', 'ug'),
        ('Nanogram', 'ng'),
        ('Day', 'day'),
        ('Hour', 'hr'),
        ('Minute', 'min'),
        ('Second', 's'),
        ('Millisecond', 'ms'),
        ('Kelvin', 'K'),
        ('Celsius', '°C'),
        ('Microsecond', 'us'),
        ('Nanosecond', 'ns'),
        ('Microgram/microlitre', 'ug/ul'),
        ('Item', 'item'),
    ]
    for item in default_measures:
        AmountMeasure.objects.create(name=item[0], symbol=item[1])

def make_itemtypes(apps, schema_editor):
    ItemType = apps.get_model('inventory', 'ItemType')
    default_itemtypes = [
        ('Item',)
    ]
    for item in default_itemtypes:
         itm = ItemType(name=item[0], lft=0, rght=0, level=0, tree_id=0)
         itm.save()

class Migration(migrations.Migration):

    dependencies = [
        ('inventory', '0016_auto_20170123_0941'),
    ]

    operations = [
        migrations.RunPython(make_measures),
        migrations.RunPython(make_locations),
        migrations.RunPython(make_itemtypes),
    ]
<commit_msg>Rebuild item type and location trees in migration<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
import mptt
import mptt.managers

def make_locations(apps, schema_editor):
    Location = apps.get_model('inventory', 'Location')
    default_locations = [
        ('Lab', 'L', 1,),
    ]
    for item in default_locations:
        l = Location(name=item[0], code=item[1], lft=0, rght=0, level=0, tree_id=item[2])
        l.save()

def make_measures(apps, schema_editor):
    AmountMeasure = apps.get_model('inventory', 'AmountMeasure')
    default_measures = [
        ('Litres', 'l'),
        ('Millilitres', 'ml'),
        ('Microlitres', 'ul'),
        ('Nanolitres', 'nl'),
        ('Metres', 'm'),
        ('Centimetres', 'cm'),
        ('Millimetres', 'mm'),
        ('Micrometres', 'um'),
        ('Nanometres', 'nm'),
        ('Millimolar', 'mM'),
        ('Micromolar', 'uM'),
        ('Nanomolar', 'nM'),
        ('Kilogram', 'kg'),
        ('Gram', 'g'),
        ('Milligram', 'mg'),
        ('Microgram', 'ug'),
        ('Nanogram', 'ng'),
        ('Day', 'day'),
        ('Hour', 'hr'),
        ('Minute', 'min'),
        ('Second', 's'),
        ('Millisecond', 'ms'),
        ('Kelvin', 'K'),
        ('Celsius', '°C'),
        ('Microsecond', 'us'),
        ('Nanosecond', 'ns'),
        ('Microgram/microlitre', 'ug/ul'),
        ('Item', 'item'),
    ]
    for item in default_measures:
        AmountMeasure.objects.create(name=item[0], symbol=item[1])

def make_itemtypes(apps, schema_editor):
    ItemType = apps.get_model('inventory', 'ItemType')
    default_itemtypes = [
        ('Item',)
    ]
    for item in default_itemtypes:
         itm = ItemType(name=item[0], lft=0, rght=0, level=0, tree_id=0)
         itm.save()

def rebuild_trees(apps, schema_editor):
    for tree in ['Location', 'ItemType']:
        manager = mptt.managers.TreeManager()
        Object = apps.get_model('inventory', tree)
        manager.model = Object
        mptt.register(Object, order_insertion_by=['display_order', 'id'])
        manager.contribute_to_class(Object, 'objects')
        manager.rebuild()

class Migration(migrations.Migration):

    dependencies = [
        ('inventory', '0016_auto_20170123_0941'),
    ]

    operations = [
        migrations.RunPython(make_measures),
        migrations.RunPython(make_locations),
        migrations.RunPython(make_itemtypes),
    ]
<|endoftext|>


<commit_before>import Analysis

def parsePayload(context, scope):
    print 'Hello World'
    print context
    print scope

my_analysis = Analysis(parsePayload, '606109f0-bda3-11e6-a1e0-f13b10826644')<commit_msg>moving analysis test to examples<commit_after><|endoftext|>


<commit_before>'''
Pure-python parsing backend.
'''
from __future__ import unicode_literals
import decimal
import re
from codecs import getreader

from ijson import common
from ijson.compat import chr

BUFSIZE = 16 * 1024
LEXEME_RE = re.compile(r'[a-z0-9eE\.\+-]+|\S')

class UnexpectedSymbol(common.JSONError):
    def __init__(self, symbol, pos):
        super(UnexpectedSymbol, self).__init__(
            'Unexpected symbol %r at %d' % (symbol, pos)
        )

def Lexer(f, buf_size=BUFSIZE):
    f = getreader('utf-8')(f)
    buf = f.read(buf_size)
    pos = 0
    discarded = 0
    while True:
        match = LEXEME_RE.search(buf, pos)
        if match:
            lexeme = match.group()
            if lexeme == '"':
                pos = match.start()
                start = pos + 1
                while True:
                    try:
                        end = buf.index('"', start)
                        escpos = end - 1
                        while buf[escpos] == '\\':
                            escpos -= 1
                        if (end - escpos) % 2 == 0:
                            start = end + 1
                        else:
                            break
                    except ValueError:
                        data = f.read(buf_size)
                        if not data:
                            raise common.JSONError('Incomplete string lexeme')
                        buf += data
                yield discarded + pos, buf[pos:end + 1]
                pos = end + 1
            else:
                while match.end() == len(buf):
                    data = f.read(buf_size)
                    if not data:
                        break
                    buf += data
                    match = LEXEME_RE.search(buf, pos)
                    lexeme = match.group()
                yield discarded + match.start(), lexeme
                pos = match.end()
        else:
            data = f.read(buf_size)
            if not data:
                break
            discarded += len(buf)
            buf = data
            pos = 0

def unescape(s):
    start = 0
    result = ''
    while start < len(s):
        pos = s.find('\\', start)
        if pos == -1:
            if start == 0:
                return s
            result += s[start:]
            break
        result += s[start:pos]
        pos += 1
        esc = s[pos]
        if esc == 'u':
            result += chr(int(s[pos + 1:pos + 5], 16))
            pos += 4
        elif esc == 'b':
            result += '\b'
        elif esc == 'f':
            result += '\f'
        elif esc == 'n':
            result += '\n'
        elif esc == 'r':
            result += '\r'
        elif esc == 't':
            result += '\t'
        else:
            result += esc
        start = pos + 1
    return result

def parse_value(lexer, symbol=None, pos=0):
    try:
        if symbol is None:
            pos, symbol = next(lexer)
        if symbol == 'null':
            yield ('null', None)
        elif symbol == 'true':
            yield ('boolean', True)
        elif symbol == 'false':
            yield ('boolean', False)
        elif symbol == '[':
            for event in parse_array(lexer):
                yield event
        elif symbol == '{':
            for event in parse_object(lexer):
                yield event
        elif symbol[0] == '"':
            yield ('string', unescape(symbol[1:-1]))
        else:
            try:
                yield ('number', common.number(symbol))
            except decimal.InvalidOperation:
                raise UnexpectedSymbol(symbol, pos)
    except StopIteration:
        raise common.JSONError('Incomplete JSON data')

def parse_array(lexer):
    yield ('start_array', None)
    pos, symbol = next(lexer)
    if symbol != ']':
        while True:
            for event in parse_value(lexer, symbol, pos):
                yield event
            pos, symbol = next(lexer)
            if symbol == ']':
                break
            if symbol != ',':
                raise UnexpectedSymbol(symbol, pos)
            pos, symbol = next(lexer)
    yield ('end_array', None)

def parse_object(lexer):
    yield ('start_map', None)
    pos, symbol = next(lexer)
    if symbol != '}':
        while True:
            if symbol[0] != '"':
                raise UnexpectedSymbol(symbol, pos)
            yield ('map_key', symbol[1:-1])
            pos, symbol = next(lexer)
            if symbol != ':':
                raise UnexpectedSymbol(symbol, pos)
            for event in parse_value(lexer, None, pos):
                yield event
            pos, symbol = next(lexer)
            if symbol == '}':
                break
            if symbol != ',':
                raise UnexpectedSymbol(symbol, pos)
            pos, symbol = next(lexer)
    yield ('end_map', None)

def basic_parse(file=None, buf_size=BUFSIZE):
    '''
    Iterator yielding unprefixed events.

    Parameters:

    - file: a readable file-like object with JSON input
    '''
    lexer = iter(Lexer(file, buf_size))
    for value in parse_value(lexer):
        yield value
    try:
        next(lexer)
    except StopIteration:
        pass
    else:
        raise common.JSONError('Additional data')

def parse(file):
    '''
    Backend-specific wrapper for ijson.common.parse.
    '''
    return common.parse(basic_parse(file))

def items(file, prefix):
    '''
    Backend-specific wrapper for ijson.common.items.
    '''
    return common.items(parse(file), prefix)
<commit_msg>Add buf_size kwarg to python parse<commit_after>'''
Pure-python parsing backend.
'''
from __future__ import unicode_literals
import decimal
import re
from codecs import getreader

from ijson import common
from ijson.compat import chr

BUFSIZE = 16 * 1024
LEXEME_RE = re.compile(r'[a-z0-9eE\.\+-]+|\S')

class UnexpectedSymbol(common.JSONError):
    def __init__(self, symbol, pos):
        super(UnexpectedSymbol, self).__init__(
            'Unexpected symbol %r at %d' % (symbol, pos)
        )

def Lexer(f, buf_size=BUFSIZE):
    f = getreader('utf-8')(f)
    buf = f.read(buf_size)
    pos = 0
    discarded = 0
    while True:
        match = LEXEME_RE.search(buf, pos)
        if match:
            lexeme = match.group()
            if lexeme == '"':
                pos = match.start()
                start = pos + 1
                while True:
                    try:
                        end = buf.index('"', start)
                        escpos = end - 1
                        while buf[escpos] == '\\':
                            escpos -= 1
                        if (end - escpos) % 2 == 0:
                            start = end + 1
                        else:
                            break
                    except ValueError:
                        data = f.read(buf_size)
                        if not data:
                            raise common.JSONError('Incomplete string lexeme')
                        buf += data
                yield discarded + pos, buf[pos:end + 1]
                pos = end + 1
            else:
                while match.end() == len(buf):
                    data = f.read(buf_size)
                    if not data:
                        break
                    buf += data
                    match = LEXEME_RE.search(buf, pos)
                    lexeme = match.group()
                yield discarded + match.start(), lexeme
                pos = match.end()
        else:
            data = f.read(buf_size)
            if not data:
                break
            discarded += len(buf)
            buf = data
            pos = 0

def unescape(s):
    start = 0
    result = ''
    while start < len(s):
        pos = s.find('\\', start)
        if pos == -1:
            if start == 0:
                return s
            result += s[start:]
            break
        result += s[start:pos]
        pos += 1
        esc = s[pos]
        if esc == 'u':
            result += chr(int(s[pos + 1:pos + 5], 16))
            pos += 4
        elif esc == 'b':
            result += '\b'
        elif esc == 'f':
            result += '\f'
        elif esc == 'n':
            result += '\n'
        elif esc == 'r':
            result += '\r'
        elif esc == 't':
            result += '\t'
        else:
            result += esc
        start = pos + 1
    return result

def parse_value(lexer, symbol=None, pos=0):
    try:
        if symbol is None:
            pos, symbol = next(lexer)
        if symbol == 'null':
            yield ('null', None)
        elif symbol == 'true':
            yield ('boolean', True)
        elif symbol == 'false':
            yield ('boolean', False)
        elif symbol == '[':
            for event in parse_array(lexer):
                yield event
        elif symbol == '{':
            for event in parse_object(lexer):
                yield event
        elif symbol[0] == '"':
            yield ('string', unescape(symbol[1:-1]))
        else:
            try:
                yield ('number', common.number(symbol))
            except decimal.InvalidOperation:
                raise UnexpectedSymbol(symbol, pos)
    except StopIteration:
        raise common.JSONError('Incomplete JSON data')

def parse_array(lexer):
    yield ('start_array', None)
    pos, symbol = next(lexer)
    if symbol != ']':
        while True:
            for event in parse_value(lexer, symbol, pos):
                yield event
            pos, symbol = next(lexer)
            if symbol == ']':
                break
            if symbol != ',':
                raise UnexpectedSymbol(symbol, pos)
            pos, symbol = next(lexer)
    yield ('end_array', None)

def parse_object(lexer):
    yield ('start_map', None)
    pos, symbol = next(lexer)
    if symbol != '}':
        while True:
            if symbol[0] != '"':
                raise UnexpectedSymbol(symbol, pos)
            yield ('map_key', symbol[1:-1])
            pos, symbol = next(lexer)
            if symbol != ':':
                raise UnexpectedSymbol(symbol, pos)
            for event in parse_value(lexer, None, pos):
                yield event
            pos, symbol = next(lexer)
            if symbol == '}':
                break
            if symbol != ',':
                raise UnexpectedSymbol(symbol, pos)
            pos, symbol = next(lexer)
    yield ('end_map', None)

def basic_parse(file=None, buf_size=BUFSIZE):
    '''
    Iterator yielding unprefixed events.

    Parameters:

    - file: a readable file-like object with JSON input
    '''
    lexer = iter(Lexer(file, buf_size))
    for value in parse_value(lexer):
        yield value
    try:
        next(lexer)
    except StopIteration:
        pass
    else:
        raise common.JSONError('Additional data')

def parse(file, buf_size=BUFSIZE):
    '''
    Backend-specific wrapper for ijson.common.parse.
    '''
    return common.parse(basic_parse(file, buf_size=buf_size))

def items(file, prefix):
    '''
    Backend-specific wrapper for ijson.common.items.
    '''
    return common.items(parse(file), prefix)
<|endoftext|>


<commit_before>from decimal import Decimal
import re

from ijson import common

BUFSIZE = 4 * 1024
NONWS = re.compile(r'\S')
STRTERM = re.compile(r'["\\]')
NUMTERM = re.compile(r'[^0-9\.-]')
ALPHATERM = re.compile(r'[^a-z]')

class Reader(object):
    def __init__(self, f):
        self.f = f
        self.buffer = ''
        self.pos = 0

    def nextsymbol(self):
        while True:
            match = NONWS.search(self.buffer, self.pos)
            if match:
                self.pos = match.start()
                char = self.buffer[self.pos]
                if 'a' <= char <= 'z':
                    return self.readuntil(ALPHATERM)
                elif '0' <= char <= '9' or char == '-':
                    return self.readuntil(NUMTERM)
                elif char == '"':
                    self.pos += 1
                    return '"' + self.readuntil(STRTERM, '\\', True)
                else:
                    self.pos += 1
                    return char
            self.buffer = self.f.read(BUFSIZE)
            self.pos = 0
            if not len(self.buffer):
                raise common.IncompleteJSONError()

    def readuntil(self, pattern, escape=None, eatterm=False):
        result = []
        while True:
            match = pattern.search(self.buffer, self.pos)
            if match:
                pos = match.start()
                terminator = self.buffer[pos:pos + 1]
                if terminator == escape:
                    if len(self.buffer) < pos + 2:
                        raise common.IncompleteJSONError()
                    result.append(self.buffer[self.pos:pos + 2])
                    self.pos = pos + 2
                else:
                    result.append(self.buffer[self.pos:pos])
                    self.pos = pos + len(terminator) if eatterm else pos
                    return ''.join(result)
            else:
                result.append(self.buffer[self.pos:])
                self.buffer = self.f.read(BUFSIZE)
                self.pos = 0
                if not self.buffer:
                    if eatterm:
                        raise common.IncompleteJSONError()
                    else:
                        return ''.join(result)

def parse_value(f, symbol=None):
    if symbol == None:
        symbol = f.nextsymbol()
    if symbol == 'null':
        yield ('null', None)
    elif symbol == 'true':
        yield ('boolean', True)
    elif symbol == 'false':
        yield ('boolean', False)
    elif symbol == '[':
        for event in parse_array(f):
            yield event
    elif symbol == '{':
        for event in parse_object(f):
            yield event
    elif symbol[0] == '"':
        yield ('string', symbol.strip('"').decode('unicode-escape'))
    else:
        try:
            number = Decimal(symbol) if '.' in symbol else int(symbol)
            yield ('number', number)
        except ValueError:
            raise common.JSONError('Unexpected symbol')

def parse_array(f):
    yield ('start_array', None)
    expect_comma = False
    while True:
        symbol = f.nextsymbol()
        if symbol == ']':
            break
        if expect_comma:
            if symbol != ',':
                raise common.JSONError('Unexpected symbol')
        else:
            for event in parse_value(f, symbol):
                yield event
        expect_comma = not expect_comma
    yield ('end_array', None)

def parse_object(f):
    yield ('start_map', None)
    while True:
        symbol = f.nextsymbol()
        if symbol[0] != '"':
            raise common.JSONError('Unexpected symbol')
        yield ('map_key', symbol.strip('"'))
        symbol = f.nextsymbol()
        if symbol != ':':
            raise common.JSONError('Unexpected symbol')
        for event in parse_value(f):
            yield event
        symbol = f.nextsymbol()
        if symbol == '}':
            break
        if symbol != ',':
            raise common.JSONError('Unexpected symbol')
    yield ('end_map', None)

def basic_parse(f):
    f = Reader(f)
    for value in parse_value(f):
        yield value
    try:
        f.nextsymbol()
    except common.IncompleteJSONError:
        pass
    else:
        raise common.JSONError('Additional data')

def parse(file):
    return common.parse(basic_parse(file))

def items(file, prefix):
    return common.items(basic_parse(file), prefix)
<commit_msg>Turn Reader into an iterator<commit_after>from decimal import Decimal
import re

from ijson import common

BUFSIZE = 4 * 1024
NONWS = re.compile(r'\S')
STRTERM = re.compile(r'["\\]')
NUMTERM = re.compile(r'[^0-9\.-]')
ALPHATERM = re.compile(r'[^a-z]')

class Reader(object):
    def __init__(self, f):
        self.f = f

    def __iter__(self):
        self.buffer = ''
        self.pos = 0
        return self

    def next(self):
        while True:
            match = NONWS.search(self.buffer, self.pos)
            if match:
                self.pos = match.start()
                char = self.buffer[self.pos]
                if 'a' <= char <= 'z':
                    return self.readuntil(ALPHATERM)
                elif '0' <= char <= '9' or char == '-':
                    return self.readuntil(NUMTERM)
                elif char == '"':
                    self.pos += 1
                    return '"' + self.readuntil(STRTERM, '\\', True)
                else:
                    self.pos += 1
                    return char
            self.buffer = self.f.read(BUFSIZE)
            self.pos = 0
            if not len(self.buffer):
                raise common.IncompleteJSONError()

    def readuntil(self, pattern, escape=None, eatterm=False):
        result = []
        while True:
            match = pattern.search(self.buffer, self.pos)
            if match:
                pos = match.start()
                terminator = self.buffer[pos:pos + 1]
                if terminator == escape:
                    if len(self.buffer) < pos + 2:
                        raise common.IncompleteJSONError()
                    result.append(self.buffer[self.pos:pos + 2])
                    self.pos = pos + 2
                else:
                    result.append(self.buffer[self.pos:pos])
                    self.pos = pos + len(terminator) if eatterm else pos
                    return ''.join(result)
            else:
                result.append(self.buffer[self.pos:])
                self.buffer = self.f.read(BUFSIZE)
                self.pos = 0
                if not self.buffer:
                    if eatterm:
                        raise common.IncompleteJSONError()
                    else:
                        return ''.join(result)

def parse_value(f, symbol=None):
    if symbol == None:
        symbol = f.next()
    if symbol == 'null':
        yield ('null', None)
    elif symbol == 'true':
        yield ('boolean', True)
    elif symbol == 'false':
        yield ('boolean', False)
    elif symbol == '[':
        for event in parse_array(f):
            yield event
    elif symbol == '{':
        for event in parse_object(f):
            yield event
    elif symbol[0] == '"':
        yield ('string', symbol.strip('"').decode('unicode-escape'))
    else:
        try:
            number = Decimal(symbol) if '.' in symbol else int(symbol)
            yield ('number', number)
        except ValueError:
            raise common.JSONError('Unexpected symbol')

def parse_array(f):
    yield ('start_array', None)
    expect_comma = False
    while True:
        symbol = f.next()
        if symbol == ']':
            break
        if expect_comma:
            if symbol != ',':
                raise common.JSONError('Unexpected symbol')
        else:
            for event in parse_value(f, symbol):
                yield event
        expect_comma = not expect_comma
    yield ('end_array', None)

def parse_object(f):
    yield ('start_map', None)
    while True:
        symbol = f.next()
        if symbol[0] != '"':
            raise common.JSONError('Unexpected symbol')
        yield ('map_key', symbol.strip('"'))
        symbol = f.next()
        if symbol != ':':
            raise common.JSONError('Unexpected symbol')
        for event in parse_value(f):
            yield event
        symbol = f.next()
        if symbol == '}':
            break
        if symbol != ',':
            raise common.JSONError('Unexpected symbol')
    yield ('end_map', None)

def basic_parse(f):
    f = iter(Reader(f))
    for value in parse_value(f):
        yield value
    try:
        f.next()
    except common.IncompleteJSONError:
        pass
    else:
        raise common.JSONError('Additional data')

def parse(file):
    return common.parse(basic_parse(file))

def items(file, prefix):
    return common.items(basic_parse(file), prefix)
<|endoftext|>