<commit_before>#!/usr/bin/env python

from __future__ import division

import sys
import itertools
import re
import warnings

from statepoint import StatePoint

alphanum = re.compile(r"[\W_]+")

err = False

################################################################################
def parse_options():
  """Process command line arguments"""

  def tallies_callback(option, opt, value, parser):
    """Option parser function for list of tallies"""
    global err
    try:
      setattr(parser.values, option.dest, [int(v) for v in value.split(',')])
    except:
      p.print_help()
      err = True

  def scores_callback(option, opt, value, parser):
    """Option parser function for list of scores"""
    global err
    try:
      scores = {}
      entries = value.split(',')
      for e in entries:
        tally,score = [int(i) for i in e.split('.')]
        if not tally in scores: scores[tally] = []
        scores[tally].append(score)
      setattr(parser.values, option.dest, scores)
    except:
      p.print_help()
      err = True

  def filters_callback(option, opt, value, parser):
    """Option parser function for list of filters"""
    global err
    try:
      filters = {}
      entries = value.split(',')
      for e in entries:
        tally,filter_,bin = [i for i in e.split('.')]
        tally,bin = int(tally),int(bin)
        if not tally in filters: filters[tally] = {}
        if not filter_ in filters[tally]: filters[tally][filter_] = []
        filters[tally][filter_].append(bin)
      setattr(parser.values, option.dest, filters)
    except:
      p.print_help()
      err = True

  from optparse import OptionParser
  usage = r"""%prog [options] <statepoint_file>

The default is to process all tallies and all scores into one file. Subsets
can be chosen using the options.  For example, to only process tallies 2 and 4
with all scores on tally 2 and only scores 1 and 3 on tally 4:

%prog -t 2,4 -s 4.1,4.3 <statepoint_file>

Likewise if you have additional filters on a tally you can specify a subset of
bins for each filter for that tally. For example to process all tallies and
scores, but only energyin bin #1 in tally 2:

%prog -f 2.energyin.1 <statepoint_file>

You can list the available tallies, scores, and filters with the -l option:

%prog -l <statepoint_file>"""
  p = OptionParser(usage=usage)
  p.add_option('-t', '--tallies', dest='tallies', type='string', default=None,
                action='callback', callback=tallies_callback,
                help='List of tally indices to process, separated by commas.' \
                     ' Default is to process all tallies.')
  p.add_option('-s', '--scores', dest='scores', type='string', default=None,
               action='callback', callback=scores_callback,
               help='List of score indices to process, separated by commas, ' \
                 'specified as {tallyid}.{scoreid}.' \
                 ' Default is to process all scores in each tally.')
  p.add_option('-f', '--filters', dest='filters', type='string', default=None,
               action='callback', callback=filters_callback,
               help='List of filter bins to process, separated by commas, ' \
                 'specified as {tallyid}.{filter}.{binid}. ' \
                 'Default is to process all filter combinaiton for each score.')
  p.add_option('-l', '--list', dest='list', action='store_true',
               help='List the tally and score indices available in the file.')
  p.add_option('-o', '--output', action='store', dest='output',
               default='tally', help='path to output SILO file.')
  p.add_option('-e', '--error', dest='valerr', default=False,
               action='store_true',
               help='Flag to extract errors instead of values.')
  p.add_option('-v', '--vtk', action='store_true', dest='vtk',
               default=False, help='Flag to convert to VTK instead of SILO.')
  parsed = p.parse_args()

  if not parsed[1]:
    p.print_help()
    return parsed, err

  if parsed[0].valerr:
    parsed[0].valerr = 1
  else:
    parsed[0].valerr = 0

  return parsed, err

################################################################################
def main(file_, o):
  """Main program"""

  sp = StatePoint(file_)
  sp.read_results()

  validate_options(sp, o)

  if o.list:
    print_available(sp)
    return

  if o.vtk:
    if not o.output[-4:] == ".vtm": o.output += ".vtm"
  else:
    if not o.output[-5:] == ".silo": o.output += ".silo"

  if o.vtk:
    try:
      import vtk
    except:
      print 'The vtk python bindings do not appear to be installed properly.\n'+\
            'On Ubuntu: sudo apt-get install python-vtk\n'+\
            'See: http://www.vtk.org/'
      return
  else:
    try:
      import silomesh
    except:
      print 'The silomesh package does not appear to be installed properly.\n'+\
            'See: https://github.com/nhorelik/silomesh/'
      return

  if o.vtk:
    blocks = vtk.vtkMultiBlockDataSet()
    blocks.SetNumberOfBlocks(5)
    block_idx = 0
  else:
    silomesh.init_silo(o.output)

  # Tally loop #################################################################
  for tally in sp.tallies:

    # skip non-mesh tallies or non-user-specified tallies
    if o.tallies and not tally.id in o.tallies: continue
    if not 'mesh' in tally.filters: continue

    print "Processing Tally {}...".format(tally.id)

    # extract filter options and mesh parameters for this tally
    filtercombos = get_filter_combos(tally)
    meshparms = get_mesh_parms(sp, tally)
    nx,ny,nz = meshparms[:3]
    ll = meshparms[3:6]
    ur = meshparms[6:9]

    if o.vtk:
      ww = [(u-l)/n for u,l,n in zip(ur,ll,(nx,ny,nz))]
      origin = [(l+w*n/2.) for n,l,w in zip((nx,ny,nz),ll,ww)]
      grid = grid = vtk.vtkImageData()
      grid.SetDimensions(nx+1,ny+1,nz+1)
      grid.SetOrigin(*ll)
      grid.SetSpacing(*ww)
    else:
      silomesh.init_mesh('Tally_{}'.format(tally.id), *meshparms)

    # Score loop ###############################################################
    for sid,score in enumerate(tally.scores):

      # skip non-user-specified scrores for this tally
      if o.scores and tally.id in o.scores and not sid in o.scores[tally.id]:
        continue

      # Filter loop ############################################################
      for filterspec in filtercombos:

        # skip non-user-specified filter bins
        skip = False
        if o.filters and tally.id in o.filters:
          for filter_,bin in filterspec[1:]:
            if filter_ in o.filters[tally.id] and \
               not bin in o.filters[tally.id][filter_]:
              skip = True
              break
        if skip: continue

        # find and sanitize the variable name for this score
        varname = get_sanitized_filterspec_name(tally, score, filterspec)
        if o.vtk:
          vtkdata = vtk.vtkDoubleArray()
          vtkdata.SetName(varname)
          dataforvtk = {}
        else:
          silomesh.init_var(varname)

        lbl = "\t Score {}.{} {}:\t\t{}".format(tally.id, sid+1, score, varname)

        # Mesh fill loop #######################################################
        for x in range(1,nx+1):
          sys.stdout.write(lbl+" {0}%\r".format(int(x/nx*100)))
          sys.stdout.flush()
          for y in range(1,ny+1):
            for z in range(1,nz+1):
              filterspec[0][1] = (x,y,z)
              val = sp.get_value(tally.id-1, filterspec, sid)[o.valerr]
              if o.vtk:
                # vtk cells go z, y, x, so we store it now and enter it later
                i = (z-1)*nx*ny + (y-1)*nx + x-1
                dataforvtk[i] = float(val)
              else:
                silomesh.set_value(float(val), x, y, z)

        # end mesh fill loop
        print
        if o.vtk:
          for i in range(nx*ny*nz):
            vtkdata.InsertNextValue(dataforvtk[i])
          grid.GetCellData().AddArray(vtkdata)
          del vtkdata

        else:
          silomesh.finalize_var()

      # end filter loop

    # end score loop
    if o.vtk:
      blocks.SetBlock(block_idx, grid)
      block_idx += 1
    else:
      silomesh.finalize_mesh()

  # end tally loop
  if o.vtk:
    writer = vtk.vtkXMLMultiBlockDataWriter()
    writer.SetFileName(o.output)
    writer.SetInput(blocks)
    writer.Write()
  else:
    silomesh.finalize_silo()

################################################################################
def get_sanitized_filterspec_name(tally, score, filterspec):
  """Returns a name fit for silo vars for a given filterspec, tally and score"""

  comboname = "_"+" ".join(["{}_{}".format(filter_, bin)
                        for filter_, bin in filterspec[1:]])
  if len(filterspec[1:]) == 0: comboname = ''
  varname = 'Tally_{}_{}{}'.format(tally.id, score, comboname)
  varname = alphanum.sub('_', varname)
  return varname

################################################################################
def get_filter_combos(tally):
  """Returns a list of all filter spec combinations, excluding meshes

  Each combo has the mesh spec as the first element, to be set later.
  These filter specs correspond with the second argument to StatePoint.get_value
  """

  specs = []

  if len(tally.filters) == 1:
    return [[['mesh', [1, 1, 1]]]]

  filters = tally.filters.keys()
  filters.pop(filters.index('mesh'))
  nbins = [tally.filters[f].length for f in filters]

  combos = [ [b] for b in range(nbins[0])]
  for i,b in enumerate(nbins[1:]):
    prod = list(itertools.product(combos, range(b)))
    if i == 0:
      combos = prod
    else:
      combos = [[v for v in p[0]] + [p[1]] for p in prod]

  for c in combos:
    spec = [['mesh', [1, 1, 1]]]
    for i,bin in enumerate(c):
      spec.append((filters[i], bin))
    specs.append(spec)

  return specs

################################################################################
def get_mesh_parms(sp, tally):
  meshid = tally.filters['mesh'].bins[0]
  for i,m in enumerate(sp.meshes):
    if m.id == meshid:
      mesh = m
  return mesh.dimension + mesh.lower_left + mesh.upper_right

################################################################################
def print_available(sp):
  """Prints available tallies/scores in a statepoint"""

  print "Available tally and score indices:"
  for tally in sp.tallies:
    mesh = ""
    if not 'mesh' in tally.filters: mesh = "(no mesh)"
    print "\tTally {} {}".format(tally.id, mesh)
    scores = ["{}.{}: {}".format(tally.id, sid, score)
              for sid, score in enumerate(tally.scores)]
    for score in scores:
      print "\t\tScore {}".format(score)
      for filter_ in tally.filters:
        if filter_ == 'mesh': continue
        for bin in range(tally.filters[filter_].length):
          print "\t\t\tFilters: {}.{}.{}".format(tally.id, filter_, bin)

################################################################################
def validate_options(sp,o):
  """Validates specified tally/score options for the current statepoint"""

  available_tallies = [t.id for t in sp.tallies]
  if o.tallies:
    for otally in o.tallies:
      if not otally in available_tallies:
        warnings.warn('Tally {} not in statepoint file'.format(otally))
        continue
      else:
        for tally in sp.tallies:
          if tally.id == otally: break
      if not 'mesh' in tally.filters:
        warnings.warn('Tally {} contains no mesh'.format(otally))
      if o.scores and otally in o.scores.keys():
        for oscore in o.scores[otally]:
          if oscore > len(tally.scores):
            warnings.warn('No score {} in tally {}'.format(oscore, otally))

  if o.scores:
    for otally in o.scores.keys():
      if not otally in available_tallies:
        warnings.warn('Tally {} not in statepoint file'.format(otally))
        continue
      if o.tallies and not otally in o.tallies:
        warnings.warn(
          'Skipping scores for tally {}, excluded by tally list'.format(otally))
        continue

  if o.filters:
    for otally in o.filters.keys():
      if not otally in available_tallies:
        warnings.warn('Tally {} not in statepoint file'.format(otally))
        continue
      if o.tallies and not otally in o.tallies:
        warnings.warn(
          'Skipping filters for tally {}, excluded by tally list'.format(otally))
        continue
      for tally in sp.tallies:
        if tally.id == otally: break
      for filter_ in o.filters[otally]:
        if filter_ == 'mesh':
          warnings.warn('Cannot specify mesh filter bins')
          continue
        if not filter_ in tally.filters.keys():
          warnings.warn(
                  'Tally {} does not contain filter {}'.format(otally, filter_))
          continue
        for bin in o.filters[otally][filter_]:
          if bin >= tally.filters[filter_].length:
            warnings.warn(
                 'No bin {} in tally {} filter {}'.format(bin, otally, filter_))

################################################################################
# monkeypatch to suppress the source echo produced by warnings
def formatwarning(message, category, filename, lineno, line):
  return "{}:{}: {}: {}\n".format(filename, lineno, category.__name__, message)
warnings.formatwarning = formatwarning

################################################################################
if __name__ == '__main__':
    (options, args), err = parse_options()
    if args and not err:
      main(args[0],options)
<commit_msg>Removed extraneous code<commit_after>#!/usr/bin/env python

from __future__ import division

import sys
import itertools
import re
import warnings

from statepoint import StatePoint

alphanum = re.compile(r"[\W_]+")

err = False

################################################################################
def parse_options():
  """Process command line arguments"""

  def tallies_callback(option, opt, value, parser):
    """Option parser function for list of tallies"""
    global err
    try:
      setattr(parser.values, option.dest, [int(v) for v in value.split(',')])
    except:
      p.print_help()
      err = True

  def scores_callback(option, opt, value, parser):
    """Option parser function for list of scores"""
    global err
    try:
      scores = {}
      entries = value.split(',')
      for e in entries:
        tally,score = [int(i) for i in e.split('.')]
        if not tally in scores: scores[tally] = []
        scores[tally].append(score)
      setattr(parser.values, option.dest, scores)
    except:
      p.print_help()
      err = True

  def filters_callback(option, opt, value, parser):
    """Option parser function for list of filters"""
    global err
    try:
      filters = {}
      entries = value.split(',')
      for e in entries:
        tally,filter_,bin = [i for i in e.split('.')]
        tally,bin = int(tally),int(bin)
        if not tally in filters: filters[tally] = {}
        if not filter_ in filters[tally]: filters[tally][filter_] = []
        filters[tally][filter_].append(bin)
      setattr(parser.values, option.dest, filters)
    except:
      p.print_help()
      err = True

  from optparse import OptionParser
  usage = r"""%prog [options] <statepoint_file>

The default is to process all tallies and all scores into one file. Subsets
can be chosen using the options.  For example, to only process tallies 2 and 4
with all scores on tally 2 and only scores 1 and 3 on tally 4:

%prog -t 2,4 -s 4.1,4.3 <statepoint_file>

Likewise if you have additional filters on a tally you can specify a subset of
bins for each filter for that tally. For example to process all tallies and
scores, but only energyin bin #1 in tally 2:

%prog -f 2.energyin.1 <statepoint_file>

You can list the available tallies, scores, and filters with the -l option:

%prog -l <statepoint_file>"""
  p = OptionParser(usage=usage)
  p.add_option('-t', '--tallies', dest='tallies', type='string', default=None,
                action='callback', callback=tallies_callback,
                help='List of tally indices to process, separated by commas.' \
                     ' Default is to process all tallies.')
  p.add_option('-s', '--scores', dest='scores', type='string', default=None,
               action='callback', callback=scores_callback,
               help='List of score indices to process, separated by commas, ' \
                 'specified as {tallyid}.{scoreid}.' \
                 ' Default is to process all scores in each tally.')
  p.add_option('-f', '--filters', dest='filters', type='string', default=None,
               action='callback', callback=filters_callback,
               help='List of filter bins to process, separated by commas, ' \
                 'specified as {tallyid}.{filter}.{binid}. ' \
                 'Default is to process all filter combinaiton for each score.')
  p.add_option('-l', '--list', dest='list', action='store_true',
               help='List the tally and score indices available in the file.')
  p.add_option('-o', '--output', action='store', dest='output',
               default='tally', help='path to output SILO file.')
  p.add_option('-e', '--error', dest='valerr', default=False,
               action='store_true',
               help='Flag to extract errors instead of values.')
  p.add_option('-v', '--vtk', action='store_true', dest='vtk',
               default=False, help='Flag to convert to VTK instead of SILO.')
  parsed = p.parse_args()

  if not parsed[1]:
    p.print_help()
    return parsed, err

  if parsed[0].valerr:
    parsed[0].valerr = 1
  else:
    parsed[0].valerr = 0

  return parsed, err

################################################################################
def main(file_, o):
  """Main program"""

  sp = StatePoint(file_)
  sp.read_results()

  validate_options(sp, o)

  if o.list:
    print_available(sp)
    return

  if o.vtk:
    if not o.output[-4:] == ".vtm": o.output += ".vtm"
  else:
    if not o.output[-5:] == ".silo": o.output += ".silo"

  if o.vtk:
    try:
      import vtk
    except:
      print 'The vtk python bindings do not appear to be installed properly.\n'+\
            'On Ubuntu: sudo apt-get install python-vtk\n'+\
            'See: http://www.vtk.org/'
      return
  else:
    try:
      import silomesh
    except:
      print 'The silomesh package does not appear to be installed properly.\n'+\
            'See: https://github.com/nhorelik/silomesh/'
      return

  if o.vtk:
    blocks = vtk.vtkMultiBlockDataSet()
    blocks.SetNumberOfBlocks(5)
    block_idx = 0
  else:
    silomesh.init_silo(o.output)

  # Tally loop #################################################################
  for tally in sp.tallies:

    # skip non-mesh tallies or non-user-specified tallies
    if o.tallies and not tally.id in o.tallies: continue
    if not 'mesh' in tally.filters: continue

    print "Processing Tally {}...".format(tally.id)

    # extract filter options and mesh parameters for this tally
    filtercombos = get_filter_combos(tally)
    meshparms = get_mesh_parms(sp, tally)
    nx,ny,nz = meshparms[:3]
    ll = meshparms[3:6]
    ur = meshparms[6:9]

    if o.vtk:
      ww = [(u-l)/n for u,l,n in zip(ur,ll,(nx,ny,nz))]
      grid = grid = vtk.vtkImageData()
      grid.SetDimensions(nx+1,ny+1,nz+1)
      grid.SetOrigin(*ll)
      grid.SetSpacing(*ww)
    else:
      silomesh.init_mesh('Tally_{}'.format(tally.id), *meshparms)

    # Score loop ###############################################################
    for sid,score in enumerate(tally.scores):

      # skip non-user-specified scrores for this tally
      if o.scores and tally.id in o.scores and not sid in o.scores[tally.id]:
        continue

      # Filter loop ############################################################
      for filterspec in filtercombos:

        # skip non-user-specified filter bins
        skip = False
        if o.filters and tally.id in o.filters:
          for filter_,bin in filterspec[1:]:
            if filter_ in o.filters[tally.id] and \
               not bin in o.filters[tally.id][filter_]:
              skip = True
              break
        if skip: continue

        # find and sanitize the variable name for this score
        varname = get_sanitized_filterspec_name(tally, score, filterspec)
        if o.vtk:
          vtkdata = vtk.vtkDoubleArray()
          vtkdata.SetName(varname)
          dataforvtk = {}
        else:
          silomesh.init_var(varname)

        lbl = "\t Score {}.{} {}:\t\t{}".format(tally.id, sid+1, score, varname)

        # Mesh fill loop #######################################################
        for x in range(1,nx+1):
          sys.stdout.write(lbl+" {0}%\r".format(int(x/nx*100)))
          sys.stdout.flush()
          for y in range(1,ny+1):
            for z in range(1,nz+1):
              filterspec[0][1] = (x,y,z)
              val = sp.get_value(tally.id-1, filterspec, sid)[o.valerr]
              if o.vtk:
                # vtk cells go z, y, x, so we store it now and enter it later
                i = (z-1)*nx*ny + (y-1)*nx + x-1
                dataforvtk[i] = float(val)
              else:
                silomesh.set_value(float(val), x, y, z)

        # end mesh fill loop
        print
        if o.vtk:
          for i in range(nx*ny*nz):
            vtkdata.InsertNextValue(dataforvtk[i])
          grid.GetCellData().AddArray(vtkdata)
          del vtkdata

        else:
          silomesh.finalize_var()

      # end filter loop

    # end score loop
    if o.vtk:
      blocks.SetBlock(block_idx, grid)
      block_idx += 1
    else:
      silomesh.finalize_mesh()

  # end tally loop
  if o.vtk:
    writer = vtk.vtkXMLMultiBlockDataWriter()
    writer.SetFileName(o.output)
    writer.SetInput(blocks)
    writer.Write()
  else:
    silomesh.finalize_silo()

################################################################################
def get_sanitized_filterspec_name(tally, score, filterspec):
  """Returns a name fit for silo vars for a given filterspec, tally and score"""

  comboname = "_"+" ".join(["{}_{}".format(filter_, bin)
                        for filter_, bin in filterspec[1:]])
  if len(filterspec[1:]) == 0: comboname = ''
  varname = 'Tally_{}_{}{}'.format(tally.id, score, comboname)
  varname = alphanum.sub('_', varname)
  return varname

################################################################################
def get_filter_combos(tally):
  """Returns a list of all filter spec combinations, excluding meshes

  Each combo has the mesh spec as the first element, to be set later.
  These filter specs correspond with the second argument to StatePoint.get_value
  """

  specs = []

  if len(tally.filters) == 1:
    return [[['mesh', [1, 1, 1]]]]

  filters = tally.filters.keys()
  filters.pop(filters.index('mesh'))
  nbins = [tally.filters[f].length for f in filters]

  combos = [ [b] for b in range(nbins[0])]
  for i,b in enumerate(nbins[1:]):
    prod = list(itertools.product(combos, range(b)))
    if i == 0:
      combos = prod
    else:
      combos = [[v for v in p[0]] + [p[1]] for p in prod]

  for c in combos:
    spec = [['mesh', [1, 1, 1]]]
    for i,bin in enumerate(c):
      spec.append((filters[i], bin))
    specs.append(spec)

  return specs

################################################################################
def get_mesh_parms(sp, tally):
  meshid = tally.filters['mesh'].bins[0]
  for i,m in enumerate(sp.meshes):
    if m.id == meshid:
      mesh = m
  return mesh.dimension + mesh.lower_left + mesh.upper_right

################################################################################
def print_available(sp):
  """Prints available tallies/scores in a statepoint"""

  print "Available tally and score indices:"
  for tally in sp.tallies:
    mesh = ""
    if not 'mesh' in tally.filters: mesh = "(no mesh)"
    print "\tTally {} {}".format(tally.id, mesh)
    scores = ["{}.{}: {}".format(tally.id, sid, score)
              for sid, score in enumerate(tally.scores)]
    for score in scores:
      print "\t\tScore {}".format(score)
      for filter_ in tally.filters:
        if filter_ == 'mesh': continue
        for bin in range(tally.filters[filter_].length):
          print "\t\t\tFilters: {}.{}.{}".format(tally.id, filter_, bin)

################################################################################
def validate_options(sp,o):
  """Validates specified tally/score options for the current statepoint"""

  available_tallies = [t.id for t in sp.tallies]
  if o.tallies:
    for otally in o.tallies:
      if not otally in available_tallies:
        warnings.warn('Tally {} not in statepoint file'.format(otally))
        continue
      else:
        for tally in sp.tallies:
          if tally.id == otally: break
      if not 'mesh' in tally.filters:
        warnings.warn('Tally {} contains no mesh'.format(otally))
      if o.scores and otally in o.scores.keys():
        for oscore in o.scores[otally]:
          if oscore > len(tally.scores):
            warnings.warn('No score {} in tally {}'.format(oscore, otally))

  if o.scores:
    for otally in o.scores.keys():
      if not otally in available_tallies:
        warnings.warn('Tally {} not in statepoint file'.format(otally))
        continue
      if o.tallies and not otally in o.tallies:
        warnings.warn(
          'Skipping scores for tally {}, excluded by tally list'.format(otally))
        continue

  if o.filters:
    for otally in o.filters.keys():
      if not otally in available_tallies:
        warnings.warn('Tally {} not in statepoint file'.format(otally))
        continue
      if o.tallies and not otally in o.tallies:
        warnings.warn(
          'Skipping filters for tally {}, excluded by tally list'.format(otally))
        continue
      for tally in sp.tallies:
        if tally.id == otally: break
      for filter_ in o.filters[otally]:
        if filter_ == 'mesh':
          warnings.warn('Cannot specify mesh filter bins')
          continue
        if not filter_ in tally.filters.keys():
          warnings.warn(
                  'Tally {} does not contain filter {}'.format(otally, filter_))
          continue
        for bin in o.filters[otally][filter_]:
          if bin >= tally.filters[filter_].length:
            warnings.warn(
                 'No bin {} in tally {} filter {}'.format(bin, otally, filter_))

################################################################################
# monkeypatch to suppress the source echo produced by warnings
def formatwarning(message, category, filename, lineno, line):
  return "{}:{}: {}: {}\n".format(filename, lineno, category.__name__, message)
warnings.formatwarning = formatwarning

################################################################################
if __name__ == '__main__':
    (options, args), err = parse_options()
    if args and not err:
      main(args[0],options)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from __future__ import unicode_literals

import re
from datetime import datetime, date

import six
import requests

__all__ = ('FioBank',)

str = six.text_type

def coerce_date(value):
    if isinstance(value, datetime):
        return value.date()
    elif isinstance(value, date):
        return value
    else:
        return datetime.strptime(value[:10], '%Y-%m-%d').date()

def sanitize_value(value, convert=None):
    if isinstance(value, six.string_types):
        value = value.strip() or None
    if convert and value is not None:
        return convert(value)
    return value

class FioBank(object):

    base_url = 'https://www.fio.cz/ib_api/rest/'

    actions = {
        'periods': 'periods/{token}/{from_date}/{to_date}/transactions.json',
        'by-id': 'by-id/{token}/{year}/{number}/transactions.json',
        'last': 'last/{token}/transactions.json',
        'set-last-id': 'set-last-id/{token}/{from_id}/',
        'set-last-date': 'set-last-date/{token}/{from_date}/',
    }

    transaction_schema = {
        'id pohybu': ('transaction_id', str),
        'datum': ('date', str),
        'objem': ('amount', float),
        'měna': ('currency', str),
        'protiúčet': ('account_number', str),
        'název protiúčtu': ('account_name', str),
        'kód banky': ('bank_code', str),
        'bic': ('bic', str),
        'název banky': ('bank_name', str),
        'ks': ('constant_symbol', str),
        'vs': ('variable_symbol', str),
        'ss': ('specific_symbol', str),
        'uživatelská identifikace': ('user_identification', str),
        'zpráva pro příjemce': ('recipient_message', str),
        'typ': ('type', str),
        'provedl': ('executor', str),
        'upřesnění': ('specification', str),
        'komentář': ('comment', str),
        'id pokynu': ('instruction_id', str),
    }

    info_schema = {
        'accountid': ('account_number', str),
        'bankid': ('bank_code', str),
        'currency': ('currency', str),
        'iban': ('iban', str),
        'bic': ('bic', str),
        'closingbalance': ('balance', float),
    }

    _amount_re = re.compile(r'\-?[\d+](\.\d+)? [A-Z]{3}')

    def __init__(self, token):
        self.token = token

    def _request(self, action, **params):
        template = self.base_url + self.actions[action]
        url = template.format(token=self.token, **params)

        response = requests.get(url)
        response.raise_for_status()

        if response.content:
            return response.json()
        return None

    def _parse_info(self, data):
        # parse data from API
        info = {}
        for key, value in data['accountStatement']['info'].items():
            key = key.lower()
            if key in self.info_schema:
                field_name, type_ = self.info_schema[key]
                value = sanitize_value(value, type_)
                info[field_name] = value

        # make some refinements
        info['account_number_full'] = (info['account_number'] +
                                       '/' + info['bank_code'])

        # return data
        return info

    def _parse_transactions(self, data):
        schema = self.transaction_schema
        try:
            entries = data['accountStatement']['transactionList']['transaction']  # NOQA
        except TypeError:
            entries = []

        for entry in entries:
            # parse entry from API
            trans = {}
            for column_name, column_data in entry.items():
                if not column_data:
                    continue
                field_name, type_ = schema[column_data['name'].lower()]
                value = sanitize_value(column_data['value'], type_)
                trans[field_name] = value

            # make some refinements
            is_amount = self._amount_re.match
            if 'specification' in trans and is_amount(trans['specification']):
                amount, currency = trans['specification'].split(' ')
                trans['original_amount'] = float(amount)
                trans['original_currency'] = currency

            if 'date' in trans:
                trans['date'] = coerce_date(trans['date'])

            if 'account_number' in trans and 'bank_code' in trans:
                trans['account_number_full'] = (trans['account_number'] +
                                                '/' + trans['bank_code'])

            # generate transaction data
            yield trans

    def info(self):
        today = date.today()
        data = self._request('periods', from_date=today, to_date=today)
        # with open('info.json', 'w') as f:
        #     import json
        #     json.dump(data, f, indent=4)
        return self._parse_info(data)

    def period(self, from_date, to_date):
        data = self._request('periods',
                             from_date=coerce_date(from_date),
                             to_date=coerce_date(to_date))
        return self._parse_transactions(data)

    def statement(self, year, number):
        data = self._request('by-id', year=year, number=number)
        return self._parse_transactions(data)

    def last(self, from_id=None, from_date=None):
        assert not (from_id and from_date), 'Only one constraint is allowed.'

        if from_id:
            self._request('set-last-id', from_id=from_id)
        elif from_date:
            self._request('set-last-date', from_date=coerce_date(from_date))

        return self._parse_transactions(self._request('last'))
<commit_msg>feat: Always add full account number property, raise ValueError instead of assertion, fixed detection of amount in specification, pair data according to column IDs<commit_after># -*- coding: utf-8 -*-

from __future__ import unicode_literals

import re
from datetime import datetime, date

import six
import requests

__all__ = ('FioBank',)

str = six.text_type

def coerce_date(value):
    if isinstance(value, datetime):
        return value.date()
    elif isinstance(value, date):
        return value
    else:
        return datetime.strptime(value[:10], '%Y-%m-%d').date()

def sanitize_value(value, convert=None):
    if isinstance(value, six.string_types):
        value = value.strip() or None
    if convert and value is not None:
        return convert(value)
    return value

class FioBank(object):

    base_url = 'https://www.fio.cz/ib_api/rest/'

    actions = {
        'periods': 'periods/{token}/{from_date}/{to_date}/transactions.json',
        'by-id': 'by-id/{token}/{year}/{number}/transactions.json',
        'last': 'last/{token}/transactions.json',
        'set-last-id': 'set-last-id/{token}/{from_id}/',
        'set-last-date': 'set-last-date/{token}/{from_date}/',
    }

    # http://www.fio.cz/xsd/IBSchema.xsd
    transaction_schema = {
        'column22': ('transaction_id', str),
        'column0': ('date', coerce_date),
        'column1': ('amount', float),
        'column14': ('currency', str),
        'column2': ('account_number', str),
        'column10': ('account_name', str),
        'column3': ('bank_code', str),
        'column26': ('bic', str),
        'column12': ('bank_name', str),
        'column4': ('constant_symbol', str),
        'column5': ('variable_symbol', str),
        'column6': ('specific_symbol', str),
        'column7': ('user_identification', str),
        'column16': ('recipient_message', str),
        'column8': ('type', str),
        'column9': ('executor', str),
        'column18': ('specification', str),
        'column25': ('comment', str),
        'column17': ('instruction_id', str),
    }

    info_schema = {
        'accountid': ('account_number', str),
        'bankid': ('bank_code', str),
        'currency': ('currency', str),
        'iban': ('iban', str),
        'bic': ('bic', str),
        'closingbalance': ('balance', float),
    }

    _amount_re = re.compile(r'\-?\d+(\.\d+)? [A-Z]{3}')

    def __init__(self, token):
        self.token = token

    def _request(self, action, **params):
        template = self.base_url + self.actions[action]
        url = template.format(token=self.token, **params)

        response = requests.get(url)
        response.raise_for_status()

        if response.content:
            return response.json()
        return None

    def _parse_info(self, data):
        # parse data from API
        info = {}
        for key, value in data['accountStatement']['info'].items():
            key = key.lower()
            if key in self.info_schema:
                field_name, convert = self.info_schema[key]
                value = sanitize_value(value, convert)
                info[field_name] = value

        # make some refinements
        self._add_account_number_full(info)

        # return data
        return info

    def _parse_transactions(self, data):
        schema = self.transaction_schema
        try:
            entries = data['accountStatement']['transactionList']['transaction']  # NOQA
        except TypeError:
            entries = []

        for entry in entries:
            # parse entry from API
            trans = {}
            for column_name, column_data in entry.items():
                if not column_data:
                    continue
                field_name, convert = schema[column_name.lower()]
                value = sanitize_value(column_data['value'], convert)
                trans[field_name] = value

            # add missing fileds with None values
            for column_data_name, (field_name, convert) in schema.items():
                trans.setdefault(field_name, None)

            # make some refinements
            is_amount = self._amount_re.match
            if trans['specification'] is not None and is_amount(trans['specification']):
                amount, currency = trans['specification'].split(' ')
                trans['original_amount'] = float(amount)
                trans['original_currency'] = currency
            self._add_account_number_full(trans)

            # generate transaction data
            yield trans

    def _add_account_number_full(self, obj):
        account_number = obj.get('account_number')
        bank_code = obj.get('bank_code')

        if account_number is not None and bank_code is not None:
            account_number_full = '{}/{}'.format(account_number, bank_code)
        else:
            account_number_full = None

        obj['account_number_full'] = account_number_full

    def info(self):
        today = date.today()
        data = self._request('periods', from_date=today, to_date=today)
        return self._parse_info(data)

    def period(self, from_date, to_date):
        data = self._request('periods',
                             from_date=coerce_date(from_date),
                             to_date=coerce_date(to_date))
        return self._parse_transactions(data)

    def statement(self, year, number):
        data = self._request('by-id', year=year, number=number)
        return self._parse_transactions(data)

    def last(self, from_id=None, from_date=None):
        if from_id and from_date:
            raise ValueError('Only one constraint is allowed.')

        if from_id:
            self._request('set-last-id', from_id=from_id)
        elif from_date:
            self._request('set-last-date', from_date=coerce_date(from_date))

        return self._parse_transactions(self._request('last'))
<|endoftext|>


<commit_before>"""pages/index
Homepage
"""

from collections import OrderedDict

import dash_bootstrap_components as dbc

from chime_dash.app.components.base import Component
from chime_dash.app.components.footer import Footer
from chime_dash.app.components.header import Header
from chime_dash.app.components.tool_details import ToolDetails
from chime_dash.app.components.intro import Intro
from chime_dash.app.components.visualizations import Visualizations

from chime_dash.app.utils import (
    get_n_switch_values,
    parameters_deserializer,
    prepare_visualization_group,
)
from chime_dash.app.utils.callbacks import ChimeCallback

from penn_chime.models import SimSirModel

class Index(Component):
    """
    """

    @staticmethod
    def toggle_tool_details(switch_value):
        return get_n_switch_values(switch_value, 1)

    @staticmethod
    def toggle_tables(switch_value):
        return get_n_switch_values(switch_value, 3)

    def __init__(self, language, defaults):
        """
        """

        def handle_model_change_helper(pars_json):
            model = None
            pars = None
            result = []
            viz_kwargs = {}
            if pars_json:
                pars = parameters_deserializer(pars_json)
                model = SimSirModel(pars)
                viz_kwargs = dict(
                    labels=pars.labels,
                    table_mod=7,
                    max_y_axis=pars.max_y_axis,
                    # show_tables=kwargs["show_tables"],
                )

            result.extend(self.components["intro"].build(model, pars))
            result.extend(self.components["tool_details"].build(model, pars))
            for df_key in ["admits_df", "census_df", "sim_sir_w_date_df"]:
                df = model.__dict__.get(df_key, None)
                result.extend(prepare_visualization_group(df, **viz_kwargs))
            return result

        super().__init__(
            language,
            defaults,
            [
                ChimeCallback(  # If user toggles show_additional_projections, show/hide the additional intro content
                    changed_elements=OrderedDict(show_tool_details="value"),
                    dom_updates=OrderedDict(more_intro_wrapper="hidden"),
                    callback_fn=Index.toggle_tool_details,
                ),
                ChimeCallback(  # If user toggles show_tables, show/hide tables
                    changed_elements=OrderedDict(show_tables="value"),
                    dom_updates=OrderedDict(
                        SIR_table_container="hidden",
                        new_admissions_table_container="hidden",
                        admitted_patients_table_container="hidden",
                    ),
                    callback_fn=Index.toggle_tables,
                ),
                ChimeCallback(  # If the parameters or model change, update the text
                    changed_elements=OrderedDict(pars="children"),
                    dom_updates=OrderedDict(
                        intro="children",
                        more_intro="children",
                        new_admissions_graph="figure",
                        new_admissions_table="children",
                        new_admissions_download="href",
                        admitted_patients_graph="figure",
                        admitted_patients_table="children",
                        admitted_patients_download="href",
                        SIR_graph="figure",
                        SIR_table="children",
                        SIR_download="href",
                    ),
                    callback_fn=handle_model_change_helper,
                ),
            ],
        )
        self.components = OrderedDict(
            header=Header(language, defaults),
            intro=Intro(language, defaults),
            tool_details=ToolDetails(language, defaults),
            visualizations=Visualizations(language, defaults),
            footer=Footer(language, defaults),
        )

    def get_html(self):
        """Initializes the content container dash html
        """
        content = dbc.Col(
            children=self.components["header"].html
            + self.components["intro"].html
            + self.components["tool_details"].html
            + self.components["visualizations"].html
            + self.components["footer"].html,
            width=9,
            className="ml-sm-auto p-5",
        )

        return [content]
<commit_msg>Handeled exception where there is no model<commit_after>"""pages/index
Homepage
"""

from collections import OrderedDict

import dash_bootstrap_components as dbc

from chime_dash.app.components.base import Component
from chime_dash.app.components.footer import Footer
from chime_dash.app.components.header import Header
from chime_dash.app.components.tool_details import ToolDetails
from chime_dash.app.components.intro import Intro
from chime_dash.app.components.visualizations import Visualizations

from chime_dash.app.utils import (
    get_n_switch_values,
    parameters_deserializer,
    prepare_visualization_group,
)
from chime_dash.app.utils.callbacks import ChimeCallback

from penn_chime.models import SimSirModel

class Index(Component):
    """
    """

    @staticmethod
    def toggle_tool_details(switch_value):
        return get_n_switch_values(switch_value, 1)

    @staticmethod
    def toggle_tables(switch_value):
        return get_n_switch_values(switch_value, 3)

    def __init__(self, language, defaults):
        """
        """

        def handle_model_change_helper(pars_json):
            model = None
            pars = None
            result = []
            viz_kwargs = {}
            if pars_json:
                pars = parameters_deserializer(pars_json)
                model = SimSirModel(pars)
                viz_kwargs = dict(
                    labels=pars.labels,
                    table_mod=7,
                    max_y_axis=pars.max_y_axis,
                    # show_tables=kwargs["show_tables"],
                )

            result.extend(self.components["intro"].build(model, pars))
            result.extend(self.components["tool_details"].build(model, pars))
            for df_key in ["admits_df", "census_df", "sim_sir_w_date_df"]:
                df = model.__dict__.get(df_key, None) if model is not None else None
                result.extend(prepare_visualization_group(df, **viz_kwargs))
            return result

        super().__init__(
            language,
            defaults,
            [
                ChimeCallback(  # If user toggles show_additional_projections, show/hide the additional intro content
                    changed_elements=OrderedDict(show_tool_details="value"),
                    dom_updates=OrderedDict(more_intro_wrapper="hidden"),
                    callback_fn=Index.toggle_tool_details,
                ),
                ChimeCallback(  # If user toggles show_tables, show/hide tables
                    changed_elements=OrderedDict(show_tables="value"),
                    dom_updates=OrderedDict(
                        SIR_table_container="hidden",
                        new_admissions_table_container="hidden",
                        admitted_patients_table_container="hidden",
                    ),
                    callback_fn=Index.toggle_tables,
                ),
                ChimeCallback(  # If the parameters or model change, update the text
                    changed_elements=OrderedDict(pars="children"),
                    dom_updates=OrderedDict(
                        intro="children",
                        more_intro="children",
                        new_admissions_graph="figure",
                        new_admissions_table="children",
                        new_admissions_download="href",
                        admitted_patients_graph="figure",
                        admitted_patients_table="children",
                        admitted_patients_download="href",
                        SIR_graph="figure",
                        SIR_table="children",
                        SIR_download="href",
                    ),
                    callback_fn=handle_model_change_helper,
                ),
            ],
        )
        self.components = OrderedDict(
            header=Header(language, defaults),
            intro=Intro(language, defaults),
            tool_details=ToolDetails(language, defaults),
            visualizations=Visualizations(language, defaults),
            footer=Footer(language, defaults),
        )

    def get_html(self):
        """Initializes the content container dash html
        """
        content = dbc.Col(
            children=self.components["header"].html
            + self.components["intro"].html
            + self.components["tool_details"].html
            + self.components["visualizations"].html
            + self.components["footer"].html,
            width=9,
            className="ml-sm-auto p-5",
        )

        return [content]
<|endoftext|>


<commit_before>import pytest

from contextlib import contextmanager
from urlparse import urlparse

from flask import json, template_rendered
from flask.testing import FlaskClient
from werkzeug.urls import url_encode

from udata import settings
from udata.app import create_app
from udata.core.user.factories import UserFactory
from udata.models import db
from udata.search import es

class TestClient(FlaskClient):
    def _build_url(self, url, kwargs):
        if 'qs' not in kwargs:
            return url
        qs = kwargs.pop('qs')
        return '?'.join([url, url_encode(qs)])

    def get(self, url, **kwargs):
        url = self._build_url(url, kwargs)
        return super(TestClient, self).get(url, **kwargs)

    def post(self, url, data=None, **kwargs):
        url = self._build_url(url, kwargs)
        return super(TestClient, self).post(url, data=data, **kwargs)

    def put(self, url, data=None, **kwargs):
        url = self._build_url(url, kwargs)
        return super(TestClient, self).put(url, data=data, **kwargs)

    def delete(self, url, data=None, **kwargs):
        url = self._build_url(url, kwargs)
        return super(TestClient, self).delete(url, data=data, **kwargs)

    def login(self, user=None):
        user = user or UserFactory()
        with self.session_transaction() as session:
            session['user_id'] = str(user.id)
            session['_fresh'] = True
        return user

@pytest.fixture
def app(request):
    test_settings, extra = get_settings(request)
    app = create_app(settings.Defaults, override=test_settings)
    app.test_client_class = TestClient
    for key, value in extra.items():
        app.config[key] = value

    if request.cls and hasattr(request.cls, 'modules'):
        from udata import frontend, api
        api.init_app(app)
        frontend.init_app(app, request.cls.modules)
    return app

@pytest.fixture
def client(app):
    '''
    Fixes https://github.com/pytest-dev/pytest-flask/issues/42
    '''
    return app.test_client()

def get_settings(request):
    '''
    Extract settings from the current test request
    '''
    marker = request.node.get_marker('settings')
    if marker:
        cls = marker.args[0] if len(marker.args) else settings.Testing
        return cls, marker.kwargs
    return getattr(request.cls, 'settings', settings.Testing), {}

def drop_db(app):
    '''Clear the database'''
    parsed_url = urlparse(app.config['MONGODB_HOST'])

    # drop the leading /
    db_name = parsed_url.path[1:]
    db.connection.drop_database(db_name)

@pytest.fixture
def clean_db(app):
    drop_db(app)
    yield
    drop_db(app)

class ApiClient(object):
    def __init__(self, client):
        self.client = client
        self._user = None

    @contextmanager
    def user(self, user=None):
        self._user = user or UserFactory()
        if not self._user.apikey:
            self._user.generate_api_key()
            self._user.save()
        yield self._user

    def perform(self, verb, url, **kwargs):
        headers = kwargs.pop('headers', {})
        headers['Content-Type'] = 'application/json'

        data = kwargs.get('data')
        if data is not None:
            data = json.dumps(data)
            headers['Content-Length'] = len(data)
            kwargs['data'] = data

        if self._user:
            headers['X-API-KEY'] = kwargs.get('X-API-KEY', self._user.apikey)

        kwargs['headers'] = headers
        method = getattr(self.client, verb)
        return method(url, **kwargs)

    def get(self, url, *args, **kwargs):
        return self.perform('get', url, *args, **kwargs)

    def post(self, url, data=None, json=True, *args, **kwargs):
        if not json:
            return self.client.post(url, data or {}, *args, **kwargs)
        return self.perform('post', url, data=data or {}, *args, **kwargs)

    def put(self, url, data=None, json=True, *args, **kwargs):
        if not json:
            return self.client.put(url, data or {}, *args, **kwargs)
        return self.perform('put', url, data=data or {}, *args, **kwargs)

    def delete(self, url, data=None, *args, **kwargs):
        return self.perform('delete', url, data=data or {}, *args, **kwargs)

@pytest.fixture
def api(client):
    api_client = ApiClient(client)
    return api_client

@pytest.fixture
def autoindex(app, clean_db):
    app.config['AUTO_INDEX'] = True
    es.initialize()
    es.cluster.health(wait_for_status='yellow', request_timeout=10)

    @contextmanager
    def cm():
        yield
        es.indices.refresh(index=es.index_name)

    yield cm

    if es.indices.exists(index=es.index_name):
        es.indices.delete(index=es.index_name)

@pytest.fixture(name='cli')
def cli_fixture(mocker, app):

    def mock_runner(*args):
        from click.testing import CliRunner
        from udata.commands import cli

        if len(args) == 1 and ' ' in args[0]:
            args = args[0].split()
        runner = CliRunner()
        # Avoid instanciating another app and reuse the app fixture
        with mocker.patch.object(cli, 'create_app', return_value=app):
            return runner.invoke(cli, args, catch_exceptions=False)

    return mock_runner

@pytest.fixture
def instance_path(app, tmpdir):
    '''Use temporary application instance_path'''
    from udata.core import storages
    from udata.core.storages.views import blueprint

    app.instance_path = str(tmpdir)
    storages.init_app(app)
    app.register_blueprint(blueprint)

    return tmpdir

class ContextVariableDoesNotExist(Exception):
    pass

class TemplateRecorder:
    @contextmanager
    def capture(self):
        self.templates = []
        template_rendered.connect(self._add_template)
        yield
        template_rendered.disconnect(self._add_template)

    def _add_template(self, app, template, context):
        self.templates.append((template, context))

    def assert_used(self, name):
        """
        Checks if a given template is used in the request.

        :param name: template name
        """
        __tracebackhide__ = True

        used_templates = []

        for template, context in self.templates:
            if template.name == name:
                return True

            used_templates.append(template)

        msg = 'Template %s not used. Templates were used: %s' % (
            name, ' '.join(repr(used_templates))
        )
        raise AssertionError(msg)

    def get_context_variable(self, name):
        """
        Returns a variable from the context passed to the template.

        :param name: name of variable
        :raises ContextVariableDoesNotExist: if does not exist.
        """
        for template, context in self.templates:
            if name in context:
                return context[name]
        raise ContextVariableDoesNotExist

@pytest.fixture
def templates():
    recorder = TemplateRecorder()
    with recorder.capture():
        yield recorder

@pytest.fixture
def httpretty():
    import httpretty
    httpretty.reset()
    httpretty.enable()
    yield httpretty
    httpretty.disable()
<commit_msg>Revert "Allow individual setting as kwargs" (should not have been pushed)<commit_after>import pytest

from contextlib import contextmanager
from urlparse import urlparse

from flask import json, template_rendered
from flask.testing import FlaskClient
from werkzeug.urls import url_encode

from udata import settings
from udata.app import create_app
from udata.core.user.factories import UserFactory
from udata.models import db
from udata.search import es

class TestClient(FlaskClient):
    def _build_url(self, url, kwargs):
        if 'qs' not in kwargs:
            return url
        qs = kwargs.pop('qs')
        return '?'.join([url, url_encode(qs)])

    def get(self, url, **kwargs):
        url = self._build_url(url, kwargs)
        return super(TestClient, self).get(url, **kwargs)

    def post(self, url, data=None, **kwargs):
        url = self._build_url(url, kwargs)
        return super(TestClient, self).post(url, data=data, **kwargs)

    def put(self, url, data=None, **kwargs):
        url = self._build_url(url, kwargs)
        return super(TestClient, self).put(url, data=data, **kwargs)

    def delete(self, url, data=None, **kwargs):
        url = self._build_url(url, kwargs)
        return super(TestClient, self).delete(url, data=data, **kwargs)

    def login(self, user=None):
        user = user or UserFactory()
        with self.session_transaction() as session:
            session['user_id'] = str(user.id)
            session['_fresh'] = True
        return user

@pytest.fixture
def app(request):
    test_settings = get_settings(request)
    app = create_app(settings.Defaults, override=test_settings)
    app.test_client_class = TestClient

    if request.cls and hasattr(request.cls, 'modules'):
        from udata import frontend, api
        api.init_app(app)
        frontend.init_app(app, request.cls.modules)
    return app

@pytest.fixture
def client(app):
    '''
    Fixes https://github.com/pytest-dev/pytest-flask/issues/42
    '''
    return app.test_client()

def get_settings(request):
    '''
    Extract settings from the current test request
    '''
    marker = request.node.get_marker('settings')
    if marker:
        return marker.args[0]
    return getattr(request.cls, 'settings', settings.Testing)

def drop_db(app):
    '''Clear the database'''
    parsed_url = urlparse(app.config['MONGODB_HOST'])

    # drop the leading /
    db_name = parsed_url.path[1:]
    db.connection.drop_database(db_name)

@pytest.fixture
def clean_db(app):
    drop_db(app)
    yield
    drop_db(app)

class ApiClient(object):
    def __init__(self, client):
        self.client = client
        self._user = None

    @contextmanager
    def user(self, user=None):
        self._user = user or UserFactory()
        if not self._user.apikey:
            self._user.generate_api_key()
            self._user.save()
        yield self._user

    def perform(self, verb, url, **kwargs):
        headers = kwargs.pop('headers', {})
        headers['Content-Type'] = 'application/json'

        data = kwargs.get('data')
        if data is not None:
            data = json.dumps(data)
            headers['Content-Length'] = len(data)
            kwargs['data'] = data

        if self._user:
            headers['X-API-KEY'] = kwargs.get('X-API-KEY', self._user.apikey)

        kwargs['headers'] = headers
        method = getattr(self.client, verb)
        return method(url, **kwargs)

    def get(self, url, *args, **kwargs):
        return self.perform('get', url, *args, **kwargs)

    def post(self, url, data=None, json=True, *args, **kwargs):
        if not json:
            return self.client.post(url, data or {}, *args, **kwargs)
        return self.perform('post', url, data=data or {}, *args, **kwargs)

    def put(self, url, data=None, json=True, *args, **kwargs):
        if not json:
            return self.client.put(url, data or {}, *args, **kwargs)
        return self.perform('put', url, data=data or {}, *args, **kwargs)

    def delete(self, url, data=None, *args, **kwargs):
        return self.perform('delete', url, data=data or {}, *args, **kwargs)

@pytest.fixture
def api(client):
    api_client = ApiClient(client)
    return api_client

@pytest.fixture
def autoindex(app, clean_db):
    app.config['AUTO_INDEX'] = True
    es.initialize()
    es.cluster.health(wait_for_status='yellow', request_timeout=10)

    @contextmanager
    def cm():
        yield
        es.indices.refresh(index=es.index_name)

    yield cm

    if es.indices.exists(index=es.index_name):
        es.indices.delete(index=es.index_name)

@pytest.fixture(name='cli')
def cli_fixture(mocker, app):

    def mock_runner(*args):
        from click.testing import CliRunner
        from udata.commands import cli

        if len(args) == 1 and ' ' in args[0]:
            args = args[0].split()
        runner = CliRunner()
        # Avoid instanciating another app and reuse the app fixture
        with mocker.patch.object(cli, 'create_app', return_value=app):
            return runner.invoke(cli, args, catch_exceptions=False)

    return mock_runner

@pytest.fixture
def instance_path(app, tmpdir):
    '''Use temporary application instance_path'''
    from udata.core import storages
    from udata.core.storages.views import blueprint

    app.instance_path = str(tmpdir)
    storages.init_app(app)
    app.register_blueprint(blueprint)

    return tmpdir

class ContextVariableDoesNotExist(Exception):
    pass

class TemplateRecorder:
    @contextmanager
    def capture(self):
        self.templates = []
        template_rendered.connect(self._add_template)
        yield
        template_rendered.disconnect(self._add_template)

    def _add_template(self, app, template, context):
        self.templates.append((template, context))

    def assert_used(self, name):
        """
        Checks if a given template is used in the request.

        :param name: template name
        """
        __tracebackhide__ = True

        used_templates = []

        for template, context in self.templates:
            if template.name == name:
                return True

            used_templates.append(template)

        msg = 'Template %s not used. Templates were used: %s' % (
            name, ' '.join(repr(used_templates))
        )
        raise AssertionError(msg)

    def get_context_variable(self, name):
        """
        Returns a variable from the context passed to the template.

        :param name: name of variable
        :raises ContextVariableDoesNotExist: if does not exist.
        """
        for template, context in self.templates:
            if name in context:
                return context[name]
        raise ContextVariableDoesNotExist

@pytest.fixture
def templates():
    recorder = TemplateRecorder()
    with recorder.capture():
        yield recorder

@pytest.fixture
def httpretty():
    import httpretty
    httpretty.reset()
    httpretty.enable()
    yield httpretty
    httpretty.disable()
<|endoftext|>


<commit_before>import asyncio
import typing
import logging
import traceback
if typing.TYPE_CHECKING:
    from types import AsyncGeneratorType

log = logging.getLogger(__name__)

def cancel_task(task: typing.Optional[asyncio.Task]):
    if task and not (task.done() or task.cancelled()):
        task.cancel()

def drain_tasks(tasks: typing.List[typing.Optional[asyncio.Task]]):
    while tasks:
        cancel_task(tasks.pop())

class AsyncGeneratorJunction:
    """
    A helper to interleave the results from multiple async generators into one
    async generator.
    """

    def __init__(self, loop: asyncio.BaseEventLoop, queue: typing.Optional[asyncio.Queue] = None):
        self.loop = loop
        self.result_queue = queue or asyncio.Queue(loop=loop)
        self.tasks: typing.List[asyncio.Task] = []
        self.running_iterators: typing.Dict[typing.AsyncGenerator, bool] = {}
        self.generator_queue: asyncio.Queue = asyncio.Queue(loop=self.loop)
        self.can_iterate = asyncio.Event(loop=self.loop)
        self.finished = asyncio.Event(loop=self.loop)

    @property
    def running(self):
        return any(self.running_iterators.values())

    async def wait_for_generators(self):
        async def iterate(iterator: typing.AsyncGenerator):
            try:
                async for item in iterator:
                    self.result_queue.put_nowait(item)
            finally:
                self.running_iterators[iterator] = False

        while True:
            async_gen: typing.Union[typing.AsyncGenerator, 'AsyncGeneratorType'] = await self.generator_queue.get()
            self.running_iterators[async_gen] = True
            self.tasks.append(self.loop.create_task(iterate(async_gen)))
            if not self.can_iterate.is_set():
                self.can_iterate.set()

    def add_generator(self, async_gen: typing.Union[typing.AsyncGenerator, 'AsyncGeneratorType']):
        """
        Add an async generator. This can be called during an iteration of the generator junction.
        """
        self.generator_queue.put_nowait(async_gen)

    def __aiter__(self):
        return self

    async def __anext__(self):
        if not self.can_iterate.is_set():
            await self.can_iterate.wait()
        if not self.running:
            raise StopAsyncIteration()
        try:
            return await self.result_queue.get()
        finally:
            self.awaiting = None

    def aclose(self):
        async def _aclose():
            for iterator in list(self.running_iterators.keys()):
                result = iterator.aclose()
                if asyncio.iscoroutine(result):
                    await result
                self.running_iterators[iterator] = False
            drain_tasks(self.tasks)
            raise StopAsyncIteration()
        if not self.finished.is_set():
            self.finished.set()
        return self.loop.create_task(_aclose())

    async def __aenter__(self):
        self.tasks.append(self.loop.create_task(self.wait_for_generators()))
        return self

    async def __aexit__(self, exc_type, exc, tb):
        try:
            await self.aclose()
        except StopAsyncIteration:
            pass
        finally:
            if exc_type:
                if exc_type not in (asyncio.CancelledError, asyncio.TimeoutError, StopAsyncIteration, GeneratorExit):
                    err = traceback.format_exception(exc_type, exc, tb)
                    log.error(err)
<commit_msg>fix generator junction<commit_after>import asyncio
import typing
import logging
import traceback
if typing.TYPE_CHECKING:
    from types import AsyncGeneratorType

log = logging.getLogger(__name__)

def cancel_task(task: typing.Optional[asyncio.Task]):
    if task and not (task.done() or task.cancelled()):
        task.cancel()

def drain_tasks(tasks: typing.List[typing.Optional[asyncio.Task]]):
    while tasks:
        cancel_task(tasks.pop())

class AsyncGeneratorJunction:
    """
    A helper to interleave the results from multiple async generators into one
    async generator.
    """

    def __init__(self, loop: asyncio.BaseEventLoop, queue: typing.Optional[asyncio.Queue] = None):
        self.loop = loop
        self.__iterator_queue = asyncio.Queue(loop=loop)
        self.result_queue = queue or asyncio.Queue(loop=loop)
        self.tasks: typing.List[asyncio.Task] = []
        self.running_iterators: typing.Dict[typing.AsyncGenerator, bool] = {}
        self.generator_queue: asyncio.Queue = asyncio.Queue(loop=self.loop)

    @property
    def running(self):
        return any(self.running_iterators.values())

    async def wait_for_generators(self):
        async def iterate(iterator: typing.AsyncGenerator):
            try:
                async for item in iterator:
                    self.result_queue.put_nowait(item)
                    self.__iterator_queue.put_nowait(item)
            finally:
                self.running_iterators[iterator] = False
                if not self.running:
                    self.__iterator_queue.put_nowait(StopAsyncIteration)

        while True:
            async_gen: typing.Union[typing.AsyncGenerator, 'AsyncGeneratorType'] = await self.generator_queue.get()
            self.running_iterators[async_gen] = True
            self.tasks.append(self.loop.create_task(iterate(async_gen)))

    def add_generator(self, async_gen: typing.Union[typing.AsyncGenerator, 'AsyncGeneratorType']):
        """
        Add an async generator. This can be called during an iteration of the generator junction.
        """
        self.generator_queue.put_nowait(async_gen)

    def __aiter__(self):
        return self

    async def __anext__(self):
        result = await self.__iterator_queue.get()
        if result is StopAsyncIteration:
            raise result
        return result

    def aclose(self):
        async def _aclose():
            for iterator in list(self.running_iterators.keys()):
                result = iterator.aclose()
                if asyncio.iscoroutine(result):
                    await result
                self.running_iterators[iterator] = False
            drain_tasks(self.tasks)
            raise StopAsyncIteration()
        return self.loop.create_task(_aclose())

    async def __aenter__(self):
        self.tasks.append(self.loop.create_task(self.wait_for_generators()))
        return self

    async def __aexit__(self, exc_type, exc, tb):
        try:
            await self.aclose()
        except StopAsyncIteration:
            pass
        finally:
            if exc_type:
                if exc_type not in (asyncio.CancelledError, asyncio.TimeoutError, StopAsyncIteration, GeneratorExit):
                    err = traceback.format_exception(exc_type, exc, tb)
                    log.error(err)
<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-
from __future__ import print_function, unicode_literals, division
import os
from bs4 import BeautifulSoup

path = os.path.dirname(os.path.realpath(__file__))

class EEP(object):
    def __init__(self):
        self.ok = False
        try:
            with open(os.path.join(path, 'EEP_2.6.1.xml'), 'r') as f:
                self.soup = BeautifulSoup(f.read())
            self.ok = True
        except IOError:
            pass

    def _get_hex(self, nmb):
        ''' Get hex-representation of number '''
        return '0x%02X' % (nmb)

    def _get_raw(self, value, bitarray):
        ''' Get raw data as integer, based on offset and size '''
        offset = int(value['offset'])
        size = int(value['size'])
        return int(''.join(['1' if digit else '0' for digit in bitarray[offset:offset + size]]), 2)

    def _get_value(self, value, data):
        ''' Get value, based on the data in XML '''
        raw_value = self._get_raw(value, data)

        rng = value.find('range')
        rng_min = float(rng.find('min').text)
        rng_max = float(rng.find('max').text)

        scl = value.find('scale')
        scl_min = float(scl.find('min').text)
        scl_max = float(scl.find('max').text)

        return {
            value['shortcut']: {
                'description': value['description'],
                'unit': value['unit'],
                'value': (scl_max - scl_min) / (rng_max - rng_min) * (raw_value - rng_min) + scl_min,
                'raw_value': raw_value,
            }
        }

    def _get_enum(self, value, data):
        ''' Get enum value, based on the data in XML '''
        raw_value = self._get_raw(value, data)
        value_desc = value.find('item', {'value': str(raw_value)})
        return {
            value['shortcut']: {
                'description': value['description'],
                'unit': value.get('unit', ''),
                'value': value_desc['description'],
                'raw_value': raw_value,
            }
        }

    def find_profile(self, rorg, func, type):
        ''' Find profile and data description, matching RORG, FUNC and TYPE '''
        if not self.ok:
            return None, None

        rorg = self.soup.find('telegram', {'rorg': self._get_hex(rorg)})
        if not rorg:
            return None, None

        func = rorg.find('profiles', {'func': self._get_hex(func)})
        if not func:
            return None, None

        profile = rorg.find('profile', {'type': self._get_hex(type)})
        if not profile:
            return None, None

        data_description = profile.find('data')
        if not data_description:
            data_description = []

        return profile, data_description

    def get_values(self, rorg, func, type, data):
        ''' Get keys and values from data, matching RORG, FUNC and TYPE '''
        if not self.ok:
            return [], {}

        profile, data_description = self.find_profile(rorg, func, type)
        if not profile or not data_description:
            return [], {}

        output = {}
        for d in data_description.contents:
            if not d.name:
                continue
            if d.name == 'value':
                output.update(self._get_value(d, data))
            if d.name == 'enum':
                output.update(self._get_enum(d, data))
        return output.keys(), output
<commit_msg>- add logging to enocean.protocol.eep<commit_after># -*- encoding: utf-8 -*-
from __future__ import print_function, unicode_literals, division
import os
from bs4 import BeautifulSoup
import logging

logger = logging.getLogger('enocean.protocol.eep')

path = os.path.dirname(os.path.realpath(__file__))

class EEP(object):
    def __init__(self):
        self.ok = False
        try:
            with open(os.path.join(path, 'EEP_2.6.1.xml'), 'r') as f:
                self.soup = BeautifulSoup(f.read())
            self.ok = True
        except IOError:
            logger.warn('Cannot load protocol file!')
            pass

    def _get_hex(self, nmb):
        ''' Get hex-representation of number '''
        return '0x%02X' % (nmb)

    def _get_raw(self, value, bitarray):
        ''' Get raw data as integer, based on offset and size '''
        offset = int(value['offset'])
        size = int(value['size'])
        return int(''.join(['1' if digit else '0' for digit in bitarray[offset:offset + size]]), 2)

    def _get_value(self, value, data):
        ''' Get value, based on the data in XML '''
        raw_value = self._get_raw(value, data)

        rng = value.find('range')
        rng_min = float(rng.find('min').text)
        rng_max = float(rng.find('max').text)

        scl = value.find('scale')
        scl_min = float(scl.find('min').text)
        scl_max = float(scl.find('max').text)

        return {
            value['shortcut']: {
                'description': value['description'],
                'unit': value['unit'],
                'value': (scl_max - scl_min) / (rng_max - rng_min) * (raw_value - rng_min) + scl_min,
                'raw_value': raw_value,
            }
        }

    def _get_enum(self, value, data):
        ''' Get enum value, based on the data in XML '''
        raw_value = self._get_raw(value, data)
        value_desc = value.find('item', {'value': str(raw_value)})
        return {
            value['shortcut']: {
                'description': value['description'],
                'unit': value.get('unit', ''),
                'value': value_desc['description'],
                'raw_value': raw_value,
            }
        }

    def find_profile(self, rorg, func, type):
        ''' Find profile and data description, matching RORG, FUNC and TYPE '''
        if not self.ok:
            return None, None

        rorg = self.soup.find('telegram', {'rorg': self._get_hex(rorg)})
        if not rorg:
            logger.warn('Cannot find rorg in EEP!')
            return None, None

        func = rorg.find('profiles', {'func': self._get_hex(func)})
        if not func:
            logger.warn('Cannot find func in EEP!')
            return None, None

        profile = rorg.find('profile', {'type': self._get_hex(type)})
        if not profile:
            logger.warn('Cannot find type in EEP!')
            return None, None

        data_description = profile.find('data')
        if not data_description:
            logger.warn('Cannot find data description in EEP!')
            data_description = []

        return profile, data_description

    def get_values(self, rorg, func, type, data):
        ''' Get keys and values from data, matching RORG, FUNC and TYPE '''
        if not self.ok:
            return [], {}

        profile, data_description = self.find_profile(rorg, func, type)
        if not profile or not data_description:
            return [], {}

        output = {}
        for d in data_description.contents:
            if not d.name:
                continue
            if d.name == 'value':
                output.update(self._get_value(d, data))
            if d.name == 'enum':
                output.update(self._get_enum(d, data))
        return output.keys(), output
<|endoftext|>


<commit_before>import os
from pdb import set_trace

import pygame

from behaviours import Collide
from src.utils.ClassGetter import get_class_that_defined_method

class Entity:

    def __init__(self, x, y, name):
        self.deletion_pending = False
        self.spawn_x = x
        self.spawn_y = y
        self.x = x  # do not modify directly, use self.set_x
        self.y = y  # do not modify directly, use self.set_y
        self.width = 1  # do not modify directly, use self.set_width
        self.height = 1  # do not modify directly, use self.set_height
        self.behaviours = {}
        self.listeners = {}
        self.name = name
        self.velocity = pygame.math.Vector2(0, 0)
        self.onFloor = False
        self.register_behaviour(Collide.Collide())
        path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "../" + name.lower() + "/" + name + ".png")
        try:
            self.sprite = pygame.image.load(path)
        except:
            print("Could not load sprite for " + name)

    def set_x(self, x):
        self.x = x

    def set_y(self, y):
        self.y = y

    def set_width(self, width):
        self.width = width

    def set_height(self, height):
        self.height = height

    def add_listener(self, func_name, callback):
        func = getattr(self, func_name)
        if func.__name__ != "patched_func":
            cls = get_class_that_defined_method(func)
            name = func.__name__
            cls_func = getattr(cls, name)

            def patched_func(self, *args, **kwargs):
                cls_func(self, *args, **kwargs)
                if func_name in self.listeners:
                    for listener in self.listeners[func_name]:
                        listener()

            setattr(cls, name, patched_func)
            func = getattr(cls, name)
        if func in self.listeners:
            self.listeners[func_name].append(callback)
        else:
            self.listeners[func_name] = [callback]

    def remove_listener(self, func_name, callback):
        if func_name in self.listeners and callback in self.listeners[func_name]:
            self.listeners[func_name].remove(callback)
        return

    def get_behaviour(self, behaviour_name):
        if not isinstance(behaviour_name, str):
            behaviour_name = behaviour_name.__name__
        if behaviour_name in self.behaviours:
            return self.behaviours[behaviour_name]
        else:
            return None

    def register_behaviour(self, behaviour):
        behaviour.set_owner(self, None, None, None)
        self.behaviours[type(behaviour).__name__] = behaviour

    def register_behaviours(self, behaviours):
        for name, behaviour in behaviours.items():
            self.register_behaviour(behaviour)

    def update(self, delta_time, keys, config, state):
        for name, behaviour in self.behaviours.items():
            behaviour.update(delta_time, keys, config)
        self.update_position(delta_time)
        if self.deletion_pending:
            self.clear()

    def update_position(self, delta_time):
        dx = delta_time * self.velocity.x
        dy = delta_time * self.velocity.y

        c = self.get_behaviour(Collide.Collide)
        if c is None:
            self.set_y(self.y + dy)
            self.set_x(self.x + dx)
        else:
            if dy < 0:
                colliding_top = c.check_top(abs(dy))
                if len(colliding_top) <= 0:
                    self.set_y(self.y + dy)
                else:
                    self.move_top_to(Collide.get_bottom_most_of(colliding_top).get_bottom())
                    self.velocity.y = 0
            else:
                colliding_bottom = c.check_bottom(abs(dy))
                if len(colliding_bottom) <= 0:
                    self.set_y(self.y + dy)
                else:
                    self.move_bottom_to(Collide.get_top_most_of(colliding_bottom).get_top())
                    self.velocity.y = 0
            if dx < 0:
                colliding_left = c.check_left(abs(dx))
                if len(colliding_left) <= 0:
                    self.set_x(self.x + dx)
                else:
                    self.move_left_to(Collide.get_right_most_of(colliding_left).get_right())
                    self.velocity.x = 0
            else:
                colliding_right = c.check_right(abs(dx))
                if len(colliding_right) <= 0:
                    self.set_x(self.x + dx)
                else:
                    self.move_right_to(Collide.get_left_most_of(colliding_right).get_left())
                    self.velocity.x = 0

    def move_top_to(self, y):
        self.set_y(y)

    def get_top(self):
        return self.y

    def move_vertical_center_to(self, y):
        self.set_y(y - self.height / 2)

    def get_vertical_center(self):
        return self.y + self.height / 2

    def move_bottom_to(self, y):
        self.set_y(y - self.height)

    def get_bottom(self):
        return self.y + self.height

    def move_left_to(self, x):
        self.set_x(x)

    def get_left(self):
        return self.x

    def move_horizontal_center_to(self, x):
        self.set_x(x - self.width / 2)

    def get_horizontal_center(self):
        return self.x + self.width / 2

    def move_right_to(self, x):
        self.set_x(x - self.width)

    def get_right(self):
        return self.x + self.width

    def clear(self):
        self.deletion_pending = True
        c = self.get_behaviour(Collide.Collide)
        if c is not None:
            c.clear()

    def __del__(self):
        self.clear()
<commit_msg>Removes entity.onFloor<commit_after>import os
from pdb import set_trace

import pygame

from behaviours import Collide
from src.utils.ClassGetter import get_class_that_defined_method

class Entity:

    def __init__(self, x, y, name):
        self.deletion_pending = False
        self.spawn_x = x
        self.spawn_y = y
        self.x = x  # do not modify directly, use self.set_x
        self.y = y  # do not modify directly, use self.set_y
        self.width = 1  # do not modify directly, use self.set_width
        self.height = 1  # do not modify directly, use self.set_height
        self.behaviours = {}
        self.listeners = {}
        self.name = name
        self.velocity = pygame.math.Vector2(0, 0)
        self.register_behaviour(Collide.Collide())
        path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "../" + name.lower() + "/" + name + ".png")
        try:
            self.sprite = pygame.image.load(path)
        except:
            print("Could not load sprite for " + name)

    def set_x(self, x):
        self.x = x

    def set_y(self, y):
        self.y = y

    def set_width(self, width):
        self.width = width

    def set_height(self, height):
        self.height = height

    def add_listener(self, func_name, callback):
        func = getattr(self, func_name)
        if func.__name__ != "patched_func":
            cls = get_class_that_defined_method(func)
            name = func.__name__
            cls_func = getattr(cls, name)

            def patched_func(self, *args, **kwargs):
                cls_func(self, *args, **kwargs)
                if func_name in self.listeners:
                    for listener in self.listeners[func_name]:
                        listener()

            setattr(cls, name, patched_func)
            func = getattr(cls, name)
        if func in self.listeners:
            self.listeners[func_name].append(callback)
        else:
            self.listeners[func_name] = [callback]

    def remove_listener(self, func_name, callback):
        if func_name in self.listeners and callback in self.listeners[func_name]:
            self.listeners[func_name].remove(callback)
        return

    def get_behaviour(self, behaviour_name):
        if not isinstance(behaviour_name, str):
            behaviour_name = behaviour_name.__name__
        if behaviour_name in self.behaviours:
            return self.behaviours[behaviour_name]
        else:
            return None

    def register_behaviour(self, behaviour):
        behaviour.set_owner(self, None, None, None)
        self.behaviours[type(behaviour).__name__] = behaviour

    def register_behaviours(self, behaviours):
        for name, behaviour in behaviours.items():
            self.register_behaviour(behaviour)

    def update(self, delta_time, keys, config, state):
        for name, behaviour in self.behaviours.items():
            behaviour.update(delta_time, keys, config)
        self.update_position(delta_time)
        if self.deletion_pending:
            self.clear()

    def update_position(self, delta_time):
        dx = delta_time * self.velocity.x
        dy = delta_time * self.velocity.y

        c = self.get_behaviour(Collide.Collide)
        if c is None:
            self.set_y(self.y + dy)
            self.set_x(self.x + dx)
        else:
            if dy < 0:
                colliding_top = c.check_top(abs(dy))
                if len(colliding_top) <= 0:
                    self.set_y(self.y + dy)
                else:
                    self.move_top_to(Collide.get_bottom_most_of(colliding_top).get_bottom())
                    self.velocity.y = 0
            else:
                colliding_bottom = c.check_bottom(abs(dy))
                if len(colliding_bottom) <= 0:
                    self.set_y(self.y + dy)
                else:
                    self.move_bottom_to(Collide.get_top_most_of(colliding_bottom).get_top())
                    self.velocity.y = 0
            if dx < 0:
                colliding_left = c.check_left(abs(dx))
                if len(colliding_left) <= 0:
                    self.set_x(self.x + dx)
                else:
                    self.move_left_to(Collide.get_right_most_of(colliding_left).get_right())
                    self.velocity.x = 0
            else:
                colliding_right = c.check_right(abs(dx))
                if len(colliding_right) <= 0:
                    self.set_x(self.x + dx)
                else:
                    self.move_right_to(Collide.get_left_most_of(colliding_right).get_left())
                    self.velocity.x = 0

    def move_top_to(self, y):
        self.set_y(y)

    def get_top(self):
        return self.y

    def move_vertical_center_to(self, y):
        self.set_y(y - self.height / 2)

    def get_vertical_center(self):
        return self.y + self.height / 2

    def move_bottom_to(self, y):
        self.set_y(y - self.height)

    def get_bottom(self):
        return self.y + self.height

    def move_left_to(self, x):
        self.set_x(x)

    def get_left(self):
        return self.x

    def move_horizontal_center_to(self, x):
        self.set_x(x - self.width / 2)

    def get_horizontal_center(self):
        return self.x + self.width / 2

    def move_right_to(self, x):
        self.set_x(x - self.width)

    def get_right(self):
        return self.x + self.width

    def clear(self):
        self.deletion_pending = True
        c = self.get_behaviour(Collide.Collide)
        if c is not None:
            c.clear()

    def __del__(self):
        self.clear()
<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import print_function
from __future__ import division

import tensorflow as tf

def record_reader(filename_queue):
    reader = tf.TFRecordReader()
    _, serialized = reader.read(filename_queue)
    sequence_features = {
        "story": tf.FixedLenSequenceFeature([], dtype=tf.int64),
        "query": tf.FixedLenSequenceFeature([], dtype=tf.int64),
    }
    context_features = {
        "answer": tf.FixedLenFeature([], dtype=tf.int64),
    }
    context_features, sequence_features = tf.parse_single_sequence_example(serialized,
        context_features=context_features,
        sequence_features=sequence_features)
    answer = context_features['answer']
    story = sequence_features['story']
    query = sequence_features['query']

    max_sentence_length = 7
    max_story_length = 10
    max_query_length = 4

    story.set_shape([max_story_length * max_sentence_length])
    query.set_shape([max_query_length])
    return story, query, answer

def input_pipeline(filenames, batch_size, num_epochs=None, shuffle=False):
    filename_queue = tf.train.string_input_producer(filenames,
        num_epochs=num_epochs,
        shuffle=shuffle)
    min_after_dequeue = 9000
    capacity = min_after_dequeue + 10 * batch_size
    records = record_reader(filename_queue)
    batches = tf.train.shuffle_batch(records,
        batch_size=batch_size,
        min_after_dequeue=min_after_dequeue,
        capacity=capacity)
    return batches
<commit_msg>Shape sentences from data module<commit_after>from __future__ import absolute_import
from __future__ import print_function
from __future__ import division

import tensorflow as tf

def record_reader(filename_queue):
    reader = tf.TFRecordReader()
    _, serialized = reader.read(filename_queue)
    sequence_features = {
        "story": tf.FixedLenSequenceFeature([], dtype=tf.int64),
        "query": tf.FixedLenSequenceFeature([], dtype=tf.int64),
    }
    context_features = {
        "answer": tf.FixedLenFeature([], dtype=tf.int64),
    }
    context_features, sequence_features = tf.parse_single_sequence_example(serialized,
        context_features=context_features,
        sequence_features=sequence_features)
    answer = context_features['answer']
    story = sequence_features['story']
    query = sequence_features['query']

    # TODO: Turn both functions into class methods with public properties
    # to the story, query and answer as well as other vars.
    max_sentence_length = 7
    max_story_length = 10
    max_query_length = 4

    story.set_shape([max_story_length * max_sentence_length])
    query.set_shape([max_query_length])

    story = tf.reshape(story, [max_story_length, max_sentence_length])
    query = tf.reshape(query, [1, max_query_length])

    return story, query, answer

def input_pipeline(filenames, batch_size, num_epochs=None, shuffle=False):
    filename_queue = tf.train.string_input_producer(filenames,
        num_epochs=num_epochs,
        shuffle=shuffle)
    min_after_dequeue = 10000
    capacity = min_after_dequeue + 10 * batch_size
    records = record_reader(filename_queue)
    batches = tf.train.shuffle_batch(records,
        batch_size=batch_size,
        min_after_dequeue=min_after_dequeue,
        capacity=capacity)
    return batches
<|endoftext|>


<commit_before>"""Generate a build.ninja file to make up for the lack of symlink support in
Chrome when working with an unpackaged app.

Re-build any time files are added/removed from the base implementation.

"""
import os
import sys
sys.path.append('..')

import ninja_syntax

SYMLINKS = ['index.html', 'assets']
ZIPFILE_EXTRAS = ['manifest.json', 'background.js', 'impl.js']
ZIPFILE_TARGET = os.path.join('../oplop-chrome_app.zip')

def find_files(paths):
    """Find all files, starting at '..', as specified by 'paths'.

    If something in 'paths' is a directory, then recursively find all files in
    that directory.
    """
    rel_paths = []
    for path in paths:
        root_path = os.path.join('..', path)
        if not os.path.exists(root_path):
            raise ValueError(root_path + ' does not exist')
        elif os.path.isfile(root_path):
            rel_paths.append(path)
        else:
            for dirpath, dirnames, filenames in os.walk(root_path,
                    followlinks=True):
                for filename in filenames:
                    if filename.startswith('.'):
                        continue
                    path = os.path.join(dirpath, filename)[3:]  # Skip ../
                    rel_paths.append(path)
    return rel_paths

def check_gitignore(*check_for):
    """Check that .gitignore has rules for ignoring the files specified."""
    repo_root = os.path.abspath('../..')
    cwd = os.getcwd()[len(repo_root)+len(os.sep):]
    with open('../../.gitignore') as file:
        gitignore = file.read()
    for path in check_for:
        rel_path = os.path.normpath('/'.join([cwd, path]))
        if rel_path not in gitignore:
            raise ValueError(rel_path + ' not in .gitignore')

def see_what_is_missing(have_so_far):
    """Alert the user to what files are not being included in the zip file."""
    for dirpath, dirnames, filenames in os.walk('.'):
        for filename in filenames:
            path = os.path.join(dirpath, filename)[2:]
            if path not in all_files:
                print('Ignoring', path)

if __name__ == '__main__':
    to_ignore = [ZIPFILE_TARGET, 'build.ninja'] + SYMLINKS
    check_gitignore(*to_ignore)
    rel_paths = find_files(SYMLINKS)
    all_files = list(sorted(ZIPFILE_EXTRAS[:]+rel_paths))
    see_what_is_missing(all_files)

    with open('build.ninja', 'w') as file:
        ninja = ninja_syntax.Writer(file)

        ninja.rule('make_zipfile', 'zip $out $in')
        ninja.rule('sync_versions', 'python sync_versions.py $in $out')
        ninja.newline()
        ninja.newline()

        ninja.comment("Keep version numbers in sync")
        ninja.build('manifest.mobile.json', 'sync_versions', 'manifest.json',
                    implicit=['sync_versions.py', 'make_ninja.py'])
        ninja.newline()

        ninja.build(ZIPFILE_TARGET, 'make_zipfile', all_files,
                    implicit=['make_ninja.py'])
<commit_msg>Don't assume symlinks are ignored by .gitignore<commit_after>"""Generate a build.ninja file to make up for the lack of symlink support in
Chrome when working with an unpackaged app.

Re-build any time files are added/removed from the base implementation.

"""
import os
import sys
sys.path.append('..')

import ninja_syntax

SYMLINKS = ['index.html', 'assets']
ZIPFILE_EXTRAS = ['manifest.json', 'background.js', 'impl.js']
ZIPFILE_TARGET = os.path.join('../oplop-chrome_app.zip')

def find_files(paths):
    """Find all files, starting at '..', as specified by 'paths'.

    If something in 'paths' is a directory, then recursively find all files in
    that directory.
    """
    rel_paths = []
    for path in paths:
        root_path = os.path.join('..', path)
        if not os.path.exists(root_path):
            raise ValueError(root_path + ' does not exist')
        elif os.path.isfile(root_path):
            rel_paths.append(path)
        else:
            for dirpath, dirnames, filenames in os.walk(root_path,
                    followlinks=True):
                for filename in filenames:
                    if filename.startswith('.'):
                        continue
                    path = os.path.join(dirpath, filename)[3:]  # Skip ../
                    rel_paths.append(path)
    return rel_paths

def check_gitignore(*check_for):
    """Check that .gitignore has rules for ignoring the files specified."""
    repo_root = os.path.abspath('../..')
    cwd = os.getcwd()[len(repo_root)+len(os.sep):]
    with open('../../.gitignore') as file:
        gitignore = file.read()
    for path in check_for:
        rel_path = os.path.normpath('/'.join([cwd, path]))
        if rel_path not in gitignore:
            raise ValueError(rel_path + ' not in .gitignore')

def see_what_is_missing(have_so_far):
    """Alert the user to what files are not being included in the zip file."""
    for dirpath, dirnames, filenames in os.walk('.'):
        for filename in filenames:
            path = os.path.join(dirpath, filename)[2:]
            if path not in all_files:
                print('Ignoring', path)

if __name__ == '__main__':
    to_ignore = [ZIPFILE_TARGET, 'build.ninja']
    check_gitignore(*to_ignore)
    rel_paths = find_files(SYMLINKS)
    all_files = list(sorted(ZIPFILE_EXTRAS[:]+rel_paths))
    see_what_is_missing(all_files)

    with open('build.ninja', 'w') as file:
        ninja = ninja_syntax.Writer(file)

        ninja.rule('make_zipfile', 'zip $out $in')
        ninja.rule('sync_versions', 'python sync_versions.py $in $out')
        ninja.newline()
        ninja.newline()

        ninja.comment("Keep version numbers in sync")
        ninja.build('manifest.mobile.json', 'sync_versions', 'manifest.json',
                    implicit=['sync_versions.py', 'make_ninja.py'])
        ninja.newline()

        ninja.build(ZIPFILE_TARGET, 'make_zipfile', all_files,
                    implicit=['make_ninja.py'])
<|endoftext|>


<commit_before>from __future__ import unicode_literals

from urwid import (Columns, ListBox, Divider)
from ubuntui.utils import Color

class Table:
    def __init__(self):
        self._rows = []
        self._row_id = []

    def addHeadings(self, headings):
        """ Takes list of headings and converts them to column header
        with appropriate color

        Params:
        headings: List of column text headings
        """
        cols = []
        for h in headings:
            h = Color.column_header(h)
            cols.append(h)
        self.addRow(Columns(cols))

    def addColumns(self, row_id, columns):
        """ Convert list of widgets to Columns and add to a table row

        Arguments:
        row_id: unique id of new row
        columns: list of columns
        """
        if row_id not in self._row_id:
            self._row_id.append(row_id)
            self.addRow(Columns(columns))

    def addRow(self, item):
        """ Appends widget to Pile
        """
        self._rows.append(item)
        self._rows.append(
            Divider("\N{BOX DRAWINGS LIGHT HORIZONTAL}"))

    def render(self):
        return ListBox(self._rows)
<commit_msg>add force row<commit_after>from __future__ import unicode_literals

from urwid import (Columns, ListBox, Divider)
from ubuntui.utils import Color

class Table:
    def __init__(self):
        self._rows = []
        self._row_id = []

    def addHeadings(self, headings):
        """ Takes list of headings and converts them to column header
        with appropriate color

        Params:
        headings: List of column text headings
        """
        cols = []
        for h in headings:
            h = Color.column_header(h)
            cols.append(h)
        self.addRow(Columns(cols))

    def addColumns(self, row_id, columns, force=False):
        """ Convert list of widgets to Columns and add to a table row

        Arguments:
        row_id: unique id of new row
        columns: list of columns
        force: force additional row regardless of existing row_id
        """
        if row_id not in self._row_id or force:
            if row_id not in self._row_id:
                self._row_id.append(row_id)
            self.addRow(Columns(columns))

    def addRow(self, item):
        """ Appends widget to Pile
        """
        self._rows.append(item)
        self._rows.append(
            Divider("\N{BOX DRAWINGS LIGHT HORIZONTAL}"))

    def render(self):
        return ListBox(self._rows)
<|endoftext|>