<commit_before>#!/usr/bin/env python
#
# dials.command_line.check_indexing_symmetry.py
#
#  Copyright (C) 2015 Diamond Light Source
#
#  Author: Graeme Winter
#
#  This code is distributed under the BSD license, a copy of which is
#  included in the root directory of this package.

from __future__ import division

import libtbx.load_env
import logging
logger = logging.getLogger(libtbx.env.dispatcher_name)
from libtbx.phil import command_line
import iotbx.phil
from cctbx import sgtbx
from dials.util.options import OptionParser
from dials.util.options import flatten_reflections, flatten_experiments

help_message = '''

This program can be used to analyse the correlation coefficients between
reflections related by the symmetry operators belonging to the space group of
the input experiment.json file. It can also check for misindexing of
the diffraction pattern, possibly as a result of an incorrect beam centre.

Examples::

  dials.check_indexing_symmetry experiment.json indexed.pickle \\
    grid=1 symop_threshold=0.7

  dials.check_indexing_symmetry experiment.json indexed.pickle \\
    grid_l=3 symop_threshold=0.7

'''

phil_scope = iotbx.phil.parse("""
d_min = 0
  .type = float
  .help = "High resolution limit to use for analysis"
d_max = 0
  .type = float
  .help = "Low resolution limit to use for analysis"
symop_threshold = 0
  .type = float
  .help = "Threshold above which we consider a symmetry operator true."
grid = 0
  .type = int
  .help = "Search scope for testing misindexing on h, k, l."
grid_h = 0
  .type = int
  .help = "Search scope for testing misindexing on h."
grid_k = 0
  .type = int
  .help = "Search scope for testing misindexing on k."
grid_l = 0
  .type = int
  .help = "Search scope for testing misindexing on l."
asu = False
  .type = bool
  .help = "Perform search comparing within ASU (assumes input symm)"
normalise = False
  .type = bool
  .help = "Normalise intensities before calculating correlation coefficients."
normalise_bins = 0
  .type = int
  .help = "Number of resolution bins for normalisation"
output {
  log = dials.check_indexing_symmetry.log
    .type = str
  debug_log = dials.check_indexing_symmetry.debug.log
    .type = str
}
""", process_includes=True)

def dump_text(filename, set0, set1):
  i0 = set0.as_double()
  i1 = set1.as_double()
  with open(filename, 'w') as fout:
    for _0, _1 in zip(i0, i1):
      assert(_0[0] == _1[0])
      fout.write('%f %f\n' % (_0[1], _1[1]))

def get_symop_correlation_coefficients(miller_array, use_binning=False):
  from copy import deepcopy
  from scitbx.array_family import flex
  from cctbx import miller
  corr_coeffs = flex.double()
  n_refs = flex.int()
  space_group = miller_array.space_group()
  for smx in space_group.smx():
    reindexed_array = miller_array.change_basis(sgtbx.change_of_basis_op(smx))
    intensity, intensity_rdx = reindexed_array.common_sets(miller_array)
    if use_binning:
      intensity.use_binning_of(miller_array)
      intensity_rdx.use_binning_of(miller_array)
      cc = intensity.correlation(
        intensity_rdx, use_binning=use_binning)
      corr_coeffs.append(
        flex.mean_weighted(
          flex.double(i for i in cc.data if i is not None),
          flex.double(
            j for i, j in zip(cc.data, cc.binner.counts()) if i is not None)))
    else:
      corr_coeffs.append(intensity.correlation(
        intensity_rdx, use_binning=use_binning).coefficient())
    n_refs.append(intensity.size())
  return corr_coeffs, n_refs

def normalise_intensities(miller_array, n_bins=10):
  miller_array.setup_binner(n_bins=n_bins)
  nomalisations = miller_array.amplitude_quasi_normalisations()
  miller_array = miller_array.customized_copy(
    data=miller_array.data()/nomalisations.data())
  return miller_array

def test_crystal_pointgroup_symmetry(reflections, experiment, params):
  crystal = experiment.crystal

  from dials.array_family import flex

  # in case we pass in reflections from integration
  reflections = reflections.select(reflections['intensity.sum.variance'] > 0)
  reflections = reflections.select(reflections['intensity.sum.value'] > 0)
  original_miller_indices = reflections['miller_index']

  space_group = crystal.get_space_group()
  unit_cell = crystal.get_unit_cell()
  from cctbx.crystal import symmetry as crystal_symmetry
  cs = crystal_symmetry(unit_cell, space_group.type().lookup_symbol())

  from cctbx.miller import set as miller_set
  ms = miller_set(cs, original_miller_indices)
  ms = ms.array(reflections['intensity.sum.value'] /
                flex.sqrt(reflections['intensity.sum.variance']))

  if params.d_min or params.d_max:
    d_spacings = ms.d_spacings().data()
    sel = (d_spacings >= params.d_min) & (d_spacings <= params.d_max)
    ms = ms.select(sel)
    reflections = reflections.select(sel)

  if params.normalise:
    if params.normalise_bins:
      ms = normalise_intensities(ms, n_bins=params.normalise_bins)
    else:
      ms = normalise_intensities(ms)

  logger.info('Check symmetry operations on %d reflections:' % ms.size())
  logger.info('')
  logger.info('%20s %6s %5s' % ('Symop', 'Nref', 'CC'))

  true_symops = []

  ccs, n_refs = get_symop_correlation_coefficients(ms)

  for smx, cc, n_ref in zip(space_group.smx(), ccs, n_refs):
    accept = ''
    if params.symop_threshold:
      if cc > params.symop_threshold:
        true_symops.append(smx)
        accept = '***'
    logger.info('%20s %6d %.3f %s' % (smx, n_ref, cc, accept))

  if params.symop_threshold:
    from cctbx.sgtbx import space_group as sgtbx_space_group
    sg = sgtbx_space_group()
    for symop in true_symops:
      sg = sg.expand_smx(symop)
    for ltr in space_group.ltr():
      sg = sg.expand_ltr(ltr)
    sg_symbols = sg.match_tabulated_settings()
    logger.info('')
    logger.info('Derived point group from symmetry operations: %s' % \
      sg_symbols.hermann_mauguin())
    logger.info('')

  return

def offset_miller_indices(miller_indices, offset):
  from dials.array_family import flex
  return flex.miller_index(
    *[mi.iround() for mi in (miller_indices.as_vec3_double() + offset).parts()])

def get_indexing_offset_correlation_coefficients(
    reflections, crystal, grid, d_min=None, d_max=None,
    map_to_asu=False, grid_h=0, grid_k=0, grid_l=0):

  from copy import deepcopy
  from dials.array_family import flex

  space_group = crystal.get_space_group()
  unit_cell = crystal.get_unit_cell()
  from cctbx.crystal import symmetry as crystal_symmetry
  cs = crystal_symmetry(unit_cell, space_group.type().lookup_symbol())

  from cctbx.miller import set as miller_set

  data = reflections['intensity.sum.value'] / \
         flex.sqrt(reflections['intensity.sum.variance'])

  ccs = flex.double()
  offsets = flex.vec3_int()
  nref = flex.size_t()

  original_miller_indices = reflections['miller_index']
  ms = miller_set(cs, original_miller_indices)
  ms = ms.array(data)

  if d_min is not None or d_max is not None:
    ms = ms.resolution_filter(d_min=d_min, d_max=d_max)

  if map_to_asu:
    ms = ms.map_to_asu()

  gh = gk = gl = grid
  if grid_h: gh = grid_h
  if grid_k: gk = grid_k
  if grid_l: gl = grid_l

  # essentially just inversion operation - this *should* have good CC
  cb_op = sgtbx.change_of_basis_op('-x,-y,-z')

  for h in range(-gh, gh + 1):
    for k in range(-gk, gk + 1):
      for l in range(-gl, gl + 1):
        offset = (2*h, 2*k, 2*l)
        miller_indices = offset_miller_indices(ms.indices(), offset)
        reindexed_miller_indices = cb_op.apply(miller_indices)
        rms = miller_set(cs, reindexed_miller_indices).array(data)
        if map_to_asu:
          rms = rms.map_to_asu()
        intensity, intensity_rdx = rms.common_sets(ms)
        cc = intensity.correlation(intensity_rdx).coefficient()
        ccs.append(cc)
        offsets.append((h, k, l))
        nref.append(intensity.size())

  return offsets, ccs, nref

def test_P1_crystal_indexing(reflections, experiment, params):
  if not (params.grid or params.grid_h or params.grid_k or params.grid_l):
    return

  logger.info('Checking HKL origin:')
  logger.info('')
  logger.info('dH dK dL %6s %5s' % ('Nref', 'CC'))

  offsets, ccs, nref = get_indexing_offset_correlation_coefficients(
    reflections, experiment.crystal,
    grid=params.grid,
    d_min=params.d_min, d_max=params.d_max, map_to_asu=params.asu,
    grid_h=params.grid_h, grid_k=params.grid_k, grid_l=params.grid_l)

  for (h, k, l), cc, n in zip(offsets, ccs, nref):
    if cc > params.symop_threshold or (h == k == l == 0):
      logger.info('%2d %2d %2d %6d %.3f' % (h, k, l, n, cc))

  logger.info('')

  return

def run(args):
  import libtbx.load_env
  from dials.array_family import flex
  from dials.util import log
  from dials.util.version import dials_version

  usage = "%s [options] experiment.json indexed.pickle" % \
    libtbx.env.dispatcher_name

  parser = OptionParser(
    usage=usage,
    phil=phil_scope,
    read_reflections=True,
    read_experiments=True,
    check_format=False,
    epilog=help_message)

  params, options = parser.parse_args(show_diff_phil=True)

  # Configure the logging
  log.config(info=params.output.log, debug=params.output.debug_log)
  logger.info(dials_version())

  reflections = flatten_reflections(params.input.reflections)
  experiments = flatten_experiments(params.input.experiments)
  if len(reflections) == 0 or len(experiments) == 0:
    parser.print_help()
    return
  assert(len(reflections) == 1)
  assert(len(experiments) == 1)
  experiment = experiments[0]
  reflections = reflections[0]

  # remove reflections with 0, 0, 0 index
  zero = (reflections['miller_index'] == (0, 0, 0))
  logger.info('Removing %d unindexed reflections' % zero.count(True))
  reflections = reflections.select(~zero)

  h, k, l = reflections['miller_index'].as_vec3_double().parts()

  h = h.iround()
  k = k.iround()
  l = l.iround()

  logger.info('Range on h: %d to %d' % (flex.min(h), flex.max(h)))
  logger.info('Range on k: %d to %d' % (flex.min(k), flex.max(k)))
  logger.info('Range on l: %d to %d' % (flex.min(l), flex.max(l)))

  test_P1_crystal_indexing(reflections, experiment, params)
  test_crystal_pointgroup_symmetry(reflections, experiment, params)

if __name__ == '__main__':
  import sys
  from libtbx.utils import show_times_at_exit
  show_times_at_exit()
  run(sys.argv[1:])
<commit_msg>Clean up code; added option for reference data<commit_after>#!/usr/bin/env python
#
# dials.command_line.check_indexing_symmetry.py
#
#  Copyright (C) 2015 Diamond Light Source
#
#  Author: Graeme Winter
#
#  This code is distributed under the BSD license, a copy of which is
#  included in the root directory of this package.

from __future__ import division

import libtbx.load_env
import logging
logger = logging.getLogger(libtbx.env.dispatcher_name)
from libtbx.phil import command_line
import iotbx.phil
from cctbx import sgtbx
from dials.util.options import OptionParser
from dials.util.options import flatten_reflections, flatten_experiments

help_message = '''

This program can be used to analyse the correlation coefficients between
reflections related by the symmetry operators belonging to the space group of
the input experiment.json file. It can also check for misindexing of
the diffraction pattern, possibly as a result of an incorrect beam centre.

Examples::

  dials.check_indexing_symmetry experiment.json indexed.pickle \\
    grid=1 symop_threshold=0.7

  dials.check_indexing_symmetry experiment.json indexed.pickle \\
    grid_l=3 symop_threshold=0.7

'''

phil_scope = iotbx.phil.parse("""
d_min = 0
  .type = float
  .help = "High resolution limit to use for analysis"
d_max = 0
  .type = float
  .help = "Low resolution limit to use for analysis"
symop_threshold = 0
  .type = float
  .help = "Threshold above which we consider a symmetry operator true."
grid = 0
  .type = int
  .help = "Search scope for testing misindexing on h, k, l."
grid_h = 0
  .type = int
  .help = "Search scope for testing misindexing on h."
grid_k = 0
  .type = int
  .help = "Search scope for testing misindexing on k."
grid_l = 0
  .type = int
  .help = "Search scope for testing misindexing on l."
asu = False
  .type = bool
  .help = "Perform search comparing within ASU (assumes input symm)"
normalise = False
  .type = bool
  .help = "Normalise intensities before calculating correlation coefficients."
normalise_bins = 0
  .type = int
  .help = "Number of resolution bins for normalisation"
reference = None
  .type = path
  .help = "Correctly indexed reference set for comparison"
output {
  log = dials.check_indexing_symmetry.log
    .type = str
  debug_log = dials.check_indexing_symmetry.debug.log
    .type = str
}
""", process_includes=True)

def dump_text(filename, set0, set1):
  i0 = set0.as_double()
  i1 = set1.as_double()
  with open(filename, 'w') as fout:
    for _0, _1 in zip(i0, i1):
      assert(_0[0] == _1[0])
      fout.write('%f %f\n' % (_0[1], _1[1]))

def get_symop_correlation_coefficients(miller_array, use_binning=False):
  from copy import deepcopy
  from scitbx.array_family import flex
  from cctbx import miller
  corr_coeffs = flex.double()
  n_refs = flex.int()
  space_group = miller_array.space_group()
  for smx in space_group.smx():
    reindexed_array = miller_array.change_basis(sgtbx.change_of_basis_op(smx))
    intensity, intensity_rdx = reindexed_array.common_sets(miller_array)
    if use_binning:
      intensity.use_binning_of(miller_array)
      intensity_rdx.use_binning_of(miller_array)
      cc = intensity.correlation(
        intensity_rdx, use_binning=use_binning)
      corr_coeffs.append(
        flex.mean_weighted(
          flex.double(i for i in cc.data if i is not None),
          flex.double(
            j for i, j in zip(cc.data, cc.binner.counts()) if i is not None)))
    else:
      corr_coeffs.append(intensity.correlation(
        intensity_rdx, use_binning=use_binning).coefficient())
    n_refs.append(intensity.size())
  return corr_coeffs, n_refs

def normalise_intensities(miller_array, n_bins=10):
  miller_array.setup_binner(n_bins=n_bins)
  nomalisations = miller_array.amplitude_quasi_normalisations()
  miller_array = miller_array.customized_copy(
    data=miller_array.data()/nomalisations.data())
  return miller_array

def test_crystal_pointgroup_symmetry(reflections, experiment, params):
  crystal = experiment.crystal

  from dials.array_family import flex

  # in case we pass in reflections from integration
  reflections = reflections.select(reflections['intensity.sum.variance'] > 0)
  reflections = reflections.select(reflections['intensity.sum.value'] > 0)
  original_miller_indices = reflections['miller_index']

  space_group = crystal.get_space_group()
  unit_cell = crystal.get_unit_cell()
  from cctbx.crystal import symmetry as crystal_symmetry
  cs = crystal_symmetry(unit_cell, space_group.type().lookup_symbol())

  from cctbx.miller import set as miller_set
  ms = miller_set(cs, original_miller_indices)
  ms = ms.array(reflections['intensity.sum.value'] /
                flex.sqrt(reflections['intensity.sum.variance']))

  if params.d_min or params.d_max:
    d_spacings = ms.d_spacings().data()
    sel = (d_spacings >= params.d_min) & (d_spacings <= params.d_max)
    ms = ms.select(sel)
    reflections = reflections.select(sel)

  if params.normalise:
    if params.normalise_bins:
      ms = normalise_intensities(ms, n_bins=params.normalise_bins)
    else:
      ms = normalise_intensities(ms)

  logger.info('Check symmetry operations on %d reflections:' % ms.size())
  logger.info('')
  logger.info('%20s %6s %5s' % ('Symop', 'Nref', 'CC'))

  true_symops = []

  ccs, n_refs = get_symop_correlation_coefficients(ms)

  for smx, cc, n_ref in zip(space_group.smx(), ccs, n_refs):
    accept = ''
    if params.symop_threshold:
      if cc > params.symop_threshold:
        true_symops.append(smx)
        accept = '***'
    logger.info('%20s %6d %.3f %s' % (smx, n_ref, cc, accept))

  if params.symop_threshold:
    from cctbx.sgtbx import space_group as sgtbx_space_group
    sg = sgtbx_space_group()
    for symop in true_symops:
      sg = sg.expand_smx(symop)
    for ltr in space_group.ltr():
      sg = sg.expand_ltr(ltr)
    sg_symbols = sg.match_tabulated_settings()
    logger.info('')
    logger.info('Derived point group from symmetry operations: %s' % \
      sg_symbols.hermann_mauguin())
    logger.info('')

  return

def offset_miller_indices(miller_indices, offset):
  from dials.array_family import flex
  return flex.miller_index(
    *[mi.iround() for mi in (miller_indices.as_vec3_double() + offset).parts()])

def get_indexing_offset_correlation_coefficients(
    reflections, crystal, grid, d_min=None, d_max=None,
    map_to_asu=False, grid_h=0, grid_k=0, grid_l=0, reference=None):

  from copy import deepcopy
  from dials.array_family import flex

  space_group = crystal.get_space_group()
  unit_cell = crystal.get_unit_cell()
  from cctbx.crystal import symmetry as crystal_symmetry
  cs = crystal_symmetry(unit_cell, space_group.type().lookup_symbol())

  from cctbx.miller import set as miller_set

  data = reflections['intensity.sum.value'] / \
         flex.sqrt(reflections['intensity.sum.variance'])

  if reference:
    reference = reference.select(reference['intensity.sum.variance'] > 0)
    reference_data = reference['intensity.sum.value'] / \
         flex.sqrt(reference['intensity.sum.variance'])
    reference_ms = miller_set(cs, reference['miller_index']).array(
      reference_data)
  else:
    reference_ms = None

  ccs = flex.double()
  offsets = flex.vec3_int()
  nref = flex.size_t()

  original_miller_indices = reflections['miller_index']
  ms = miller_set(cs, original_miller_indices).array(data)

  if d_min is not None or d_max is not None:
    ms = ms.resolution_filter(d_min=d_min, d_max=d_max)

  if map_to_asu:
    ms = ms.map_to_asu()

  gh = gk = gl = grid
  if grid_h: gh = grid_h
  if grid_k: gk = grid_k
  if grid_l: gl = grid_l

  # essentially just inversion operation - this *should* have good CC - unless
  # we are working on a reference set where we don't reindex
  if reference:
    cb_op = sgtbx.change_of_basis_op('x,y,z')
  else:
    cb_op = sgtbx.change_of_basis_op('-x,-y,-z')

  for h in range(-gh, gh + 1):
    for k in range(-gk, gk + 1):
      for l in range(-gl, gl + 1):
        miller_indices = offset_miller_indices(ms.indices(), (h, k, l))
        reindexed_miller_indices = cb_op.apply(miller_indices)
        rms = miller_set(cs, reindexed_miller_indices).array(data)
        if reference_ms:
          _ms = reference_ms
        else:
          _ms = miller_set(cs, miller_indices).array(data)
        if map_to_asu:
          rms = rms.map_to_asu()
          _ms = _ms.map_to_asu()
        intensity, intensity_rdx = rms.common_sets(_ms)
        cc = intensity.correlation(intensity_rdx).coefficient()
        ccs.append(cc)
        offsets.append((h, k, l))
        nref.append(intensity.size())

  return offsets, ccs, nref

def test_P1_crystal_indexing(reflections, experiment, params):
  if not (params.grid or params.grid_h or params.grid_k or params.grid_l):
    return

  logger.info('Checking HKL origin:')
  logger.info('')
  logger.info('dH dK dL %6s %5s' % ('Nref', 'CC'))

  if params.reference:
    from dials.array_family import flex # implicit dependency
    import cPickle as pickle
    reference = pickle.load(open(params.reference))
  else:
    reference = None

  offsets, ccs, nref = get_indexing_offset_correlation_coefficients(
    reflections, experiment.crystal,
    grid=params.grid,
    d_min=params.d_min, d_max=params.d_max, map_to_asu=params.asu,
    grid_h=params.grid_h, grid_k=params.grid_k, grid_l=params.grid_l,
    reference=reference)

  for (h, k, l), cc, n in zip(offsets, ccs, nref):
    if cc > params.symop_threshold or (h == k == l == 0):
      logger.info('%2d %2d %2d %6d %.3f' % (h, k, l, n, cc))

  logger.info('')

  return

def run(args):
  import libtbx.load_env
  from dials.array_family import flex
  from dials.util import log
  from dials.util.version import dials_version

  usage = "%s [options] experiment.json indexed.pickle" % \
    libtbx.env.dispatcher_name

  parser = OptionParser(
    usage=usage,
    phil=phil_scope,
    read_reflections=True,
    read_experiments=True,
    check_format=False,
    epilog=help_message)

  params, options = parser.parse_args(show_diff_phil=True)

  # Configure the logging
  log.config(info=params.output.log, debug=params.output.debug_log)
  logger.info(dials_version())

  reflections = flatten_reflections(params.input.reflections)
  experiments = flatten_experiments(params.input.experiments)
  if len(reflections) == 0 or len(experiments) == 0:
    parser.print_help()
    return
  assert(len(reflections) == 1)
  assert(len(experiments) == 1)
  experiment = experiments[0]
  reflections = reflections[0]

  # remove reflections with 0, 0, 0 index
  zero = (reflections['miller_index'] == (0, 0, 0))
  logger.info('Removing %d unindexed reflections' % zero.count(True))
  reflections = reflections.select(~zero)

  h, k, l = reflections['miller_index'].as_vec3_double().parts()

  h = h.iround()
  k = k.iround()
  l = l.iround()

  logger.info('Range on h: %d to %d' % (flex.min(h), flex.max(h)))
  logger.info('Range on k: %d to %d' % (flex.min(k), flex.max(k)))
  logger.info('Range on l: %d to %d' % (flex.min(l), flex.max(l)))

  test_P1_crystal_indexing(reflections, experiment, params)
  test_crystal_pointgroup_symmetry(reflections, experiment, params)

if __name__ == '__main__':
  import sys
  from libtbx.utils import show_times_at_exit
  show_times_at_exit()
  run(sys.argv[1:])
<|endoftext|>


<commit_before>import os
import pandas as pd
import pandas.util.testing as tm
from castra import Castra

A = pd.DataFrame({'x': [1, 2],
                  'y': [1., 2.]},
                 columns=['x', 'y'],
                 index=[1, 2])

B = pd.DataFrame({'x': [10, 20],
                  'y': [10., 20.]},
                 columns=['x', 'y'],
                 index=[10, 20])

def test_Castra():
    c = Castra(columns=A.columns,
               dtypes=A.dtypes,
               index_dtype=A.index.dtype)
    c.extend(A)
    c.extend(B)

    assert c.columns == ['x', 'y']

    tm.assert_frame_equal(c[0:100], pd.concat([A, B]))
    tm.assert_frame_equal(c[:5], A)
    tm.assert_frame_equal(c[5:], B)

    tm.assert_frame_equal(c[2:5], A[1:])
    tm.assert_frame_equal(c[2:15], pd.concat([A[1:], B[:1]]))

def test_del():
    c = Castra(columns=A.columns, dtypes=A.dtypes, index_dtype=A.index.dtype)
    assert os.path.exists(c.path)
    c.__del__()
    assert not os.path.exists(c.path)

def test_context_manager():
    with Castra(columns=A.columns, dtypes=A.dtypes, index_dtype=A.index.dtype) as c:
        assert os.path.exists(c.path)
    assert not os.path.exists(c.path)
<commit_msg>test for loading data<commit_after>import os
import pandas as pd
import pandas.util.testing as tm
from castra import Castra
import tempfile

A = pd.DataFrame({'x': [1, 2],
                  'y': [1., 2.]},
                 columns=['x', 'y'],
                 index=[1, 2])

B = pd.DataFrame({'x': [10, 20],
                  'y': [10., 20.]},
                 columns=['x', 'y'],
                 index=[10, 20])

def test_Castra():
    c = Castra(columns=A.columns,
               dtypes=A.dtypes,
               index_dtype=A.index.dtype)
    c.extend(A)
    c.extend(B)

    assert c.columns == ['x', 'y']

    tm.assert_frame_equal(c[0:100], pd.concat([A, B]))
    tm.assert_frame_equal(c[:5], A)
    tm.assert_frame_equal(c[5:], B)

    tm.assert_frame_equal(c[2:5], A[1:])
    tm.assert_frame_equal(c[2:15], pd.concat([A[1:], B[:1]]))

def test_del():
    c = Castra(columns=A.columns, dtypes=A.dtypes, index_dtype=A.index.dtype)
    assert os.path.exists(c.path)
    c.__del__()
    assert not os.path.exists(c.path)

def test_context_manager():
    with Castra(columns=A.columns, dtypes=A.dtypes, index_dtype=A.index.dtype) as c:
        assert os.path.exists(c.path)
    assert not os.path.exists(c.path)

def test_load_Castra():
    path = tempfile.mkdtemp(prefix='castra-')
    c = Castra(path=path, columns=A.columns, dtypes=A.dtypes, index_dtype=A.index.dtype)
    c.extend(A)
    c.extend(B)

    loaded = Castra(path=path)
    tm.assert_frame_equal(c[:], loaded[:])
<|endoftext|>


<commit_before>import nipype.pipeline.engine as pe
import nipype.interfaces.io as nio
import nipype.interfaces.utility as util
import os
from nipype.interfaces.freesurfer.preprocess import ReconAll

subject_list = ['s1', 's3']
data_dir = os.path.abspath('data')
subjects_dir = os.path.abspath('amri_freesurfer_tutorial/subjects_dir')

wf = pe.Workflow(name="l1workflow")
wf.base_dir = os.path.abspath('amri_freesurfer_tutorial/workdir')

datasource = pe.MapNode(interface=nio.DataGrabber(infields=['subject_id'],
                                                  outfields=['struct']),
                        name='datasource',
                        iterfield=['subject_id'])
datasource.inputs.base_directory = data_dir
datasource.inputs.template = '%s/%s.nii'
datasource.inputs.template_args = dict(struct=[['subject_id', 'struct']])
datasource.inputs.subject_id = subject_list

recon_all = pe.MapNode(interface=ReconAll(), name='recon_all',
                       iterfield=['subject_id', 'T1_files'])
recon_all.inputs.subject_id = subject_list
if not os.path.exists(subjects_dir):
    os.mkdir(subjects_dir)
recon_all.inputs.subjects_dir = subjects_dir

wf.connect(datasource, 'struct', recon_all, 'T1_files')

def MakeAverageSubject(subjects_dir, subjects_list, out_name):
    from nipype.interfaces.base import CommandLine
    mas = CommandLine(command='make_average_subject')
    mas.inputs.args = "--sdir %s --subjects %s --out %s"%(subjects_dir[0], " "%subjects_list, out_name)
    mas.run()
    return subjects_dir, out_name

average = pe.Node(interface=util.Function(input_names=['subjects_dir',
                                                       'subjects_list',
                                                       'out_name'],
                                          output_names=['subjects_dir',
                                                        'out_name'],
                                          function=MakeAverageSubject),
                  name="average")

average.inputs.out_name = "average"

wf.connect(recon_all, 'subjects_dir', average, 'subjects_dir')
wf.connect(recon_all, 'subject_id', average, 'subjects_list')

wf.run("MultiProc", plugin_args={'n_procs': 4})
<commit_msg>Fixed join issue.<commit_after>import nipype.pipeline.engine as pe
import nipype.interfaces.io as nio
import nipype.interfaces.utility as util
import os
from nipype.interfaces.freesurfer.preprocess import ReconAll

subject_list = ['s1', 's3']
data_dir = os.path.abspath('data')
subjects_dir = os.path.abspath('amri_freesurfer_tutorial/subjects_dir')

wf = pe.Workflow(name="l1workflow")
wf.base_dir = os.path.abspath('amri_freesurfer_tutorial/workdir')

datasource = pe.MapNode(interface=nio.DataGrabber(infields=['subject_id'],
                                                  outfields=['struct']),
                        name='datasource',
                        iterfield=['subject_id'])
datasource.inputs.base_directory = data_dir
datasource.inputs.template = '%s/%s.nii'
datasource.inputs.template_args = dict(struct=[['subject_id', 'struct']])
datasource.inputs.subject_id = subject_list

recon_all = pe.MapNode(interface=ReconAll(), name='recon_all',
                       iterfield=['subject_id', 'T1_files'])
recon_all.inputs.subject_id = subject_list
if not os.path.exists(subjects_dir):
    os.mkdir(subjects_dir)
recon_all.inputs.subjects_dir = subjects_dir

wf.connect(datasource, 'struct', recon_all, 'T1_files')

def MakeAverageSubject(subjects_dir, subjects_list, out_name):
    from nipype.interfaces.base import CommandLine
    mas = CommandLine(command='make_average_subject')
    mas.inputs.args = "--sdir %s --subjects %s --out %s"%(subjects_dir[0], " ".join(subjects_list), out_name)
    mas.run()
    return subjects_dir, out_name

average = pe.Node(interface=util.Function(input_names=['subjects_dir',
                                                       'subjects_list',
                                                       'out_name'],
                                          output_names=['subjects_dir',
                                                        'out_name'],
                                          function=MakeAverageSubject),
                  name="average")

average.inputs.out_name = "average"

wf.connect(recon_all, 'subjects_dir', average, 'subjects_dir')
wf.connect(recon_all, 'subject_id', average, 'subjects_list')

wf.run("MultiProc", plugin_args={'n_procs': 4})
<|endoftext|>


<commit_before>"""Tests related to the @value object."""
import pytest

from dependencies import Injector
from dependencies import value
from dependencies.exceptions import DependencyError
from helpers import CodeCollector

def test_define_value():
    """Evaluate @value decorated function during dependency injection process."""

    class Get:
        def __init__(self, result):
            self.result = result

    class Container(Injector):

        get = Get
        foo = 1
        bar = 2
        baz = 3

        @value
        def result(foo, bar, baz):
            return foo + bar + baz

    assert Container.get.result == 6

def test_keyword_arguments():
    """@value decorated function should support keyword arguments.

    If keyword argument is missing in the Injector subclass the
    default value should be used.

    """

    class Get:
        def __init__(self, result):
            self.result = result

    class Container(Injector):

        get = Get
        foo = 1
        bar = 2

        @value
        def result(foo, bar, baz=3):
            return foo + bar + baz

    assert Container.get.result == 6

def test_protect_against_classes():
    """Deny to apply @value to classes."""
    with pytest.raises(DependencyError) as exc_info:

        @value
        class Foo:
            pass

    assert str(exc_info.value) == "'value' decorator can not be used on classes"

deny_method = CodeCollector()

@deny_method.parametrize
def test_protect_against_self(code):
    """Deny to define a value with argument called `self`."""

    class Foo:
        pass

    @value
    def method(self, foo, bar):
        raise RuntimeError

    with pytest.raises(DependencyError) as exc_info:
        code(Foo, method)

    assert str(exc_info.value) == "'value' decorator can not be used on methods"

@deny_method
def _sUIvAUUeQIde(Foo, arg):
    class Container(Injector):
        foo = Foo
        method = arg

    Container.foo

@deny_method
def _nVlMKQghCDAQ(Foo, arg):
    Injector(foo=Foo, method=arg).foo

deny_direct_resolve = CodeCollector()

@deny_direct_resolve.parametrize
def test_direct_value_resolve(touch, code):
    """Attempt to resolve value directly should raise exception.

    Values are allowed to be used as dependencies for classes.

    """
    with pytest.raises(DependencyError) as exc_info:
        touch(code(), "a")
    expected = "'value' dependencies could only be used to instantiate classes"
    assert str(exc_info.value) == expected

@deny_direct_resolve
def _n3XOmcCoWc0W():
    class Container(Injector):
        @value
        def a():
            return 1

    return Container

@deny_direct_resolve
def _pZKCM0OCHMML():
    @value
    def a():
        return 1

    return Injector(a=a)
<commit_msg>test: @value object is aware of decorated functions #299<commit_after>"""Tests related to the @value object."""
from functools import lru_cache
from random import randint

import pytest

from dependencies import Injector
from dependencies import value
from dependencies.exceptions import DependencyError
from helpers import CodeCollector

def test_define_value():
    """Evaluate @value decorated function during dependency injection process."""

    class Get:
        def __init__(self, result):
            self.result = result

    class Container(Injector):

        get = Get
        foo = 1
        bar = 2
        baz = 3

        @value
        def result(foo, bar, baz):
            return foo + bar + baz

    assert Container.get.result == 6

def test_keyword_arguments():
    """@value decorated function should support keyword arguments.

    If keyword argument is missing in the Injector subclass the
    default value should be used.

    """

    class Get:
        def __init__(self, result):
            self.result = result

    class Container(Injector):

        get = Get
        foo = 1
        bar = 2

        @value
        def result(foo, bar, baz=3):
            return foo + bar + baz

    assert Container.get.result == 6

def test_protect_against_classes():
    """Deny to apply @value to classes."""
    with pytest.raises(DependencyError) as exc_info:

        @value
        class Foo:
            pass

    assert str(exc_info.value) == "'value' decorator can not be used on classes"

deny_method = CodeCollector()

@deny_method.parametrize
def test_protect_against_self(code):
    """Deny to define a value with argument called `self`."""

    class Foo:
        pass

    @value
    def method(self, foo, bar):
        raise RuntimeError

    with pytest.raises(DependencyError) as exc_info:
        code(Foo, method)

    assert str(exc_info.value) == "'value' decorator can not be used on methods"

@deny_method
def _sUIvAUUeQIde(Foo, arg):
    class Container(Injector):
        foo = Foo
        method = arg

    Container.foo

@deny_method
def _nVlMKQghCDAQ(Foo, arg):
    Injector(foo=Foo, method=arg).foo

deny_direct_resolve = CodeCollector()

@deny_direct_resolve.parametrize
def test_direct_value_resolve(touch, code):
    """Attempt to resolve value directly should raise exception.

    Values are allowed to be used as dependencies for classes.

    """
    with pytest.raises(DependencyError) as exc_info:
        touch(code(), "a")
    expected = "'value' dependencies could only be used to instantiate classes"
    assert str(exc_info.value) == expected

@deny_direct_resolve
def _n3XOmcCoWc0W():
    class Container(Injector):
        @value
        def a():
            return 1

    return Container

@deny_direct_resolve
def _pZKCM0OCHMML():
    @value
    def a():
        return 1

    return Injector(a=a)

decorated_functions = CodeCollector()

@decorated_functions.parametrize
def test_allow_decorated_functions(touch, code):
    """Decorators applied to functions should keep working."""

    class Foo:
        def __init__(self, bar):
            self.bar = bar

    Container = code(Foo)
    assert touch(Container, "foo").bar == touch(Container, "foo").bar

@decorated_functions
def _z4nIvE3QRpik(Foo):
    class Container(Injector):
        foo = Foo

        @value
        @lru_cache()
        def bar():
            return randint(0, 1000)

    return Container

@decorated_functions
def _q6PYNbsibDGG(Foo):
    @value
    @lru_cache()
    def bar():
        return randint(0, 1000)

    return Injector(foo=Foo, bar=bar)
<|endoftext|>


<commit_before>from unittest import TestCase

import sanitize

class TestSanitizeHTML(TestCase):
    def _html(self, html_source, expected_data, base_uri=None, add_nofollow=False):
        """
        :type html_source: str
        :type expected_data: str
        :type base_uri: str
        :type add_nofollow: bool
        """
        self.assertEqual(
            sanitize.HTML(
                htmlSource=html_source,
                baseuri=base_uri,
                addnofollow=add_nofollow
            ),
            expected_data
        )
<commit_msg>Add test_basics<commit_after>from unittest import TestCase

import sanitize

class TestSanitizeHTML(TestCase):
    def _html(self, html_source, expected_data, base_uri=None, add_nofollow=False):
        """
        :type html_source: str
        :type expected_data: str
        :type base_uri: str
        :type add_nofollow: bool
        """
        self.assertEqual(
            sanitize.HTML(
                htmlSource=html_source,
                baseuri=base_uri,
                addnofollow=add_nofollow
            ),
            expected_data
        )

    def test_basics(self):
        self._html("", "")
        self._html("hello", "hello")
<|endoftext|>


<commit_before># test_serialization.py

from __future__ import unicode_literals

import io
import random

import pytest

from concepts import Context, Definition
from concepts.lattices import Lattice

SERIALIZED = {
    'objects': (
        '1sg', '1pl', '2sg', '2pl', '3sg', '3pl',
    ),
    'properties': (
        '+1', '-1', '+2', '-2', '+3', '-3', '+sg', '+pl', '-sg', '-pl',
    ),
    'context': [
        (0, 3, 5, 6, 9),
        (0, 3, 5, 7, 8),
        (1, 2, 5, 6, 9),
        (1, 2, 5, 7, 8),
        (1, 3, 4, 6, 9),
        (1, 3, 4, 7, 8),
    ],
    'lattice': [
        ((), (0, 1, 2, 3, 4, 5, 6, 7, 8, 9), (1, 2, 3, 4, 5, 6), ()),
        ((0,), (0, 3, 5, 6, 9), (7, 8, 9), (0,)),
        ((1,), (0, 3, 5, 7, 8), (7, 10, 11), (0,)),
        ((2,), (1, 2, 5, 6, 9), (8, 12, 13), (0,)),
        ((3,), (1, 2, 5, 7, 8), (10, 12, 14), (0,)),
        ((4,), (1, 3, 4, 6, 9), (9, 13, 15), (0,)),
        ((5,), (1, 3, 4, 7, 8), (11, 14, 15), (0,)),
        ((0, 1), (0, 3, 5), (18, 19), (1, 2)),
        ((0, 2), (5, 6, 9), (16, 18), (1, 3)),
        ((0, 4), (3, 6, 9), (16, 19), (1, 5)),
        ((1, 3), (5, 7, 8), (17, 18), (2, 4)),
        ((1, 5), (3, 7, 8), (17, 19), (2, 6)),
        ((2, 3), (1, 2, 5), (18, 20), (3, 4)),
        ((2, 4), (1, 6, 9), (16, 20), (3, 5)),
        ((3, 5), (1, 7, 8), (17, 20), (4, 6)),
        ((4, 5), (1, 3, 4), (19, 20), (5, 6)),
        ((0, 2, 4), (6, 9), (21,), (8, 9, 13)),
        ((1, 3, 5), (7, 8), (21,), (10, 11, 14)),
        ((0, 1, 2, 3), (5,), (21,), (7, 8, 10, 12)),
        ((0, 1, 4, 5), (3,), (21,), (7, 9, 11, 15)),
        ((2, 3, 4, 5), (1,), (21,), (12, 13, 14, 15)),
        ((0, 1, 2, 3, 4, 5), (), (), (18, 19, 20, 16, 17)),
    ],
}

SERIALIZED_NOLATTICE = {'objects': SERIALIZED['objects'],
                        'properties': SERIALIZED['properties'],
                        'context': list(SERIALIZED['context'])}

@pytest.fixture(params=[SERIALIZED, SERIALIZED_NOLATTICE])
def d(request):
    return request.param

def test_todict(context, d):
    assert 'lattice' not in context.__dict__
    if 'lattice' not in d:
        assert context.todict() == context.todict(include_lattice=False) == d
        assert 'lattice' not in context.__dict__
    else:
        context = Context(context.objects, context.properties, context.bools)
        assert 'lattice' not in context.__dict__
        assert context.todict(include_lattice=True) == context.todict() == d
        assert 'lattice' in context.__dict__

@pytest.fixture
def d_invalid():
    return {'objects': SERIALIZED['objects'],
            'properties': SERIALIZED['properties'],
            'context': list(SERIALIZED['context']),
            'lattice': list(SERIALIZED['lattice'])}

@pytest.mark.parametrize('missing',
                         ['objects', 'properties', 'context', 'lattice'])
def test_fromdict_missing(d_invalid, missing):
    del d_invalid[missing]
    with pytest.raises(ValueError, match=r'missing .*%s' % missing):
        Context.fromdict(d_invalid, require_lattice=(missing == 'lattice'))

@pytest.mark.parametrize('nonstring', ['objects', 'properties'])
def test_fromdict_nonstring(d_invalid, nonstring):
    d_invalid[nonstring] = (42,) + d_invalid[nonstring][1:]
    with pytest.raises(ValueError, match=r'non-string %s' % nonstring):
        Context.fromdict(d_invalid)

@pytest.mark.parametrize('short', ['objects', 'context'])
def test_fromdict_mismatch(d_invalid, short):
    d_invalid[short] = d_invalid[short][1:]
    lens = (5, 6) if short == 'objects' else (6, 5)
    match = r'mismatch: %d objects with %d context' % lens
    with pytest.raises(ValueError, match=match):
        Context.fromdict(d_invalid)

def test_fromdict_context_duplicates(d_invalid):
    first = d_invalid['context'][0]
    d_invalid['context'][0] = (first[0], first[0]) + first[2:]
    with pytest.raises(ValueError, match='duplicate'):
        Context.fromdict(d_invalid)

def test_fromdict_context_invalid_index(d_invalid):
    first = d_invalid['context'][0]
    d_invalid['context'][0] = (42,) + first[1:]
    with pytest.raises(ValueError, match='invalid index'):
        Context.fromdict(d_invalid)

def test_fromdict_empty_lattice(d_invalid):
    d_invalid['lattice'] = []
    with pytest.raises(ValueError, match='empty lattice'):
        Context.fromdict(d_invalid)

@pytest.fixture(params=[False, True])
def require_lattice(request):
    return request.param

@pytest.fixture(params=[False, True])
def ignore_lattice(request):
    return request.param

@pytest.fixture(params=[False, True])
def raw(request):
    return request.param

def test_fromdict(context, lattice, d, require_lattice, ignore_lattice, raw):
    if require_lattice and 'lattice' not in d:
        return

    result = Context.fromdict(d,
                              require_lattice=require_lattice,
                              ignore_lattice=ignore_lattice,
                              raw=raw)

    assert result == context

    if ignore_lattice or 'lattice' not in d:
        assert 'lattice' not in result.__dict__
    else:
        assert 'lattice' in result.__dict__
        assert result.lattice == lattice

def test_raw(context, lattice, d, raw):
    def shuffled(items):
        result = list(items)
        random.shuffle(result)
        return result

    _lattice = d.get('lattice')
    d = {'objects': d['objects'], 'properties': d['properties'],
         'context': [shuffled(intent) for intent in d['context']]}

    if _lattice is not None:
        pairs = shuffled(enumerate(_lattice))
        index_map = {old: new for new, (old, _) in enumerate(pairs)}
        d['lattice'] = [(shuffled(ex), shuffled(in_),
                         shuffled(index_map[i] for i in up),
                         shuffled(index_map[i] for i in lo))
                        for _, (ex, in_, up, lo) in pairs]

    result = Context.fromdict(d, raw=raw)

    assert isinstance(result, Context)
    assert result == context
    if _lattice is not None:
        if raw:
            assert result.lattice == lattice
        else:
            # instance broken by shuffled(d['lattice'])
            assert result.lattice != lattice

@pytest.mark.parametrize('include_lattice', [False, None, True])
def test_roundtrip(context, include_lattice):
    context = Context(context.objects, context.properties, context.bools)
    assert 'lattice' not in context.__dict__

    d = context.todict(include_lattice=include_lattice)

    assert isinstance(d, dict) and d
    assert all(d[k] for k in ('objects', 'properties', 'context'))
    if include_lattice:
        assert 'lattice' in context.__dict__
        assert d['lattice']
    else:
        assert 'lattice' not in context.__dict__
        assert 'lattice' not in d

    result = Context.fromdict(d)

    assert isinstance(result, Context)
    assert result == context
    if include_lattice:
        assert 'lattice' in result.__dict__
        assert result.lattice == context.lattice

def test_json_invalid_path(context):
    with pytest.raises(TypeError, match=r'path_or_fileobj'):
        context.tojson(object())
    with pytest.raises(TypeError, match=r'path_or_fileobj'):
        Context.fromjson(object())

@pytest.fixture(scope='module')
def make_stringio(py2):
    return io.BytesIO if py2 else io.StringIO

@pytest.fixture(params=range(3))
def path_or_fileobj(request, make_stringio, tmp_path, filename='context.json'):
    if request.param == 0:
        return str(tmp_path / filename), False
    elif request.param == 1:
        return tmp_path / filename, False
    elif request.param == 2:
        return make_stringio(), True
    raise RuntimeError

@pytest.fixture(params=['utf-8', None])
def encoding(request):
    return request.param

def test_json_roundtrip(context, path_or_fileobj, encoding):
    f, is_fileobj = path_or_fileobj
    kwargs = {'encoding': encoding} if encoding is not None else {}

    context = Context(context.objects, context.properties, context.bools)
    assert 'lattice' not in context.__dict__

    context.tojson(f, **kwargs)
    if is_fileobj:
        f.seek(0)
    assert 'lattice' not in context.__dict__

    deserialized = Context.fromjson(f, **kwargs)
    if is_fileobj:
        f.seek(0)
    assert 'lattice' not in deserialized.__dict__

    assert deserialized == context

    assert isinstance(context.lattice, Lattice)
    assert 'lattice' in context.__dict__

    context.tojson(f, **kwargs)
    if is_fileobj:
        f.seek(0)

    deserialized = Context.fromjson(f, **kwargs)
    if is_fileobj:
        f.close()
    assert 'lattice' in deserialized.__dict__

    assert deserialized == context
    assert deserialized.lattice == context.lattice

def test_json_indent(make_stringio, context, encoding):
    assert 'lattice' not in context.__dict__
    kwargs = {'encoding': encoding} if encoding is not None else {}

    with make_stringio() as f:
        context.tojson(f, indent=4, sort_keys=True, **kwargs)

        assert 'lattice' not in context.__dict__
        serialized = f.getvalue()

    assert serialized.startswith('{\n    "context": [')

def test_json_newlinedelmited(make_stringio, context, encoding):
    assert 'lattice' not in context.__dict__
    kwargs = {'encoding': encoding} if encoding is not None else {}

    with make_stringio() as f:
        context.tojson(f, sort_keys=True,  **kwargs)
        assert 'lattice' not in context.__dict__
        f.write(str('\n'))
        serialized = f.getvalue()

        assert serialized.startswith('{"context": [')

        context.tojson(f, sort_keys=True, **kwargs)
        assert 'lattice' not in context.__dict__
        f.write(str('\n'))
        second = f.getvalue()

        assert second == serialized * 2

@pytest.fixture(scope='module')
def nonascii_context(abba=(u'Agneta F\xe4ltskog', u'Anni-Frid Lyngstat',
                           u'Benny Andersson', u'Bj\xf6rn Ulvaeus')):
    d = Definition()
    for o in abba:
        d.add_object(o, [u'human', u'singer'])
    d.add_property(u'female', abba[:2])
    d.add_property(u'male', abba[2:])
    d.add_property(u'keyboarder', [abba[2]])
    d.add_property(u'guitarrist', [abba[3]])
    d.add_property(u'sch\xf6n', abba[::2])

    return Context(*d)

def test_json_nonascii_context(make_stringio, nonascii_context, encoding):
    assert isinstance(nonascii_context.lattice, Lattice)
    assert 'lattice' in nonascii_context.__dict__
    kwargs = {'encoding': encoding} if encoding is not None else {}

    with make_stringio() as f:
        nonascii_context.tojson(f, **kwargs)
        serialized = f.getvalue()
        f.seek(0)
        deserialized = Context.fromjson(f, **kwargs)
        assert 'lattice' in deserialized.__dict__
        assert deserialized == nonascii_context
        assert deserialized.lattice == nonascii_context.lattice
        assert u'"Agneta F\\u00e4ltskog"' in serialized
        assert u'"Bj\\u00f6rn Ulvaeus"' in serialized
        assert u'"sch\\u00f6n"' in serialized
<commit_msg>use better test names<commit_after># test_serialization.py

from __future__ import unicode_literals

import io
import random

import pytest

from concepts import Context, Definition
from concepts.lattices import Lattice

SERIALIZED = {
    'objects': (
        '1sg', '1pl', '2sg', '2pl', '3sg', '3pl',
    ),
    'properties': (
        '+1', '-1', '+2', '-2', '+3', '-3', '+sg', '+pl', '-sg', '-pl',
    ),
    'context': [
        (0, 3, 5, 6, 9),
        (0, 3, 5, 7, 8),
        (1, 2, 5, 6, 9),
        (1, 2, 5, 7, 8),
        (1, 3, 4, 6, 9),
        (1, 3, 4, 7, 8),
    ],
    'lattice': [
        ((), (0, 1, 2, 3, 4, 5, 6, 7, 8, 9), (1, 2, 3, 4, 5, 6), ()),
        ((0,), (0, 3, 5, 6, 9), (7, 8, 9), (0,)),
        ((1,), (0, 3, 5, 7, 8), (7, 10, 11), (0,)),
        ((2,), (1, 2, 5, 6, 9), (8, 12, 13), (0,)),
        ((3,), (1, 2, 5, 7, 8), (10, 12, 14), (0,)),
        ((4,), (1, 3, 4, 6, 9), (9, 13, 15), (0,)),
        ((5,), (1, 3, 4, 7, 8), (11, 14, 15), (0,)),
        ((0, 1), (0, 3, 5), (18, 19), (1, 2)),
        ((0, 2), (5, 6, 9), (16, 18), (1, 3)),
        ((0, 4), (3, 6, 9), (16, 19), (1, 5)),
        ((1, 3), (5, 7, 8), (17, 18), (2, 4)),
        ((1, 5), (3, 7, 8), (17, 19), (2, 6)),
        ((2, 3), (1, 2, 5), (18, 20), (3, 4)),
        ((2, 4), (1, 6, 9), (16, 20), (3, 5)),
        ((3, 5), (1, 7, 8), (17, 20), (4, 6)),
        ((4, 5), (1, 3, 4), (19, 20), (5, 6)),
        ((0, 2, 4), (6, 9), (21,), (8, 9, 13)),
        ((1, 3, 5), (7, 8), (21,), (10, 11, 14)),
        ((0, 1, 2, 3), (5,), (21,), (7, 8, 10, 12)),
        ((0, 1, 4, 5), (3,), (21,), (7, 9, 11, 15)),
        ((2, 3, 4, 5), (1,), (21,), (12, 13, 14, 15)),
        ((0, 1, 2, 3, 4, 5), (), (), (18, 19, 20, 16, 17)),
    ],
}

SERIALIZED_NOLATTICE = {'objects': SERIALIZED['objects'],
                        'properties': SERIALIZED['properties'],
                        'context': list(SERIALIZED['context'])}

@pytest.fixture(params=[SERIALIZED, SERIALIZED_NOLATTICE])
def d(request):
    return request.param

def test_todict(context, d):
    assert 'lattice' not in context.__dict__
    if 'lattice' in d:
        context = Context(context.objects, context.properties, context.bools)
        assert 'lattice' not in context.__dict__

        assert context.todict(include_lattice=True) == context.todict() == d
        assert 'lattice' in context.__dict__
    else:
        assert context.todict() == context.todict(include_lattice=False) == d
        assert 'lattice' not in context.__dict__

@pytest.fixture
def d_invalid():
    return {'objects': SERIALIZED['objects'],
            'properties': SERIALIZED['properties'],
            'context': list(SERIALIZED['context']),
            'lattice': list(SERIALIZED['lattice'])}

@pytest.mark.parametrize('missing',
                         ['objects', 'properties', 'context', 'lattice'])
def test_fromdict_missing(d_invalid, missing):
    del d_invalid[missing]
    with pytest.raises(ValueError, match=r'missing .*%s' % missing):
        Context.fromdict(d_invalid, require_lattice=(missing == 'lattice'))

@pytest.mark.parametrize('nonstring', ['objects', 'properties'])
def test_fromdict_nonstring(d_invalid, nonstring):
    d_invalid[nonstring] = (42,) + d_invalid[nonstring][1:]
    with pytest.raises(ValueError, match=r'non-string %s' % nonstring):
        Context.fromdict(d_invalid)

@pytest.mark.parametrize('short', ['objects', 'context'])
def test_fromdict_mismatch(d_invalid, short):
    d_invalid[short] = d_invalid[short][1:]
    lens = (5, 6) if short == 'objects' else (6, 5)
    match = r'mismatch: %d objects with %d context' % lens
    with pytest.raises(ValueError, match=match):
        Context.fromdict(d_invalid)

def test_fromdict_context_duplicates(d_invalid):
    first = d_invalid['context'][0]
    d_invalid['context'][0] = (first[0], first[0]) + first[2:]
    with pytest.raises(ValueError, match='duplicate'):
        Context.fromdict(d_invalid)

def test_fromdict_context_invalid_index(d_invalid):
    first = d_invalid['context'][0]
    d_invalid['context'][0] = (42,) + first[1:]
    with pytest.raises(ValueError, match='invalid index'):
        Context.fromdict(d_invalid)

def test_fromdict_empty_lattice(d_invalid):
    d_invalid['lattice'] = []
    with pytest.raises(ValueError, match='empty lattice'):
        Context.fromdict(d_invalid)

@pytest.fixture(params=[False, True])
def require_lattice(request):
    return request.param

@pytest.fixture(params=[False, True])
def ignore_lattice(request):
    return request.param

@pytest.fixture(params=[False, True])
def raw(request):
    return request.param

def test_fromdict(context, lattice, d, require_lattice, ignore_lattice, raw):
    if require_lattice and 'lattice' not in d:
        return

    result = Context.fromdict(d,
                              require_lattice=require_lattice,
                              ignore_lattice=ignore_lattice,
                              raw=raw)

    assert result == context

    if ignore_lattice or 'lattice' not in d:
        assert 'lattice' not in result.__dict__
    else:
        assert 'lattice' in result.__dict__
        assert result.lattice == lattice

def test_fromdict_raw(context, lattice, d, raw):
    def shuffled(items):
        result = list(items)
        random.shuffle(result)
        return result

    _lattice = d.get('lattice')
    d = {'objects': d['objects'], 'properties': d['properties'],
         'context': [shuffled(intent) for intent in d['context']]}

    if _lattice is not None:
        pairs = shuffled(enumerate(_lattice))
        index_map = {old: new for new, (old, _) in enumerate(pairs)}
        d['lattice'] = [(shuffled(ex), shuffled(in_),
                         shuffled(index_map[i] for i in up),
                         shuffled(index_map[i] for i in lo))
                        for _, (ex, in_, up, lo) in pairs]

    result = Context.fromdict(d, raw=raw)

    assert isinstance(result, Context)
    assert result == context
    if _lattice is not None:
        if raw:
            assert result.lattice == lattice
        else:
            # instance broken by shuffled(d['lattice'])
            assert result.lattice != lattice

@pytest.mark.parametrize('include_lattice', [False, None, True])
def test_dict_roundtrip(context, include_lattice):
    context = Context(context.objects, context.properties, context.bools)
    assert 'lattice' not in context.__dict__

    d = context.todict(include_lattice=include_lattice)

    assert isinstance(d, dict) and d
    assert all(d[k] for k in ('objects', 'properties', 'context'))
    if include_lattice:
        assert 'lattice' in context.__dict__
        assert d['lattice']
    else:
        assert 'lattice' not in context.__dict__
        assert 'lattice' not in d

    result = Context.fromdict(d)

    assert isinstance(result, Context)
    assert result == context
    if include_lattice:
        assert 'lattice' in result.__dict__
        assert result.lattice == context.lattice

def test_json_invalid_path(context):
    with pytest.raises(TypeError, match=r'path_or_fileobj'):
        context.tojson(object())
    with pytest.raises(TypeError, match=r'path_or_fileobj'):
        Context.fromjson(object())

@pytest.fixture(scope='module')
def make_stringio(py2):
    return io.BytesIO if py2 else io.StringIO

@pytest.fixture(params=['str', 'pathlike', 'fileobj'])
def path_or_fileobj(request, tmp_path, make_stringio, filename='context.json'):
    if request.param == 'str':
        return str(tmp_path / filename), False
    elif request.param == 'pathlike':
        return tmp_path / filename, False
    elif request.param == 'fileobj':
        return make_stringio(), True
    raise RuntimeError

@pytest.fixture(params=['utf-8', None])
def encoding(request):
    return request.param

def test_json_roundtrip(context, path_or_fileobj, encoding):
    f, is_fileobj = path_or_fileobj
    kwargs = {'encoding': encoding} if encoding is not None else {}

    context = Context(context.objects, context.properties, context.bools)
    assert 'lattice' not in context.__dict__

    context.tojson(f, **kwargs)
    if is_fileobj:
        f.seek(0)
    assert 'lattice' not in context.__dict__

    deserialized = Context.fromjson(f, **kwargs)
    if is_fileobj:
        f.seek(0)
    assert 'lattice' not in deserialized.__dict__

    assert deserialized == context

    assert isinstance(context.lattice, Lattice)
    assert 'lattice' in context.__dict__

    context.tojson(f, **kwargs)
    if is_fileobj:
        f.seek(0)

    deserialized = Context.fromjson(f, **kwargs)
    if is_fileobj:
        f.close()
    assert 'lattice' in deserialized.__dict__

    assert deserialized == context
    assert deserialized.lattice == context.lattice

def test_tojson_indent4(make_stringio, context, encoding):
    assert 'lattice' not in context.__dict__
    kwargs = {'encoding': encoding} if encoding is not None else {}

    with make_stringio() as f:
        context.tojson(f, indent=4, sort_keys=True, **kwargs)

        assert 'lattice' not in context.__dict__
        serialized = f.getvalue()

    assert serialized.startswith('{\n    "context": [')

def test_tojson_newlinedelmited(make_stringio, context, encoding):
    assert 'lattice' not in context.__dict__
    kwargs = {'encoding': encoding} if encoding is not None else {}

    with make_stringio() as f:
        context.tojson(f, sort_keys=True,  **kwargs)
        assert 'lattice' not in context.__dict__
        f.write(str('\n'))
        serialized = f.getvalue()

        assert serialized.startswith('{"context": [')

        context.tojson(f, sort_keys=True, **kwargs)
        assert 'lattice' not in context.__dict__
        f.write(str('\n'))
        second = f.getvalue()

        assert second == serialized * 2

@pytest.fixture(scope='module')
def nonascii_context(abba=(u'Agneta F\xe4ltskog', u'Anni-Frid Lyngstat',
                           u'Benny Andersson', u'Bj\xf6rn Ulvaeus')):
    d = Definition()
    for o in abba:
        d.add_object(o, [u'human', u'singer'])
    d.add_property(u'female', abba[:2])
    d.add_property(u'male', abba[2:])
    d.add_property(u'keyboarder', [abba[2]])
    d.add_property(u'guitarrist', [abba[3]])
    d.add_property(u'sch\xf6n', abba[::2])

    return Context(*d)

def test_json_roundtrip_nonascii_context(make_stringio, nonascii_context, encoding):
    assert isinstance(nonascii_context.lattice, Lattice)
    assert 'lattice' in nonascii_context.__dict__
    kwargs = {'encoding': encoding} if encoding is not None else {}

    with make_stringio() as f:
        nonascii_context.tojson(f, **kwargs)
        serialized = f.getvalue()
        f.seek(0)

        deserialized = Context.fromjson(f, **kwargs)
        assert 'lattice' in deserialized.__dict__
        assert deserialized == nonascii_context
        assert deserialized.lattice == nonascii_context.lattice

        assert u'"Agneta F\\u00e4ltskog"' in serialized
        assert u'"Bj\\u00f6rn Ulvaeus"' in serialized
        assert u'"sch\\u00f6n"' in serialized
<|endoftext|>


<commit_before>import os

from celery.loaders.base import BaseLoader

DEFAULT_CONFIG_MODULE = "celeryconfig"

DEFAULT_SETTINGS = {
    "DEBUG": False,
    "DATABASE_ENGINE": "sqlite3",
    "DATABASE_NAME": "celery.sqlite",
    "INSTALLED_APPS": ("celery", ),
}

def wanted_module_item(item):
    return not item.startswith("_")

class Settings(dict):

    def __getattr__(self, key):
        try:
            return self[key]
        except KeyError:
            raise AttributeError(key)

    def __setattr_(self, key, value):
        self[key] = value

class Loader(BaseLoader):
    """The default loader.

    See the FAQ for example usage.

    """

    def setup_django_env(self, settingsdict):
        settings = Settings(DEFAULT_SETTINGS, **settingsdict)
        installed_apps = set(list(DEFAULT_SETTINGS["INSTALLED_APPS"]) + \
                             list(settings.INSTALLED_APPS))
        settings.INSTALLED_APPS = tuple(installed_apps)

        return settings

    def read_configuration(self):
        """Read configuration from ``celeryconfig.py`` and configure
        celery and Django so it can be used by regular Python."""
        configname = os.environ.get("CELERY_CONFIG_MODULE",
                                    DEFAULT_CONFIG_MODULE)
        celeryconfig = __import__(configname, {}, {}, [''])
        usercfg = dict((key, getattr(celeryconfig, key))
                            for key in dir(celeryconfig)
                                if wanted_module_item(key))
        return self.setup_django_env(usercfg)

    def on_worker_init(self):
        """Imports modules at worker init so tasks can be registered
        and used by the worked.

        The list of modules to import is taken from the ``CELERY_IMPORTS``
        setting in ``celeryconf.py``.

        """
        self.import_default_modules()
<commit_msg>DefaultLoader.setup_django_env renamed to .setup_settings<commit_after>import os

from celery.loaders.base import BaseLoader

DEFAULT_CONFIG_MODULE = "celeryconfig"

DEFAULT_SETTINGS = {
    "DEBUG": False,
    "DATABASE_ENGINE": "sqlite3",
    "DATABASE_NAME": "celery.sqlite",
    "INSTALLED_APPS": ("celery", ),
}

def wanted_module_item(item):
    return not item.startswith("_")

class Settings(dict):

    def __getattr__(self, key):
        try:
            return self[key]
        except KeyError:
            raise AttributeError(key)

    def __setattr_(self, key, value):
        self[key] = value

class Loader(BaseLoader):
    """The default loader.

    See the FAQ for example usage.

    """

    def setup_settings(self, settingsdict):
        settings = Settings(DEFAULT_SETTINGS, **settingsdict)
        installed_apps = set(list(DEFAULT_SETTINGS["INSTALLED_APPS"]) + \
                             list(settings.INSTALLED_APPS))
        settings.INSTALLED_APPS = tuple(installed_apps)

        return settings

    def read_configuration(self):
        """Read configuration from ``celeryconfig.py`` and configure
        celery and Django so it can be used by regular Python."""
        configname = os.environ.get("CELERY_CONFIG_MODULE",
                                    DEFAULT_CONFIG_MODULE)
        celeryconfig = __import__(configname, {}, {}, [''])
        usercfg = dict((key, getattr(celeryconfig, key))
                            for key in dir(celeryconfig)
                                if wanted_module_item(key))
        return self.setup_settings(usercfg)

    def on_worker_init(self):
        """Imports modules at worker init so tasks can be registered
        and used by the worked.

        The list of modules to import is taken from the ``CELERY_IMPORTS``
        setting in ``celeryconf.py``.

        """
        self.import_default_modules()
<|endoftext|>


<commit_before>import os
import unittest
import multiprocessing
import time

from urllib.parse import urlparse
from werkzeug.security import generate_password_hash
from splinter import Browser

# Configure app to user the testing database
os.environ["CONFIG_PATH"] = "fight_simulator.config.TestingConfig"

from fight_simulator import app
from fight_simulator.database import Base, engine, session, User, History, Fighter

class TestViews(unittest.TestCase):
    def setUp(self):
        """ Test setup """
        self.browser = Browser("phantomjs")
        # Resize browser window to make sure all elements are visible for tests
        self.browser.driver.set_window_size(1920, 1080)

        # Set up the tables in the database
        Base.metadata.create_all(engine)

        self.process = multiprocessing.Process(target=app.run, kwargs={"port": 8080})
        self.process.start()
        time.sleep(1)

    """ Acceptance tests for user and login system """

    def test_register_new_user(self):
        self.browser.visit("http://127.0.0.1:8080/create_user")
        self.browser.fill("email", "testmail@test.com")
        self.browser.fill("password", "testpass")
        button = self.browser.find_by_css("button[type=submit]")
        button.click()
        self.assertEqual(self.browser.url, "http://127.0.0.1:8080/login")

    def test_register_user_exists(self):
        self.test_register_new_user()
        self.browser.visit("http://127.0.0.1:8080/create_user")
        self.browser.fill("email", "testmail@test.com")
        self.browser.fill("password", "testpass")
        button = self.browser.find_by_css("button[type=submit]")
        button.click()
        self.assertEqual(self.browser.url, "http://127.0.0.1:8080/create_user")

    def test_login_correct(self):
        self.test_register_new_user()
        self.browser.visit("http://127.0.0.1:8080/login")
        self.browser.fill("email", "testmail@test.com")
        self.browser.fill("password", "testpass")
        button = self.browser.find_by_css("button[type=submit]")
        button.click()
        login_link = self.browser.is_element_present_by_text('Login')
        self.assertFalse(login_link)
        self.assertEqual(self.browser.url, "http://127.0.0.1:8080/fight")

    def test_login_incorrect(self):
        self.test_register_new_user()
        self.browser.visit("http://127.0.0.1:8080/login")
        self.browser.fill("email", "wrongmail@test.com")
        self.browser.fill("password", "testpass")
        button = self.browser.find_by_css("button[type=submit]")
        button.click()
        self.assertEqual(self.browser.url, "http://127.0.0.1:8080/login")

    def test_logout(self):
        self.test_login_correct()
        self.browser.click_link_by_partial_href('logout')
        logout_link = self.browser.is_element_present_by_text('Logout')
        self.assertFalse(logout_link)
        self.assertEqual(self.browser.url, "http://127.0.0.1:8080/")

    def tearDown(self):
        """ Test teardown """
        # Remove the tables and their data from the database
        self.process.terminate()
        session.close()
        engine.dispose()
        Base.metadata.drop_all(engine)
        self.browser.quit()

if __name__ == "__main__":
    unittest.main()
<commit_msg>added assertion to 'test_login_incorrect' to improve test accuracy<commit_after>import os
import unittest
import multiprocessing
import time

from urllib.parse import urlparse
from werkzeug.security import generate_password_hash
from splinter import Browser

# Configure app to user the testing database
os.environ["CONFIG_PATH"] = "fight_simulator.config.TestingConfig"

from fight_simulator import app
from fight_simulator.database import Base, engine, session, User, History, Fighter

class TestViews(unittest.TestCase):
    def setUp(self):
        """ Test setup """
        self.browser = Browser("phantomjs")
        # Resize browser window to make sure all elements are visible for tests
        self.browser.driver.set_window_size(1920, 1080)

        # Set up the tables in the database
        Base.metadata.create_all(engine)

        self.process = multiprocessing.Process(target=app.run, kwargs={"port": 8080})
        self.process.start()
        time.sleep(1)

    """ Acceptance tests for user and login system """

    def test_register_new_user(self):
        self.browser.visit("http://127.0.0.1:8080/create_user")
        self.browser.fill("email", "testmail@test.com")
        self.browser.fill("password", "testpass")
        button = self.browser.find_by_css("button[type=submit]")
        button.click()
        self.assertEqual(self.browser.url, "http://127.0.0.1:8080/login")

    def test_register_user_exists(self):
        self.test_register_new_user()
        self.browser.visit("http://127.0.0.1:8080/create_user")
        self.browser.fill("email", "testmail@test.com")
        self.browser.fill("password", "testpass")
        button = self.browser.find_by_css("button[type=submit]")
        button.click()
        self.assertEqual(self.browser.url, "http://127.0.0.1:8080/create_user")

    def test_login_correct(self):
        self.test_register_new_user()
        self.browser.visit("http://127.0.0.1:8080/login")
        self.browser.fill("email", "testmail@test.com")
        self.browser.fill("password", "testpass")
        button = self.browser.find_by_css("button[type=submit]")
        button.click()
        login_link = self.browser.is_element_present_by_text('Login')
        self.assertFalse(login_link)
        self.assertEqual(self.browser.url, "http://127.0.0.1:8080/fight")

    def test_login_incorrect(self):
        self.test_register_new_user()
        self.browser.visit("http://127.0.0.1:8080/login")
        self.browser.fill("email", "wrongmail@test.com")
        self.browser.fill("password", "testpass")
        button = self.browser.find_by_css("button[type=submit]")
        button.click()
        login_link = self.browser.is_element_present_by_text('Login')
        self.assertTrue(login_link)
        self.assertEqual(self.browser.url, "http://127.0.0.1:8080/login")

    def test_logout(self):
        self.test_login_correct()
        self.browser.click_link_by_partial_href('logout')
        logout_link = self.browser.is_element_present_by_text('Logout')
        self.assertFalse(logout_link)
        self.assertEqual(self.browser.url, "http://127.0.0.1:8080/")

    def tearDown(self):
        """ Test teardown """
        # Remove the tables and their data from the database
        self.process.terminate()
        session.close()
        engine.dispose()
        Base.metadata.drop_all(engine)
        self.browser.quit()

if __name__ == "__main__":
    unittest.main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
from django.core.management import execute_manager
import sys
from hvad.test_utils.project import settings
import os

try:
except ImportError:
    sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__)
    sys.exit(1)

BASEDIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
sys.path.insert(0, BASEDIR)

if __name__ == "__main__":
    execute_manager(settings)
<commit_msg>fix: python syntax error (annoying when precompiling all py)<commit_after>#!/usr/bin/env python
from django.core.management import execute_manager
import sys
import os

try:
    from hvad.test_utils.project import settings
except ImportError:
    sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__)
    sys.exit(1)

BASEDIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
sys.path.insert(0, BASEDIR)

if __name__ == "__main__":
    execute_manager(settings)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for the Profile plugin."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import json
import os
import tensorflow as tf

from tensorboard.backend.event_processing import plugin_asset_util
from tensorboard.plugins import base_plugin
from tensorboard.plugins.profile import profile_plugin
from tensorboard.plugins.profile import trace_events_pb2

class ProfilePluginTest(tf.test.TestCase):

  def setUp(self):
    # Populate the log directory with runs and traces.
    self.logdir = self.get_temp_dir()
    plugin_logdir = plugin_asset_util.PluginDirectory(
        self.logdir, profile_plugin.ProfilePlugin.plugin_name)
    os.makedirs(plugin_logdir)
    self.run_to_tools = {
        'foo': ['trace_viewer'],
        'bar': ['unsupported'],
        'baz': ['trace_viewer'],
        'empty': [],
    }
    self.run_to_host = {
        'foo': 'host0',
        'bar': 'host1',
        'baz': 'host2',
        'empty': "",
    }
    for run in self.run_to_tools:
      run_dir = os.path.join(plugin_logdir, run)
      os.mkdir(run_dir)
      for tool in self.run_to_tools[run]:
        if tool not in profile_plugin.TOOLS:
          continue
        host = self.run_to_host[run]
        file_name = host + profile_plugin.TOOLS[tool]
        tool_file = os.path.join(run_dir, file_name)
        if tool == 'trace_viewer':
          trace = trace_events_pb2.Trace()
          trace.devices[0].name = run
          data = trace.SerializeToString()
        else:
          data = tool
        with open(tool_file, 'wb') as f:
          f.write(data)
    with open(os.path.join(plugin_logdir, 'noise'), 'w') as f:
      f.write('Not a dir, not a run.')

    # The profiler plugin does not use the multiplexer, so we do not bother to
    # construct a meaningful one right now. In fact, the profiler plugin does
    # not use this context object in general. Its constructor has to accept it
    # though, and it may use the context in the future.
    context = base_plugin.TBContext(logdir=self.logdir, multiplexer=None)
    self.plugin = profile_plugin.ProfilePlugin(context)
    self.apps = self.plugin.get_plugin_apps()

  def testRuns(self):
    runs = self.plugin.index_impl()
    self.assertItemsEqual(runs.keys(), self.run_to_tools.keys())
    self.assertItemsEqual(runs['foo'], self.run_to_tools['foo'])
    self.assertItemsEqual(runs['bar'], [])
    self.assertItemsEqual(runs['empty'], [])

  def testData(self):
    trace = json.loads(self.plugin.data_impl('foo', 'trace_viewer', self.run_to_host['foo']))
    self.assertEqual(trace,
                     dict(
                         displayTimeUnit='ns',
                         metadata={'highres-ticks': True},
                         traceEvents=[
                             dict(
                                 ph='M',
                                 pid=0,
                                 name='process_name',
                                 args=dict(name='foo')),
                             dict(
                                 ph='M',
                                 pid=0,
                                 name='process_sort_index',
                                 args=dict(sort_index=0)), {}
                         ]))

    # Invalid tool/run.
    self.assertEqual(None, self.plugin.data_impl('foo', 'nonono', 'host0'))
    self.assertEqual(None, self.plugin.data_impl('foo', 'trace_viewer', 'host1'))
    self.assertEqual(None, self.plugin.data_impl('bar', 'unsupported', 'host1'))
    self.assertEqual(None, self.plugin.data_impl('empty', 'trace_viewer', 'host3'))

  def testActive(self):
    self.assertTrue(self.plugin.is_active())

if __name__ == '__main__':
  tf.test.main()
<commit_msg>Added test for host route.<commit_after># -*- coding: utf-8 -*-
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for the Profile plugin."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import json
import os
import tensorflow as tf

from tensorboard.backend.event_processing import plugin_asset_util
from tensorboard.plugins import base_plugin
from tensorboard.plugins.profile import profile_plugin
from tensorboard.plugins.profile import trace_events_pb2

class ProfilePluginTest(tf.test.TestCase):

  def setUp(self):
    # Populate the log directory with runs and traces.
    self.logdir = self.get_temp_dir()
    plugin_logdir = plugin_asset_util.PluginDirectory(
        self.logdir, profile_plugin.ProfilePlugin.plugin_name)
    os.makedirs(plugin_logdir)
    self.run_to_tools = {
        'foo': ['trace_viewer'],
        'bar': ['unsupported'],
        'baz': ['trace_viewer'],
        'empty': [],
    }
    self.run_to_hosts = {
        'foo': ['host0','host1'],
        'bar': ['host1'],
        'baz': ['host2'],
        'empty': [],
    }
    for run in self.run_to_tools:
      run_dir = os.path.join(plugin_logdir, run)
      os.mkdir(run_dir)
      for tool in self.run_to_tools[run]:
        if tool not in profile_plugin.TOOLS:
          continue
        for host in self.run_to_hosts[run]:
          file_name = host + profile_plugin.TOOLS[tool]
          tool_file = os.path.join(run_dir, file_name)
          if tool == 'trace_viewer':
            trace = trace_events_pb2.Trace()
            trace.devices[0].name = run
            data = trace.SerializeToString()
          else:
            data = tool
          with open(tool_file, 'wb') as f:
            f.write(data)
    with open(os.path.join(plugin_logdir, 'noise'), 'w') as f:
      f.write('Not a dir, not a run.')

    # The profiler plugin does not use the multiplexer, so we do not bother to
    # construct a meaningful one right now. In fact, the profiler plugin does
    # not use this context object in general. Its constructor has to accept it
    # though, and it may use the context in the future.
    context = base_plugin.TBContext(logdir=self.logdir, multiplexer=None)
    self.plugin = profile_plugin.ProfilePlugin(context)
    self.apps = self.plugin.get_plugin_apps()

  def testRuns(self):
    runs = self.plugin.index_impl()
    self.assertItemsEqual(runs.keys(), self.run_to_tools.keys())
    self.assertItemsEqual(runs['foo'], self.run_to_tools['foo'])
    self.assertItemsEqual(runs['bar'], [])
    self.assertItemsEqual(runs['empty'], [])

  def testHosts(self):
    hosts = self.plugin.host_impl('foo', 'trace_viewer')
    self.assertItemsEqual(['host0', 'host1'], sorted(hosts))

  def testData(self):
    trace = json.loads(self.plugin.data_impl('foo', 'trace_viewer', 'host0'))
    self.assertEqual(trace,
                     dict(
                         displayTimeUnit='ns',
                         metadata={'highres-ticks': True},
                         traceEvents=[
                             dict(
                                 ph='M',
                                 pid=0,
                                 name='process_name',
                                 args=dict(name='foo')),
                             dict(
                                 ph='M',
                                 pid=0,
                                 name='process_sort_index',
                                 args=dict(sort_index=0)), {}
                         ]))

    # Invalid tool/run.
    self.assertEqual(None, self.plugin.data_impl('foo', 'nonono', 'host0'))
    self.assertEqual(None, self.plugin.data_impl('foo', 'trace_viewer', 'host2'))
    self.assertEqual(None, self.plugin.data_impl('bar', 'unsupported', 'host1'))
    self.assertEqual(None, self.plugin.data_impl('empty', 'trace_viewer', 'host3'))

  def testActive(self):
    self.assertTrue(self.plugin.is_active())

if __name__ == '__main__':
  tf.test.main()
<|endoftext|>


<commit_before>"""Kraken - base config module.

Classes:
BaseConfig -- Base config object used to configure builders.

"""

class BaseConfig(object):
    """Base Configuration for Kraken builders."""

    def __init__(self):
        super(BaseConfig, self).__init__()

        self.nameTemplate = {
                             "sides": ['L', 'R', 'M'],
                             "separator": "_",
                             "types": {
                                       "default": "null",
                                       "MyControl": "ctrl",
                                       "MyObj": "null"
                                      },
                             "formats":
                                       {
                                        "default": ['component', 'sep', 'side', 'sep', 'name', 'sep', 'type'],
                                        "MyControl2": ['side', 'sep', 'component', 'sep', 'name', 'sep', 'type']
                                       }
                            }

    def getNameTemplate(self):
        """Returns the naming template for this configuration.

        Return:
        Dict, naming template.

        """

        return self.nameTemplate<commit_msg>Setup better default naming.<commit_after>"""Kraken - base config module.

Classes:
BaseConfig -- Base config object used to configure builders.

"""

class BaseConfig(object):
    """Base Configuration for Kraken builders."""

    def __init__(self):
        super(BaseConfig, self).__init__()

        self.nameTemplate = {
                             "sides": ['L', 'R', 'M'],
                             "separator": "_",
                             "types": {
                                       "default": "null",
                                       "Control": "ctrl",
                                       "Null": "null"
                                      },
                             "formats":
                                       {
                                        "default": ['component', 'sep', 'side', 'sep', 'name', 'sep', 'type'],
                                        "Rig": ['name'],
                                        "Layer": ['name']
                                       }
                            }

    def getNameTemplate(self):
        """Returns the naming template for this configuration.

        Return:
        Dict, naming template.

        """

        return self.nameTemplate<|endoftext|>


<commit_before>import elasticsearch, re, sys
from teafacto.util import tokenize, argprun

class SimpleQuestionsLabelIndex(object):
    def __init__(self, host="drogon", index="simplequestions_labels"):
        self.host = host
        self.indexp = index

    def index(self, labelp="labels.map"):
        es = elasticsearch.Elasticsearch(hosts=[self.host])
        es.indices.create(index=self.indexp,
                          body={
                              "settings": {
                                  "index": {
                                      "analysis": {
                                          "analyzer": {
                                              "keylower": {
                                                  "tokenizer": "keyword",
                                                  "filter": "lowercase"
                                              }
                                          }
                                      }
                                  }
                              },
                              "mappings": {
                                  "labelmap": {
                                      "properties": {
                                          "label": {"type": "string", "analyzer": "keylower"}
                                      }
                                  }
                              }
                          })
        i = 1
        for line in open(labelp):
            k, v = line[:-1].split("\t")
            vt = tokenize(v)
            es.index(index=self.indexp, doc_type="labelmap", id=i,
                     body={"label": " ".join(vt), "fbid": k})
            if i % 1000 == 0:
                print i
            i += 1
        print "indexed labels"

    def search(self, query, top=10):
        es = elasticsearch.Elasticsearch(hosts=[self.host])
        res = es.search(index=self.indexp, q="label:%s" % query, size=top)
        acc = {}
        for r in res["hits"]["hits"]:
            self._merge(acc, {r["_source"]["fbid"]: (r["_score"], r["_source"]["label"])})
        return acc

    @staticmethod
    def _merge(acc, d):
        for k, v in d.items():
            if k in acc:
                if acc[k][0] < v[0]:
                    acc[k] = v
            else:
                acc[k] = v

    def searchsentence(self, s, top=None, topsize=None):
        s = tokenize(s)
        ngrams = self.getallngrams(s, topsize)
        return self.searchallngrams(ngrams, top)

    def getallngrams(self, s, topsize=None):
        topsize = len(s) if topsize is None else topsize
        ngrams = set()
        i = 0
        while i < len(s):
            j = i + 1
            while j <= min(len(s), i + topsize):
                ngram = tuple(s[i:j])
                j += 1
                ngrams.add(ngram)
            i += 1
        return ngrams

    def searchallngrams(self, ngrams, top=None):
        #print ngrams
        es = elasticsearch.Elasticsearch(hosts=[self.host])
        searchbody = []
        header = {"index": self.indexp, "type": "labelmap"}
        for ngram in ngrams:
            body = {
                        "query": {
                            "filtered": {
                                "query": {
                                    "match_phrase": {
                                        "label":  '"%s"' % " ".join(ngram)
                                    }
                                }
                            }
                        }
                    }
            """
            body = {
                "query": {
                    "constant_score": {
                        "filter": {
                            "term": {
                                "label": " ".join(ngram)
                            }
                        }
                    }
                }
            }
            """
            if top is not None:
                body.update({"size": top, "from": 0})
            searchbody.append(header)
            searchbody.append(body)
        ngramres = es.msearch(body=searchbody)
        cans = {}
        for response in ngramres["responses"]:
            try:
                for r in response["hits"]["hits"]:
                    self._merge(cans, {r["_source"]["fbid"]: (r["_score"], r["_source"]["label"])})
            except KeyError, e:
                print response
        return cans

def run(index=False, indexp="labels.map", indexname="sq_subjnames_fb2m",
        search="e mc", host="drogon"):
    idx = SimpleQuestionsLabelIndex(host=host, index=indexname)
    if index is True and indexp is not None:
        idx.index(labelp=indexp)
        sys.exit()
    #res = idx.search("e", top=10)
    res = idx.searchsentence(search)
    sres = sorted(res.items(), key=lambda (x, y): y[0], reverse=True)
    for x in sres:
        print x
    print len(sres)

if __name__ == "__main__":
    argprun(run)<commit_msg>* label search exact match<commit_after>import elasticsearch, re, sys
from teafacto.util import tokenize, argprun

class SimpleQuestionsLabelIndex(object):
    def __init__(self, host="drogon", index="simplequestions_labels"):
        self.host = host
        self.indexp = index

    def index(self, labelp="labels.map"):
        es = elasticsearch.Elasticsearch(hosts=[self.host])
        try:
            es.indices.delete(index=self.indexp)
        except Exception, e:
            print "could not delete index %s" % self.indexp
        es.indices.create(index=self.indexp,
                          body={
                              "settings": {
                                  "index": {
                                      "analysis": {
                                          "analyzer": {
                                              "keylower": {
                                                  "tokenizer": "keyword",
                                                  "filter": "lowercase"
                                              }
                                          }
                                      }
                                  }
                              },
                              "mappings": {
                                  "labelmap": {
                                      "properties": {
                                          "label": {"type": "string", "analyzer": "keylower"}
                                      }
                                  }
                              }
                          })
        i = 1
        for line in open(labelp):
            k, v = line[:-1].split("\t")
            vt = tokenize(v)
            es.index(index=self.indexp, doc_type="labelmap", id=i,
                     body={"label": " ".join(vt), "fbid": k})
            if i % 1000 == 0:
                print i
            i += 1
        print "indexed labels"

    def search(self, query, top=10):
        es = elasticsearch.Elasticsearch(hosts=[self.host])
        res = es.search(index=self.indexp, q="label:%s" % query, size=top)
        acc = {}
        for r in res["hits"]["hits"]:
            self._merge(acc, {r["_source"]["fbid"]: (r["_score"], r["_source"]["label"])})
        return acc

    @staticmethod
    def _merge(acc, d):
        for k, v in d.items():
            if k in acc:
                if acc[k][0] < v[0]:
                    acc[k] = v
            else:
                acc[k] = v

    def searchsentence(self, s, top=None, topsize=None):
        s = tokenize(s)
        ngrams = self.getallngrams(s, topsize)
        return self.searchallngrams(ngrams, top)

    def getallngrams(self, s, topsize=None):
        topsize = len(s) if topsize is None else topsize
        ngrams = set()
        i = 0
        while i < len(s):
            j = i + 1
            while j <= min(len(s), i + topsize):
                ngram = tuple(s[i:j])
                j += 1
                ngrams.add(ngram)
            i += 1
        return ngrams

    def searchallngrams(self, ngrams, top=None):
        #print ngrams
        es = elasticsearch.Elasticsearch(hosts=[self.host])
        searchbody = []
        header = {"index": self.indexp, "type": "labelmap"}
        for ngram in ngrams:
            body = {
                        "query": {
                            "filtered": {
                                "query": {
                                    "match_phrase": {
                                        "label":  '"%s"' % " ".join(ngram)
                                    }
                                }
                            }
                        }
                    }
            """
            body = {
                "query": {
                    "constant_score": {
                        "filter": {
                            "term": {
                                "label": " ".join(ngram)
                            }
                        }
                    }
                }
            }
            """
            if top is not None:
                body.update({"size": top, "from": 0})
            searchbody.append(header)
            searchbody.append(body)
        ngramres = es.msearch(body=searchbody)
        cans = {}
        for response in ngramres["responses"]:
            try:
                for r in response["hits"]["hits"]:
                    self._merge(cans, {r["_source"]["fbid"]: (r["_score"], r["_source"]["label"])})
            except KeyError, e:
                print response
        return cans

def run(index=False, indexp="labels.map", indexname="sq_subjnames_fb2m",
        search="e mc", host="drogon"):
    idx = SimpleQuestionsLabelIndex(host=host, index=indexname)
    if index is True and indexp is not None:
        idx.index(labelp=indexp)
        sys.exit()
    #res = idx.search("e", top=10)
    res = idx.searchsentence(search)
    sres = sorted(res.items(), key=lambda (x, y): y[0], reverse=True)
    for x in sres:
        print x
    print len(sres)

if __name__ == "__main__":
    argprun(run)<|endoftext|>


<commit_before>import numpy as np
from matplotlib import pyplot as plt
from pylab import rcParams

def plot_buoyancy(cwd=''):
    """
    Plotting routine for the cross section of the buoyancy

    Args:
        cwd (string): current working directory
    """

    xx = np.load(cwd + 'xaxis.npy')
    uend = np.load(cwd + 'sdc.npy')
    udirk = np.load(cwd + 'dirk.npy')
    uimex = np.load(cwd + 'rkimex.npy')
    uref = np.load(cwd + 'uref.npy')
    usplit = np.load(cwd + 'split.npy')

    print("Estimated discretisation error split explicit:  %5.3e" %
          (np.linalg.norm(usplit.flatten() - uref.flatten(), np.inf) / np.linalg.norm(uref.flatten(), np.inf)))
    print("Estimated discretisation error of DIRK: %5.3e" %
          (np.linalg.norm(udirk.flatten() - uref.flatten(), np.inf) / np.linalg.norm(uref.flatten(), np.inf)))
    print("Estimated discretisation error of RK-IMEX:  %5.3e" %
          (np.linalg.norm(uimex.flatten() - uref.flatten(), np.inf) / np.linalg.norm(uref.flatten(), np.inf)))
    print("Estimated discretisation error of SDC:  %5.3e" %
          (np.linalg.norm(uend.flatten() - uref.flatten(), np.inf) / np.linalg.norm(uref.flatten(), np.inf)))

    fs = 8
    rcParams['figure.figsize'] = 5.0, 2.5
    plt.figure()
    plt.plot(xx[:, 5], udirk[2, :, 5], '--', color='g', markersize=fs - 2, label='DIRK(4)', dashes=(3, 3))
    plt.plot(xx[:, 5], uend[2, :, 5], '-', color='b', label='SDC(4)')
    plt.plot(xx[:, 5], uimex[2, :, 5], '--', color='r', markersize=fs - 2, label='IMEX(4)', dashes=(3, 3))
    plt.legend(loc='lower left', fontsize=fs, prop={'size': fs})
    plt.yticks(fontsize=fs)
    plt.xticks(fontsize=fs)
    plt.xlabel('x [km]', fontsize=fs, labelpad=0)
    plt.ylabel('Bouyancy', fontsize=fs, labelpad=1)
    filename = 'boussinesq.png'
    plt.savefig(filename, bbox_inches='tight')

if __name__ == "__main__":
    plot_buoyancy()
<commit_msg>more sanity checking<commit_after>import numpy as np
from matplotlib import pyplot as plt
from pylab import rcParams

def plot_buoyancy(cwd=''):
    """
    Plotting routine for the cross section of the buoyancy

    Args:
        cwd (string): current working directory
    """

    xx = np.load(cwd + 'xaxis.npy')
    uend = np.load(cwd + 'sdc.npy')
    udirk = np.load(cwd + 'dirk.npy')
    uimex = np.load(cwd + 'rkimex.npy')
    uref = np.load(cwd + 'uref.npy')
    usplit = np.load(cwd + 'split.npy')

    err_split = np.linalg.norm(usplit.flatten() - uref.flatten(), np.inf) / np.linalg.norm(uref.flatten(), np.inf)
    err_dirk = np.linalg.norm(udirk.flatten() - uref.flatten(), np.inf) / np.linalg.norm(uref.flatten(), np.inf)
    err_imex = np.linalg.norm(uimex.flatten() - uref.flatten(), np.inf) / np.linalg.norm(uref.flatten(), np.inf)
    err_sdc = np.linalg.norm(uend.flatten() - uref.flatten(), np.inf) / np.linalg.norm(uref.flatten(), np.inf)

    assert err_split < 4.821E-02, 'ERROR: split error is too high, got %s' % err_split
    assert err_dirk < 1.495e-01, 'ERROR: dirk error is too high, got %s' % err_dirk
    assert err_imex < 1.305e-01, 'ERROR: imex error is too high, got %s' % err_imex
    assert err_sdc < 9.548e-02, 'ERROR: sdc error is too high, got %s' % err_sdc

    print("Estimated discretisation error split explicit:  %5.3e" % err_split)
    print("Estimated discretisation error of DIRK: %5.3e" % err_dirk)
    print("Estimated discretisation error of RK-IMEX:  %5.3e" % err_imex)
    print("Estimated discretisation error of SDC:  %5.3e" % err_sdc)

    fs = 8
    rcParams['figure.figsize'] = 5.0, 2.5
    plt.figure()
    plt.plot(xx[:, 5], udirk[2, :, 5], '--', color='g', markersize=fs - 2, label='DIRK(4)', dashes=(3, 3))
    plt.plot(xx[:, 5], uend[2, :, 5], '-', color='b', label='SDC(4)')
    plt.plot(xx[:, 5], uimex[2, :, 5], '--', color='r', markersize=fs - 2, label='IMEX(4)', dashes=(3, 3))
    plt.legend(loc='lower left', fontsize=fs, prop={'size': fs})
    plt.yticks(fontsize=fs)
    plt.xticks(fontsize=fs)
    plt.xlabel('x [km]', fontsize=fs, labelpad=0)
    plt.ylabel('Bouyancy', fontsize=fs, labelpad=1)
    filename = 'boussinesq.png'
    plt.savefig(filename, bbox_inches='tight')

if __name__ == "__main__":
    plot_buoyancy()
<|endoftext|>


<commit_before>
from __future__ import print_function

import os
import sys
import signal
import threading
import pyuv

if sys.version_info >= (3, 0):
    LINESEP = os.linesep.encode()
else:
    LINESEP = os.linesep

def on_read(handle, ip_port, data, error):
    data = data.strip()
    if data:
        ip, port = ip_port
        handle.send(data+LINESEP, (ip, port))

def async_exit(async):
    async.close()
    signal_h.close()
    server.close()

def signal_cb(sig, frame):
    async.send(async_exit)

print("PyUV version %s" % pyuv.__version__)

loop = pyuv.Loop()
async = pyuv.Async(loop)

server = pyuv.UDP(loop)
server.bind(("0.0.0.0", 1234))
server.start_recv(on_read)

signal_h = pyuv.Signal(loop)
signal_h.start()

t = threading.Thread(target=loop.run)
t.start()

signal.signal(signal.SIGINT, signal_cb)
signal.pause()

t.join()

print("Stopped!")

<commit_msg>Fixed UDP example after API change<commit_after>
from __future__ import print_function

import os
import sys
import signal
import threading
import pyuv

if sys.version_info >= (3, 0):
    LINESEP = os.linesep.encode()
else:
    LINESEP = os.linesep

def on_read(handle, ip_port, data, error):
    data = data.strip()
    if data:
        ip, port = ip_port
        handle.send((ip, port), data+LINESEP)

def async_exit(async):
    async.close()
    signal_h.close()
    server.close()

def signal_cb(sig, frame):
    async.send(async_exit)

print("PyUV version %s" % pyuv.__version__)

loop = pyuv.Loop()
async = pyuv.Async(loop)

server = pyuv.UDP(loop)
server.bind(("0.0.0.0", 1234))
server.start_recv(on_read)

signal_h = pyuv.Signal(loop)
signal_h.start()

t = threading.Thread(target=loop.run)
t.start()

signal.signal(signal.SIGINT, signal_cb)
signal.pause()

t.join()

print("Stopped!")

<|endoftext|>


<commit_before>"""
Scraper for Massachusetts Appeals Court
CourtID: massapp
Court Short Name: MS
Author: William Palin
Court Contact: SJCReporter@sjc.state.ma.us (617) 557-1030
Reviewer:
Date: 2020-01-17
"""

from juriscraper.opinions.united_states.state import mass

class Site(mass.Site):
    def __init__(self, *args, **kwargs):
        super(Site, self).__init__(*args, **kwargs)
        self.url = "https://128archive.com"
        self.court_id = self.__module__
        self.court_identifier = "AC"
        self.set_local_variables()
<commit_msg>docs(mass128): Comment on url<commit_after>"""
Scraper for Massachusetts Appeals Court
CourtID: massapp
Court Short Name: MS
Author: William Palin
Court Contact: SJCReporter@sjc.state.ma.us (617) 557-1030
Reviewer:
Date: 2020-01-17
"""

from juriscraper.opinions.united_states.state import mass

class Site(mass.Site):
    def __init__(self, *args, **kwargs):
        super(Site, self).__init__(*args, **kwargs)
        self.url = "https://128archive.com"
        self.court_id = self.__module__
        self.court_identifier = "AC"
        self.set_local_variables()
        # https://128archive.com/Disposition/DownloadDisposition/{1 thru 14291}?dispositionId=14304
<|endoftext|>