<commit_before>#!/usr/bin/env python
#
# dials.process.py
#
#  Copyright (C) 2013 Diamond Light Source
#
#  Author: James Parkhurst
#
#  This code is distributed under the BSD license, a copy of which is
#  included in the root directory of this package.

from __future__ import division
from dials.util.script import ScriptRunner

class Script(ScriptRunner):
  '''A class for running the script.'''

  def __init__(self):
    '''Initialise the script.'''

    # The script usage
    usage = "usage: %prog [options] [param.phil] datablock.json"

    # Initialise the base class
    ScriptRunner.__init__(self, usage=usage)

    # Add a verbosity option
    self.config().add_option(
      "-v",
      dest="verbosity",
      action="count", default=1,
      help="set verbosity level; -vv gives verbosity level 2")

    # Add an options for spot finder output
    self.config().add_option(
      "--strong-filename",
      dest="strong_filename",
      type="str", default=None,
      help="The output filename for found spots")

  def main(self, params, options, args):
    '''Execute the script.'''
    from dials.util.command_line import Importer
    from dials.util.command_line import Command
    from time import time

    # Save the options
    self.options = options
    self.params = params

    st = time()

    # Preamble stuff
    print '*' * 80
    print ''
    print '                       mmmm   mmmmm    mm   m       mmmm            '
    print '                       #   "m   #      ##   #      #"   "           '
    print '                      m#mm  #   #     #  #  #      "#mmm            '
    print '                       #    #   #     #mm#  #          "#           '
    print '                       #mmm"  mm#mm  #    # #mmmmm "mmm#"           '
    print ''
    print 'Launching dials.process'
    print ''
    print 'The following tasks will be performed:'
    print ' 1) Strong spots will be found (dials.find_spots)'
    print ' 2) The strong spots will be indexed (dials.index)'
    print ' 3) A profile model will be created (dials.create_profile_model)'
    print ' 4) The reflections will be integrated (dials.integrate)'
    print ''
    print 'Please be patient, this may take a few minutes'
    print ''
    print '*' * 80
    print ''

    # Import stuff
    Command.start('Importing datablocks')
    importer = Importer(args, include=["images", "datablocks"])
    assert(len(importer.datablocks) == 1)
    datablock = importer.datablocks[0]
    Command.end('Imported datablocks')

    # Check the unhandled arguments
    if len(importer.unhandled_arguments) > 0:
      print '-' * 80
      print 'The following command line arguments weren\'t handled'
      for arg in importer.unhandled_arguments:
        print '  ' + arg
    print ''

    # Do the processing
    observed = self.find_spots(datablock)
    experiments, indexed = self.index(datablock, observed)
    profile = self.create_profile_model(experiments, indexed)
    integrated = self.integrate(experiments, profile, indexed)

    # Total Time
    print ""
    print "Total Time Taken = %f seconds" % (time() - st)

  def find_spots(self, datablock):
    from time import time
    from dials.array_family import flex
    from dials.util.command_line import Command
    st = time()

    print '*' * 80
    print 'Finding Strong Spots'
    print '*' * 80

    # Find the strong spots
    observed = flex.reflection_table.from_observations(datablock)

    # Save the reflections to file
    print '\n' + '-' * 80
    if self.options.strong_filename:
      Command.start('Saving {0} reflections to {1}'.format(
          len(observed), self.options.strong_filename))
      observed.as_pickle(self.options.strong_filename)
      Command.end('Saved {0} observed to {1}'.format(
          len(observed), self.options.strong_filename))

    print ''
    print 'Time Taken = %f seconds' % (time() - st)
    return observed

  def index(self, datablock, reflections):
    from dials.algorithms.indexing.indexer2 import master_phil_scope
    from libtbx.phil import command_line
    from time import time
    st = time()

    print '*' * 80
    print 'Indexing Strong Spots'
    print '*' * 80

    imagesets = datablock.extract_imagesets()
    if len(imagesets) > 1:
      raise RuntimeError("Only one imageset can be processed at a time")
    imageset = imagesets[0]

    cmd_line = command_line.argument_interpreter(master_params=master_phil_scope)
    working_phil = cmd_line.process_and_fetch(args=[])
    working_phil.show()

    gonio = imageset.get_goniometer()
    detector = imageset.get_detector()
    scan = imageset.get_scan()
    beam = imageset.get_beam()
    print detector
    print scan
    print gonio
    print beam

    params = working_phil.extract()
    params.refinement.reflections.use_all_reflections=True
    if params.method == "fft3d":
      from dials.algorithms.indexing.fft3d import indexer_fft3d as indexer
    elif params.method == "fft1d":
      from dials.algorithms.indexing.fft1d import indexer_fft1d as indexer
    elif params.method == "real_space_grid_search":
      from dials.algorithms.indexing.real_space_grid_search \
           import indexer_real_space_grid_search as indexer
    idxr = indexer(reflections, imageset, params=params)
    idxr.index()

    from dials.array_family import flex
    from dials.model.experiment.experiment_list import ExperimentListFactory
    indexed = flex.reflection_table.from_pickle("indexed.pickle")
    experiments = ExperimentListFactory.from_json_file("experiments.json")

    print ''
    print 'Time Taken = %f seconds' % (time() - st)
    return experiments, indexed

  def create_profile_model(self, experiments, reflections):
    from dials.algorithms.profile_model.profile_model import ProfileModel
    from time import time
    from dials.util.command_line import Command
    from math import pi
    st = time()

    print '*' * 80
    print 'Creating Profile Model'
    print '*' * 80

    print 'Starting with %d reflections' % len(reflections)

    from dials.array_family import flex
    Command.start('Removing invalid coordinates')
    xyz = reflections['xyzcal.mm']
    mask = flex.bool([x == (0, 0, 0) for x in xyz])
    reflections.del_selected(mask)
    Command.end('Removed invalid coordinates, %d remaining' % len(reflections))

    # Create the profile model
    profile_model = ProfileModel(experiments[0], reflections)
    sigma_b = profile_model.sigma_b() * 180.0 / pi
    sigma_m = profile_model.sigma_m() * 180.0 / pi
    print 'Sigma B: %f' % sigma_b
    print 'Sigma M: %f' % sigma_m

    # Write the parameters
    from dials.framework.registry import Registry
    registry = Registry()

    # Get the parameters
    params = registry.config().params()
    params.shoebox.sigma_b = sigma_b
    params.shoebox.sigma_m = sigma_m

    print ''
    print 'Time Taken = %f seconds' % (time() - st)
    return (sigma_b, sigma_m)

  def integrate(self, experiments, profile, indexed):
    from dials.algorithms.integration import Integrator
    from time import time
    from dials.framework.registry import Registry
    registry = Registry()
    assert(registry.params().shoebox.sigma_b > 0)
    assert(registry.params().shoebox.sigma_m > 0)

    st = time()

    print '*' * 80
    print 'Integrating Reflections'
    print '*' * 80

    # Get the integrator from the input parameters
    print 'Configurating integrator from input parameters'
    integrate = Integrator(self.params.shoebox.n_sigma,
                           self.params.shoebox.n_blocks,
                           self.params.integration.filter.by_zeta)

    # Intregate the sweep's reflections
    print 'Integrating reflections'
    reflections = integrate(experiments, reference=indexed, extracted=None)

    print ''
    print 'Time Taken = %f seconds' % (time() - st)
    return reflections

if __name__ == '__main__':
  script = Script()
  script.run()
<commit_msg>Updated and fixed a bit of dials.process<commit_after>#!/usr/bin/env python
#
# dials.process.py
#
#  Copyright (C) 2013 Diamond Light Source
#
#  Author: James Parkhurst
#
#  This code is distributed under the BSD license, a copy of which is
#  included in the root directory of this package.

from __future__ import division
from dials.util.script import ScriptRunner

class Script(ScriptRunner):
  '''A class for running the script.'''

  def __init__(self):
    '''Initialise the script.'''

    # The script usage
    usage = "usage: %prog [options] [param.phil] datablock.json"

    # Initialise the base class
    ScriptRunner.__init__(self, usage=usage)

    # Add a verbosity option
    self.config().add_option(
      "-v",
      dest="verbosity",
      action="count", default=1,
      help="set verbosity level; -vv gives verbosity level 2")

    # Add an options for spot finder output
    self.config().add_option(
      "--strong-filename",
      dest="strong_filename",
      type="str", default=None,
      help="The output filename for found spots")

  def main(self, params, options, args):
    '''Execute the script.'''
    from dials.util.command_line import Importer
    from dials.util.command_line import Command
    from time import time

    # Save the options
    self.options = options
    self.params = params

    st = time()

    # Preamble stuff
    print '*' * 80
    print ''
    print '                       mmmm   mmmmm    mm   m       mmmm            '
    print '                       #   "m   #      ##   #      #"   "           '
    print '                      m#mm  #   #     #  #  #      "#mmm            '
    print '                       #    #   #     #mm#  #          "#           '
    print '                       #mmm"  mm#mm  #    # #mmmmm "mmm#"           '
    print ''
    print 'Launching dials.process'
    print ''
    print 'The following tasks will be performed:'
    print ' 1) Strong spots will be found (dials.find_spots)'
    print ' 2) The strong spots will be indexed (dials.index)'
    print ' 3) The reflections will be integrated (dials.integrate)'
    print ''
    print 'Please be patient, this may take a few minutes'
    print ''
    print '*' * 80
    print ''

    # Import stuff
    Command.start('Importing datablocks')
    importer = Importer(args, include=["images", "datablocks"])
    assert(len(importer.datablocks) == 1)
    datablock = importer.datablocks[0]
    Command.end('Imported datablocks')

    from dxtbx.datablock import DataBlockDumper
    dump = DataBlockDumper(datablock)
    dump.as_json("datablock.json")

    # Check the unhandled arguments
    if len(importer.unhandled_arguments) > 0:
      print '-' * 80
      print 'The following command line arguments weren\'t handled'
      for arg in importer.unhandled_arguments:
        print '  ' + arg
    print ''

    # Do the processing
    observed = self.find_spots(datablock)
    experiments, indexed = self.index(datablock, observed)
    integrated = self.integrate(experiments, indexed)

    # Total Time
    print ""
    print "Total Time Taken = %f seconds" % (time() - st)

  def find_spots(self, datablock):
    from time import time
    from dials.array_family import flex
    from dials.util.command_line import Command
    st = time()

    print '*' * 80
    print 'Finding Strong Spots'
    print '*' * 80

    # Find the strong spots
    observed = flex.reflection_table.from_observations(datablock)

    # Save the reflections to file
    print '\n' + '-' * 80
    if self.options.strong_filename:
      Command.start('Saving {0} reflections to {1}'.format(
          len(observed), self.options.strong_filename))
      observed.as_pickle(self.options.strong_filename)
      Command.end('Saved {0} observed to {1}'.format(
          len(observed), self.options.strong_filename))

    observed.as_pickle("strong.pickle")

    print ''
    print 'Time Taken = %f seconds' % (time() - st)
    return observed

  def index(self, datablock, reflections):
    from dials.algorithms.indexing.indexer2 import master_phil_scope
    from libtbx.phil import command_line
    from time import time
    st = time()

    print '*' * 80
    print 'Indexing Strong Spots'
    print '*' * 80

    imagesets = datablock.extract_imagesets()
    if len(imagesets) > 1:
      raise RuntimeError("Only one imageset can be processed at a time")
    imageset = imagesets[0]

    cmd_line = command_line.argument_interpreter(master_params=master_phil_scope)
    working_phil = cmd_line.process_and_fetch(args=[])
    working_phil.show()

    gonio = imageset.get_goniometer()
    detector = imageset.get_detector()
    scan = imageset.get_scan()
    beam = imageset.get_beam()
    print detector
    print scan
    print gonio
    print beam

    params = working_phil.extract()
    if params.method == "fft3d":
      from dials.algorithms.indexing.fft3d import indexer_fft3d as indexer
    elif params.method == "fft1d":
      from dials.algorithms.indexing.fft1d import indexer_fft1d as indexer
    elif params.method == "real_space_grid_search":
      from dials.algorithms.indexing.real_space_grid_search \
           import indexer_real_space_grid_search as indexer
    idxr = indexer(reflections, imageset, params=params)
    idxr.index()

    from dials.array_family import flex
    from dials.model.experiment.experiment_list import ExperimentListFactory
    indexed = flex.reflection_table.from_pickle("indexed.pickle")
    experiments = ExperimentListFactory.from_json_file("experiments.json")

    print ''
    print 'Time Taken = %f seconds' % (time() - st)
    return experiments, indexed

  def integrate(self, experiments, indexed):
    from dials.algorithms.integration import Integrator
    from time import time
    from dials.util.command_line import Command

    st = time()

    print '*' * 80
    print 'Integrating Reflections'
    print '*' * 80

    from dials.array_family import flex
    Command.start('Removing invalid coordinates')
    xyz = indexed['xyzcal.mm']
    mask = flex.bool([x == (0, 0, 0) for x in xyz])
    indexed.del_selected(mask)
    Command.end('Removed invalid coordinates, %d remaining' % len(indexed))

    # Get the integrator from the input parameters
    print 'Configurating integrator from input parameters'
    integrate = Integrator(self.params.shoebox.n_sigma,
                           self.params.shoebox.n_blocks,
                           self.params.integration.filter.by_zeta)

    # Intregate the sweep's reflections
    print 'Integrating reflections'
    reflections = integrate(experiments, reference=indexed, extracted=None)

    reflections.as_pickle("integrated.pickle")

    print ''
    print 'Time Taken = %f seconds' % (time() - st)
    return reflections

if __name__ == '__main__':
  script = Script()
  script.run()
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# dials.process.py
#
#  Copyright (C) 2013 Diamond Light Source
#
#  Author: James Parkhurst
#
#  This code is distributed under the BSD license, a copy of which is
#  included in the root directory of this package.

from __future__ import division
from dials.util.script import ScriptRunner

class Script(ScriptRunner):
  '''A class for running the script.'''

  def __init__(self):
    '''Initialise the script.'''

    # The script usage
    usage = "usage: %prog [options] [param.phil] datablock.json"

    # Initialise the base class
    ScriptRunner.__init__(self, usage=usage)

    # Add a verbosity option
    self.config().add_option(
      "-v",
      dest="verbosity",
      action="count", default=1,
      help="set verbosity level; -vv gives verbosity level 2")

    # Add an options for spot finder output
    self.config().add_option(
      "--strong-filename",
      dest="strong_filename",
      type="str", default=None,
      help="The output filename for found spots")

  def main(self, params, options, args):
    '''Execute the script.'''
    from dials.util.command_line import Importer
    from dials.util.command_line import Command
    from time import time

    # Save the options
    self.options = options
    self.params = params

    st = time()

    # Preamble stuff
    print '*' * 80
    print ''
    print '                       mmmm   mmmmm    mm   m       mmmm            '
    print '                       #   "m   #      ##   #      #"   "           '
    print '                      m#mm  #   #     #  #  #      "#mmm            '
    print '                       #    #   #     #mm#  #          "#           '
    print '                       #mmm"  mm#mm  #    # #mmmmm "mmm#"           '
    print ''
    print 'Launching dials.process'
    print ''
    print 'The following tasks will be performed:'
    print ' 1) Strong spots will be found (dials.find_spots)'
    print ' 2) The strong spots will be indexed (dials.index)'
    print ' 3) The reflections will be integrated (dials.integrate)'
    print ''
    print 'Please be patient, this may take a few minutes'
    print ''
    print '*' * 80
    print ''

    # Import stuff
    Command.start('Importing datablocks')
    importer = Importer(args, include=["images", "datablocks"])
    assert(len(importer.datablocks) == 1)
    datablock = importer.datablocks[0]
    Command.end('Imported datablocks')

    from dxtbx.datablock import DataBlockDumper
    dump = DataBlockDumper(datablock)
    dump.as_json("datablock.json")

    # Check the unhandled arguments
    if len(importer.unhandled_arguments) > 0:
      print '-' * 80
      print 'The following command line arguments weren\'t handled'
      for arg in importer.unhandled_arguments:
        print '  ' + arg
    print ''

    # Do the processing
    observed = self.find_spots(datablock)
    experiments, indexed = self.index(datablock, observed)
    integrated = self.integrate(experiments, indexed)

    # Total Time
    print ""
    print "Total Time Taken = %f seconds" % (time() - st)

  def find_spots(self, datablock):
    from time import time
    from dials.array_family import flex
    from dials.util.command_line import Command
    st = time()

    print '*' * 80
    print 'Finding Strong Spots'
    print '*' * 80

    # Find the strong spots
    observed = flex.reflection_table.from_observations(datablock)

    # Save the reflections to file
    print '\n' + '-' * 80
    if self.options.strong_filename:
      Command.start('Saving {0} reflections to {1}'.format(
          len(observed), self.options.strong_filename))
      observed.as_pickle(self.options.strong_filename)
      Command.end('Saved {0} observed to {1}'.format(
          len(observed), self.options.strong_filename))

    observed.as_pickle("strong.pickle")

    print ''
    print 'Time Taken = %f seconds' % (time() - st)
    return observed

  def index(self, datablock, reflections):
    from dials.algorithms.indexing.indexer2 import master_phil_scope
    from libtbx.phil import command_line
    from time import time
    st = time()

    print '*' * 80
    print 'Indexing Strong Spots'
    print '*' * 80

    imagesets = datablock.extract_imagesets()
    if len(imagesets) > 1:
      raise RuntimeError("Only one imageset can be processed at a time")
    imageset = imagesets[0]

    cmd_line = command_line.argument_interpreter(master_params=master_phil_scope)
    working_phil = cmd_line.process_and_fetch(args=[])
    working_phil.show()

    gonio = imageset.get_goniometer()
    detector = imageset.get_detector()
    scan = imageset.get_scan()
    beam = imageset.get_beam()
    print detector
    print scan
    print gonio
    print beam

    params = working_phil.extract()
    if params.method == "fft3d":
      from dials.algorithms.indexing.fft3d import indexer_fft3d as indexer
    elif params.method == "fft1d":
      from dials.algorithms.indexing.fft1d import indexer_fft1d as indexer
    elif params.method == "real_space_grid_search":
      from dials.algorithms.indexing.real_space_grid_search \
           import indexer_real_space_grid_search as indexer
    idxr = indexer(reflections, imageset, params=params)
    idxr.index()

    from dials.array_family import flex
    from dials.model.experiment.experiment_list import ExperimentListFactory
    indexed = flex.reflection_table.from_pickle("indexed.pickle")
    experiments = ExperimentListFactory.from_json_file("experiments.json")

    print ''
    print 'Time Taken = %f seconds' % (time() - st)
    return experiments, indexed

  def integrate(self, experiments, indexed):
    from dials.algorithms.integration import Integrator
    from time import time
    from dials.util.command_line import Command

    st = time()

    print '*' * 80
    print 'Integrating Reflections'
    print '*' * 80

    from dials.array_family import flex
    Command.start('Removing invalid coordinates')
    xyz = indexed['xyzcal.mm']
    mask = flex.bool([x == (0, 0, 0) for x in xyz])
    indexed.del_selected(mask)
    Command.end('Removed invalid coordinates, %d remaining' % len(indexed))

    # Get the integrator from the input parameters
    print 'Configurating integrator from input parameters'
    integrate = Integrator(self.params.shoebox.n_sigma,
                           self.params.shoebox.n_blocks,
                           self.params.integration.filter.by_zeta)

    # Intregate the sweep's reflections
    print 'Integrating reflections'
    reflections = integrate(experiments, reference=indexed, extracted=None)

    reflections.as_pickle("integrated.pickle")

    print ''
    print 'Time Taken = %f seconds' % (time() - st)
    return reflections

if __name__ == '__main__':
  script = Script()
  script.run()
<commit_msg>insert a refinement step into dials.process<commit_after>#!/usr/bin/env python
#
# dials.process.py
#
#  Copyright (C) 2013 Diamond Light Source
#
#  Author: James Parkhurst
#
#  This code is distributed under the BSD license, a copy of which is
#  included in the root directory of this package.

from __future__ import division
from dials.util.script import ScriptRunner

class Script(ScriptRunner):
  '''A class for running the script.'''

  def __init__(self):
    '''Initialise the script.'''

    # The script usage
    usage = "usage: %prog [options] [param.phil] datablock.json"

    # Initialise the base class
    ScriptRunner.__init__(self, usage=usage)

    # Add a verbosity option
    self.config().add_option(
      "-v",
      dest="verbosity",
      action="count", default=1,
      help="set verbosity level; -vv gives verbosity level 2")

    # Add an options for spot finder output
    self.config().add_option(
      "--strong-filename",
      dest="strong_filename",
      type="str", default=None,
      help="The output filename for found spots")

  def main(self, params, options, args):
    '''Execute the script.'''
    from dials.util.command_line import Importer
    from dials.util.command_line import Command
    from time import time

    # Save the options
    self.options = options
    self.params = params

    st = time()

    # Preamble stuff
    print '*' * 80
    print ''
    print '                       mmmm   mmmmm    mm   m       mmmm            '
    print '                       #   "m   #      ##   #      #"   "           '
    print '                      m#mm  #   #     #  #  #      "#mmm            '
    print '                       #    #   #     #mm#  #          "#           '
    print '                       #mmm"  mm#mm  #    # #mmmmm "mmm#"           '
    print ''
    print 'Launching dials.process'
    print ''
    print 'The following tasks will be performed:'
    print ' 1) Strong spots will be found (dials.find_spots)'
    print ' 2) The strong spots will be indexed (dials.index)'
    print ' 3) The reflections will be integrated (dials.integrate)'
    print ''
    print 'Please be patient, this may take a few minutes'
    print ''
    print '*' * 80
    print ''

    # Import stuff
    Command.start('Importing datablocks')
    importer = Importer(args, include=["images", "datablocks"])
    assert(len(importer.datablocks) == 1)
    datablock = importer.datablocks[0]
    Command.end('Imported datablocks')

    from dxtbx.datablock import DataBlockDumper
    dump = DataBlockDumper(datablock)
    dump.as_json("datablock.json")

    # Check the unhandled arguments
    if len(importer.unhandled_arguments) > 0:
      print '-' * 80
      print 'The following command line arguments weren\'t handled'
      for arg in importer.unhandled_arguments:
        print '  ' + arg
    print ''

    # Do the processing
    observed = self.find_spots(datablock)
    experiments, indexed = self.index(datablock, observed)
    experiments = self.refine(experiments, indexed)
    integrated = self.integrate(experiments, indexed)

    # Total Time
    print ""
    print "Total Time Taken = %f seconds" % (time() - st)

  def find_spots(self, datablock):
    from time import time
    from dials.array_family import flex
    from dials.util.command_line import Command
    st = time()

    print '*' * 80
    print 'Finding Strong Spots'
    print '*' * 80

    # Find the strong spots
    observed = flex.reflection_table.from_observations(datablock)

    # Save the reflections to file
    print '\n' + '-' * 80
    if self.options.strong_filename:
      Command.start('Saving {0} reflections to {1}'.format(
          len(observed), self.options.strong_filename))
      observed.as_pickle(self.options.strong_filename)
      Command.end('Saved {0} observed to {1}'.format(
          len(observed), self.options.strong_filename))

    observed.as_pickle("strong.pickle")

    print ''
    print 'Time Taken = %f seconds' % (time() - st)
    return observed

  def index(self, datablock, reflections):
    from dials.algorithms.indexing.indexer2 import master_phil_scope
    from libtbx.phil import command_line, parse
    from time import time
    st = time()

    print '*' * 80
    print 'Indexing Strong Spots'
    print '*' * 80

    imagesets = datablock.extract_imagesets()
    if len(imagesets) > 1:
      raise RuntimeError("Only one imageset can be processed at a time")
    imageset = imagesets[0]

    cmd_line = command_line.argument_interpreter(master_params=master_phil_scope)
    # switch off scan-varying refinement for indexing, leaving it set (if
    # requested) for the refinement step
    extra_src = parse(
      "refinement.parameterisation.crystal.scan_varying=False")
    working_phil = cmd_line.process_and_fetch(args=[],extra_sources=[extra_src])
    working_phil.show()

    gonio = imageset.get_goniometer()
    detector = imageset.get_detector()
    scan = imageset.get_scan()
    beam = imageset.get_beam()
    print detector
    print scan
    print gonio
    print beam

    params = working_phil.extract()
    if params.method == "fft3d":
      from dials.algorithms.indexing.fft3d import indexer_fft3d as indexer
    elif params.method == "fft1d":
      from dials.algorithms.indexing.fft1d import indexer_fft1d as indexer
    elif params.method == "real_space_grid_search":
      from dials.algorithms.indexing.real_space_grid_search \
           import indexer_real_space_grid_search as indexer
    idxr = indexer(reflections, imageset, params=params)
    idxr.index()

    from dials.array_family import flex
    from dials.model.experiment.experiment_list import ExperimentListFactory
    indexed = flex.reflection_table.from_pickle("indexed.pickle")
    experiments = ExperimentListFactory.from_json_file("experiments.json")

    print ''
    print 'Time Taken = %f seconds' % (time() - st)
    return experiments, indexed

  def refine(self, experiments, centroids):
    from dials.algorithms.refinement import RefinerFactory
    from time import time
    st = time()

    refiner = RefinerFactory.from_parameters_data_experiments(
      self.params, centroids, experiments, self.options.verbosity)

    print '*' * 80
    print 'Refining Model'
    print '*' * 80

    refiner.run()
    experiments = refiner.get_experiments()

    print ''
    print 'Time Taken = %f seconds' % (time() - st)

    return experiments

  def integrate(self, experiments, indexed):
    from dials.algorithms.integration import Integrator
    from time import time
    from dials.util.command_line import Command

    st = time()

    print '*' * 80
    print 'Integrating Reflections'
    print '*' * 80

    from dials.array_family import flex
    Command.start('Removing invalid coordinates')
    xyz = indexed['xyzcal.mm']
    mask = flex.bool([x == (0, 0, 0) for x in xyz])
    indexed.del_selected(mask)
    Command.end('Removed invalid coordinates, %d remaining' % len(indexed))

    # Get the integrator from the input parameters
    print 'Configurating integrator from input parameters'
    integrate = Integrator(self.params.shoebox.n_sigma,
                           self.params.shoebox.n_blocks,
                           self.params.integration.filter.by_zeta)

    # Integrate the sweep's reflections
    print 'Integrating reflections'
    reflections = integrate(experiments, reference=indexed, extracted=None)

    reflections.as_pickle("integrated.pickle")

    print ''
    print 'Time Taken = %f seconds' % (time() - st)
    return reflections

if __name__ == '__main__':
  script = Script()
  script.run()
<|endoftext|>


<commit_before>import numpy as np
import tensorflow as tf
from tensorflow.python.framework import ops

from models.rbm_models import dbn
from utils import datasets

# #################### #
#   Flags definition   #
# #################### #
flags = tf.app.flags
FLAGS = flags.FLAGS

# Global configuration
flags.DEFINE_string('dataset', 'mnist', 'Which dataset to use. ["mnist", "cifar10", "custom"]')
flags.DEFINE_string('train_dataset', '', 'Path to train set .npy file.')
flags.DEFINE_string('train_labels', '', 'Path to train labels .npy file.')
flags.DEFINE_string('valid_dataset', '', 'Path to valid set .npy file.')
flags.DEFINE_string('valid_labels', '', 'Path to valid labels .npy file.')
flags.DEFINE_string('test_dataset', '', 'Path to test set .npy file.')
flags.DEFINE_string('test_labels', '', 'Path to test labels .npy file.')
flags.DEFINE_string('cifar_dir', '', 'Path to the cifar 10 dataset directory.')
flags.DEFINE_string('model_name', 'srbm', 'Name of the model.')
flags.DEFINE_boolean('do_pretrain', True, 'Whether or not pretrain the network.')

flags.DEFINE_integer('verbose', 0, 'Level of verbosity. 0 - silent, 1 - print accuracy.')
flags.DEFINE_string('main_dir', 'dbn/', 'Directory to store data relative to the algorithm.')
# RBMs layers specific parameters
flags.DEFINE_string('rbm_names', 'rbm', 'Name for the rbm stored_models.')
flags.DEFINE_string('layers', '256,', 'Comma-separated values for the layers in the sdae.')
flags.DEFINE_boolean('gauss_visible', False, 'Whether to use Gaussian units for the visible layer.')
flags.DEFINE_float('stddev', 0.1, 'Standard deviation for Gaussian visible units.')
flags.DEFINE_string('rbm_learning_rate', '0.01,', 'Initial learning rate.')
flags.DEFINE_string('rbm_num_epochs', '10,', 'Number of epochs.')
flags.DEFINE_string('rbm_batch_size', '10,', 'Size of each mini-batch.')
flags.DEFINE_string('rbm_gibbs_k', '1,', 'Gibbs sampling steps.')
# Supervised fine tuning parameters
flags.DEFINE_string('act_func', 'relu', 'Activation function.')
flags.DEFINE_float('learning_rate', 0.01, 'Learning rate.')
flags.DEFINE_float('momentum', 0.7, 'Momentum parameter.')
flags.DEFINE_integer('num_epochs', 10, 'Number of epochs.')
flags.DEFINE_integer('batch_size', 10, 'Size of each mini-batch.')
flags.DEFINE_string('opt', 'gradient_descent', 'Which optimizer to use.')
flags.DEFINE_string('loss_func', 'mean_squared', 'Loss function.')
flags.DEFINE_float('dropout', 1, 'Dropout parameter.')

# Conversion of Autoencoder layers parameters from string to their specific type
rbm_names = [_ for _ in FLAGS.rbm_names.split(',') if _]
layers = [int(_) for _ in FLAGS.layers.split(',') if _]
rbm_learning_rate = [float(_) for _ in FLAGS.rbm_learning_rate.split(',') if _]
rbm_num_epochs = [int(_) for _ in FLAGS.rbm_num_epochs.split(',') if _]
rbm_batch_size = [int(_) for _ in FLAGS.rbm_batch_size.split(',') if _]
rbm_gibbs_k = [float(_) for _ in FLAGS.rbm_gibbs_k.split(',') if _]

# Parameters normalization: if a parameter is not specified, it must be made of the same length of the others
dae_params = {'layers': layers,  'learning_rate': rbm_learning_rate, 'num_epochs': rbm_num_epochs,
              'batch_size': rbm_batch_size, 'gibbs_k': rbm_gibbs_k, 'rbm_names': rbm_names}

for p in dae_params:
    if len(dae_params[p]) != len(layers):
        # The current parameter is not specified by the user, should default it for all the layers
        dae_params[p] = [dae_params[p][0] for _ in layers]

# Parameters validation
assert FLAGS.dataset in ['mnist', 'cifar10', 'custom']
assert FLAGS.act_func in ['sigmoid', 'tanh', 'relu']
assert len(layers) > 0

if __name__ == '__main__':

    if FLAGS.dataset == 'mnist':

        # ################# #
        #   MNIST Dataset   #
        # ################# #

        trX, trY, vlX, vlY, teX, teY = datasets.load_mnist_dataset(mode='supervised')

    elif FLAGS.dataset == 'cifar10':

        # ################### #
        #   Cifar10 Dataset   #
        # ################### #

        trX, trY, teX, teY = datasets.load_cifar10_dataset(FLAGS.cifar_dir, mode='supervised')
        vlX = teX[:5000]  # Validation set is the first half of the test set
        vlY = teY[:5000]

    elif FLAGS.dataset == 'custom':

        # ################## #
        #   Custom Dataset   #
        # ################## #

        def load_from_np(dataset_path):
            if dataset_path != '':
                return np.load(dataset_path)
            else:
                return None

        trX, trY = load_from_np(FLAGS.train_dataset), load_from_np(FLAGS.train_labels)
        vlX, vlY = load_from_np(FLAGS.valid_dataset), load_from_np(FLAGS.valid_labels)
        teX, teY = load_from_np(FLAGS.test_dataset), load_from_np(FLAGS.test_labels)

    else:
        trX, trY, vlX, vlY, teX, teY = None, None, None, None, None, None

    # Create the object
    srbm = dbn.DBN(
        model_name=FLAGS.model_name, rbm_names=dae_params['rbm_names'], do_pretrain=FLAGS.do_pretrain,
        layers=dae_params['layers'], dataset=FLAGS.dataset, main_dir=FLAGS.main_dir, act_func=FLAGS.act_func,
        rbm_learning_rate=dae_params['learning_rate'], rbm_gibbs_k=dae_params['gibbs_k'],
        verbose=FLAGS.verbose, rbm_num_epochs=dae_params['num_epochs'], momentum=FLAGS.momentum,
        rbm_batch_size=dae_params['batch_size'], learning_rate=FLAGS.learning_rate, num_epochs=FLAGS.num_epochs,
        batch_size=FLAGS.batch_size, opt=FLAGS.opt, loss_func=FLAGS.loss_func, dropout=FLAGS.dropout,
        gauss_visible=FLAGS.gauss_visible, stddev=FLAGS.stddev)

    # Fit the model (unsupervised pretraining)
    srbm.pretrain(trX, vlX)

    ops.reset_default_graph()

    # finetuning
    print('Start deep belief net finetuning...')
    srbm.build_model(trX.shape[1], trY.shape[1])
    srbm.fit(trX, trY, vlX, vlY)

    params = srbm.get_model_parameters()

    np.savez('data/dbn/{}_W_params'.format(FLAGS.model_name), params['W'])
    np.savez('data/dbn/{}_bh_params'.format(FLAGS.model_name), params['bh'])
    np.savez('data/dbn/{}_bv_params'.format(FLAGS.model_name), params['bv'])

    # Test the model
    print('Test set accuracy: {}'.format(srbm.predict(teX, teY)))
<commit_msg>fixed little bug in run_dbn.py<commit_after>import numpy as np
import tensorflow as tf
from tensorflow.python.framework import ops

from models.rbm_models import dbn
from utils import datasets

# #################### #
#   Flags definition   #
# #################### #
flags = tf.app.flags
FLAGS = flags.FLAGS

# Global configuration
flags.DEFINE_string('dataset', 'mnist', 'Which dataset to use. ["mnist", "cifar10", "custom"]')
flags.DEFINE_string('train_dataset', '', 'Path to train set .npy file.')
flags.DEFINE_string('train_labels', '', 'Path to train labels .npy file.')
flags.DEFINE_string('valid_dataset', '', 'Path to valid set .npy file.')
flags.DEFINE_string('valid_labels', '', 'Path to valid labels .npy file.')
flags.DEFINE_string('test_dataset', '', 'Path to test set .npy file.')
flags.DEFINE_string('test_labels', '', 'Path to test labels .npy file.')
flags.DEFINE_string('cifar_dir', '', 'Path to the cifar 10 dataset directory.')
flags.DEFINE_string('model_name', 'srbm', 'Name of the model.')
flags.DEFINE_boolean('do_pretrain', True, 'Whether or not pretrain the network.')

flags.DEFINE_integer('verbose', 0, 'Level of verbosity. 0 - silent, 1 - print accuracy.')
flags.DEFINE_string('main_dir', 'dbn/', 'Directory to store data relative to the algorithm.')
# RBMs layers specific parameters
flags.DEFINE_string('rbm_names', 'rbm', 'Name for the rbm stored_models.')
flags.DEFINE_string('layers', '256,', 'Comma-separated values for the layers in the sdae.')
flags.DEFINE_boolean('gauss_visible', False, 'Whether to use Gaussian units for the visible layer.')
flags.DEFINE_float('stddev', 0.1, 'Standard deviation for Gaussian visible units.')
flags.DEFINE_string('rbm_learning_rate', '0.01,', 'Initial learning rate.')
flags.DEFINE_string('rbm_num_epochs', '10,', 'Number of epochs.')
flags.DEFINE_string('rbm_batch_size', '10,', 'Size of each mini-batch.')
flags.DEFINE_string('rbm_gibbs_k', '1,', 'Gibbs sampling steps.')
# Supervised fine tuning parameters
flags.DEFINE_string('act_func', 'relu', 'Activation function.')
flags.DEFINE_float('learning_rate', 0.01, 'Learning rate.')
flags.DEFINE_float('momentum', 0.7, 'Momentum parameter.')
flags.DEFINE_integer('num_epochs', 10, 'Number of epochs.')
flags.DEFINE_integer('batch_size', 10, 'Size of each mini-batch.')
flags.DEFINE_string('opt', 'gradient_descent', 'Which optimizer to use.')
flags.DEFINE_string('loss_func', 'mean_squared', 'Loss function.')
flags.DEFINE_float('dropout', 1, 'Dropout parameter.')

# Conversion of Autoencoder layers parameters from string to their specific type
rbm_names = [_ for _ in FLAGS.rbm_names.split(',') if _]
layers = [int(_) for _ in FLAGS.layers.split(',') if _]
rbm_learning_rate = [float(_) for _ in FLAGS.rbm_learning_rate.split(',') if _]
rbm_num_epochs = [int(_) for _ in FLAGS.rbm_num_epochs.split(',') if _]
rbm_batch_size = [int(_) for _ in FLAGS.rbm_batch_size.split(',') if _]
rbm_gibbs_k = [float(_) for _ in FLAGS.rbm_gibbs_k.split(',') if _]

# Parameters normalization: if a parameter is not specified, it must be made of the same length of the others
dae_params = {'layers': layers,  'learning_rate': rbm_learning_rate, 'num_epochs': rbm_num_epochs,
              'batch_size': rbm_batch_size, 'gibbs_k': rbm_gibbs_k, 'rbm_names': rbm_names}

for p in dae_params:
    if len(dae_params[p]) != len(layers):
        # The current parameter is not specified by the user, should default it for all the layers
        dae_params[p] = [dae_params[p][0] for _ in layers]

# Parameters validation
assert FLAGS.dataset in ['mnist', 'cifar10', 'custom']
assert FLAGS.act_func in ['sigmoid', 'tanh', 'relu']
assert len(layers) > 0

if __name__ == '__main__':

    if FLAGS.dataset == 'mnist':

        # ################# #
        #   MNIST Dataset   #
        # ################# #

        trX, trY, vlX, vlY, teX, teY = datasets.load_mnist_dataset(mode='supervised')

    elif FLAGS.dataset == 'cifar10':

        # ################### #
        #   Cifar10 Dataset   #
        # ################### #

        trX, trY, teX, teY = datasets.load_cifar10_dataset(FLAGS.cifar_dir, mode='supervised')
        vlX = teX[:5000]  # Validation set is the first half of the test set
        vlY = teY[:5000]

    elif FLAGS.dataset == 'custom':

        # ################## #
        #   Custom Dataset   #
        # ################## #

        def load_from_np(dataset_path):
            if dataset_path != '':
                return np.load(dataset_path)
            else:
                return None

        trX, trY = load_from_np(FLAGS.train_dataset), load_from_np(FLAGS.train_labels)
        vlX, vlY = load_from_np(FLAGS.valid_dataset), load_from_np(FLAGS.valid_labels)
        teX, teY = load_from_np(FLAGS.test_dataset), load_from_np(FLAGS.test_labels)

    else:
        trX, trY, vlX, vlY, teX, teY = None, None, None, None, None, None

    # Create the object
    srbm = dbn.DBN(
        model_name=FLAGS.model_name, rbm_names=dae_params['rbm_names'], do_pretrain=FLAGS.do_pretrain,
        layers=dae_params['layers'], dataset=FLAGS.dataset, main_dir=FLAGS.main_dir, act_func=FLAGS.act_func,
        rbm_learning_rate=dae_params['learning_rate'], rbm_gibbs_k=dae_params['gibbs_k'],
        verbose=FLAGS.verbose, rbm_num_epochs=dae_params['num_epochs'], momentum=FLAGS.momentum,
        rbm_batch_size=dae_params['batch_size'], learning_rate=FLAGS.learning_rate, num_epochs=FLAGS.num_epochs,
        batch_size=FLAGS.batch_size, opt=FLAGS.opt, loss_func=FLAGS.loss_func, dropout=FLAGS.dropout,
        gauss_visible=FLAGS.gauss_visible, stddev=FLAGS.stddev)

    # Fit the model (unsupervised pretraining)
    if FLAGS.do_pretrain:
        srbm.pretrain(trX, vlX)

    ops.reset_default_graph()

    # finetuning
    print('Start deep belief net finetuning...')
    srbm.build_model(trX.shape[1], trY.shape[1])
    srbm.fit(trX, trY, vlX, vlY)

    params = srbm.get_model_parameters()

    np.savez('data/dbn/{}_W_params'.format(FLAGS.model_name), params['W'])
    np.savez('data/dbn/{}_bh_params'.format(FLAGS.model_name), params['bh'])
    np.savez('data/dbn/{}_bv_params'.format(FLAGS.model_name), params['bv'])

    # Test the model
    print('Test set accuracy: {}'.format(srbm.predict(teX, teY)))
<|endoftext|>


<commit_before>#!/usr/bin/python

import os
import re
import string

import lldb
import fblldbbase as fb
import fblldbobjcruntimehelpers as runtimeHelpers

def lldbcommands():
  return [
    FBPrintMethods()
  ]

class FBPrintMethods(fb.FBCommand):
  def name(self):
    return 'pmethods'

  def description(self):
    return 'Print the class instance methods.'

  def options(self):
    return [
      fb.FBCommandArgument(short='-a', long='--address', arg='showaddr', help='Print the implementation address of the method', default=False, boolean=True),
      fb.FBCommandArgument(short='-i', long='--instance', arg='insmethod', help='Print the instance methods', default=False, boolean=True),
      fb.FBCommandArgument(short='-c', long='--class', arg='clsmethod', help='Print the class methods', default=False, boolean=True)
    ]

  def args(self):
    return [ fb.FBCommandArgument(arg='class or instance', type='id or Class', help='an Objective-C Class.') ]

  def run(self, arguments, options):
    cls = arguments[0]
    if not isClassObject(cls):
        cls = runtimeHelpers.object_getClass(cls)
        if not isClassObject(cls):
            raise Exception('Invalid argument. Please specify an instance or a Class.')

    if options.clsmethod:
      print 'Class Methods:'
      printClassMethods(cls, options.showaddr)

    if options.insmethod:
      print '\nInstance Methods:'
      printInstanceMethods(cls, options.showaddr)

    if not options.clsmethod and not options.insmethod:
      print 'Class Methods:'
      printClassMethods(cls, options.showaddr)
      print '\nInstance Methods:'
      printInstanceMethods(cls, options.showaddr)

def isClassObject(arg):
    return runtimeHelpers.class_isMetaClass(runtimeHelpers.object_getClass(arg))

def printInstanceMethods(cls, showaddr=False, prefix='-'):
    ocarray = instanceMethodsOfClass(cls)
    if not ocarray:
      print "No instance methods were found."
      return

    methodAddrs = covertOCArrayToPyArray(ocarray)
    methods = []
    for i in methodAddrs:
      method = createMethodFromOCMethod(i)
      if method is not None:
        methods.append(method)
        if showaddr:
          print prefix + ' ' + method.prettyPrint() + ' ' + method.imp
        else:
          print prefix + ' ' + method.prettyPrint()

def printClassMethods(cls, showaddr=False):
    printInstanceMethods(runtimeHelpers.object_getClass(cls), showaddr, '+')

# Use numberWithLongLong: rather than -[NSString stringWithFormat:]
# since evaluateExpression doesn't work with variable arguments.
# I remove the free(methods) because it would cause evaluateExpressionValue fail some time.
def instanceMethodsOfClass(klass):
  tmpString = """
    unsigned int outCount;
    void **methods = (void **)class_copyMethodList((Class)$cls, &outCount);
    NSMutableArray *result = [NSMutableArray array];
    for (int i = 0; i < outCount; i++) {
      NSNumber *num = (NSNumber *)[NSNumber numberWithLongLong:(long long)methods[i]];
      [result addObject:num];
    }
    id ret = result.count ? [result copy] : nil;
    ret;
  """

  command = string.Template(tmpString).substitute(cls=klass)
  command = '({' + command + '})'
  ret = fb.evaluateExpressionValue(command)
  if not ret.GetError().Success():
    return None

  ret = ret.GetValue()
  if int(ret, 16) == 0: # return nil
    ret = None
  return ret

# OC array only can hold id,
# @return an array whose instance type is str of the oc object`s address

def covertOCArrayToPyArray(oc_array):
  is_array = fb.evaluateBooleanExpression("[{} isKindOfClass:[NSArray class]]".format(oc_array))
  if not is_array:
    return None

  result = []
  count = fb.evaluateExpression("(int)[{} count]".format(oc_array))

  for i in range(int(count)):
    value = fb.evaluateExpression("(id)[{} objectAtIndex:{}]".format(oc_array, i))
    value = fb.evaluateExpression("(long long)[{} longLongValue]".format(value))
    result.append(value)

  return result

class Method:

  encodeMap = {
    'c': 'char',
    'i': 'int',
    's': 'short',
    'l': 'long',
    'q': 'long long',

    'C': 'unsigned char',
    'I': 'unsigned int',
    'S': 'unsigned short',
    'L': 'unsigned long',
    'Q': 'unsigned long long',

    'f': 'float',
    'd': 'double',
    'B': 'bool',
    'v': 'void',
    '*': 'char *',
    '@': 'id',
    '#': 'Class',
    ':': 'SEL',
  }

  def __init__(self, name, type_encoding, imp, oc_method):
    self.name = name
    self.type = type_encoding
    self.imp = imp
    self.oc_method = self.toHex(oc_method)

  def prettyPrint(self):
    argnum = fb.evaluateIntegerExpression("method_getNumberOfArguments({})".format(self.oc_method))
    names = self.name.split(':')

    # the argnum count must be bigger then 2, index 0 for self, index 1 for SEL
    for i in range(2, argnum):
      arg_type = fb.evaluateCStringExpression("(char *)method_copyArgumentType({}, {})".format(self.oc_method, i))
      names[i-2] = names[i-2] + ":(" +  self.decode(arg_type) + ")arg" + str(i-2)

    string = " ".join(names)

    ret_type = fb.evaluateCStringExpression("(char *)method_copyReturnType({})".format(self.oc_method))
    return "({}){}".format(self.decode(ret_type), string)

  def decode(self, type):
    ret = type
    if type in Method.encodeMap:
      ret = Method.encodeMap[type]
    return ret

  def toHex(self, addr):
    return addr

  def __str__(self):
    return "<Method:" + self.oc_method + "> " + self.name + " --- " + self.type + " --- " + self.imp

def createMethodFromOCMethod(method):
  process = lldb.debugger.GetSelectedTarget().GetProcess()
  error = lldb.SBError()

  nameValue = fb.evaluateExpression("(char *)method_getName({})".format(method))
  name = process.ReadCStringFromMemory(int(nameValue, 16), 256, error)

  if not error.Success():
    print error
    return None

  typeEncodingValue = fb.evaluateExpression("(char *)method_getTypeEncoding({})".format(method))
  type_encoding = process.ReadCStringFromMemory(int(typeEncodingValue, 16), 256, error)

  if not error.Success():
    print error
    return None

  imp = fb.evaluateExpression("(void *)method_getImplementation({})".format(method))
  return Method(name, type_encoding, imp, method)
<commit_msg>change the impletation of pmethods with json<commit_after>#!/usr/bin/python

import os
import re
import string
import json
import sys

import lldb
import fblldbbase as fb
import fblldbobjcruntimehelpers as runtimeHelpers

def lldbcommands():
  return [
    FBPrintMethods()
  ]

class FBPrintMethods(fb.FBCommand):
  def name(self):
    return 'pmethods'

  def description(self):
    return 'Print the class instance methods.'

  def options(self):
    return [
      fb.FBCommandArgument(short='-a', long='--address', arg='showaddr', help='Print the implementation address of the method', default=False, boolean=True),
      fb.FBCommandArgument(short='-i', long='--instance', arg='insmethod', help='Print the instance methods', default=False, boolean=True),
      fb.FBCommandArgument(short='-c', long='--class', arg='clsmethod', help='Print the class methods', default=False, boolean=True)
    ]

  def args(self):
    return [ fb.FBCommandArgument(arg='class or instance', type='id or Class', help='an Objective-C Class.') ]

  def run(self, arguments, options):
    cls = arguments[0]
    if not isClassObject(cls):
        cls = runtimeHelpers.object_getClass(cls)
        if not isClassObject(cls):
            raise Exception('Invalid argument. Please specify an instance or a Class.')

    if options.clsmethod:
      print 'Class Methods:'
      printClassMethods(cls, options.showaddr)

    if options.insmethod:
      print '\nInstance Methods:'
      printInstanceMethods(cls, options.showaddr)

    if not options.clsmethod and not options.insmethod:
      print 'Class Methods:'
      printClassMethods(cls, options.showaddr)
      print '\nInstance Methods:'
      printInstanceMethods(cls, options.showaddr)

def isClassObject(arg):
    return runtimeHelpers.class_isMetaClass(runtimeHelpers.object_getClass(arg))

def printInstanceMethods(cls, showaddr=False, prefix='-'):

    json_method_array = get_oc_methods_json(cls)
    if json_method_array:
      for m in json_method_array:
        method = Method(m)

        if showaddr:
          print prefix + ' ' + method.prettyPrintString() + ' ' + str(method.imp)
        else:
          print prefix + ' ' + method.prettyPrintString()

def printClassMethods(cls, showaddr=False):
    printInstanceMethods(runtimeHelpers.object_getClass(cls), showaddr, '+')

# Notice that evaluateExpression doesn't work with variable arguments. such as -[NSString stringWithFormat:]
# I remove the free(methods) because it would cause evaluateExpressionValue fail some time.
def get_oc_methods_json(klass):
  tmpString = """
    unsigned int outCount;
    Method *methods = (Method *)class_copyMethodList((Class)$cls, &outCount);
    NSMutableArray *result2 = (id)[NSMutableArray array];

    for (int i = 0; i < outCount; i++) {

        NSMutableDictionary *m = (id)[NSMutableDictionary dictionary];

        SEL name = (SEL)method_getName(methods[i]);
        [m setObject:(id)NSStringFromSelector(name) forKey:@"name"];

        char * ecodeing = (char *)method_getTypeEncoding(methods[i]);
        [m setObject:(id)[NSString stringWithUTF8String:ecodeing] forKey:@"type_encoding"];

        NSMutableArray *types = (id)[NSMutableArray array];
        NSInteger args = (NSInteger)method_getNumberOfArguments(methods[i]);
        for (int idx = 0; idx < args; idx++) {
            char *type = (char *)method_copyArgumentType(methods[i], idx);
            [types addObject:(id)[NSString stringWithUTF8String:type]];
        }
        [m setObject:types forKey:@"peremeters_type"];

        char *ret_type = (char *)method_copyReturnType(methods[i]);
        [m setObject:(id)[NSString stringWithUTF8String:ret_type] forKey:@"return_type"];

        long imp = (long)method_getImplementation(methods[i]);
        [m setObject:[NSNumber numberWithLongLong:imp] forKey:@"impletation"];

        [result2 addObject:m];
    }
    RET(result2);
  """
  command = string.Template(tmpString).substitute(cls=klass)
  return eval(command)

class Method:

  encodeMap = {
    'c': 'char',
    'i': 'int',
    's': 'short',
    'l': 'long',
    'q': 'long long',

    'C': 'unsigned char',
    'I': 'unsigned int',
    'S': 'unsigned short',
    'L': 'unsigned long',
    'Q': 'unsigned long long',

    'f': 'float',
    'd': 'double',
    'B': 'bool',
    'v': 'void',
    '*': 'char *',
    '@': 'id',
    '#': 'Class',
    ':': 'SEL',
  }

  def __init__(self, json):
    self.name = json['name']
    self.type_encoding = json['type_encoding']
    self.peremeters_type = json['peremeters_type']
    self.return_type = json['return_type']
    self.imp = self.toHex(json['impletation'])

  def prettyPrintString(self):
    argnum = len(self.peremeters_type)
    names = self.name.split(':')

    # the argnum count must be bigger then 2, index 0 for self, index 1 for SEL
    for i in range(2, argnum):
      arg_type = self.peremeters_type[i]
      names[i-2] = names[i-2] + ":(" +  self.decode(arg_type) + ")arg" + str(i-2)

    string = " ".join(names)
    return "({}){}".format(self.decode(self.return_type), string)

  def decode(self, type):
    ret = type
    if type in Method.encodeMap:
      ret = Method.encodeMap[type]
    return ret

  def toHex(self, addr):
    return hex(addr)

  def __str__(self):
    return "<Method:" + self.oc_method + "> " + self.name + " --- " + self.type + " --- " + self.imp

# For eval
# TODO: move to base

RET_MACRO = """
#define IS_JSON_OBJ(obj)\
    (obj != nil && ((bool)[NSJSONSerialization isValidJSONObject:obj] ||\
    (bool)[obj isKindOfClass:[NSString class]] ||\
    (bool)[obj isKindOfClass:[NSNumber class]]))
#define RET(ret) ({\
    if (!IS_JSON_OBJ(ret)) {\
        (void)[NSException raise:@"RET error" format:@"Invalied return type"];\
    }\
    NSDictionary *__dict = @{@"return":ret};\
    NSData *__data = (id)[NSJSONSerialization dataWithJSONObject:__dict options:1 error:NULL];\
    NSString *__str = (id)[[NSString alloc] initWithData:__data encoding:4];\
    (char *)[__str UTF8String];})
#define RETCString(ret)\
    ({NSString *___cstring_ret = [NSString stringWithUTF8String:ret];\
    RET(___cstring_ret);})
"""

def check_expr(expr):
    return expr.strip().split('\n')[-1].find('RET') != -1

# evaluates a batch of OC expression, the last expression must contain a RET marco
# and it will automatic transform the RET OC object to python object
# Example:
#       >>> fblldbbase.eval('NSString *str = @"hello world"; RET(@{@"key": str});')
#       {u'key': u'hello world'}
#
def eval(expr):
    if not check_expr(expr):
        raise Exception("expr not Invalied, the last expression should include a RET* marco")

    command = "({" + RET_MACRO + '\n' + expr + "})"
    ret = fb.evaluateExpressionValue(command, True)
    if not ret.GetError().Success():
        raise Exception("eval expression error occur")
    else:
        process = lldb.debugger.GetSelectedTarget().GetProcess()
        error = lldb.SBError()
        ret = process.ReadCStringFromMemory(int(ret.GetValue(), 16), 2**20, error)
        if not error.Success():
            print error
            return None
        else:
            ret = json.loads(ret)
            return ret['return']
<|endoftext|>


<commit_before>#
# Copyright (c) 2004-2005 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/cpl.php.
#
# This program is distributed in the hope that it will be useful, but
# without any waranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#
"""
Provides the output for the "conary showcs" command
"""

import itertools
import time
import sys

#conary
from conary import conaryclient
from conary.conaryclient import cmdline
from conary import display, query
from conary import files
from conary.lib import log
from conary.lib.sha1helper import sha1ToString
from conary.repository import repository, trovesource

def usage():
    print "conary showcs   <changeset> [trove[=version]]"
    print "showcs flags:   "
    print "                --full-versions   Print full version strings instead of "
    print "                                  attempting to shorten them"
    print "                --deps            Print dependency information about the troves"
    print "                --ls              (Recursive) list file contents"
    print "                --show-changes    For modifications, show the old "
    print "                                  file version next to new one"
    print "                --tags            Show tagged files (use with ls to "
    print "                                  show tagged and untagged)"
    print "                --sha1s           Show sha1s for files"
    print "                --ids             Show fileids"
    print "                --all             Combine above tags except show-changes"
    print ""

def displayChangeSet(db, cs, troveSpecs, cfg, ls = False, tags = False,
                     showChanges=False, info=False,
                     all=False, deps=False, sha1s=False, ids=False,
                     asJob=False):

    if all:
        ls = deps = sha1s = ids = tags = True

    client = conaryclient.ConaryClient(cfg)
    repos = client.getRepos()

    if not asJob and not showChanges and cs.isAbsolute():
        changeSetSource = trovesource.ChangesetFilesTroveSource(None)
        changeSetSource.addChangeSet(cs)

        if not troveSpecs:
            troveTups = cs.getPrimaryTroveList()
            primary = True
            namesOnly = True
        else:
            troveTups, namesOnly, primary  = query.getTrovesToDisplay(
                                                         changeSetSource,
                                                         troveSpecs)

        dcfg = display.DisplayConfig(changeSetSource, ls=ls, ids=ids,
                             sha1s=sha1s, fullVersions=cfg.fullVersions,
                             tags=tags, deps=deps, info=info,
                             showFlavors=cfg.fullFlavors,
                             iterChildren=not namesOnly)
        if primary:
            dcfg.setPrimaryTroves(set(troveTups))
        formatter = display.TroveFormatter(dcfg)
        display.displayTroves(dcfg, formatter, troveTups)
    else:
        changeSetSource = trovesource.ChangeSetJobSource(repos,
                                             trovesource.stack(db, repos))
        changeSetSource.addChangeSet(cs)

        jobs, namesOnly = getJobsToDisplay(changeSetSource, troveSpecs)

        dcfg = display.JobDisplayConfig(changeSetSource,
                                        ls=ls, ids=ids, sha1s=sha1s,
                                        info=info, tags=tags, deps=deps,
                                        showChanges=showChanges,
                                        iterChildren=not namesOnly)

        formatter = display.JobFormatter(dcfg)
        display.displayJobs(dcfg, formatter, jobs)

def getJobsToDisplay(jobSource, jobSpecs):
    namesOnly = True
    if jobSpecs:
        jobSpecs = cmdline.parseChangeList(jobSpecs, allowChangeSets=False)
    else:
        jobSpecs = []

    for jobSpec in jobSpecs:
        if jobSpec[1] != (None, None) or jobSpec[2] != (None, None):
            namesOnly = False

    if jobSpecs:
        results = jobSource.findJobs(jobSpecs)
        jobs = list(itertools.chain(*results.itervalues()))
    else:
        jobs = list(jobSource.iterAllJobs())

    return jobs, namesOnly
<commit_msg>* don't display pathIds or sha1s when using --all in showcs.   * default to listing all troves in a changeset if there are no primary troves * fall back to the repository if a particular referenced trove is not in an   absolute changeset<commit_after>#
# Copyright (c) 2004-2005 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/cpl.php.
#
# This program is distributed in the hope that it will be useful, but
# without any waranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#
"""
Provides the output for the "conary showcs" command
"""

import itertools
import time
import sys

#conary
from conary import conaryclient
from conary.conaryclient import cmdline
from conary import display, query
from conary import files
from conary.lib import log
from conary.lib.sha1helper import sha1ToString
from conary.repository import repository, trovesource

def usage():
    print "conary showcs   <changeset> [trove[=version]]"
    print "showcs flags:   "
    print "                --full-versions   Print full version strings instead of "
    print "                                  attempting to shorten them"
    print "                --deps            Print dependency information about the troves"
    print "                --ls              (Recursive) list file contents"
    print "                --show-changes    For modifications, show the old "
    print "                                  file version next to new one"
    print "                --tags            Show tagged files (use with ls to "
    print "                                  show tagged and untagged)"
    print "                --sha1s           Show sha1s for files"
    print "                --ids             Show fileids"
    print "                --all             Combine above tags except show-changes"
    print ""

def displayChangeSet(db, cs, troveSpecs, cfg, ls = False, tags = False,
                     showChanges=False, info=False,
                     all=False, deps=False, sha1s=False, ids=False,
                     asJob=False):

    if all:
        ls = deps = tags = True

    client = conaryclient.ConaryClient(cfg)
    repos = client.getRepos()

    if not asJob and not showChanges and cs.isAbsolute():
        changeSetSource = trovesource.ChangesetFilesTroveSource(None)
        changeSetSource.addChangeSet(cs)

        if not troveSpecs:
            troveTups = cs.getPrimaryTroveList()
            primary = True
            namesOnly = True
            if not troveTups:
                log.warning('No primary troves in changeset, listing all troves')
                troveTups = [(x.getName(), x.getNewVersion(), x.getNewFlavor())\
                                            for x in cs.iterNewTroveList()]
        else:
            troveTups, namesOnly, primary  = query.getTrovesToDisplay(
                                                         changeSetSource,
                                                         troveSpecs)
        querySource = trovesource.stack(changeSetSource, client.getRepos())

        dcfg = display.DisplayConfig(querySource, ls=ls, ids=ids,
                             sha1s=sha1s, fullVersions=cfg.fullVersions,
                             tags=tags, deps=deps, info=info,
                             showFlavors=cfg.fullFlavors,
                             iterChildren=not namesOnly)
        if primary:
            dcfg.setPrimaryTroves(set(troveTups))
        formatter = display.TroveFormatter(dcfg)
        display.displayTroves(dcfg, formatter, troveTups)
    else:
        changeSetSource = trovesource.ChangeSetJobSource(repos,
                                             trovesource.stack(db, repos))
        changeSetSource.addChangeSet(cs)

        jobs, namesOnly = getJobsToDisplay(changeSetSource, troveSpecs)

        dcfg = display.JobDisplayConfig(changeSetSource,
                                        ls=ls, ids=ids, sha1s=sha1s,
                                        info=info, tags=tags, deps=deps,
                                        showChanges=showChanges,
                                        iterChildren=not namesOnly)

        formatter = display.JobFormatter(dcfg)
        display.displayJobs(dcfg, formatter, jobs)

def getJobsToDisplay(jobSource, jobSpecs):
    namesOnly = True
    if jobSpecs:
        jobSpecs = cmdline.parseChangeList(jobSpecs, allowChangeSets=False)
    else:
        jobSpecs = []

    for jobSpec in jobSpecs:
        if jobSpec[1] != (None, None) or jobSpec[2] != (None, None):
            namesOnly = False

    if jobSpecs:
        results = jobSource.findJobs(jobSpecs)
        jobs = list(itertools.chain(*results.itervalues()))
    else:
        jobs = list(jobSource.iterAllJobs())

    return jobs, namesOnly
<|endoftext|>


<commit_before>__version__ = "2.5.0"
<commit_msg>Bump version<commit_after>__version__ = "2.5.2"
<|endoftext|>


<commit_before>from anytree import NodeMixin, PreOrderIter, RenderTree, ContStyle

from devito.ir.iet.nodes import ArrayCast, Call, Callable, Expression, List
from devito.ir.iet.scheduler import iet_insert_C_decls
from devito.ir.iet.utils import derive_parameters
from devito.ir.iet.visitors import FindSymbols, FindNodes
from devito.tools import as_tuple

__all__ = ['make_efunc', 'EFuncNode']

class ElementalFunction(Callable):

    """
    A Callable performing a computation over an abstract convex iteration space.

    A Call to an ElementalFunction will "instantiate" such iteration space by
    supplying bounds and step increment for each Dimension listed in
    ``dynamic_parameters``.
    """

    def __init__(self, name, body, retval, parameters=None, prefix=('static', 'inline'),
                 dynamic_parameters=None):
        super(ElementalFunction, self).__init__(name, body, retval, parameters, prefix)

        self._mapper = {}
        for i in as_tuple(dynamic_parameters):
            if i.is_Dimension:
                self._mapper[i] = (parameters.index(i.symbolic_min),
                                   parameters.index(i.symbolic_max))
            else:
                self._mapper[i] = (parameters.index(i),)

    def make_call(self, dynamic_parameters_mapper=None):
        dynamic_parameters_mapper = dynamic_parameters_mapper or {}
        arguments = list(self.parameters)
        for k, v in dynamic_parameters_mapper.items():
            # Sanity check
            if k not in self._mapper:
                raise ValueError("`k` is not a dynamic parameter" % k)
            if len(self._mapper[k]) != len(v):
                raise ValueError("Expected %d values for dynamic parameter `%s`, given %d"
                                 % (len(self._mapper[k]), k, len(v)))
            # Create the argument list
            for i, j in zip(self._mapper[k], v):
                arguments[i] = j
        return Call(self.name, tuple(arguments))

def make_efunc(name, iet, dynamic_parameters=None, retval='void', prefix='static'):
    """
    Create an ElementalFunction from (a sequence of) perfectly nested Iterations.
    """
    # Arrays are by definition (vector) temporaries, so if they are written
    # within `iet`, they can also be declared and allocated within the `efunc`
    items = FindSymbols().visit(iet)
    local = [i.write for i in FindNodes(Expression).visit(iet) if i.write.is_Array]
    external = [i for i in items if i.is_Tensor and i not in local]

    # Insert array casts
    casts = [ArrayCast(i) for i in external]
    iet = List(body=casts + [iet])

    # Insert declarations
    iet = iet_insert_C_decls(iet, external)

    # The Callable parameters
    params = [i for i in derive_parameters(iet) if i not in local]

    return ElementalFunction(name, iet, retval, params, prefix, dynamic_parameters)

class EFuncNode(NodeMixin):

    """A simple utility class to keep track of ElementalFunctions and Call sites."""

    def __init__(self, iet, parent=None, name=None):
        self.iet = iet
        if name is not None:
            self.name = name
        else:
            assert iet.is_Callable
            self.name = iet.name
        self.parent = parent

        self._rendered_name = "<%s>" % self.name

    def __repr__(self):
        return RenderTree(self, style=ContStyle()).by_attr('_rendered_name')

    def visit(self):
        for i in PreOrderIter(self):
            yield i
<commit_msg>ir: Add ElementalCall, move logic from ElementalFunction<commit_after>from anytree import NodeMixin, PreOrderIter, RenderTree, ContStyle

from devito.ir.iet.nodes import ArrayCast, Call, Callable, Expression, List
from devito.ir.iet.scheduler import iet_insert_C_decls
from devito.ir.iet.utils import derive_parameters
from devito.ir.iet.visitors import FindSymbols, FindNodes
from devito.tools import as_tuple

__all__ = ['ElementalFunction', 'ElementalCall', 'EFuncNode', 'make_efunc']

class ElementalFunction(Callable):

    """
    A Callable performing a computation over an abstract convex iteration space.

    A Call to an ElementalFunction will "instantiate" such iteration space by
    supplying bounds and step increment for each Dimension listed in
    ``dynamic_parameters``.
    """

    def __init__(self, name, body, retval, parameters=None, prefix=('static', 'inline'),
                 dynamic_parameters=None):
        super(ElementalFunction, self).__init__(name, body, retval, parameters, prefix)

        self._mapper = {}
        for i in as_tuple(dynamic_parameters):
            if i.is_Dimension:
                self._mapper[i] = (parameters.index(i.symbolic_min),
                                   parameters.index(i.symbolic_max))
            else:
                self._mapper[i] = (parameters.index(i),)

    @property
    def dynamic_defaults(self):
        return {k : tuple(self.parameters[i] for i in v) for k, v in self._mapper.items()}

    def make_call(self, dynamic_params_mapper=None, incr=False):
        return ElementalCall(self.name, list(self.parameters), dict(self._mapper),
                             dynamic_params_mapper, incr)

class ElementalCall(Call):

    def __init__(self, name, arguments=None, mapper=None, dynamic_params_mapper=None,
                 incr=False):
        self._mapper = mapper or {}

        arguments = list(as_tuple(arguments))
        dynamic_params_mapper = dynamic_params_mapper or {}
        for k, v in dynamic_params_mapper.items():
            # Sanity check
            if k not in self._mapper:
                raise ValueError("`k` is not a dynamic parameter" % k)
            if len(self._mapper[k]) != len(v):
                raise ValueError("Expected %d values for dynamic parameter `%s`, given %d"
                                 % (len(self._mapper[k]), k, len(v)))
            # Create the argument list
            for i, j in zip(self._mapper[k], v):
                arguments[i] = j if incr is False else (arguments[i] + j)

        super(ElementalCall, self).__init__(name, arguments)

    @property
    def dynamic_defaults(self):
        return {k : tuple(self.params[i] for i in v) for k, v in self._mapper.items()}

def make_efunc(name, iet, dynamic_parameters=None, retval='void', prefix='static'):
    """
    Create an ElementalFunction from (a sequence of) perfectly nested Iterations.
    """
    # Arrays are by definition (vector) temporaries, so if they are written
    # within `iet`, they can also be declared and allocated within the `efunc`
    items = FindSymbols().visit(iet)
    local = [i.write for i in FindNodes(Expression).visit(iet) if i.write.is_Array]
    external = [i for i in items if i.is_Tensor and i not in local]

    # Insert array casts
    casts = [ArrayCast(i) for i in external]
    iet = List(body=casts + [iet])

    # Insert declarations
    iet = iet_insert_C_decls(iet, external)

    # The Callable parameters
    params = [i for i in derive_parameters(iet) if i not in local]

    return ElementalFunction(name, iet, retval, params, prefix, dynamic_parameters)

class EFuncNode(NodeMixin):

    """A simple utility class to keep track of ElementalFunctions and Call sites."""

    def __init__(self, iet, parent=None, name=None):
        self.iet = iet
        if name is not None:
            self.name = name
        else:
            assert iet.is_Callable
            self.name = iet.name
        self.parent = parent

        self._rendered_name = "<%s>" % self.name

    def __repr__(self):
        return RenderTree(self, style=ContStyle()).by_attr('_rendered_name')

    def visit(self):
        for i in PreOrderIter(self):
            yield i
<|endoftext|>