<commit_before># DIALS_ENABLE_COMMAND_LINE_COMPLETION
"""
Refine the diffraction geometry of input experiments against the input indexed
reflections. For rotation scans, the model may be either static (the same for
all reflections) or scan-varying (dependent on image number in the scan).
Other basic parameters include control over output filenames, fixing of
certain parameters of each model and options that control the number of
reflections used in refinement.

Examples::

  dials.refine indexed.expt indexed.refl

  dials.refine indexed.expt indexed.refl scan_varying=(False/True/Auto)
"""

from __future__ import absolute_import, division, print_function
import sys
import logging
from time import time
import dials.util
import libtbx.phil
from libtbx import Auto
from dials.array_family import flex
from dials.algorithms.refinement import RefinerFactory
from dials.algorithms.refinement import DialsRefineConfigError, DialsRefineRuntimeError
from dials.algorithms.refinement.corrgram import create_correlation_plots
from dials.util import Sorry

logger = logging.getLogger("dials.command_line.refine")

# The phil scope
phil_scope = libtbx.phil.parse(
    """

  output {
    experiments = refined.expt
      .type = str
      .help = "The filename for refined experimental models"

    reflections = refined.refl
      .type = str
      .help = "The filename for reflections with updated predictions"

    include_unused_reflections = True
      .type = bool
      .help = "If True, keep reflections unused in refinement in updated"
              "reflections file. Otherwise, remove them"
      .expert_level = 1

    matches = None
      .type = str
      .help = "The filename for output of the reflection table for reflections"
              "used in refinement, containing extra columns used internally."
              "Intended for debugging purposes only"
      .expert_level = 2

    centroids = None
      .type = str
      .help = "The filename for the table of centroids at the end of"
              "refinement"
      .expert_level = 1

    parameter_table = None
      .type = str
      .help = "The filename for the table of scan varying parameter values"
      .expert_level = 1

    log = dials.refine.log
      .type = str

    include scope dials.algorithms.refinement.corrgram.phil_scope

    history = None
      .type = str
      .help = "The filename for output of the refinement history json"
      .expert_level = 1
  }

  n_static_macrocycles = 1
    .type = int(value_min=1)
    .help = "Number of macro-cycles of static refinement to perform"

  include scope dials.algorithms.refinement.refiner.phil_scope
""",
    process_includes=True,
)

# local overrides for refiner.phil_scope
phil_overrides = libtbx.phil.parse(
    """
  refinement
  {
    parameterisation.scan_varying = Auto
  }
"""
)

working_phil = phil_scope.fetch(sources=[phil_overrides])

def write_centroids_table(refiner, filename):
    """Write a table of centroids from refinement for debugging.

    Args:
        refiner: A dials.algorithms.refinement.refiner.Refiner.
        filename (str): The name of the file to which to write.

    Returns:
        None
    """

    matches = refiner.get_matches()

    header = "H\tK\tL\tFrame_obs\tX_obs\tY_obs\tPhi_obs\tX_calc\t" "Y_calc\tPhi_calc"
    msg_temp = "%d\t%d\t%d\t%d\t%5.3f\t%5.3f\t%9.6f\t%5.3f\t%5.3f\t%9.6f"
    has_del_psi = "delpsical.rad" in matches
    if has_del_psi:
        header += "\tDelta_Psi"
        msg_temp += "\t%9.6f"
    header += "\n"
    msg_temp += "\n"

    with open(filename, "w") as f:
        f.write(header)

        for m in matches:
            (h, k, l) = m["miller_index"]
            frame = m["xyzobs.px.value"][2]
            x_obs, y_obs, phi_obs = m["xyzobs.mm.value"]
            x_calc, y_calc, phi_calc = m["xyzcal.mm"]
            if has_del_psi:
                del_psi = m["delpsical.rad"]
                msg = msg_temp % (
                    h,
                    k,
                    l,
                    frame,
                    x_obs,
                    y_obs,
                    phi_obs,
                    x_calc,
                    y_calc,
                    phi_calc,
                    del_psi,
                )
            else:
                msg = msg_temp % (
                    h,
                    k,
                    l,
                    frame,
                    x_obs,
                    y_obs,
                    phi_obs,
                    x_calc,
                    y_calc,
                    phi_calc,
                )
            f.write(msg)

def run_macrocycle(params, reflections, experiments):
    """Run one macrocycle of refinement.

    One macrocycle of refinement is run, as specified by the PHIL
    parameters, using the centroids from the supplied reflections
    and the initial experimental geometry taken from experiments.

    Args:
        params: The working PHIL parameters.
        reflections: A reflection table containing observed centroids
        experiments: The initial dxtbx experimental geometry models

    Returns:
        tuple: The Refiner, the reflection table with updated predictions
            and flags, and the refinement history object.
    """
    # Get the refiner
    logger.info("Configuring refiner")
    try:
        refiner = RefinerFactory.from_parameters_data_experiments(
            params, reflections, experiments
        )
    except DialsRefineConfigError as e:
        raise Sorry(e.message)

    # Refine the geometry
    nexp = len(experiments)
    if nexp == 1:
        logger.info("Performing refinement of a single Experiment...")
    else:
        logger.info("Performing refinement of {} Experiments...".format(nexp))

    # Refine and get the refinement history
    try:
        history = refiner.run()
    except DialsRefineRuntimeError as e:
        raise Sorry(e.message)

    # Update predictions for all indexed reflections
    logger.info("Updating predictions for indexed reflections")
    preds = refiner.predict_for_indexed()

    # just copy over the columns of interest or columns that may have been
    # updated, leaving behind things added by e.g. scan-varying refinement
    # such as 'block', the U, B and UB matrices and gradients.
    for key in preds.rows():
        if key in reflections.keys() or key in [
            "s1",
            "xyzcal.mm",
            "xyzcal.px",
            "entering",
            "delpsical.rad",
        ]:
            reflections[key] = preds[key]

    # set refinement flags
    assert len(preds) == len(reflections)
    reflections.unset_flags(
        flex.size_t_range(len(reflections)),
        reflections.flags.excluded_for_refinement
        | reflections.flags.used_in_refinement
        | reflections.flags.centroid_outlier
        | reflections.flags.predicted,
    )
    reflections.set_flags(
        preds.get_flags(preds.flags.excluded_for_refinement),
        reflections.flags.excluded_for_refinement,
    )
    reflections.set_flags(
        preds.get_flags(preds.flags.centroid_outlier),
        reflections.flags.centroid_outlier,
    )
    reflections.set_flags(
        preds.get_flags(preds.flags.used_in_refinement),
        reflections.flags.used_in_refinement,
    )
    reflections.set_flags(
        preds.get_flags(preds.flags.predicted), reflections.flags.predicted
    )

    return refiner, reflections, history

def run_dials_refine(experiments, reflections, params):
    """Functional interface to tasks performed by the program dials.refine.

    This runs refinement according to the PHIL parameters in params, using
    taking the initial experimental geometry from experiments, and the
    observed reflection centroids from reflections.

    Static refinement can be done in a number of macrocycles. By default,
    for scans, this will be followed by a macrocycle of scan-varying
    refinement.

    Args:
        experiments: The initial dxtbx experimental geometry models
        reflections: A reflection table containing observed centroids
        params: The working PHIL parameters

    Returns:
        tuple: The refined experiments, the updated reflection table, the
            Refiner object and the refinement history object.
    """

    # Modify options if necessary
    if params.output.correlation_plot.filename is not None:
        params.refinement.refinery.journal.track_parameter_correlation = True

    # scan_varying=Auto is special to dials.refine. Refiner expects it to be
    # True or False, so catch and set here
    scan_varying = params.refinement.parameterisation.scan_varying
    if scan_varying is Auto:
        params.refinement.parameterisation.scan_varying = False

    if params.n_static_macrocycles == 1:
        refiner, reflections, history = run_macrocycle(params, reflections, experiments)
        experiments = refiner.get_experiments()
    else:
        for i in range(params.n_static_macrocycles):
            logger.info("\nStatic refinement macrocycle {}".format(i + 1))
            refiner, reflections, history = run_macrocycle(
                params, reflections, experiments
            )
            experiments = refiner.get_experiments()

    # Scan-varying macrocycle, if appropriate
    if scan_varying is Auto and refiner.experiment_type == "scans":
        logger.info("\nScan-varying refinement")
        params.refinement.parameterisation.scan_varying = True
        refiner, reflections, history = run_macrocycle(params, reflections, experiments)
        experiments = refiner.get_experiments()

    return experiments, reflections, refiner, history

def run(args=None, phil=working_phil):
    """
    Set up refinement from command line options, files and PHIL parameters.
    Run refinement and save output files as specified.

    Called when running dials.refine as a command-line program

    Args:
        args (list): Additional command-line arguments
        phil: The working PHIL parameters

    Returns:
        None
    """

    import dials.util.log
    from dials.util.options import OptionParser
    from dials.util.options import flatten_reflections
    from dials.util.options import flatten_experiments
    import libtbx.load_env

    start_time = time()

    # The script usage
    usage = (
        "usage: %s [options] [param.phil] "
        "models.expt observations.refl" % libtbx.env.dispatcher_name
    )

    # Create the parser
    parser = OptionParser(
        usage=usage,
        phil=phil,
        read_reflections=True,
        read_experiments=True,
        check_format=False,
        epilog=__doc__,
    )

    # Parse the command line
    params, options = parser.parse_args(args=args, show_diff_phil=False)
    reflections = flatten_reflections(params.input.reflections)
    experiments = flatten_experiments(params.input.experiments)

    # Configure the logging
    dials.util.log.config(verbosity=options.verbose, logfile=params.output.log)

    # Try to load the models and data
    nexp = len(experiments)
    if nexp == 0 or len(reflections) == 0:
        parser.print_help()
        return
    if len(reflections) > 1:
        sys.exit("Only one reflections list can be imported at present")
    reflections = reflections[0]

    # check input is suitable
    msg = (
        "The supplied reflection table does not have the required data " + "column: {0}"
    )
    for key in ["xyzobs.mm.value", "xyzobs.mm.variance"]:
        if key not in reflections:
            msg = msg.format(key)
            raise dials.util.Sorry(msg)

    from dials.util.version import dials_version

    logger.info(dials_version())

    # Log the diff phil
    diff_phil = parser.diff_phil.as_str()
    if diff_phil:
        logger.info("The following parameters have been modified:\n")
        logger.info(diff_phil)

    # Warn about potentially unhelpful options
    if params.refinement.mp.nproc > 1:
        logger.warning(
            "WARNING: setting nproc > 1 is only helpful in rare "
            "circumstances. It is not recommended for typical data processing "
            "tasks.\n"
        )

    # Run refinement
    experiments, reflections, refiner, history = run_dials_refine(
        experiments, reflections, params
    )

    # For the usual case of refinement of one crystal, print that model for information
    crystals = experiments.crystals()
    if len(crystals) == 1:
        logger.info("")
        logger.info("Final refined crystal model:")
        logger.info(crystals[0])

    # Write table of centroids to file, if requested
    if params.output.centroids:
        logger.info(
            "Writing table of centroids to '{}'".format(params.output.centroids)
        )
        write_centroids_table(refiner, params.output.centroids)

    # Write scan-varying parameters to file, if there were any
    if params.output.parameter_table:
        scans = experiments.scans()
        if len(scans) > 1:
            logger.info(
                "Writing a scan-varying parameter table is only supported "
                "for refinement of a single scan"
            )
        else:
            scan = scans[0]
            text = refiner.get_param_reporter().varying_params_vs_image_number(
                scan.get_array_range()
            )
            if text:
                logger.info(
                    "Writing scan-varying parameter table to {}".format(
                        params.output.parameter_table
                    )
                )
                f = open(params.output.parameter_table, "w")
                f.write(text)
                f.close()
            else:
                logger.info("No scan-varying parameter table to write")

    # Save the refined experiments to file
    output_experiments_filename = params.output.experiments
    logger.info("Saving refined experiments to {}".format(output_experiments_filename))
    experiments.as_file(output_experiments_filename)

    # Save reflections with updated predictions if requested (allow to switch
    # this off if it is a time-consuming step)
    if params.output.reflections:
        logger.info(
            "Saving reflections with updated predictions to {}".format(
                params.output.reflections
            )
        )
        if params.output.include_unused_reflections:
            reflections.as_file(params.output.reflections)
        else:
            sel = reflections.get_flags(reflections.flags.used_in_refinement)
            reflections.select(sel).as_file(params.output.reflections)

    # Save matches to file for debugging
    if params.output.matches:
        matches = refiner.get_matches()
        logger.info(
            "Saving matches (use for debugging purposes) to {}".format(
                params.output.matches
            )
        )
        matches.as_file(params.output.matches)

    # Create correlation plots
    if params.output.correlation_plot.filename is not None:
        create_correlation_plots(refiner, params.output)

    # Save refinement history
    if params.output.history:
        logger.info(
            "Saving refinement step history to {}".format(params.output.history)
        )
        history.to_json_file(params.output.history)

    # Log the total time taken
    logger.info("\nTotal time taken: {:.2f}s".format(time() - start_time))

if __name__ == "__main__":
    with dials.util.show_mail_on_error():
        run()
<commit_msg>Document expected exceptions from run_dials_refine<commit_after># DIALS_ENABLE_COMMAND_LINE_COMPLETION
"""
Refine the diffraction geometry of input experiments against the input indexed
reflections. For rotation scans, the model may be either static (the same for
all reflections) or scan-varying (dependent on image number in the scan).
Other basic parameters include control over output filenames, fixing of
certain parameters of each model and options that control the number of
reflections used in refinement.

Examples::

  dials.refine indexed.expt indexed.refl

  dials.refine indexed.expt indexed.refl scan_varying=(False/True/Auto)
"""

from __future__ import absolute_import, division, print_function
import sys
import logging
from time import time
import dials.util
import libtbx.phil
from libtbx import Auto
from dials.array_family import flex
from dials.algorithms.refinement import RefinerFactory
from dials.algorithms.refinement import DialsRefineConfigError, DialsRefineRuntimeError
from dials.algorithms.refinement.corrgram import create_correlation_plots
from dials.util import Sorry

logger = logging.getLogger("dials.command_line.refine")

# The phil scope
phil_scope = libtbx.phil.parse(
    """

  output {
    experiments = refined.expt
      .type = str
      .help = "The filename for refined experimental models"

    reflections = refined.refl
      .type = str
      .help = "The filename for reflections with updated predictions"

    include_unused_reflections = True
      .type = bool
      .help = "If True, keep reflections unused in refinement in updated"
              "reflections file. Otherwise, remove them"
      .expert_level = 1

    matches = None
      .type = str
      .help = "The filename for output of the reflection table for reflections"
              "used in refinement, containing extra columns used internally."
              "Intended for debugging purposes only"
      .expert_level = 2

    centroids = None
      .type = str
      .help = "The filename for the table of centroids at the end of"
              "refinement"
      .expert_level = 1

    parameter_table = None
      .type = str
      .help = "The filename for the table of scan varying parameter values"
      .expert_level = 1

    log = dials.refine.log
      .type = str

    include scope dials.algorithms.refinement.corrgram.phil_scope

    history = None
      .type = str
      .help = "The filename for output of the refinement history json"
      .expert_level = 1
  }

  n_static_macrocycles = 1
    .type = int(value_min=1)
    .help = "Number of macro-cycles of static refinement to perform"

  include scope dials.algorithms.refinement.refiner.phil_scope
""",
    process_includes=True,
)

# local overrides for refiner.phil_scope
phil_overrides = libtbx.phil.parse(
    """
  refinement
  {
    parameterisation.scan_varying = Auto
  }
"""
)

working_phil = phil_scope.fetch(sources=[phil_overrides])

def write_centroids_table(refiner, filename):
    """Write a table of centroids from refinement for debugging.

    Args:
        refiner: A dials.algorithms.refinement.refiner.Refiner.
        filename (str): The name of the file to which to write.

    Returns:
        None
    """

    matches = refiner.get_matches()

    header = "H\tK\tL\tFrame_obs\tX_obs\tY_obs\tPhi_obs\tX_calc\t" "Y_calc\tPhi_calc"
    msg_temp = "%d\t%d\t%d\t%d\t%5.3f\t%5.3f\t%9.6f\t%5.3f\t%5.3f\t%9.6f"
    has_del_psi = "delpsical.rad" in matches
    if has_del_psi:
        header += "\tDelta_Psi"
        msg_temp += "\t%9.6f"
    header += "\n"
    msg_temp += "\n"

    with open(filename, "w") as f:
        f.write(header)

        for m in matches:
            (h, k, l) = m["miller_index"]
            frame = m["xyzobs.px.value"][2]
            x_obs, y_obs, phi_obs = m["xyzobs.mm.value"]
            x_calc, y_calc, phi_calc = m["xyzcal.mm"]
            if has_del_psi:
                del_psi = m["delpsical.rad"]
                msg = msg_temp % (
                    h,
                    k,
                    l,
                    frame,
                    x_obs,
                    y_obs,
                    phi_obs,
                    x_calc,
                    y_calc,
                    phi_calc,
                    del_psi,
                )
            else:
                msg = msg_temp % (
                    h,
                    k,
                    l,
                    frame,
                    x_obs,
                    y_obs,
                    phi_obs,
                    x_calc,
                    y_calc,
                    phi_calc,
                )
            f.write(msg)

def run_macrocycle(params, reflections, experiments):
    """Run one macrocycle of refinement.

    One macrocycle of refinement is run, as specified by the PHIL
    parameters, using the centroids from the supplied reflections
    and the initial experimental geometry taken from experiments.

    Args:
        params: The working PHIL parameters.
        reflections: A reflection table containing observed centroids
        experiments: The initial dxtbx experimental geometry models

    Returns:
        tuple: The Refiner, the reflection table with updated predictions
            and flags, and the refinement history object.
    """
    # Get the refiner
    logger.info("Configuring refiner")
    try:
        refiner = RefinerFactory.from_parameters_data_experiments(
            params, reflections, experiments
        )
    except DialsRefineConfigError as e:
        raise Sorry(e.message)

    # Refine the geometry
    nexp = len(experiments)
    if nexp == 1:
        logger.info("Performing refinement of a single Experiment...")
    else:
        logger.info("Performing refinement of {} Experiments...".format(nexp))

    # Refine and get the refinement history
    try:
        history = refiner.run()
    except DialsRefineRuntimeError as e:
        raise Sorry(e.message)

    # Update predictions for all indexed reflections
    logger.info("Updating predictions for indexed reflections")
    preds = refiner.predict_for_indexed()

    # just copy over the columns of interest or columns that may have been
    # updated, leaving behind things added by e.g. scan-varying refinement
    # such as 'block', the U, B and UB matrices and gradients.
    for key in preds.rows():
        if key in reflections.keys() or key in [
            "s1",
            "xyzcal.mm",
            "xyzcal.px",
            "entering",
            "delpsical.rad",
        ]:
            reflections[key] = preds[key]

    # set refinement flags
    assert len(preds) == len(reflections)
    reflections.unset_flags(
        flex.size_t_range(len(reflections)),
        reflections.flags.excluded_for_refinement
        | reflections.flags.used_in_refinement
        | reflections.flags.centroid_outlier
        | reflections.flags.predicted,
    )
    reflections.set_flags(
        preds.get_flags(preds.flags.excluded_for_refinement),
        reflections.flags.excluded_for_refinement,
    )
    reflections.set_flags(
        preds.get_flags(preds.flags.centroid_outlier),
        reflections.flags.centroid_outlier,
    )
    reflections.set_flags(
        preds.get_flags(preds.flags.used_in_refinement),
        reflections.flags.used_in_refinement,
    )
    reflections.set_flags(
        preds.get_flags(preds.flags.predicted), reflections.flags.predicted
    )

    return refiner, reflections, history

def run_dials_refine(experiments, reflections, params):
    """Functional interface to tasks performed by the program dials.refine.

    This runs refinement according to the PHIL parameters in params, using
    taking the initial experimental geometry from experiments, and the
    observed reflection centroids from reflections.

    Static refinement can be done in a number of macrocycles. By default,
    for scans, this will be followed by a macrocycle of scan-varying
    refinement.

    Args:
        experiments: The initial dxtbx experimental geometry models
        reflections: A reflection table containing observed centroids
        params: The working PHIL parameters

    Returns:
        tuple: The refined experiments, the updated reflection table, the
            Refiner object and the refinement history object.

    Raises:
        dials.util.Sorry: Refinement failed.
    """

    # Modify options if necessary
    if params.output.correlation_plot.filename is not None:
        params.refinement.refinery.journal.track_parameter_correlation = True

    # scan_varying=Auto is special to dials.refine. Refiner expects it to be
    # True or False, so catch and set here
    scan_varying = params.refinement.parameterisation.scan_varying
    if scan_varying is Auto:
        params.refinement.parameterisation.scan_varying = False

    if params.n_static_macrocycles == 1:
        refiner, reflections, history = run_macrocycle(params, reflections, experiments)
        experiments = refiner.get_experiments()
    else:
        for i in range(params.n_static_macrocycles):
            logger.info("\nStatic refinement macrocycle {}".format(i + 1))
            refiner, reflections, history = run_macrocycle(
                params, reflections, experiments
            )
            experiments = refiner.get_experiments()

    # Scan-varying macrocycle, if appropriate
    if scan_varying is Auto and refiner.experiment_type == "scans":
        logger.info("\nScan-varying refinement")
        params.refinement.parameterisation.scan_varying = True
        refiner, reflections, history = run_macrocycle(params, reflections, experiments)
        experiments = refiner.get_experiments()

    return experiments, reflections, refiner, history

def run(args=None, phil=working_phil):
    """
    Set up refinement from command line options, files and PHIL parameters.
    Run refinement and save output files as specified.

    Called when running dials.refine as a command-line program

    Args:
        args (list): Additional command-line arguments
        phil: The working PHIL parameters

    Returns:
        None
    """

    import dials.util.log
    from dials.util.options import OptionParser
    from dials.util.options import flatten_reflections
    from dials.util.options import flatten_experiments
    import libtbx.load_env

    start_time = time()

    # The script usage
    usage = (
        "usage: %s [options] [param.phil] "
        "models.expt observations.refl" % libtbx.env.dispatcher_name
    )

    # Create the parser
    parser = OptionParser(
        usage=usage,
        phil=phil,
        read_reflections=True,
        read_experiments=True,
        check_format=False,
        epilog=__doc__,
    )

    # Parse the command line
    params, options = parser.parse_args(args=args, show_diff_phil=False)
    reflections = flatten_reflections(params.input.reflections)
    experiments = flatten_experiments(params.input.experiments)

    # Configure the logging
    dials.util.log.config(verbosity=options.verbose, logfile=params.output.log)

    # Try to load the models and data
    nexp = len(experiments)
    if nexp == 0 or len(reflections) == 0:
        parser.print_help()
        return
    if len(reflections) > 1:
        sys.exit("Only one reflections list can be imported at present")
    reflections = reflections[0]

    # check input is suitable
    msg = (
        "The supplied reflection table does not have the required data " + "column: {0}"
    )
    for key in ["xyzobs.mm.value", "xyzobs.mm.variance"]:
        if key not in reflections:
            msg = msg.format(key)
            raise dials.util.Sorry(msg)

    from dials.util.version import dials_version

    logger.info(dials_version())

    # Log the diff phil
    diff_phil = parser.diff_phil.as_str()
    if diff_phil:
        logger.info("The following parameters have been modified:\n")
        logger.info(diff_phil)

    # Warn about potentially unhelpful options
    if params.refinement.mp.nproc > 1:
        logger.warning(
            "WARNING: setting nproc > 1 is only helpful in rare "
            "circumstances. It is not recommended for typical data processing "
            "tasks.\n"
        )

    # Run refinement
    experiments, reflections, refiner, history = run_dials_refine(
        experiments, reflections, params
    )

    # For the usual case of refinement of one crystal, print that model for information
    crystals = experiments.crystals()
    if len(crystals) == 1:
        logger.info("")
        logger.info("Final refined crystal model:")
        logger.info(crystals[0])

    # Write table of centroids to file, if requested
    if params.output.centroids:
        logger.info(
            "Writing table of centroids to '{}'".format(params.output.centroids)
        )
        write_centroids_table(refiner, params.output.centroids)

    # Write scan-varying parameters to file, if there were any
    if params.output.parameter_table:
        scans = experiments.scans()
        if len(scans) > 1:
            logger.info(
                "Writing a scan-varying parameter table is only supported "
                "for refinement of a single scan"
            )
        else:
            scan = scans[0]
            text = refiner.get_param_reporter().varying_params_vs_image_number(
                scan.get_array_range()
            )
            if text:
                logger.info(
                    "Writing scan-varying parameter table to {}".format(
                        params.output.parameter_table
                    )
                )
                f = open(params.output.parameter_table, "w")
                f.write(text)
                f.close()
            else:
                logger.info("No scan-varying parameter table to write")

    # Save the refined experiments to file
    output_experiments_filename = params.output.experiments
    logger.info("Saving refined experiments to {}".format(output_experiments_filename))
    experiments.as_file(output_experiments_filename)

    # Save reflections with updated predictions if requested (allow to switch
    # this off if it is a time-consuming step)
    if params.output.reflections:
        logger.info(
            "Saving reflections with updated predictions to {}".format(
                params.output.reflections
            )
        )
        if params.output.include_unused_reflections:
            reflections.as_file(params.output.reflections)
        else:
            sel = reflections.get_flags(reflections.flags.used_in_refinement)
            reflections.select(sel).as_file(params.output.reflections)

    # Save matches to file for debugging
    if params.output.matches:
        matches = refiner.get_matches()
        logger.info(
            "Saving matches (use for debugging purposes) to {}".format(
                params.output.matches
            )
        )
        matches.as_file(params.output.matches)

    # Create correlation plots
    if params.output.correlation_plot.filename is not None:
        create_correlation_plots(refiner, params.output)

    # Save refinement history
    if params.output.history:
        logger.info(
            "Saving refinement step history to {}".format(params.output.history)
        )
        history.to_json_file(params.output.history)

    # Log the total time taken
    logger.info("\nTotal time taken: {:.2f}s".format(time() - start_time))

if __name__ == "__main__":
    with dials.util.show_mail_on_error():
        run()
<|endoftext|>


<commit_before>from os.path import join

import pyrax
pyrax.set_setting("identity_type", "rackspace")

from insight_reloaded.insight_settings import (
    CLOUDFILES_USERNAME, CLOUDFILES_API_KEY, CLOUDFILES_COUNTAINER,
    CLOUDFILES_SERVICENET, CLOUDFILES_DEFAULT_REGION
)

class CloudFilesStorage(object):

    def __init__(self, document_url, document_hash, cloudfiles_location=None):
        self.document_url = document_url
        self.document_hash = document_hash

        if cloudfiles_location is None:
            cloudfiles_location = CLOUDFILES_DEFAULT_REGION

        pyrax.set_default_region(cloudfiles_location)
        pyrax.set_credentials(CLOUDFILES_USERNAME, CLOUDFILES_API_KEY,
                              region=cloudfiles_location)
        self.paths = {}

        self.client = pyrax.connect_to_cloudfiles(
            cloudfiles_location, public=not CLOUDFILES_SERVICENET)
        self.container = self.client.create_container(CLOUDFILES_COUNTAINER)

    def prepare(self):
        """Do nothing
        """
        pass

    def get_path(self, filename):
        return self.paths[filename]

    def save(self, path, filename):
        self.paths[filename] = path
        self.container.upload_file(path, join(self.document_hash, filename))

    def get_base_document_url(self):
        return self.document_hash
<commit_msg>Add error message if region is not available for cloudfiles account<commit_after>from os.path import join

import pyrax
pyrax.set_setting("identity_type", "rackspace")

from insight_reloaded.insight_settings import (
    CLOUDFILES_USERNAME, CLOUDFILES_API_KEY, CLOUDFILES_COUNTAINER,
    CLOUDFILES_SERVICENET, CLOUDFILES_DEFAULT_REGION
)
from insight_reloaded.preview import PreviewException

class CloudFilesStorage(object):

    def __init__(self, document_url, document_hash, cloudfiles_location=None):
        self.document_url = document_url
        self.document_hash = document_hash

        if cloudfiles_location is None:
            cloudfiles_location = CLOUDFILES_DEFAULT_REGION

        pyrax.set_default_region(cloudfiles_location)
        pyrax.set_credentials(CLOUDFILES_USERNAME, CLOUDFILES_API_KEY,
                              region=cloudfiles_location)
        self.paths = {}

        self.client = pyrax.connect_to_cloudfiles(
            cloudfiles_location, public=not CLOUDFILES_SERVICENET)
        if self.client is None:
            err_msg = ('Error during connection to cloudfiles, region {0} is'
                       ' likely to be wrong.'.format(cloudfiles_location))
            raise PreviewException(err_msg)
        self.container = self.client.create_container(CLOUDFILES_COUNTAINER)

    def prepare(self):
        """Do nothing
        """
        pass

    def get_path(self, filename):
        return self.paths[filename]

    def save(self, path, filename):
        self.paths[filename] = path
        self.container.upload_file(path, join(self.document_hash, filename))

    def get_base_document_url(self):
        return self.document_hash
<|endoftext|>


<commit_before>#-*- coding: utf-8 -*-

import object_detector.extractor as extractor
import object_detector.file_io as file_io
import object_detector.descriptor as descriptor

CONFIGURATION_FILE = "conf/new_format.json"
PATCH_SIZE = (32, 96)

if __name__ == "__main__":
    conf = file_io.FileJson().read(CONFIGURATION_FILE)

    # 1. Initialize Descriptor instance
    # Todo : Factory Method to create hog-descriptor

    hog = descriptor.HOG(conf["descriptor"]["parameters"]["orientations"],
                         conf["descriptor"]["parameters"]["pixels_per_cell"],
                         conf["descriptor"]["parameters"]["cells_per_block"])

    # 2. Initialize FeatureGetter instance
    getter = extractor.FeatureExtractor(descriptor=hog, patch_size=PATCH_SIZE)

    # 3. Get Feature sets
    getter.add_positive_sets(image_dir=conf["dataset"]["pos_data_dir"],
                             pattern=conf["dataset"]["pos_format"],
                             annotation_path=conf["dataset"]['annotations_dir'],
                             padding=conf["extractor"]['padding'],
                             )

    # Todo : positive sample 숫자에 따라 negative sample 숫자를 자동으로 정할 수 있도록 설정
    getter.add_negative_sets(image_dir=conf["dataset"]["neg_data_dir"],
                             pattern=conf["dataset"]["neg_format"],
                             n_samples_per_img=conf["extractor"]["num_patches_per_negative_image"],
                             sample_ratio=conf["extractor"]["sampling_ratio_for_negative_images"])

    getter.summary()

    # 4. Save dataset
    getter.save(data_file=conf["extractor"]["output_file"])
    del getter

    # 5. Test Loading dataset
    getter = extractor.FeatureExtractor.load(descriptor=hog, patch_size=PATCH_SIZE, data_file=conf["extractor"]["output_file"])
    getter.summary()

<commit_msg>DescriptorFactory class implemented<commit_after>#-*- coding: utf-8 -*-

import object_detector.extractor as extractor
import object_detector.file_io as file_io
import object_detector.descriptor as descriptor

CONFIGURATION_FILE = "conf/new_format.json"
PATCH_SIZE = (32, 96)

class DescriptorFactory:

    def __init__(self):
        pass

    @staticmethod
    def create(algorithm, params):
        desc = None
        if algorithm == "hog":
            desc = descriptor.HOG(**params)

        if desc is None:
            raise ValueError('Such algorithm is not supported.')

        return desc

if __name__ == "__main__":
    conf = file_io.FileJson().read(CONFIGURATION_FILE)

    # 1. Initialize Descriptor instance
    desc = DescriptorFactory.create(conf["descriptor"]["algorithm"], conf["descriptor"]["parameters"])

    # 2. Initialize FeatureGetter instance
    getter = extractor.FeatureExtractor(descriptor=desc, patch_size=PATCH_SIZE)

    # 3. Get Feature sets
    getter.add_positive_sets(image_dir=conf["dataset"]["pos_data_dir"],
                             pattern=conf["dataset"]["pos_format"],
                             annotation_path=conf["dataset"]['annotations_dir'],
                             padding=conf["extractor"]['padding'],
                             )

    # Todo : positive sample 숫자에 따라 negative sample 숫자를 자동으로 정할 수 있도록 설정
    getter.add_negative_sets(image_dir=conf["dataset"]["neg_data_dir"],
                             pattern=conf["dataset"]["neg_format"],
                             n_samples_per_img=conf["extractor"]["num_patches_per_negative_image"],
                             sample_ratio=conf["extractor"]["sampling_ratio_for_negative_images"])

    getter.summary()

    # 4. Save dataset
    getter.save(data_file=conf["extractor"]["output_file"])
    del getter

    # 5. Test Loading dataset
    getter = extractor.FeatureExtractor.load(descriptor=desc, patch_size=PATCH_SIZE, data_file=conf["extractor"]["output_file"])
    getter.summary()

<|endoftext|>


<commit_before># -*- coding: utf8 -*-
from flask import request, redirect, url_for
from flask.ext.restplus import Resource, reqparse, fields, abort
from flask.ext.security import login_required, roles_required,\
        roles_accepted, current_user
from .. import ns_hail, db, api
from ..models import Hail as HailModel, Customer as CustomerModel, Taxi as TaxiModel
from datetime import datetime
from ..utils.make_model import make_model

hail_model = make_model('hail', 'Hail')

parser_put = reqparse.RequestParser()
parser_put.add_argument('customer_lon', type=float, required=True,
        location='hail')
parser_put.add_argument('customer_lat', type=float, required=True,
        location='hail')
parser_put.add_argument('status', type=str, required=True,
                        choices=['received_by_taxi',
                                 'accepted_by_taxi',
                                 'declined_by_taxi',
                                 'incident_taxi',
                                 'incident_customer'],
                        location='hail')
argument_names = map(lambda f: f.name, parser_put.args)
hail_expect_put_details = api.model('hail_expect_put_details',
                                dict(filter(lambda f: f[0] in argument_names, HailModel.marshall_obj().items())))
hail_expect_put = api.model('hail_expect_put', {'data': fields.List(fields.Nested(hail_expect_put_details))})

@login_required
@roles_accepted('moteur', 'operateur')
@ns_hail.route('/<int:hail_id>/', endpoint='hailid')
class HailId(Resource):

    @api.marshal_with(hail_model, envelope='hail')
    def get(self, hail_id):
        hail = HailModel.query.get_or_404(hail_id)
        return hail.to_dict()

    @api.marshal_with(hail_model)
    @api.expect(hail_expect_put)
    def put(self, hail_id):
        root_parser = reqparse.RequestParser()
        root_parser.add_argument('hail', type=dict, location='json')
        hj = parser_put.parse_args(req=root_parser.parse_args())
        hail = HailModel.query.get_or_404(hail_id)
        #We change the status
        if hasattr(hail, hj['status']):
            getattr(hail, hj['status'])()
        if current_user.has_role('moteur'):
            hail.customer_lon = hj['customer_lon']
            hail.customer_lat = hj['customer_lat']
        db.session.commit()
        return hail.to_dict()

parser_post = reqparse.RequestParser()
parser_post.add_argument('customer_id', type=str,
                         required=True, location='hail')
parser_post.add_argument('customer_lon', type=float,
                         required=True, location='hail')
parser_post.add_argument('customer_lat', type=float,
                         required=True, location='hail')
parser_post.add_argument('taxi_id', type=int,
                         required=True, location='hail')
argument_names = map(lambda f: f.name, parser_post.args)
hail_expect_post_details = api.model('hail_expect_post_details',
                                dict(filter(lambda f: f[0] in argument_names, HailModel.marshall_obj().items())))
hail_expect = api.model('hail_expect_post', {'data': fields.List(fields.Nested(hail_expect_post_details))})

@ns_hail.route('/', endpoint='hail_endpoint')
class Hail(Resource):

    @api.expect(hail_expect)
    @api.marshal_with(hail_model)
    @login_required
    @roles_required('moteur')
    def post(self):
        root_parser = reqparse.RequestParser()
        root_parser.add_argument('hail', type=dict, location='json')
        req = root_parser.parse_args()
        hj = parser_post.parse_args(req=req)
        taxi = TaxiModel.query.get(hj['taxi_id'])
        if not taxi:
            return abort(404, message="Unable to find taxi")
        if taxi.status != 'free':
            return abort(403, message="The taxi is not available")
        taxi.status = 'answering'
        db.session.commit()
        #@TODO: checker que le status est emitted???
        customer = CustomerModel.query.filter_by(id=hj['customer_id'],
                operateur_id=current_user.id)
        if not customer:
            customer = CustomerModel()
            customer.id = hj['customer_id']
            customer.operateur_id = current_user.id
            customer.nb_sanctions = 0
            customer.added_via = 'api'
            db.session.add(customer)
        hail = HailModel()
        hail.creation_datetime = datetime.now().isoformat()
        hail.customer_id = hj['customer_id']
        hail.customer_lon = hj['customer_lon']
        hail.customer_lat = hj['customer_lat']
        hail.added_via = 'api'
        hail.taxi_id = hj['taxi_id']
        db.session.add(hail)
        db.session.commit()
        #send hail to operateur
        hail.received()
        db.session.commit()
        return redirect(url_for('hailid', hail_id=hail.id))

<commit_msg>taxi id is a string<commit_after># -*- coding: utf8 -*-
from flask import request, redirect, url_for
from flask.ext.restplus import Resource, reqparse, fields, abort
from flask.ext.security import login_required, roles_required,\
        roles_accepted, current_user
from .. import ns_hail, db, api
from ..models import Hail as HailModel, Customer as CustomerModel, Taxi as TaxiModel
from datetime import datetime
from ..utils.make_model import make_model

hail_model = make_model('hail', 'Hail')

parser_put = reqparse.RequestParser()
parser_put.add_argument('customer_lon', type=float, required=True,
        location='hail')
parser_put.add_argument('customer_lat', type=float, required=True,
        location='hail')
parser_put.add_argument('status', type=str, required=True,
                        choices=['received_by_taxi',
                                 'accepted_by_taxi',
                                 'declined_by_taxi',
                                 'incident_taxi',
                                 'incident_customer'],
                        location='hail')
argument_names = map(lambda f: f.name, parser_put.args)
hail_expect_put_details = api.model('hail_expect_put_details',
                                dict(filter(lambda f: f[0] in argument_names, HailModel.marshall_obj().items())))
hail_expect_put = api.model('hail_expect_put', {'data': fields.List(fields.Nested(hail_expect_put_details))})

@login_required
@roles_accepted('moteur', 'operateur')
@ns_hail.route('/<int:hail_id>/', endpoint='hailid')
class HailId(Resource):

    @api.marshal_with(hail_model, envelope='hail')
    def get(self, hail_id):
        hail = HailModel.query.get_or_404(hail_id)
        return hail.to_dict()

    @api.marshal_with(hail_model)
    @api.expect(hail_expect_put)
    def put(self, hail_id):
        root_parser = reqparse.RequestParser()
        root_parser.add_argument('hail', type=dict, location='json')
        hj = parser_put.parse_args(req=root_parser.parse_args())
        hail = HailModel.query.get_or_404(hail_id)
        #We change the status
        if hasattr(hail, hj['status']):
            getattr(hail, hj['status'])()
        if current_user.has_role('moteur'):
            hail.customer_lon = hj['customer_lon']
            hail.customer_lat = hj['customer_lat']
        db.session.commit()
        return hail.to_dict()

parser_post = reqparse.RequestParser()
parser_post.add_argument('customer_id', type=str,
                         required=True, location='hail')
parser_post.add_argument('customer_lon', type=float,
                         required=True, location='hail')
parser_post.add_argument('customer_lat', type=float,
                         required=True, location='hail')
parser_post.add_argument('taxi_id', type=str,
                         required=True, location='hail')
argument_names = map(lambda f: f.name, parser_post.args)
hail_expect_post_details = api.model('hail_expect_post_details',
                                dict(filter(lambda f: f[0] in argument_names, HailModel.marshall_obj().items())))
hail_expect = api.model('hail_expect_post', {'data': fields.List(fields.Nested(hail_expect_post_details))})

@ns_hail.route('/', endpoint='hail_endpoint')
class Hail(Resource):

    @api.expect(hail_expect)
    @api.marshal_with(hail_model)
    @login_required
    @roles_required('moteur')
    def post(self):
        root_parser = reqparse.RequestParser()
        root_parser.add_argument('hail', type=dict, location='json')
        req = root_parser.parse_args()
        hj = parser_post.parse_args(req=req)
        taxi = TaxiModel.query.get(hj['taxi_id'])
        if not taxi:
            return abort(404, message="Unable to find taxi")
        if taxi.status != 'free':
            return abort(403, message="The taxi is not available")
        taxi.status = 'answering'
        db.session.commit()
        #@TODO: checker que le status est emitted???
        customer = CustomerModel.query.filter_by(id=hj['customer_id'],
                operateur_id=current_user.id)
        if not customer:
            customer = CustomerModel()
            customer.id = hj['customer_id']
            customer.operateur_id = current_user.id
            customer.nb_sanctions = 0
            customer.added_via = 'api'
            db.session.add(customer)
        hail = HailModel()
        hail.creation_datetime = datetime.now().isoformat()
        hail.customer_id = hj['customer_id']
        hail.customer_lon = hj['customer_lon']
        hail.customer_lat = hj['customer_lat']
        hail.added_via = 'api'
        hail.taxi_id = hj['taxi_id']
        db.session.add(hail)
        db.session.commit()
        #send hail to operateur
        hail.received()
        db.session.commit()
        return redirect(url_for('hailid', hail_id=hail.id))

<|endoftext|>


<commit_before>import logging
import logging.handlers
from . import ToolHelper as TH
from path_maker import PathMaker

class Explog(PathMaker):
    """
    logger
    """
    def __init__(self, root, name):
        """
        usage:
            Explog(self.__class__.__name__)
        """
        self.enable_stream = False
        self.__add_logger()
        self.__add_handlers()
        self.__done()

    def __add_logger(self):
        lg = self.__get_logger()
        lg.setLevel(logging.INFO)
        lg.propagate = 0
        self.elog = lg

    def __add_handlers(self):
        """
        Should run after self.elog is inited
        """
        fmts = self.__format_str()
        fmt = logging.Formatter(**fmts)
        fn = self.make('log', 'log', True, False)
        self.__add_file_handler(fn, fmt)
        if self.enable_stream:
            self.__add_stream_handler(fn, fmt)

    def __add_stream_handler(self, fn, fmt):
        sh = self.__stream_handle(fn, fmt)
        if sh is not None:
            self.elog.addHandler(sh)

    def __add_file_handler(self, fn, fmt):
        fh = self.__file_handle(fn, fmt)
        if fh is not None:
            self.elog.addHandler(fh)

    def __done(self):
        cn = TH.underscore(self.__class__.__name__)
        raw = ">> ===== {} inited ===== <<".format(cn)
        self.elog.info(raw)

    def __format_str(self):
        items = ["%(asctime)s", "%(levelname)s", "%(name)s",
                 "%(funcName)s", "%(lineno)d", "%(message)s"]
        fmt = "{},{},{},{},{},{}".format(*items)
        dft = "%m/%d/%Y %H:%M:%S"
        return dict(fmt=fmt, datefmt=dft)

    def __get_logger(self):
        cn = TH.underscore(self.__class__.__name__)
        lgn = "{}.{}.{}".format(self.root, self.name, cn)
        print "debug: from logger: {}".format(lgn)
        logger = logging.getLogger(lgn)
        return logger

    def __file_handler_names(self):
        return [lh.name for lh in self.elog.handlers]

    def __create_file_handle(self, lfp, fmt, fh_name):
        fh = logging.handlers.\
            RotatingFileHandler(lfp,
                                maxBytes=10485760,
                                backupCount=5)
        fh.name = fh_name
        fh.setFormatter(fmt)
        return fh

    def __file_handle(self, lfp, fmt):
        fhn = lfp + "_fh"
        fhns = self.__file_handler_names()
        if fhn not in fhns:
            return self.__create_file_handle(lfp, fmt, fhn)
        return None

    def __create_stream_handle(self, lfp, fmt, sh_name):
        sh = logging.StreamHandler()
        sh.name = sh_name
        sh.setFormatter(fmt)
        return sh

    def __stream_handle(self, lfp, fmt):
        fhns = self.__file_handler_names()
        shn = lfp + "_sh"
        if shn not in fhns:
            return self.__create_stream_handle(lfp, fmt, shn)
        return None
<commit_msg>Remove debug info<commit_after>import logging
import logging.handlers
from . import ToolHelper as TH
from path_maker import PathMaker

class Explog(PathMaker):
    """
    logger
    """
    def __init__(self, root, name):
        """
        usage:
            Explog(self.__class__.__name__)
        """
        self.enable_stream = False
        self.__add_logger()
        self.__add_handlers()
        self.__done()

    def __add_logger(self):
        lg = self.__get_logger()
        lg.setLevel(logging.INFO)
        lg.propagate = 0
        self.elog = lg

    def __add_handlers(self):
        """
        Should run after self.elog is inited
        """
        fmts = self.__format_str()
        fmt = logging.Formatter(**fmts)
        fn = self.make('log', 'log', True, False)
        self.__add_file_handler(fn, fmt)
        if self.enable_stream:
            self.__add_stream_handler(fn, fmt)

    def __add_stream_handler(self, fn, fmt):
        sh = self.__stream_handle(fn, fmt)
        if sh is not None:
            self.elog.addHandler(sh)

    def __add_file_handler(self, fn, fmt):
        fh = self.__file_handle(fn, fmt)
        if fh is not None:
            self.elog.addHandler(fh)

    def __done(self):
        cn = TH.underscore(self.__class__.__name__)
        raw = ">> ===== {} inited ===== <<".format(cn)
        self.elog.info(raw)

    def __format_str(self):
        items = ["%(asctime)s", "%(levelname)s", "%(name)s",
                 "%(funcName)s", "%(lineno)d", "%(message)s"]
        fmt = "{},{},{},{},{},{}".format(*items)
        dft = "%m/%d/%Y %H:%M:%S"
        return dict(fmt=fmt, datefmt=dft)

    def __get_logger(self):
        cn = TH.underscore(self.__class__.__name__)
        lgn = "{}.{}.{}".format(self.root, self.name, cn)
        logger = logging.getLogger(lgn)
        return logger

    def __file_handler_names(self):
        return [lh.name for lh in self.elog.handlers]

    def __create_file_handle(self, lfp, fmt, fh_name):
        fh = logging.handlers.\
            RotatingFileHandler(lfp,
                                maxBytes=10485760,
                                backupCount=5)
        fh.name = fh_name
        fh.setFormatter(fmt)
        return fh

    def __file_handle(self, lfp, fmt):
        fhn = lfp + "_fh"
        fhns = self.__file_handler_names()
        if fhn not in fhns:
            return self.__create_file_handle(lfp, fmt, fhn)
        return None

    def __create_stream_handle(self, lfp, fmt, sh_name):
        sh = logging.StreamHandler()
        sh.name = sh_name
        sh.setFormatter(fmt)
        return sh

    def __stream_handle(self, lfp, fmt):
        fhns = self.__file_handler_names()
        shn = lfp + "_sh"
        if shn not in fhns:
            return self.__create_stream_handle(lfp, fmt, shn)
        return None
<|endoftext|>


<commit_before># pylab-free version of simpletest.py
import sys
from matplotlib.toolkits.basemap import Basemap
from matplotlib.figure import Figure
from matplotlib.mlab import meshgrid
from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
import matplotlib.numerix as nx
import matplotlib.cm as cm
import cPickle
# read in topo data from pickle (on a regular lat/lon grid)
topodict = cPickle.load(open('etopo20.pickle','rb'))
etopo = topodict['data']; lons = topodict['lons']; lats = topodict['lats']
# create figure with same aspect ratio as map.
fig = Figure()
canvas = FigureCanvas(fig)
ax = fig.add_axes([0.125,0.175,0.75,0.75],frameon=False)
# create Basemap instance for Robinson projection.
# set 'ax' keyword so pylab won't be imported.
m = Basemap(projection='robin',lon_0=0.5*(lons[0]+lons[-1]),ax=ax)
# reset figure size to have same aspect ratio as map.
fig.set_figsize_inches((8,m.aspect*8.))
# make filled contour plot.
x, y = m(*meshgrid(lons, lats))
levels, colls = m.contourf(x,y,etopo,30,cmap=cm.jet,colors=None)
# draw coastlines.
m.drawcoastlines()
# draw a line around the map region.
m.drawmapboundary()
# draw parallels and meridians.
m.drawparallels(nx.arange(-60.,90.,30.),labels=[1,0,0,0],fontsize=10)
m.drawmeridians(nx.arange(0.,420.,60.),labels=[0,0,0,1],fontsize=10)
# add a title.
ax.set_title('Robinson Projection')
# add a single colorbar.
cax = fig.add_axes([0.25, 0.05, 0.5, 0.05],frameon=False)
fig.colorbar(colls.mappable, cax=cax, tickfmt='%d', orientation='horizontal')
canvas.print_figure('simpletest')
<commit_msg>force use of Agg backend<commit_after># pylab-free version of simpletest.py
import matplotlib
matplotlib.use('Agg')
from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
from matplotlib.toolkits.basemap import Basemap
from matplotlib.figure import Figure
from matplotlib.mlab import meshgrid
import matplotlib.numerix as nx
import matplotlib.cm as cm
import cPickle
# read in topo data from pickle (on a regular lat/lon grid)
topodict = cPickle.load(open('etopo20.pickle','rb'))
etopo = topodict['data']; lons = topodict['lons']; lats = topodict['lats']
# create figure with same aspect ratio as map.
fig = Figure()
canvas = FigureCanvas(fig)
ax = fig.add_axes([0.125,0.175,0.75,0.75],frameon=False)
# create Basemap instance for Robinson projection.
# set 'ax' keyword so pylab won't be imported.
m = Basemap(projection='robin',lon_0=0.5*(lons[0]+lons[-1]),ax=ax)
# reset figure size to have same aspect ratio as map.
fig.set_figsize_inches((8,m.aspect*8.))
# make filled contour plot.
x, y = m(*meshgrid(lons, lats))
levels, colls = m.contourf(x,y,etopo,30,cmap=cm.jet,colors=None)
# draw coastlines.
m.drawcoastlines()
# draw a line around the map region.
m.drawmapboundary()
# draw parallels and meridians.
m.drawparallels(nx.arange(-60.,90.,30.),labels=[1,0,0,0],fontsize=10)
m.drawmeridians(nx.arange(0.,420.,60.),labels=[0,0,0,1],fontsize=10)
# add a title.
ax.set_title('Robinson Projection')
# add a single colorbar.
cax = fig.add_axes([0.25, 0.05, 0.5, 0.05],frameon=False)
fig.colorbar(colls.mappable, cax=cax, tickfmt='%d', orientation='horizontal')
canvas.print_figure('simpletest')
print 'image saved in simpletest.png'
<|endoftext|>


<commit_before>from math import log
import re
from termcolor import colored

from . import ffi, lib, seq, CffiObject
from .homopolymeric import hp_tokenize

global lib
"""Values for enum `align_type`.
"""
ALIGN_GLOBAL = lib.GLOBAL
ALIGN_LOCAL = lib.LOCAL
ALIGN_START_ANCHORED = lib.START_ANCHORED
ALIGN_END_ANCHORED = lib.END_ANCHORED
ALIGN_OVERLAP = lib.OVERLAP

class AlignParams(CffiObject):
    """Wraps the C struct align_params, see `libalign.h`

    Attributes:
        alphabet (Alphabet): alphabet used to represent the sequences.
        _c_subst_rows_ka (list[cffi.cdata]): has ownership of (keeps alive) C pointers
            to rows in the substitution matrix.
        _c_subst_full_ka (cffi.cdata): has ownership of (keeps alive) the C
            pointer to the full substitution matrix.
        c_obj (cffi.cdata): points to the underlying `align_params` struct.
    """
    def __init__(self, alphabet=None, subst_scores=[],
        go_score=0, ge_score=0, max_diversion=-1):
        self.alphabet = alphabet
        # each row in the subst matrix must be "owned" by an object that's kept
        # alive.
        self._c_subst_rows_ka = [ffi.new('double[]', subst_scores[i]) for i in range(self.alphabet.length)]
        self._c_subst_full_ka = ffi.new('double *[]', self._c_subst_rows_ka)
        self.c_obj = ffi.new('align_params*', {
            'alphabet': self.alphabet.c_obj,
            'subst_scores': self._c_subst_full_ka,
            'gap_open_score': go_score,
            'gap_extend_score': ge_score,
            'max_diversion': max_diversion,
        })

    @classmethod
    def subst_scores_from_probs(cls, subst_probs, alphabet, letter_dist=None):
        """Converts a substitution probability matrix to a substitution score
        matrix using a null-hypothesis letters distribution. The scores are
        natural logs of odds ratios.

        :param subst_probs(list[list]): as in seq.Sequence.mutate()
        :param alphabet(Alphabet): the underlying alphabet, needed since
            probabilities are in order of letter index in alphabet.
        :param letter_dist(list[float]): probability distributions of each
            letter of the alphabet in the null (random) hypothesis.

        :return subst_scores(list[list]): as expected in AlignParams.__init__().
        """
        L = alphabet.length
        if letter_dist is None:
            letter_dist = [1.0/L for k in range(L)]
        subst_scores = [[0 for _ in range(L)] for _ in range(L)]
        for i in range(L):
            assert(abs(1-sum([subst_probs[i][j] for j in range(L)])) < 0.001)
            for j in range(L):
                assert(subst_probs[i][j] > 0)
                assert(letter_dist[i] * letter_dist[j] != 0)
                subst_scores[i][j] = log(subst_probs[i][j]) \
                    - log(letter_dist[i]) - log(letter_dist[j])
        return subst_scores

    @classmethod
    def gap_scores_from_probs(cls, go_prob, ge_prob):
        """Converts gap open/extend probabilities to gap open/extend scores
        in an affine gap penalty scheme. If go_prob = 1 (which means gap
        extension does not require a separate "opening" event) we get a linear
        gap penalty scheme.
        """
        return log(go_prob), log(ge_prob)

    def __getattr__(self, name):
        """Allow attributes to access members of the underlying `align_params`
        struct. Additionally provides a `subst_scores` attribute which builds
        a python list of lists from the corresponding C data structure.
        """
        if name == 'subst_scores':
            idx = range(self.alphabet.length)
            return [[self.c_obj.subst_scores[i][j] for j in idx] for i in idx]
        else:
            return super(AlignParams, self).__getattr__(name)

    def score(self, S, T, opseq, S_min_idx=0, T_min_idx=0):
        """Calculates the score for an arbitray opseq over given sequences.
        Opseqs are allowed to be partial alignments (i.e finishing before
        reaching the end of frame).

            C = AlignParams(...)
            C.score('ACC', 'AGC', 'MSM')

        :param S(seq.Sequence)
        :param T(seq.Sequence)
        :param opseq(str)
        """
        score = 0.0
        i, j = S_min_idx, T_min_idx
        for op,num in hp_tokenize(opseq):
            if op in 'MS':
                for k in range(num):
                    score += self.subst_scores[S.c_idxseq[i+k]][T.c_idxseq[j+k]]
                i, j = i + num, j + num
            elif op in 'ID':
                score += self.gap_open_score + self.gap_extend_score * num
                if op == 'I':
                    j = j + num
                else:
                    i = i + num
            else:
                raise ValueError('Invalid edit operation: %c' % op)
        return score

class AlignProblem(CffiObject):
    """Wraps the C struct `align_problem', see `libalign.h`

    Attributes:
        S (Sequence): "from" sequence.
        T (Sequence): "to" sequence.
        params (AlignParams): parameters for the alignment problem.
        align_type (int): any of the ALIGN_* constants defined here (pointing to
            values of C enum `align_type`).
        c_obj (cffi.cdata): points to the underlying `align_problem` struct.
        c_dp_table (cffi.cdata): points to the underlying `double **` DP table.
    """
    def __init__(self, S=None, T=None, params=None, align_type=ALIGN_GLOBAL,
        S_min_idx=0, T_min_idx=0, S_max_idx=None, T_max_idx=None):

        # protect against index overflows
        S_max_idx = S.length if S_max_idx else S.length
        T_max_idx = T.length if T_max_idx else T.length
        assert(S_max_idx <= S.length)
        assert(T_max_idx <= T.length)

        self.S, self.T, self.params, self.align_type = S, T, params, align_type
        self.c_obj = ffi.new('align_problem*', {
            'S': S.c_idxseq,
            'T': T.c_idxseq,
            'S_min_idx': S_min_idx,
            'S_max_idx': S_max_idx,
            'T_min_idx': T_min_idx,
            'T_max_idx': T_max_idx,
            'type': align_type,
            'params': params.c_obj
        })
        self.c_dp_table = ffi.NULL;
        global lib

    def score(self, opseq):
        """Calculates the score for an arbitray opseq. Opseqs are allowed to be
        partial alignments (i.e finishing before reaching the end of frame).

            P = AlignProblem(...)
            P.score('MMMSSISSD') #=> 23.50
        """
        return self.params.score(self.S, self.T, opseq, self.S_min_idx, self.T_min_idx)

    def solve(self, print_dp_table=False):
        """Populates the DP table and traces back an (any) optimal alignment.

        :param print_dp_table(optional): whether or not (truthy) to print the
            fully calculated DP table.
        :returns: a transcript string with the specified format.
        """
        global lib
        if self.c_dp_table != ffi.NULL:
            lib.free_dp_table(self.c_obj, self.c_dp_table)
        #print 'S range: %d, %d' % (self.S_min_idx, self.S_max_idx)
        #print 'T range: %d, %d' % (self.T_min_idx, self.T_max_idx)
        self.c_dp_table = lib.define(self.c_obj)
        if self.c_dp_table == -1:
            raise('Got -1 from align.define().')

        self.opt = lib.solve(self.c_dp_table, self.c_obj)
        if print_dp_table:
            mat = self.dp_table
            for i in range(len(mat)):
                print [round(f,2) for f in mat[i]]
        if self.opt == ffi.NULL:
            print 'Err: No Alignment found (go=%.2f, ge=%.2f, max_div=%d)' % \
                (self.params.gap_open_score, self.params.gap_extend_score, self.params.max_diversion)
            self.opt = None
            return None, None
        score = self.opt.choices[0].score
        return self.opt, score

    def traceback(self):
        if self.opt is None:
            return None

        raw_transcript = lib.traceback(self.c_dp_table, self.c_obj, self.opt)
        if raw_transcript == ffi.NULL:
            return None
        return Transcript(raw_transcript=ffi.string(raw_transcript))

    def __getattr__(self, name):
        """Allow attributes to access members of the underlying `align_problem`
        struct. Additionally provide a `dp_table` attribute which builds a
        python list of lists from the corresponding C data structure.
        """
        if name == 'dp_table':
            i_idx = range(self.S_max_idx - self.S_min_idx + 1)
            j_idx = range(self.T_max_idx - self.T_min_idx + 1)
            def score(i,j):
                if self.c_dp_table[i][j].num_choices != 0:
                    return self.c_dp_table[i][j].choices[0].score
                else:
                    return None
            return [[score(i,j) for j in j_idx] for i in i_idx]
        else:
            return super(AlignProblem, self).__getattr__(name)

    def __setattr__(self, name, value):
        """TODO
        """
        if name in ['S_min_idx', 'S_max_idx', 'T_min_idx', 'T_max_idx', 'type']:
            setattr(self.c_obj, name, value)
        else:
            return super(AlignProblem, self).__setattr__(name, value)

# TODO make this a namedtuple as well
class Transcript(object):
    """A wrapper for alignment transcripts. Solutions to the alignment problem are
    represented by transcript strings with the following format:

        (<Si,Tj>),<score>:...

    Si and Tj are integers specifying the positiong along each string where
    the alignment begins. Score is the score of the transcript to 2 decimal
    places. What follows the ':' is a sequence of "ops" defined as follows:
        M match
        S substitution
        I insert
        D delete
    All op sequences begin with a B and insertion/deletions are meant to
    mean "from S to T".
    """
    def __init__(self, idx_S=0, idx_T=0, score=0.0, opseq='', raw_transcript=None):
        """FIXME Parses a raw transcript as given by `libalign.so` and populates
        class attributes. Raw transcripts are expected to have the following
        format:

            (<idx_S,idx_T>),<score>:<opseq>
        """
        if raw_transcript is None:
            self.idx_S, self.idx_T = idx_S, idx_T
            self.score, self.opseq = score, opseq
            return

        assert(re.match('\([0-9]+,[0-9]+\),[0-9-\.]+:[MISD]+', raw_transcript) is not None)
        infostr, opseq = raw_transcript.split(':', 1)
        indices, score = infostr.rsplit(',', 1)
        idx_S, idx_T = indices[1:-1].split(',') # skip the open/close parens
        self.idx_S, self.idx_T = int(idx_S), int(idx_T)
        self.opseq, self.score = opseq, float(score)

    def __repr__(self):
        return '(%d,%d),%.2f:%s' % (self.idx_S, self.idx_T, self.score, self.opseq)

    # TODO allow appending another transcript to the front or back

    def pretty_print(self, S, T, f, width=120, margin=20, colors=True):
        """Pretty prints a transcript to f.

        :param S(Sequence): "from" sequence.
        :param T(Sequence): "to" sequence.
        :param f: file handle to write the output to; for standard output use
            `sys.stdout`.
        :param width(optional): terminal width (int) used for wrapping; default
            is 120.
        :param margin(optional): length (int) of leading and trailing sequences
            in S and T before and after the alignment; default 20.
        :param colors(optional): whether or not (truthy) to use colors in
            output; default True.
        """
        assert(S.alphabet.letter_length == T.alphabet.letter_length)
        assert(S.alphabet.letters == T.alphabet.letters)
        letlen = S.alphabet.letter_length
        idx_S, idx_T = self.idx_S, self.idx_T

        slines = tlines = []
        sline = tline = ''

        def print_lines(sline, tline, f):
            maxlen = max(len(sline), len(tline))
            sline, tline = sline.rjust(maxlen), tline.rjust(maxlen)
            f.write('%s\n%s\n' % (sline,tline))

        def new_line(sline, tline, _idx_S, _idx_T, f):
            print_lines(sline, tline, f)
            sline, tline = 'S[%d]: ' % _idx_S, 'T[%d]: ' % _idx_T
            return (max(len(sline), len(tline)), sline, tline)

        # The pre margin:
        pre_margin = min(margin, max(idx_S, idx_T))
        sline = 'S[%d]: ' % max(0, idx_S - pre_margin)
        tline = 'T[%d]: ' % max(0, idx_T - pre_margin)
        counter = max(len(sline), len(tline))
        for i in reversed(range(1, pre_margin)):
            if counter >= width:
                counter, sline, tline = new_line(sline, tline, idx_S+i, idx_T+i, f)
            sline += S[idx_S-i] if i <= idx_S else ' '
            tline += T[idx_T-i] if i <= idx_T else ' '
            counter += letlen

        # The alignment itself:
        for i,op in enumerate(self.opseq):
            if counter >= width:
                counter, sline, tline = new_line(sline, tline, idx_S, idx_T, f)
            if op in 'MS':
                s, t = S[idx_S], T[idx_T]
                idx_S += 1
                idx_T += 1
            elif op == 'I':
                s, t = '-', T[idx_T]
                idx_T += 1
            elif op == 'D':
                s, t = S[idx_S], '-'
                idx_S += 1
            on_color = color = None
            if colors:
                if op in 'MS':
                    color = 'green' if op == 'M' else 'red'
                elif op == 'I':
                    on_color = 'on_red'
                elif op == 'D':
                    on_color = 'on_red'
            sline += colored(s, color=color, on_color=on_color)
            tline += colored(t, color=color, on_color=on_color)
            counter += letlen

        # The post margin:
        post_margin = min(margin, max(S.length-idx_S, T.length-idx_T))
        for i in range(post_margin):
            if counter >= width:
                counter, sline, tline = new_line(
                    sline, tline, idx_S+i, idx_T+i, f)
            sline += S[idx_S+i] if idx_S + i < S.length else ' '
            tline += T[idx_T+i] if idx_T + i < T.length else ' '
            counter += letlen

        print_lines(sline, tline, f)
<commit_msg>don't print an error if no alignment is found<commit_after>from math import log
import re
from termcolor import colored

from . import ffi, lib, seq, CffiObject
from .homopolymeric import hp_tokenize

global lib
"""Values for enum `align_type`.
"""
ALIGN_GLOBAL = lib.GLOBAL
ALIGN_LOCAL = lib.LOCAL
ALIGN_START_ANCHORED = lib.START_ANCHORED
ALIGN_END_ANCHORED = lib.END_ANCHORED
ALIGN_OVERLAP = lib.OVERLAP

class AlignParams(CffiObject):
    """Wraps the C struct align_params, see `libalign.h`

    Attributes:
        alphabet (Alphabet): alphabet used to represent the sequences.
        _c_subst_rows_ka (list[cffi.cdata]): has ownership of (keeps alive) C pointers
            to rows in the substitution matrix.
        _c_subst_full_ka (cffi.cdata): has ownership of (keeps alive) the C
            pointer to the full substitution matrix.
        c_obj (cffi.cdata): points to the underlying `align_params` struct.
    """
    def __init__(self, alphabet=None, subst_scores=[],
        go_score=0, ge_score=0, max_diversion=-1):
        self.alphabet = alphabet
        # each row in the subst matrix must be "owned" by an object that's kept
        # alive.
        self._c_subst_rows_ka = [ffi.new('double[]', subst_scores[i]) for i in range(self.alphabet.length)]
        self._c_subst_full_ka = ffi.new('double *[]', self._c_subst_rows_ka)
        self.c_obj = ffi.new('align_params*', {
            'alphabet': self.alphabet.c_obj,
            'subst_scores': self._c_subst_full_ka,
            'gap_open_score': go_score,
            'gap_extend_score': ge_score,
            'max_diversion': max_diversion,
        })

    @classmethod
    def subst_scores_from_probs(cls, subst_probs, alphabet, letter_dist=None):
        """Converts a substitution probability matrix to a substitution score
        matrix using a null-hypothesis letters distribution. The scores are
        natural logs of odds ratios.

        :param subst_probs(list[list]): as in seq.Sequence.mutate()
        :param alphabet(Alphabet): the underlying alphabet, needed since
            probabilities are in order of letter index in alphabet.
        :param letter_dist(list[float]): probability distributions of each
            letter of the alphabet in the null (random) hypothesis.

        :return subst_scores(list[list]): as expected in AlignParams.__init__().
        """
        L = alphabet.length
        if letter_dist is None:
            letter_dist = [1.0/L for k in range(L)]
        subst_scores = [[0 for _ in range(L)] for _ in range(L)]
        for i in range(L):
            assert(abs(1-sum([subst_probs[i][j] for j in range(L)])) < 0.001)
            for j in range(L):
                assert(subst_probs[i][j] > 0)
                assert(letter_dist[i] * letter_dist[j] != 0)
                subst_scores[i][j] = log(subst_probs[i][j]) \
                    - log(letter_dist[i]) - log(letter_dist[j])
        return subst_scores

    @classmethod
    def gap_scores_from_probs(cls, go_prob, ge_prob):
        """Converts gap open/extend probabilities to gap open/extend scores
        in an affine gap penalty scheme. If go_prob = 1 (which means gap
        extension does not require a separate "opening" event) we get a linear
        gap penalty scheme.
        """
        return log(go_prob), log(ge_prob)

    def __getattr__(self, name):
        """Allow attributes to access members of the underlying `align_params`
        struct. Additionally provides a `subst_scores` attribute which builds
        a python list of lists from the corresponding C data structure.
        """
        if name == 'subst_scores':
            idx = range(self.alphabet.length)
            return [[self.c_obj.subst_scores[i][j] for j in idx] for i in idx]
        else:
            return super(AlignParams, self).__getattr__(name)

    def score(self, S, T, opseq, S_min_idx=0, T_min_idx=0):
        """Calculates the score for an arbitray opseq over given sequences.
        Opseqs are allowed to be partial alignments (i.e finishing before
        reaching the end of frame).

            C = AlignParams(...)
            C.score('ACC', 'AGC', 'MSM')

        :param S(seq.Sequence)
        :param T(seq.Sequence)
        :param opseq(str)
        """
        score = 0.0
        i, j = S_min_idx, T_min_idx
        for op,num in hp_tokenize(opseq):
            if op in 'MS':
                for k in range(num):
                    score += self.subst_scores[S.c_idxseq[i+k]][T.c_idxseq[j+k]]
                i, j = i + num, j + num
            elif op in 'ID':
                score += self.gap_open_score + self.gap_extend_score * num
                if op == 'I':
                    j = j + num
                else:
                    i = i + num
            else:
                raise ValueError('Invalid edit operation: %c' % op)
        return score

class AlignProblem(CffiObject):
    """Wraps the C struct `align_problem', see `libalign.h`

    Attributes:
        S (Sequence): "from" sequence.
        T (Sequence): "to" sequence.
        params (AlignParams): parameters for the alignment problem.
        align_type (int): any of the ALIGN_* constants defined here (pointing to
            values of C enum `align_type`).
        c_obj (cffi.cdata): points to the underlying `align_problem` struct.
        c_dp_table (cffi.cdata): points to the underlying `double **` DP table.
    """
    def __init__(self, S=None, T=None, params=None, align_type=ALIGN_GLOBAL,
        S_min_idx=0, T_min_idx=0, S_max_idx=None, T_max_idx=None):

        # protect against index overflows
        S_max_idx = S.length if S_max_idx else S.length
        T_max_idx = T.length if T_max_idx else T.length
        assert(S_max_idx <= S.length)
        assert(T_max_idx <= T.length)

        self.S, self.T, self.params, self.align_type = S, T, params, align_type
        self.c_obj = ffi.new('align_problem*', {
            'S': S.c_idxseq,
            'T': T.c_idxseq,
            'S_min_idx': S_min_idx,
            'S_max_idx': S_max_idx,
            'T_min_idx': T_min_idx,
            'T_max_idx': T_max_idx,
            'type': align_type,
            'params': params.c_obj
        })
        self.c_dp_table = ffi.NULL;
        global lib

    def score(self, opseq):
        """Calculates the score for an arbitray opseq. Opseqs are allowed to be
        partial alignments (i.e finishing before reaching the end of frame).

            P = AlignProblem(...)
            P.score('MMMSSISSD') #=> 23.50
        """
        return self.params.score(self.S, self.T, opseq, self.S_min_idx, self.T_min_idx)

    def solve(self, print_dp_table=False):
        """Populates the DP table and traces back an (any) optimal alignment.

        :param print_dp_table(optional): whether or not (truthy) to print the
            fully calculated DP table.
        :returns: a transcript string with the specified format.
        """
        global lib
        if self.c_dp_table != ffi.NULL:
            lib.free_dp_table(self.c_obj, self.c_dp_table)
        self.c_dp_table = lib.define(self.c_obj)
        if self.c_dp_table == -1:
            raise('Got -1 from align.define().')

        self.opt = lib.solve(self.c_dp_table, self.c_obj)
        if print_dp_table:
            mat = self.dp_table
            for i in range(len(mat)):
                print [round(f,2) for f in mat[i]]
        if self.opt == ffi.NULL:
            self.opt = None
            return None, None
        score = self.opt.choices[0].score
        return self.opt, score

    def traceback(self):
        if self.opt is None:
            return None

        raw_transcript = lib.traceback(self.c_dp_table, self.c_obj, self.opt)
        if raw_transcript == ffi.NULL:
            return None
        return Transcript(raw_transcript=ffi.string(raw_transcript))

    def __getattr__(self, name):
        """Allow attributes to access members of the underlying `align_problem`
        struct. Additionally provide a `dp_table` attribute which builds a
        python list of lists from the corresponding C data structure.
        """
        if name == 'dp_table':
            i_idx = range(self.S_max_idx - self.S_min_idx + 1)
            j_idx = range(self.T_max_idx - self.T_min_idx + 1)
            def score(i,j):
                if self.c_dp_table[i][j].num_choices != 0:
                    return self.c_dp_table[i][j].choices[0].score
                else:
                    return None
            return [[score(i,j) for j in j_idx] for i in i_idx]
        else:
            return super(AlignProblem, self).__getattr__(name)

    def __setattr__(self, name, value):
        """TODO
        """
        if name in ['S_min_idx', 'S_max_idx', 'T_min_idx', 'T_max_idx', 'type']:
            setattr(self.c_obj, name, value)
        else:
            return super(AlignProblem, self).__setattr__(name, value)

# TODO make this a namedtuple as well
class Transcript(object):
    """A wrapper for alignment transcripts. Solutions to the alignment problem are
    represented by transcript strings with the following format:

        (<Si,Tj>),<score>:...

    Si and Tj are integers specifying the positiong along each string where
    the alignment begins. Score is the score of the transcript to 2 decimal
    places. What follows the ':' is a sequence of "ops" defined as follows:
        M match
        S substitution
        I insert
        D delete
    All op sequences begin with a B and insertion/deletions are meant to
    mean "from S to T".
    """
    def __init__(self, idx_S=0, idx_T=0, score=0.0, opseq='', raw_transcript=None):
        """FIXME Parses a raw transcript as given by `libalign.so` and populates
        class attributes. Raw transcripts are expected to have the following
        format:

            (<idx_S,idx_T>),<score>:<opseq>
        """
        if raw_transcript is None:
            self.idx_S, self.idx_T = idx_S, idx_T
            self.score, self.opseq = score, opseq
            return

        assert(re.match('\([0-9]+,[0-9]+\),[0-9-\.]+:[MISD]+', raw_transcript) is not None)
        infostr, opseq = raw_transcript.split(':', 1)
        indices, score = infostr.rsplit(',', 1)
        idx_S, idx_T = indices[1:-1].split(',') # skip the open/close parens
        self.idx_S, self.idx_T = int(idx_S), int(idx_T)
        self.opseq, self.score = opseq, float(score)

    def __repr__(self):
        return '(%d,%d),%.2f:%s' % (self.idx_S, self.idx_T, self.score, self.opseq)

    # TODO allow appending another transcript to the front or back

    def pretty_print(self, S, T, f, width=120, margin=20, colors=True):
        """Pretty prints a transcript to f.

        :param S(Sequence): "from" sequence.
        :param T(Sequence): "to" sequence.
        :param f: file handle to write the output to; for standard output use
            `sys.stdout`.
        :param width(optional): terminal width (int) used for wrapping; default
            is 120.
        :param margin(optional): length (int) of leading and trailing sequences
            in S and T before and after the alignment; default 20.
        :param colors(optional): whether or not (truthy) to use colors in
            output; default True.
        """
        assert(S.alphabet.letter_length == T.alphabet.letter_length)
        assert(S.alphabet.letters == T.alphabet.letters)
        letlen = S.alphabet.letter_length
        idx_S, idx_T = self.idx_S, self.idx_T

        slines = tlines = []
        sline = tline = ''

        def print_lines(sline, tline, f):
            maxlen = max(len(sline), len(tline))
            sline, tline = sline.rjust(maxlen), tline.rjust(maxlen)
            f.write('%s\n%s\n' % (sline,tline))

        def new_line(sline, tline, _idx_S, _idx_T, f):
            print_lines(sline, tline, f)
            sline, tline = 'S[%d]: ' % _idx_S, 'T[%d]: ' % _idx_T
            return (max(len(sline), len(tline)), sline, tline)

        # The pre margin:
        pre_margin = min(margin, max(idx_S, idx_T))
        sline = 'S[%d]: ' % max(0, idx_S - pre_margin)
        tline = 'T[%d]: ' % max(0, idx_T - pre_margin)
        counter = max(len(sline), len(tline))
        for i in reversed(range(1, pre_margin)):
            if counter >= width:
                counter, sline, tline = new_line(sline, tline, idx_S+i, idx_T+i, f)
            sline += S[idx_S-i] if i <= idx_S else ' '
            tline += T[idx_T-i] if i <= idx_T else ' '
            counter += letlen

        # The alignment itself:
        for i,op in enumerate(self.opseq):
            if counter >= width:
                counter, sline, tline = new_line(sline, tline, idx_S, idx_T, f)
            if op in 'MS':
                s, t = S[idx_S], T[idx_T]
                idx_S += 1
                idx_T += 1
            elif op == 'I':
                s, t = '-', T[idx_T]
                idx_T += 1
            elif op == 'D':
                s, t = S[idx_S], '-'
                idx_S += 1
            on_color = color = None
            if colors:
                if op in 'MS':
                    color = 'green' if op == 'M' else 'red'
                elif op == 'I':
                    on_color = 'on_red'
                elif op == 'D':
                    on_color = 'on_red'
            sline += colored(s, color=color, on_color=on_color)
            tline += colored(t, color=color, on_color=on_color)
            counter += letlen

        # The post margin:
        post_margin = min(margin, max(S.length-idx_S, T.length-idx_T))
        for i in range(post_margin):
            if counter >= width:
                counter, sline, tline = new_line(
                    sline, tline, idx_S+i, idx_T+i, f)
            sline += S[idx_S+i] if idx_S + i < S.length else ' '
            tline += T[idx_T+i] if idx_T + i < T.length else ' '
            counter += letlen

        print_lines(sline, tline, f)
<|endoftext|>


<commit_before>"""a pyqt test app that embeds javascript using QtScript,
defines qt properties and slots in py, and exposes them to js"""

import sys
from PyQt4 import QtGui, QtCore
from PyQt4.QtScript import QScriptEngine, QScriptValue
from PyQt4.QtCore import QObject, pyqtSignature, pyqtProperty, QVariant, QTimer

ps1 = ">>> "
ps2 = "... "

def printhello():
    print "Hello."

app = QtGui.QApplication(sys.argv)
window = QtGui.QWidget()
# Create our main window using a plain QWidget.
window.setWindowTitle("Signals")
# Set our window's title as "Signals".
button = QtGui.QPushButton("Press", window)
# Create, with "Press" as its caption,
# a child button in the window.
# By specifying a parent object,
# this new widget is automatically added to the same.
button.resize(200, 40)
# Resize our button to (200, 40) -> (X, Y)

#old style signals, works in pyqt 4.4
button.connect(button, QtCore.SIGNAL("clicked()"),\
    printhello) #QtCore.SLOT("quit()"))

#new style signals in 4.5, must upgdate to get these
#button.clicked.connect(printhello)

# Connect the button's click signal to the QApplication's quit() slot.
window.show()
# Show our window.

class Python(QObject):
    def __init__(self):
        QObject.__init__(self)
        self.setObjectName("python")
        # Does not work as expected :(
        self.setProperty("app", QVariant(self))
        self.t = QTimer(self)
        self.t.setObjectName("timer")

    @pyqtSignature("QString")
    def hello(self, name):
        print "Hello,", name

    def get_test(self):
        return 123

    test = pyqtProperty("int", get_test)

engine = QScriptEngine()
engine.evaluate("function dir(obj) { for(o in obj) print(o); }")

py = Python()
spy = engine.newQObject(py)
engine.globalObject().setProperty("python", spy)

#app.exec_()

print "Ctrl+D to quit"
prompt = ps1
code = ""
while True:
    app.processEvents()
    line = raw_input(prompt)
    if not line.strip():
        continue
    code = code + line + "\n"
    if engine.canEvaluate(code):
        result = engine.evaluate(code)
        if engine.hasUncaughtException():
            bt = engine.uncaughtExceptionBacktrace()
            print "Traceback:"
            print "\n".join(["  %s" % l for l in list(bt)])
            print engine.uncaughtException().toString()
        else:
            print result.toString()
        code = ""
        prompt = ps1
    else:
        prompt = ps2
<commit_msg>switched to the new style pyqt signal connecting, works on 4.5<commit_after>"""a pyqt test app that embeds javascript using QtScript,
defines qt properties and slots in py, and exposes them to js"""

import sys
from PyQt4 import QtGui, QtCore
from PyQt4.QtScript import QScriptEngine, QScriptValue
from PyQt4.QtCore import QObject, pyqtSignature, pyqtProperty, QVariant, QTimer

ps1 = ">>> "
ps2 = "... "

def printhello():
    print "Hello."

app = QtGui.QApplication(sys.argv)
window = QtGui.QWidget()
# Create our main window using a plain QWidget.
window.setWindowTitle("Signals")
# Set our window's title as "Signals".
button = QtGui.QPushButton("Press", window)
# Create, with "Press" as its caption,
# a child button in the window.
# By specifying a parent object,
# this new widget is automatically added to the same.
button.resize(200, 40)
# Resize our button to (200, 40) -> (X, Y)

"""old style signals, works in pyqt 4.4"""
#button.connect(button, QtCore.SIGNAL("clicked()"), printhello) #QtCore.SLOT("quit()"))

#new style signals in 4.5, must upgdate to get these
button.clicked.connect(printhello)

# Connect the button's click signal to the QApplication's quit() slot.
window.show()
# Show our window.

class Python(QObject):
    def __init__(self):
        QObject.__init__(self)
        self.setObjectName("python")
        # Does not work as expected :(
        self.setProperty("app", QVariant(self))
        self.t = QTimer(self)
        self.t.setObjectName("timer")

    @pyqtSignature("QString")
    def hello(self, name):
        print "Hello,", name

    def get_test(self):
        return 123

    test = pyqtProperty("int", get_test)

engine = QScriptEngine()
engine.evaluate("function dir(obj) { for(o in obj) print(o); }")

py = Python()
spy = engine.newQObject(py)
engine.globalObject().setProperty("python", spy)

#app.exec_()

print "Ctrl+D to quit"
prompt = ps1
code = ""
while True:
    app.processEvents()
    line = raw_input(prompt)
    if not line.strip():
        continue
    code = code + line + "\n"
    if engine.canEvaluate(code):
        result = engine.evaluate(code)
        if engine.hasUncaughtException():
            bt = engine.uncaughtExceptionBacktrace()
            print "Traceback:"
            print "\n".join(["  %s" % l for l in list(bt)])
            print engine.uncaughtException().toString()
        else:
            print result.toString()
        code = ""
        prompt = ps1
    else:
        prompt = ps2
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
Functions that we use for each and every script in the folder
/afs/psi.ch/project/EssentialMed/Dev/Analysis
"""

def AskUser(Blurb, Choices):
    """
    Ask for user input.
    Based on function in MasterThesisIvan.ini
    """
    print(Blurb)
    for Counter, Item in enumerate(sorted(Choices)):
        print '    * [' + str(Counter) + ']:', Item
    Selection = []
    while Selection not in range(len(Choices)):
        try:
            Selection = int(input(' '.join(['Please enter the choice you',
                                            'want [0-' +
                                            str(len(Choices) - 1) +
                                            ']:'])))
        except:
            print 'You actually have to select *something*'
        if Selection not in range(len(Choices)):
            print 'Try again with a valid choice'
    print 'You selected', sorted(Choices)[Selection]
    return sorted(Choices)[Selection]

def get_git_hash():
    """
    Get the current git hash from the repository.
    Good for saving this information into the log files of process.
    Based on http://stackoverflow.com/a/949391/323100 and
    http://stackoverflow.com/a/18283905/323100
    """
    from subprocess import Popen, PIPE
    gitprocess = Popen(['git', '--git-dir',
        '/afs/psi.ch/user/h/haberthuer/EssentialMed/Dev/.git', 'rev-parse',
        '--short', '--verify', 'HEAD'], stdout=PIPE)
    (output, _) = gitprocess.communicate()
    return output.strip()

def myLogger(Folder, LogFileName):
    """
    Since logging in a loop does always write to the first instaniated file,
    we make a little wrapper around the logger function to write to a defined
    log file.
    Based on http://stackoverflow.com/a/2754216/323100
    """
    import logging
    import os
    logger = logging.getLogger(LogFileName)
    # either set INFO or DEBUG
    #~ logger.setLevel(logging.DEBUG)
    logger.setLevel(logging.INFO)
    handler = logging.FileHandler(os.path.join(Folder, LogFileName), 'w')
    logger.addHandler(handler)
    return logger

def get_experiment_list(StartingFolder):
    """
    Get all folders (Experiment) and ExperimentIDs inside StartingFolder
    """
    import os
    from progressbar import ProgressBar, Percentage, Bar, ETA
    widgets = ['Reading: ', Percentage(), ' ', Bar(), ' ', ETA()]
    pbar = ProgressBar(widgets=widgets, maxval=2222).start()
    Experiment = []
    ExperimentID = []
    for root, dirs, files in os.walk(StartingFolder):
        #~ print 'Looking for experiment IDs in folder', os.path.basename(root)
        if (len(os.path.basename(root)) == 7
            or len(os.path.basename(root)) == 8) \
            and not os.path.basename(root).startswith('2014') \
            and not 'RECYCLE' in os.path.basename(root) \
            and not 'Pingseng' in os.path.basename(root) \
            and not 'Toshiba' in os.path.basename(root) \
            and not 'MT9' in os.path.basename(root) \
            and not 'AR0' in os.path.basename(root):
            Experiment.append(root)
            ExperimentID.append(os.path.basename(root))
            pbar.update(len(ExperimentID))
    pbar.finish()
    return Experiment, ExperimentID

def distance(Folder, chatty=False):
    import os
    import glob
    RawFileName = glob.glob(os.path.join(Folder, '*.raw'))[0]
    ScintillatorCMOSDistance = int(RawFileName.split('_')[5][:-5])
    if chatty:
        print 'Experiment', os.path.basename(Folder), \
            'was performed with a scintillator-CMOS-distance of', \
            ScintillatorCMOSDistance, 'mm'
    return ScintillatorCMOSDistance

def estimate_image_noise(image):
    '''
    # Noise estimation according to http://stackoverflow.com/a/25436112/323100
    # based on Immerkær1996
    '''
    height, width = image.shape
    M = [[1, -2, 1],
        [-2, 4, -2],
        [1, -2, 1]]
    from scipy.signal import convolve2d
    import numpy as np
    sigma = np.sum(np.sum(np.absolute(convolve2d(image, M))))
    sigma = sigma * np.sqrt(0.5 * np.pi) / (6 * (width - 2) * (height - 2))
    return sigma
<commit_msg>No funny UTF8 business here, please!<commit_after># -*- coding: utf-8 -*-

"""
Functions that we use for each and every script in the folder
/afs/psi.ch/project/EssentialMed/Dev/Analysis
"""

def AskUser(Blurb, Choices):
    """
    Ask for user input.
    Based on function in MasterThesisIvan.ini
    """
    print(Blurb)
    for Counter, Item in enumerate(sorted(Choices)):
        print '    * [' + str(Counter) + ']:', Item
    Selection = []
    while Selection not in range(len(Choices)):
        try:
            Selection = int(input(' '.join(['Please enter the choice you',
                                            'want [0-' +
                                            str(len(Choices) - 1) +
                                            ']:'])))
        except:
            print 'You actually have to select *something*'
        if Selection not in range(len(Choices)):
            print 'Try again with a valid choice'
    print 'You selected', sorted(Choices)[Selection]
    return sorted(Choices)[Selection]

def get_git_hash():
    """
    Get the current git hash from the repository.
    Good for saving this information into the log files of process.
    Based on http://stackoverflow.com/a/949391/323100 and
    http://stackoverflow.com/a/18283905/323100
    """
    from subprocess import Popen, PIPE
    gitprocess = Popen(['git', '--git-dir',
        '/afs/psi.ch/user/h/haberthuer/EssentialMed/Dev/.git', 'rev-parse',
        '--short', '--verify', 'HEAD'], stdout=PIPE)
    (output, _) = gitprocess.communicate()
    return output.strip()

def myLogger(Folder, LogFileName):
    """
    Since logging in a loop does always write to the first instaniated file,
    we make a little wrapper around the logger function to write to a defined
    log file.
    Based on http://stackoverflow.com/a/2754216/323100
    """
    import logging
    import os
    logger = logging.getLogger(LogFileName)
    # either set INFO or DEBUG
    #~ logger.setLevel(logging.DEBUG)
    logger.setLevel(logging.INFO)
    handler = logging.FileHandler(os.path.join(Folder, LogFileName), 'w')
    logger.addHandler(handler)
    return logger

def get_experiment_list(StartingFolder):
    """
    Get all folders (Experiment) and ExperimentIDs inside StartingFolder
    """
    import os
    from progressbar import ProgressBar, Percentage, Bar, ETA
    widgets = ['Reading: ', Percentage(), ' ', Bar(), ' ', ETA()]
    pbar = ProgressBar(widgets=widgets, maxval=2222).start()
    Experiment = []
    ExperimentID = []
    for root, dirs, files in os.walk(StartingFolder):
        #~ print 'Looking for experiment IDs in folder', os.path.basename(root)
        if (len(os.path.basename(root)) == 7
            or len(os.path.basename(root)) == 8) \
            and not os.path.basename(root).startswith('2014') \
            and not 'RECYCLE' in os.path.basename(root) \
            and not 'Pingseng' in os.path.basename(root) \
            and not 'Toshiba' in os.path.basename(root) \
            and not 'MT9' in os.path.basename(root) \
            and not 'AR0' in os.path.basename(root):
            Experiment.append(root)
            ExperimentID.append(os.path.basename(root))
            pbar.update(len(ExperimentID))
    pbar.finish()
    return Experiment, ExperimentID

def distance(Folder, chatty=False):
    import os
    import glob
    RawFileName = glob.glob(os.path.join(Folder, '*.raw'))[0]
    ScintillatorCMOSDistance = int(RawFileName.split('_')[5][:-5])
    if chatty:
        print 'Experiment', os.path.basename(Folder), \
            'was performed with a scintillator-CMOS-distance of', \
            ScintillatorCMOSDistance, 'mm'
    return ScintillatorCMOSDistance

def estimate_image_noise(image):
    '''
    # Noise estimation according to http://stackoverflow.com/a/25436112/323100
    # based on Immerkaer1996
    '''
    height, width = image.shape
    M = [[1, -2, 1],
        [-2, 4, -2],
        [1, -2, 1]]
    from scipy.signal import convolve2d
    import numpy as np
    sigma = np.sum(np.sum(np.absolute(convolve2d(image, M))))
    sigma = sigma * np.sqrt(0.5 * np.pi) / (6 * (width - 2) * (height - 2))
    return sigma
<|endoftext|>