<commit_before>'''
Created on Aug 27, 2014

@author: David Zwicker <dzwicker@seas.harvard.edu>

Provides a dictionary with default parameters for the mouse tracking.
This can also be seen as some kind of documentation of the available
parameters.
'''

from __future__ import division

from collections import namedtuple
import warnings

# enum of different units that we use
class UNIT(object):
    FACTOR = 1
    FRACTION = 2
    FOLDER = 3
    SUBFOLDER = 4
    COLOR = 5
    BOOLEAN = 6
    LENGTH_PIXEL = 11
    AREA_PIXEL = 12
    TIME_FRAMES = 20
    RATE_FRAMES = 21
    SPEED_PIXEL_FRAME = 30

Parameter = namedtuple('Parameter', ['key', 'default_value', 'unit', 'description'])

PARAMETER_LIST = [
    Parameter('base_folder', '.', UNIT.FOLDER,
              'Base folder in which all files are kept'),

    # Video input
    Parameter('video/filename_pattern', './raw_video/*.MTS', UNIT.SUBFOLDER,
              'Filename pattern used to look for videos'),
    Parameter('video/initial_adaptation_frames', 100, UNIT.TIME_FRAMES,
              'Number of initial frames to skip during analysis'),
    Parameter('video/blur_radius', 3, UNIT.LENGTH_PIXEL,
              'Radius of the blur filter to remove noise'),
    Parameter('video/blur_sigma_color', 10, UNIT.COLOR,
              'Standard deviation in color space of the bilateral filter'),
    Parameter('video/frames', None, None,
              'Frames of the video which are analyzed [start and end index should be given]'),
    Parameter('video/cropping_rect', None, None,
              "Rectangle to which the video is cropped. This can be either four "
              "numbers [left, top, width, height] or some string like "
              "'upper left', 'lower right', etc."),

    # Logging
    Parameter('logging/enabled',  True, UNIT.BOOLEAN,
              'Flag indicating whether logging is enabled'),
    Parameter('logging/folder', './logging/', UNIT.SUBFOLDER,
              'Folder to which the log file is written'),
    Parameter('logging/level_stderr', 'WARN', None,
              'Level of messages to log to stderr [standard python logging levels]'),
    Parameter('logging/level_file', 'INFO', None,
              'Level of messages to log to file if folder is set '
              '[standard python logging levels]'),

    # Debug
    Parameter('debug/output', [], None,
              'List of identifiers determining what debug output is produced'),
    Parameter('debug/use_multiprocessing', True, UNIT.BOOLEAN,
              'Flag indicating whether multiprocessing should be used to read '
              'and display videos'),
    Parameter('debug/folder', './debug/', UNIT.SUBFOLDER,
              'Folder to which debug videos are written'),
    Parameter('debug/output_period', 100, UNIT.TIME_FRAMES,
              'How often are frames written to the output file'),
    Parameter('debug/window_position', None, None,
              'Position (x, y) of the top-left corner of the debug window'),

    # Output
    Parameter('output/folder', './results/', UNIT.SUBFOLDER,
              'Folder to which the YAML and HDF5 result files are written'),
    Parameter('output/video/folder', './results/', UNIT.SUBFOLDER,
              'Folder to which the result video is written'),
    Parameter('output/output_period', 1, UNIT.TIME_FRAMES,
              'How often are frames written to the output file or shown on the '
              'screen'),
    Parameter('output/video/enabled', True, UNIT.BOOLEAN,
              'Flag determining whether the final video should be produced'),
    Parameter('output/video/extension', '.mov', None,
              'File extension used for debug videos'),
    Parameter('output/video/codec', 'libx264', None,
              'ffmpeg video codec used for debug videos'),
    Parameter('output/video/bitrate', '2000k', None,
              'Bitrate used for debug videos'),
    Parameter('output/hdf5_compression', 'gzip', None,
              'Compression algorithm to be used for the HDF5 data. Possible '
              'options might be None, "gzip", "lzf", and "szip".'),

    # Cage
    Parameter('cage/determine_boundaries', True, UNIT.BOOLEAN,
              'Flag indicating whether the cropping rectangle should be determined '
              'automatically. If False, we assume that the original video is '
              'already cropped'),
    Parameter('cage/boundary_detection_thresholds', [0.3, 0.1, 0.3, 0.9], None,
              'Thresholds for the boundary detection algorithm. The four values '
              'are the fraction of bright pixels necessary to define the '
              'boundary for [left, top, right, bottom], respectively.'),
    Parameter('cage/width_min', 600, UNIT.LENGTH_PIXEL,
              'Minimal width of the cage. This is only used to make a '
              'plausibility test of the results'),
    Parameter('cage/width_max', 800, UNIT.LENGTH_PIXEL,
              'Maximal width of the cage. This is only used to make a '
              'plausibility test of the results'),
    Parameter('cage/height_min', 350, UNIT.LENGTH_PIXEL,
              'Minimal height of the cage. This is only used to make a '
              'plausibility test of the results'),
    Parameter('cage/height_max', 500, UNIT.LENGTH_PIXEL,
              'Maximal height of the cage. This is only used to make a '
              'plausibility test of the results'),
    Parameter('cage/linescan_width', 30, UNIT.LENGTH_PIXEL,
              'Width of the line scan use to detect the cage frame'),
    Parameter('cage/linescan_smooth', 5, UNIT.LENGTH_PIXEL,
              'Standard deviation used for smoothing the line scan profile'),

    # Colors
    Parameter('colors/adaptation_interval', 1000, UNIT.TIME_FRAMES,
              'How often are the color estimates adapted'),
    Parameter('colors/std_min', 5, UNIT.COLOR,
              'Minimal standard deviation of sky and sand colors'),

    # Background and explored area
    Parameter('background/adaptation_rate', 1e-2, UNIT.RATE_FRAMES,
              'Rate at which the background is adapted'),
    Parameter('explored_area/adaptation_rate_outside', 1e-3, UNIT.RATE_FRAMES,
              'Rate at which the explored area is adapted outside of burrows'),
    Parameter('explored_area/adaptation_rate_burrows', 0, UNIT.RATE_FRAMES,
              'Rate at which the explored area is adapted inside burrows'),

    # Ground
    Parameter('ground/point_spacing', 20, UNIT.LENGTH_PIXEL,
              'Spacing of the support points describing the ground profile'),
    Parameter('ground/linescan_length', 50, UNIT.LENGTH_PIXEL,
              'Length of the line scan used to determine the ground profile'),
    Parameter('ground/slope_detector_max_factor', 0.4, UNIT.FACTOR,
              'Factor important in the ridge detection step, where the ridge '
              'is roughly located by looking at vertical line scans and points '
              'with large slopes are located. The smaller this factor, the more '
              'such points are detected and the further up the profile is '
              'estimated to be'),
    Parameter('ground/length_max', 1500, UNIT.LENGTH_PIXEL,
              'Maximal length of the ground profile above which it is rejected'),
    Parameter('ground/curvature_energy_factor', 2, UNIT.FACTOR,
              'Relative strength of the curvature energy to the image energy '
              'in the snake model of the ground line'),
    Parameter('ground/snake_energy_max', 5, UNIT.FACTOR,
              'Determines the maximal energy the snake is allowed to have'),
    Parameter('ground/slope_max', 3, UNIT.FRACTION,
              'Maximal slope of the side ridges'),
    Parameter('ground/frame_margin', 100, UNIT.LENGTH_PIXEL,
              'Width of the margin to the frame in which the ground profile is '
              'not determined'),
    Parameter('ground/grabcut_uncertainty_margin', 50, UNIT.LENGTH_PIXEL,
              'Width of the region around the estimated profile, in which '
              'the GrabCut algorithm may optimize'),
    Parameter('ground/adaptation_interval', 100, UNIT.TIME_FRAMES,
              'How often is the ground profile adapted'),
    Parameter('ground/ridge_width', 5, UNIT.LENGTH_PIXEL,
              'Width of the ground profile ridge'),
    Parameter('ground/smoothing_sigma', 300, UNIT.TIME_FRAMES,
              'Standard deviation for Gaussian smoothing over time'),

    # Mouse and the associated tracking
    Parameter('mouse/intensity_threshold', 1, UNIT.FACTOR,
              'Determines how much brighter than the background (usually the '
              'sky) the mouse has to be. This value is measured in terms of '
              'standard deviations of the sky color'),
    Parameter('mouse/model_radius', 25, UNIT.LENGTH_PIXEL,
              'Radius of the mouse model'),
    Parameter('mouse/area_max', 5000, UNIT.AREA_PIXEL,
              'Maximal area of a feature to be considered in tracking'),
    Parameter('mouse/area_min', 100, UNIT.AREA_PIXEL,
              'Minimal area of a feature to be considered in tracking'),
    Parameter('mouse/area_mean', 700, UNIT.AREA_PIXEL,
              'Mean area of a mouse, which is used to score the mouse'),
    Parameter('mouse/speed_max', 30, UNIT.SPEED_PIXEL_FRAME,
              'Maximal speed of the mouse'),
    Parameter('mouse/max_rel_area_change', 0.5, UNIT.FACTOR,
              'Maximal area change allowed between consecutive frames'),

    Parameter('tracking/weight', 0.5, UNIT.FACTOR,
              'Relative weight of distance vs. size of objects for matching them'),
    Parameter('tracking/moving_window', 20, UNIT.TIME_FRAMES,
              'Number of consecutive frames used for motion detection'),
    Parameter('tracking/moving_threshold', 15, UNIT.SPEED_PIXEL_FRAME,
              'Threshold speed above which an object is said to be moving'),
    Parameter('tracking/object_count_max', 7, None,
              'Maximal number of objects allowed in a single frame. If there are '
              'more objects, the entire frame is discarded'),
    Parameter('tracking/time_scale', 10, UNIT.TIME_FRAMES,
              'Time duration of not seeing the mouse after which we do not '
              'know where it is anymore'),
    Parameter('tracking/tolerated_overlap', 10, UNIT.TIME_FRAMES,
              'How much are two consecutive tracks allowed to overlap'),
    Parameter('tracking/initial_score_threshold', 1000, UNIT.FACTOR,
              'Initial threshold for building the tracking graph'),
    Parameter('tracking/end_node_interval', 1000, UNIT.TIME_FRAMES,
              'What time duration do we consider for start and end nodes'),
    Parameter('tracking/splitting_duration_min', 10, UNIT.TIME_FRAMES,
              'Track duration above which two overlapping tracks are split'),

    # Burrows
    Parameter('burrows/adaptation_interval', 100, UNIT.TIME_FRAMES,
              'How often are the burrow shapes adapted'),
    Parameter('burrows/cage_margin', 30, UNIT.LENGTH_PIXEL,
              'Margin of a potential burrow to the cage boundary'),
    Parameter('burrows/width', 20, UNIT.LENGTH_PIXEL,
              'Typical width of a burrow'),
    Parameter('burrows/width_min', 10, UNIT.LENGTH_PIXEL,
              'Minimal width of a burrow'),
    Parameter('burrows/area_min', 400, UNIT.AREA_PIXEL,
              'Minimal area a burrow cross section has to have'),
    Parameter('burrows/ground_point_distance', 10, UNIT.LENGTH_PIXEL,
              'Maximal distance of ground profile to outline points that are '
              'considered exit points'),
    Parameter('burrows/centerline_segment_length', 25, UNIT.LENGTH_PIXEL,
              'Length of a segment of the center line of a burrow'),
    Parameter('burrows/curvature_radius_max', 50, UNIT.LENGTH_PIXEL,
              'Maximal radius of curvature the centerline is allowed to have'),
    Parameter('burrows/grabcut_burrow_core_area_min', 500, UNIT.AREA_PIXEL,
              'Minimal area the sure region of the mask for the grab cut '
              'algorithm is supposed to have'),
    Parameter('burrows/fitting_length_threshold', 100, UNIT.LENGTH_PIXEL,
              'Length above which burrows are refined by fitting'),
    Parameter('burrows/fitting_width_threshold', 30, UNIT.LENGTH_PIXEL,
              'Width below which burrows are refined by fitting'),
    Parameter('burrows/fitting_edge_width', 3, UNIT.LENGTH_PIXEL,
              'Width of the burrow edge used in the template for fitting'),
    Parameter('burrows/fitting_edge_R2min', -10, UNIT.FACTOR,
              'Minimal value of the Coefficient of Determination (R^2) above '
              'which the fit of a burrow edge is considered good enough and '
              'will be used'),
    Parameter('burrows/outline_simplification_threshold', 0.005, UNIT.FACTOR,
              'Determines how much the burrow outline might be simplified. '
              'The quantity determines by what fraction the total outline '
              'length is allowed to change'),
    Parameter('burrows/simplification_threshold_area', 50, UNIT.AREA_PIXEL,
              'Burrow outline points are removed when the resulting effective '
              'change is below this threshold'),

    Parameter('factor_length', 1, UNIT.FACTOR,
              'A factor by which all length scales will be scaled'),

    # Computation resources
    Parameter('project/symlink_folder', None, None,
              'If set, a symlink pointing to the base_folder will be created '
              'in this directory when a project is created.'),
    Parameter('resources/notification_email', 'dzwicker@seas.harvard.edu', None,
              'Email address of the user to be notified in case of problems.'),
    Parameter('resources/slurm_partition', 'general', None,
              'Name of the slurm partition to use for submitting jobs'),
    Parameter('resources/pass1/cores', 4, None, 'Number of cores for pass 1'),
    Parameter('resources/pass1/time', 30*60, None, 'Maximal computation minutes for pass 1'),
    Parameter('resources/pass1/memory', 1000, None, 'Maximal RAM per core for pass 1 [in MB]'),
    Parameter('resources/pass2/cores', 2, None, 'Number of cores for pass 2'),
    Parameter('resources/pass2/time', 20*60, None, 'Maximal computation minutes for pass 2'),
    Parameter('resources/pass2/memory', 8000, None, 'Maximal RAM per core for pass 2 [in MB]'),
]

PARAMETERS = {p.key: p for p in PARAMETER_LIST}
PARAMETERS_DEFAULT = {p.key: p.default_value for p in PARAMETER_LIST}

def set_base_folder(parameters, folder, include_default=False):
    """ changes the base folder of all folders given in the parameter
    dictionary.
    include_default is a flag indicating whether the default parameters
    describing folders should also be included. """

    warnings.warn("Base folder is a parameter now.", DeprecationWarning)

    # convert to plain dictionary if it is anything else
    parameters_type = type(parameters)
    if parameters_type != dict:
        parameters = parameters.to_dict(flatten=True)

    # add all default folders, which will be changed later
    if include_default:
        for p in PARAMETER_LIST:
            if p.unit == UNIT.SUBFOLDER and p.key not in parameters:
                parameters[p.key] = p.default_value

    # set the base folder
    parameters['base_folder'] = folder

    # return the result as the original type
    return parameters_type(parameters)

def scale_parameters(parameters, factor_length=1, factor_time=1):
    """ takes a dictionary of parameters and scales them according to their
    unit and the given scale factors """
    # convert to plain dictionary if it is anything else
    parameters_type = type(parameters)
    if parameters_type != dict:
        parameters = parameters.to_dict(flatten=True)

    # scale each parameter in the list
    for key in parameters:
        unit = PARAMETERS[key].unit
        if unit == UNIT.LENGTH_PIXEL:
            parameters[key] *= factor_length
        elif unit == UNIT.AREA_PIXEL:
            parameters[key] *= factor_length**2
        elif unit == UNIT.TIME_FRAMES:
            parameters[key] *= factor_time
        elif unit == UNIT.RATE_FRAMES:
            parameters[key] /= factor_time
        elif unit == UNIT.SPEED_PIXEL_FRAME:
            parameters[key] *= factor_length/factor_time

    # return the result as the original type
    return parameters_type(parameters)

<commit_msg>Changed default parameters for cage finding<commit_after>'''
Created on Aug 27, 2014

@author: David Zwicker <dzwicker@seas.harvard.edu>

Provides a dictionary with default parameters for the mouse tracking.
This can also be seen as some kind of documentation of the available
parameters.
'''

from __future__ import division

from collections import namedtuple
import warnings

# enum of different units that we use
class UNIT(object):
    FACTOR = 1
    FRACTION = 2
    FOLDER = 3
    SUBFOLDER = 4
    COLOR = 5
    BOOLEAN = 6
    LENGTH_PIXEL = 11
    AREA_PIXEL = 12
    TIME_FRAMES = 20
    RATE_FRAMES = 21
    SPEED_PIXEL_FRAME = 30

Parameter = namedtuple('Parameter', ['key', 'default_value', 'unit', 'description'])

PARAMETER_LIST = [
    Parameter('base_folder', '.', UNIT.FOLDER,
              'Base folder in which all files are kept'),

    # Video input
    Parameter('video/filename_pattern', './raw_video/*.MTS', UNIT.SUBFOLDER,
              'Filename pattern used to look for videos'),
    Parameter('video/initial_adaptation_frames', 100, UNIT.TIME_FRAMES,
              'Number of initial frames to skip during analysis'),
    Parameter('video/blur_radius', 3, UNIT.LENGTH_PIXEL,
              'Radius of the blur filter to remove noise'),
    Parameter('video/blur_sigma_color', 10, UNIT.COLOR,
              'Standard deviation in color space of the bilateral filter'),
    Parameter('video/frames', None, None,
              'Frames of the video which are analyzed [start and end index should be given]'),
    Parameter('video/cropping_rect', None, None,
              "Rectangle to which the video is cropped. This can be either four "
              "numbers [left, top, width, height] or some string like "
              "'upper left', 'lower right', etc."),

    # Logging
    Parameter('logging/enabled',  True, UNIT.BOOLEAN,
              'Flag indicating whether logging is enabled'),
    Parameter('logging/folder', './logging/', UNIT.SUBFOLDER,
              'Folder to which the log file is written'),
    Parameter('logging/level_stderr', 'WARN', None,
              'Level of messages to log to stderr [standard python logging levels]'),
    Parameter('logging/level_file', 'INFO', None,
              'Level of messages to log to file if folder is set '
              '[standard python logging levels]'),

    # Debug
    Parameter('debug/output', [], None,
              'List of identifiers determining what debug output is produced'),
    Parameter('debug/use_multiprocessing', True, UNIT.BOOLEAN,
              'Flag indicating whether multiprocessing should be used to read '
              'and display videos'),
    Parameter('debug/folder', './debug/', UNIT.SUBFOLDER,
              'Folder to which debug videos are written'),
    Parameter('debug/output_period', 100, UNIT.TIME_FRAMES,
              'How often are frames written to the output file'),
    Parameter('debug/window_position', None, None,
              'Position (x, y) of the top-left corner of the debug window'),

    # Output
    Parameter('output/folder', './results/', UNIT.SUBFOLDER,
              'Folder to which the YAML and HDF5 result files are written'),
    Parameter('output/video/folder', './results/', UNIT.SUBFOLDER,
              'Folder to which the result video is written'),
    Parameter('output/output_period', 1, UNIT.TIME_FRAMES,
              'How often are frames written to the output file or shown on the '
              'screen'),
    Parameter('output/video/enabled', True, UNIT.BOOLEAN,
              'Flag determining whether the final video should be produced'),
    Parameter('output/video/extension', '.mov', None,
              'File extension used for debug videos'),
    Parameter('output/video/codec', 'libx264', None,
              'ffmpeg video codec used for debug videos'),
    Parameter('output/video/bitrate', '2000k', None,
              'Bitrate used for debug videos'),
    Parameter('output/hdf5_compression', 'gzip', None,
              'Compression algorithm to be used for the HDF5 data. Possible '
              'options might be None, "gzip", "lzf", and "szip".'),

    # Cage
    Parameter('cage/determine_boundaries', True, UNIT.BOOLEAN,
              'Flag indicating whether the cropping rectangle should be determined '
              'automatically. If False, we assume that the original video is '
              'already cropped'),
    Parameter('cage/boundary_detection_thresholds', [0.1, 0.1, 0.1, 0.9], None,
              'Thresholds for the boundary detection algorithm. The four values '
              'are the fraction of bright pixels necessary to define the '
              'boundary for [left, top, right, bottom], respectively.'),
    Parameter('cage/width_min', 600, UNIT.LENGTH_PIXEL,
              'Minimal width of the cage. This is only used to make a '
              'plausibility test of the results'),
    Parameter('cage/width_max', 800, UNIT.LENGTH_PIXEL,
              'Maximal width of the cage. This is only used to make a '
              'plausibility test of the results'),
    Parameter('cage/height_min', 350, UNIT.LENGTH_PIXEL,
              'Minimal height of the cage. This is only used to make a '
              'plausibility test of the results'),
    Parameter('cage/height_max', 500, UNIT.LENGTH_PIXEL,
              'Maximal height of the cage. This is only used to make a '
              'plausibility test of the results'),
    Parameter('cage/linescan_width', 30, UNIT.LENGTH_PIXEL,
              'Width of the line scan use to detect the cage frame'),
    Parameter('cage/linescan_smooth', 5, UNIT.LENGTH_PIXEL,
              'Standard deviation used for smoothing the line scan profile'),

    # Colors
    Parameter('colors/adaptation_interval', 1000, UNIT.TIME_FRAMES,
              'How often are the color estimates adapted'),
    Parameter('colors/std_min', 5, UNIT.COLOR,
              'Minimal standard deviation of sky and sand colors'),

    # Background and explored area
    Parameter('background/adaptation_rate', 1e-2, UNIT.RATE_FRAMES,
              'Rate at which the background is adapted'),
    Parameter('explored_area/adaptation_rate_outside', 1e-3, UNIT.RATE_FRAMES,
              'Rate at which the explored area is adapted outside of burrows'),
    Parameter('explored_area/adaptation_rate_burrows', 0, UNIT.RATE_FRAMES,
              'Rate at which the explored area is adapted inside burrows'),

    # Ground
    Parameter('ground/point_spacing', 20, UNIT.LENGTH_PIXEL,
              'Spacing of the support points describing the ground profile'),
    Parameter('ground/linescan_length', 50, UNIT.LENGTH_PIXEL,
              'Length of the line scan used to determine the ground profile'),
    Parameter('ground/slope_detector_max_factor', 0.4, UNIT.FACTOR,
              'Factor important in the ridge detection step, where the ridge '
              'is roughly located by looking at vertical line scans and points '
              'with large slopes are located. The smaller this factor, the more '
              'such points are detected and the further up the profile is '
              'estimated to be'),
    Parameter('ground/length_max', 1500, UNIT.LENGTH_PIXEL,
              'Maximal length of the ground profile above which it is rejected'),
    Parameter('ground/curvature_energy_factor', 2, UNIT.FACTOR,
              'Relative strength of the curvature energy to the image energy '
              'in the snake model of the ground line'),
    Parameter('ground/snake_energy_max', 5, UNIT.FACTOR,
              'Determines the maximal energy the snake is allowed to have'),
    Parameter('ground/slope_max', 3, UNIT.FRACTION,
              'Maximal slope of the side ridges'),
    Parameter('ground/frame_margin', 100, UNIT.LENGTH_PIXEL,
              'Width of the margin to the frame in which the ground profile is '
              'not determined'),
    Parameter('ground/grabcut_uncertainty_margin', 50, UNIT.LENGTH_PIXEL,
              'Width of the region around the estimated profile, in which '
              'the GrabCut algorithm may optimize'),
    Parameter('ground/adaptation_interval', 100, UNIT.TIME_FRAMES,
              'How often is the ground profile adapted'),
    Parameter('ground/ridge_width', 5, UNIT.LENGTH_PIXEL,
              'Width of the ground profile ridge'),
    Parameter('ground/smoothing_sigma', 300, UNIT.TIME_FRAMES,
              'Standard deviation for Gaussian smoothing over time'),

    # Mouse and the associated tracking
    Parameter('mouse/intensity_threshold', 1, UNIT.FACTOR,
              'Determines how much brighter than the background (usually the '
              'sky) the mouse has to be. This value is measured in terms of '
              'standard deviations of the sky color'),
    Parameter('mouse/model_radius', 25, UNIT.LENGTH_PIXEL,
              'Radius of the mouse model'),
    Parameter('mouse/area_max', 5000, UNIT.AREA_PIXEL,
              'Maximal area of a feature to be considered in tracking'),
    Parameter('mouse/area_min', 100, UNIT.AREA_PIXEL,
              'Minimal area of a feature to be considered in tracking'),
    Parameter('mouse/area_mean', 700, UNIT.AREA_PIXEL,
              'Mean area of a mouse, which is used to score the mouse'),
    Parameter('mouse/speed_max', 30, UNIT.SPEED_PIXEL_FRAME,
              'Maximal speed of the mouse'),
    Parameter('mouse/max_rel_area_change', 0.5, UNIT.FACTOR,
              'Maximal area change allowed between consecutive frames'),

    Parameter('tracking/weight', 0.5, UNIT.FACTOR,
              'Relative weight of distance vs. size of objects for matching them'),
    Parameter('tracking/moving_window', 20, UNIT.TIME_FRAMES,
              'Number of consecutive frames used for motion detection'),
    Parameter('tracking/moving_threshold', 15, UNIT.SPEED_PIXEL_FRAME,
              'Threshold speed above which an object is said to be moving'),
    Parameter('tracking/object_count_max', 7, None,
              'Maximal number of objects allowed in a single frame. If there are '
              'more objects, the entire frame is discarded'),
    Parameter('tracking/time_scale', 10, UNIT.TIME_FRAMES,
              'Time duration of not seeing the mouse after which we do not '
              'know where it is anymore'),
    Parameter('tracking/tolerated_overlap', 10, UNIT.TIME_FRAMES,
              'How much are two consecutive tracks allowed to overlap'),
    Parameter('tracking/initial_score_threshold', 1000, UNIT.FACTOR,
              'Initial threshold for building the tracking graph'),
    Parameter('tracking/end_node_interval', 1000, UNIT.TIME_FRAMES,
              'What time duration do we consider for start and end nodes'),
    Parameter('tracking/splitting_duration_min', 10, UNIT.TIME_FRAMES,
              'Track duration above which two overlapping tracks are split'),

    # Burrows
    Parameter('burrows/adaptation_interval', 100, UNIT.TIME_FRAMES,
              'How often are the burrow shapes adapted'),
    Parameter('burrows/cage_margin', 30, UNIT.LENGTH_PIXEL,
              'Margin of a potential burrow to the cage boundary'),
    Parameter('burrows/width', 20, UNIT.LENGTH_PIXEL,
              'Typical width of a burrow'),
    Parameter('burrows/width_min', 10, UNIT.LENGTH_PIXEL,
              'Minimal width of a burrow'),
    Parameter('burrows/area_min', 400, UNIT.AREA_PIXEL,
              'Minimal area a burrow cross section has to have'),
    Parameter('burrows/ground_point_distance', 10, UNIT.LENGTH_PIXEL,
              'Maximal distance of ground profile to outline points that are '
              'considered exit points'),
    Parameter('burrows/centerline_segment_length', 25, UNIT.LENGTH_PIXEL,
              'Length of a segment of the center line of a burrow'),
    Parameter('burrows/curvature_radius_max', 50, UNIT.LENGTH_PIXEL,
              'Maximal radius of curvature the centerline is allowed to have'),
    Parameter('burrows/grabcut_burrow_core_area_min', 500, UNIT.AREA_PIXEL,
              'Minimal area the sure region of the mask for the grab cut '
              'algorithm is supposed to have'),
    Parameter('burrows/fitting_length_threshold', 100, UNIT.LENGTH_PIXEL,
              'Length above which burrows are refined by fitting'),
    Parameter('burrows/fitting_width_threshold', 30, UNIT.LENGTH_PIXEL,
              'Width below which burrows are refined by fitting'),
    Parameter('burrows/fitting_edge_width', 3, UNIT.LENGTH_PIXEL,
              'Width of the burrow edge used in the template for fitting'),
    Parameter('burrows/fitting_edge_R2min', -10, UNIT.FACTOR,
              'Minimal value of the Coefficient of Determination (R^2) above '
              'which the fit of a burrow edge is considered good enough and '
              'will be used'),
    Parameter('burrows/outline_simplification_threshold', 0.005, UNIT.FACTOR,
              'Determines how much the burrow outline might be simplified. '
              'The quantity determines by what fraction the total outline '
              'length is allowed to change'),
    Parameter('burrows/simplification_threshold_area', 50, UNIT.AREA_PIXEL,
              'Burrow outline points are removed when the resulting effective '
              'change is below this threshold'),

    Parameter('factor_length', 1, UNIT.FACTOR,
              'A factor by which all length scales will be scaled'),

    # Computation resources
    Parameter('project/symlink_folder', None, None,
              'If set, a symlink pointing to the base_folder will be created '
              'in this directory when a project is created.'),
    Parameter('resources/notification_email', 'dzwicker@seas.harvard.edu', None,
              'Email address of the user to be notified in case of problems.'),
    Parameter('resources/slurm_partition', 'general', None,
              'Name of the slurm partition to use for submitting jobs'),
    Parameter('resources/pass1/cores', 4, None, 'Number of cores for pass 1'),
    Parameter('resources/pass1/time', 30*60, None, 'Maximal computation minutes for pass 1'),
    Parameter('resources/pass1/memory', 1000, None, 'Maximal RAM per core for pass 1 [in MB]'),
    Parameter('resources/pass2/cores', 2, None, 'Number of cores for pass 2'),
    Parameter('resources/pass2/time', 20*60, None, 'Maximal computation minutes for pass 2'),
    Parameter('resources/pass2/memory', 8000, None, 'Maximal RAM per core for pass 2 [in MB]'),
]

PARAMETERS = {p.key: p for p in PARAMETER_LIST}
PARAMETERS_DEFAULT = {p.key: p.default_value for p in PARAMETER_LIST}

def set_base_folder(parameters, folder, include_default=False):
    """ changes the base folder of all folders given in the parameter
    dictionary.
    include_default is a flag indicating whether the default parameters
    describing folders should also be included. """

    warnings.warn("Base folder is a parameter now.", DeprecationWarning)

    # convert to plain dictionary if it is anything else
    parameters_type = type(parameters)
    if parameters_type != dict:
        parameters = parameters.to_dict(flatten=True)

    # add all default folders, which will be changed later
    if include_default:
        for p in PARAMETER_LIST:
            if p.unit == UNIT.SUBFOLDER and p.key not in parameters:
                parameters[p.key] = p.default_value

    # set the base folder
    parameters['base_folder'] = folder

    # return the result as the original type
    return parameters_type(parameters)

def scale_parameters(parameters, factor_length=1, factor_time=1):
    """ takes a dictionary of parameters and scales them according to their
    unit and the given scale factors """
    # convert to plain dictionary if it is anything else
    parameters_type = type(parameters)
    if parameters_type != dict:
        parameters = parameters.to_dict(flatten=True)

    # scale each parameter in the list
    for key in parameters:
        unit = PARAMETERS[key].unit
        if unit == UNIT.LENGTH_PIXEL:
            parameters[key] *= factor_length
        elif unit == UNIT.AREA_PIXEL:
            parameters[key] *= factor_length**2
        elif unit == UNIT.TIME_FRAMES:
            parameters[key] *= factor_time
        elif unit == UNIT.RATE_FRAMES:
            parameters[key] /= factor_time
        elif unit == UNIT.SPEED_PIXEL_FRAME:
            parameters[key] *= factor_length/factor_time

    # return the result as the original type
    return parameters_type(parameters)

<|endoftext|>


<commit_before>import os
import subprocess
from moviepy.tools import subprocess_call

def gif_to_directory(gif_file,dirName=None):
    """
    Stores all the frames of the given .gif file
    into the directory ``dirName``. If ``dirName``
    is not provided, the directory has the same name
    as the .gif file. Supports transparency.
    Returns the directory name.

    Example:

    >>> d = gif_to_directory("animated-earth.gif")
    >>> clip = DirectoryClip(d,fps=3)

    """

    if dirName is None:
        name, ext = os.path.splitext(gif_file)
        dirName = name

    try:
        os.mkdir(dirName)
    except:
        pass

    subprocess_call ["convert", "-coalesce", gif_file,
             os.path.join(dirName,"%04d.png")])
<commit_msg>Fixed syntax error in io/imageMagick_tools.py<commit_after>import os
import subprocess
from moviepy.tools import subprocess_call

def gif_to_directory(gif_file,dirName=None):
    """
    Stores all the frames of the given .gif file
    into the directory ``dirName``. If ``dirName``
    is not provided, the directory has the same name
    as the .gif file. Supports transparency.
    Returns the directory name.

    Example:

    >>> d = gif_to_directory("animated-earth.gif")
    >>> clip = DirectoryClip(d,fps=3)

    """

    if dirName is None:
        name, ext = os.path.splitext(gif_file)
        dirName = name

    try:
        os.mkdir(dirName)
    except:
        pass

    subprocess_call(["convert", "-coalesce", gif_file,
             os.path.join(dirName,"%04d.png")])
<|endoftext|>


<commit_before>import os
os.environ["KIVY_NO_ARGS"] = "1"

from queue import Queue

import time
from kivy import Config, Logger
from kivy.base import runTouchApp, EventLoop
from kivy.clock import Clock
from kivy.uix.widget import Widget

import mpfmc
from mpf.core.utility_functions import Util
from mpf.tests.MpfBcpTestCase import MockBcpClient
from mpf.tests.MpfTestCase import MpfTestCase
from mpfmc.core.config_processor import ConfigProcessor
from mpfmc.core.utils import load_machine_config

class TestBcpClient(MockBcpClient):
    def __init__(self, machine, name, bcp):
        super().__init__(machine, name, bcp)
        self.queue = Queue()
        self.exit_on_close = False

    def send(self, bcp_command, kwargs):
        self.queue.put((bcp_command, kwargs))

    def receive(self, bcp_command, callback=None, rawbytes=None, **kwargs):
        if rawbytes:
            kwargs['rawbytes'] = rawbytes
        self.receive_queue.put_nowait((bcp_command, kwargs))

        if callback:
            callback()

class MpfIntegrationTest(MpfTestCase):

    def get_options(self):
        return dict(machine_path=self.getAbsoluteMachinePath(),
                    mcconfigfile='mpfmc/mcconfig.yaml',
                    configfile=Util.string_to_list(self.getConfigFile()),
                    bcp=False)

    def preprocess_config(self, config):
        # TODO this method is copied from the mc.py launcher. Prob a better way
        kivy_config = config['kivy_config']

        try:
            kivy_config['graphics'].update(config['displays']['window'])
        except KeyError:
            pass

        try:
            kivy_config['graphics'].update(config['window'])
        except KeyError:
            pass

        if 'top' in kivy_config['graphics'] and 'left' in kivy_config['graphics']:
            kivy_config['graphics']['position'] = 'custom'

        for section, settings in kivy_config.items():
            for k, v in settings.items():
                try:
                    if k in Config[section]:
                        Config.set(section, k, v)
                except KeyError:
                    continue

    def _start_app_as_slave(self):
        # from app::run
        if not self.mc.built:
            self.mc.load_config()
            self.mc.load_kv(filename=self.mc.kv_file)
            root = self.mc.build()
            if root:
                self.mc.root = root
        if self.mc.root:
            if not isinstance(self.mc.root, Widget):
                Logger.critical('App.root must be an _instance_ of Widget')
                raise Exception('Invalid instance in App.root')
            from kivy.core.window import Window
            Window.add_widget(self.mc.root)

        # Check if the window is already created
        from kivy.base import EventLoop
        window = EventLoop.window
        if window:
            self.mc._app_window = window
            window.set_title(self.mc.get_application_name())
            icon = self.mc.get_application_icon()
            if icon:
                window.set_icon(icon)
            self.mc._install_settings_keys(window)
        else:
            Logger.critical("Application: No window is created."
                            " Terminating application run.")
            return

        self.mc.dispatch('on_start')
        runTouchApp(slave=True)  # change is here

        while not self.mc.is_init_done:
            EventLoop.idle()

    def _start_mc(self):
        from mpfmc.core.mc import MpfMc

        # prevent sleep in clock
        Clock._max_fps = 0

        mpf_config = ConfigProcessor.load_config_file(os.path.abspath(
            os.path.join(mpfmc.__path__[0], os.pardir,
                         self.get_options()['mcconfigfile'])), 'machine')

        machine_path = os.path.abspath(os.path.join(
            mpfmc.__path__[0], os.pardir, 'mpfmc', self.getAbsoluteMachinePath()))

        mpf_config = load_machine_config(
            Util.string_to_list(self.getConfigFile()),
            machine_path,
            mpf_config['mpf-mc']['paths']['config'], mpf_config)
        self.preprocess_config(mpf_config)

        self.mc = MpfMc(options=self.get_options(),
                        config=mpf_config,
                        machine_path=machine_path)

        from kivy.core.window import Window
        Window.create_window()
        Window.canvas.clear()

        self._start_app_as_slave()

    def _mc_time(self):
        return self._start_time + self.loop._time

    def _run_mc(self, dt):
        del dt
        Clock.time = self._mc_time
        if self.unittest_verbosity() > 1:
            time.sleep(.05)
        EventLoop.idle()

    def get_enable_plugins(self):
        return True

    def __init__(self, methodName):
        super().__init__(methodName)
        del self.machine_config_patches['mpf']['plugins']
        self.machine_config_patches['bcp'] = \
            {"connections": {"local_display": {"type": "mpf.integration.MpfIntegrationTest.TestBcpClient"}}}
        self.machine_config_patches['bcp']['servers'] = []

    def setUp(self):
        super().setUp()
        self._start_time = time.time()
        self._start_mc()
        self.clock.schedule_interval(self._run_mc, 1 / 30)

        client = self.machine.bcp.transport.get_named_client("local_display")
        bcp_mc = self.mc.bcp_processor
        bcp_mc.send = client.receive
        bcp_mc._client_connected()
        self.machine_run()
        while not client.queue.empty():
            bcp_mc.receive_queue.put(client.queue.get())
        client.queue = bcp_mc.receive_queue
        self.machine_run()

    def tearDown(self):
        self.mc.stop()
        EventLoop.close()
        super().tearDown()
        Clock._events = [[] for i in range(256)]
<commit_msg>improve test<commit_after>import os

os.environ["KIVY_NO_ARGS"] = "1"

from queue import Queue

import time
from kivy import Config, Logger
from kivy.base import runTouchApp, EventLoop
from kivy.clock import Clock
from kivy.uix.widget import Widget

import mpfmc
from mpf.core.utility_functions import Util
from mpf.tests.MpfBcpTestCase import MockBcpClient
from mpf.tests.MpfTestCase import MpfTestCase
from mpfmc.core.config_processor import ConfigProcessor
from mpfmc.core.utils import load_machine_config

class TestBcpClient(MockBcpClient):
    def __init__(self, machine, name, bcp):
        super().__init__(machine, name, bcp)
        self.queue = Queue()
        self.exit_on_close = False

    def send(self, bcp_command, kwargs):
        self.queue.put((bcp_command, kwargs))

    def receive(self, bcp_command, callback=None, rawbytes=None, **kwargs):
        if rawbytes:
            kwargs['rawbytes'] = rawbytes
        self.receive_queue.put_nowait((bcp_command, kwargs))

        if callback:
            callback()

class MpfIntegrationTest(MpfTestCase):

    def get_options(self):
        return dict(machine_path=self.getAbsoluteMachinePath(),
                    mcconfigfile='mpfmc/mcconfig.yaml',
                    configfile=Util.string_to_list(self.getConfigFile()),
                    bcp=False)

    def preprocess_config(self, config):
        # TODO this method is copied from the mc.py launcher. Prob a better way
        kivy_config = config['kivy_config']

        try:
            kivy_config['graphics'].update(config['displays']['window'])
        except KeyError:
            pass

        try:
            kivy_config['graphics'].update(config['window'])
        except KeyError:
            pass

        if 'top' in kivy_config['graphics'] and 'left' in kivy_config['graphics']:
            kivy_config['graphics']['position'] = 'custom'

        for section, settings in kivy_config.items():
            for k, v in settings.items():
                try:
                    if k in Config[section]:
                        Config.set(section, k, v)
                except KeyError:
                    continue

    def _start_app_as_slave(self):
        # from app::run
        if not self.mc.built:
            self.mc.load_config()
            self.mc.load_kv(filename=self.mc.kv_file)
            root = self.mc.build()
            if root:
                self.mc.root = root
        if self.mc.root:
            if not isinstance(self.mc.root, Widget):
                Logger.critical('App.root must be an _instance_ of Widget')
                raise Exception('Invalid instance in App.root')
            from kivy.core.window import Window
            Window.add_widget(self.mc.root)

        # Check if the window is already created
        from kivy.base import EventLoop
        window = EventLoop.window
        if window:
            self.mc._app_window = window
            window.set_title(self.mc.get_application_name())
            icon = self.mc.get_application_icon()
            if icon:
                window.set_icon(icon)
            self.mc._install_settings_keys(window)
        else:
            Logger.critical("Application: No window is created."
                            " Terminating application run.")
            return

        self.mc.dispatch('on_start')
        runTouchApp(slave=True)  # change is here

        while not self.mc.is_init_done:
            EventLoop.idle()
        print("+")

    def _start_mc(self):
        from mpfmc.core.mc import MpfMc

        # prevent sleep in clock
        Clock._max_fps = 0

        mpf_config = ConfigProcessor.load_config_file(os.path.abspath(
            os.path.join(mpfmc.__path__[0], os.pardir,
                         self.get_options()['mcconfigfile'])), 'machine')

        machine_path = os.path.abspath(os.path.join(
            mpfmc.__path__[0], os.pardir, 'mpfmc', self.getAbsoluteMachinePath()))

        mpf_config = load_machine_config(
            Util.string_to_list(self.getConfigFile()),
            machine_path,
            mpf_config['mpf-mc']['paths']['config'], mpf_config)
        self.preprocess_config(mpf_config)

        self.mc = MpfMc(options=self.get_options(),
                        config=mpf_config,
                        machine_path=machine_path)

        from kivy.core.window import Window
        Window.create_window()
        Window.canvas.clear()

        self._start_app_as_slave()

    def _mc_time(self):
        return self._start_time + self.loop._time

    def _run_mc(self, dt):
        del dt
        if self.unittest_verbosity() > 1:
            time.sleep(.05)
        EventLoop.idle()

    def get_enable_plugins(self):
        return True

    def __init__(self, methodName):
        super().__init__(methodName)
        del self.machine_config_patches['mpf']['plugins']
        self.machine_config_patches['bcp'] = \
            {"connections": {"local_display": {"type": "mpf.integration.MpfIntegrationTest.TestBcpClient"}}}
        self.machine_config_patches['bcp']['servers'] = []

    def setUp(self):
        super().setUp()
        self._start_time = time.time()
        Clock._start_tick = self._start_time
        Clock._last_tick = self._start_time
        Clock.time = self._mc_time
        Clock._events = [[] for i in range(256)]
        self._start_mc()
        self.mc_task = self.clock.schedule_interval(self._run_mc, 1 / 30)

        client = self.machine.bcp.transport.get_named_client("local_display")
        bcp_mc = self.mc.bcp_processor
        bcp_mc.send = client.receive
        bcp_mc._client_connected()
        self.machine_run()
        while not client.queue.empty():
            bcp_mc.receive_queue.put(client.queue.get())
        client.queue = bcp_mc.receive_queue
        self.machine_run()

    def tearDown(self):
        self.mc.stop()
        self.clock.unschedule(self.mc_task)
        super().tearDown()
        EventLoop.close()

<|endoftext|>


<commit_before>"""
OMEGA iTHX Series Temperature and Humidity Chart Recorder.

This class is compatible with the following model numbers:

* iTHX-W3
* iTHX-D3
* iTHX-SD
* iTHX-M
* iTHX-W
* iTHX-2
"""
import os
import re
import time
import socket
import sqlite3
import datetime
try:
    ConnectionResetError
except NameError:
    ConnectionResetError = socket.error  # for Python 2.7

from msl.equipment.exceptions import OmegaError, MSLTimeoutError
from msl.equipment.connection_socket import ConnectionSocket
from msl.equipment.resources import register

@register(manufacturer=r'OMEGA', model=r'iTHX-[2DMSW][3D]*', flags=re.IGNORECASE)
class iTHX(ConnectionSocket):

    def __init__(self, record):
        """OMEGA iTHX Series Temperature and Humidity Chart Recorder.

        Do not instantiate this class directly. Use the :meth:`~.EquipmentRecord.connect`
        method to connect to the equipment.

        Parameters
        ----------
        record : :class:`~.EquipmentRecord`
            A record from an :ref:`equipment_database`.
        """
        super(iTHX, self).__init__(record)
        self.set_exception_class(OmegaError)

    def temperature(self, probe=1, celsius=True):
        """Read the temperature.

        Parameters
        ----------
        probe : :class:`int`, optional
            The probe number to read the temperature of (for iTHX's that contain multiple probes).
        celsius : class:`bool`, optional
            :data:`True` to return the temperature in celsius, :data:`False` for fahrenheit.

        Returns
        -------
        :class:`float` or :class:`tuple` of :class:`float`
            The current temperature.
        """
        msg = 'TC' if celsius else 'TF'
        return self._get(msg, probe)

    def humidity(self, probe=1):
        """Read the percent humidity.

        Parameters
        ----------
        probe : :class:`int`, optional
            The probe number to read the humidity of (for iTHX's that contain multiple probes).

        Returns
        -------
        :class:`float` or :class:`tuple` of :class:`float`
            The current percent humidity.
        """
        return self._get('H', probe)

    def dewpoint(self, probe=1, celsius=True):
        """Read the dew point.

        Parameters
        ----------
        probe : :class:`int`, optional
            The probe number to read the dew point of (for iTHX's that contain multiple probes).
        celsius : :class:`bool`, optional
            :data:`True` to return the dew point in celsius, :data:`False` for fahrenheit.

        Returns
        -------
        :class:`float` or :class:`tuple` of :class:`float`
            The current dew point.
        """
        msg = 'D' if celsius else 'DF'
        return self._get(msg, probe)

    def temperature_humidity(self, probe=1, celsius=True):
        """Read the temperature and the humidity.

        Parameters
        ----------
        probe : :class:`int`, optional
            The probe number to read the temperature and humidity of (for iTHX's that contain multiple probes).
        celsius : :class:`bool`, optional
            :data:`True` to return the temperature in celsius, :data:`False` for fahrenheit.

        Returns
        -------
        :class:`float`
            The current temperature.
        :class:`float`
            The current humidity.
        """
        msg = 'B' if celsius else 'BF'
        # since the returned bytes are of the form b'019.4\r,057.0\r'
        # the _get method would stop reading bytes at the first instance of '\r'
        # therefore, we will specify the number of bytes to read to get both values
        return self._get(msg, probe, size=13)

    def temperature_humidity_dewpoint(self, probe=1, celsius=True):
        """Read the temperature, the humidity and the dew point.

        Parameters
        ----------
        probe : :class:`int`, optional
            The probe number to read the temperature, humidity and dew point of
            (for iTHX's that contain multiple probes).
        celsius : :class:`bool`, optional
            :data:`True` to return the temperature and dew point in celsius, :data:`False` for fahrenheit.

        Returns
        -------
        :class:`float`
            The current temperature.
        :class:`float`
            The current humidity.
        :class:`float`
            The current dew point.
        """
        t, h = self.temperature_humidity(probe=probe, celsius=celsius)
        return t, h, self.dewpoint(probe=probe, celsius=celsius)

    def start_logging(self, path, wait=60, num_probes=1):
        """Start logging the temperature, humidity and dew point to the specified path.

        The information is logged to a SQLite_ database. To stop logging press ``CTRL+C``.

        .. _SQLite: https://www.sqlite.org/index.html

        Parameters
        ----------
        path : :class:`str`
            The path to the SQLite_ database. If you only specify a folder then a database
            with the default filename, ``model_serial.sqlite3``, is created/opened in this folder.
        wait : :class:`int`, optional
            The number of seconds to wait between each log.
        num_probes : :class:`int`, optional
            The number of probes that the iServer has (1 or 2).
        """
        if os.path.isdir(path):
            filename=self.equipment_record.model + '_' + self.equipment_record.serial + '.sqlite3'
            path = os.path.join(path, filename)

        db_timeout = 10.0

        db = sqlite3.connect(path, timeout=db_timeout)
        self.log_info('start logging to {}'.format(path))

        if num_probes == 1:
            db.execute(
                'CREATE TABLE IF NOT EXISTS data ('
                'timestamp TIMESTAMP, '
                'temperature FLOAT, '
                'humidity FLOAT, '
                'dewpoint FLOAT);'
            )
        elif num_probes == 2:
            db.execute(
                'CREATE TABLE IF NOT EXISTS data ('
                'timestamp TIMESTAMP, '
                'temperature1 FLOAT, '
                'humidity1 FLOAT, '
                'dewpoint1 FLOAT, '
                'temperature2 FLOAT, '
                'humidity2 FLOAT, '
                'dewpoint2 FLOAT);'
            )
        else:
            raise ValueError('The number-of-probes value must be either 1 or 2. Got {}'.format(num_probes))

        db.commit()
        db.close()

        msg = 'Sn:{} - {}:{} => '.format(self.equipment_record.serial, self._ip_address, self._port)

        try:
            while True:
                t0 = time.time()
                results = [datetime.datetime.now()]

                # get the values
                try:
                    data = self.temperature_humidity_dewpoint(probe=1, celsius=True)
                    if num_probes == 2:
                        data += self.temperature_humidity_dewpoint(probe=2, celsius=True)
                        self.log_info(msg+'T1={:.1f} H1={:.1f} DP1={:.1f} T2={:.1f} H2={:.1f} DP2={:.1f}'.format(*data))
                    else:
                        self.log_info(msg+'T={:.1f} H={:.1f} DP={:.1f}'.format(*data))
                except MSLTimeoutError:
                    while True:
                        try:
                            self._connect()
                        except MSLTimeoutError:
                            pass
                        else:
                            break
                    continue
                else:
                    results.extend(data)

                # save the values to the database and then wait
                try:
                    db = sqlite3.connect(path, timeout=db_timeout)
                    if num_probes == 1:
                        db.execute('INSERT INTO data VALUES (?, ?, ?, ?);', results)
                    else:
                        db.execute('INSERT INTO data VALUES (?, ?, ?, ?, ?, ?, ?);', results)
                    db.commit()
                    db.close()
                except sqlite3.OperationalError as e:  # database is locked, someone is reading data
                    db.close()
                    self.log_error('sqlite3.OperationalError: ' + str(e))
                else:
                    time.sleep(max(0.0, wait - (time.time() - t0)))

        except (KeyboardInterrupt, SystemExit):
            pass

        db.close()
        self.log_info('stopped logging to {}'.format(path))

    @staticmethod
    def data(path, date1=None, date2=None, as_datetime=True, select='*'):
        """Fetch all the log records between two dates.

        Parameters
        ----------
        path : :class:`str`
            The path to the SQLite_ database.
        date1 : :class:`datetime.datetime` or :class:`str`, optional
            Include all records that have a timestamp > `date1`. If :class:`str` then in
            ``yyyy-mm-dd`` or ``yyyy-mm-dd HH:MM:SS`` format.
        date2 : :class:`datetime.datetime` or :class:`str`, optional
            Include all records that have a timestamp < `date2`. If :class:`str` then in
            ``yyyy-mm-dd`` or ``yyyy-mm-dd HH:MM:SS`` format.
        as_datetime : :class:`bool`, optional
            Whether to fetch the timestamps from the database as :class:`datetime.datetime` objects.
            If :data:`False` then the timestamps will be of type :class:`str` and this function
            will return much faster if requesting data over a large date range.
        select : :class:`str` or :class:`list` of :class:`str`, optional
            The column(s) in the database to use with the ``SELECT`` SQL command.

        Returns
        -------
        :class:`list` of :class:`tuple`
            A list of ``(timestamp, temperature, humidity, dewpoint)`` log records,
            depending on the value of `select`.
        """
        if not os.path.isfile(path):
            raise IOError('Cannot find {}'.format(path))

        detect_types = sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES if as_datetime else 0
        db = sqlite3.connect(path, timeout=10.0, detect_types=detect_types)
        cursor = db.cursor()

        if select != '*':
            if isinstance(select, (list, tuple, set)):
                select = ','.join(select)
        base = 'SELECT {} FROM data'.format(select)

        if date1 is None and date2 is None:
            cursor.execute(base + ';')
        elif date1 is not None and date2 is None:
            cursor.execute(base + ' WHERE timestamp > ?;', (date1,))
        elif date1 is None and date2 is not None:
            cursor.execute(base + ' WHERE timestamp < ?;', (date2,))
        else:
            cursor.execute(base + ' WHERE timestamp BETWEEN ? AND ?;', (date1, date2))

        data = cursor.fetchall()
        cursor.close()
        db.close()
        return data

    def _get(self, message, probe, size=None):
        if not 1 <= probe <= 3:
            self.raise_exception('Invalid probe number, {}. Must be either 1, 2, or 3'.format(probe))

        command = '*SR' + message
        if probe > 1:
            command += str(probe)

        try:
            ret = self.query(command, size=size)
        except ConnectionResetError:
            # for some reason the socket closes if a certain amount of time passes and no
            # messages have been sent. For example, querying the temperature, humidity and
            # dew point every >60 seconds raised:
            #   [Errno errno.ECONNRESET] An existing connection was forcibly closed by the remote host
            self._connect()  # reconnect
            return self._get(message, probe, size=size)  # retry
        else:
            values = tuple(float(v) for v in re.split(r'[,;]', ret))
            if len(values) == 1:
                return values[0]
            else:
                return values
<commit_msg>get OMEGA iTHX working with iTHX-2 or iTHX-W and with non-term. replies<commit_after>"""
OMEGA iTHX Series Temperature and Humidity Chart Recorder.

This class is compatible with the following model numbers:

* iTHX-W3
* iTHX-D3
* iTHX-SD
* iTHX-M
* iTHX-W
* iTHX-2
"""
import os
import re
import time
import socket
import sqlite3
import datetime
try:
    ConnectionResetError
except NameError:
    ConnectionResetError = socket.error  # for Python 2.7

from msl.equipment.exceptions import OmegaError, MSLTimeoutError
from msl.equipment.connection_socket import ConnectionSocket
from msl.equipment.resources import register

@register(manufacturer=r'OMEGA', model=r'iTHX-[2DMSW][3D]*', flags=re.IGNORECASE)
class iTHX(ConnectionSocket):

    def __init__(self, record):
        """OMEGA iTHX Series Temperature and Humidity Chart Recorder.

        Do not instantiate this class directly. Use the :meth:`~.EquipmentRecord.connect`
        method to connect to the equipment.

        Parameters
        ----------
        record : :class:`~.EquipmentRecord`
            A record from an :ref:`equipment_database`.
        """
        super(iTHX, self).__init__(record)
        self.set_exception_class(OmegaError)
        self.IS_W_OR_2 = record.model.upper() in ['ITHX-2', 'ITHX-W']

    def temperature(self, probe=1, celsius=True, nbytes=None):
        """Read the temperature.

        Parameters
        ----------
        probe : :class:`int`, optional
            The probe number to read the temperature of (for iTHX's that contain multiple probes).
        celsius : class:`bool`, optional
            :data:`True` to return the temperature in celsius, :data:`False` for fahrenheit.
        nbytes : class:`int`, optional
            The number of bytes to read. If :data:`None` then read until the termination
            character sequence.

        Returns
        -------
        :class:`float` or :class:`tuple` of :class:`float`
            The current temperature.
        """
        msg = 'TC' if celsius else 'TF'
        return self._get(msg, probe, size=nbytes)

    def humidity(self, probe=1, nbytes=None):
        """Read the percent humidity.

        Parameters
        ----------
        probe : :class:`int`, optional
            The probe number to read the humidity of (for iTHX's that contain multiple probes).
        nbytes : class:`int`, optional
            The number of bytes to read. If :data:`None` then read until the termination
            character sequence.

        Returns
        -------
        :class:`float` or :class:`tuple` of :class:`float`
            The current percent humidity.
        """
        return self._get('H', probe, size=nbytes)

    def dewpoint(self, probe=1, celsius=True, nbytes=None):
        """Read the dew point.

        Parameters
        ----------
        probe : :class:`int`, optional
            The probe number to read the dew point of (for iTHX's that contain multiple probes).
        celsius : :class:`bool`, optional
            :data:`True` to return the dew point in celsius, :data:`False` for fahrenheit.
        nbytes : class:`int`, optional
            The number of bytes to read. If :data:`None` then read until the termination
            character sequence.

        Returns
        -------
        :class:`float` or :class:`tuple` of :class:`float`
            The current dew point.
        """
        msg = 'D' if celsius else 'DF'
        return self._get(msg, probe, size=nbytes)

    def temperature_humidity(self, probe=1, celsius=True, nbytes=None):
        """Read the temperature and the humidity.

        Parameters
        ----------
        probe : :class:`int`, optional
            The probe number to read the temperature and humidity of (for iTHX's that contain multiple probes).
        celsius : :class:`bool`, optional
            :data:`True` to return the temperature in celsius, :data:`False` for fahrenheit.
        nbytes : class:`int`, optional
            The number of bytes to read. If :data:`None` then the default value is 13 bytes.

        Returns
        -------
        :class:`float`
            The current temperature.
        :class:`float`
            The current humidity.
        """
        if nbytes is None:
            # since the returned bytes are of the form b'019.4\r,057.0\r'
            # the _get method would stop reading bytes at the first instance of '\r'
            # therefore, we will specify the number of bytes to read to get both values
            nbytes = 13

        if self.IS_W_OR_2:
            # iTHX-W and iTHX-2 do not support the 'B' command
            n = nbytes//2
            return self.temperature(probe=probe, celsius=celsius, nbytes=n), self.humidity(probe=probe, nbytes=n)

        msg = 'B' if celsius else 'BF'
        return self._get(msg, probe, size=nbytes)

    def temperature_humidity_dewpoint(self, probe=1, celsius=True, nbytes=None):
        """Read the temperature, the humidity and the dew point.

        Parameters
        ----------
        probe : :class:`int`, optional
            The probe number to read the temperature, humidity and dew point of
            (for iTHX's that contain multiple probes).
        celsius : :class:`bool`, optional
            :data:`True` to return the temperature and dew point in celsius, :data:`False` for fahrenheit.
        nbytes : class:`int`, optional
            The number of bytes to read. If :data:`None` then read until the termination
            character sequence. For example, if no termination is returned and each value is
            represented by 6 bytes then specify `nbytes`=18 to get all 3 values.

        Returns
        -------
        :class:`float`
            The current temperature.
        :class:`float`
            The current humidity.
        :class:`float`
            The current dew point.
        """
        nth = None if nbytes is None else (nbytes*2)//3
        nd = None if nbytes is None else nbytes//3
        t, h = self.temperature_humidity(probe=probe, celsius=celsius, nbytes=nth)
        return t, h, self.dewpoint(probe=probe, celsius=celsius, nbytes=nd)

    def start_logging(self, path, wait=60, nprobes=1, nbytes=None):
        """Start logging the temperature, humidity and dew point to the specified path.

        The information is logged to a SQLite_ database. To stop logging press ``CTRL+C``.

        .. _SQLite: https://www.sqlite.org/index.html

        Parameters
        ----------
        path : :class:`str`
            The path to the SQLite_ database. If you only specify a folder then a database
            with the default filename, ``model_serial.sqlite3``, is created/opened in this folder.
        wait : :class:`int`, optional
            The number of seconds to wait between each log.
        nprobes : :class:`int`, optional
            The number of probes that the iServer has (1 or 2).
        nbytes : class:`int`, optional
            The number of bytes to read. If :data:`None` then read until the termination
            character sequence. For example, if no termination is used and each value is
            represented by 6 bytes then specify `nbytes`=18 to get all three values.
        """
        if os.path.isdir(path):
            filename = self.equipment_record.model + '_' + self.equipment_record.serial + '.sqlite3'
            path = os.path.join(path, filename)

        db_timeout = 10.0

        db = sqlite3.connect(path, timeout=db_timeout)
        self.log_info('start logging to {}'.format(path))

        if nprobes == 1:
            db.execute(
                'CREATE TABLE IF NOT EXISTS data ('
                'timestamp TIMESTAMP, '
                'temperature FLOAT, '
                'humidity FLOAT, '
                'dewpoint FLOAT);'
            )
        elif nprobes == 2:
            db.execute(
                'CREATE TABLE IF NOT EXISTS data ('
                'timestamp TIMESTAMP, '
                'temperature1 FLOAT, '
                'humidity1 FLOAT, '
                'dewpoint1 FLOAT, '
                'temperature2 FLOAT, '
                'humidity2 FLOAT, '
                'dewpoint2 FLOAT);'
            )
        else:
            raise ValueError('The number-of-probes value must be either 1 or 2. Got {}'.format(nprobes))

        db.commit()
        db.close()

        msg = 'Sn:{} - {}:{} => '.format(self.equipment_record.serial, self._ip_address, self._port)

        try:
            while True:
                t0 = time.time()
                results = [datetime.datetime.now()]

                # get the values
                try:
                    data = self.temperature_humidity_dewpoint(probe=1, celsius=True, nbytes=nbytes)
                    if nprobes == 2:
                        data += self.temperature_humidity_dewpoint(probe=2, celsius=True, nbytes=nbytes)
                        self.log_info(msg+'T1={:.1f} H1={:.1f} DP1={:.1f} T2={:.1f} H2={:.1f} DP2={:.1f}'.format(*data))
                    else:
                        self.log_info(msg+'T={:.1f} H={:.1f} DP={:.1f}'.format(*data))
                except MSLTimeoutError:
                    while True:
                        try:
                            self._connect()
                        except MSLTimeoutError:
                            pass
                        else:
                            break
                    continue
                else:
                    results.extend(data)

                # save the values to the database and then wait
                try:
                    db = sqlite3.connect(path, timeout=db_timeout)
                    if nprobes == 1:
                        db.execute('INSERT INTO data VALUES (?, ?, ?, ?);', results)
                    else:
                        db.execute('INSERT INTO data VALUES (?, ?, ?, ?, ?, ?, ?);', results)
                    db.commit()
                    db.close()
                except sqlite3.OperationalError as e:  # database is locked, someone is reading data
                    db.close()
                    self.log_error('sqlite3.OperationalError: ' + str(e))
                else:
                    time.sleep(max(0.0, wait - (time.time() - t0)))

        except (KeyboardInterrupt, SystemExit):
            pass

        db.close()
        self.log_info('stopped logging to {}'.format(path))

    @staticmethod
    def data(path, date1=None, date2=None, as_datetime=True, select='*'):
        """Fetch all the log records between two dates.

        Parameters
        ----------
        path : :class:`str`
            The path to the SQLite_ database.
        date1 : :class:`datetime.datetime` or :class:`str`, optional
            Include all records that have a timestamp > `date1`. If :class:`str` then in
            ``yyyy-mm-dd`` or ``yyyy-mm-dd HH:MM:SS`` format.
        date2 : :class:`datetime.datetime` or :class:`str`, optional
            Include all records that have a timestamp < `date2`. If :class:`str` then in
            ``yyyy-mm-dd`` or ``yyyy-mm-dd HH:MM:SS`` format.
        as_datetime : :class:`bool`, optional
            Whether to fetch the timestamps from the database as :class:`datetime.datetime` objects.
            If :data:`False` then the timestamps will be of type :class:`str` and this function
            will return much faster if requesting data over a large date range.
        select : :class:`str` or :class:`list` of :class:`str`, optional
            The column(s) in the database to use with the ``SELECT`` SQL command.

        Returns
        -------
        :class:`list` of :class:`tuple`
            A list of ``(timestamp, temperature, humidity, dewpoint)`` log records,
            depending on the value of `select`.
        """
        if not os.path.isfile(path):
            raise IOError('Cannot find {}'.format(path))

        detect_types = sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES if as_datetime else 0
        db = sqlite3.connect(path, timeout=10.0, detect_types=detect_types)
        cursor = db.cursor()

        if select != '*':
            if isinstance(select, (list, tuple, set)):
                select = ','.join(select)
        base = 'SELECT {} FROM data'.format(select)

        if date1 is None and date2 is None:
            cursor.execute(base + ';')
        elif date1 is not None and date2 is None:
            cursor.execute(base + ' WHERE timestamp > ?;', (date1,))
        elif date1 is None and date2 is not None:
            cursor.execute(base + ' WHERE timestamp < ?;', (date2,))
        else:
            cursor.execute(base + ' WHERE timestamp BETWEEN ? AND ?;', (date1, date2))

        data = cursor.fetchall()
        cursor.close()
        db.close()
        return data

    def _get(self, message, probe, size=None):
        if not 1 <= probe <= 3:
            self.raise_exception('Invalid probe number, {}. Must be either 1, 2, or 3'.format(probe))

        command = '*SR' + message
        if probe > 1:
            command += str(probe)

        try:
            ret = self.query(command, size=size)
        except ConnectionResetError:
            # for some reason the socket closes if a certain amount of time passes and no
            # messages have been sent. For example, querying the temperature, humidity and
            # dew point every >60 seconds raised:
            #   [Errno errno.ECONNRESET] An existing connection was forcibly closed by the remote host
            self._connect()  # reconnect
            return self._get(message, probe, size=size)  # retry
        else:
            values = tuple(float(v) for v in re.split(r'[,;]', ret))
            if len(values) == 1:
                return values[0]
            else:
                return values
<|endoftext|>


<commit_before>'''game.py for applequest.fallenofftheedge.com'''
import library as lib
import cgitb
cgitb.enable()

def html_print(value='', print_data='', newcookie=''):
    '''Prints the HTML to the client with varibals
       $$   printdata  $$
       $$   command    $$
       $$ newcookie    $$
       $$ oldcookie    $$'''
    html = lib.get_html(lib.HTML_DIR + 'game.html')

    if True:
        html = html.replace('$$oldcookie$$$', cookie_read())
    else:
        html = html.replace('$$oldcookie$$$', '')

    if newcookie != '':
        html = html.replace('$$newcookie$$$', newcookie)
    else:
        html = html.replace('$$newcookie$$$', '')

    if print_data != '':
        html = html.replace('$$printdata$$', print_data)
    else:
        html = html.replace('$$printdata$$', '')

    if value != '':
        html = html.replace('$$command$$', value)
    else:
        html = html.replace('$$command$$', '')

    if True is not False:
        html = html.replace('$$printcommand$$', '')
    else:
        print('This is not the case you are looking for!')
        print('P.S. The world is about to end!!!')

    print(html)

def cookie_read():
    '''Reads the cookies sent in the request headers and prints the back
    to the client'''
    cookie = lib.get_cookies()
    if cookie is None:
        return 'No saved Cookies'
    else:
        return str(cookie)

def main():
    '''main'''
    print_data = ''
    newcookie = 'No New Cookie'
    cookie_read()
    form = lib.get_cgi_data()
    command = form.getfirst("command")
    renew = form.getfirst("newid")
    if command is None:
        command = ''
    elif type(command) != str:
        command = str(command)

    cookie = ''
    if renew == 'true':
        cookies = lib.get_cookies()
        sessioninfo = renew_session_id(cookies['id'],cookies['username'])
        if type(sessioninfo) == str:
            print_data += 'Could not renew\n'
        else:
            newcookie = lib.cookie_wright(sessioninfo)

    if command == '301':
        print_data += '103\n'
    elif command == '302':
        print_data += '203\n'
    elif command == '303':
        print_data += '303\n'
    else:
        print_data += '003\n'

    lib.print_header(cookie)
    html_print(command, print_data, newcookie)

if __name__ == '__main__':
    main()
<commit_msg>Fix get val of cook<commit_after>'''game.py for applequest.fallenofftheedge.com'''
import library as lib
import cgitb
cgitb.enable()

def html_print(value='', print_data='', newcookie=''):
    '''Prints the HTML to the client with varibals
       $$   printdata  $$
       $$   command    $$
       $$ newcookie    $$
       $$ oldcookie    $$'''
    html = lib.get_html(lib.HTML_DIR + 'game.html')

    if True:
        html = html.replace('$$oldcookie$$$', cookie_read())
    else:
        html = html.replace('$$oldcookie$$$', '')

    if newcookie != '':
        html = html.replace('$$newcookie$$$', newcookie)
    else:
        html = html.replace('$$newcookie$$$', '')

    if print_data != '':
        html = html.replace('$$printdata$$', print_data)
    else:
        html = html.replace('$$printdata$$', '')

    if value != '':
        html = html.replace('$$command$$', value)
    else:
        html = html.replace('$$command$$', '')

    if True is not False:
        html = html.replace('$$printcommand$$', '')
    else:
        print('This is not the case you are looking for!')
        print('P.S. The world is about to end!!!')

    print(html)

def cookie_read():
    '''Reads the cookies sent in the request headers and prints the back
    to the client'''
    cookie = lib.get_cookies()
    if cookie is None:
        return 'No saved Cookies'
    else:
        return str(cookie)

def main():
    '''main'''
    print_data = ''
    newcookie = 'No New Cookie'
    cookie_read()
    form = lib.get_cgi_data()
    command = form.getfirst("command")
    renew = form.getfirst("newid")
    if command is None:
        command = ''
    elif type(command) != str:
        command = str(command)

    cookie = ''
    if renew == 'true':
        cookies = lib.get_cookies()
        sessioninfo = renew_session_id(cookies['id'].value,cookies['username'].value)
        if type(sessioninfo) == str:
            print_data += 'Could not renew\n'
        else:
            newcookie = lib.cookie_wright(sessioninfo)

    if command == '301':
        print_data += '103\n'
    elif command == '302':
        print_data += '203\n'
    elif command == '303':
        print_data += '303\n'
    else:
        print_data += '003\n'

    lib.print_header(cookie)
    html_print(command, print_data, newcookie)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>try:  # IPython/Jupyter 4.x
    from ipywidgets import Widget, widget_serialization
    from traitlets import Bool, Int, Float, Unicode, List, Instance
except ImportError:  # IPython 3.x
    from IPython.html.widgets import Widget
    from IPython.utils.traitlets import Bool, Int, Float, Unicode, List, Instance
    widget_serialization = {}

class Button(Widget):
    """Represents a gamepad or joystick button"""
    value = Float(min=0.0, max=1.0, read_only=True, sync=True)
    pressed = Bool(read_only=True, sync=True)

    _view_name = Unicode('Button', sync=True)
    _view_module = Unicode('nbextensions/gamepad/gamepad', sync=True)

class Axis(Widget):
    """Represents a gamepad or joystick axis"""
    value = Float(min=-1.0, max=1.0, read_only=True, sync=True)

    _view_name = Unicode('Axis', sync=True)
    _view_module = Unicode('nbextensions/gamepad/gamepad', sync=True)

class GamePad(Widget):
    """Represents a game controller"""
    index = Int(sync=True)

    # General information about the gamepad, button and axes mapping, id.
    # These values are all read-only and set by the JavaScript side.
    id = Unicode(read_only=True, sync=True)
    mapping = Unicode(read_only=True, sync=True)
    connected = Bool(read_only=True, sync=True)
    timestamp = Float(read_only=True, sync=True)

    # Buttons and axes - read-only
    buttons = List(trait=Instance(Button), read_only=True,
                   sync=True, **widget_serialization)
    axes = List(trait=Instance(Axis), read_only=True, sync=True,
                **widget_serialization)

    _view_name = Unicode('GamepadView', sync=True)
    _view_module = Unicode('nbextensions/gamepad/gamepad', sync=True)

    _model_name = Unicode('Gamepad', sync=True)
    _model_module = Unicode('nbextensions/gamepad/gamepad', sync=True)

<commit_msg>pep8 and 0.1.3<commit_after>try:  # IPython/Jupyter 4.x
    from ipywidgets import Widget, widget_serialization
    from traitlets import Bool, Int, Float, Unicode, List, Instance
except ImportError:  # IPython 3.x
    from IPython.html.widgets import Widget
    from IPython.utils.traitlets import Bool, Int, Float, Unicode, List, Instance
    widget_serialization = {}

class Button(Widget):
    """Represents a gamepad or joystick button"""
    value = Float(min=0.0, max=1.0, read_only=True, sync=True)
    pressed = Bool(read_only=True, sync=True)

    _view_name = Unicode('Button', sync=True)
    _view_module = Unicode('nbextensions/gamepad/gamepad', sync=True)

class Axis(Widget):
    """Represents a gamepad or joystick axis"""
    value = Float(min=-1.0, max=1.0, read_only=True, sync=True)

    _view_name = Unicode('Axis', sync=True)
    _view_module = Unicode('nbextensions/gamepad/gamepad', sync=True)

class Gamepad(Widget):
    """Represents a game controller"""
    index = Int(sync=True)

    # General information about the gamepad, button and axes mapping, id.
    # These values are all read-only and set by the JavaScript side.
    id = Unicode(read_only=True, sync=True)
    mapping = Unicode(read_only=True, sync=True)
    connected = Bool(read_only=True, sync=True)
    timestamp = Float(read_only=True, sync=True)

    # Buttons and axes - read-only
    buttons = List(trait=Instance(Button), read_only=True,
                   sync=True, **widget_serialization)
    axes = List(trait=Instance(Axis), read_only=True, sync=True,
                **widget_serialization)

    _view_name = Unicode('GamepadView', sync=True)
    _view_module = Unicode('nbextensions/gamepad/gamepad', sync=True)

    _model_name = Unicode('Gamepad', sync=True)
    _model_module = Unicode('nbextensions/gamepad/gamepad', sync=True)

<|endoftext|>