<commit_before>import time
import logging
import numpy as np

from analysis.plotting.plotting import plot_occupancy
from daq.readout import get_col_row_array_from_data_record_array, convert_data_array, data_array_from_data_dict_iterable, is_data_record, is_data_from_channel

from scan.scan import ScanBase
from daq.readout import open_raw_data_file

logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)-8s] (%(threadName)-10s) %(message)s")

class NoiseOccupancyScan(ScanBase):
    def __init__(self, config_file, definition_file=None, bit_file=None, device=None, scan_identifier="scan_noise_occupancy", scan_data_path=None):
        super(NoiseOccupancyScan, self).__init__(config_file=config_file, definition_file=definition_file, bit_file=bit_file, device=device, scan_identifier=scan_identifier, scan_data_path=scan_data_path)

    def scan(self, occupancy_limit=10 ** (-5), triggers=10000000, consecutive_lvl1=16, disable_for_mask=['Enable'], enable_for_mask=['Imon'], overwrite_mask=False, col_span=[1, 80], row_span=[1, 336], timeout_no_data=10):
        '''Masking pixels with occupancy above certain limit.

        Parameters
        ----------
        occupancy_limit : float
            Occupancy limit which is multiplied with measured number of hits for each pixel. Any pixel above 1 will be masked.
        triggers : int
            Total number of triggers sent to FE. From 1 to 4294967295 (32-bit unsigned int).
        consecutive_lvl1 : int
            Number of consecutive LVL1 triggers. From 1 to 16.
        disable_for_mask : list, tuple
            List of masks for which noisy pixels will be disabled.
        enable_for_mask : list, tuple
            List of masks for which noisy pixels will be enabled.
        overwrite_mask : bool
            Overwrite masks (disable_for_mask, enable_for_mask) if set to true. If set to false, make a combination of existing mask and new mask.
        col_span : list, tuple
            Column range (from minimum to maximum value). From 1 to 80.
        row_span : list, tuple
            Row range (from minimum to maximum value). From 1 to 336.
        timeout_no_data : int
            In seconds; if no data, stop scan after given time.
        scan_timeout : int
            In seconds; stop scan after given time.

        Note
        ----
        The total number of trigger is triggers * consecutive_lvl1.
        Please note that a high trigger rate leads to an effective lower threshold.
        '''
        # create restore point
        self.register.create_restore_point()

        if occupancy_limit * triggers * consecutive_lvl1 < 1:
            logging.warning('Number of triggers too low for given occupancy limit')

        commands = []
        mask = self.register_utils.make_box_pixel_mask_from_col_row(column=col_span, row=row_span)
        pixel_reg = "Enable"
        commands.extend(self.register.get_commands("confmode"))
        self.register.set_pixel_register_value(pixel_reg, mask)
        commands.extend(self.register.get_commands("wrfrontend", same_mask_for_all_dc=False, name=pixel_reg))
        # generate mask for Imon mask
        pixel_reg = "Imon"
#         mask = self.register_utils.make_box_pixel_mask_from_col_row(column=col_span, row=row_span, default=1, value=0)
        self.register.set_pixel_register_value(pixel_reg, 1)
        commands.extend(self.register.get_commands("wrfrontend", same_mask_for_all_dc=True, name=pixel_reg))
        # disable C_inj mask
        pixel_reg = "C_High"
        self.register.set_pixel_register_value(pixel_reg, 0)
        commands.extend(self.register.get_commands("wrfrontend", same_mask_for_all_dc=True, name=pixel_reg))
        pixel_reg = "C_Low"
        self.register.set_pixel_register_value(pixel_reg, 0)
        commands.extend(self.register.get_commands("wrfrontend", same_mask_for_all_dc=True, name=pixel_reg))
#         self.register.set_global_register_value("Trig_Lat", 232)  # set trigger latency
        self.register.set_global_register_value("Trig_Count", (0 if consecutive_lvl1 == 16 else consecutive_lvl1))  # set number of consecutive triggers
        commands.extend(self.register.get_commands("wrregister", name=["Trig_Count"]))
        # setting FE into runmode
        commands.extend(self.register.get_commands("runmode"))
        self.register_utils.send_commands(commands)

        self.col_arr = np.array([], dtype=np.dtype('>u1'))
        self.row_arr = np.array([], dtype=np.dtype('>u1'))

        with open_raw_data_file(filename=self.scan_data_filename, title=self.scan_identifier) as raw_data_file:
            self.readout.start()

            # preload command
            command_delay = 400 # 100kHz
            lvl1_command = self.register.get_commands("lv1")[0] + self.register.get_commands("zeros", length=command_delay)[0]
            commnd_lenght = lvl1_command.length()
            logging.info('Estimated scan time: %ds' % int(commnd_lenght * 25 * (10 ** -9) * triggers))
            logging.info('Please stand by...')
            self.register_utils.send_command(lvl1_command, repeat=triggers, wait_for_finish=False, set_length=True, clear_memory=False)

            wait_for_first_data = False
            last_iteration = time.time()
            saw_no_data_at_time = last_iteration
            saw_data_at_time = last_iteration
            no_data_at_time = last_iteration
            while not self.stop_thread_event.wait(self.readout.readout_interval):
                last_iteration = time.time()
                try:
                    data = (self.readout.data.popleft(), )
                    raw_data_file.append(data)
                    col_arr_tmp, row_arr_tmp = convert_data_array(data_array_from_data_dict_iterable(data), filter_func=is_data_record, converter_func=get_col_row_array_from_data_record_array)
                    self.col_arr = np.concatenate((self.col_arr, col_arr_tmp))
                    self.row_arr = np.concatenate((self.row_arr, row_arr_tmp))
                    #logging.info('data words')
                except IndexError:  # no data
                    #logging.info('no data words')
                    no_data_at_time = last_iteration
                    if self.register_utils.is_ready:
                        self.stop_thread_event.set()
                        logging.info('Finished sending %d triggers' % triggers)
                    elif wait_for_first_data == False and saw_no_data_at_time > (saw_data_at_time + timeout_no_data):
                        logging.info('Reached no data timeout. Stopping Scan...')
                        self.stop_thread_event.set()
                    elif wait_for_first_data == False:
                        saw_no_data_at_time = no_data_at_time
                    elif self.reaout_utils.is_ready:
                        self.stop_thread_event.set()

                    continue

                saw_data_at_time = last_iteration

                if wait_for_first_data == True:
                    logging.info('Taking data...')
                    wait_for_first_data = False

            self.readout.stop()

            self.register.restore()

            occ_hist, _, _ = np.histogram2d(self.col_arr, self.row_arr, bins=(80, 336), range=[[1, 80], [1, 336]])

            self.occ_mask = np.zeros(shape=occ_hist.shape, dtype=np.dtype('>u1'))
            # noisy pixels are set to 1
            self.occ_mask[occ_hist > occupancy_limit * triggers * consecutive_lvl1] = 1
            # make inverse
            self.inv_occ_mask = self.register_utils.invert_pixel_mask(self.occ_mask)
            self.disable_for_mask = disable_for_mask
            if overwrite_mask:
                self.register.set_pixel_register_value(disable_for_mask, self.inv_occ_mask)
            else:
                for mask in disable_for_mask:
                    enable_mask = np.logical_and(self.inv_occ_mask, self.register.get_pixel_register_value(mask))
                    self.register.set_pixel_register_value(mask, enable_mask)

            self.enable_for_mask = enable_for_mask
            if overwrite_mask:
                self.register.set_pixel_register_value(enable_for_mask, self.occ_mask)
            else:
                for mask in enable_for_mask:
                    disable_mask = np.logical_or(self.occ_mask, self.register.get_pixel_register_value(mask))
                    self.register.set_pixel_register_value(mask, disable_mask)

#             plot_occupancy(self.col_arr, self.row_arr, max_occ=None, filename=self.scan_data_filename + "_occupancy.pdf")

    def analyze(self):
        from analysis.analyze_raw_data import AnalyzeRawData
        output_file = self.scan_data_filename + "_interpreted.h5"
        with AnalyzeRawData(raw_data_file=scan.scan_data_filename, analyzed_data_file=output_file, create_pdf=True) as analyze_raw_data:
            analyze_raw_data.create_source_scan_hist = True
#             analyze_raw_data.create_hit_table = True
#             analyze_raw_data.interpreter.debug_events(0, 0, True)  # events to be printed onto the console for debugging, usually deactivated
            analyze_raw_data.interpreter.set_trig_count(self.register.get_global_register_value("Trig_Count"))
            analyze_raw_data.interpreter.set_warning_output(False)
            analyze_raw_data.interpret_word_table(fei4b=scan.register.fei4b)
            analyze_raw_data.interpreter.print_summary()
            analyze_raw_data.plot_histograms()
            plot_occupancy(self.occ_mask.T, title='Noisy Pixels', z_max=1, filename=analyze_raw_data.output_pdf)
            for mask in self.disable_for_mask:
                mask_name = self.register.get_pixel_register_attributes("full_name", do_sort=True, name=[mask])[0]
                plot_occupancy(self.register.get_pixel_register_value(mask).T, title='%s Mask' % mask_name, z_max=1, filename=analyze_raw_data.output_pdf)
            for mask in self.enable_for_mask:
                mask_name = self.register.get_pixel_register_attributes("full_name", do_sort=True, name=[mask])[0]
                plot_occupancy(self.register.get_pixel_register_value(mask).T, title='%s Mask' % mask_name, z_max=1, filename=analyze_raw_data.output_pdf)

if __name__ == "__main__":
    import configuration
    scan = NoiseOccupancyScan(config_file=configuration.config_file, bit_file=configuration.bit_file, scan_data_path=configuration.scan_data_path)
    scan.start(configure=True, use_thread=True, occupancy_limit=10 ** (-5), triggers=10000000, consecutive_lvl1=16, disable_for_mask=['Enable'], enable_for_mask=['Imon'], overwrite_mask=False, col_span=[1, 80], row_span=[1, 336], timeout_no_data=10)
    scan.stop()
    scan.analyze()
    scan.register.save_configuration(configuration.config_file)
<commit_msg>ENH: set trigger count (consecutive triggers) to 1 by default BUG: analysis use correct number of trigger count<commit_after>import time
import logging
import numpy as np

from analysis.plotting.plotting import plot_occupancy
from daq.readout import get_col_row_array_from_data_record_array, convert_data_array, data_array_from_data_dict_iterable, is_data_record, is_data_from_channel

from scan.scan import ScanBase
from daq.readout import open_raw_data_file

logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)-8s] (%(threadName)-10s) %(message)s")

class NoiseOccupancyScan(ScanBase):
    def __init__(self, config_file, definition_file=None, bit_file=None, device=None, scan_identifier="scan_noise_occupancy", scan_data_path=None):
        super(NoiseOccupancyScan, self).__init__(config_file=config_file, definition_file=definition_file, bit_file=bit_file, device=device, scan_identifier=scan_identifier, scan_data_path=scan_data_path)

    def scan(self, occupancy_limit=10 ** (-5), triggers=10000000, consecutive_lvl1=1, disable_for_mask=['Enable'], enable_for_mask=['Imon'], overwrite_mask=False, col_span=[1, 80], row_span=[1, 336], timeout_no_data=10):
        '''Masking pixels with occupancy above certain limit.

        Parameters
        ----------
        occupancy_limit : float
            Occupancy limit which is multiplied with measured number of hits for each pixel. Any pixel above 1 will be masked.
        triggers : int
            Total number of triggers sent to FE. From 1 to 4294967295 (32-bit unsigned int).
        consecutive_lvl1 : int
            Number of consecutive LVL1 triggers. From 1 to 16.
        disable_for_mask : list, tuple
            List of masks for which noisy pixels will be disabled.
        enable_for_mask : list, tuple
            List of masks for which noisy pixels will be enabled.
        overwrite_mask : bool
            Overwrite masks (disable_for_mask, enable_for_mask) if set to true. If set to false, make a combination of existing mask and new mask.
        col_span : list, tuple
            Column range (from minimum to maximum value). From 1 to 80.
        row_span : list, tuple
            Row range (from minimum to maximum value). From 1 to 336.
        timeout_no_data : int
            In seconds; if no data, stop scan after given time.
        scan_timeout : int
            In seconds; stop scan after given time.

        Note
        ----
        The total number of trigger is triggers * consecutive_lvl1.
        Please note that a high trigger rate leads to an effective lower threshold.
        '''
        # create restore point
        self.register.create_restore_point()

        if occupancy_limit * triggers * consecutive_lvl1 < 1:
            logging.warning('Number of triggers too low for given occupancy limit')

        commands = []
        mask = self.register_utils.make_box_pixel_mask_from_col_row(column=col_span, row=row_span)
        pixel_reg = "Enable"
        commands.extend(self.register.get_commands("confmode"))
        self.register.set_pixel_register_value(pixel_reg, mask)
        commands.extend(self.register.get_commands("wrfrontend", same_mask_for_all_dc=False, name=pixel_reg))
        # generate mask for Imon mask
        pixel_reg = "Imon"
#         mask = self.register_utils.make_box_pixel_mask_from_col_row(column=col_span, row=row_span, default=1, value=0)
        self.register.set_pixel_register_value(pixel_reg, 1)
        commands.extend(self.register.get_commands("wrfrontend", same_mask_for_all_dc=True, name=pixel_reg))
        # disable C_inj mask
        pixel_reg = "C_High"
        self.register.set_pixel_register_value(pixel_reg, 0)
        commands.extend(self.register.get_commands("wrfrontend", same_mask_for_all_dc=True, name=pixel_reg))
        pixel_reg = "C_Low"
        self.register.set_pixel_register_value(pixel_reg, 0)
        commands.extend(self.register.get_commands("wrfrontend", same_mask_for_all_dc=True, name=pixel_reg))
#         self.register.set_global_register_value("Trig_Lat", 232)  # set trigger latency
        self.consecutive_lvl1 = consecutive_lvl1
        self.register.set_global_register_value("Trig_Count", (0 if consecutive_lvl1 == 16 else consecutive_lvl1))  # set number of consecutive triggers
        commands.extend(self.register.get_commands("wrregister", name=["Trig_Count"]))
        # setting FE into runmode
        commands.extend(self.register.get_commands("runmode"))
        self.register_utils.send_commands(commands)

        self.col_arr = np.array([], dtype=np.dtype('>u1'))
        self.row_arr = np.array([], dtype=np.dtype('>u1'))

        with open_raw_data_file(filename=self.scan_data_filename, title=self.scan_identifier) as raw_data_file:
            self.readout.start()

            # preload command
            command_delay = 400 # 100kHz
            lvl1_command = self.register.get_commands("lv1")[0] + self.register.get_commands("zeros", length=command_delay)[0]
            commnd_lenght = lvl1_command.length()
            logging.info('Estimated scan time: %ds' % int(commnd_lenght * 25 * (10 ** -9) * triggers))
            logging.info('Please stand by...')
            self.register_utils.send_command(lvl1_command, repeat=triggers, wait_for_finish=False, set_length=True, clear_memory=False)

            wait_for_first_data = False
            last_iteration = time.time()
            saw_no_data_at_time = last_iteration
            saw_data_at_time = last_iteration
            no_data_at_time = last_iteration
            while not self.stop_thread_event.wait(self.readout.readout_interval):
                last_iteration = time.time()
                try:
                    data = (self.readout.data.popleft(), )
                    raw_data_file.append(data)
                    col_arr_tmp, row_arr_tmp = convert_data_array(data_array_from_data_dict_iterable(data), filter_func=is_data_record, converter_func=get_col_row_array_from_data_record_array)
                    self.col_arr = np.concatenate((self.col_arr, col_arr_tmp))
                    self.row_arr = np.concatenate((self.row_arr, row_arr_tmp))
                    #logging.info('data words')
                except IndexError:  # no data
                    #logging.info('no data words')
                    no_data_at_time = last_iteration
                    if self.register_utils.is_ready:
                        self.stop_thread_event.set()
                        logging.info('Finished sending %d triggers' % triggers)
                    elif wait_for_first_data == False and saw_no_data_at_time > (saw_data_at_time + timeout_no_data):
                        logging.info('Reached no data timeout. Stopping Scan...')
                        self.stop_thread_event.set()
                    elif wait_for_first_data == False:
                        saw_no_data_at_time = no_data_at_time
                    elif self.reaout_utils.is_ready:
                        self.stop_thread_event.set()

                    continue

                saw_data_at_time = last_iteration

                if wait_for_first_data == True:
                    logging.info('Taking data...')
                    wait_for_first_data = False

            self.readout.stop()

            self.register.restore()

            occ_hist, _, _ = np.histogram2d(self.col_arr, self.row_arr, bins=(80, 336), range=[[1, 80], [1, 336]])

            self.occ_mask = np.zeros(shape=occ_hist.shape, dtype=np.dtype('>u1'))
            # noisy pixels are set to 1
            self.occ_mask[occ_hist > occupancy_limit * triggers * consecutive_lvl1] = 1
            # make inverse
            self.inv_occ_mask = self.register_utils.invert_pixel_mask(self.occ_mask)
            self.disable_for_mask = disable_for_mask
            if overwrite_mask:
                self.register.set_pixel_register_value(disable_for_mask, self.inv_occ_mask)
            else:
                for mask in disable_for_mask:
                    enable_mask = np.logical_and(self.inv_occ_mask, self.register.get_pixel_register_value(mask))
                    self.register.set_pixel_register_value(mask, enable_mask)

            self.enable_for_mask = enable_for_mask
            if overwrite_mask:
                self.register.set_pixel_register_value(enable_for_mask, self.occ_mask)
            else:
                for mask in enable_for_mask:
                    disable_mask = np.logical_or(self.occ_mask, self.register.get_pixel_register_value(mask))
                    self.register.set_pixel_register_value(mask, disable_mask)

#             plot_occupancy(self.col_arr, self.row_arr, max_occ=None, filename=self.scan_data_filename + "_occupancy.pdf")

    def analyze(self):
        from analysis.analyze_raw_data import AnalyzeRawData
        output_file = self.scan_data_filename + "_interpreted.h5"
        with AnalyzeRawData(raw_data_file=scan.scan_data_filename, analyzed_data_file=output_file, create_pdf=True) as analyze_raw_data:
            analyze_raw_data.interpreter.set_trig_count(self.consecutive_lvl1)
            analyze_raw_data.create_source_scan_hist = True
#             analyze_raw_data.create_hit_table = True
#             analyze_raw_data.interpreter.debug_events(0, 0, True)  # events to be printed onto the console for debugging, usually deactivated
            analyze_raw_data.interpreter.set_warning_output(False)
            analyze_raw_data.interpret_word_table(fei4b=scan.register.fei4b)
            analyze_raw_data.interpreter.print_summary()
            analyze_raw_data.plot_histograms()
            plot_occupancy(self.occ_mask.T, title='Noisy Pixels', z_max=1, filename=analyze_raw_data.output_pdf)
            for mask in self.disable_for_mask:
                mask_name = self.register.get_pixel_register_attributes("full_name", do_sort=True, name=[mask])[0]
                plot_occupancy(self.register.get_pixel_register_value(mask).T, title='%s Mask' % mask_name, z_max=1, filename=analyze_raw_data.output_pdf)
            for mask in self.enable_for_mask:
                mask_name = self.register.get_pixel_register_attributes("full_name", do_sort=True, name=[mask])[0]
                plot_occupancy(self.register.get_pixel_register_value(mask).T, title='%s Mask' % mask_name, z_max=1, filename=analyze_raw_data.output_pdf)

if __name__ == "__main__":
    import configuration
    scan = NoiseOccupancyScan(config_file=configuration.config_file, bit_file=configuration.bit_file, scan_data_path=configuration.scan_data_path)
    scan.start(configure=True, use_thread=True, occupancy_limit=10 ** (-5), triggers=10000000, consecutive_lvl1=1, disable_for_mask=['Enable'], enable_for_mask=['Imon'], overwrite_mask=False, col_span=[1, 80], row_span=[1, 336], timeout_no_data=10)
    scan.stop()
    scan.analyze()
    scan.register.save_configuration(configuration.config_file)
<|endoftext|>


<commit_before>"""
SciPy --- A scientific computing package for Python

Available subpackages
======================
"""

#try:
    #import gui_thread
#except:
    #print "Warning:  wxPython not loaded"

from scipy_version import scipy_version as __version__

# SciPy has levels
# Level 0 -- Numeric and core routines in scipy_base
#
# Level 1 -- Level 0 + fft, special, linalg, stats
# Level 1a -- Core routines which depend on Level 1.

# Level 2 -- plotting interface.
#
# Level 3
# Packages which define own functions plus depend on Levels 0-2.
#
# Level 0, 1, 2 should be imported in order and then other levels imported
#   as available.

import Numeric
import os,sys
import string

# Level 0
# modules to import under the scipy namespace
from scipy_base import *
import Matrix
mat = Matrix.Matrix

from helpmod import *

try:
    id(help)
except NameError:
    help = info

# Level 1
# these modules will just be imported (not subsumed)
import special, io, linalg, stats, fftpack
from fftpack import fft, fftn, fft2, ifft, ifft2, ifftn, fftshift, ifftshift, cont_ft, fftfreq, zeropad
from stats import mean, median, std, cov, corrcoef
from special import isinf, isfinite, isnan

# Functions to be subsumed that need Level 0 and Level 1
from common import *

try:
    from pilutil import *
except ImportError:
    pass

try:
    import fftw
except ImportError:
    pass  # Not default anymore

# Level 2
import optimize, integrate, signal, special, interpolate, cow, ga, cluster, weave

# Level 3
_plot = []
try:
    if sys.platform != 'win32':
        import xplt
        #__all__.append('xplt')
        _plot.append('xplt')
except ImportError:
    pass

try:
#   gplt on win32 messes up focus and takes up 99%
#   of processor -- works fine on *nix.
    if sys.platform != 'win32':
        import gplt
        #__all__.append('gplt')
        _plot.append('gplt')
except ImportError:
    pass

#if _plot == []:
#    print "Warning: No plotting available."
#else:
#    print "Plotting methods available: ", _plot

# removed gplt because it screws up imports on win32
_pkgs = ['cluster','cow','fftpack','fftw','ga','integrate',
         'interpolate', 'io', 'linalg', 'optimize', 'plt',
         'signal', 'sparse', 'special', 'stats', 'weave',
         'xplt'] #,'gplt']

if sys.platform != 'win32':
    _pkgs.append('gplt')

_pkg_doc = ['Vector Quantization / Kmeans',
            'Cluster of Workstations',
            'FFT algorithms',
            'FFT algorithms using fftw',
            'Genetic Algorithms',
            'Plotting with gnuplot',
            'Integration',
            'Interpolation',
            'Input/Output',
            'Linear Algebra',
            'Optimization',
            'Plotting with wxPython',
            'Signal Processing',
            'Sparse Matrices',
            'Special Functions',
            'Statistics',
            'C/C++ integration',
            'Plotting with X']

def _extend_doc(st,p,pd):
    _lengths = [len(x) for x in p]
    _ml = max(_lengths)
    k = 0
    for name in p:
        try:
            exec "import %s" % name
            st += "%s%s --- %s\n" % (name, ' '*(_ml-_lengths[k]), pd[k])
        except ImportError:
            pass
        k += 1
    return st

__doc__ = _extend_doc(__doc__,_pkgs,_pkg_doc)

#---- testing ----#

def test(level=1):
    """ From this top level, there are possibly many many tests.
        Test only the quick tests by default.
    """
    import unittest
    runner = unittest.TextTestRunner()
    runner.run(test_suite(level))
    return runner

def test_all(level=10):
    test(level)

def test_suite(level = 1):
    import scipy_test.testing
    import scipy
    ignore = ['xplt','plt','gplt','gui_thread','sparse','scipy_version']
    suites = [scipy_test.testing.harvest_test_suites(scipy,ignore,level=level)]
    suites += [scipy_test.testing.harvest_test_suites(weave,ignore,level=level)]
    import unittest
    return unittest.TestSuite(suites)<commit_msg>removed plt from _pkgs so that it isn't imported at start up.  This causes problems on Unix machines when DISPLAY isn't set and GTK isn't available.<commit_after>"""
SciPy --- A scientific computing package for Python

Available subpackages
======================
"""

#try:
    #import gui_thread
#except:
    #print "Warning:  wxPython not loaded"

from scipy_version import scipy_version as __version__

# SciPy has levels
# Level 0 -- Numeric and core routines in scipy_base
#
# Level 1 -- Level 0 + fft, special, linalg, stats
# Level 1a -- Core routines which depend on Level 1.

# Level 2 -- plotting interface.
#
# Level 3
# Packages which define own functions plus depend on Levels 0-2.
#
# Level 0, 1, 2 should be imported in order and then other levels imported
#   as available.

import Numeric
import os,sys
import string

# Level 0
# modules to import under the scipy namespace
from scipy_base import *
import Matrix
mat = Matrix.Matrix

from helpmod import *

try:
    id(help)
except NameError:
    help = info

# Level 1
# these modules will just be imported (not subsumed)
import special, io, linalg, stats, fftpack
from fftpack import fft, fftn, fft2, ifft, ifft2, ifftn, fftshift, ifftshift, cont_ft, fftfreq, zeropad
from stats import mean, median, std, cov, corrcoef
from special import isinf, isfinite, isnan

# Functions to be subsumed that need Level 0 and Level 1
from common import *

try:
    from pilutil import *
except ImportError:
    pass

try:
    import fftw
except ImportError:
    pass  # Not default anymore

# Level 2
import optimize, integrate, signal, special, interpolate, cow, ga, cluster, weave

# Level 3
_plot = []
try:
    if sys.platform != 'win32':
        import xplt
        #__all__.append('xplt')
        _plot.append('xplt')
except ImportError:
    pass

try:
#   gplt on win32 messes up focus and takes up 99%
#   of processor -- works fine on *nix.
    if sys.platform != 'win32':
        import gplt
        #__all__.append('gplt')
        _plot.append('gplt')
except ImportError:
    pass

#if _plot == []:
#    print "Warning: No plotting available."
#else:
#    print "Plotting methods available: ", _plot

# removed gplt because it screws up imports on win32
# removed plt cause it screws up unix when gtk isn't available.
_pkgs = ['cluster','cow','fftpack','fftw','ga','integrate',
         'interpolate', 'io', 'linalg', 'optimize',
         'signal', 'sparse', 'special', 'stats', 'weave',
         'xplt'] #,'plt', 'gplt']

if sys.platform != 'win32':
    _pkgs.append('gplt')

_pkg_doc = ['Vector Quantization / Kmeans',
            'Cluster of Workstations',
            'FFT algorithms',
            'FFT algorithms using fftw',
            'Genetic Algorithms',
            'Plotting with gnuplot',
            'Integration',
            'Interpolation',
            'Input/Output',
            'Linear Algebra',
            'Optimization',
            'Plotting with wxPython',
            'Signal Processing',
            'Sparse Matrices',
            'Special Functions',
            'Statistics',
            'C/C++ integration',
            'Plotting with X']

def _extend_doc(st,p,pd):
    _lengths = [len(x) for x in p]
    _ml = max(_lengths)
    k = 0
    for name in p:
        try:
            exec "import %s" % name
            st += "%s%s --- %s\n" % (name, ' '*(_ml-_lengths[k]), pd[k])
        except ImportError:
            pass
        k += 1
    return st

__doc__ = _extend_doc(__doc__,_pkgs,_pkg_doc)

#---- testing ----#

def test(level=1):
    """ From this top level, there are possibly many many tests.
        Test only the quick tests by default.
    """
    import unittest
    runner = unittest.TextTestRunner()
    runner.run(test_suite(level))
    return runner

def test_all(level=10):
    test(level)

def test_suite(level = 1):
    import scipy_test.testing
    import scipy
    ignore = ['xplt','plt','gplt','gui_thread','sparse','scipy_version']
    suites = [scipy_test.testing.harvest_test_suites(scipy,ignore,level=level)]
    suites += [scipy_test.testing.harvest_test_suites(weave,ignore,level=level)]
    import unittest
    return unittest.TestSuite(suites)<|endoftext|>


<commit_before>import math
from sqlalchemy import or_
from collections import Counter

from server import jobs
from server.models import Assignment, Score, Backup, db

@jobs.background_job
def grade_on_effort(assignment_id, full_credit, late_multiplier, required_questions, grading_url):
    logger = jobs.get_job_logger()

    current_user = jobs.get_current_job().user
    assignment = Assignment.query.get(assignment_id)
    submissions = assignment.course_submissions(include_empty=False)

    # archive all previous effort scores for this assignment
    scores = Score.query.filter(
        Score.kind == 'effort',
        assignment_id == assignment_id).all()
    for score in scores:
        db.session.delete(score)

    seen = set()
    stats = Counter()
    manual, late = [], []
    for i, subm in enumerate(submissions, 1):
        user_id = int(subm['user']['id'])
        if user_id in seen:
            continue

        backup = Backup.query.get(subm['backup']['id'])

        if backup.submission_time > assignment.due_date:
            submitter_id = backup.submitter_id
            backups = (
                find_ontime(submitter_id, assignment_id, assignment.due_date),
                find_ontime(submitter_id, assignment_id, assignment.lock_date),
                backup # default to the late backup
            )
            backup = best_scoring(backups, full_credit, required_questions)

        try:
            score, messages = effort_score(backup, full_credit, required_questions)
        except AssertionError:
            pass

        if backup.submission_time > assignment.lock_date:
            late.append(backup.hashid)
            messages.append('\nLate - No Credit')
            score = 0
        elif backup.submission_time > assignment.due_date:
            late.append(backup.hashid)
            late_percent = 100 - round(late_multiplier * 100)
            messages.append('\nLate - {}% off'.format(late_percent))
            score = math.floor(score * late_multiplier)

        messages.append('\nFinal Score: {}'.format(score))

        new_score = Score(score=score, kind='effort',
                message='\n'.join(messages),
                user_id=backup.submitter_id,
                assignment=assignment, backup=backup,
                grader=current_user)
        db.session.add(new_score)

        if i % 100 == 0:
            logger.info('Scored {}/{}'.format(i, len(submissions)))

        if subm['group']:
            member_ids = {int(id) for id in subm['group']['group_member'].split(',')}
            seen |= member_ids
            stats[score] += len(member_ids)
        else:
            seen.add(user_id)
            stats[score] += 1

    logger.info('Scored {}/{}'.format(i, len(submissions)))
    logger.info('done!')

    if len(late) > 0:
        logger.info('\n{} Late:'.format(len(late)))
        for backup_id in late:
            logger.info('  {}'.format(grading_url + backup_id))

    logger.info('\nScore Distribution:')
    sorted_scores = sorted(stats.items(), key=lambda p: -p[0])
    for score, count in sorted_scores:
        logger.info('  {} - {}'.format(str(score).rjust(3), count))

    if len(manual) > 0:
        logger.info('\n{} Backups Need Manual Grading:'.format(len(manual)))
        for backup_id in manual:
            logger.info(grading_url + backup_id)

    db.session.commit()
    return '/admin/course/{cid}/assignments/{aid}/scores'.format(
                cid=jobs.get_current_job().course_id,
                aid=assignment_id)

def best_scoring(backups, full_credit, required_questions):
    def effort_grade(backup):
        score = 0
        try:
            score, _ = effort_score(backup, full_credit, required_questions)
        except AssertionError:
            pass
        return score
    non_none = (b for b in backups if b is not None)
    return max(non_none, key=effort_grade)

def find_ontime(submitter_id, assignment_id, due_date):
    submission = Backup.query.filter(
            Backup.assignment_id == assignment_id,
            Backup.submitter_id == submitter_id,
            Backup.submit == True,
            or_(Backup.created < due_date,
                Backup.custom_submission_time < due_date)
        ).order_by(Backup.created.desc()).first()
    backup = Backup.query.filter(
        Backup.assignment_id == assignment_id,
        Backup.submitter_id == submitter_id,
        or_(Backup.created < due_date,
            Backup.custom_submission_time < due_date)
    ).order_by(Backup.created.desc()).first()
    return submission or backup

def effort_score(backup, full_credit, required_questions):
    """
    Gives a score based on "effort" instead of correctness.

    Effort credit for a question is given if either
        1.  The question is correct
        2.  The question has at least one testcase passed
        3.  Greater than 3 attempts were made
    """
    grading = backup.grading()
    analytics = backup.analytics()

    with_effort = 0
    messages = ['Effort Breakdown']

    history = analytics and analytics.get('history') and analytics['history']['questions']
    questions = grading.keys()
    if history:
        questions |= history.keys()

    assert len(questions) > 0

    for question in questions:
        correct, showed_effort = False, False
        if history and history.get(question):
            info = history[question]
            correct |= info['solved']
            showed_effort |= info['attempts'] >= 3
        if grading and grading.get(question):
            info = grading[question]
            correct |= info['locked'] == 0 and info['failed'] == 0
            showed_effort |= info['passed'] >= 1

        if correct or showed_effort:
            with_effort += 1

        if correct:
            message = 'Correct'
        elif showed_effort:
            message = 'Sufficient Effort'
        else:
            message = 'Not Sufficient Effort'
        messages.append('    {}: {}'.format(question, message))

    score = math.ceil(full_credit * with_effort / required_questions)
    return min(score, full_credit), messages
<commit_msg>erge branch 'master' of https://github.com/Cal-CS-61A-Staff/ok<commit_after>import math
from sqlalchemy import or_
from collections import Counter

from server import jobs
from server.models import Assignment, Score, Backup, db

@jobs.background_job
def grade_on_effort(assignment_id, full_credit, late_multiplier, required_questions, grading_url):
    logger = jobs.get_job_logger()

    current_user = jobs.get_current_job().user
    assignment = Assignment.query.get(assignment_id)
    submissions = assignment.course_submissions(include_empty=False)

    # archive all previous effort scores for this assignment
    scores = Score.query.filter(
        Score.kind == 'effort',
        Score.assignment_id == assignment_id).all()
    for score in scores:
        db.session.delete(score)

    seen = set()
    stats = Counter()
    manual, late = [], []
    for i, subm in enumerate(submissions, 1):
        user_id = int(subm['user']['id'])
        if user_id in seen:
            continue

        backup = Backup.query.get(subm['backup']['id'])

        if backup.submission_time > assignment.due_date:
            submitter_id = backup.submitter_id
            backups = (
                find_ontime(submitter_id, assignment_id, assignment.due_date),
                find_ontime(submitter_id, assignment_id, assignment.lock_date),
                backup # default to the late backup
            )
            backup = best_scoring(backups, full_credit, required_questions)

        try:
            score, messages = effort_score(backup, full_credit, required_questions)
        except AssertionError:
            pass

        if backup.submission_time > assignment.lock_date:
            late.append(backup.hashid)
            messages.append('\nLate - No Credit')
            score = 0
        elif backup.submission_time > assignment.due_date:
            late.append(backup.hashid)
            late_percent = 100 - round(late_multiplier * 100)
            messages.append('\nLate - {}% off'.format(late_percent))
            score = math.floor(score * late_multiplier)

        messages.append('\nFinal Score: {}'.format(score))

        new_score = Score(score=score, kind='effort',
                message='\n'.join(messages),
                user_id=backup.submitter_id,
                assignment=assignment, backup=backup,
                grader=current_user)
        db.session.add(new_score)

        if i % 100 == 0:
            logger.info('Scored {}/{}'.format(i, len(submissions)))

        if subm['group']:
            member_ids = {int(id) for id in subm['group']['group_member'].split(',')}
            seen |= member_ids
            stats[score] += len(member_ids)
        else:
            seen.add(user_id)
            stats[score] += 1

    logger.info('Scored {}/{}'.format(i, len(submissions)))
    logger.info('done!')

    if len(late) > 0:
        logger.info('\n{} Late:'.format(len(late)))
        for backup_id in late:
            logger.info('  {}'.format(grading_url + backup_id))

    logger.info('\nScore Distribution:')
    sorted_scores = sorted(stats.items(), key=lambda p: -p[0])
    for score, count in sorted_scores:
        logger.info('  {} - {}'.format(str(score).rjust(3), count))

    if len(manual) > 0:
        logger.info('\n{} Backups Need Manual Grading:'.format(len(manual)))
        for backup_id in manual:
            logger.info(grading_url + backup_id)

    db.session.commit()
    return '/admin/course/{cid}/assignments/{aid}/scores'.format(
                cid=jobs.get_current_job().course_id,
                aid=assignment_id)

def best_scoring(backups, full_credit, required_questions):
    def effort_grade(backup):
        score = 0
        try:
            score, _ = effort_score(backup, full_credit, required_questions)
        except AssertionError:
            pass
        return score
    non_none = (b for b in backups if b is not None)
    return max(non_none, key=effort_grade)

def find_ontime(submitter_id, assignment_id, due_date):
    submission = Backup.query.filter(
            Backup.assignment_id == assignment_id,
            Backup.submitter_id == submitter_id,
            Backup.submit == True,
            or_(Backup.created < due_date,
                Backup.custom_submission_time < due_date)
        ).order_by(Backup.created.desc()).first()
    backup = Backup.query.filter(
        Backup.assignment_id == assignment_id,
        Backup.submitter_id == submitter_id,
        or_(Backup.created < due_date,
            Backup.custom_submission_time < due_date)
    ).order_by(Backup.created.desc()).first()
    return submission or backup

def effort_score(backup, full_credit, required_questions):
    """
    Gives a score based on "effort" instead of correctness.

    Effort credit for a question is given if either
        1.  The question is correct
        2.  The question has at least one testcase passed
        3.  Greater than 3 attempts were made
    """
    grading = backup.grading()
    analytics = backup.analytics()

    with_effort = 0
    messages = ['Effort Breakdown']

    history = analytics and analytics.get('history') and analytics['history']['questions']
    questions = grading.keys()
    if history:
        questions |= history.keys()

    assert len(questions) > 0

    for question in questions:
        correct, showed_effort = False, False
        if history and history.get(question):
            info = history[question]
            correct |= info['solved']
            showed_effort |= info['attempts'] >= 3
        if grading and grading.get(question):
            info = grading[question]
            correct |= info['locked'] == 0 and info['failed'] == 0
            showed_effort |= info['passed'] >= 1

        if correct or showed_effort:
            with_effort += 1

        if correct:
            message = 'Correct'
        elif showed_effort:
            message = 'Sufficient Effort'
        else:
            message = 'Not Sufficient Effort'
        messages.append('    {}: {}'.format(question, message))

    score = math.ceil(full_credit * with_effort / required_questions)
    return min(score, full_credit), messages
<|endoftext|>


<commit_before>"""Create an applet from a Python script.

This puts up a dialog asking for a Python source file ('TEXT').
The output is a file with the same name but its ".py" suffix dropped.
It is created by copying an applet template and then adding a 'PYC '
resource named __main__ containing the compiled, marshalled script.
"""

import sys
sys.stdout = sys.stderr

import string
import os
import marshal
import imp
import macfs
import MacOS
from Res import *

# .pyc file (and 'PYC ' resource magic number)
MAGIC = imp.get_magic()

# Template file (searched on sys.path)
TEMPLATE = "PythonApplet"

# Specification of our resource
RESTYPE = 'PYC '
RESNAME = '__main__'

# A resource with this name sets the "owner" (creator) of the destination
OWNERNAME = "owner resource"

# OpenResFile mode parameters
READ = 1
WRITE = 2

def main():

	# Find the template
	# (there's no point in proceeding if we can't find it)

	for p in sys.path:
		template = os.path.join(p, TEMPLATE)
		try:
			tmpl = open(template, "rb")
			tmpl.close()
			break
		except IOError:
			continue
	else:
		die("Template %s not found" % `template`)
		return

	# Ask for source text if not specified in sys.argv[1:]

	if not sys.argv[1:]:
		srcfss, ok = macfs.StandardGetFile('TEXT')
		if not ok:
			return
		filename = srcfss.as_pathname()
		if not sys.argv: sys.argv.append('')
		sys.argv.append(filename)

	# Loop over all files to be processed

	for filename in sys.argv[1:]:
		process(template, filename)

undefs = ('????', '    ', '\0\0\0\0', 'BINA')

def process(template, filename):

	print "Processing", `filename`, "..."

	# Read the source and compile it
	# (there's no point overwriting the destination if it has a syntax error)

	fp = open(filename)
	text = fp.read()
	fp.close()
	try:
		code = compile(text, filename, "exec")
	except (SyntaxError, EOFError):
		die("Syntax error in script %s" % `filename`)
		return

	# Set the destination file name

	if string.lower(filename[-3:]) == ".py":
		destname = filename[:-3]
		rsrcname = destname + '.rsrc'
	else:
		destname = filename + ".applet"
		rsrcname = filename + '.rsrc'

	# Copy the data from the template (creating the file as well)

	tmpl = open(template, "rb")
	dest = open(destname, "wb")
	data = tmpl.read()
	if data:
		dest.write(data)
	dest.close()
	tmpl.close()

	# Copy the creator of the template to the destination
	# unless it already got one.  Set type to APPL

	tctor, ttype = MacOS.GetCreatorAndType(template)
	ctor, type = MacOS.GetCreatorAndType(destname)
	if type in undefs: type = 'APPL'
	if ctor in undefs: ctor = tctor

	# Open the output resource fork

	try:
		output = FSpOpenResFile(destname, WRITE)
	except MacOS.Error:
		print "Creating resource fork..."
		CreateResFile(destname)
		output = FSpOpenResFile(destname, WRITE)

	# Copy the resources from the template

	input = FSpOpenResFile(template, READ)
	newctor = copyres(input, output)
	CloseResFile(input)
	if newctor: ctor = newctor

	# Copy the resources from the target specific resource template, if any

	try:
		input = FSpOpenResFile(rsrcname, READ)
	except MacOS.Error:
		pass
	else:
		newctor = copyres(input, output)
		CloseResFile(input)
		if newctor: ctor = newctor

	# Now set the creator and type of the destination

	MacOS.SetCreatorAndType(destname, ctor, type)

	# Make sure we're manipulating the output resource file now

	UseResFile(output)

	# Delete any existing 'PYC 'resource named __main__

	try:
		res = Get1NamedResource(RESTYPE, RESNAME)
		res.RmveResource()
	except Error:
		pass

	# Create the raw data for the resource from the code object

	data = marshal.dumps(code)
	del code
	data = (MAGIC + '\0\0\0\0') + data

	# Create the resource and write it

	id = 0
	while id < 128:
		id = Unique1ID(RESTYPE)
	res = Resource(data)
	res.AddResource(RESTYPE, id, RESNAME)
	res.WriteResource()
	res.ReleaseResource()

	# Close the output file

	CloseResFile(output)

	# Give positive feedback

	message("Applet %s created." % `destname`)

# Copy resources between two resource file descriptors.
# Exception: don't copy a __main__ resource.
# If a resource's name is "owner resource", its type is returned
# (so the caller can use it to set the destination's creator)

def copyres(input, output):
	ctor = None
	UseResFile(input)
	ntypes = Count1Types()
	for itype in range(1, 1+ntypes):
		type = Get1IndType(itype)
		nresources = Count1Resources(type)
		for ires in range(1, 1+nresources):
			res = Get1IndResource(type, ires)
			id, type, name = res.GetResInfo()
			lcname = string.lower(name)
			if (type, lcname) == (RESTYPE, RESNAME):
				continue # Don't copy __main__ from template
			if lcname == OWNERNAME: ctor = type
			size = res.SizeResource()
			attrs = res.GetResAttrs()
			print id, type, name, size, hex(attrs)
			res.LoadResource()
			res.DetachResource()
			UseResFile(output)
			try:
				res2 = Get1Resource(type, id)
			except MacOS.Error:
				res2 = None
			if res2:
				print "Overwriting..."
				res2.RmveResource()
			res.AddResource(type, id, name)
			res.WriteResource()
			attrs = attrs | res.GetResAttrs()
			print "New attrs =", hex(attrs)
			res.SetResAttrs(attrs)
			UseResFile(input)
	return ctor

# Show a message and exit

def die(str):
	message(str)
	sys.exit(1)

# Show a message

def message(str, id = 256):
	from Dlg import *
	d = GetNewDialog(id, -1)
	if not d:
		print "Error:", `str`
		print "DLOG id =", id, "not found."
		return
	tp, h, rect = d.GetDItem(2)
	SetIText(h, str)
	while 1:
		n = ModalDialog(None)
		if n == 1: break
	del d

if __name__ == '__main__':
	main()

<commit_msg>In interactive mode ask for output file<commit_after>"""Create an applet from a Python script.

This puts up a dialog asking for a Python source file ('TEXT').
The output is a file with the same name but its ".py" suffix dropped.
It is created by copying an applet template and then adding a 'PYC '
resource named __main__ containing the compiled, marshalled script.
"""

import sys
sys.stdout = sys.stderr

import string
import os
import marshal
import imp
import macfs
import MacOS
from Res import *

# .pyc file (and 'PYC ' resource magic number)
MAGIC = imp.get_magic()

# Template file (searched on sys.path)
TEMPLATE = "PythonApplet"

# Specification of our resource
RESTYPE = 'PYC '
RESNAME = '__main__'

# A resource with this name sets the "owner" (creator) of the destination
OWNERNAME = "owner resource"

# OpenResFile mode parameters
READ = 1
WRITE = 2

def main():

	# Find the template
	# (there's no point in proceeding if we can't find it)

	for p in sys.path:
		template = os.path.join(p, TEMPLATE)
		try:
			tmpl = open(template, "rb")
			tmpl.close()
			break
		except IOError:
			continue
	else:
		die("Template %s not found" % `template`)
		return

	# Ask for source text if not specified in sys.argv[1:]

	if not sys.argv[1:]:
		srcfss, ok = macfs.StandardGetFile('TEXT')
		if not ok:
			return
		filename = srcfss.as_pathname()
		tp, tf = os.path.split(filename)
		if tf[-3:] == '.py':
			tf = tf[:-3]
		else:
			tf = tf + '.applet'
		dstfss, ok = macfs.StandardPutFile('Save application as:', tf)
		if not ok: return
		process(template, filename, dstfss.as_pathname())
	else:

		# Loop over all files to be processed
		for filename in sys.argv[1:]:
			process(template, filename, '')

undefs = ('????', '    ', '\0\0\0\0', 'BINA')

def process(template, filename, output):

	print "Processing", `filename`, "..."

	# Read the source and compile it
	# (there's no point overwriting the destination if it has a syntax error)

	fp = open(filename)
	text = fp.read()
	fp.close()
	try:
		code = compile(text, filename, "exec")
	except (SyntaxError, EOFError):
		die("Syntax error in script %s" % `filename`)
		return

	# Set the destination file name

	if string.lower(filename[-3:]) == ".py":
		destname = filename[:-3]
		rsrcname = destname + '.rsrc'
	else:
		destname = filename + ".applet"
		rsrcname = filename + '.rsrc'

	if output:
		destname = output
	# Copy the data from the template (creating the file as well)

	tmpl = open(template, "rb")
	dest = open(destname, "wb")
	data = tmpl.read()
	if data:
		dest.write(data)
	dest.close()
	tmpl.close()

	# Copy the creator of the template to the destination
	# unless it already got one.  Set type to APPL

	tctor, ttype = MacOS.GetCreatorAndType(template)
	ctor, type = MacOS.GetCreatorAndType(destname)
	if type in undefs: type = 'APPL'
	if ctor in undefs: ctor = tctor

	# Open the output resource fork

	try:
		output = FSpOpenResFile(destname, WRITE)
	except MacOS.Error:
		print "Creating resource fork..."
		CreateResFile(destname)
		output = FSpOpenResFile(destname, WRITE)

	# Copy the resources from the template

	input = FSpOpenResFile(template, READ)
	newctor = copyres(input, output)
	CloseResFile(input)
	if newctor: ctor = newctor

	# Copy the resources from the target specific resource template, if any

	try:
		input = FSpOpenResFile(rsrcname, READ)
	except MacOS.Error:
		pass
	else:
		newctor = copyres(input, output)
		CloseResFile(input)
		if newctor: ctor = newctor

	# Now set the creator and type of the destination

	MacOS.SetCreatorAndType(destname, ctor, type)

	# Make sure we're manipulating the output resource file now

	UseResFile(output)

	# Delete any existing 'PYC 'resource named __main__

	try:
		res = Get1NamedResource(RESTYPE, RESNAME)
		res.RmveResource()
	except Error:
		pass

	# Create the raw data for the resource from the code object

	data = marshal.dumps(code)
	del code
	data = (MAGIC + '\0\0\0\0') + data

	# Create the resource and write it

	id = 0
	while id < 128:
		id = Unique1ID(RESTYPE)
	res = Resource(data)
	res.AddResource(RESTYPE, id, RESNAME)
	res.WriteResource()
	res.ReleaseResource()

	# Close the output file

	CloseResFile(output)

	# Give positive feedback

	message("Applet %s created." % `destname`)

# Copy resources between two resource file descriptors.
# Exception: don't copy a __main__ resource.
# If a resource's name is "owner resource", its type is returned
# (so the caller can use it to set the destination's creator)

def copyres(input, output):
	ctor = None
	UseResFile(input)
	ntypes = Count1Types()
	for itype in range(1, 1+ntypes):
		type = Get1IndType(itype)
		nresources = Count1Resources(type)
		for ires in range(1, 1+nresources):
			res = Get1IndResource(type, ires)
			id, type, name = res.GetResInfo()
			lcname = string.lower(name)
			if (type, lcname) == (RESTYPE, RESNAME):
				continue # Don't copy __main__ from template
			if lcname == OWNERNAME: ctor = type
			size = res.SizeResource()
			attrs = res.GetResAttrs()
			print id, type, name, size, hex(attrs)
			res.LoadResource()
			res.DetachResource()
			UseResFile(output)
			try:
				res2 = Get1Resource(type, id)
			except MacOS.Error:
				res2 = None
			if res2:
				print "Overwriting..."
				res2.RmveResource()
			res.AddResource(type, id, name)
			res.WriteResource()
			attrs = attrs | res.GetResAttrs()
			print "New attrs =", hex(attrs)
			res.SetResAttrs(attrs)
			UseResFile(input)
	return ctor

# Show a message and exit

def die(str):
	message(str)
	sys.exit(1)

# Show a message

def message(str, id = 256):
	from Dlg import *
	d = GetNewDialog(id, -1)
	if not d:
		print "Error:", `str`
		print "DLOG id =", id, "not found."
		return
	tp, h, rect = d.GetDItem(2)
	SetIText(h, str)
	while 1:
		n = ModalDialog(None)
		if n == 1: break
	del d

if __name__ == '__main__':
	main()

<|endoftext|>


<commit_before>#!/usr/bin/env python
'''

Usage:
    validate.py [path]

Validates all xml files against the DocBook 5 RELAX NG schema, and
attempts to build all books.

Options:
    path     Root directory, defaults to <repo root>/doc/src/doc/docbkx

Ignores pom.xml files and subdirectories named "target".

Requires:
    - Python 2.7 or greater (for argparse)
    - lxml Python library
    - Maven

'''
from lxml import etree

import argparse
import os
import subprocess
import sys
import urllib2

# These are files that are known to not be in DocBook format
FILE_EXCEPTIONS = ['ha-guide-docinfo.xml']

# These are books that we aren't checking yet
BOOK_EXCEPTIONS = ['openstack-training']

def get_schema():
    """Return the DocBook RELAX NG schema"""
    url = "http://www.oasis-open.org/docbook/xml/5.0b5/rng/docbookxi.rng"
    relaxng_doc = etree.parse(urllib2.urlopen(url))
    return etree.RelaxNG(relaxng_doc)

def validation_failed(schema, doc):
    """Return True if the parsed doc fails against the schema

    This will ignore validation failures of the type: IDREF attribute linkend
    references an unknown ID. This is because we are validating individual
    files that are being imported, and sometimes the reference isn't present
    in the current file."""
    return not schema.validate(doc) and \
        any(log.type_name != "DTD_UNKNOWN_ID" for log in schema.error_log)

def verify_section_tags_have_xmid(doc):
    """Check that all section tags have an xml:id attribute

    Will throw an exception if there's at least one missing"""
    ns = {"docbook": "http://docbook.org/ns/docbook"}
    for node in doc.xpath('//docbook:section', namespaces=ns):
        if "{http://www.w3.org/XML/1998/namespace}id" not in node.attrib:
            raise ValueError("section missing xml:id attribute, line %d" %
                             node.sourceline)

def error_message(error_log):
    """Return a string that contains the error message.

    We use this to filter out false positives related to IDREF attributes
    """
    errs = [str(x) for x in error_log if x.type_name != 'DTD_UNKNOWN_ID']

    # Reverse output so that earliest failures are reported first
    errs.reverse()
    return "\n".join(errs)

def validate_individual_files(rootdir, exceptions):
    schema = get_schema()

    any_failures = False

    for root, dirs, files in os.walk(rootdir):
        # Don't descend into 'target' subdirectories
        try:
            ind = dirs.index('target')
            del dirs[ind]
        except ValueError:
            pass

        for f in files:
            # Ignore maven files, which are called pom.xml
            if f.endswith('.xml') and f != 'pom.xml' \
                                  and f not in exceptions:
                try:
                    path = os.path.abspath(os.path.join(root, f))
                    doc = etree.parse(path)
                    if validation_failed(schema, doc):
                        any_failures = True
                        print error_message(schema.error_log)
                    verify_section_tags_have_xmid(doc)
                except etree.XMLSyntaxError as e:
                    any_failures = True
                    print "%s: %s" % (path, e)
                except ValueError as e:
                    any_failures = True
                    print "%s: %s" % (path, e)

    if any_failures:
        sys.exit(1)

def build_all_books(rootdir, exceptions):
    """ Build all of the books.

    Looks for all directories with "pom.xml" in them and runs
    "mvn clean generate-sources" in that directory.

    This will throw an exception if a book fails to build
    """
    for root, dirs, files in os.walk(rootdir):
        book = os.path.basename(root)
        if ("pom.xml" in files) and (book not in exceptions):
            print "Building %s" % book
            os.chdir(root)
            subprocess.check_call(["mvn", "clean", "generate-sources"])

def main(rootdir):
    validate_individual_files(rootdir, FILE_EXCEPTIONS)
    build_all_books(rootdir, BOOK_EXCEPTIONS)

def default_root():
    """Return the location of openstack-manuals/doc/src/docbkx

    The current working directory must be inside of the openstack-manuals
    repository for this method to succeed"""
    args = ["git", "rev-parse", "--show-toplevel"]
    gitroot = subprocess.check_output(args).rstrip()
    return os.path.join(gitroot, "doc/src/docbkx")

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Validate XML files against "
                                     "the DocBook 5 RELAX NG schema")
    parser.add_argument('path', nargs='?', default=default_root(),
                        help="Root directory that contains DocBook files, "
                        "defaults to `git rev-parse --show-toplevel`/doc/src/"
                        "docbkx")
    args = parser.parse_args()
    main(args.path)
<commit_msg>validate.py variable changes<commit_after>#!/usr/bin/env python
'''

Usage:
    validate.py [path]

Validates all xml files against the DocBook 5 RELAX NG schema, and
attempts to build all books.

Options:
    path     Root directory, defaults to <repo root>/doc/src/doc/docbkx

Ignores pom.xml files and subdirectories named "target".

Requires:
    - Python 2.7 or greater (for argparse)
    - lxml Python library
    - Maven

'''
from lxml import etree

import argparse
import os
import subprocess
import sys
import urllib2

# These are files that are known to not be in DocBook format
FILE_EXCEPTIONS = ['ha-guide-docinfo.xml','bk001-ch003-associate-general.xml']

# These are books that we aren't checking yet
BOOK_EXCEPTIONS = []

def get_schema():
    """Return the DocBook RELAX NG schema"""
    url = "http://www.oasis-open.org/docbook/xml/5.0b5/rng/docbookxi.rng"
    relaxng_doc = etree.parse(urllib2.urlopen(url))
    return etree.RelaxNG(relaxng_doc)

def validation_failed(schema, doc):
    """Return True if the parsed doc fails against the schema

    This will ignore validation failures of the type: IDREF attribute linkend
    references an unknown ID. This is because we are validating individual
    files that are being imported, and sometimes the reference isn't present
    in the current file."""
    return not schema.validate(doc) and \
        any(log.type_name != "DTD_UNKNOWN_ID" for log in schema.error_log)

def verify_section_tags_have_xmid(doc):
    """Check that all section tags have an xml:id attribute

    Will throw an exception if there's at least one missing"""
    ns = {"docbook": "http://docbook.org/ns/docbook"}
    for node in doc.xpath('//docbook:section', namespaces=ns):
        if "{http://www.w3.org/XML/1998/namespace}id" not in node.attrib:
            raise ValueError("section missing xml:id attribute, line %d" %
                             node.sourceline)

def error_message(error_log):
    """Return a string that contains the error message.

    We use this to filter out false positives related to IDREF attributes
    """
    errs = [str(x) for x in error_log if x.type_name != 'DTD_UNKNOWN_ID']

    # Reverse output so that earliest failures are reported first
    errs.reverse()
    return "\n".join(errs)

def validate_individual_files(rootdir, exceptions):
    schema = get_schema()

    any_failures = False

    for root, dirs, files in os.walk(rootdir):
        # Don't descend into 'target' subdirectories
        try:
            ind = dirs.index('target')
            del dirs[ind]
        except ValueError:
            pass

        for f in files:
            # Ignore maven files, which are called pom.xml
            if f.endswith('.xml') and f != 'pom.xml' \
                                  and f not in exceptions:
                try:
                    path = os.path.abspath(os.path.join(root, f))
                    doc = etree.parse(path)
                    if validation_failed(schema, doc):
                        any_failures = True
                        print error_message(schema.error_log)
                    verify_section_tags_have_xmid(doc)
                except etree.XMLSyntaxError as e:
                    any_failures = True
                    print "%s: %s" % (path, e)
                except ValueError as e:
                    any_failures = True
                    print "%s: %s" % (path, e)

    if any_failures:
        sys.exit(1)

def build_all_books(rootdir, exceptions):
    """ Build all of the books.

    Looks for all directories with "pom.xml" in them and runs
    "mvn clean generate-sources" in that directory.

    This will throw an exception if a book fails to build
    """
    for root, dirs, files in os.walk(rootdir):
        book = os.path.basename(root)
        if ("pom.xml" in files) and (book not in exceptions):
            print "Building %s" % book
            os.chdir(root)
            subprocess.check_call(["mvn", "clean", "generate-sources"])

def main(rootdir):
    validate_individual_files(rootdir, FILE_EXCEPTIONS)
    build_all_books(rootdir, BOOK_EXCEPTIONS)

def default_root():
    """Return the location of openstack-manuals/doc/src/docbkx

    The current working directory must be inside of the openstack-manuals
    repository for this method to succeed"""
    args = ["git", "rev-parse", "--show-toplevel"]
    gitroot = subprocess.check_output(args).rstrip()
    return os.path.join(gitroot, "doc/src/docbkx")

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Validate XML files against "
                                     "the DocBook 5 RELAX NG schema")
    parser.add_argument('path', nargs='?', default=default_root(),
                        help="Root directory that contains DocBook files, "
                        "defaults to `git rev-parse --show-toplevel`/doc/src/"
                        "docbkx")
    args = parser.parse_args()
    main(args.path)
<|endoftext|>


<commit_before># Copyright 2014, Sandia Corporation. Under the terms of Contract
# DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains certain
# rights in this software.

from __future__ import division

import numbers
import toyplot.compatibility

def region(xmin, xmax, ymin, ymax, bounds=None, rect=None, corner=None, grid=None, gutter=40):
  def length(min, max, value):
    if isinstance(value, numbers.Number):
      return value
    if isinstance(value, toyplot.compatibility.string_type):
      value = value.strip()
      if value[-1] == "%":
        value = float(value[:-1]) * 0.01
        return ((1.0 - value) * min) + (value * max)
      else:
        return float(value)

  # Specify explicit bounds for the region
  if bounds is not None:
    if isinstance(bounds, tuple) and len(bounds) == 4:
      return (length(xmin, xmax, bounds[0]), length(xmin, xmax, bounds[1]), length(ymin, ymax, bounds[2]), length(ymin, ymax, bounds[3]))
    raise ValueError("Unrecognized bounds type")
  # Specify an explicit rectangle for the region
  if rect is not None:
    if isinstance(rect, tuple) and len(rect) == 4:
      x = length(xmin, xmax, rect[0])
      y = length(ymin, ymax, rect[1])
      width = length(xmin, xmax, rect[2])
      height = length(ymin, ymax, rect[3])
      return (x, x + width, y, y + height)
    raise ValueError("Unrecognized rect type")
  # Offset a rectangle from a corner
  if corner is not None:
    if isinstance(corner, tuple) and len(corner) == 4:
      position = corner[0]
      width = length(xmin, xmax, corner[1])
      height = length(ymin, ymax, corner[2])
      inset = float(corner[3])
    else:
      raise ValueError("Unrecognized corner type")
    if position == "top":
      return ((xmin + xmax - width) / 2, (xmin + xmax + width) / 2, ymin + inset, ymin + inset + height)
    elif position == "top-right":
      return (xmax - width - inset, xmax - inset, ymin + inset, ymin + inset + height)
    elif position == "right":
      return (xmax - width - inset, xmax - inset, (ymin + ymax - height) / 2, (ymin + ymax + height) / 2)
    elif position == "bottom-right":
      return (xmax - width - inset, xmax - inset, ymax - inset - height, ymax - inset)
    elif position == "bottom":
      return ((xmin + xmax - width) / 2, (xmin + xmax + width) / 2, ymax - inset - height, ymax - inset)
    elif position == "bottom-left":
      return (xmin + inset, xmin + inset + width, ymax - inset - height, ymax - inset)
    elif position == "left":
      return (xmin + inset, xmin + inset + width, (ymin + ymax - height) / 2, (ymin + ymax + height) / 2)
    elif position == "top-left":
      return (xmin + inset, xmin + inset + width, ymin + inset, ymin + inset + height)
    else:
      raise ValueError("Unrecognized corner")
  # Choose a cell from an MxN grid, with optional column/row spanning.
  if grid is not None:
    if len(grid) == 3: # Cell n (in left-to-right, top-to-bottom order) of an M x N grid
      M, N, n = grid
      i = n // N
      j = n % N
      rowspan, colspan = (1, 1)
    elif len(grid) == 4: # Cell i,j of an M x N grid
      M, N, i, j = grid
      rowspan, colspan = (1, 1)
    elif len(grid) == 6: # Cells [i, i+rowspan), [j, j+colspan) of an M x N grid
      M, N, i, rowspan, j, colspan = grid

    cell_width = (xmax - xmin) / N
    cell_height = (ymax - ymin) / M

    return (
      (j * cell_width) + gutter,
      ((j + colspan) * cell_width) - gutter,
      (i * cell_height) + gutter,
      ((i + rowspan) * cell_height) - gutter,
      )
  # If nothing else fits, consume the entire region
  return (xmin + gutter, xmax - gutter, ymin + gutter, ymax - gutter)

<commit_msg>Update layout.py<commit_after># Copyright 2014, Sandia Corporation. Under the terms of Contract
# DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains certain
# rights in this software.

"""Provides layout algorithms.
"""

from __future__ import division

import numbers
import toyplot.compatibility

def region(xmin, xmax, ymin, ymax, bounds=None, rect=None, corner=None, grid=None, gutter=40):
  def length(min, max, value):
    if isinstance(value, numbers.Number):
      return value
    if isinstance(value, toyplot.compatibility.string_type):
      value = value.strip()
      if value[-1] == "%":
        value = float(value[:-1]) * 0.01
        return ((1.0 - value) * min) + (value * max)
      else:
        return float(value)

  # Specify explicit bounds for the region
  if bounds is not None:
    if isinstance(bounds, tuple) and len(bounds) == 4:
      return (length(xmin, xmax, bounds[0]), length(xmin, xmax, bounds[1]), length(ymin, ymax, bounds[2]), length(ymin, ymax, bounds[3]))
    raise ValueError("Unrecognized bounds type")
  # Specify an explicit rectangle for the region
  if rect is not None:
    if isinstance(rect, tuple) and len(rect) == 4:
      x = length(xmin, xmax, rect[0])
      y = length(ymin, ymax, rect[1])
      width = length(xmin, xmax, rect[2])
      height = length(ymin, ymax, rect[3])
      return (x, x + width, y, y + height)
    raise ValueError("Unrecognized rect type")
  # Offset a rectangle from a corner
  if corner is not None:
    if isinstance(corner, tuple) and len(corner) == 4:
      position = corner[0]
      width = length(xmin, xmax, corner[1])
      height = length(ymin, ymax, corner[2])
      inset = float(corner[3])
    else:
      raise ValueError("Unrecognized corner type")
    if position == "top":
      return ((xmin + xmax - width) / 2, (xmin + xmax + width) / 2, ymin + inset, ymin + inset + height)
    elif position == "top-right":
      return (xmax - width - inset, xmax - inset, ymin + inset, ymin + inset + height)
    elif position == "right":
      return (xmax - width - inset, xmax - inset, (ymin + ymax - height) / 2, (ymin + ymax + height) / 2)
    elif position == "bottom-right":
      return (xmax - width - inset, xmax - inset, ymax - inset - height, ymax - inset)
    elif position == "bottom":
      return ((xmin + xmax - width) / 2, (xmin + xmax + width) / 2, ymax - inset - height, ymax - inset)
    elif position == "bottom-left":
      return (xmin + inset, xmin + inset + width, ymax - inset - height, ymax - inset)
    elif position == "left":
      return (xmin + inset, xmin + inset + width, (ymin + ymax - height) / 2, (ymin + ymax + height) / 2)
    elif position == "top-left":
      return (xmin + inset, xmin + inset + width, ymin + inset, ymin + inset + height)
    else:
      raise ValueError("Unrecognized corner")
  # Choose a cell from an MxN grid, with optional column/row spanning.
  if grid is not None:
    if len(grid) == 3: # Cell n (in left-to-right, top-to-bottom order) of an M x N grid
      M, N, n = grid
      i = n // N
      j = n % N
      rowspan, colspan = (1, 1)
    elif len(grid) == 4: # Cell i,j of an M x N grid
      M, N, i, j = grid
      rowspan, colspan = (1, 1)
    elif len(grid) == 6: # Cells [i, i+rowspan), [j, j+colspan) of an M x N grid
      M, N, i, rowspan, j, colspan = grid

    cell_width = (xmax - xmin) / N
    cell_height = (ymax - ymin) / M

    return (
      (j * cell_width) + gutter,
      ((j + colspan) * cell_width) - gutter,
      (i * cell_height) + gutter,
      ((i + rowspan) * cell_height) - gutter,
      )
  # If nothing else fits, consume the entire region
  return (xmin + gutter, xmax - gutter, ymin + gutter, ymax - gutter)

<|endoftext|>


<commit_before># Django settings for cavedbmanager project.

import os

DEBUG = True
TEMPLATE_DEBUG = DEBUG

ADMINS = (
    ('Your Name',   'user@domain.org'),
)

ALLOWED_HOSTS = ( 'localhost' )

ADMIN_SITE_HEADER = "My Cave Database"

MANAGERS = ADMINS

DATABASES = {
    'default': {
	'ENGINE': 'django.db.backends.postgresql_psycopg2',
	'NAME': 'cavedb',
	'HOST': '/var/run/postgresql/',
    }
}

TEST_RUNNER = 'django.test.runner.DiscoverRunner'

TIME_ZONE = 'America/New_York'
USE_TZ = False

LANGUAGE_CODE = 'en-us'
SITE_ID = 1

INSTALL_ROOT = os.path.abspath(os.path.dirname(__file__))
if os.sep != '/': INSTALL_ROOT = INSTALL_ROOT.replace(os.sep, '/')

# Specifies where the application is deployed
CONTEXT_PATH = '/'

MEDIA_ROOT = '/usr/local/cavedbmanager-data'
MEDIA_URL = CONTEXT_PATH + 'cavedb/'
STATIC_URL = '/media/'
STATIC_ROOT = 'media/'

SECRET_KEY = 'FIXME_CHANGE_THIS_SECRET_KEY'
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
    #'django.template.loaders.eggs.load_template_source',
)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
#    'django.middleware.security.SecurityMiddleware',
    'cavedb.middleware.ThreadLocals',
)

ROOT_URLCONF = 'cavedb.urls'

# FIXME - this came from a brand new run of `python manage.py startproject`
#TEMPLATES = [
#    {
#        'BACKEND': 'django.template.backends.django.DjangoTemplates',
#        'DIRS': [],
#        'APP_DIRS': True,
#        'OPTIONS': {
#            'context_processors': [
#                'django.template.context_processors.debug',
#                'django.template.context_processors.request',
#                'django.contrib.auth.context_processors.auth',
#                'django.contrib.messages.context_processors.messages',
#            ],
#        },
#    },
#]

TEMPLATE_DIRS = (
    INSTALL_ROOT + "/templates",
)

WSGI_APPLICATION = 'cavedb.wsgi.application'

INSTALLED_APPS = (
    'django.contrib.admin',
    'django.contrib.admindocs',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',

    'cavedb',
)

GIS_CONNECTION_TYPE = 'postgis'
GIS_CONNECTION = 'dbname=wvgis'

<commit_msg>fixed media root<commit_after># Django settings for cavedbmanager project.

import os

DEBUG = True
TEMPLATE_DEBUG = DEBUG

ADMINS = (
    ('Your Name',   'user@domain.org'),
)

ALLOWED_HOSTS = ( 'localhost' )

ADMIN_SITE_HEADER = "My Cave Database"

MANAGERS = ADMINS

DATABASES = {
    'default': {
	'ENGINE': 'django.db.backends.postgresql_psycopg2',
	'NAME': 'cavedb',
	'HOST': '/var/run/postgresql/',
    }
}

TEST_RUNNER = 'django.test.runner.DiscoverRunner'

TIME_ZONE = 'America/New_York'
USE_TZ = False

LANGUAGE_CODE = 'en-us'
SITE_ID = 1

INSTALL_ROOT = os.path.abspath(os.path.dirname(__file__))
if os.sep != '/': INSTALL_ROOT = INSTALL_ROOT.replace(os.sep, '/')

# Specifies where the application is deployed
CONTEXT_PATH = '/'

MEDIA_ROOT = '/usr/local/cavedbmanager-data/'
MEDIA_URL = CONTEXT_PATH + 'cavedb/'
STATIC_URL = '/media/'
STATIC_ROOT = 'media/'

SECRET_KEY = 'FIXME_CHANGE_THIS_SECRET_KEY'
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
    #'django.template.loaders.eggs.load_template_source',
)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
#    'django.middleware.security.SecurityMiddleware',
    'cavedb.middleware.ThreadLocals',
)

ROOT_URLCONF = 'cavedb.urls'

# FIXME - this came from a brand new run of `python manage.py startproject`
#TEMPLATES = [
#    {
#        'BACKEND': 'django.template.backends.django.DjangoTemplates',
#        'DIRS': [],
#        'APP_DIRS': True,
#        'OPTIONS': {
#            'context_processors': [
#                'django.template.context_processors.debug',
#                'django.template.context_processors.request',
#                'django.contrib.auth.context_processors.auth',
#                'django.contrib.messages.context_processors.messages',
#            ],
#        },
#    },
#]

TEMPLATE_DIRS = (
    INSTALL_ROOT + "/templates",
)

WSGI_APPLICATION = 'cavedb.wsgi.application'

INSTALLED_APPS = (
    'django.contrib.admin',
    'django.contrib.admindocs',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',

    'cavedb',
)

GIS_CONNECTION_TYPE = 'postgis'
GIS_CONNECTION = 'dbname=wvgis'

<|endoftext|>


<commit_before>import numpy as np
from cbamf.util import Tile, cdd, amin, amax

class SphereCollectionRealSpace(object):
    def __init__(self, pos, rad, shape, support_size=4, typ=None, pad=None):
        self.support_size = support_size
        self.pos = pos.astype('float')
        self.rad = rad.astype('float')
        self.N = rad.shape[0]

        if typ is None:
            self.typ = np.ones(self.N)
            if pad is not None and pad <= self.N:
                self.typ[-pad:] = 0
        else:
            self.typ = typ.astype('float')

        self.shape = shape
        self._setup()

    def _setup(self):
        z,y,x = Tile(self.shape).coords()
        self.rvecs = np.rollaxis(np.array(np.broadcast_arrays(z,y,x)), 0, 4)
        self.particles = np.zeros(self.shape)
        self._diff_field = np.zeros(self.shape)

    def _particle(self, pos, rad, zscale, sign=1, dodiff=False):
        p = np.round(pos)
        r = np.round(np.array([1.0/zscale,1,1])*np.ceil(rad)+self.support_size)

        tile = Tile(p-r, p+r, 0, self.shape)
        subr = self.rvecs[tile.slicer + (np.s_[:],)]
        rvec = (subr - pos)

        # apply the zscale and find the distances to make a ellipsoid
        # note: the appearance of PI in the last line is because of leastsq
        # fits to the correct Fourier version of the sphere, j_{3/2} / r^{3/2}
        # happened to fit right at pi -- what?!
        rvec[...,0] *= zscale
        rdist = np.sqrt((rvec**2).sum(axis=-1))

        t = sign/(1.0 + np.exp(5.0*(rdist - rad)))
        self.particles[tile.slicer] += t

        if dodiff:
            self._diff_field[tile.slicer] += t

    def _update_particle(self, n, p, r, t, zscale, dodiff=True):
        if self.typ[n] == 1:
            self._particle(self.pos[n], self.rad[n], zscale, -1, dodiff=dodiff)

        self.pos[n] = p
        self.rad[n] = r
        self.typ[n] = t

        if self.typ[n] == 1:
            self._particle(self.pos[n], self.rad[n], zscale, +1, dodiff=dodiff)

    def initialize(self, zscale):
        if len(self.pos.shape) != 2:
            raise AttributeError("Position array needs to be (-1,3) shaped, (z,y,x) order")

        self.particles = np.zeros(self.shape)
        for p0, r0, t0 in zip(self.pos, self.rad, self.typ):
            if t0 == 1:
                self._particle(p0, r0, zscale)

    def set_tile(self, tile):
        self.tile = tile

    def update(self, ns, pos, rad, typ, zscale, difference=True):
        for n, p, r, t in zip(ns, pos, rad, typ):
            self._update_particle(n, p, r, t, zscale, dodiff=difference)

    def get_field(self):
        return self.particles[self.tile.slicer]

    def get_diff_field(self):
        c = self._diff_field[self.tile.slicer].copy()
        self._diff_field[self.tile.slicer] *= 0
        return c

    def get_support_size(self, p0, r0, t0, p1, r1, t1, zscale):
        rsc = self.support_size

        zsc = np.array([1.0/zscale, 1, 1])
        r0, r1 = zsc*r0, zsc*r1

        off0 = r0 + rsc
        off1 = r1 + rsc

        if t0[0] == 1 and t1[0] == 1:
            pl = amin(p0-off0-1, p1-off1-1)
            pr = amax(p0+off0+1, p1+off1+1)
        if t0[0] != 1 and t1[0] == 1:
            pl = (p1-off1-1)
            pr = (p1+off1+1)
        if t0[0] == 1 and t1[0] != 1:
            pl = (p0-off0-1)
            pr = (p0+off0+1)
        if t0[0] != 1 and t1[0] != 1:
            c = np.array(self.shape)
            pl = c/2 - c/8
            pr = c/2 + c/8

        if len(pl.shape) > 1:
            pl = pl[0]
            pr = pr[0]
        return pl, pr

    def get_params(self):
        return np.hstack([self.pos.ravel(), self.rad])

    def get_params_pos(self):
        return self.pos.ravel()

    def get_params_rad(self):
        return self.rad

    def get_params_typ(self):
        return self.typ

    def __getstate__(self):
        odict = self.__dict__.copy()
        cdd(odict, ['rvecs', 'particles', '_diff_field'])
        return odict

    def __setstate__(self, idict):
        self.__dict__.update(idict)
        self._setup()

class Slab(object):
    def __init__(self, pos, shape, normal=(1,0,0), support_size=4, typ=None, pad=None):
        self.support_size = support_size

        self.pos = np.array(pos).astype('float')
        self.normal = np.array(normal).astype('float')
        self.normal /= np.sqrt(self.normal.dot(self.normal))

        self.shape = shape
        self._setup()

    def _setup(self):
        z,y,x = Tile(self.shape).coords()
        self.rvecs = np.rollaxis(np.array(np.broadcast_arrays(z,y,x)), 0, 4)
        self.image = np.zeros(self.shape)

    def _slab(self, pos, norm, sign=1):
        p = (self.rvecs - pos).dot(norm)
        t = sign/(1.0 + np.exp(np.pi*p))
        self.image += t

    def initialize(self):
        self.image = np.zeros(self.shape)
        self._slab(self.pos, self.normal)

    def set_tile(self, tile):
        self.tile = tile

    def update(self, pos, norm):
        self._slab(self.pos, self.normal, -1)
        self.pos = pos
        self.normal = norm / np.sqrt(norm.dot(norm))
        self._slab(self.pos, self.normal, +1)

    def get_field(self):
        return self.image[self.tile.slicer]

    def get_support_size(self, p=None):
        return pl, pr

    def get_params(self):
        return np.hstack([self.pos.ravel(), self.normal.ravel()])

    def __getstate__(self):
        odict = self.__dict__.copy()
        cdd(odict, ['rvecs', 'image'])
        return odict

    def __setstate__(self, idict):
        self.__dict__.update(idict)
        self._setup()
<commit_msg>switching platonic aliasing to lerp by default with options for other methods<commit_after>import numpy as np
from cbamf.util import Tile, cdd, amin, amax

class SphereCollectionRealSpace(object):
    def __init__(self, pos, rad, shape, support_size=4, typ=None, pad=None, method='lerp'):
        """
        method can be one of ['lerp', 'logistic', 'triangle']
        """
        self.support_size = support_size
        self.pos = pos.astype('float')
        self.rad = rad.astype('float')
        self.N = rad.shape[0]
        self.method = method

        if typ is None:
            self.typ = np.ones(self.N)
            if pad is not None and pad <= self.N:
                self.typ[-pad:] = 0
        else:
            self.typ = typ.astype('float')

        self.shape = shape
        self._setup()

    def _setup(self):
        z,y,x = Tile(self.shape).coords()
        self.rvecs = np.rollaxis(np.array(np.broadcast_arrays(z,y,x)), 0, 4)
        self.particles = np.zeros(self.shape)
        self._diff_field = np.zeros(self.shape)

    def _particle(self, pos, rad, zscale, sign=1, dodiff=False):
        p = np.round(pos)
        r = np.round(np.array([1.0/zscale,1,1])*np.ceil(rad)+self.support_size)

        tile = Tile(p-r, p+r, 0, self.shape)
        subr = self.rvecs[tile.slicer + (np.s_[:],)]
        rvec = (subr - pos)

        # apply the zscale and find the distances to make a ellipsoid
        rvec[...,0] *= zscale
        rdist = np.sqrt((rvec**2).sum(axis=-1))

        # keep backwards compatibility for save files with older logistic coefficient
        if not hasattr(self, 'method'):
            self.method = 'logistic'
            lalpha = 5.0
        else:
            lalpha = 6.5

        if self.method == 'lerp':
            # lerp the pixel values over a range very close to one pixel.  from
            # fits to the fourier-space sphere, we find slightly less than a pixel
            # as given by the value alpha
            alpha = 0.4539
            t = sign*(1-np.clip((rdist-(rad-alpha)) / (2*alpha), 0, 1))

        if self.method == 'logistic':
            # logistic interpolation of pixel edges, with a fix parameter of 6.5 (see above)
            t = sign/(1.0 + np.exp(lalpha*(rdist - rad)))

        if self.method == 'triangle':
            # triangle CDF interpolation of pixel edges
            alpha = 0.6618
            p0 = (rdist-rad+alpha)**2/(2*alpha**2)*(rad > rdist)*(rdist>rad-alpha)
            p1 = 1*(rdist>rad)-(rad+alpha-rdist)**2/(2*alpha**2)*(rad<rdist)*(rdist<rad+alpha)
            t = sign*(1-np.clip(p0+p1, 0, 1))

        self.particles[tile.slicer] += t

        if dodiff:
            self._diff_field[tile.slicer] += t

    def _update_particle(self, n, p, r, t, zscale, dodiff=True):
        if self.typ[n] == 1:
            self._particle(self.pos[n], self.rad[n], zscale, -1, dodiff=dodiff)

        self.pos[n] = p
        self.rad[n] = r
        self.typ[n] = t

        if self.typ[n] == 1:
            self._particle(self.pos[n], self.rad[n], zscale, +1, dodiff=dodiff)

    def initialize(self, zscale):
        if len(self.pos.shape) != 2:
            raise AttributeError("Position array needs to be (-1,3) shaped, (z,y,x) order")

        self.particles = np.zeros(self.shape)
        for p0, r0, t0 in zip(self.pos, self.rad, self.typ):
            if t0 == 1:
                self._particle(p0, r0, zscale)

    def set_tile(self, tile):
        self.tile = tile

    def update(self, ns, pos, rad, typ, zscale, difference=True):
        for n, p, r, t in zip(ns, pos, rad, typ):
            self._update_particle(n, p, r, t, zscale, dodiff=difference)

    def get_field(self):
        return self.particles[self.tile.slicer]

    def get_diff_field(self):
        c = self._diff_field[self.tile.slicer].copy()
        self._diff_field[self.tile.slicer] *= 0
        return c

    def get_support_size(self, p0, r0, t0, p1, r1, t1, zscale):
        rsc = self.support_size

        zsc = np.array([1.0/zscale, 1, 1])
        r0, r1 = zsc*r0, zsc*r1

        off0 = r0 + rsc
        off1 = r1 + rsc

        if t0[0] == 1 and t1[0] == 1:
            pl = amin(p0-off0-1, p1-off1-1)
            pr = amax(p0+off0+1, p1+off1+1)
        if t0[0] != 1 and t1[0] == 1:
            pl = (p1-off1-1)
            pr = (p1+off1+1)
        if t0[0] == 1 and t1[0] != 1:
            pl = (p0-off0-1)
            pr = (p0+off0+1)
        if t0[0] != 1 and t1[0] != 1:
            c = np.array(self.shape)
            pl = c/2 - c/8
            pr = c/2 + c/8

        if len(pl.shape) > 1:
            pl = pl[0]
            pr = pr[0]
        return pl, pr

    def get_params(self):
        return np.hstack([self.pos.ravel(), self.rad])

    def get_params_pos(self):
        return self.pos.ravel()

    def get_params_rad(self):
        return self.rad

    def get_params_typ(self):
        return self.typ

    def __getstate__(self):
        odict = self.__dict__.copy()
        cdd(odict, ['rvecs', 'particles', '_diff_field'])
        return odict

    def __setstate__(self, idict):
        self.__dict__.update(idict)
        self._setup()

class Slab(object):
    def __init__(self, pos, shape, normal=(1,0,0), support_size=4, typ=None, pad=None):
        self.support_size = support_size

        self.pos = np.array(pos).astype('float')
        self.normal = np.array(normal).astype('float')
        self.normal /= np.sqrt(self.normal.dot(self.normal))

        self.shape = shape
        self._setup()

    def _setup(self):
        z,y,x = Tile(self.shape).coords()
        self.rvecs = np.rollaxis(np.array(np.broadcast_arrays(z,y,x)), 0, 4)
        self.image = np.zeros(self.shape)

    def _slab(self, pos, norm, sign=1):
        p = (self.rvecs - pos).dot(norm)
        t = sign/(1.0 + np.exp(np.pi*p))
        self.image += t

    def initialize(self):
        self.image = np.zeros(self.shape)
        self._slab(self.pos, self.normal)

    def set_tile(self, tile):
        self.tile = tile

    def update(self, pos, norm):
        self._slab(self.pos, self.normal, -1)
        self.pos = pos
        self.normal = norm / np.sqrt(norm.dot(norm))
        self._slab(self.pos, self.normal, +1)

    def get_field(self):
        return self.image[self.tile.slicer]

    def get_support_size(self, p=None):
        return pl, pr

    def get_params(self):
        return np.hstack([self.pos.ravel(), self.normal.ravel()])

    def __getstate__(self):
        odict = self.__dict__.copy()
        cdd(odict, ['rvecs', 'image'])
        return odict

    def __setstate__(self, idict):
        self.__dict__.update(idict)
        self._setup()
<|endoftext|>