<commit_before>import logging
from typing import Optional

import matplotlib
import matplotlib.cm
import matplotlib.colors
import numpy as np
from PyQt5 import QtWidgets
from matplotlib.axes import Axes
from matplotlib.backends.backend_qt5agg import NavigationToolbar2QT, FigureCanvasQTAgg
from matplotlib.colorbar import Colorbar
from matplotlib.figure import Figure
from matplotlib.image import AxesImage
from matplotlib.lines import Line2D
from sastool.classes2 import Exposure

from .plotimage_ui import Ui_Form

logger=logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)

maskcmap = matplotlib.colors.ListedColormap([(0.0, 0.0, 0.0, 0.0), (1.0, 1.0, 1.0, 0.7)])

class PlotImage(QtWidgets.QWidget, Ui_Form):
    figure: Figure = None
    canvas: FigureCanvasQTAgg = None
    figToolbar: NavigationToolbar2QT = None
    axes: Axes = None
    mask: Optional[np.ndarray] = None
    matrix: Optional[np.ndarray] = None
    beamx: Optional[float] = None
    beamy: Optional[float] = None
    pixelsize: Optional[float] = None
    distance: Optional[float] = None
    wavelength: Optional[float] = None

    _cmapaxis: Optional[Colorbar] = None
    _imghandle: Optional[AxesImage] = None
    _maskhandle: Optional[AxesImage] = None
    _xcrosshairhandle: Optional[Line2D] = None
    _ycrosshairhandle: Optional[Line2D] = None

    _normalizationdict = {
        'linear': matplotlib.colors.Normalize(),
        'log10': matplotlib.colors.LogNorm(),
        'square': matplotlib.colors.PowerNorm(2),
        'sqrt': matplotlib.colors.PowerNorm(0.5),
    }

    def __init__(self, parent: QtWidgets.QWidget = None):
        super().__init__(parent)
        self.mask = None
        self.matrix = None
        self.setupUi(self)

    def setupUi(self, Form):
        super().setupUi(Form)
        self.figure = Figure(figsize=(6, 4), constrained_layout=True)
        self.canvas = FigureCanvasQTAgg(self.figure)
        self.figToolbar = NavigationToolbar2QT(self.canvas, self)
        self.layout().addWidget(self.figToolbar)
        self.layout().addWidget(self.canvas, 1)
        self.canvas.mpl_connect('resize_event', self.onCanvasResize)
        self.canvas.setSizePolicy(
            QtWidgets.QSizePolicy.MinimumExpanding,
            QtWidgets.QSizePolicy.MinimumExpanding)
        gs = self.figure.add_gridspec(1, 1)
        self.axes = self.figure.add_subplot(gs[:, :])
        self.axes.set_facecolor('black')
        self.canvas.draw_idle()
        self.paletteComboBox.addItems(sorted(matplotlib.cm.cmap_d))
        self.paletteComboBox.setCurrentIndex(self.paletteComboBox.findText(matplotlib.rcParams['image.cmap']))
        self.paletteComboBox.currentIndexChanged.connect(self.replot)
        self.colourScaleComboBox.currentIndexChanged.connect(self.replot)
        self.axesComboBox.currentIndexChanged.connect(self.axisScaleChanged)
        self.showColourBarToolButton.toggled.connect(self.showColourBar)
        self.showBeamToolButton.toggled.connect(self.showBeam)
        self.showMaskToolButton.toggled.connect(self.showMask)
        self.equalAspectToolButton.toggled.connect(self.changeAspect)

    def axisScaleChanged(self):
        self.replot()
        (left, right, bottom, top), center = self._get_extent()
        self.axes.axis([left, right, bottom, top])
        self.canvas.draw_idle()
        self.figToolbar.update()

    def showColourBar(self, showit: bool):
        if self._cmapaxis is not None:
            self._cmapaxis.ax.set_visible(showit)
            self.canvas.draw_idle()

    def showMask(self, showit: bool):
        if self._maskhandle is not None:
            self._maskhandle.set_visible(showit)
            self.canvas.draw_idle()

    def showBeam(self, showit: bool):
        if self._xcrosshairhandle is not None:
            self._xcrosshairhandle.set_visible(showit)
            self.canvas.draw_idle()
        if self._ycrosshairhandle is not None:
            self._ycrosshairhandle.set_visible(showit)
            self.canvas.draw_idle()

    def changeAspect(self, equalaspect: bool):
        self.axes.set_aspect('equal' if equalaspect else 'auto')
        self.canvas.draw_idle()

    def onCanvasResize(self, event):
        pass

    def _get_extent(self):
        # extent: left, right, bottom, top
        if self.axesComboBox.currentText() == 'abs. pixel':
            extent = (
                -0.5,
                self.matrix.shape[1] - 0.5,
                self.matrix.shape[0] - 0.5,
                -0.5)
            center = (self.beamx, self.beamy)
        elif self.axesComboBox.currentText() == 'rel. pixel':
            extent = (-0.5 - self.beamx,
                      self.matrix.shape[1] - self.beamx - 0.5,
                      self.matrix.shape[0] - self.beamy - 0.5,
                      -0.5 - self.beamy)
            center = (0.0, 0.0)
        elif self.axesComboBox.currentText() == 'detector radius':
            extent = ((-0.5 - self.beamx) * self.pixelsize,
                      (self.matrix.shape[1] - self.beamx - 0.5) * self.pixelsize,
                      (self.matrix.shape[0] - self.beamy - 0.5) * self.pixelsize,
                      (-0.5 - self.beamy) * self.pixelsize)
            center = (0.0, 0.0)
        elif self.axesComboBox.currentText() == 'twotheta':
            extent = (
                180 / np.pi * np.arctan((-0.5 - self.beamx) * self.pixelsize / self.distance),
                180 / np.pi * np.arctan((self.matrix.shape[1] - self.beamx - 0.5) * self.pixelsize / self.distance),
                180 / np.pi * np.arctan((self.matrix.shape[0] - self.beamy - 0.5) * self.pixelsize / self.distance),
                180 / np.pi * np.arctan((-0.5 - self.beamy) * self.pixelsize / self.distance),
            )
            center = (0.0, 0.0)
        elif self.axesComboBox.currentText() == 'q':
            extent = (
                4 * np.pi * np.sin(
                    0.5 * np.arctan(
                        (-0.5 - self.beamx) * self.pixelsize / self.distance)) / self.wavelength,
                4 * np.pi * np.sin(
                    0.5 * np.arctan(
                        (self.matrix.shape[1] - self.beamx - 0.5) * self.pixelsize / self.distance)) / self.wavelength,
                4 * np.pi * np.sin(
                    0.5 * np.arctan(
                        (self.matrix.shape[0] - self.beamy - 0.5) * self.pixelsize / self.distance)) / self.wavelength,
                4 * np.pi * np.sin(
                    0.5 * np.arctan(
                        (-0.5 - self.beamy) * self.pixelsize / self.distance)) / self.wavelength,
            )
            center = (0.0, 0.0)
        else:
            assert False
        return extent, center

    def replot(self):
        extent, center = self._get_extent()
        # now plot the matrix
        if self._imghandle is None:
            self._imghandle = self.axes.imshow(
                self.matrix,
                cmap=self.paletteComboBox.currentText(),
                norm=self._normalizationdict[self.colourScaleComboBox.currentText()],
                aspect='equal' if self.equalAspectToolButton.isChecked() else 'auto',
                interpolation='nearest',
                alpha=1.0,
                origin='upper',
                extent=extent,
            )
        else:
            self._imghandle.set_data(self.matrix)
            self._imghandle.set_cmap(self.paletteComboBox.currentText())
            self._imghandle.set_norm(self._normalizationdict[self.colourScaleComboBox.currentText()])
            self._imghandle.set_extent(extent)
            self._imghandle.autoscale()
            self._imghandle.changed()
        # color bar
        if self._cmapaxis is None:
            self._cmapaxis = self.figure.colorbar(
                self._imghandle,
                cax=self._cmapaxis.ax if self._cmapaxis is not None else None,
                ax=None if self._cmapaxis is not None else self.axes)
        else:
            logger.debug('Updating cmap')
        self._cmapaxis.ax.set_visible(self.showColourBarToolButton.isChecked())

        # now plot the mask
        logger.debug(f'Mask: {(self.mask != 0).sum()} nonzero, {(self.mask == 0).sum()} zero pixels')
        if self._maskhandle is None:
            self._maskhandle = self.axes.imshow(
                self.mask,
                cmap=maskcmap,
                aspect='equal' if self.equalAspectToolButton.isChecked() else 'auto',
                interpolation='nearest',
                # alpha=0.7,  # no need to specify: the color map does the job.
                origin='upper',
                extent=extent,
            )
        else:
            self._maskhandle.set_data(self.mask)
            self._maskhandle.set_extent(extent)
            self._maskhandle.changed()
        self._maskhandle.set_visible(self.showMaskToolButton.isChecked())

        # plot the crosshair
        if self._xcrosshairhandle is None:
            self._xcrosshairhandle = self.axes.axhline(center[1], color='w', ls='--', lw=0.5)
        else:
            self._xcrosshairhandle.set_ydata([center[1], center[1]])
        if self._ycrosshairhandle is None:
            self._ycrosshairhandle = self.axes.axvline(center[0], color='w', ls='--', lw=0.5)
        else:
            self._ycrosshairhandle.set_xdata([center[0], center[0]])
        self._xcrosshairhandle.set_visible(self.showBeamToolButton.isChecked())
        self._ycrosshairhandle.set_visible(self.showBeamToolButton.isChecked())

        # add axis labels
        if self.axesComboBox.currentText() == 'abs. pixel':
            self.axes.set_xlabel('Pixel coordinate')
            self.axes.set_ylabel('Pixel coordinate')
        elif self.axesComboBox.currentText() == 'rel. pixel':
            self.axes.set_xlabel('Distance from the center (pixel)')
            self.axes.set_ylabel('Distance from the center (pixel)')
        elif self.axesComboBox.currentText() == 'detector radius':
            self.axes.set_xlabel('Distance from the center (mm)')
            self.axes.set_ylabel('Distance from the center (mm)')
        elif self.axesComboBox.currentText() == 'twotheta':
            self.axes.set_xlabel(r'Scattering angle $2\theta$ ($^\circ$)')
            self.axes.set_ylabel(r'Scattering angle $2\theta$ ($^\circ$)')
        elif self.axesComboBox.currentText() == 'q':
            self.axes.set_xlabel('$q$ (nm$^{-1}$)')
            self.axes.set_ylabel('$q$ (nm$^{-1}$)')
        else:
            assert False
        self.canvas.draw_idle()

    def setExposure(self, exposure: Exposure):
        self.matrix = exposure.intensity
        self.mask = exposure.mask == 0
        self.wavelength = float(exposure.header.wavelength)
        self.pixelsize = float(exposure.header.pixelsizex)
        self.beamx = float(exposure.header.beamcenterx)
        self.beamy = float(exposure.header.beamcentery)
        self.distance = float(exposure.header.distance)
        self.replot()
<commit_msg>Bug fix in PlotImage<commit_after>import logging
from typing import Optional

import matplotlib
import matplotlib.cm
import matplotlib.colors
import numpy as np
from PyQt5 import QtWidgets
from matplotlib.axes import Axes
from matplotlib.backends.backend_qt5agg import NavigationToolbar2QT, FigureCanvasQTAgg
from matplotlib.colorbar import Colorbar
from matplotlib.figure import Figure
from matplotlib.image import AxesImage
from matplotlib.lines import Line2D

from .plotimage_ui import Ui_Form
from ...core2.dataclasses import Exposure

logger=logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)

maskcmap = matplotlib.colors.ListedColormap([(0.0, 0.0, 0.0, 0.0), (1.0, 1.0, 1.0, 0.7)])

class PlotImage(QtWidgets.QWidget, Ui_Form):
    figure: Figure = None
    canvas: FigureCanvasQTAgg = None
    figToolbar: NavigationToolbar2QT = None
    axes: Axes = None
    mask: Optional[np.ndarray] = None
    matrix: Optional[np.ndarray] = None
    beamx: Optional[float] = None
    beamy: Optional[float] = None
    pixelsize: Optional[float] = None
    distance: Optional[float] = None
    wavelength: Optional[float] = None

    _cmapaxis: Optional[Colorbar] = None
    _imghandle: Optional[AxesImage] = None
    _maskhandle: Optional[AxesImage] = None
    _xcrosshairhandle: Optional[Line2D] = None
    _ycrosshairhandle: Optional[Line2D] = None

    _normalizationdict = {
        'linear': matplotlib.colors.Normalize(),
        'log10': matplotlib.colors.LogNorm(),
        'square': matplotlib.colors.PowerNorm(2),
        'sqrt': matplotlib.colors.PowerNorm(0.5),
    }

    def __init__(self, parent: QtWidgets.QWidget = None):
        super().__init__(parent)
        self.mask = None
        self.matrix = None
        self.setupUi(self)

    def setupUi(self, Form):
        super().setupUi(Form)
        self.figure = Figure(figsize=(6, 4), constrained_layout=True)
        self.canvas = FigureCanvasQTAgg(self.figure)
        self.figToolbar = NavigationToolbar2QT(self.canvas, self)
        self.layout().addWidget(self.figToolbar)
        self.layout().addWidget(self.canvas, 1)
        self.canvas.mpl_connect('resize_event', self.onCanvasResize)
        self.canvas.setSizePolicy(
            QtWidgets.QSizePolicy.MinimumExpanding,
            QtWidgets.QSizePolicy.MinimumExpanding)
        gs = self.figure.add_gridspec(1, 1)
        self.axes = self.figure.add_subplot(gs[:, :])
        self.axes.set_facecolor('black')
        self.canvas.draw_idle()
        self.paletteComboBox.addItems(sorted(matplotlib.cm.cmap_d))
        self.paletteComboBox.setCurrentIndex(self.paletteComboBox.findText(matplotlib.rcParams['image.cmap']))
        self.paletteComboBox.currentIndexChanged.connect(self.replot)
        self.colourScaleComboBox.currentIndexChanged.connect(self.replot)
        self.axesComboBox.currentIndexChanged.connect(self.axisScaleChanged)
        self.showColourBarToolButton.toggled.connect(self.showColourBar)
        self.showBeamToolButton.toggled.connect(self.showBeam)
        self.showMaskToolButton.toggled.connect(self.showMask)
        self.equalAspectToolButton.toggled.connect(self.changeAspect)

    def axisScaleChanged(self):
        self.replot()
        (left, right, bottom, top), center = self._get_extent()
        self.axes.axis([left, right, bottom, top])
        self.canvas.draw_idle()
        self.figToolbar.update()

    def showColourBar(self, showit: bool):
        if self._cmapaxis is not None:
            self._cmapaxis.ax.set_visible(showit)
            self.canvas.draw_idle()

    def showMask(self, showit: bool):
        if self._maskhandle is not None:
            self._maskhandle.set_visible(showit)
            self.canvas.draw_idle()

    def showBeam(self, showit: bool):
        if self._xcrosshairhandle is not None:
            self._xcrosshairhandle.set_visible(showit)
            self.canvas.draw_idle()
        if self._ycrosshairhandle is not None:
            self._ycrosshairhandle.set_visible(showit)
            self.canvas.draw_idle()

    def changeAspect(self, equalaspect: bool):
        self.axes.set_aspect('equal' if equalaspect else 'auto')
        self.canvas.draw_idle()

    def onCanvasResize(self, event):
        pass

    def _get_extent(self):
        # extent: left, right, bottom, top
        if self.axesComboBox.currentText() == 'abs. pixel':
            extent = (
                -0.5,
                self.matrix.shape[1] - 0.5,
                self.matrix.shape[0] - 0.5,
                -0.5)
            center = (self.beamx, self.beamy)
        elif self.axesComboBox.currentText() == 'rel. pixel':
            extent = (-0.5 - self.beamx,
                      self.matrix.shape[1] - self.beamx - 0.5,
                      self.matrix.shape[0] - self.beamy - 0.5,
                      -0.5 - self.beamy)
            center = (0.0, 0.0)
        elif self.axesComboBox.currentText() == 'detector radius':
            extent = ((-0.5 - self.beamx) * self.pixelsize,
                      (self.matrix.shape[1] - self.beamx - 0.5) * self.pixelsize,
                      (self.matrix.shape[0] - self.beamy - 0.5) * self.pixelsize,
                      (-0.5 - self.beamy) * self.pixelsize)
            center = (0.0, 0.0)
        elif self.axesComboBox.currentText() == 'twotheta':
            extent = (
                180 / np.pi * np.arctan((-0.5 - self.beamx) * self.pixelsize / self.distance),
                180 / np.pi * np.arctan((self.matrix.shape[1] - self.beamx - 0.5) * self.pixelsize / self.distance),
                180 / np.pi * np.arctan((self.matrix.shape[0] - self.beamy - 0.5) * self.pixelsize / self.distance),
                180 / np.pi * np.arctan((-0.5 - self.beamy) * self.pixelsize / self.distance),
            )
            center = (0.0, 0.0)
        elif self.axesComboBox.currentText() == 'q':
            extent = (
                4 * np.pi * np.sin(
                    0.5 * np.arctan(
                        (-0.5 - self.beamx) * self.pixelsize / self.distance)) / self.wavelength,
                4 * np.pi * np.sin(
                    0.5 * np.arctan(
                        (self.matrix.shape[1] - self.beamx - 0.5) * self.pixelsize / self.distance)) / self.wavelength,
                4 * np.pi * np.sin(
                    0.5 * np.arctan(
                        (self.matrix.shape[0] - self.beamy - 0.5) * self.pixelsize / self.distance)) / self.wavelength,
                4 * np.pi * np.sin(
                    0.5 * np.arctan(
                        (-0.5 - self.beamy) * self.pixelsize / self.distance)) / self.wavelength,
            )
            center = (0.0, 0.0)
        else:
            assert False
        return extent, center

    def replot(self):
        extent, center = self._get_extent()
        # now plot the matrix
        if self._imghandle is None:
            self._imghandle = self.axes.imshow(
                self.matrix,
                cmap=self.paletteComboBox.currentText(),
                norm=self._normalizationdict[self.colourScaleComboBox.currentText()],
                aspect='equal' if self.equalAspectToolButton.isChecked() else 'auto',
                interpolation='nearest',
                alpha=1.0,
                origin='upper',
                extent=extent,
                picker=True,
            )
        else:
            self._imghandle.set_data(self.matrix)
            self._imghandle.set_cmap(self.paletteComboBox.currentText())
            self._imghandle.set_norm(self._normalizationdict[self.colourScaleComboBox.currentText()])
            self._imghandle.set_extent(extent)
            self._imghandle.autoscale()
            self._imghandle.changed()
        # color bar
        if self._cmapaxis is None:
            self._cmapaxis = self.figure.colorbar(
                self._imghandle,
                cax=self._cmapaxis.ax if self._cmapaxis is not None else None,
                ax=None if self._cmapaxis is not None else self.axes)
        else:
            logger.debug('Updating cmap')
        self._cmapaxis.ax.set_visible(self.showColourBarToolButton.isChecked())

        # now plot the mask
        logger.debug(f'Mask: {(self.mask != 0).sum()} nonzero, {(self.mask == 0).sum()} zero pixels')
        if self._maskhandle is None:
            self._maskhandle = self.axes.imshow(
                self.mask,
                cmap=maskcmap,
                aspect='equal' if self.equalAspectToolButton.isChecked() else 'auto',
                interpolation='nearest',
                # alpha=0.7,  # no need to specify: the color map does the job.
                origin='upper',
                extent=extent,
            )
        else:
            self._maskhandle.set_data(self.mask)
            self._maskhandle.set_extent(extent)
            self._maskhandle.changed()
        self._maskhandle.set_visible(self.showMaskToolButton.isChecked())

        # plot the crosshair
        if self._xcrosshairhandle is None:
            self._xcrosshairhandle = self.axes.axhline(center[1], color='w', ls='--', lw=0.5)
        else:
            self._xcrosshairhandle.set_ydata([center[1], center[1]])
        if self._ycrosshairhandle is None:
            self._ycrosshairhandle = self.axes.axvline(center[0], color='w', ls='--', lw=0.5)
        else:
            self._ycrosshairhandle.set_xdata([center[0], center[0]])
        self._xcrosshairhandle.set_visible(self.showBeamToolButton.isChecked())
        self._ycrosshairhandle.set_visible(self.showBeamToolButton.isChecked())

        # add axis labels
        if self.axesComboBox.currentText() == 'abs. pixel':
            self.axes.set_xlabel('Pixel coordinate')
            self.axes.set_ylabel('Pixel coordinate')
        elif self.axesComboBox.currentText() == 'rel. pixel':
            self.axes.set_xlabel('Distance from the center (pixel)')
            self.axes.set_ylabel('Distance from the center (pixel)')
        elif self.axesComboBox.currentText() == 'detector radius':
            self.axes.set_xlabel('Distance from the center (mm)')
            self.axes.set_ylabel('Distance from the center (mm)')
        elif self.axesComboBox.currentText() == 'twotheta':
            self.axes.set_xlabel(r'Scattering angle $2\theta$ ($^\circ$)')
            self.axes.set_ylabel(r'Scattering angle $2\theta$ ($^\circ$)')
        elif self.axesComboBox.currentText() == 'q':
            self.axes.set_xlabel('$q$ (nm$^{-1}$)')
            self.axes.set_ylabel('$q$ (nm$^{-1}$)')
        else:
            assert False
        self.canvas.draw_idle()

    def setExposure(self, exposure: Exposure):
        self.matrix = exposure.intensity
        self.mask = exposure.mask == 0
        self.wavelength = float(exposure.header.wavelength[0])
        self.pixelsize = float(exposure.header.pixelsize[0])
        self.beamx = float(exposure.header.beamposcol[0])
        self.beamy = float(exposure.header.beamposrow[0])
        self.distance = float(exposure.header.distance[0])
        self.replot()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
Many aspects of the salt payload need to be managed, from the return of
encrypted keys to general payload dynamics and packaging, these happen
in here
'''

# Import python libs
#import sys  # Use of sys is commented out below
import logging

# Import salt libs
import salt.log
import salt.crypt
from salt.exceptions import SaltReqTimeoutError

# Import third party libs
try:
    import zmq
except ImportError:
    # No need for zeromq in local mode
    pass

log = logging.getLogger(__name__)

try:
    # Attempt to import msgpack
    import msgpack
    # There is a serialization issue on ARM and potentially other platforms
    # for some msgpack bindings, check for it
    if msgpack.loads(msgpack.dumps([1, 2, 3]), use_list=True) is None:
        raise ImportError
except ImportError:
    # Fall back to msgpack_pure
    try:
        import msgpack_pure as msgpack
    except ImportError:
        # TODO: Come up with a sane way to get a configured logfile
        #       and write to the logfile when this error is hit also
        LOG_FORMAT = '[%(levelname)-8s] %(message)s'
        salt.log.setup_console_logger(log_format=LOG_FORMAT)
        log.fatal('Unable to import msgpack or msgpack_pure python modules')
        # Don't exit if msgpack is not available, this is to make local mode
        # work without msgpack
        #sys.exit(salt.exitcodes.EX_GENERIC)

def package(payload):
    '''
    This method for now just wraps msgpack.dumps, but it is here so that
    we can make the serialization a custom option in the future with ease.
    '''
    return msgpack.dumps(payload)

def unpackage(package_):
    '''
    Unpackages a payload
    '''
    return msgpack.loads(package_, use_list=True)

def format_payload(enc, **kwargs):
    '''
    Pass in the required arguments for a payload, the enc type and the cmd,
    then a list of keyword args to generate the body of the load dict.
    '''
    payload = {'enc': enc}
    load = {}
    for key in kwargs:
        load[key] = kwargs[key]
    payload['load'] = load
    return package(payload)

class Serial(object):
    '''
    Create a serialization object, this object manages all message
    serialization in Salt
    '''
    def __init__(self, opts):
        if isinstance(opts, dict):
            self.serial = opts.get('serial', 'msgpack')
        elif isinstance(opts, str):
            self.serial = opts
        else:
            self.serial = 'msgpack'

    def loads(self, msg):
        '''
        Run the correct loads serialization format
        '''
        return msgpack.loads(msg, use_list=True)

    def load(self, fn_):
        '''
        Run the correct serialization to load a file
        '''
        data = fn_.read()
        fn_.close()
        return self.loads(data)

    def dumps(self, msg):
        '''
        Run the correct dumps serialization format
        '''
        try:
            return msgpack.dumps(msg)
        except TypeError:
            if msgpack.version >= (0, 2, 0):
                # Should support OrderedDict serialization, so, let's
                # raise the exception
                raise

            # msgpack is < 0.2.0, let's make its life easier
            # Since OrderedDict is identified as a dictionary, we can't
            # make use of msgpack custom types, we will need to convert by
            # hand.
            # This means iterating through all elements of a dictionary or
            # list/tuple
            def odict_encoder(obj):
                if isinstance(obj, dict):
                    for key, value in obj.copy().iteritems():
                        obj[key] = odict_encoder(value)
                    return dict(obj)
                elif isinstance(obj, (list, tuple)):
                    obj = list(obj)
                    for idx, entry in enumerate(obj):
                        obj[idx] = odict_encoder(entry)
                    return obj
                return obj
            return msgpack.dumps(odict_encoder(msg))

    def dump(self, msg, fn_):
        '''
        Serialize the correct data into the named file object
        '''
        fn_.write(self.dumps(msg))
        fn_.close()

class SREQ(object):
    '''
    Create a generic interface to wrap salt zeromq req calls.
    '''
    def __init__(self, master, id_='', serial='msgpack', linger=0):
        self.master = master
        self.id_ = id_
        self.serial = Serial(serial)
        self.linger = linger
        self.context = zmq.Context()
        self.poller = zmq.Poller()

    @property
    def socket(self):
        '''
        Lazily create the socket.
        '''
        if not hasattr(self, '_socket'):
            # create a new one
            self._socket = self.context.socket(zmq.REQ)
            if hasattr(zmq, 'RECONNECT_IVL_MAX'):
                self._socket.setsockopt(
                    zmq.RECONNECT_IVL_MAX, 5000
                )

            if self.master.startswith('tcp://[') and hasattr(zmq, 'IPV4ONLY'):
                # IPv6 sockets work for both IPv6 and IPv4 addresses
                self._socket.setsockopt(zmq.IPV4ONLY, 0)
            self._socket.linger = self.linger
            if self.id_:
                self._socket.setsockopt(zmq.IDENTITY, self.id_)
            self._socket.connect(self.master)
        return self._socket

    def clear_socket(self):
        '''
        delete socket if you have it
        '''
        if hasattr(self, '_socket'):
            if isinstance(self.poller.sockets, dict):
                for socket in self.poller.sockets.keys():
                    self.poller.unregister(socket)
            else:
                for socket in self.poller.sockets:
                    self.poller.unregister(socket[0])
            del self._socket

    def send(self, enc, load, tries=1, timeout=60):
        '''
        Takes two arguments, the encryption type and the base payload
        '''
        payload = {'enc': enc}
        payload['load'] = load
        pkg = self.serial.dumps(payload)
        self.socket.send(pkg)
        self.poller.register(self.socket, zmq.POLLIN)
        tried = 0
        while True:
            polled = self.poller.poll(timeout * 1000)
            tried += 1
            if polled:
                break
            if tries > 1:
                log.info('SaltReqTimeoutError: after {0} seconds. (Try {1} of {2})'.format(
                  timeout, tried, tries))
            if tried >= tries:
                self.clear_socket()
                raise SaltReqTimeoutError(
                    'SaltReqTimeoutError: after {0} seconds, ran {1} tries'.format(timeout * tried, tried)
                )
        return self.serial.loads(self.socket.recv())

    def send_auto(self, payload, tries=1, timeout=60):
        '''
        Detect the encryption type based on the payload
        '''
        enc = payload.get('enc', 'clear')
        load = payload.get('load', {})
        return self.send(enc, load, tries, timeout)

    def destroy(self):
        if isinstance(self.poller.sockets, dict):
            for socket in self.poller.sockets.keys():
                if socket.closed is False:
                    socket.setsockopt(zmq.LINGER, 1)
                    socket.close()
                self.poller.unregister(socket)
        else:
            for socket in self.poller.sockets:
                if socket[0].closed is False:
                    socket[0].setsockopt(zmq.LINGER, 1)
                    socket[0].close()
                self.poller.unregister(socket[0])
        if self.socket.closed is False:
            self.socket.setsockopt(zmq.LINGER, 1)
            self.socket.close()
        if self.context.closed is False:
            self.context.term()

    def __del__(self):
        self.destroy()
<commit_msg>Catch exceptions when deserializing<commit_after># -*- coding: utf-8 -*-
'''
Many aspects of the salt payload need to be managed, from the return of
encrypted keys to general payload dynamics and packaging, these happen
in here
'''

# Import python libs
#import sys  # Use of sys is commented out below
import logging

# Import salt libs
import salt.log
import salt.crypt
from salt.exceptions import SaltReqTimeoutError

# Import third party libs
try:
    import zmq
except ImportError:
    # No need for zeromq in local mode
    pass

log = logging.getLogger(__name__)

try:
    # Attempt to import msgpack
    import msgpack
    # There is a serialization issue on ARM and potentially other platforms
    # for some msgpack bindings, check for it
    if msgpack.loads(msgpack.dumps([1, 2, 3]), use_list=True) is None:
        raise ImportError
except ImportError:
    # Fall back to msgpack_pure
    try:
        import msgpack_pure as msgpack
    except ImportError:
        # TODO: Come up with a sane way to get a configured logfile
        #       and write to the logfile when this error is hit also
        LOG_FORMAT = '[%(levelname)-8s] %(message)s'
        salt.log.setup_console_logger(log_format=LOG_FORMAT)
        log.fatal('Unable to import msgpack or msgpack_pure python modules')
        # Don't exit if msgpack is not available, this is to make local mode
        # work without msgpack
        #sys.exit(salt.exitcodes.EX_GENERIC)

def package(payload):
    '''
    This method for now just wraps msgpack.dumps, but it is here so that
    we can make the serialization a custom option in the future with ease.
    '''
    return msgpack.dumps(payload)

def unpackage(package_):
    '''
    Unpackages a payload
    '''
    return msgpack.loads(package_, use_list=True)

def format_payload(enc, **kwargs):
    '''
    Pass in the required arguments for a payload, the enc type and the cmd,
    then a list of keyword args to generate the body of the load dict.
    '''
    payload = {'enc': enc}
    load = {}
    for key in kwargs:
        load[key] = kwargs[key]
    payload['load'] = load
    return package(payload)

class Serial(object):
    '''
    Create a serialization object, this object manages all message
    serialization in Salt
    '''
    def __init__(self, opts):
        if isinstance(opts, dict):
            self.serial = opts.get('serial', 'msgpack')
        elif isinstance(opts, str):
            self.serial = opts
        else:
            self.serial = 'msgpack'

    def loads(self, msg):
        '''
        Run the correct loads serialization format
        '''
        try:
            return msgpack.loads(msg, use_list=True)
        except Exception as exc:
            log.critical('Could not deserialize msgpack message: {0}'
                         'In an attempt to keep Salt running, returning an empty dict.'
                         'This often happens when trying to read a file not in binary mode.'
                         'Please open an issue and include the following error: {1}'.format(msg, exc))
            return {}

    def load(self, fn_):
        '''
        Run the correct serialization to load a file
        '''
        data = fn_.read()
        fn_.close()
        return self.loads(data)

    def dumps(self, msg):
        '''
        Run the correct dumps serialization format
        '''
        try:
            return msgpack.dumps(msg)
        except TypeError:
            if msgpack.version >= (0, 2, 0):
                # Should support OrderedDict serialization, so, let's
                # raise the exception
                raise

            # msgpack is < 0.2.0, let's make its life easier
            # Since OrderedDict is identified as a dictionary, we can't
            # make use of msgpack custom types, we will need to convert by
            # hand.
            # This means iterating through all elements of a dictionary or
            # list/tuple
            def odict_encoder(obj):
                if isinstance(obj, dict):
                    for key, value in obj.copy().iteritems():
                        obj[key] = odict_encoder(value)
                    return dict(obj)
                elif isinstance(obj, (list, tuple)):
                    obj = list(obj)
                    for idx, entry in enumerate(obj):
                        obj[idx] = odict_encoder(entry)
                    return obj
                return obj
            return msgpack.dumps(odict_encoder(msg))

    def dump(self, msg, fn_):
        '''
        Serialize the correct data into the named file object
        '''
        fn_.write(self.dumps(msg))
        fn_.close()

class SREQ(object):
    '''
    Create a generic interface to wrap salt zeromq req calls.
    '''
    def __init__(self, master, id_='', serial='msgpack', linger=0):
        self.master = master
        self.id_ = id_
        self.serial = Serial(serial)
        self.linger = linger
        self.context = zmq.Context()
        self.poller = zmq.Poller()

    @property
    def socket(self):
        '''
        Lazily create the socket.
        '''
        if not hasattr(self, '_socket'):
            # create a new one
            self._socket = self.context.socket(zmq.REQ)
            if hasattr(zmq, 'RECONNECT_IVL_MAX'):
                self._socket.setsockopt(
                    zmq.RECONNECT_IVL_MAX, 5000
                )

            if self.master.startswith('tcp://[') and hasattr(zmq, 'IPV4ONLY'):
                # IPv6 sockets work for both IPv6 and IPv4 addresses
                self._socket.setsockopt(zmq.IPV4ONLY, 0)
            self._socket.linger = self.linger
            if self.id_:
                self._socket.setsockopt(zmq.IDENTITY, self.id_)
            self._socket.connect(self.master)
        return self._socket

    def clear_socket(self):
        '''
        delete socket if you have it
        '''
        if hasattr(self, '_socket'):
            if isinstance(self.poller.sockets, dict):
                for socket in self.poller.sockets.keys():
                    self.poller.unregister(socket)
            else:
                for socket in self.poller.sockets:
                    self.poller.unregister(socket[0])
            del self._socket

    def send(self, enc, load, tries=1, timeout=60):
        '''
        Takes two arguments, the encryption type and the base payload
        '''
        payload = {'enc': enc}
        payload['load'] = load
        pkg = self.serial.dumps(payload)
        self.socket.send(pkg)
        self.poller.register(self.socket, zmq.POLLIN)
        tried = 0
        while True:
            polled = self.poller.poll(timeout * 1000)
            tried += 1
            if polled:
                break
            if tries > 1:
                log.info('SaltReqTimeoutError: after {0} seconds. (Try {1} of {2})'.format(
                  timeout, tried, tries))
            if tried >= tries:
                self.clear_socket()
                raise SaltReqTimeoutError(
                    'SaltReqTimeoutError: after {0} seconds, ran {1} tries'.format(timeout * tried, tried)
                )
        return self.serial.loads(self.socket.recv())

    def send_auto(self, payload, tries=1, timeout=60):
        '''
        Detect the encryption type based on the payload
        '''
        enc = payload.get('enc', 'clear')
        load = payload.get('load', {})
        return self.send(enc, load, tries, timeout)

    def destroy(self):
        if isinstance(self.poller.sockets, dict):
            for socket in self.poller.sockets.keys():
                if socket.closed is False:
                    socket.setsockopt(zmq.LINGER, 1)
                    socket.close()
                self.poller.unregister(socket)
        else:
            for socket in self.poller.sockets:
                if socket[0].closed is False:
                    socket[0].setsockopt(zmq.LINGER, 1)
                    socket[0].close()
                self.poller.unregister(socket[0])
        if self.socket.closed is False:
            self.socket.setsockopt(zmq.LINGER, 1)
            self.socket.close()
        if self.context.closed is False:
            self.context.term()

    def __del__(self):
        self.destroy()
<|endoftext|>


<commit_before>"""
mshpScraper.py
Missouri State Highway Patrol scraper for crash data!
[azide0x37] Alexander Templeton (nepenthe.me)

Arguments:
    Production:
        optional:
            date:
            name:
            county:
            severity:
            age_of_occupant:

    Devel:
        optional:
            url: of webpage

Returns: pandas Data.Frame object

Class usage template
myScrape = mshpScraper()
myScrape()
"""

import urllib2
import geopy
import pandas as pd
import numpy as np
import sklearn.preprocessing, sklearn.decomposition, sklearn.linear_model, sklearn.pipeline, sklearn.metrics
from sklearn_pandas import DataFrameMapper
from bs4 import BeautifulSoup
from collections import OrderedDict
from datetime import datetime

class mshpScraper:

    def __init__(
                self,
                url = 'http://www.mshp.dps.missouri.gov/HP68/SearchAction',
                colNames = ['Name', 'Age', 'Hometown', 'Severity', 'Date', 'Time', 'County', 'Location', 'Troop'],
                county = 'Jefferson'):

        self.url = url
        self.colNames = colNames

        #Use a proper useragent to evade the anti-hack software
        opener = urllib2.build_opener()
        opener.addheaders = [('User-agent', 'Mozilla/5.0')]

        #Cache our response
        self.response = opener.open(self.url)

        #Set better formatting for testing console print
        pd.set_option('display.max_rows', 500)
        pd.set_option('display.max_columns', 500)
        pd.set_option('display.width', 200)

    def __str__(self):
        try:
            return self.url
        except:
            print "Unable to print url."

    #Setting this default method allows one to call an instance like a function. NEATO!
    def __call__(self, refreshCache = False):

        if refreshCache:
            self.__init__()

        webpageSouped = BeautifulSoup(self.response.read())

        #This is the particular table with the data we need.
        #TODO: Find a better way to pick the right one?
        table = webpageSouped.find('table', summary="Table listing details of the accident.")
        rows = table.findAll('tr')

        dataset = []

        for tr in rows:
            cols = tr.findAll('td')
            row_data = OrderedDict()
            counter = 1

            #TODO: the zeroth element of cols is null; why? Fix.
            for td in cols[1:]:
                text = ''.join(td.find(text=True))
                try:
                    row_data[self.colNames[counter]] = text
                    counter += 1
                except:
                    counter = 0
                    continue
            dataset.append(row_data)

        #Convert dataset (List of Dicts) to pandas DataFrame
        #Extract date and time colums to column of dateTime objects, then drop

        def datetimeCoerce(row):
            return datetime(row['Date'], axis = 1)

        returnData = pd.DataFrame(dataset)
        #returnData['DateTime'] = returnData.apply(lambda row: datetimeCoerce(row))

        #FIXME drop these unused columns
        #returnData.drop('Date', 1, inplace = True)

        #FIXME this doesn't work at all

        #Location coersion
        #geolocator = geopy.geocoders.OpenMapQuest()

        #location = geolocator.geocode(returnData['Location'])

        #returnData['Location'] = returnData.apply(lambda row: geolocator.geocode(row.Location + ", " + row.County + " County, MO"))
        #returnData['Latitude'] = returnData.apply(lambda row: geolocator.geocode(row['Location']).latitude)
        #returnData['Longitude'] = returnData.apply(lambda row: geolocator.geocode(row['Location']).longitude)

        returnData = returnData.drop(, axis=1)
        #Ship it!
        return returnData

myScrape = mshpScraper()
data = myScrape()
mapper = DataFrameMapper([
    ('Age', sklearn.preprocessing.StandardScaler()),
    ('Hometown', sklearn.preprocessing.LabelBinarizer()),
    ('Severity', sklearn.preprocessing.LabelBinarizer()),
    ('County', sklearn.preprocessing.LabelBinarizer()),
    ('Troop', sklearn.preprocessing.LabelBinarizer())
    ])

print np.round(mapper.fit_transform(data), 2)
<commit_msg>trying to fix mapping<commit_after>"""
mshpScraper.py
Missouri State Highway Patrol scraper for crash data!
[azide0x37] Alexander Templeton (nepenthe.me)

Arguments:
    Production:
        optional:
            date:
            name:
            county:
            severity:
            age_of_occupant:

    Devel:
        optional:
            url: of webpage

Returns: pandas Data.Frame object

Class usage template
myScrape = mshpScraper()
myScrape()
"""

import urllib2
import geopy
import pandas as pd
import numpy as np
import sklearn.preprocessing, sklearn.decomposition, sklearn.linear_model, sklearn.pipeline, sklearn.metrics
from sklearn_pandas import DataFrameMapper
from bs4 import BeautifulSoup
from collections import OrderedDict
from datetime import datetime

class mshpScraper:

    def __init__(
                self,
                url = 'http://www.mshp.dps.missouri.gov/HP68/SearchAction',
                colNames = ['Name', 'Age', 'Hometown', 'Severity', 'Date', 'Time', 'County', 'Location', 'Troop'],
                county = 'Jefferson'):

        self.url = url
        self.colNames = colNames

        #Use a proper useragent to evade the anti-hack software
        opener = urllib2.build_opener()
        opener.addheaders = [('User-agent', 'Mozilla/5.0')]

        #Cache our response
        self.response = opener.open(self.url)

        #Set better formatting for testing console print
        pd.set_option('display.max_rows', 500)
        pd.set_option('display.max_columns', 500)
        pd.set_option('display.width', 200)

    def __str__(self):
        try:
            return self.url
        except:
            print "Unable to print url."

    #Setting this default method allows one to call an instance like a function. NEATO!
    def __call__(self, refreshCache = False):

        if refreshCache:
            self.__init__()

        webpageSouped = BeautifulSoup(self.response.read())

        #This is the particular table with the data we need.
        #TODO: Find a better way to pick the right one?
        table = webpageSouped.find('table', summary="Table listing details of the accident.")
        rows = table.findAll('tr')

        dataset = []

        for tr in rows:
            cols = tr.findAll('td')
            row_data = OrderedDict()
            counter = 1

            #TODO: the zeroth element of cols is null; why? Fix.
            for td in cols[1:]:
                text = ''.join(td.find(text=True))
                try:
                    row_data[self.colNames[counter]] = text
                    counter += 1
                except:
                    counter = 0
                    continue
            dataset.append(row_data)

        #Convert dataset (List of Dicts) to pandas DataFrame
        #Extract date and time colums to column of dateTime objects, then drop

        def datetimeCoerce(row):
            return datetime(row['Date'], axis = 1)

        returnData = pd.DataFrame(dataset)
        #returnData['DateTime'] = returnData.apply(lambda row: datetimeCoerce(row))

        #FIXME drop these unused columns
        #returnData.drop('Date', 1, inplace = True)

        #FIXME this doesn't work at all

        #Location coersion
        #geolocator = geopy.geocoders.OpenMapQuest()

        #location = geolocator.geocode(returnData['Location'])

        #returnData['Location'] = returnData.apply(lambda row: geolocator.geocode(row.Location + ", " + row.County + " County, MO"))
        #returnData['Latitude'] = returnData.apply(lambda row: geolocator.geocode(row['Location']).latitude)
        #returnData['Longitude'] = returnData.apply(lambda row: geolocator.geocode(row['Location']).longitude)

        returnData = returnData.drop('Name', axis=1)
        returnData = returnData.drop('Hometown', axis=1)
        returnData = returnData.drop('Date', axis=1)
        returnData = returnData.drop('Time', axis=1)
        returnData = returnData.drop('Location', axis=1)
        #Ship it!
        return returnData

myScrape = mshpScraper()
data = myScrape()
mapper = DataFrameMapper([
    ('Age', sklearn.preprocessing.StandardScaler()),
    ('Severity', sklearn.preprocessing.LabelBinarizer()),
    ('County', sklearn.preprocessing.LabelBinarizer()),
    ('Troop', sklearn.preprocessing.LabelBinarizer())
    ])

print np.round(mapper.fit_transform(data), 2)
<|endoftext|>


<commit_before>
#
# script.py
#
# Distributed under the MIT/X11 software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
#

import struct

# push value
OP_0 = 0x00
OP_FALSE = OP_0
OP_PUSHDATA1 = 0x4c
OP_PUSHDATA2 = 0x4d
OP_PUSHDATA4 = 0x4e
OP_1NEGATE = 0x4f
OP_RESERVED = 0x50
OP_1 = 0x51
OP_TRUE=OP_1
OP_2 = 0x52
OP_3 = 0x53
OP_4 = 0x54
OP_5 = 0x55
OP_6 = 0x56
OP_7 = 0x57
OP_8 = 0x58
OP_9 = 0x59
OP_10 = 0x5a
OP_11 = 0x5b
OP_12 = 0x5c
OP_13 = 0x5d
OP_14 = 0x5e
OP_15 = 0x5f
OP_16 = 0x60

# control
OP_NOP = 0x61
OP_VER = 0x62
OP_IF = 0x63
OP_NOTIF = 0x64
OP_VERIF = 0x65
OP_VERNOTIF = 0x66
OP_ELSE = 0x67
OP_ENDIF = 0x68
OP_VERIFY = 0x69
OP_RETURN = 0x6a

# stack ops
OP_TOALTSTACK = 0x6b
OP_FROMALTSTACK = 0x6c
OP_2DROP = 0x6d
OP_2DUP = 0x6e
OP_3DUP = 0x6f
OP_2OVER = 0x70
OP_2ROT = 0x71
OP_2SWAP = 0x72
OP_IFDUP = 0x73
OP_DEPTH = 0x74
OP_DROP = 0x75
OP_DUP = 0x76
OP_NIP = 0x77
OP_OVER = 0x78
OP_PICK = 0x79
OP_ROLL = 0x7a
OP_ROT = 0x7b
OP_SWAP = 0x7c
OP_TUCK = 0x7d

# splice ops
OP_CAT = 0x7e
OP_SUBSTR = 0x7f
OP_LEFT = 0x80
OP_RIGHT = 0x81
OP_SIZE = 0x82

# bit logic
OP_INVERT = 0x83
OP_AND = 0x84
OP_OR = 0x85
OP_XOR = 0x86
OP_EQUAL = 0x87
OP_EQUALVERIFY = 0x88
OP_RESERVED1 = 0x89
OP_RESERVED2 = 0x8a

# numeric
OP_1ADD = 0x8b
OP_1SUB = 0x8c
OP_2MUL = 0x8d
OP_2DIV = 0x8e
OP_NEGATE = 0x8f
OP_ABS = 0x90
OP_NOT = 0x91
OP_0NOTEQUAL = 0x92

OP_ADD = 0x93
OP_SUB = 0x94
OP_MUL = 0x95
OP_DIV = 0x96
OP_MOD = 0x97
OP_LSHIFT = 0x98
OP_RSHIFT = 0x99

OP_BOOLAND = 0x9a
OP_BOOLOR = 0x9b
OP_NUMEQUAL = 0x9c
OP_NUMEQUALVERIFY = 0x9d
OP_NUMNOTEQUAL = 0x9e
OP_LESSTHAN = 0x9f
OP_GREATERTHAN = 0xa0
OP_LESSTHANOREQUAL = 0xa1
OP_GREATERTHANOREQUAL = 0xa2
OP_MIN = 0xa3
OP_MAX = 0xa4

OP_WITHIN = 0xa5

# crypto
OP_RIPEMD160 = 0xa6
OP_SHA1 = 0xa7
OP_SHA256 = 0xa8
OP_HASH160 = 0xa9
OP_HASH256 = 0xaa
OP_CODESEPARATOR = 0xab
OP_CHECKSIG = 0xac
OP_CHECKSIGVERIFY = 0xad
OP_CHECKMULTISIG = 0xae
OP_CHECKMULTISIGVERIFY = 0xaf

# expansion
OP_NOP1 = 0xb0
OP_NOP2 = 0xb1
OP_NOP3 = 0xb2
OP_NOP4 = 0xb3
OP_NOP5 = 0xb4
OP_NOP6 = 0xb5
OP_NOP7 = 0xb6
OP_NOP8 = 0xb7
OP_NOP9 = 0xb8
OP_NOP10 = 0xb9

# template matching params
OP_SMALLINTEGER = 0xfa
OP_PUBKEYS = 0xfb
OP_PUBKEYHASH = 0xfd
OP_PUBKEY = 0xfe

OP_INVALIDOPCODE = 0xff

# pseudo-opcodes used internally, always > 0xff
OP_PUSHDATA = 0x1000

VALID_OPCODES = {
	OP_1NEGATE : True,
	OP_RESERVED : True,
	OP_1 : True,
	OP_2 : True,
	OP_3 : True,
	OP_4 : True,
	OP_5 : True,
	OP_6 : True,
	OP_7 : True,
	OP_8 : True,
	OP_9 : True,
	OP_10 : True,
	OP_11 : True,
	OP_12 : True,
	OP_13 : True,
	OP_14 : True,
	OP_15 : True,
	OP_16 : True,

	OP_NOP : True,
	OP_VER : True,
	OP_IF : True,
	OP_NOTIF : True,
	OP_VERIF : True,
	OP_VERNOTIF : True,
	OP_ELSE : True,
	OP_ENDIF : True,
	OP_VERIFY : True,
	OP_RETURN : True,

	OP_TOALTSTACK : True,
	OP_FROMALTSTACK : True,
	OP_2DROP : True,
	OP_2DUP : True,
	OP_3DUP : True,
	OP_2OVER : True,
	OP_2ROT : True,
	OP_2SWAP : True,
	OP_IFDUP : True,
	OP_DEPTH : True,
	OP_DROP : True,
	OP_DUP : True,
	OP_NIP : True,
	OP_OVER : True,
	OP_PICK : True,
	OP_ROLL : True,
	OP_ROT : True,
	OP_SWAP : True,
	OP_TUCK : True,

	OP_CAT : True,
	OP_SUBSTR : True,
	OP_LEFT : True,
	OP_RIGHT : True,
	OP_SIZE : True,

	OP_INVERT : True,
	OP_AND : True,
	OP_OR : True,
	OP_XOR : True,
	OP_EQUAL : True,
	OP_EQUALVERIFY : True,
	OP_RESERVED1 : True,
	OP_RESERVED2 : True,

	OP_1ADD : True,
	OP_1SUB : True,
	OP_2MUL : True,
	OP_2DIV : True,
	OP_NEGATE : True,
	OP_ABS : True,
	OP_NOT : True,
	OP_0NOTEQUAL : True,

	OP_ADD : True,
	OP_SUB : True,
	OP_MUL : True,
	OP_DIV : True,
	OP_MOD : True,
	OP_LSHIFT : True,
	OP_RSHIFT : True,

	OP_BOOLAND : True,
	OP_BOOLOR : True,
	OP_NUMEQUAL : True,
	OP_NUMEQUALVERIFY : True,
	OP_NUMNOTEQUAL : True,
	OP_LESSTHAN : True,
	OP_GREATERTHAN : True,
	OP_LESSTHANOREQUAL : True,
	OP_GREATERTHANOREQUAL : True,
	OP_MIN : True,
	OP_MAX : True,

	OP_WITHIN : True,

	OP_RIPEMD160 : True,
	OP_SHA1 : True,
	OP_SHA256 : True,
	OP_HASH160 : True,
	OP_HASH256 : True,
	OP_CODESEPARATOR : True,
	OP_CHECKSIG : True,
	OP_CHECKSIGVERIFY : True,
	OP_CHECKMULTISIG : True,
	OP_CHECKMULTISIGVERIFY : True,

	OP_NOP1 : True,
	OP_NOP2 : True,
	OP_NOP3 : True,
	OP_NOP4 : True,
	OP_NOP5 : True,
	OP_NOP6 : True,
	OP_NOP7 : True,
	OP_NOP8 : True,
	OP_NOP9 : True,
	OP_NOP10 : True,

	OP_SMALLINTEGER : True,
	OP_PUBKEYS : True,
	OP_PUBKEYHASH : True,
	OP_PUBKEY : True,
}

class CScriptOp(object):
	def __init__(self):
		self.op = OP_INVALIDOPCODE
		self.data = ''

class CScript(object):
	def __init__(self, vch=None):
		self.ops = []
		self.valid = True

		if vch is not None:
			self.tokenize(vch)

	def tokenize(self, vch_in):
		vch = vch_in
		self.valid = False
		while len(vch) > 0:
			opcode = ord(vch[0])

			sop = CScriptOp()
			sop.op = opcode

			vch = vch[1:]

			if opcode > OP_PUSHDATA4:
				if opcode not in VALID_OPCODES:
					return False
				self.ops.append(sop)
				continue

			if opcode < OP_PUSHDATA1:
				datasize = opcode

			elif opcode == OP_PUSHDATA1:
				if len(vch) < 1:
					return False
				datasize = ord(vch[0])
				vch = vch[1:]

			elif opcode == OP_PUSHDATA2:
				if len(vch) < 2:
					return False
				datasize = struct.unpack("<H", vch[:2])[0]
				vch = vch[2:]

			elif opcode == OP_PUSHDATA4:
				if len(vch) < 4:
					return False
				datasize = struct.unpack("<I", vch[:4])[0]
				vch = vch[4:]

			if len(vch) < datasize:
				return False

			sop.op = OP_PUSHDATA
			sop.data = vch[:datasize]
			vch = vch[datasize:]

			self.ops.append(sop)

		self.valid = True
		return True

<commit_msg>script: for pushdata ops, do not obscure original opcode<commit_after>
#
# script.py
#
# Distributed under the MIT/X11 software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
#

import struct

# push value
OP_0 = 0x00
OP_FALSE = OP_0
OP_PUSHDATA1 = 0x4c
OP_PUSHDATA2 = 0x4d
OP_PUSHDATA4 = 0x4e
OP_1NEGATE = 0x4f
OP_RESERVED = 0x50
OP_1 = 0x51
OP_TRUE=OP_1
OP_2 = 0x52
OP_3 = 0x53
OP_4 = 0x54
OP_5 = 0x55
OP_6 = 0x56
OP_7 = 0x57
OP_8 = 0x58
OP_9 = 0x59
OP_10 = 0x5a
OP_11 = 0x5b
OP_12 = 0x5c
OP_13 = 0x5d
OP_14 = 0x5e
OP_15 = 0x5f
OP_16 = 0x60

# control
OP_NOP = 0x61
OP_VER = 0x62
OP_IF = 0x63
OP_NOTIF = 0x64
OP_VERIF = 0x65
OP_VERNOTIF = 0x66
OP_ELSE = 0x67
OP_ENDIF = 0x68
OP_VERIFY = 0x69
OP_RETURN = 0x6a

# stack ops
OP_TOALTSTACK = 0x6b
OP_FROMALTSTACK = 0x6c
OP_2DROP = 0x6d
OP_2DUP = 0x6e
OP_3DUP = 0x6f
OP_2OVER = 0x70
OP_2ROT = 0x71
OP_2SWAP = 0x72
OP_IFDUP = 0x73
OP_DEPTH = 0x74
OP_DROP = 0x75
OP_DUP = 0x76
OP_NIP = 0x77
OP_OVER = 0x78
OP_PICK = 0x79
OP_ROLL = 0x7a
OP_ROT = 0x7b
OP_SWAP = 0x7c
OP_TUCK = 0x7d

# splice ops
OP_CAT = 0x7e
OP_SUBSTR = 0x7f
OP_LEFT = 0x80
OP_RIGHT = 0x81
OP_SIZE = 0x82

# bit logic
OP_INVERT = 0x83
OP_AND = 0x84
OP_OR = 0x85
OP_XOR = 0x86
OP_EQUAL = 0x87
OP_EQUALVERIFY = 0x88
OP_RESERVED1 = 0x89
OP_RESERVED2 = 0x8a

# numeric
OP_1ADD = 0x8b
OP_1SUB = 0x8c
OP_2MUL = 0x8d
OP_2DIV = 0x8e
OP_NEGATE = 0x8f
OP_ABS = 0x90
OP_NOT = 0x91
OP_0NOTEQUAL = 0x92

OP_ADD = 0x93
OP_SUB = 0x94
OP_MUL = 0x95
OP_DIV = 0x96
OP_MOD = 0x97
OP_LSHIFT = 0x98
OP_RSHIFT = 0x99

OP_BOOLAND = 0x9a
OP_BOOLOR = 0x9b
OP_NUMEQUAL = 0x9c
OP_NUMEQUALVERIFY = 0x9d
OP_NUMNOTEQUAL = 0x9e
OP_LESSTHAN = 0x9f
OP_GREATERTHAN = 0xa0
OP_LESSTHANOREQUAL = 0xa1
OP_GREATERTHANOREQUAL = 0xa2
OP_MIN = 0xa3
OP_MAX = 0xa4

OP_WITHIN = 0xa5

# crypto
OP_RIPEMD160 = 0xa6
OP_SHA1 = 0xa7
OP_SHA256 = 0xa8
OP_HASH160 = 0xa9
OP_HASH256 = 0xaa
OP_CODESEPARATOR = 0xab
OP_CHECKSIG = 0xac
OP_CHECKSIGVERIFY = 0xad
OP_CHECKMULTISIG = 0xae
OP_CHECKMULTISIGVERIFY = 0xaf

# expansion
OP_NOP1 = 0xb0
OP_NOP2 = 0xb1
OP_NOP3 = 0xb2
OP_NOP4 = 0xb3
OP_NOP5 = 0xb4
OP_NOP6 = 0xb5
OP_NOP7 = 0xb6
OP_NOP8 = 0xb7
OP_NOP9 = 0xb8
OP_NOP10 = 0xb9

# template matching params
OP_SMALLINTEGER = 0xfa
OP_PUBKEYS = 0xfb
OP_PUBKEYHASH = 0xfd
OP_PUBKEY = 0xfe

OP_INVALIDOPCODE = 0xff

VALID_OPCODES = {
	OP_1NEGATE : True,
	OP_RESERVED : True,
	OP_1 : True,
	OP_2 : True,
	OP_3 : True,
	OP_4 : True,
	OP_5 : True,
	OP_6 : True,
	OP_7 : True,
	OP_8 : True,
	OP_9 : True,
	OP_10 : True,
	OP_11 : True,
	OP_12 : True,
	OP_13 : True,
	OP_14 : True,
	OP_15 : True,
	OP_16 : True,

	OP_NOP : True,
	OP_VER : True,
	OP_IF : True,
	OP_NOTIF : True,
	OP_VERIF : True,
	OP_VERNOTIF : True,
	OP_ELSE : True,
	OP_ENDIF : True,
	OP_VERIFY : True,
	OP_RETURN : True,

	OP_TOALTSTACK : True,
	OP_FROMALTSTACK : True,
	OP_2DROP : True,
	OP_2DUP : True,
	OP_3DUP : True,
	OP_2OVER : True,
	OP_2ROT : True,
	OP_2SWAP : True,
	OP_IFDUP : True,
	OP_DEPTH : True,
	OP_DROP : True,
	OP_DUP : True,
	OP_NIP : True,
	OP_OVER : True,
	OP_PICK : True,
	OP_ROLL : True,
	OP_ROT : True,
	OP_SWAP : True,
	OP_TUCK : True,

	OP_CAT : True,
	OP_SUBSTR : True,
	OP_LEFT : True,
	OP_RIGHT : True,
	OP_SIZE : True,

	OP_INVERT : True,
	OP_AND : True,
	OP_OR : True,
	OP_XOR : True,
	OP_EQUAL : True,
	OP_EQUALVERIFY : True,
	OP_RESERVED1 : True,
	OP_RESERVED2 : True,

	OP_1ADD : True,
	OP_1SUB : True,
	OP_2MUL : True,
	OP_2DIV : True,
	OP_NEGATE : True,
	OP_ABS : True,
	OP_NOT : True,
	OP_0NOTEQUAL : True,

	OP_ADD : True,
	OP_SUB : True,
	OP_MUL : True,
	OP_DIV : True,
	OP_MOD : True,
	OP_LSHIFT : True,
	OP_RSHIFT : True,

	OP_BOOLAND : True,
	OP_BOOLOR : True,
	OP_NUMEQUAL : True,
	OP_NUMEQUALVERIFY : True,
	OP_NUMNOTEQUAL : True,
	OP_LESSTHAN : True,
	OP_GREATERTHAN : True,
	OP_LESSTHANOREQUAL : True,
	OP_GREATERTHANOREQUAL : True,
	OP_MIN : True,
	OP_MAX : True,

	OP_WITHIN : True,

	OP_RIPEMD160 : True,
	OP_SHA1 : True,
	OP_SHA256 : True,
	OP_HASH160 : True,
	OP_HASH256 : True,
	OP_CODESEPARATOR : True,
	OP_CHECKSIG : True,
	OP_CHECKSIGVERIFY : True,
	OP_CHECKMULTISIG : True,
	OP_CHECKMULTISIGVERIFY : True,

	OP_NOP1 : True,
	OP_NOP2 : True,
	OP_NOP3 : True,
	OP_NOP4 : True,
	OP_NOP5 : True,
	OP_NOP6 : True,
	OP_NOP7 : True,
	OP_NOP8 : True,
	OP_NOP9 : True,
	OP_NOP10 : True,

	OP_SMALLINTEGER : True,
	OP_PUBKEYS : True,
	OP_PUBKEYHASH : True,
	OP_PUBKEY : True,
}

class CScriptOp(object):
	def __init__(self):
		self.op = OP_INVALIDOPCODE
		self.data = ''

class CScript(object):
	def __init__(self, vch=None):
		self.ops = []
		self.valid = True

		if vch is not None:
			self.tokenize(vch)

	def tokenize(self, vch_in):
		vch = vch_in
		self.valid = False
		while len(vch) > 0:
			opcode = ord(vch[0])

			sop = CScriptOp()
			sop.op = opcode

			vch = vch[1:]

			if opcode > OP_PUSHDATA4:
				if opcode not in VALID_OPCODES:
					return False
				self.ops.append(sop)
				continue

			if opcode < OP_PUSHDATA1:
				datasize = opcode

			elif opcode == OP_PUSHDATA1:
				if len(vch) < 1:
					return False
				datasize = ord(vch[0])
				vch = vch[1:]

			elif opcode == OP_PUSHDATA2:
				if len(vch) < 2:
					return False
				datasize = struct.unpack("<H", vch[:2])[0]
				vch = vch[2:]

			elif opcode == OP_PUSHDATA4:
				if len(vch) < 4:
					return False
				datasize = struct.unpack("<I", vch[:4])[0]
				vch = vch[4:]

			if len(vch) < datasize:
				return False

			sop.data = vch[:datasize]
			vch = vch[datasize:]

			self.ops.append(sop)

		self.valid = True
		return True

<|endoftext|>


<commit_before>from django.conf import settings
from django.contrib.contenttypes import generic
from django.contrib.contenttypes.models import ContentType
from django.contrib.sites.models import Site
from django.db import models
from django.utils.translation import get_language, ugettext_lazy as _

from .managers import MetadataManager

LANGUAGES = tuple((code, _(name)) for code, name in settings.LANGUAGES)

class Metadata(models.Model):

    url_path = models.CharField(
        verbose_name=_('URL path'),
        max_length=255,
        db_index=True,
    )
    title = models.CharField(
        verbose_name=_(u'page title'),
        max_length=255,
        blank=True,
        help_text=_(u'Displayed in browser window title.'),
    )
    title_extend = models.BooleanField(
        verbose_name=_(u'extend page title with site name'),
        default=True,
        help_text=_(u'For example, Page title - Site name'),
    )
    keywords = models.TextField(
        verbose_name=_(u'page keywords'),
        blank=True,
        help_text=_(
            u'Keywords are terms describing web page content and used '
            u'by search engines. Here you can enumerate some words '
            u'divided by commas.'
        ),
    )
    description = models.TextField(
        verbose_name=_(u'page description'),
        blank=True,
        help_text=_(
            u'Here you can set short description of this page '
            u'for search engines.'
        ),
    )
    enabled = models.BooleanField(
        verbose_name=_(u'enabled'),
        default=False,
        help_text=_(u'If not set, meta tags will not be used on page.'),
    )
    language = models.CharField(
        verbose_name=_(u'language'),
        max_length=10,
        db_index=True,
        choices=LANGUAGES,
        default=get_language,
    )
    sites = models.ManyToManyField(
        to=Site,
        null=True,
        blank=True,
        related_name='+',
        verbose_name=_(u'sites'),
    )
    content_type = models.ForeignKey(ContentType, null=True, editable=False)
    object_id = models.PositiveIntegerField(null=True, editable=False)
    content_object = generic.GenericForeignKey('content_type', 'object_id')

    objects = MetadataManager()

    class Meta:
        verbose_name = _(u'page metadata')
        verbose_name_plural = _(u'pages metadata')

    def __unicode__(self):
        return self.url_path

    def save(self, *args, **kwargs):
        self.update_url_path(commit=False)
        self.update_language(commit=False)
        super(Metadata, self).save(*args, **kwargs)
        self.update_sites()

    def update_url_path(self, commit=True):
        obj = self.content_object
        if obj:
            from . import site
            model_metadata = site._registry.get(obj.__class__)
            url_path = model_metadata.url_path(obj)
            if url_path:
                self.url_path = url_path
                if commit:
                    super(Metadata, self).save(False, False)

    def update_language(self, commit=True):
        obj = self.content_object
        if obj:
            from . import site
            model_metadata = site._registry.get(obj.__class__)
            language = model_metadata.language(obj)
            if language:
                self.language = language
                if commit:
                    super(Metadata, self).save(False, False)

    def update_sites(self):
        obj = self.content_object
        if obj:
            from . import site
            model_metadata = site._registry.get(obj.__class__)
            sites = model_metadata.sites(obj)
            self.sites.clear()
            if sites:
                self.sites.add(*sites)
<commit_msg>Regroup model's fields<commit_after>from django.conf import settings
from django.contrib.contenttypes import generic
from django.contrib.contenttypes.models import ContentType
from django.contrib.sites.models import Site
from django.db import models
from django.utils.translation import get_language, ugettext_lazy as _

from .managers import MetadataManager

LANGUAGES = tuple((code, _(name)) for code, name in settings.LANGUAGES)

class Metadata(models.Model):

    url_path = models.CharField(
        verbose_name=_('URL path'),
        max_length=255,
        db_index=True,
    )
    sites = models.ManyToManyField(
        to=Site,
        null=True,
        blank=True,
        related_name='+',
        verbose_name=_(u'sites'),
    )
    language = models.CharField(
        verbose_name=_(u'language'),
        max_length=10,
        db_index=True,
        choices=LANGUAGES,
        default=get_language,
    )

    title = models.CharField(
        verbose_name=_(u'page title'),
        max_length=255,
        blank=True,
        help_text=_(u'Displayed in browser window title.'),
    )
    title_extend = models.BooleanField(
        verbose_name=_(u'extend page title with site name'),
        default=True,
        help_text=_(u'For example, Page title - Site name'),
    )
    keywords = models.TextField(
        verbose_name=_(u'page keywords'),
        blank=True,
        help_text=_(
            u'Keywords are terms describing web page content and used '
            u'by search engines. Here you can enumerate some words '
            u'divided by commas.'
        ),
    )
    description = models.TextField(
        verbose_name=_(u'page description'),
        blank=True,
        help_text=_(
            u'Here you can set short description of this page '
            u'for search engines.'
        ),
    )

    enabled = models.BooleanField(
        verbose_name=_(u'enabled'),
        default=False,
        help_text=_(u'If not set, meta tags will not be used on page.'),
    )

    content_type = models.ForeignKey(ContentType, null=True, editable=False)
    object_id = models.PositiveIntegerField(null=True, editable=False)
    content_object = generic.GenericForeignKey('content_type', 'object_id')

    objects = MetadataManager()

    class Meta:
        verbose_name = _(u'page metadata')
        verbose_name_plural = _(u'pages metadata')

    def __unicode__(self):
        return self.url_path

    def save(self, *args, **kwargs):
        self.update_url_path(commit=False)
        self.update_language(commit=False)
        super(Metadata, self).save(*args, **kwargs)
        self.update_sites()

    def update_url_path(self, commit=True):
        obj = self.content_object
        if obj:
            from . import site
            model_metadata = site._registry.get(obj.__class__)
            url_path = model_metadata.url_path(obj)
            if url_path:
                self.url_path = url_path
                if commit:
                    super(Metadata, self).save(False, False)

    def update_language(self, commit=True):
        obj = self.content_object
        if obj:
            from . import site
            model_metadata = site._registry.get(obj.__class__)
            language = model_metadata.language(obj)
            if language:
                self.language = language
                if commit:
                    super(Metadata, self).save(False, False)

    def update_sites(self):
        obj = self.content_object
        if obj:
            from . import site
            model_metadata = site._registry.get(obj.__class__)
            sites = model_metadata.sites(obj)
            self.sites.clear()
            if sites:
                self.sites.add(*sites)
<|endoftext|>


<commit_before>## $Id$
##
## This file is part of CDS Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007 CERN.
##
## CDS Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

"""CDS Invenio Access Control Config. """

__revision__ = \
    "$Id$"

# pylint: disable-msg=C0301

from invenio.config import cdsname, weburl, sweburl, supportemail, CFG_CERN_SITE
import cPickle
from zlib import compress
from invenio.messages import gettext_set_language

class InvenioWebAccessFireroleError(Exception):
    """Just an Exception to discover if it's a FireRole problem"""
    pass

# VALUES TO BE EXPORTED
# CURRENTLY USED BY THE FILES access_control_engine.py access_control_admin.py webaccessadmin_lib.py

# name of the role giving superadmin rights
SUPERADMINROLE      = 'superadmin'

# name of the webaccess webadmin role
WEBACCESSADMINROLE  = 'webaccessadmin'

# name of the action allowing roles to access the web administrator interface
WEBACCESSACTION     = 'cfgwebaccess'

# name of the action allowing roles to access the web administrator interface
VIEWRESTRCOLL     = 'accrestrcoll'

# name of the action allowing roles to delegate the rights to other roles
# ex: libraryadmin to delegate libraryworker
DELEGATEADDUSERROLE = 'accdelegaterole'

# max number of users to display in the drop down selects
MAXSELECTUSERS = 25

# max number of users to display in a page (mainly for user area)
MAXPAGEUSERS = 25

""" Serialized compiled default role definition"""
CFG_ACC_EMPTY_ROLE_DEFINITION_SER=compress(cPickle.dumps((False, False, ()), -1))

""" Source of the default role definition"""
CFG_ACC_EMPTY_ROLE_DEFINITION_SRC='deny any'

# Use external source for access control?
# Atleast one must be added
# Adviced not to change the name, since it is used to identify the account
# Format is:   System name: (System class, Default True/Flase), atleast one
# must be default
CFG_EXTERNAL_AUTHENTICATION = {"Local": (None, True)}

# Variables to set to the SSO Authentication name if using SSO
CFG_EXTERNAL_AUTH_USING_SSO = False
CFG_EXTERNAL_AUTH_LOGOUT_SSO = None

if CFG_CERN_SITE:
    if False: #FIXME set this to True when we deploy SSO at CERN
        import external_authentication_sso as ea_sso
        CFG_EXTERNAL_AUTH_USING_SSO = "CERN"
        # Link to reach in order to logout from SSO
        CFG_EXTERNAL_AUTH_LOGOUT_SSO = 'https://login.cern.ch/adfs/ls/?wa=wsignout1.0'
        CFG_EXTERNAL_AUTHENTICATION = {CFG_EXTERNAL_AUTH_USING_SSO : (ea_sso.ExternalAuthSSO(), True)}
    else:
        import external_authentication_cern as ea_cern
        CFG_EXTERNAL_AUTHENTICATION = {"Local": (None, False), \
        "CERN": (ea_cern.ExternalAuthCern(), True)}

# default data for the add_default_settings function
# Note: by default the definition is set to deny any. This won't be a problem
# because userid directly connected with roles will still be allowed.
# roles
#           name          description          definition
DEF_ROLES = ((SUPERADMINROLE, 'superuser with all rights', 'deny any'),
             ('photoadmin',   'Photo collection administrator', 'deny any'),
             (WEBACCESSADMINROLE,     'WebAccess administrator', 'deny any'))

# users
# list of e-mail addresses
DEF_USERS = []

# actions
#            name                  desc     allowedkeywords   optional
DEF_ACTIONS = (
               ('cfgwebsearch',         'configure WebSearch',       '',              'no'),
               ('cfgbibformat',         'configure BibFormat',       '',              'no'),
               ('cfgwebsubmit',         'configure WebSubmit',       '',              'no'),
               ('runbibindex',          'run BibIndex',       '',              'no'),
               ('runbibupload',         'run BibUpload',       '',              'no'),
               ('runwebcoll',           'run webcoll',       'collection',    'yes'),
               ('runbibformat',         'run BibFormat',       'format',        'yes'),
               (WEBACCESSACTION,        'configure WebAccess',       '',              'no'),
               (DELEGATEADDUSERROLE,    'delegate subroles inside WebAccess',       'role',          'no'),
               (VIEWRESTRCOLL, 'view restricted collection', 'collection', 'no'),
               ('runbibtaskex',         'run BibTaskEx example',       '',              'no'),
               ('referee',                  'referee document type doctype/category categ', 'doctype,categ',    'yes'),
               ('submit',                   'use webSubmit',    'doctype,act',  'yes'),
               ('runbibrank',         'run BibRank',       '',              'no'),
               ('cfgbibrank',         'configure BibRank',       '',              'no'),
               ('cfgbibharvest',         'configure BibHarvest',       '',              'no'),
               ('runoaiharvest',         'run oaiharvest task',       '',              'no'),
               ('cfgwebcomment',         'configure WebComment',       '',              'no'),
               ('runoaiarchive',         'run oaiarchive task',       '',              'no'),
               ('runbibedit',         'run BibEdit',       '',              'no'),
               ('runwebstatadmin', 'run WebStadAdmin', '', 'no'),
               ('viewrestrdoc', 'view restricted document', 'status', 'no'),
              )

# authorizations
#          role              action          arglistid  optional   arguments
DEF_AUTHS = (
             (SUPERADMINROLE,    'cfgwebsearch',         -1,      0,       {}),
             (SUPERADMINROLE,    'cfgbibformat',         -1,      0,       {}),
             (SUPERADMINROLE,    'cfgwebsubmit',         -1,      0,       {}),
             (SUPERADMINROLE,    'runbibindex',          -1,      0,       {}),
             (SUPERADMINROLE,    'runbibupload',         -1,      0,       {}),
             (SUPERADMINROLE,    'runbibformat',         -1,      1,       {}),
             (SUPERADMINROLE,    WEBACCESSACTION,        -1,      0,       {}),
             ('photoadmin',      'runwebcoll',           -1,      0,       {'collection': 'Pictures'}),
             (WEBACCESSADMINROLE,WEBACCESSACTION,        -1,      0,       {}),
             (SUPERADMINROLE,    'runtaskex',            -1,      0,       {}),
             (SUPERADMINROLE,    'referee',         -1,      1,       {}),
             (SUPERADMINROLE,    'submit',         -1,      1,       {}),
             (SUPERADMINROLE,    'runbibrank',           -1,      0,       {}),
             (SUPERADMINROLE,    'cfgbibrank',           -1,      0,       {}),
             (SUPERADMINROLE,    'cfgbibharvest',           -1,      0,       {}),
             (SUPERADMINROLE,    'runoaiharvest',            -1,      0,       {}),
             (SUPERADMINROLE,    'cfgwebcomment',            -1,      0,       {}),
             (SUPERADMINROLE,    'runoaiarchive',            -1,      0,       {}),
             (SUPERADMINROLE,    'runbibedit',            -1,      0,       {}),
            )

_ = gettext_set_language('en')
CFG_ACC_ACTIVITIES_URLS = {
    'runbibedit' : (_("Run BibEdit"), "%s/admin/bibedit/bibeditadmin.py?ln=%%s" % weburl),
    'cfgbibformat' : (_("Configure BibFormat"), "%s/admin/bibformat/bibformatadmin.py?ln=%%s" % weburl),
    'cfgbibharvest' : (_("Configure BibHarvest"), "%s/admin/bibharvest/?ln=%%s" % weburl),
    'cfgbibindex' : (_("Configure BibIndex"), "%s/admin/bibindex/bibindexadmin.py?ln=%%s" % weburl),
    'cfgbibrank' : (_("Configure BibRank"), "%s/admin/bibrank/bibrankadmin.py?ln=%%s" % weburl),
    'cfgwebaccess' : (_("Configure WebAccess"), "%s/admin/webaccess/webaccessadmin.py?ln=%%s" % weburl),
    'cfgwebcomment' : (_("Configure WebComment"), "%s/admin/webcomment/webcommentadmin.py?ln=%%s" % weburl),
    'cfgwebsearch' : (_("Configure WebSearch"), "%s/admin/websearch/websearchadmin.py?ln=%%s" % weburl),
    'cfgwebsubmit' : (_("Configure WebSubmit"), "%s/admin/websubmit/websubmitadmin.py?ln=%%s" % weburl),
}

CFG_WEBACCESS_MSGS = {
                                0: 'Try to login with another account.' % (sweburl),
                                1: '
If you think this is not correct, please contact: %s' % (supportemail, supportemail),
                                2: '
If you have any questions, please write to %s' % (supportemail, supportemail),
                                3: 'Guest users are not allowed, please login.' % sweburl,
                                4: 'The site is temporarily closed for maintenance.  Please come back soon.',
                                5: 'Authorization failure',
                                6: '%s temporarily closed' % cdsname,
                                7: 'This functionality is temporarily closed due to server maintenance. Please use only the search engine in the meantime.',
                                8: 'Functionality temporarily closed'
        }

CFG_WEBACCESS_WARNING_MSGS = {
                                0: 'Authorization granted',
                                1: 'Error(1): You are not authorized to perform this action.',
                                2: 'Error(2): You are not authorized to perform any action.',
                                3: 'Error(3): The action %s does not exist.',
                                4: 'Error(4): Unexpected error occurred.',
                                5: 'Error(5): Missing mandatory keyword argument(s) for this action.',
                                6: 'Error(6): Guest accounts are not authorized to perform this action.',
                                7: 'Error(7): Not enough arguments, user ID and action name required.',
                                8: 'Error(8): Incorrect keyword argument(s) for this action.',
                                9: """Error(9): Account '%s' is not yet activated.""",
                               10: """Error(10): You were not authorized by the authentication method '%s'.""",
                               11: """Error(11): The selected login method '%s' is not the default method for this account, please try another one.""",
                               12: """Error(12): Selected login method '%s' does not exist.""",
                               13: """Error(13): Could not register '%s' account.""",
                               14: """Error(14): Could not login using '%s', because this user is unknown.""",
                               15: """Error(15): Could not login using your '%s' account, because you have introduced a wrong password.""",
                               16: """Error(16): External authentication troubles using '%s' (maybe temporary network problems).""",
                               17: """Error(17): You have not yet confirmed the email address for the '%s' authentication method.""",
                               18: """Error(18): The administrator has not yet activated your account for the '%s' authentication method.""",
                               19: """Error(19): The site is having troubles in sending you an email for confirming your email address. The error has been logged and will be taken care of as soon as possible."""
        }

<commit_msg>Made a clearer distinction between BibHarvest and OAIRepository. Added cfgoairepository action.<commit_after>## $Id$
##
## This file is part of CDS Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007 CERN.
##
## CDS Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

"""CDS Invenio Access Control Config. """

__revision__ = \
    "$Id$"

# pylint: disable-msg=C0301

from invenio.config import cdsname, weburl, sweburl, supportemail, CFG_CERN_SITE
import cPickle
from zlib import compress
from invenio.messages import gettext_set_language

class InvenioWebAccessFireroleError(Exception):
    """Just an Exception to discover if it's a FireRole problem"""
    pass

# VALUES TO BE EXPORTED
# CURRENTLY USED BY THE FILES access_control_engine.py access_control_admin.py webaccessadmin_lib.py

# name of the role giving superadmin rights
SUPERADMINROLE      = 'superadmin'

# name of the webaccess webadmin role
WEBACCESSADMINROLE  = 'webaccessadmin'

# name of the action allowing roles to access the web administrator interface
WEBACCESSACTION     = 'cfgwebaccess'

# name of the action allowing roles to access the web administrator interface
VIEWRESTRCOLL     = 'accrestrcoll'

# name of the action allowing roles to delegate the rights to other roles
# ex: libraryadmin to delegate libraryworker
DELEGATEADDUSERROLE = 'accdelegaterole'

# max number of users to display in the drop down selects
MAXSELECTUSERS = 25

# max number of users to display in a page (mainly for user area)
MAXPAGEUSERS = 25

""" Serialized compiled default role definition"""
CFG_ACC_EMPTY_ROLE_DEFINITION_SER=compress(cPickle.dumps((False, False, ()), -1))

""" Source of the default role definition"""
CFG_ACC_EMPTY_ROLE_DEFINITION_SRC='deny any'

# Use external source for access control?
# Atleast one must be added
# Adviced not to change the name, since it is used to identify the account
# Format is:   System name: (System class, Default True/Flase), atleast one
# must be default
CFG_EXTERNAL_AUTHENTICATION = {"Local": (None, True)}

# Variables to set to the SSO Authentication name if using SSO
CFG_EXTERNAL_AUTH_USING_SSO = False
CFG_EXTERNAL_AUTH_LOGOUT_SSO = None

if CFG_CERN_SITE:
    if False: #FIXME set this to True when we deploy SSO at CERN
        import external_authentication_sso as ea_sso
        CFG_EXTERNAL_AUTH_USING_SSO = "CERN"
        # Link to reach in order to logout from SSO
        CFG_EXTERNAL_AUTH_LOGOUT_SSO = 'https://login.cern.ch/adfs/ls/?wa=wsignout1.0'
        CFG_EXTERNAL_AUTHENTICATION = {CFG_EXTERNAL_AUTH_USING_SSO : (ea_sso.ExternalAuthSSO(), True)}
    else:
        import external_authentication_cern as ea_cern
        CFG_EXTERNAL_AUTHENTICATION = {"Local": (None, False), \
        "CERN": (ea_cern.ExternalAuthCern(), True)}

# default data for the add_default_settings function
# Note: by default the definition is set to deny any. This won't be a problem
# because userid directly connected with roles will still be allowed.
# roles
#           name          description          definition
DEF_ROLES = ((SUPERADMINROLE, 'superuser with all rights', 'deny any'),
             ('photoadmin',   'Photo collection administrator', 'deny any'),
             (WEBACCESSADMINROLE,     'WebAccess administrator', 'deny any'))

# users
# list of e-mail addresses
DEF_USERS = []

# actions
#            name                  desc     allowedkeywords   optional
DEF_ACTIONS = (
               ('cfgwebsearch',         'configure WebSearch',       '',              'no'),
               ('cfgbibformat',         'configure BibFormat',       '',              'no'),
               ('cfgwebsubmit',         'configure WebSubmit',       '',              'no'),
               ('runbibindex',          'run BibIndex',       '',              'no'),
               ('runbibupload',         'run BibUpload',       '',              'no'),
               ('runwebcoll',           'run webcoll',       'collection',    'yes'),
               ('runbibformat',         'run BibFormat',       'format',        'yes'),
               (WEBACCESSACTION,        'configure WebAccess',       '',              'no'),
               (DELEGATEADDUSERROLE,    'delegate subroles inside WebAccess',       'role',          'no'),
               (VIEWRESTRCOLL, 'view restricted collection', 'collection', 'no'),
               ('runbibtaskex',         'run BibTaskEx example',       '',              'no'),
               ('referee',                  'referee document type doctype/category categ', 'doctype,categ',    'yes'),
               ('submit',                   'use webSubmit',    'doctype,act',  'yes'),
               ('runbibrank',         'run BibRank',       '',              'no'),
               ('cfgbibrank',         'configure BibRank',       '',              'no'),
               ('cfgbibharvest',         'configure BibHarvest',       '',              'no'),
               ('runoaiharvest',         'run oaiharvest task',       '',              'no'),
               ('cfgwebcomment',         'configure WebComment',       '',              'no'),
               ('runoaiarchive',         'run oaiarchive task',       '',              'no'),
               ('runbibedit',         'run BibEdit',       '',              'no'),
               ('runwebstatadmin', 'run WebStadAdmin', '', 'no'),
               ('viewrestrdoc', 'view restricted document', 'status', 'no'),
              )

# authorizations
#          role              action          arglistid  optional   arguments
DEF_AUTHS = (
             (SUPERADMINROLE,    'cfgwebsearch',         -1,      0,       {}),
             (SUPERADMINROLE,    'cfgbibformat',         -1,      0,       {}),
             (SUPERADMINROLE,    'cfgwebsubmit',         -1,      0,       {}),
             (SUPERADMINROLE,    'runbibindex',          -1,      0,       {}),
             (SUPERADMINROLE,    'runbibupload',         -1,      0,       {}),
             (SUPERADMINROLE,    'runbibformat',         -1,      1,       {}),
             (SUPERADMINROLE,    WEBACCESSACTION,        -1,      0,       {}),
             ('photoadmin',      'runwebcoll',           -1,      0,       {'collection': 'Pictures'}),
             (WEBACCESSADMINROLE,WEBACCESSACTION,        -1,      0,       {}),
             (SUPERADMINROLE,    'runtaskex',            -1,      0,       {}),
             (SUPERADMINROLE,    'referee',         -1,      1,       {}),
             (SUPERADMINROLE,    'submit',         -1,      1,       {}),
             (SUPERADMINROLE,    'runbibrank',           -1,      0,       {}),
             (SUPERADMINROLE,    'cfgbibrank',           -1,      0,       {}),
             (SUPERADMINROLE,    'cfgbibharvest',           -1,      0,       {}),
             (SUPERADMINROLE,    'runoaiharvest',            -1,      0,       {}),
             (SUPERADMINROLE,    'cfgwebcomment',            -1,      0,       {}),
             (SUPERADMINROLE,    'runoaiarchive',            -1,      0,       {}),
             (SUPERADMINROLE,    'runbibedit',            -1,      0,       {}),
            )

_ = gettext_set_language('en')
CFG_ACC_ACTIVITIES_URLS = {
    'runbibedit' : (_("Run BibEdit"), "%s/admin/bibedit/bibeditadmin.py?ln=%%s" % weburl),
    'cfgbibformat' : (_("Configure BibFormat"), "%s/admin/bibformat/bibformatadmin.py?ln=%%s" % weburl),
    'cfgbibharvest' : (_("Configure BibHarvest"), "%s/admin/bibharvest/bibharvestadmin.py?ln=%%s" % weburl),
    'cfgoairepository' : (_("Configure OAI Repository"), "%s/admin/bibharvest/oaiarchiveadmin.py?ln=%%s" % weburl),
    'cfgbibindex' : (_("Configure BibIndex"), "%s/admin/bibindex/bibindexadmin.py?ln=%%s" % weburl),
    'cfgbibrank' : (_("Configure BibRank"), "%s/admin/bibrank/bibrankadmin.py?ln=%%s" % weburl),
    'cfgwebaccess' : (_("Configure WebAccess"), "%s/admin/webaccess/webaccessadmin.py?ln=%%s" % weburl),
    'cfgwebcomment' : (_("Configure WebComment"), "%s/admin/webcomment/webcommentadmin.py?ln=%%s" % weburl),
    'cfgwebsearch' : (_("Configure WebSearch"), "%s/admin/websearch/websearchadmin.py?ln=%%s" % weburl),
    'cfgwebsubmit' : (_("Configure WebSubmit"), "%s/admin/websubmit/websubmitadmin.py?ln=%%s" % weburl),
}

CFG_WEBACCESS_MSGS = {
                                0: 'Try to login with another account.' % (sweburl),
                                1: '
If you think this is not correct, please contact: %s' % (supportemail, supportemail),
                                2: '
If you have any questions, please write to %s' % (supportemail, supportemail),
                                3: 'Guest users are not allowed, please login.' % sweburl,
                                4: 'The site is temporarily closed for maintenance.  Please come back soon.',
                                5: 'Authorization failure',
                                6: '%s temporarily closed' % cdsname,
                                7: 'This functionality is temporarily closed due to server maintenance. Please use only the search engine in the meantime.',
                                8: 'Functionality temporarily closed'
        }

CFG_WEBACCESS_WARNING_MSGS = {
                                0: 'Authorization granted',
                                1: 'Error(1): You are not authorized to perform this action.',
                                2: 'Error(2): You are not authorized to perform any action.',
                                3: 'Error(3): The action %s does not exist.',
                                4: 'Error(4): Unexpected error occurred.',
                                5: 'Error(5): Missing mandatory keyword argument(s) for this action.',
                                6: 'Error(6): Guest accounts are not authorized to perform this action.',
                                7: 'Error(7): Not enough arguments, user ID and action name required.',
                                8: 'Error(8): Incorrect keyword argument(s) for this action.',
                                9: """Error(9): Account '%s' is not yet activated.""",
                               10: """Error(10): You were not authorized by the authentication method '%s'.""",
                               11: """Error(11): The selected login method '%s' is not the default method for this account, please try another one.""",
                               12: """Error(12): Selected login method '%s' does not exist.""",
                               13: """Error(13): Could not register '%s' account.""",
                               14: """Error(14): Could not login using '%s', because this user is unknown.""",
                               15: """Error(15): Could not login using your '%s' account, because you have introduced a wrong password.""",
                               16: """Error(16): External authentication troubles using '%s' (maybe temporary network problems).""",
                               17: """Error(17): You have not yet confirmed the email address for the '%s' authentication method.""",
                               18: """Error(18): The administrator has not yet activated your account for the '%s' authentication method.""",
                               19: """Error(19): The site is having troubles in sending you an email for confirming your email address. The error has been logged and will be taken care of as soon as possible."""
        }

<|endoftext|>


<commit_before># TODO:
# x change the algorithm definitions. Done for LilUCB only
# o explore the dashboard, see what you need to change
# ? modify the widgets?
import json
import numpy

import next.apps.SimpleTargetManager
import next.utils as utils
class ActiveRanking(object):
    def __init__(self,db):
        self.app_id = 'ActiveRanking'
        self.TargetManager = next.apps.SimpleTargetManager.SimpleTargetManager(db)

    def initExp(self, butler, init_algs, args):
        """
        This function is meant to store an additional components in the
        databse.

        In the implementation of two apps, DuelingBanditsPureExploration and
        PoolBasedTripletMDS, we only managed targets in this function. We
        stored the targets to the database than deleted the 'targets' key
        from args, replacing it with ``args['n']`` to
        represent a list of n targets. This is easier when doing numerical
        computation.

        Inputs
        ------
        exp_uid : The unique identifier to represent an experiment.
        args : The keys specified in the app specific YAML file in the
                   initExp section.
        butler : The wrapper for database writes. See next/apps/Butler.py for
                 more documentation.

        Returns
        -------
        args: The experiment data, potentially modified.
        """
        # TODO: change this in every app type coded thus far!
        if 'targetset' in args['targets'].keys():
            n = len(args['targets']['targetset'])
            self.TargetManager.set_targetset(butler.exp_uid, args['targets']['targetset'])
        else:
            n = args['targets']['n']
        args['n'] = n
        del args['targets']

        alg_data = {}
        algorithm_keys = ['n']
        for key in algorithm_keys:
            alg_data[key] = args[key]

        init_algs(alg_data)
        return args

    def getQuery(self, butler, alg, args):
        alg_response = alg({'participant_uid':args['participant_uid']})
        targets = [self.TargetManager.get_target_item(butler.exp_uid, alg_response[i])
                   for i in [0, 1]]

        targets_list = [{'target':targets[0],'label':'left'},
                        {'target':targets[1],'label':'right'}]

        #if targets[0]['target_id'] == targets[-1]['target_id']:
        #    targets_list[0]['flag'] = 1
        #    targets_list[1]['flag'] = 0
        #else:
        #    targets_list[0]['flag'] = 0
        #    targets_list[1]['flag'] = 1

        return_dict = {'target_indices':targets_list}

        experiment_dict = butler.experiment.get()

        #if 'labels' in experiment_dict['args']['rating_scale']:
            #labels = experiment_dict['args']['rating_scale']['labels']
            #return_dict.update({'labels':labels})

        if 'context' in experiment_dict['args'] and 'context_type' in experiment_dict['args']:
            return_dict.update({'context':experiment_dict['args']['context'],'context_type':experiment_dict['args']['context_type']})

        return return_dict

    def processAnswer(self, butler, alg, args):
        query = butler.queries.get(uid=args['query_uid'])
        targets = query['target_indices']
        left_id = target[0]['target']
        right_id = target[1]['target']

        winner_id = args['target_winner']
        butler.experiment.increment(key='num_reported_answers_for_' + query['alg_label'])

        alg({'left_id':left_id,
             'right_id':right_id,
             'winner_id':winner_id})
        return {'winner_id':winner_id}

    def getModel(self, butler, alg, args):
        scores, precisions = alg()
        ranks = (-numpy.array(scores)).argsort().tolist()
        n = len(scores)
        indexes = numpy.array(range(n))[ranks]
        scores = numpy.array(scores)[ranks]
        precisions = numpy.array(precisions)[ranks]
        ranks = range(n)

        targets = []
        for index in range(n):
          targets.append( {'index':indexes[index],
                           'target':self.TargetManager.get_target_item(butler.exp_uid, indexes[index]),
                           'rank':ranks[index],
                           'score':scores[index],
                           'precision':precisions[index]} )
        num_reported_answers = butler.experiment.get('num_reported_answers')
        return {'targets': targets, 'num_reported_answers':num_reported_answers}

<commit_msg>Update ActiveRanking.py<commit_after># TODO:
# x change the algorithm definitions. Done for LilUCB only
# o explore the dashboard, see what you need to change
# ? modify the widgets?
import json
import numpy

import next.apps.SimpleTargetManager
import next.utils as utils
class ActiveRanking(object):
    def __init__(self,db):
        self.app_id = 'ActiveRanking'
        self.TargetManager = next.apps.SimpleTargetManager.SimpleTargetManager(db)

    def initExp(self, butler, init_algs, args):
        """
        This function is meant to store an additional components in the
        databse.

        In the implementation of two apps, DuelingBanditsPureExploration and
        PoolBasedTripletMDS, we only managed targets in this function. We
        stored the targets to the database than deleted the 'targets' key
        from args, replacing it with ``args['n']`` to
        represent a list of n targets. This is easier when doing numerical
        computation.

        Inputs
        ------
        exp_uid : The unique identifier to represent an experiment.
        args : The keys specified in the app specific YAML file in the
                   initExp section.
        butler : The wrapper for database writes. See next/apps/Butler.py for
                 more documentation.

        Returns
        -------
        args: The experiment data, potentially modified.
        """
        # TODO: change this in every app type coded thus far!
        if 'targetset' in args['targets'].keys():
            n = len(args['targets']['targetset'])
            self.TargetManager.set_targetset(butler.exp_uid, args['targets']['targetset'])
        else:
            n = args['targets']['n']
        args['n'] = n
        del args['targets']

        alg_data = {}
        algorithm_keys = ['n']
        for key in algorithm_keys:
            alg_data[key] = args[key]

        init_algs(alg_data)
        return args

    def getQuery(self, butler, alg, args):
        alg_response = alg({'participant_uid':args['participant_uid']})
        targets = [self.TargetManager.get_target_item(butler.exp_uid, alg_response[i])
                   for i in [0, 1]]

        targets_list = [{'target':targets[0],'label':'left'},
                        {'target':targets[1],'label':'right'}]

        #if targets[0]['target_id'] == targets[-1]['target_id']:
        #    targets_list[0]['flag'] = 1
        #    targets_list[1]['flag'] = 0
        #else:
        #    targets_list[0]['flag'] = 0
        #    targets_list[1]['flag'] = 1

        return_dict = {'target_indices':targets_list, 'quicksort_id':alg_response[3]}

        experiment_dict = butler.experiment.get()

        #if 'labels' in experiment_dict['args']['rating_scale']:
            #labels = experiment_dict['args']['rating_scale']['labels']
            #return_dict.update({'labels':labels})

        if 'context' in experiment_dict['args'] and 'context_type' in experiment_dict['args']:
            return_dict.update({'context':experiment_dict['args']['context'],'context_type':experiment_dict['args']['context_type']})

        return return_dict

    def processAnswer(self, butler, alg, args):
        query = butler.queries.get(uid=args['query_uid'])
        targets = query['target_indices']
        left_id = target[0]['target']
        right_id = target[1]['target']
        quicksort_id = query['quicksort_id']
        winner_id = args['target_winner']
        butler.experiment.increment(key='num_reported_answers_for_' + query['alg_label'])

        alg({'left_id':left_id,
             'right_id':right_id,
             'winner_id':winner_id})
        return {'winner_id':winner_id, 'quicksort_id':quicksort_id}

    def getModel(self, butler, alg, args):
        scores, precisions = alg()
        ranks = (-numpy.array(scores)).argsort().tolist()
        n = len(scores)
        indexes = numpy.array(range(n))[ranks]
        scores = numpy.array(scores)[ranks]
        precisions = numpy.array(precisions)[ranks]
        ranks = range(n)

        targets = []
        for index in range(n):
          targets.append( {'index':indexes[index],
                           'target':self.TargetManager.get_target_item(butler.exp_uid, indexes[index]),
                           'rank':ranks[index],
                           'score':scores[index],
                           'precision':precisions[index]} )
        num_reported_answers = butler.experiment.get('num_reported_answers')
        return {'targets': targets, 'num_reported_answers':num_reported_answers}

<|endoftext|>