<commit_before>"""Generate the storage of NARR 3 hourly products"""
from __future__ import print_function
import datetime
import sys

import numpy as np
import pygrib
from pyiem.util import ncopen

# This exists on dev laptop :/
TEMPLATE_FN = ("/mesonet/ARCHIVE/data/1980/01/01/model/"
               "NARR/apcp_198001010000.grib")
BASEDIR = "/mesonet/data/iemre"

def init_year(ts):
    """
    Create a new NetCDF file for a year of our specification!
    """
    # Load up the example grib file to base our file on
    grbs = pygrib.open(TEMPLATE_FN)
    grb = grbs[1]
    # grid shape is y, x
    lats, lons = grb.latlons()

    fp = "%s/%s_narr.nc" % (BASEDIR, ts.year)
    nc = ncopen(fp, 'w')
    nc.title = "IEM Packaged NARR for %s" % (ts.year,)
    nc.platform = "Grided Reanalysis"
    nc.description = "NARR Data"
    nc.institution = "Iowa State University, Ames, IA, USA"
    nc.source = "Iowa Environmental Mesonet"
    nc.project_id = "IEM"
    nc.realization = 1
    nc.Conventions = 'CF-1.0'
    nc.contact = "Daryl Herzmann, akrherz@iastate.edu, 515-294-5978"
    nc.history = ("%s Generated"
                  ) % (datetime.datetime.now().strftime("%d %B %Y"),)
    nc.comment = "No Comment at this time"

    # Setup Dimensions
    nc.createDimension('x', lats.shape[1])
    nc.createDimension('y', lats.shape[0])
    nc.createDimension('bnds', 2)
    ts2 = datetime.datetime(ts.year + 1, 1, 1)
    days = (ts2 - ts).days
    print('Year %s has %s days' % (ts.year, days))
    nc.createDimension('time', int(days) * 8)

    # Setup Coordinate Variables
    lat = nc.createVariable('lat', np.float, ('y', 'x'))
    lat.units = "degrees_north"
    lat.long_name = "Latitude"
    lat.standard_name = "latitude"
    lat.axis = "Y"
    lat[:] = lats

    lon = nc.createVariable('lon', np.float, ('y', 'x'))
    lon.units = "degrees_east"
    lon.long_name = "Longitude"
    lon.standard_name = "longitude"
    lon.axis = "X"
    lon[:] = lons

    tm = nc.createVariable('time', np.float, ('time',))
    tm.units = "Hours since %s-01-01 00:00:0.0" % (ts.year,)
    tm.long_name = "Time"
    tm.standard_name = "time"
    tm.axis = "T"
    tm.calendar = "gregorian"
    tm[:] = np.arange(0, int(days) * 8)

    tmb = nc.createVariable('time_bnds', 'd', ('time', 'bnds'))
    tmb[:, 0] = np.arange(0, int(days) * 8) - 1
    tmb[:, 1] = np.arange(0, int(days) * 8)

    # 3 hour accum, 0 to 655.35
    apcp = nc.createVariable('apcp', np.ushort, ('time', 'y', 'x'),
                             fill_value=65536)
    apcp.scale_factor = 0.01
    apcp.add_offset = 0.
    apcp.units = 'mm'
    apcp.long_name = 'Precipitation'
    apcp.standard_name = 'Precipitation'
    apcp.coordinates = "lon lat"
    apcp.description = "Precipitation accumulation for the previous 3 hours"
    apcp.bounds = 'time_bnds'

    nc.close()

if __name__ == '__main__':
    init_year(datetime.datetime(int(sys.argv[1]), 1, 1))
<commit_msg>fix time bug with NARR netcdf initialization<commit_after>"""Generate the storage of NARR 3 hourly products"""
from __future__ import print_function
import datetime
import sys

import numpy as np
import pygrib
from pyiem.util import ncopen

# This exists on dev laptop :/
TEMPLATE_FN = ("/mesonet/ARCHIVE/data/1980/01/01/model/"
               "NARR/apcp_198001010000.grib")
BASEDIR = "/mesonet/data/iemre"

def init_year(ts):
    """
    Create a new NetCDF file for a year of our specification!
    """
    # Load up the example grib file to base our file on
    grbs = pygrib.open(TEMPLATE_FN)
    grb = grbs[1]
    # grid shape is y, x
    lats, lons = grb.latlons()

    fp = "%s/%s_narr.nc" % (BASEDIR, ts.year)
    nc = ncopen(fp, 'w')
    nc.title = "IEM Packaged NARR for %s" % (ts.year,)
    nc.platform = "Grided Reanalysis"
    nc.description = "NARR Data"
    nc.institution = "Iowa State University, Ames, IA, USA"
    nc.source = "Iowa Environmental Mesonet"
    nc.project_id = "IEM"
    nc.realization = 1
    nc.Conventions = 'CF-1.0'
    nc.contact = "Daryl Herzmann, akrherz@iastate.edu, 515-294-5978"
    nc.history = ("%s Generated"
                  ) % (datetime.datetime.now().strftime("%d %B %Y"),)
    nc.comment = "No Comment at this time"

    # Setup Dimensions
    nc.createDimension('x', lats.shape[1])
    nc.createDimension('y', lats.shape[0])
    nc.createDimension('bnds', 2)
    ts2 = datetime.datetime(ts.year + 1, 1, 1)
    days = (ts2 - ts).days
    print('Year %s has %s days' % (ts.year, days))
    nc.createDimension('time', int(days) * 8)

    # Setup Coordinate Variables
    lat = nc.createVariable('lat', np.float, ('y', 'x'))
    lat.units = "degrees_north"
    lat.long_name = "Latitude"
    lat.standard_name = "latitude"
    lat.axis = "Y"
    lat[:] = lats

    lon = nc.createVariable('lon', np.float, ('y', 'x'))
    lon.units = "degrees_east"
    lon.long_name = "Longitude"
    lon.standard_name = "longitude"
    lon.axis = "X"
    lon[:] = lons

    tm = nc.createVariable('time', np.float, ('time',))
    tm.units = "Hours since %s-01-01 00:00:0.0" % (ts.year,)
    tm.long_name = "Time"
    tm.standard_name = "time"
    tm.axis = "T"
    tm.calendar = "gregorian"
    tm[:] = np.arange(0, int(days) * 8) * 3

    tmb = nc.createVariable('time_bnds', 'd', ('time', 'bnds'))
    tmb[:, 0] = np.arange(0, int(days) * 8) * 3 - 1
    tmb[:, 1] = np.arange(0, int(days) * 8) * 3

    # 3 hour accum, 0 to 655.35
    apcp = nc.createVariable('apcp', np.ushort, ('time', 'y', 'x'),
                             fill_value=65536)
    apcp.scale_factor = 0.01
    apcp.add_offset = 0.
    apcp.units = 'mm'
    apcp.long_name = 'Precipitation'
    apcp.standard_name = 'Precipitation'
    apcp.coordinates = "lon lat"
    apcp.description = "Precipitation accumulation for the previous 3 hours"
    apcp.bounds = 'time_bnds'

    nc.close()

if __name__ == '__main__':
    init_year(datetime.datetime(int(sys.argv[1]), 1, 1))
<|endoftext|>


<commit_before>import numpy as np
from PyQt4.QtGui import QPalette
import vispy.scene as scene
from vispy.scene.cameras.base_camera import BaseCamera

class VisPyCanvas(scene.SceneCanvas):

    def __init__(self, config=None):

        scene.SceneCanvas.__init__(self, keys=None, config=config)
        self.unfreeze()

        back_color = str(QPalette().color(QPalette.Window).name())

        self.central_widget.bgcolor = back_color
        self.central_widget.border_color = back_color

        grid = self.central_widget.add_grid(margin=10)
        grid.spacing = 0

        top_padding = grid.add_widget(row=0, col=0, col_span=2)
        top_padding.height_max = 24

        yaxis = scene.AxisWidget(orientation='left', axis_color='black', text_color='black', font_size=12)
        yaxis.width_max = 60
        grid.add_widget(yaxis, row=1, col=0)

        xaxis = scene.AxisWidget(orientation='bottom', axis_color='black', text_color='black', font_size=12)
        xaxis.height_max = 40
        grid.add_widget(xaxis, row=2, col=1)

        right_padding = grid.add_widget(row=0, col=2, row_span=2)
        right_padding.width_max = 24

        view = grid.add_view(row=1, col=1, border_color='black', bgcolor='white')
        view.camera = Camera(aspect=1)

        grid1 = scene.GridLines(parent=view.scene, color='gray')
        grid1.set_gl_state(depth_test=False)

        xaxis.link_view(view)
        yaxis.link_view(view)

        # shapes = scene.Line(parent=view.scene)
        # view.add(shapes)

        self.grid = grid1
        self.view = view
        self.freeze()

        # self.measure_fps()

    def translate_coords(self, pos):
        tr = self.grid.get_transform('canvas', 'visual')
        return tr.map(pos)

class Camera(scene.PanZoomCamera):

    def __init__(self, **kwargs):
        super(Camera, self).__init__(**kwargs)

        self.minimum_scene_size = 0.01
        self.maximum_scene_size = 10000

    def zoom(self, factor, center=None):
        center = center if (center is not None) else self.center
        super(Camera, self).zoom(factor, center)

    def viewbox_mouse_event(self, event):
        """
        The SubScene received a mouse event; update transform
        accordingly.

        Parameters
        ----------
        event : instance of Event
            The event.
        """
        if event.handled or not self.interactive:
            return

        # Scrolling
        BaseCamera.viewbox_mouse_event(self, event)

        if event.type == 'mouse_wheel':
            center = self._scene_transform.imap(event.pos)
            scale = (1 + self.zoom_factor) ** (-event.delta[1] * 30)
            self.limited_zoom(scale, center)
            event.handled = True

        elif event.type == 'mouse_move':
            if event.press_event is None:
                return

            modifiers = event.mouse_event.modifiers
            p1 = event.mouse_event.press_event.pos
            p2 = event.mouse_event.pos

            if event.button in [2, 3] and not modifiers:
                # Translate
                p1 = np.array(event.last_event.pos)[:2]
                p2 = np.array(event.pos)[:2]
                p1s = self._transform.imap(p1)
                p2s = self._transform.imap(p2)
                self.pan(p1s-p2s)
                event.handled = True
            elif event.button in [2, 3] and 'Shift' in modifiers:
                # Zoom
                p1c = np.array(event.last_event.pos)[:2]
                p2c = np.array(event.pos)[:2]
                scale = ((1 + self.zoom_factor) **
                         ((p1c-p2c) * np.array([1, -1])))
                center = self._transform.imap(event.press_event.pos[:2])
                self.limited_zoom(scale, center)
                event.handled = True
            else:
                event.handled = False
        elif event.type == 'mouse_press':
            # accept the event if it is button 1 or 2.
            # This is required in order to receive future events
            event.handled = event.button in [1, 2, 3]
        else:
            event.handled = False

    def limited_zoom(self, scale, center):

        try:
            zoom_in = scale[1] < 1
        except IndexError:
            zoom_in = scale < 1

        if (not zoom_in and self.rect.width < self.maximum_scene_size) \
                or (zoom_in and self.rect.width > self.minimum_scene_size):
            self.zoom(scale, center)
<commit_msg>Mouse move events are limited now<commit_after>import numpy as np
from PyQt4.QtGui import QPalette
import vispy.scene as scene
from vispy.scene.cameras.base_camera import BaseCamera
import time

class VisPyCanvas(scene.SceneCanvas):

    def __init__(self, config=None):

        scene.SceneCanvas.__init__(self, keys=None, config=config)
        self.unfreeze()

        back_color = str(QPalette().color(QPalette.Window).name())

        self.central_widget.bgcolor = back_color
        self.central_widget.border_color = back_color

        grid = self.central_widget.add_grid(margin=10)
        grid.spacing = 0

        top_padding = grid.add_widget(row=0, col=0, col_span=2)
        top_padding.height_max = 24

        yaxis = scene.AxisWidget(orientation='left', axis_color='black', text_color='black', font_size=12)
        yaxis.width_max = 60
        grid.add_widget(yaxis, row=1, col=0)

        xaxis = scene.AxisWidget(orientation='bottom', axis_color='black', text_color='black', font_size=12)
        xaxis.height_max = 40
        grid.add_widget(xaxis, row=2, col=1)

        right_padding = grid.add_widget(row=0, col=2, row_span=2)
        right_padding.width_max = 24

        view = grid.add_view(row=1, col=1, border_color='black', bgcolor='white')
        view.camera = Camera(aspect=1)

        grid1 = scene.GridLines(parent=view.scene, color='gray')
        grid1.set_gl_state(depth_test=False)

        xaxis.link_view(view)
        yaxis.link_view(view)

        # shapes = scene.Line(parent=view.scene)
        # view.add(shapes)

        self.grid = grid1
        self.view = view
        self.freeze()

        self.measure_fps()

    def translate_coords(self, pos):
        tr = self.grid.get_transform('canvas', 'visual')
        return tr.map(pos)

class Camera(scene.PanZoomCamera):

    def __init__(self, **kwargs):
        super(Camera, self).__init__(**kwargs)

        self.minimum_scene_size = 0.01
        self.maximum_scene_size = 10000

        self.last_event = None
        self.last_time = 0

    def zoom(self, factor, center=None):
        center = center if (center is not None) else self.center
        super(Camera, self).zoom(factor, center)

    def viewbox_mouse_event(self, event):
        """
        The SubScene received a mouse event; update transform
        accordingly.

        Parameters
        ----------
        event : instance of Event
            The event.
        """
        if event.handled or not self.interactive:
            return

        # Limit mouse move events
        last_event = event.last_event
        t = time.time()
        if t - self.last_time > 0.015:
            self.last_time = t
            if self.last_event:
                last_event = self.last_event
                self.last_event = None
        else:
            if not self.last_event:
                self.last_event = last_event
            event.handled = True
            return

        # Scrolling
        BaseCamera.viewbox_mouse_event(self, event)

        if event.type == 'mouse_wheel':
            center = self._scene_transform.imap(event.pos)
            scale = (1 + self.zoom_factor) ** (-event.delta[1] * 30)
            self.limited_zoom(scale, center)
            event.handled = True

        elif event.type == 'mouse_move':
            if event.press_event is None:
                return

            modifiers = event.mouse_event.modifiers

            if event.button in [2, 3] and not modifiers:
                # Translate
                p1 = np.array(last_event.pos)[:2]
                p2 = np.array(event.pos)[:2]
                p1s = self._transform.imap(p1)
                p2s = self._transform.imap(p2)
                self.pan(p1s-p2s)
                event.handled = True
            elif event.button in [2, 3] and 'Shift' in modifiers:
                # Zoom
                p1c = np.array(last_event.pos)[:2]
                p2c = np.array(event.pos)[:2]
                scale = ((1 + self.zoom_factor) **
                         ((p1c-p2c) * np.array([1, -1])))
                center = self._transform.imap(event.press_event.pos[:2])
                self.limited_zoom(scale, center)
                event.handled = True
            else:
                event.handled = False
        elif event.type == 'mouse_press':
            # accept the event if it is button 1 or 2.
            # This is required in order to receive future events
            event.handled = event.button in [1, 2, 3]
        else:
            event.handled = False

    def limited_zoom(self, scale, center):

        try:
            zoom_in = scale[1] < 1
        except IndexError:
            zoom_in = scale < 1

        if (not zoom_in and self.rect.width < self.maximum_scene_size) \
                or (zoom_in and self.rect.width > self.minimum_scene_size):
            self.zoom(scale, center)
<|endoftext|>


<commit_before>from fjagepy import *
from fjagepy import Message as _Message
from fjagepy import MessageClass as _MessageClass
from fjagepy import Performative as _Performative
from fjagepy import Gateway as _Gateway
from fjagepy import AgentID as _AgentID
from types import MethodType as _mt
from warnings import warn as _warn

_ParameterReq = _MessageClass('org.arl.unet.ParameterReq')
_ParameterRsp = _MessageClass('org.arl.unet.ParameterRsp')

def _repr_pretty_(self, p, cycle):
    if cycle:
        p.text('...')
    elif self.perf is not None:
        p.text(self.perf)

setattr(_Message, '_repr_pretty_', _repr_pretty_)

def _short(p):
    return p.split('.')[-1]

def _value(v):
    # TODO: support complex objects
    if isinstance(v, dict):
        if 'clazz' in v:
            if v['clazz'] == 'java.util.Date':
                return v['data']
            if v['clazz'] == 'java.util.ArrayList':
                return v['data']
            p = _Prettify()
            p.__dict__.update(v)
            return p
        if 'data' in v:
            return v['data']
    return v

class _Prettify:

    def __init__(self, **kwargs):
        self.__dict__.update(kwargs)

    def __repr__(self):
        return self.__dict__['clazz'] + '(...)'

class Services:
    NODE_INFO = 'org.arl.unet.Services.NODE_INFO'
    ADDRESS_RESOLUTION = 'org.arl.unet.Services.ADDRESS_RESOLUTION'
    DATAGRAM = 'org.arl.unet.Services.DATAGRAM'
    PHYSICAL = 'org.arl.unet.Services.PHYSICAL'
    RANGING = 'org.arl.unet.Services.RANGING'
    BASEBAND = 'org.arl.unet.Services.BASEBAND'
    LINK = 'org.arl.unet.Services.LINK'
    MAC = 'org.arl.unet.Services.MAC'
    ROUTING = 'org.arl.unet.Services.ROUTING'
    ROUTE_MAINTENANCE = 'org.arl.unet.Services.ROUTE_MAINTENANCE'
    TRANSPORT = 'org.arl.unet.Services.TRANSPORT'
    REMOTE = 'org.arl.unet.Services.REMOTE'
    STATE_MANAGER = 'org.arl.unet.Services.STATE_MANAGER'

class ParameterReq(_ParameterReq):

    def __init__(self, index=-1, **kwargs):
        super().__init__()
        self.index = index
        self.requests = []
        self.perf = _Performative.REQUEST
        self.__dict__.update(kwargs)

    def get(self, param):
        self.requests.append({'param': param});
        return self

    def set(self, param, value):
        self.requests.append({'param': param, 'value': value});
        return self

    def __str__(self):
        p = ' '.join([_short(str(request['param'])) + ':' + (str(request['value']) if 'value' in request else '?') for request in self.requests])
        return self.__class__.__name__ + ':' + self.perf + '[' + (('index:' + str(self.index)) if self.index > 0 else '') + p.strip() + ']'

    def _repr_pretty_(self, p, cycle):
        p.text(str(self) if not cycle else '...')

class ParameterRsp(_ParameterRsp):

    def __init__(self, **kwargs):
        super().__init__()
        self.index = -1
        self.values = dict()
        self.perf = _Performative.REQUEST
        self.__dict__.update(kwargs)

    def get(self, param):
        if 'param' in self.__dict__ and self.param == param:
            return _value(self.value)
        if 'values' in self.__dict__ and param in self.values:
            return _value(self.values[param])
        if 'param' in self.__dict__ and _short(self.param) == param:
            return _value(self.value)
        if 'values' not in self.__dict__:
            return None
        for v in self.values:
            if _short(v) == param:
                return _value(self.values[v])
        return None

    def parameters(self):
        if 'values' in self.__dict__:
            p = self.values.copy()
        else:
            p = {}
        if 'param' in self.__dict__:
            p[self.param] = self.value
        for k in p:
            if isinstance(p[k], dict):
                p[k] = _value(p[k])
        return p

    def __str__(self):
        p = ''
        if 'param' in self.__dict__:
            p += _short(str(self.param)) + ':' + str(_value(self.value)) + ' '
        if 'values' in self.__dict__ and len(self.values) > 0:
            p += ' '.join([_short(str(v)) + ':' + str(_value(self.values[v])) for v in self.values])
        return self.__class__.__name__ + ':' + self.perf + '[' + (('index:' + str(self.index)) if self.index > 0 else '') + p.strip() + ']'

    def _repr_pretty_(self, p, cycle):
        p.text(str(self) if not cycle else '...')

class AgentID(_AgentID):

    def __init__(self, gw, name, is_topic=False):
        self.is_topic = is_topic
        self.name = name
        self.index = -1
        self.gw = gw

    def send(self, msg):
        msg.recipient = self.name
        return self.gw.send(msg)

    def request(self, msg, timeout=1000):
        msg.recipient = self.name
        return self.gw.request(msg, timeout)

    def __lshift__(self, msg):
        return self.request(msg)

    def __getattr__(self, param):
        rsp = self.request(ParameterReq(index=self.index).get(param))
        if rsp is None:
            return None
        ursp = ParameterRsp()
        ursp.__dict__.update(rsp.__dict__)
        if 'value' in list(ursp.__dict__.keys()):
            return ursp.get(param)
        else:
            return None

    def __setattr__(self, param, value):
        if param in ['name', 'gw', 'is_topic', 'index']:
            self.__dict__[param] = value
            return value
        rsp = self.request(ParameterReq(index=self.index).set(param, value))
        if rsp is None:
            _warn('Could not set parameter ' + param)
            return None
        ursp = ParameterRsp()
        ursp.__dict__.update(rsp.__dict__)
        v = ursp.get(param)
        if v != value:
            _warn('Parameter ' + param + ' set to ' + str(v))
        return v

    def __getitem__(self, index):
        c = AgentID(self.gw, self.name)
        c.index = index
        return c

    def __str__(self):
        peer = self.gw.socket.getpeername()
        return self.name + ' on ' + peer[0] + ':' + str(peer[1])

    def _repr_pretty_(self, p, cycle):
        if cycle:
            p.text('...')
            return
        rsp = self.request(ParameterReq(index=self.index))
        if rsp is None:
            p.text(self.__str__())
            return
        ursp = ParameterRsp()
        ursp.__dict__.update(rsp.__dict__)
        params = ursp.parameters()
        oprefix = ''
        for param in sorted(params):
            pp = param.split('.')
            prefix = '.'.join(pp[:-1]) if len(pp) > 1 else ''
            if prefix != oprefix:
                oprefix = prefix
                p.text('\n[' + prefix + ']\n')
            p.text('  ' + pp[-1] + ' = ' + str(params[param]) + '\n')

class UnetGateway(_Gateway):

    def __init__(self, hostname, port=1100, name=None):
        super().__init__(hostname, port, name)

    def topic(self, name):
        a = super().topic(name)
        return AgentID(self, a.name, a.is_topic)

    def agent(self, name):
        return AgentID(self, name)

    def agentForService(self, service):
        a = super().agentForService(service)
        if a is not None:
            if isinstance(a, str):
                a = AgentID(self, a)
            else:
                a = AgentID(self, a.name, a.is_topic)
        return a

    def agentsForService(self, service):
        a = super().agentsForService(service)
        if a is not None:
            for j in range(len(a)):
                if isinstance(a[j], str):
                    a[j] = AgentID(self, a[j])
                else:
                    a[j] = AgentID(self, a[j].name)
        return a

    def agent(self, name):
        return AgentID(self, name)
<commit_msg>refactor(unetpy): changed the prettifier class name<commit_after>from fjagepy import *
from fjagepy import Message as _Message
from fjagepy import MessageClass as _MessageClass
from fjagepy import Performative as _Performative
from fjagepy import Gateway as _Gateway
from fjagepy import AgentID as _AgentID
from types import MethodType as _mt
from warnings import warn as _warn

_ParameterReq = _MessageClass('org.arl.unet.ParameterReq')
_ParameterRsp = _MessageClass('org.arl.unet.ParameterRsp')

def _repr_pretty_(self, p, cycle):
    if cycle:
        p.text('...')
    elif self.perf is not None:
        p.text(self.perf)

setattr(_Message, '_repr_pretty_', _repr_pretty_)

def _short(p):
    return p.split('.')[-1]

def _value(v):
    # TODO: support complex objects
    if isinstance(v, dict):
        if 'clazz' in v:
            if v['clazz'] == 'java.util.Date':
                return v['data']
            if v['clazz'] == 'java.util.ArrayList':
                return v['data']
            p = _GenericObject()
            p.__dict__.update(v)
            return p
        if 'data' in v:
            return v['data']
    return v

class _GenericObject:

    def __init__(self, **kwargs):
        self.__dict__.update(kwargs)

    def __repr__(self):
        return self.__dict__['clazz'] + '(...)'

class Services:
    NODE_INFO = 'org.arl.unet.Services.NODE_INFO'
    ADDRESS_RESOLUTION = 'org.arl.unet.Services.ADDRESS_RESOLUTION'
    DATAGRAM = 'org.arl.unet.Services.DATAGRAM'
    PHYSICAL = 'org.arl.unet.Services.PHYSICAL'
    RANGING = 'org.arl.unet.Services.RANGING'
    BASEBAND = 'org.arl.unet.Services.BASEBAND'
    LINK = 'org.arl.unet.Services.LINK'
    MAC = 'org.arl.unet.Services.MAC'
    ROUTING = 'org.arl.unet.Services.ROUTING'
    ROUTE_MAINTENANCE = 'org.arl.unet.Services.ROUTE_MAINTENANCE'
    TRANSPORT = 'org.arl.unet.Services.TRANSPORT'
    REMOTE = 'org.arl.unet.Services.REMOTE'
    STATE_MANAGER = 'org.arl.unet.Services.STATE_MANAGER'

class ParameterReq(_ParameterReq):

    def __init__(self, index=-1, **kwargs):
        super().__init__()
        self.index = index
        self.requests = []
        self.perf = _Performative.REQUEST
        self.__dict__.update(kwargs)

    def get(self, param):
        self.requests.append({'param': param});
        return self

    def set(self, param, value):
        self.requests.append({'param': param, 'value': value});
        return self

    def __str__(self):
        p = ' '.join([_short(str(request['param'])) + ':' + (str(request['value']) if 'value' in request else '?') for request in self.requests])
        return self.__class__.__name__ + ':' + self.perf + '[' + (('index:' + str(self.index)) if self.index > 0 else '') + p.strip() + ']'

    def _repr_pretty_(self, p, cycle):
        p.text(str(self) if not cycle else '...')

class ParameterRsp(_ParameterRsp):

    def __init__(self, **kwargs):
        super().__init__()
        self.index = -1
        self.values = dict()
        self.perf = _Performative.REQUEST
        self.__dict__.update(kwargs)

    def get(self, param):
        if 'param' in self.__dict__ and self.param == param:
            return _value(self.value)
        if 'values' in self.__dict__ and param in self.values:
            return _value(self.values[param])
        if 'param' in self.__dict__ and _short(self.param) == param:
            return _value(self.value)
        if 'values' not in self.__dict__:
            return None
        for v in self.values:
            if _short(v) == param:
                return _value(self.values[v])
        return None

    def parameters(self):
        if 'values' in self.__dict__:
            p = self.values.copy()
        else:
            p = {}
        if 'param' in self.__dict__:
            p[self.param] = self.value
        for k in p:
            if isinstance(p[k], dict):
                p[k] = _value(p[k])
        return p

    def __str__(self):
        p = ''
        if 'param' in self.__dict__:
            p += _short(str(self.param)) + ':' + str(_value(self.value)) + ' '
        if 'values' in self.__dict__ and len(self.values) > 0:
            p += ' '.join([_short(str(v)) + ':' + str(_value(self.values[v])) for v in self.values])
        return self.__class__.__name__ + ':' + self.perf + '[' + (('index:' + str(self.index)) if self.index > 0 else '') + p.strip() + ']'

    def _repr_pretty_(self, p, cycle):
        p.text(str(self) if not cycle else '...')

class AgentID(_AgentID):

    def __init__(self, gw, name, is_topic=False):
        self.is_topic = is_topic
        self.name = name
        self.index = -1
        self.gw = gw

    def send(self, msg):
        msg.recipient = self.name
        return self.gw.send(msg)

    def request(self, msg, timeout=1000):
        msg.recipient = self.name
        return self.gw.request(msg, timeout)

    def __lshift__(self, msg):
        return self.request(msg)

    def __getattr__(self, param):
        rsp = self.request(ParameterReq(index=self.index).get(param))
        if rsp is None:
            return None
        ursp = ParameterRsp()
        ursp.__dict__.update(rsp.__dict__)
        if 'value' in list(ursp.__dict__.keys()):
            return ursp.get(param)
        else:
            return None

    def __setattr__(self, param, value):
        if param in ['name', 'gw', 'is_topic', 'index']:
            self.__dict__[param] = value
            return value
        rsp = self.request(ParameterReq(index=self.index).set(param, value))
        if rsp is None:
            _warn('Could not set parameter ' + param)
            return None
        ursp = ParameterRsp()
        ursp.__dict__.update(rsp.__dict__)
        v = ursp.get(param)
        if v != value:
            _warn('Parameter ' + param + ' set to ' + str(v))
        return v

    def __getitem__(self, index):
        c = AgentID(self.gw, self.name)
        c.index = index
        return c

    def __str__(self):
        peer = self.gw.socket.getpeername()
        return self.name + ' on ' + peer[0] + ':' + str(peer[1])

    def _repr_pretty_(self, p, cycle):
        if cycle:
            p.text('...')
            return
        rsp = self.request(ParameterReq(index=self.index))
        if rsp is None:
            p.text(self.__str__())
            return
        ursp = ParameterRsp()
        ursp.__dict__.update(rsp.__dict__)
        params = ursp.parameters()
        oprefix = ''
        for param in sorted(params):
            pp = param.split('.')
            prefix = '.'.join(pp[:-1]) if len(pp) > 1 else ''
            if prefix != oprefix:
                oprefix = prefix
                p.text('\n[' + prefix + ']\n')
            p.text('  ' + pp[-1] + ' = ' + str(params[param]) + '\n')

class UnetGateway(_Gateway):

    def __init__(self, hostname, port=1100, name=None):
        super().__init__(hostname, port, name)

    def topic(self, name):
        a = super().topic(name)
        return AgentID(self, a.name, a.is_topic)

    def agent(self, name):
        return AgentID(self, name)

    def agentForService(self, service):
        a = super().agentForService(service)
        if a is not None:
            if isinstance(a, str):
                a = AgentID(self, a)
            else:
                a = AgentID(self, a.name, a.is_topic)
        return a

    def agentsForService(self, service):
        a = super().agentsForService(service)
        if a is not None:
            for j in range(len(a)):
                if isinstance(a[j], str):
                    a[j] = AgentID(self, a[j])
                else:
                    a[j] = AgentID(self, a[j].name)
        return a

    def agent(self, name):
        return AgentID(self, name)
<|endoftext|>


<commit_before>from ffmpeg import FFmpeg, Sound

class Application(object):
    app_config = None
    @staticmethod
    def run(config):
        Application.app_config = config
        print "app is running"
        ffmpeg = FFmpeg(config.FFMPEG_PATH)
        sound = Sound('/Users/ihor-pyvovarnyk/Documents/Workspace/oae-sound-processing-tool/resources/sound.wav')
        ffmpeg.convert(sound, 'mp3')
<commit_msg>Changed app instance code for test updated ffmpeg package<commit_after>import sys

from ffmpeg import FFmpeg

class Application(object):
    app_config = None
    @staticmethod
    def run(config):
        Application.app_config = config
        print "app is running"
        ffmpeg = FFmpeg(config.FFMPEG_PATH)
        ffmpeg.test()
<|endoftext|>


<commit_before>#!/usr/bin/env python3

from PWM import PWMController
from KeyDispatcher import KeyDispatcher
from Display import Display
from DataLogger import SQLiteLogger
import sys
import time

DEVICE = "PWM/Servo"

class Handler:
    def __init__(self, logger, pwm, channel):
        self.logger = logger
        self.pwm = pwm
        self.channel = channel

        self.frequency = 60
        self.on = 0
        self.off = 512
        self.pwm.set_frequency(self.frequency)
        self.pwm.set_pwm(self.channel, self.on, self.off)

        self.logger.log(DEVICE, "running", 1)

    def decrease_frequency(self):
        if self.frequency > 40:
            self.frequency -= 1
            self.pwm.set_frequency(self.frequency)
            self.logger.log(DEVICE, "frequency", self.frequency)

        return True

    def increase_frequency(self):
        if self.frequency < 1000:
            self.frequency += 1
            self.pwm.set_frequency(self.frequency)
            self.logger.log(DEVICE, "frequency", self.frequency)

        return True

    def decrease_on(self):
        if self.on > 0:
            self.on -= 1
            self.pwm.set_pwm(self.channel, self.on, self.off)
            self.logger.log(DEVICE, "on", self.on)

        return True

    def increase_on(self):
        if self.on < 4095:
            self.on += 1
            self.pwm.set_pwm(self.channel, self.on, self.off)
            self.logger.log(DEVICE, "on", self.on)

        return True

    def decrease_off(self):
        if self.off > 0:
            self.off -= 1
            self.pwm.set_pwm(self.channel, self.on, self.off)
            self.logger.log(DEVICE, "off", self.off)

        return True

    def increase_off(self):
        if self.off < 4095:
            self.off += 1
            self.pwm.set_pwm(self.channel, self.on, self.off)
            self.logger.log(DEVICE, "off", self.off)

        return True

    def quit(self):
        self.logger.log(DEVICE, "running", 0)
        return False

    def get_properties(self):
        return [
            "channel",
            "frequency",
            "on",
            "off"
        ]

    def get_data(self):
        now = time.time()

        return [
            (now, "channel", self.channel),
            (now, "frequency", self.frequency),
            (now, "on", self.on),
            (now, "off", self.off)
        ]

def update(display, handler):
    values = {}

    for item in handler.get_data():
        values[item[1]] = item[2]

    display.show_properties(values, handler.get_properties())

if len(sys.argv) == 2:
    channel = int(sys.argv[1])
else:
    channel = 0

with KeyDispatcher() as dispatcher, SQLiteLogger() as logger:
    pwm = PWMController()

    # setup key handlers
    # TODO: get name and channel from command-line
    handler = Handler(logger, pwm, channel)

    dispatcher.add("[", handler, "decrease_frequency")
    dispatcher.add("]", handler, "increase_frequency")
    dispatcher.add("_", handler, "decrease_on")
    dispatcher.add("+", handler, "increase_on")
    dispatcher.add("-", handler, "decrease_off")
    dispatcher.add("=", handler, "increase_off")
    dispatcher.add("q", handler, "quit")

    # setup display and start processing key presses
    display = Display(DEVICE, "[_/+] start [-/=] end [[/]] frequency [q]uit")

    update(display, handler)

    while dispatcher.process_key():
        update(display, handler)
<commit_msg>Add support for multiple PWM devices<commit_after>#!/usr/bin/env python3

from PWM import PWMController
from KeyDispatcher import KeyDispatcher
from Display import Display
from DataLogger import SQLiteLogger
import time

DEVICE = "PWM/Servo"

class Device:
    def __init__(self, name, channel, on, off):
        self.name = name
        self.channel = channel
        self.on = on
        self.off = off

class Handler:
    def __init__(self, logger, pwm):
        self.logger = logger
        self.pwm = pwm
        self.devices = []
        self.current_index = 0
        self.frequency = 60
        self.pwm.set_frequency(self.frequency)
        self.logger.log(DEVICE, "running", 1)

    def add_device(self, name, channel, on, off):
        device = Device(name, channel, on, off)
        self.devices.append(device)
        self.pwm.set_pwm(device.channel, device.on, device.off)

    def previous_device(self):
        self.current_index = (self.current_index - 1) % len(self.devices)
        return True

    def next_device(self):
        self.current_index = (self.current_index + 1) % len(self.devices)
        return True

    def decrease_frequency(self):
        if self.frequency > 40:
            self.frequency -= 1
            self.pwm.set_frequency(self.frequency)
            self.logger.log(DEVICE, "frequency", self.frequency)

        return True

    def increase_frequency(self):
        if self.frequency < 1000:
            self.frequency += 1
            self.pwm.set_frequency(self.frequency)
            self.logger.log(DEVICE, "frequency", self.frequency)

        return True

    def decrease_on(self):
        device = self.devices[self.current_index]

        if device.on > 0:
            device.on -= 1
            self.pwm.set_pwm(device.channel, device.on, device.off)
            self.logger.log(device.name, "on", device.on)

        return True

    def increase_on(self):
        device = self.devices[self.current_index]

        if device.on < 4095:
            device.on += 1
            self.pwm.set_pwm(device.channel, device.on, device.off)
            self.logger.log(device.name, "on", device.on)

        return True

    def decrease_off(self):
        device = self.devices[self.current_index]

        if device.off > 0:
            device.off -= 1
            self.pwm.set_pwm(device.channel, device.on, device.off)
            self.logger.log(device.name, "off", device.off)

        return True

    def increase_off(self):
        device = self.devices[self.current_index]

        if device.off < 4095:
            device.off += 1
            self.pwm.set_pwm(device.channel, device.on, device.off)
            self.logger.log(device.name, "off", device.off)

        return True

    def quit(self):
        self.logger.log(DEVICE, "running", 0)
        return False

    def get_properties(self):
        return [
            "channel",
            "name",
            "frequency",
            "on",
            "off"
        ]

    def get_data(self):
        now = time.time()
        device = self.devices[self.current_index]

        return [
            (now, "channel", device.channel),
            (now, "name", device.name),
            (now, "frequency", self.frequency),
            (now, "on", device.on),
            (now, "off", device.off)
        ]

def update(display, handler):
    values = {}

    for item in handler.get_data():
        values[item[1]] = item[2]

    display.show_properties(values, handler.get_properties())

with KeyDispatcher() as dispatcher, SQLiteLogger() as logger:
    # setup PWM controller
    pwm = PWMController()
    pwm.add_device("PWM Light", 0, 0, 512)
    pwm.add_device("PWM Motor", 1, 0, 512)

    # setup key handlers
    # TODO: get name and channel from command-line
    handler = Handler(logger, pwm)

    dispatcher.add("p", handler, "previous_device")
    dispatcher.add("n", handler, "next_device")
    dispatcher.add("[", handler, "decrease_frequency")
    dispatcher.add("]", handler, "increase_frequency")
    dispatcher.add("_", handler, "decrease_on")
    dispatcher.add("+", handler, "increase_on")
    dispatcher.add("-", handler, "decrease_off")
    dispatcher.add("=", handler, "increase_off")
    dispatcher.add("q", handler, "quit")

    # setup display and start processing key presses
    display = Display(DEVICE, "[p]revious [n]ext [_/+] start [-/=] end [[/]] frequency [q]uit")

    update(display, handler)

    while dispatcher.process_key():
        update(display, handler)
<|endoftext|>


<commit_before>"""Interface converters for Keras 1 support in Keras 2.
"""
import six
import warnings

def raise_duplicate_arg_error(old_arg, new_arg):
    raise TypeError('For the `' + new_arg + '` argument, '
                    'the layer received both '
                    'the legacy keyword argument '
                    '`' + old_arg + '` and the Keras 2 keyword argument '
                    '`' + new_arg + '`. Stick with the latter!')

def convert_legacy_kwargs(layer_name,
                          args,
                          kwargs,
                          conversions,
                          converted=None):
    converted = converted or []
    for old_arg, new_arg in conversions:
        if old_arg in kwargs:
            if new_arg in kwargs:
                raise_duplicate_arg_error(old_arg, new_arg)
            arg_value = kwargs.pop(old_arg)
            kwargs[new_arg] = arg_value
            converted.append((new_arg, arg_value))

    if converted:
        signature = '`' + layer_name + '('
        for value in args:
            if isinstance(value, six.string_types):
                signature += '"' + value + '"'
            else:
                signature += str(value)
            signature += ', '
        for i, (name, value) in enumerate(converted):
            signature += name + '='
            if isinstance(value, six.string_types):
                signature += '"' + value + '"'
            else:
                signature += str(value)
            if i < len(converted) - 1:
                signature += ', '
        signature += ')`'
        warnings.warn('Update your `' + layer_name + '` layer call '
                      'to the Keras 2 API: ' + signature)
    return kwargs

def legacy_dense_support(func):
    """Function wrapper to convert the `Dense` constructor from Keras 1 to 2.

    # Arguments
        func: `__init__` method of `Dense`.

    # Returns
        A constructor conversion wrapper.
    """
    @six.wraps(func)
    def wrapper(*args, **kwargs):
        if len(args) > 2:
            # The first entry in `args` is `self`.
            raise TypeError('The `Dense` layer can have at most '
                            'one positional argument (the `units` argument).')

        # output_dim
        if 'output_dim' in kwargs:
            if len(args) > 1:
                raise TypeError('Got both a positional argument '
                                'and keyword argument for argument '
                                '`units` '
                                '(`output_dim` in the legacy interface).')
            if 'units' in kwargs:
                raise_duplicate_arg_error('output_dim', 'units')
            output_dim = kwargs.pop('output_dim')
            args = (args[0], output_dim)

        # Remaining kwargs.
        conversions = [
            ('init', 'kernel_initializer'),
            ('W_regularizer', 'kernel_regularizer'),
            ('b_regularizer', 'bias_regularizer'),
            ('W_constraint', 'kernel_constraint'),
            ('b_constraint', 'bias_constraint'),
            ('bias', 'use_bias'),
        ]
        kwargs = convert_legacy_kwargs('Dense',
                                       args[1:],
                                       kwargs,
                                       conversions)
        return func(*args, **kwargs)
    return wrapper

def legacy_dropout_support(func):
    """Function wrapper to convert the `Dropout` constructor from Keras 1 to 2.

    # Arguments
        func: `__init__` method of `Dropout`.

    # Returns
        A constructor conversion wrapper.
    """
    @six.wraps(func)
    def wrapper(*args, **kwargs):
        if len(args) > 2:
            # The first entry in `args` is `self`.
            raise TypeError('The `Dropout` layer can have at most '
                            'one positional argument (the `rate` argument).')

        # Convert `p` to `rate` if keyword arguement is used
        if 'p' in kwargs:
            if len(args) > 1:
                raise TypeError('Got both a positional argument '
                                'and keyword argument for argument '
                                '`rate` '
                                '(`p` in the legacy interface).')
            if 'rate' in kwargs:
                raise_duplicate_arg_error('p', 'rate')
            rate = kwargs.pop('p')
            args = (args[0], rate)
            signature = '`Dropout(' + str(args[1])
            for kwarg in kwargs:
                signature += ', ' + kwarg + '='
                if isinstance(kwargs[kwarg], six.string_types):
                    signature += ('"' + kwargs[kwarg] + '"')
                else:
                    signature += str(kwargs[kwarg])
            signature += ')`'
            warnings.warn('Update your `Dropout` layer call '
                          'to the Keras 2 API: ' + signature)

        return func(*args, **kwargs)
    return wrapper

def legacy_pooling1d_support(func):
    """Function wrapper to convert `MaxPooling1D` or `AvgPooling1D` constructor from Keras 1 to 2.

    # Arguments
        func: `__init__` method of `MaxPooling1D` or `AvgPooling1D`.

    # Returns
        A constructor conversion wrapper.
    """
    @six.wraps(func)
    def wrapper(*args, **kwargs):
        if len(args) > 2:
            # The first entry in `args` is `self`.
            raise TypeError(args[0].__name__ + ' layer can have at most '
                            'one positional argument (the `pool_size` argument).')

        # make sure that only keyword argument 'pool_size'(or pool_length' in the legacy interface)
        # can be also used as positional argument, which is keyword argument originally.
        if 'pool_length' in kwargs:
            if len(args) > 1:
                raise TypeError('Got both a positional argument '
                                'and keyword argument for argument '
                                '`pool_size` '
                                '(`pool_length` in the legacy interface).')

        elif 'pool_size' in kwargs:
            if len(args) > 1:
                raise TypeError('Got both a positional argument '
                                'and keyword argument for argument '
                                '`pool_size`. ')

        # Remaining kwargs.
        conversions = [
            ('pool_length', 'pool_size'),
            ('border_mode', 'padding'),
        ]
        kwargs = convert_legacy_kwargs(args[0].__name__,
                                       args[1:],
                                       kwargs,
                                       conversions)
        return func(*args, **kwargs)
    return wrapper
<commit_msg>Fix the *Pooling1D layer conversion interface (#5677)<commit_after>"""Interface converters for Keras 1 support in Keras 2.
"""
import six
import warnings

def raise_duplicate_arg_error(old_arg, new_arg):
    raise TypeError('For the `' + new_arg + '` argument, '
                    'the layer received both '
                    'the legacy keyword argument '
                    '`' + old_arg + '` and the Keras 2 keyword argument '
                    '`' + new_arg + '`. Stick with the latter!')

def convert_legacy_kwargs(layer_name,
                          args,
                          kwargs,
                          conversions,
                          converted=None):
    converted = converted or []
    for old_arg, new_arg in conversions:
        if old_arg in kwargs:
            if new_arg in kwargs:
                raise_duplicate_arg_error(old_arg, new_arg)
            arg_value = kwargs.pop(old_arg)
            kwargs[new_arg] = arg_value
            converted.append((new_arg, arg_value))

    if converted:
        signature = '`' + layer_name + '('
        for value in args:
            if isinstance(value, six.string_types):
                signature += '"' + value + '"'
            else:
                signature += str(value)
            signature += ', '
        for i, (name, value) in enumerate(converted):
            signature += name + '='
            if isinstance(value, six.string_types):
                signature += '"' + value + '"'
            else:
                signature += str(value)
            if i < len(converted) - 1:
                signature += ', '
        signature += ')`'
        warnings.warn('Update your `' + layer_name + '` layer call '
                      'to the Keras 2 API: ' + signature)
    return kwargs

def legacy_dense_support(func):
    """Function wrapper to convert the `Dense` constructor from Keras 1 to 2.

    # Arguments
        func: `__init__` method of `Dense`.

    # Returns
        A constructor conversion wrapper.
    """
    @six.wraps(func)
    def wrapper(*args, **kwargs):
        if len(args) > 2:
            # The first entry in `args` is `self`.
            raise TypeError('The `Dense` layer can have at most '
                            'one positional argument (the `units` argument).')

        # output_dim
        if 'output_dim' in kwargs:
            if len(args) > 1:
                raise TypeError('Got both a positional argument '
                                'and keyword argument for argument '
                                '`units` '
                                '(`output_dim` in the legacy interface).')
            if 'units' in kwargs:
                raise_duplicate_arg_error('output_dim', 'units')
            output_dim = kwargs.pop('output_dim')
            args = (args[0], output_dim)

        # Remaining kwargs.
        conversions = [
            ('init', 'kernel_initializer'),
            ('W_regularizer', 'kernel_regularizer'),
            ('b_regularizer', 'bias_regularizer'),
            ('W_constraint', 'kernel_constraint'),
            ('b_constraint', 'bias_constraint'),
            ('bias', 'use_bias'),
        ]
        kwargs = convert_legacy_kwargs('Dense',
                                       args[1:],
                                       kwargs,
                                       conversions)
        return func(*args, **kwargs)
    return wrapper

def legacy_dropout_support(func):
    """Function wrapper to convert the `Dropout` constructor from Keras 1 to 2.

    # Arguments
        func: `__init__` method of `Dropout`.

    # Returns
        A constructor conversion wrapper.
    """
    @six.wraps(func)
    def wrapper(*args, **kwargs):
        if len(args) > 2:
            # The first entry in `args` is `self`.
            raise TypeError('The `Dropout` layer can have at most '
                            'one positional argument (the `rate` argument).')

        # Convert `p` to `rate` if keyword arguement is used
        if 'p' in kwargs:
            if len(args) > 1:
                raise TypeError('Got both a positional argument '
                                'and keyword argument for argument '
                                '`rate` '
                                '(`p` in the legacy interface).')
            if 'rate' in kwargs:
                raise_duplicate_arg_error('p', 'rate')
            rate = kwargs.pop('p')
            args = (args[0], rate)
            signature = '`Dropout(' + str(args[1])
            for kwarg in kwargs:
                signature += ', ' + kwarg + '='
                if isinstance(kwargs[kwarg], six.string_types):
                    signature += ('"' + kwargs[kwarg] + '"')
                else:
                    signature += str(kwargs[kwarg])
            signature += ')`'
            warnings.warn('Update your `Dropout` layer call '
                          'to the Keras 2 API: ' + signature)

        return func(*args, **kwargs)
    return wrapper

def legacy_pooling1d_support(func):
    """Function wrapper to convert `MaxPooling1D` or `AvgPooling1D` constructor from Keras 1 to 2.

    # Arguments
        func: `__init__` method of `MaxPooling1D` or `AvgPooling1D`.

    # Returns
        A constructor conversion wrapper.
    """
    @six.wraps(func)
    def wrapper(*args, **kwargs):
        if len(args) > 2:
            # The first entry in `args` is `self`.
            raise TypeError(args[0].__class__.__name__ + ' layer can have at most '
                            'one positional argument (the `pool_size` argument).')

        # make sure that only keyword argument 'pool_size'(or pool_length' in the legacy interface)
        # can be also used as positional argument, which is keyword argument originally.
        if 'pool_length' in kwargs:
            if len(args) > 1:
                raise TypeError('Got both a positional argument '
                                'and keyword argument for argument '
                                '`pool_size` '
                                '(`pool_length` in the legacy interface).')

        elif 'pool_size' in kwargs:
            if len(args) > 1:
                raise TypeError('Got both a positional argument '
                                'and keyword argument for argument '
                                '`pool_size`. ')

        # Remaining kwargs.
        conversions = [
            ('pool_length', 'pool_size'),
            ('border_mode', 'padding'),
        ]
        kwargs = convert_legacy_kwargs(args[0].__class__.__name__,
                                       args[1:],
                                       kwargs,
                                       conversions)
        return func(*args, **kwargs)
    return wrapper
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Artman smoke tests.

It generates GAPIC client libraries for Google APIs in googleapis
repo. The test will fails if any generation fails unless its artman yaml file
is on the whitelist.
"""

import argparse
import glob
import json
import logging
import os
import re
import subprocess
import sys
import yaml

from google.protobuf import json_format

from artman.config.proto import config_pb2

# Generation failure of artifact on the whitelist won't be counted as failure
# but warning. It is defined at artifact level, and one can use regex to
# whitelist the whole API or whole folder.
WHITELIST = [
    # The two configs below are failing: https://github.com/googleapis/artman/issues/483
    'python_gapic@google/devtools/containeranalysis/artman_containeranalysis_v1beta1.yaml',

    'nodejs_gapic@google/cloud/videointelligence/artman_videointelligence_v1.yaml',
    # TODO(ethanbao): Figure out the rootcause
    'java_gapic@google/cloud/dialogflow/artman_dialogflow_v2beta1_java.yaml',
    '.+@google/api/servicemanagement/artman_servicemanagement_v1.yaml',
    # TODO(pongad): talk to the API owners
    '(?:python|php)_gapic@google/api/expr/artman_cel.yaml',

    # whitelisted because of
    # java.lang.IllegalStateException: A HttpRule option must be defined.
    # (affects only PHP that generates code for JSON over HTTP)
    # https://github.com/googleapis/gapic-generator/issues/2691
    'php_gapic@google/cloud/bigquery/storage/artman_bigquerystorage_v1beta1.yaml',
    'php_gapic@google/cloud/bigquery/artman_bigquery_v2.yaml',

    # whitelisted because protos are missing Java annotations
    # necessary for client library generation. CL in flight.
    'java_gapic@google/cloud/binaryauthorization/artman_binaryauthorization_v1beta1.yaml',

    # TODO(hzyi): whitelisted to unblock artman release. CL coming.
    'csharp_gapic@google/firestore/artman_firestore_v1.yaml',
    'csharp_gapic@google/firestore/artman_firestore.yaml',

    # whitelisted four weird configs
    'java_grpc@google/artman_core.yaml',
    'java_grpc@google/artman_logging_external_types.yaml',
    'java_grpc@google/appengine/artman_appengine.yaml',
    'java_gapic@google/cloud/dialogflow/v2beta1/artman_dialogflow_v2beta1_java.yaml',

    # whitelisted for no HTTP annotations
    'php_gapic@google/devtools/cloudbuild/artman_cloudbuild',
    'php_gapic@google/cloud/osconfig/agentendpoint/v1beta/artman_osconfig_v1beta',

    # whitelisted because of broken GAPIC config
    # TODO(fenster): to fix that
    '.*@google/monitoring/v3/artman_monitoring.yaml',

    # securitycenter C# does not use gapic-generator
    'csharp_gapic@google/cloud/securitycenter/artman_securitycenter_v1p1beta1.yaml',

    # whitelisted because ruby grpc can't generate a proto2 descriptor
    # which this API depends on in protobuf.proto
    'ruby_gapic@google/cloud/bigquery/artman_bigquerystorage_v1alpha2.yaml',
]

logger = logging.getLogger('smoketest')
logger.setLevel(logging.INFO)

def run_smoke_test(root_dir, user_config, log):
    log_file = _setup_logger(log)
    failure = []
    success = []
    known_fail = []
    for artman_yaml_path in glob.glob('%s/google/**/artman_*.yaml' % root_dir,
                                      recursive=True):
        artman_config = _parse(artman_yaml_path)
        # Reorder the artifacts by moving the gapic_config artifact to the end.
        # Otherwise, the following GAPIC library generation is going to use the
        # newly-generated GAPIC config instead of the one in github, leading to
        # false negative and false positive in the test result.
        artifacts = sorted(
            artman_config.artifacts,
            key=lambda a: config_pb2.Artifact.GAPIC_CONFIG == a.type)

        for artifact in artifacts:
            logger.info('Start artifact generation for %s of %s'
                        % (artifact.name, artman_yaml_path))
            if _generate_artifact(artman_yaml_path,
                                  user_config,
                                  artifact.name,
                                  root_dir,
                                  log_file):
                msg = 'Failed to generate %s of %s.' % (
                    artifact.name, artman_yaml_path)
                if _is_whitelisted(artman_yaml_path, root_dir, artifact.name):
                    known_fail.append(msg)
                else:
                    failure.append(msg)
            else:
                msg = 'Succeded to generate %s of %s.' % (
                    artifact.name, artman_yaml_path)
                success.append(msg)
            logger.info(msg)
    logger.info('================ Smoketest summary ================')
    logger.info('Success:')
    for msg in success:
        logger.info(msg)

    if known_fail:
        logger.info('Known Whitelisted Failures:')
        for msg in known_fail:
            logger.info(msg)

    if failure:
        logger.info('Failure:')
        for msg in failure:
            logger.error(msg)
        sys.exit('Smoke test failed.')

def _parse(artman_yaml_path):
    """Parse artman yaml config into corresponding protobuf."""
    with open(artman_yaml_path, 'r') as f:
        # Convert yaml into json file as protobuf python load support parsing
        # of protobuf in json or text format, not yaml.
        artman_config_json_string = json.dumps(yaml.load(f, Loader=yaml.FullLoader))
    config_pb = config_pb2.Config()
    json_format.Parse(artman_config_json_string, config_pb)

    return config_pb

def _generate_artifact(artman_config, user_config, artifact_name, root_dir, log_file):
    with open(log_file, 'a') as log:
        grpc_pipeline_args = [
            'artman',
            '--local',
            '--verbose',
            '--config', artman_config,
            '--root-dir', root_dir,
            '--user-config', user_config,
            'generate', artifact_name
        ]
        return subprocess.call(grpc_pipeline_args, stdout=log, stderr=log)

def _is_whitelisted(artman_yaml_path, root_dir, artifact_name):
    artifact = '%s@%s' % (artifact_name,
                          os.path.relpath(artman_yaml_path, root_dir))
    for whitelist in WHITELIST:
        if re.match(whitelist, artifact):
            return True
    return False

def parse_args(*args):
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--root-dir',
        default='/googleapis',
        type=str,
        help='Specify where googleapis local repo lives.')
    parser.add_argument(
        '--log',
        default='/tmp/smoketest.log',
        type=str,
        help='Specify where smoketest log should be stored.')
    parser.add_argument(
        '--user-config',
        default='/artman/artman-user-config-in-docker.yaml',
        type=str,
        help='Specify where artman user config is located')
    return parser.parse_args(args=args)

def _setup_logger(log_file):
    """Setup logger with a logging FileHandler."""
    log_file_handler = logging.FileHandler(log_file, mode='a+')
    logger.addHandler(log_file_handler)
    logger.addHandler(logging.StreamHandler())
    return log_file

if __name__ == '__main__':
    flags = parse_args(*sys.argv[1:])

    run_smoke_test(os.path.abspath(flags.root_dir), os.path.abspath(flags.user_config), os.path.abspath(flags.log))
<commit_msg>test: revert bq storage v1alpha2 whitelist (#799)<commit_after>#!/usr/bin/env python

# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Artman smoke tests.

It generates GAPIC client libraries for Google APIs in googleapis
repo. The test will fails if any generation fails unless its artman yaml file
is on the whitelist.
"""

import argparse
import glob
import json
import logging
import os
import re
import subprocess
import sys
import yaml

from google.protobuf import json_format

from artman.config.proto import config_pb2

# Generation failure of artifact on the whitelist won't be counted as failure
# but warning. It is defined at artifact level, and one can use regex to
# whitelist the whole API or whole folder.
WHITELIST = [
    # The two configs below are failing: https://github.com/googleapis/artman/issues/483
    'python_gapic@google/devtools/containeranalysis/artman_containeranalysis_v1beta1.yaml',

    'nodejs_gapic@google/cloud/videointelligence/artman_videointelligence_v1.yaml',
    # TODO(ethanbao): Figure out the rootcause
    'java_gapic@google/cloud/dialogflow/artman_dialogflow_v2beta1_java.yaml',
    '.+@google/api/servicemanagement/artman_servicemanagement_v1.yaml',
    # TODO(pongad): talk to the API owners
    '(?:python|php)_gapic@google/api/expr/artman_cel.yaml',

    # whitelisted because of
    # java.lang.IllegalStateException: A HttpRule option must be defined.
    # (affects only PHP that generates code for JSON over HTTP)
    # https://github.com/googleapis/gapic-generator/issues/2691
    'php_gapic@google/cloud/bigquery/storage/artman_bigquerystorage_v1beta1.yaml',
    'php_gapic@google/cloud/bigquery/artman_bigquery_v2.yaml',

    # whitelisted because protos are missing Java annotations
    # necessary for client library generation. CL in flight.
    'java_gapic@google/cloud/binaryauthorization/artman_binaryauthorization_v1beta1.yaml',

    # TODO(hzyi): whitelisted to unblock artman release. CL coming.
    'csharp_gapic@google/firestore/artman_firestore_v1.yaml',
    'csharp_gapic@google/firestore/artman_firestore.yaml',

    # whitelisted four weird configs
    'java_grpc@google/artman_core.yaml',
    'java_grpc@google/artman_logging_external_types.yaml',
    'java_grpc@google/appengine/artman_appengine.yaml',
    'java_gapic@google/cloud/dialogflow/v2beta1/artman_dialogflow_v2beta1_java.yaml',

    # whitelisted for no HTTP annotations
    'php_gapic@google/devtools/cloudbuild/artman_cloudbuild',
    'php_gapic@google/cloud/osconfig/agentendpoint/v1beta/artman_osconfig_v1beta',

    # whitelisted because of broken GAPIC config
    # TODO(fenster): to fix that
    '.*@google/monitoring/v3/artman_monitoring.yaml',

    # securitycenter C# does not use gapic-generator
    'csharp_gapic@google/cloud/securitycenter/artman_securitycenter_v1p1beta1.yaml',

]

logger = logging.getLogger('smoketest')
logger.setLevel(logging.INFO)

def run_smoke_test(root_dir, user_config, log):
    log_file = _setup_logger(log)
    failure = []
    success = []
    known_fail = []
    for artman_yaml_path in glob.glob('%s/google/**/artman_*.yaml' % root_dir,
                                      recursive=True):
        artman_config = _parse(artman_yaml_path)
        # Reorder the artifacts by moving the gapic_config artifact to the end.
        # Otherwise, the following GAPIC library generation is going to use the
        # newly-generated GAPIC config instead of the one in github, leading to
        # false negative and false positive in the test result.
        artifacts = sorted(
            artman_config.artifacts,
            key=lambda a: config_pb2.Artifact.GAPIC_CONFIG == a.type)

        for artifact in artifacts:
            logger.info('Start artifact generation for %s of %s'
                        % (artifact.name, artman_yaml_path))
            if _generate_artifact(artman_yaml_path,
                                  user_config,
                                  artifact.name,
                                  root_dir,
                                  log_file):
                msg = 'Failed to generate %s of %s.' % (
                    artifact.name, artman_yaml_path)
                if _is_whitelisted(artman_yaml_path, root_dir, artifact.name):
                    known_fail.append(msg)
                else:
                    failure.append(msg)
            else:
                msg = 'Succeded to generate %s of %s.' % (
                    artifact.name, artman_yaml_path)
                success.append(msg)
            logger.info(msg)
    logger.info('================ Smoketest summary ================')
    logger.info('Success:')
    for msg in success:
        logger.info(msg)

    if known_fail:
        logger.info('Known Whitelisted Failures:')
        for msg in known_fail:
            logger.info(msg)

    if failure:
        logger.info('Failure:')
        for msg in failure:
            logger.error(msg)
        sys.exit('Smoke test failed.')

def _parse(artman_yaml_path):
    """Parse artman yaml config into corresponding protobuf."""
    with open(artman_yaml_path, 'r') as f:
        # Convert yaml into json file as protobuf python load support parsing
        # of protobuf in json or text format, not yaml.
        artman_config_json_string = json.dumps(yaml.load(f, Loader=yaml.FullLoader))
    config_pb = config_pb2.Config()
    json_format.Parse(artman_config_json_string, config_pb)

    return config_pb

def _generate_artifact(artman_config, user_config, artifact_name, root_dir, log_file):
    with open(log_file, 'a') as log:
        grpc_pipeline_args = [
            'artman',
            '--local',
            '--verbose',
            '--config', artman_config,
            '--root-dir', root_dir,
            '--user-config', user_config,
            'generate', artifact_name
        ]
        return subprocess.call(grpc_pipeline_args, stdout=log, stderr=log)

def _is_whitelisted(artman_yaml_path, root_dir, artifact_name):
    artifact = '%s@%s' % (artifact_name,
                          os.path.relpath(artman_yaml_path, root_dir))
    for whitelist in WHITELIST:
        if re.match(whitelist, artifact):
            return True
    return False

def parse_args(*args):
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--root-dir',
        default='/googleapis',
        type=str,
        help='Specify where googleapis local repo lives.')
    parser.add_argument(
        '--log',
        default='/tmp/smoketest.log',
        type=str,
        help='Specify where smoketest log should be stored.')
    parser.add_argument(
        '--user-config',
        default='/artman/artman-user-config-in-docker.yaml',
        type=str,
        help='Specify where artman user config is located')
    return parser.parse_args(args=args)

def _setup_logger(log_file):
    """Setup logger with a logging FileHandler."""
    log_file_handler = logging.FileHandler(log_file, mode='a+')
    logger.addHandler(log_file_handler)
    logger.addHandler(logging.StreamHandler())
    return log_file

if __name__ == '__main__':
    flags = parse_args(*sys.argv[1:])

    run_smoke_test(os.path.abspath(flags.root_dir), os.path.abspath(flags.user_config), os.path.abspath(flags.log))
<|endoftext|>


<commit_before>''' The speed up version of the normal threshold scan where the first and the last PlsrDAC setting is determined automatically to minimize the scan time. The step size is changed automatically.
'''
import numpy as np
from scan.scan import ScanBase
from daq.readout import open_raw_data_file, get_col_row_array_from_data_record_array, convert_data_array, data_array_from_data_dict_iterable, is_data_record, is_data_from_channel, logical_and

from analysis.analyze_raw_data import AnalyzeRawData
from analysis.analysis_utils import AnalysisUtils

import logging
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - [%(levelname)-8s] (%(threadName)-10s) %(message)s")

class ThresholdScanFast(ScanBase):
    def __init__(self, config_file, definition_file=None, bit_file=None, device=None, scan_identifier="scan_threshold_fast", scan_data_path=None):
        super(ThresholdScanFast, self).__init__(config_file=config_file, definition_file=definition_file, bit_file=bit_file, device=device, scan_identifier=scan_identifier, scan_data_path=scan_data_path)
        self.scan_parameter_start = 0  # holding last start value (e.g. used in GDAC threshold scan)

    def scan(self, mask_steps=3, repeat_command=100, scan_parameter='PlsrDAC', scan_parameter_range=None, scan_parameter_stepsize=2, search_distance=10, minimum_data_points=15, ignore_columns=(1, 78, 79, 80)):
        '''Scan loop

        Parameters
        ----------
        mask_steps : int
            Number of mask steps.
        repeat_command : int
            Number of injections per scan step.
        scan_parameter : string
            Name of global register.
        scan_parameter_range : list, tuple
            Specify the minimum and maximum value for scan parameter range. Upper value not included.
        scan_parameter_stepsize : int
            The minimum step size of the parameter. Used when start condition is not triggered.
        search_distance : int
            The parameter step size if the start condition is not triggered.
        minimum_data_points : int
            The minimum data points that are taken for sure until scan finished. Saves also calculation time.
        ignore_columns : list, tuple
            All columns that are neither scanned nor taken into account to set the scan range are mentioned here. Usually the edge columns are ignored. From 1 to 80.
        '''

        self.start_condition_triggered = False  # set to true if the start condition is true once
        self.stop_condition_triggered = False  # set to true if the stop condition is true once

        self.start_at = 0.01  # if more than start_at*activated_pixel see at least one hit the precise scanning is started
        self.stop_at = 0.99  # if more than stop_at*activated_pixel see the maximum numbers of injection, the scan is stopped

        self.record_data = False  # set to true to activate data storage, so far not everything is recorded to ease data analysis

        if scan_parameter_range is None or not scan_parameter_range:
            scan_parameter_range = (0, (2 ** self.register.get_global_register_objects(name=[scan_parameter])[0].bitlength))
        logging.info("Scanning %s from %d to %d" % (scan_parameter, scan_parameter_range[0], scan_parameter_range[1]))
        self.scan_parameter_value = scan_parameter_range[0]  # set to start value
        self.search_distance = search_distance
        data_points = 0  # counter variable to count the data points already recorded, have to be at least minimum_data_ponts

        # calculate DCs to scan from the columns to ignore
        enable_double_columns = range(0, 40)
        if 1 in ignore_columns:
            enable_double_columns.remove(0)
        if set((78, 79, 80)).issubset(ignore_columns):
            enable_double_columns.remove(39)
        for double_column in range(1, 39):
            if set((double_column * 2, (double_column * 2) + 1)).issubset(ignore_columns):
                enable_double_columns.remove(double_column)
        logging.info("Use DCs: %s" % str(enable_double_columns))

        self.select_arr_columns = range(0, 80)
        for column in ignore_columns:
            self.select_arr_columns.remove(column-1)

        self.repeat_command = repeat_command

        with open_raw_data_file(filename=self.scan_data_filename, title=self.scan_identifier, scan_parameters=[scan_parameter]) as raw_data_file:
            while self.scan_parameter_value < scan_parameter_range[1]:  # scan as long as scan parameter is smaller than defined maximum
                if self.stop_thread_event.is_set():
                    break
                if self.record_data:
                    logging.info("Scan step %d (%s %d)" % (data_points, scan_parameter, self.scan_parameter_value))

                commands = []
                commands.extend(self.register.get_commands("confmode"))
                self.register.set_global_register_value(scan_parameter, self.scan_parameter_value)
                commands.extend(self.register.get_commands("wrregister", name=[scan_parameter]))
                self.register_utils.send_commands(commands)

                self.readout.start()

                cal_lvl1_command = self.register.get_commands("cal")[0] + self.register.get_commands("zeros", length=40)[0] + self.register.get_commands("lv1")[0]
                self.scan_loop(cal_lvl1_command, repeat_command=repeat_command, hardware_repeat=True, use_delay=True, mask_steps=mask_steps, enable_mask_steps=None, enable_double_columns=enable_double_columns, same_mask_for_all_dc=True, eol_function=None, digital_injection=False, enable_c_high=None, enable_c_low=None, shift_masks=["Enable", "C_High", "C_Low"], restore_shift_masks=False, mask=None)

                self.readout.stop()

                if not self.start_condition_triggered or data_points > minimum_data_points:  # speed up, only create histograms when needed. Python is much too slow here.
                    if not self.start_condition_triggered and not self.record_data:
                        logging.info('Testing for start condition: %s %d' % (scan_parameter, self.scan_parameter_value))
                    if not self.stop_condition_triggered and self.record_data:
                        logging.info('Testing for stop condition: %s %d' % (scan_parameter, self.scan_parameter_value))
                    occupancy_array = np.histogram2d(*convert_data_array(data_array_from_data_dict_iterable(self.readout.data), filter_func=is_data_record, converter_func=get_col_row_array_from_data_record_array), bins=(80, 336), range=[[1, 80], [1, 336]])[0]
                    self.scan_condition(occupancy_array)

                # start condition is met for the first time
                if self.start_condition_triggered and not self.record_data:
                    self.scan_parameter_value = self.scan_parameter_value - self.search_distance + scan_parameter_stepsize
                    if self.scan_parameter_value < 0:
                        self.scan_parameter_value = 0
                    logging.info('Starting threshold scan at %s %d' % (scan_parameter, self.scan_parameter_value))
                    self.scan_parameter_start = self.scan_parameter_value
                    self.record_data = True
                    continue

                # saving data
                if self.record_data:
                    data_points = data_points + 1
                    raw_data_file.append(self.readout.data, scan_parameters={scan_parameter: self.scan_parameter_value})

                # stop condition is met for the first time
                if self.stop_condition_triggered and self.record_data:
                    logging.info('Stopping threshold scan at %s %d' % (scan_parameter, self.scan_parameter_value))
                    break

                # increase scan parameter value
                if not self.start_condition_triggered:
                    self.scan_parameter_value = self.scan_parameter_value + self.search_distance
                else:
                    self.scan_parameter_value = self.scan_parameter_value + scan_parameter_stepsize

    def scan_condition(self, occupancy_array):
        occupancy_array_select = occupancy_array[self.select_arr_columns, :]  # only select not ignored columns
        # stop precise scanning actions
        pixels_with_full_hits = np.ma.array(occupancy_array_select, mask=(occupancy_array_select >= self.repeat_command))  # select pixels that see all injections
        pixels_with_full_hits_count = np.ma.count_masked(pixels_with_full_hits)  # count pixels that see all injections
        stop_pixel_cnt = int(np.product(occupancy_array_select.shape) * self.stop_at)
        if pixels_with_full_hits_count >= stop_pixel_cnt and not self.stop_condition_triggered:  # stop precise scanning if this triggers
            logging.info("Triggering stop condition: %d pixel(s) with %d hits or more >= %d pixel(s)" % (pixels_with_full_hits_count, self.repeat_command, stop_pixel_cnt))
            self.stop_condition_triggered = True
        # start precise scanning actions
        pixels_with_hits = np.ma.array(occupancy_array_select, mask=(occupancy_array_select != 0))  # select pixels that see at least one hit
        pixels_with_hits_count = np.ma.count_masked(pixels_with_hits)  # count pixels that see hits
        start_pixel_cnt = int(np.product(occupancy_array_select.shape) * self.start_at)
        if pixels_with_hits_count >= start_pixel_cnt and not self.start_condition_triggered:  # start precise scanning if this is true
            logging.info("Triggering start condition: %d pixel(s) with more than 0 hits >= %d pixel(s)" % (pixels_with_hits_count, start_pixel_cnt))
            self.start_condition_triggered = True

    def analyze(self, create_plots=True):
        with AnalyzeRawData(raw_data_file=self.scan_data_filename + ".h5", analyzed_data_file=self.scan_data_filename + "_interpreted.h5") as analyze_raw_data:
            analyze_raw_data.interpreter.set_trig_count(self.register.get_global_register_value("Trig_Count"))
            analyze_raw_data.create_tot_hist = False
            analyze_raw_data.create_threshold_hists = True
            analyze_raw_data.create_fitted_threshold_hists = True
            analyze_raw_data.create_threshold_mask = True
            analyze_raw_data.n_injections = 100
            analyze_raw_data.interpreter.set_warning_output(False)  # so far the data structure in a threshold scan was always bad, too many warnings given
            analyze_raw_data.interpret_word_table(fei4b=self.register.fei4b)
            analyze_raw_data.interpreter.print_summary()
            if create_plots:
                analyze_raw_data.plot_histograms(scan_data_filename=self.scan_data_filename)

if __name__ == "__main__":
    import configuration
    scan = ThresholdScanFast(config_file=configuration.config_file, bit_file=configuration.bit_file, scan_data_path=configuration.scan_data_path)
    scan.start(use_thread=True, scan_parameter_range=None, scan_parameter_stepsize=2, search_distance=10, minimum_data_points=10, ignore_columns=(1, 78, 79, 80))
    scan.stop()
    scan.analyze()
<commit_msg>ENH: make number of data points global to be able to speed up the threshold scan when used in a loop<commit_after>''' The speed up version of the normal threshold scan where the first and the last PlsrDAC setting is determined automatically to minimize the scan time. The step size is changed automatically.
'''
import numpy as np
from scan.scan import ScanBase
from daq.readout import open_raw_data_file, get_col_row_array_from_data_record_array, convert_data_array, data_array_from_data_dict_iterable, is_data_record, is_data_from_channel, logical_and

from analysis.analyze_raw_data import AnalyzeRawData
from analysis.analysis_utils import AnalysisUtils

import logging
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - [%(levelname)-8s] (%(threadName)-10s) %(message)s")

class ThresholdScanFast(ScanBase):
    def __init__(self, config_file, definition_file=None, bit_file=None, device=None, scan_identifier="scan_threshold_fast", scan_data_path=None):
        super(ThresholdScanFast, self).__init__(config_file=config_file, definition_file=definition_file, bit_file=bit_file, device=device, scan_identifier=scan_identifier, scan_data_path=scan_data_path)
        self.scan_parameter_start = 0  # holding last start value (e.g. used in GDAC threshold scan)
        self.data_points = 10

    def scan(self, mask_steps=3, repeat_command=100, scan_parameter='PlsrDAC', scan_parameter_range=None, scan_parameter_stepsize=2, search_distance=10, minimum_data_points=15, ignore_columns=(1, 78, 79, 80)):
        '''Scan loop

        Parameters
        ----------
        mask_steps : int
            Number of mask steps.
        repeat_command : int
            Number of injections per scan step.
        scan_parameter : string
            Name of global register.
        scan_parameter_range : list, tuple
            Specify the minimum and maximum value for scan parameter range. Upper value not included.
        scan_parameter_stepsize : int
            The minimum step size of the parameter. Used when start condition is not triggered.
        search_distance : int
            The parameter step size if the start condition is not triggered.
        minimum_data_points : int
            The minimum data points that are taken for sure until scan finished. Saves also calculation time.
        ignore_columns : list, tuple
            All columns that are neither scanned nor taken into account to set the scan range are mentioned here. Usually the edge columns are ignored. From 1 to 80.
        '''

        self.start_condition_triggered = False  # set to true if the start condition is true once
        self.stop_condition_triggered = False  # set to true if the stop condition is true once

        self.start_at = 0.01  # if more than start_at*activated_pixel see at least one hit the precise scanning is started
        self.stop_at = 0.99  # if more than stop_at*activated_pixel see the maximum numbers of injection, the scan is stopped

        self.record_data = False  # set to true to activate data storage, so far not everything is recorded to ease data analysis

        if scan_parameter_range is None or not scan_parameter_range:
            scan_parameter_range = (0, (2 ** self.register.get_global_register_objects(name=[scan_parameter])[0].bitlength))
        logging.info("Scanning %s from %d to %d" % (scan_parameter, scan_parameter_range[0], scan_parameter_range[1]))
        self.scan_parameter_value = scan_parameter_range[0]  # set to start value
        self.search_distance = search_distance
        self.data_points = 0  # counter variable to count the data points already recorded, have to be at least minimum_data_ponts

        # calculate DCs to scan from the columns to ignore
        enable_double_columns = range(0, 40)
        if 1 in ignore_columns:
            enable_double_columns.remove(0)
        if set((78, 79, 80)).issubset(ignore_columns):
            enable_double_columns.remove(39)
        for double_column in range(1, 39):
            if set((double_column * 2, (double_column * 2) + 1)).issubset(ignore_columns):
                enable_double_columns.remove(double_column)
        logging.info("Use DCs: %s" % str(enable_double_columns))

        self.select_arr_columns = range(0, 80)
        for column in ignore_columns:
            self.select_arr_columns.remove(column - 1)

        self.repeat_command = repeat_command

        with open_raw_data_file(filename=self.scan_data_filename, title=self.scan_identifier, scan_parameters=[scan_parameter]) as raw_data_file:
            while self.scan_parameter_value < scan_parameter_range[1]:  # scan as long as scan parameter is smaller than defined maximum
                if self.stop_thread_event.is_set():
                    break
                if self.record_data:
                    logging.info("Scan step %d (%s %d)" % (self.data_points, scan_parameter, self.scan_parameter_value))

                commands = []
                commands.extend(self.register.get_commands("confmode"))
                self.register.set_global_register_value(scan_parameter, self.scan_parameter_value)
                commands.extend(self.register.get_commands("wrregister", name=[scan_parameter]))
                self.register_utils.send_commands(commands)

                self.readout.start()

                cal_lvl1_command = self.register.get_commands("cal")[0] + self.register.get_commands("zeros", length=40)[0] + self.register.get_commands("lv1")[0]
                self.scan_loop(cal_lvl1_command, repeat_command=repeat_command, hardware_repeat=True, use_delay=True, mask_steps=mask_steps, enable_mask_steps=None, enable_double_columns=enable_double_columns, same_mask_for_all_dc=True, eol_function=None, digital_injection=False, enable_c_high=None, enable_c_low=None, shift_masks=["Enable", "C_High", "C_Low"], restore_shift_masks=False, mask=None)

                self.readout.stop()

                if not self.start_condition_triggered or self.data_points > minimum_data_points:  # speed up, only create histograms when needed. Python is much too slow here.
                    if not self.start_condition_triggered and not self.record_data:
                        logging.info('Testing for start condition: %s %d' % (scan_parameter, self.scan_parameter_value))
                    if not self.stop_condition_triggered and self.record_data:
                        logging.info('Testing for stop condition: %s %d' % (scan_parameter, self.scan_parameter_value))
                    occupancy_array = np.histogram2d(*convert_data_array(data_array_from_data_dict_iterable(self.readout.data), filter_func=is_data_record, converter_func=get_col_row_array_from_data_record_array), bins=(80, 336), range=[[1, 80], [1, 336]])[0]
                    self.scan_condition(occupancy_array)

                # start condition is met for the first time
                if self.start_condition_triggered and not self.record_data:
                    self.scan_parameter_value = self.scan_parameter_value - self.search_distance + scan_parameter_stepsize
                    if self.scan_parameter_value < 0:
                        self.scan_parameter_value = 0
                    logging.info('Starting threshold scan at %s %d' % (scan_parameter, self.scan_parameter_value))
                    self.scan_parameter_start = self.scan_parameter_value
                    self.record_data = True
                    continue

                # saving data
                if self.record_data:
                    self.data_points = self.data_points + 1
                    raw_data_file.append(self.readout.data, scan_parameters={scan_parameter: self.scan_parameter_value})

                # stop condition is met for the first time
                if self.stop_condition_triggered and self.record_data:
                    logging.info('Stopping threshold scan at %s %d' % (scan_parameter, self.scan_parameter_value))
                    break

                # increase scan parameter value
                if not self.start_condition_triggered:
                    self.scan_parameter_value = self.scan_parameter_value + self.search_distance
                else:
                    self.scan_parameter_value = self.scan_parameter_value + scan_parameter_stepsize

    def scan_condition(self, occupancy_array):
        occupancy_array_select = occupancy_array[self.select_arr_columns, :]  # only select not ignored columns
        # stop precise scanning actions
        pixels_with_full_hits = np.ma.array(occupancy_array_select, mask=(occupancy_array_select >= self.repeat_command))  # select pixels that see all injections
        pixels_with_full_hits_count = np.ma.count_masked(pixels_with_full_hits)  # count pixels that see all injections
        stop_pixel_cnt = int(np.product(occupancy_array_select.shape) * self.stop_at)
        if pixels_with_full_hits_count >= stop_pixel_cnt and not self.stop_condition_triggered:  # stop precise scanning if this triggers
            logging.info("Triggering stop condition: %d pixel(s) with %d hits or more >= %d pixel(s)" % (pixels_with_full_hits_count, self.repeat_command, stop_pixel_cnt))
            self.stop_condition_triggered = True
        # start precise scanning actions
        pixels_with_hits = np.ma.array(occupancy_array_select, mask=(occupancy_array_select != 0))  # select pixels that see at least one hit
        pixels_with_hits_count = np.ma.count_masked(pixels_with_hits)  # count pixels that see hits
        start_pixel_cnt = int(np.product(occupancy_array_select.shape) * self.start_at)
        if pixels_with_hits_count >= start_pixel_cnt and not self.start_condition_triggered:  # start precise scanning if this is true
            logging.info("Triggering start condition: %d pixel(s) with more than 0 hits >= %d pixel(s)" % (pixels_with_hits_count, start_pixel_cnt))
            self.start_condition_triggered = True

    def analyze(self, create_plots=True):
        with AnalyzeRawData(raw_data_file=self.scan_data_filename + ".h5", analyzed_data_file=self.scan_data_filename + "_interpreted.h5") as analyze_raw_data:
            analyze_raw_data.interpreter.set_trig_count(self.register.get_global_register_value("Trig_Count"))
            analyze_raw_data.create_tot_hist = False
            analyze_raw_data.create_threshold_hists = True
            analyze_raw_data.create_fitted_threshold_hists = True
            analyze_raw_data.create_threshold_mask = True
            analyze_raw_data.n_injections = 100
            analyze_raw_data.interpreter.set_warning_output(False)  # so far the data structure in a threshold scan was always bad, too many warnings given
            analyze_raw_data.interpret_word_table(fei4b=self.register.fei4b)
            analyze_raw_data.interpreter.print_summary()
            if create_plots:
                analyze_raw_data.plot_histograms(scan_data_filename=self.scan_data_filename)

if __name__ == "__main__":
    import configuration
    scan = ThresholdScanFast(config_file=configuration.config_file, bit_file=configuration.bit_file, scan_data_path=configuration.scan_data_path)
    scan.start(use_thread=True, scan_parameter_range=None, scan_parameter_stepsize=2, search_distance=10, minimum_data_points=10, ignore_columns=(1, 78, 79, 80))
    scan.stop()
    scan.analyze()
<|endoftext|>


<commit_before>import os
from flask import render_template, Blueprint, send_file

main = Blueprint('main', __name__, template_folder='templates', static_folder='static/gen', static_url_path='/static')

@main.route('/templates/<path:partial>')
def render_partial(partial=None):
    return render_template(partial + '.html')

@main.route('/favicon.ico')
def send_favicon():
    return send_file('static/favicon.ico')

@main.route('/upload/<path:name>')
def upload(name=None)
    return send_file(os.path.join('/upload/'+name))

@main.route('/')
@main.route('/<path:path>')
def index(path=None):
    return render_template('index.html')

<commit_msg>Wowo.<commit_after>import os
from flask import render_template, Blueprint, send_file

main = Blueprint('main', __name__, template_folder='templates', static_folder='static/gen', static_url_path='/static')

@main.route('/templates/<path:partial>')
def render_partial(partial=None):
    return render_template(partial + '.html')

@main.route('/favicon.ico')
def send_favicon():
    return send_file('static/favicon.ico')

@main.route('/upload/<path:name>')
def upload(name=None):
    return send_file(os.path.join('/upload/'+name))

@main.route('/')
@main.route('/<path:path>')
def index(path=None):
    return render_template('index.html')

<|endoftext|>