<commit_before>#---------------------------------------------------------------------------
#
#   Albow - Root widget
#
#---------------------------------------------------------------------------

import sys
import pygame
from pygame.locals import *
#from pygame.time import get_ticks
from pygame.event import Event

from glbackground import *
import widget
from widget import Widget
from controls import Label

from datetime import datetime, timedelta
from albow.dialogs import wrapped_label
start_time = datetime.now()

mod_cmd = KMOD_LCTRL | KMOD_RCTRL | KMOD_LMETA | KMOD_RMETA
double_click_time = timedelta(0, 0, 300000)  # days, seconds, microseconds

import logging
log = logging.getLogger(__name__)

modifiers = dict(
    shift=False,
    ctrl=False,
    alt=False,
    meta=False,
)

modkeys = {
    K_LSHIFT: 'shift', K_RSHIFT: 'shift',
    K_LCTRL:  'ctrl', K_RCTRL:  'ctrl',
    K_LALT:   'alt', K_RALT:   'alt',
    K_LMETA:  'meta', K_RMETA:  'meta',
}

MUSIC_END_EVENT = USEREVENT + 1

last_mouse_event = Event(0, pos=(0, 0), local=(0, 0))
last_mouse_event_handler = None
root_widget = None     # Root of the containment hierarchy
top_widget = None      # Initial dispatch target
clicked_widget = None  # Target of mouse_drag and mouse_up events

#---------------------------------------------------------------------------

class Cancel(Exception):
    pass

#---------------------------------------------------------------------------

def set_modifier(key, value):
    attr = modkeys.get(key)
    if attr:
        modifiers[attr] = value

def add_modifiers(event):
    d = event.dict
    d.update(modifiers)
    d['cmd'] = event.ctrl or event.meta

def get_root():
    return root_widget

def get_top_widget():
    return top_widget

def get_focus():
    return top_widget.get_focus()

#---------------------------------------------------------------------------

class RootWidget(Widget):
    #  surface   Pygame display surface
    #  is_gl     True if OpenGL surface

    redraw_every_frame = False
    do_draw = False
    _is_gl_container = True

    def __init__(self, surface):
        global root_widget
        Widget.__init__(self, surface.get_rect())
        self.surface = surface
        root_widget = self
        widget.root_widget = self
        self.is_gl = surface.get_flags() & OPENGL != 0
        self.idle_handlers = []

    def set_timer(self, ms):
        pygame.time.set_timer(USEREVENT, ms)

    def run(self):
        self.run_modal(None)

    captured_widget = None

    def capture_mouse(self, widget):
        #put the mouse in "virtual mode" and pass mouse moved events to the
        #specified widget
        if widget:
            pygame.mouse.set_visible(False)
            pygame.event.set_grab(True)
            get_root().captured_widget = widget
        else:
            pygame.mouse.set_visible(True)
            pygame.event.set_grab(False)
            get_root().captured_widget = None

    frames = 0
    hover_widget = None

    def run_modal(self, modal_widget):
        old_captured_widget = None

        if self.captured_widget:
            old_captured_widget = self.captured_widget
            self.capture_mouse(None)

        global last_mouse_event, last_mouse_event_handler
        global top_widget, clicked_widget
        is_modal = modal_widget is not None
        modal_widget = modal_widget or self
        from OpenGL import GL
        try:
            old_top_widget = top_widget
            top_widget = modal_widget
            was_modal = modal_widget.is_modal
            modal_widget.is_modal = True
            modal_widget.modal_result = None
            if not modal_widget.focus_switch:
                modal_widget.tab_to_first()
            mouse_widget = None
            if clicked_widget:
                clicked_widget = modal_widget
            num_clicks = 0
            last_click_time = start_time
            last_click_button = 0
            self.do_draw = True

            while modal_widget.modal_result is None:
                try:
                    self.hover_widget = self.find_widget(mouse.get_pos())
                    if self.do_draw:
                        if self.is_gl:
                            self.gl_clear()
                            self.gl_draw_all(self, (0, 0))
                            GL.glFlush()
                        else:
                            self.draw_all(self.surface)
                        self.do_draw = False
                        pygame.display.flip()
                        self.frames += 1
                    #events = [pygame.event.wait()]
                    events = [pygame.event.poll()]
                    events.extend(pygame.event.get())
                    for event in events:
                        if event.type:
                            log.debug("%s", event)

                        type = event.type
                        if type == QUIT:
                            self.quit()
                        elif type == MOUSEBUTTONDOWN:
                            self.do_draw = True
                            t = datetime.now()
                            if t - last_click_time <= double_click_time and event.button == last_click_button:
                                num_clicks += 1
                            else:
                                num_clicks = 1
                            last_click_button = event.button
                            last_click_time = t
                            event.dict['num_clicks'] = num_clicks
                            add_modifiers(event)
                            mouse_widget = self.find_widget(event.pos)
                            if self.captured_widget:
                                mouse_widget = self.captured_widget

                            if not mouse_widget.is_inside(modal_widget):
                                mouse_widget = modal_widget
                            #if event.button == 1:
                            clicked_widget = mouse_widget
                            last_mouse_event_handler = mouse_widget
                            last_mouse_event = event
                            mouse_widget.notify_attention_loss()
                            mouse_widget.handle_mouse('mouse_down', event)
                        elif type == MOUSEMOTION:
                            self.do_draw = True
                            add_modifiers(event)
                            modal_widget.dispatch_key('mouse_delta', event)
                            last_mouse_event = event

                            mouse_widget = self.update_tooltip(event.pos)

                            if clicked_widget:
                                last_mouse_event_handler = clicked_widget
                                clicked_widget.handle_mouse('mouse_drag', event)
                            else:
                                if not mouse_widget.is_inside(modal_widget):
                                    mouse_widget = modal_widget
                                last_mouse_event_handler = mouse_widget
                                mouse_widget.handle_mouse('mouse_move', event)
                        elif type == MOUSEBUTTONUP:
                            add_modifiers(event)
                            self.do_draw = True
                            mouse_widget = self.find_widget(event.pos)
                            if self.captured_widget:
                                mouse_widget = self.captured_widget
                            if clicked_widget:
                                last_mouse_event_handler = clicked_widget
                                event.dict['clicked_widget'] = clicked_widget
                            else:
                                last_mouse_event_handler = mouse_widget
                                event.dict['clicked_widget'] = None

                            last_mouse_event = event
                            clicked_widget = None
                            last_mouse_event_handler.handle_mouse('mouse_up', event)
                        elif type == KEYDOWN:
                            key = event.key
                            set_modifier(key, True)
                            self.do_draw = True
                            self.send_key(modal_widget, 'key_down', event)
                            if last_mouse_event_handler:
                                event.dict['pos'] = last_mouse_event.pos
                                event.dict['local'] = last_mouse_event.local
                                last_mouse_event_handler.setup_cursor(event)
                        elif type == KEYUP:
                            key = event.key
                            set_modifier(key, False)
                            self.do_draw = True
                            self.send_key(modal_widget, 'key_up', event)
                            if last_mouse_event_handler:
                                event.dict['pos'] = last_mouse_event.pos
                                event.dict['local'] = last_mouse_event.local
                                last_mouse_event_handler.setup_cursor(event)
                        elif type == MUSIC_END_EVENT:
                            self.music_end()
                        elif type == USEREVENT:
                            make_scheduled_calls()
                            if not is_modal:
                                self.do_draw = self.redraw_every_frame
                                if last_mouse_event_handler:
                                    event.dict['pos'] = last_mouse_event.pos
                                    event.dict['local'] = last_mouse_event.local
                                    add_modifiers(event)
                                    last_mouse_event_handler.setup_cursor(event)
                                self.begin_frame()
                        elif type == VIDEORESIZE:
                            #add_modifiers(event)
                            self.do_draw = True
                            self.size = (event.w, event.h)
                            #self.dispatch_key('reshape', event)
                        elif type == ACTIVEEVENT:
                            add_modifiers(event)
                            self.dispatch_key('activeevent', event)
                        elif type == NOEVENT:
                            add_modifiers(event)
                            self.call_idle_handlers(event)

                except Cancel:
                    pass
        finally:
            modal_widget.is_modal = was_modal
            top_widget = old_top_widget
            if old_captured_widget:
                self.capture_mouse(old_captured_widget)

        clicked_widget = None

    def call_idle_handlers(self, event):
        def call(ref):
            widget = ref()
            if widget:
                widget.idleevent(event)
            else:
                print "Idle ref died!"
            return bool(widget)

        self.idle_handlers = filter(call, self.idle_handlers)

    def add_idle_handler(self, widget):
        from weakref import ref
        self.idle_handlers.append(ref(widget))

    def remove_idle_handler(self, widget):
        from weakref import ref
        self.idle_handlers.remove(ref(widget))

    def send_key(self, widget, name, event):
        add_modifiers(event)
        widget.dispatch_key(name, event)

    def begin_frame(self):
        pass

    def get_root(self):
        return self

    labelClass = lambda s, t:wrapped_label(t, 45)

    def show_tooltip(self, widget, pos):

        if hasattr(self, 'currentTooltip'):
            if self.currentTooltip != None:
                self.remove(self.currentTooltip)

            self.currentTooltip = None

        def TextTooltip(text):
            tooltipBacking = Panel()
            tooltipBacking.bg_color = (0.0, 0.0, 0.0, 0.6)
            tooltipBacking.add(self.labelClass(text))
            tooltipBacking.shrink_wrap()
            return tooltipBacking

        def showTip(tip):
            tip.topleft = pos
            tip.top += 20
            if (tip.bottom > self.bottom) or hasattr(widget, 'tooltipsUp'):
                tip.bottomleft = pos
                tip.top -= 4
            if tip.right > self.right:
                tip.right = pos[0]

            self.add(tip)
            self.currentTooltip = tip

        if widget.tooltip is not None:
            tip = widget.tooltip
            showTip(tip)

        else:
            ttext = widget.tooltipText
            if ttext is not None:
                tip = TextTooltip(ttext)
                showTip(tip)

    def update_tooltip(self, pos=None):
        if pos is None:
            pos = mouse.get_pos()
        if self.captured_widget:
            mouse_widget = self.captured_widget
            pos = mouse_widget.center
        else:
            mouse_widget = self.find_widget(pos)

        self.show_tooltip(mouse_widget, pos)
        return mouse_widget

    def has_focus(self):
        return True

    def quit(self):
        if self.confirm_quit():
            self.capture_mouse(None)
            sys.exit(0)

    def confirm_quit(self):
        return True

    def get_mouse_for(self, widget):
        last = last_mouse_event
        event = Event(0, last.dict)
        event.dict['local'] = widget.global_to_local(event.pos)
        add_modifiers(event)
        return event

    def gl_clear(self):
        from OpenGL import GL
        bg = self.bg_color
        if bg:
            r = bg[0] / 255.0
            g = bg[1] / 255.0
            b = bg[2] / 255.0
            GL.glClearColor(r, g, b, 0.0)
        GL.glClear(GL.GL_COLOR_BUFFER_BIT | GL.GL_DEPTH_BUFFER_BIT)

    def music_end(self):
        import music
        music.music_end()

#---------------------------------------------------------------------------

from time import time
from bisect import insort

scheduled_calls = []

def make_scheduled_calls():
    sched = scheduled_calls
    t = time()
    while sched and sched[0][0] <= t:
        sched[0][1]()
        sched.pop(0)

def schedule(delay, func):
    """Arrange for the given function to be called after the specified
    delay in seconds. Scheduled functions are called synchronously from
    the event loop, and only when the frame timer is running."""
    t = time() + delay
    insort(scheduled_calls, (t, func))
<commit_msg>pep8 compliance: E231 missing whitespace after ':'<commit_after>#---------------------------------------------------------------------------
#
#   Albow - Root widget
#
#---------------------------------------------------------------------------

import sys
import pygame
from pygame.locals import *
#from pygame.time import get_ticks
from pygame.event import Event

from glbackground import *
import widget
from widget import Widget
from controls import Label

from datetime import datetime, timedelta
from albow.dialogs import wrapped_label
start_time = datetime.now()

mod_cmd = KMOD_LCTRL | KMOD_RCTRL | KMOD_LMETA | KMOD_RMETA
double_click_time = timedelta(0, 0, 300000)  # days, seconds, microseconds

import logging
log = logging.getLogger(__name__)

modifiers = dict(
    shift=False,
    ctrl=False,
    alt=False,
    meta=False,
)

modkeys = {
    K_LSHIFT: 'shift', K_RSHIFT: 'shift',
    K_LCTRL:  'ctrl', K_RCTRL:  'ctrl',
    K_LALT:   'alt', K_RALT:   'alt',
    K_LMETA:  'meta', K_RMETA:  'meta',
}

MUSIC_END_EVENT = USEREVENT + 1

last_mouse_event = Event(0, pos=(0, 0), local=(0, 0))
last_mouse_event_handler = None
root_widget = None     # Root of the containment hierarchy
top_widget = None      # Initial dispatch target
clicked_widget = None  # Target of mouse_drag and mouse_up events

#---------------------------------------------------------------------------

class Cancel(Exception):
    pass

#---------------------------------------------------------------------------

def set_modifier(key, value):
    attr = modkeys.get(key)
    if attr:
        modifiers[attr] = value

def add_modifiers(event):
    d = event.dict
    d.update(modifiers)
    d['cmd'] = event.ctrl or event.meta

def get_root():
    return root_widget

def get_top_widget():
    return top_widget

def get_focus():
    return top_widget.get_focus()

#---------------------------------------------------------------------------

class RootWidget(Widget):
    #  surface   Pygame display surface
    #  is_gl     True if OpenGL surface

    redraw_every_frame = False
    do_draw = False
    _is_gl_container = True

    def __init__(self, surface):
        global root_widget
        Widget.__init__(self, surface.get_rect())
        self.surface = surface
        root_widget = self
        widget.root_widget = self
        self.is_gl = surface.get_flags() & OPENGL != 0
        self.idle_handlers = []

    def set_timer(self, ms):
        pygame.time.set_timer(USEREVENT, ms)

    def run(self):
        self.run_modal(None)

    captured_widget = None

    def capture_mouse(self, widget):
        #put the mouse in "virtual mode" and pass mouse moved events to the
        #specified widget
        if widget:
            pygame.mouse.set_visible(False)
            pygame.event.set_grab(True)
            get_root().captured_widget = widget
        else:
            pygame.mouse.set_visible(True)
            pygame.event.set_grab(False)
            get_root().captured_widget = None

    frames = 0
    hover_widget = None

    def run_modal(self, modal_widget):
        old_captured_widget = None

        if self.captured_widget:
            old_captured_widget = self.captured_widget
            self.capture_mouse(None)

        global last_mouse_event, last_mouse_event_handler
        global top_widget, clicked_widget
        is_modal = modal_widget is not None
        modal_widget = modal_widget or self
        from OpenGL import GL
        try:
            old_top_widget = top_widget
            top_widget = modal_widget
            was_modal = modal_widget.is_modal
            modal_widget.is_modal = True
            modal_widget.modal_result = None
            if not modal_widget.focus_switch:
                modal_widget.tab_to_first()
            mouse_widget = None
            if clicked_widget:
                clicked_widget = modal_widget
            num_clicks = 0
            last_click_time = start_time
            last_click_button = 0
            self.do_draw = True

            while modal_widget.modal_result is None:
                try:
                    self.hover_widget = self.find_widget(mouse.get_pos())
                    if self.do_draw:
                        if self.is_gl:
                            self.gl_clear()
                            self.gl_draw_all(self, (0, 0))
                            GL.glFlush()
                        else:
                            self.draw_all(self.surface)
                        self.do_draw = False
                        pygame.display.flip()
                        self.frames += 1
                    #events = [pygame.event.wait()]
                    events = [pygame.event.poll()]
                    events.extend(pygame.event.get())
                    for event in events:
                        if event.type:
                            log.debug("%s", event)

                        type = event.type
                        if type == QUIT:
                            self.quit()
                        elif type == MOUSEBUTTONDOWN:
                            self.do_draw = True
                            t = datetime.now()
                            if t - last_click_time <= double_click_time and event.button == last_click_button:
                                num_clicks += 1
                            else:
                                num_clicks = 1
                            last_click_button = event.button
                            last_click_time = t
                            event.dict['num_clicks'] = num_clicks
                            add_modifiers(event)
                            mouse_widget = self.find_widget(event.pos)
                            if self.captured_widget:
                                mouse_widget = self.captured_widget

                            if not mouse_widget.is_inside(modal_widget):
                                mouse_widget = modal_widget
                            #if event.button == 1:
                            clicked_widget = mouse_widget
                            last_mouse_event_handler = mouse_widget
                            last_mouse_event = event
                            mouse_widget.notify_attention_loss()
                            mouse_widget.handle_mouse('mouse_down', event)
                        elif type == MOUSEMOTION:
                            self.do_draw = True
                            add_modifiers(event)
                            modal_widget.dispatch_key('mouse_delta', event)
                            last_mouse_event = event

                            mouse_widget = self.update_tooltip(event.pos)

                            if clicked_widget:
                                last_mouse_event_handler = clicked_widget
                                clicked_widget.handle_mouse('mouse_drag', event)
                            else:
                                if not mouse_widget.is_inside(modal_widget):
                                    mouse_widget = modal_widget
                                last_mouse_event_handler = mouse_widget
                                mouse_widget.handle_mouse('mouse_move', event)
                        elif type == MOUSEBUTTONUP:
                            add_modifiers(event)
                            self.do_draw = True
                            mouse_widget = self.find_widget(event.pos)
                            if self.captured_widget:
                                mouse_widget = self.captured_widget
                            if clicked_widget:
                                last_mouse_event_handler = clicked_widget
                                event.dict['clicked_widget'] = clicked_widget
                            else:
                                last_mouse_event_handler = mouse_widget
                                event.dict['clicked_widget'] = None

                            last_mouse_event = event
                            clicked_widget = None
                            last_mouse_event_handler.handle_mouse('mouse_up', event)
                        elif type == KEYDOWN:
                            key = event.key
                            set_modifier(key, True)
                            self.do_draw = True
                            self.send_key(modal_widget, 'key_down', event)
                            if last_mouse_event_handler:
                                event.dict['pos'] = last_mouse_event.pos
                                event.dict['local'] = last_mouse_event.local
                                last_mouse_event_handler.setup_cursor(event)
                        elif type == KEYUP:
                            key = event.key
                            set_modifier(key, False)
                            self.do_draw = True
                            self.send_key(modal_widget, 'key_up', event)
                            if last_mouse_event_handler:
                                event.dict['pos'] = last_mouse_event.pos
                                event.dict['local'] = last_mouse_event.local
                                last_mouse_event_handler.setup_cursor(event)
                        elif type == MUSIC_END_EVENT:
                            self.music_end()
                        elif type == USEREVENT:
                            make_scheduled_calls()
                            if not is_modal:
                                self.do_draw = self.redraw_every_frame
                                if last_mouse_event_handler:
                                    event.dict['pos'] = last_mouse_event.pos
                                    event.dict['local'] = last_mouse_event.local
                                    add_modifiers(event)
                                    last_mouse_event_handler.setup_cursor(event)
                                self.begin_frame()
                        elif type == VIDEORESIZE:
                            #add_modifiers(event)
                            self.do_draw = True
                            self.size = (event.w, event.h)
                            #self.dispatch_key('reshape', event)
                        elif type == ACTIVEEVENT:
                            add_modifiers(event)
                            self.dispatch_key('activeevent', event)
                        elif type == NOEVENT:
                            add_modifiers(event)
                            self.call_idle_handlers(event)

                except Cancel:
                    pass
        finally:
            modal_widget.is_modal = was_modal
            top_widget = old_top_widget
            if old_captured_widget:
                self.capture_mouse(old_captured_widget)

        clicked_widget = None

    def call_idle_handlers(self, event):
        def call(ref):
            widget = ref()
            if widget:
                widget.idleevent(event)
            else:
                print "Idle ref died!"
            return bool(widget)

        self.idle_handlers = filter(call, self.idle_handlers)

    def add_idle_handler(self, widget):
        from weakref import ref
        self.idle_handlers.append(ref(widget))

    def remove_idle_handler(self, widget):
        from weakref import ref
        self.idle_handlers.remove(ref(widget))

    def send_key(self, widget, name, event):
        add_modifiers(event)
        widget.dispatch_key(name, event)

    def begin_frame(self):
        pass

    def get_root(self):
        return self

    labelClass = lambda s, t: wrapped_label(t, 45)

    def show_tooltip(self, widget, pos):

        if hasattr(self, 'currentTooltip'):
            if self.currentTooltip != None:
                self.remove(self.currentTooltip)

            self.currentTooltip = None

        def TextTooltip(text):
            tooltipBacking = Panel()
            tooltipBacking.bg_color = (0.0, 0.0, 0.0, 0.6)
            tooltipBacking.add(self.labelClass(text))
            tooltipBacking.shrink_wrap()
            return tooltipBacking

        def showTip(tip):
            tip.topleft = pos
            tip.top += 20
            if (tip.bottom > self.bottom) or hasattr(widget, 'tooltipsUp'):
                tip.bottomleft = pos
                tip.top -= 4
            if tip.right > self.right:
                tip.right = pos[0]

            self.add(tip)
            self.currentTooltip = tip

        if widget.tooltip is not None:
            tip = widget.tooltip
            showTip(tip)

        else:
            ttext = widget.tooltipText
            if ttext is not None:
                tip = TextTooltip(ttext)
                showTip(tip)

    def update_tooltip(self, pos=None):
        if pos is None:
            pos = mouse.get_pos()
        if self.captured_widget:
            mouse_widget = self.captured_widget
            pos = mouse_widget.center
        else:
            mouse_widget = self.find_widget(pos)

        self.show_tooltip(mouse_widget, pos)
        return mouse_widget

    def has_focus(self):
        return True

    def quit(self):
        if self.confirm_quit():
            self.capture_mouse(None)
            sys.exit(0)

    def confirm_quit(self):
        return True

    def get_mouse_for(self, widget):
        last = last_mouse_event
        event = Event(0, last.dict)
        event.dict['local'] = widget.global_to_local(event.pos)
        add_modifiers(event)
        return event

    def gl_clear(self):
        from OpenGL import GL
        bg = self.bg_color
        if bg:
            r = bg[0] / 255.0
            g = bg[1] / 255.0
            b = bg[2] / 255.0
            GL.glClearColor(r, g, b, 0.0)
        GL.glClear(GL.GL_COLOR_BUFFER_BIT | GL.GL_DEPTH_BUFFER_BIT)

    def music_end(self):
        import music
        music.music_end()

#---------------------------------------------------------------------------

from time import time
from bisect import insort

scheduled_calls = []

def make_scheduled_calls():
    sched = scheduled_calls
    t = time()
    while sched and sched[0][0] <= t:
        sched[0][1]()
        sched.pop(0)

def schedule(delay, func):
    """Arrange for the given function to be called after the specified
    delay in seconds. Scheduled functions are called synchronously from
    the event loop, and only when the frame timer is running."""
    t = time() + delay
    insort(scheduled_calls, (t, func))
<|endoftext|>


<commit_before>import qi
import argparse
import sys
import time
import threading
import math
import os
import threading
from naoqi import ALProxy
import vision_definitions
import cv2 as cv
import numpy as np

import action_base
from action_base import *

import conditions
from conditions import set_condition

"""
This action launches the object detection framework. In order to work, it needs darknetsrv.py running in the GPU and exposing the service.
It uses only 1 param: time between consecutive image captures/detections. The higher, the more network load I will be putting.

Upon detection, it will be publishing into ALMemory, under the address Actions/DarknetPerception/[detected object name]

Timestamp in seconds from the used image:
"Actions/DarknetPerception/"+name+"/timestamp"
Detection probability or confidence of the NN
"Actions/DarknetPerception/"+name+"/Confidence"

Bounding box data (relative to the image)
"Actions/DarknetPerception/"+name+"/BBox/Xmin"
"Actions/DarknetPerception/"+name+"/BBox/Ymin"
"Actions/DarknetPerception/"+name+"/BBox/Xmax"
"Actions/DarknetPerception/"+name+"/BBox/Ymax"

"""

actionName = "iswaving"

memory_service = None
video_service = None
DarknetSRV = None

# bounding box height ratio upper/lower chunks
factor = 1/3.0

# minimum global flow to start considering waving. Should filter small flows, as noise
global_flow_thres = 0.4

# minimum probability to trigger event
flow_event_thres = 0.5

def imcrop(img, bbox):
    x1,y1,x2,y2 = bbox
    if x1 < 0 or y1 < 0 or x2 > img.shape[1] or y2 > img.shape[0]:
        img, x1, x2, y1, y2 = pad_img_to_fit_bbox(img, x1, x2, y1, y2)

    if img.ndim==3:
        return img[y1:y2, x1:x2, :]
    else:
        return img[y1:y2, x1:x2]

def pad_img_to_fit_bbox(img, x1, x2, y1, y2):
    if img.ndim==3:
        img = np.pad(img, ((np.abs(np.minimum(0, y1)), np.maximum(y2 - img.shape[0], 0)),
               (np.abs(np.minimum(0, x1)), np.maximum(x2 - img.shape[1], 0)), (0,0)), mode="constant")
    else:
        img = np.pad(img, ((np.abs(np.minimum(0, y1)), np.maximum(y2 - img.shape[0], 0)),
                  (np.abs(np.minimum(0, x1)), np.maximum(x2 - img.shape[1], 0))), mode="constant")

    y1 += np.abs(np.minimum(0, y1))
    y2 += np.abs(np.minimum(0, y1))
    x1 += np.abs(np.minimum(0, x1))
    x2 += np.abs(np.minimum(0, x1))
    return img, x1, x2, y1, y2

def biggestTuple(tA,tB):
    (Ax_min, Ay_min, Ax_max, Ay_max )= tA
    (Bx_min, By_min, Bx_max, By_max )= tB

    tC = ( min(Ax_min,Bx_min) , min(Ay_min,By_min), max(Ax_max,Bx_max), max(Ay_max,By_max))
    return tC

def getPixelT(ri):
    x = ri[2][0]
    y = ri[2][1]
    w = ri[2][2]
    z = ri[2][3]

    x_max = int(round((2*x+w)/2))
    x_min = int(round((2*x-w)/2))
    y_min = int(round((2*y-z)/2))
    y_max = int(round((2*y+z)/2))

    pixel_list = [ x_min, y_min, x_max, y_max]
    pixel_tuple = (x_min, y_min, x_max, y_max)
    return (pixel_list, pixel_tuple)

def isCorrectImage(result):
    ans = False
    if result == None:
        print 'cannot capture.'
    elif result[6] == None:
        print 'no image data string.'
    else:
        ans = True
    return ans

def image_qi2np(dataImage):
    image = None
    if( dataImage != None ):
        image = np.reshape( np.frombuffer(dataImage[6], dtype='%iuint8' % dataImage[2]), (dataImage[1], dataImage[0], dataImage[2]))
        # image = np.fromstring(str(alImage[6]), dtype=np.uint8).reshape( alImage[1],alImage[0], dataImage[2])
    return image

def image_np2cv(npImage):
        # dirty way to use in cv2 or cv3
        if cv.__version__ == '3.3.1-dev':
            open_cv_image = cv.cvtColor(npImage, cv.COLOR_BGR2RGB)
        else:
            open_cv_image = cv.cvtColor(npImage, cv.cv.CV_BGR2RGB)
        return open_cv_image

def image_qi2cv(qiImg):
    npImg = image_qi2np(qiImg)
    cvImg = image_np2cv(npImg)
    return cvImg

def module(x,y):
    return np.sqrt((x*x)+(y*y))

def wavingThread (params):
    # This is awful....
    global actionName
    global memory_service
    global video_service
    global imgClient
    global DarknetSRV

    global factor
    global global_flow_thres
    global flow_event_thres

    t = threading.currentThread()

    paramList = params.split('_')
    sampleInterval = float(paramList[0])
    throttleInterval = float(paramList[1])

    print actionName+" thread started. Sample Period: "+ str(sampleInterval) + ", throttle Period: " + str(throttleInterval)

    while getattr(t, "do_run", True):
        # get two images with time spacing...
        isOk = False
        while not isOk:
            img0 = video_service.getImageRemote(imgClient)
            time.sleep(sampleInterval)
            img1 = video_service.getImageRemote(imgClient)
            isOk = isCorrectImage(img0) and isCorrectImage(img1)

        timestampSecs =  img1[4]
        timestampMicrosecs =  img1[5]

        # detect stuff on them
        r0 = DarknetSRV.identify(img0)
        r1 = DarknetSRV.identify(img1)

        img0 = image_qi2cv(img0)
        img1 = image_qi2cv(img1)

        if (r0 != []) and (r1 != []):
            cnt = 0
            peopleCounter = 0
            while cnt < (min(len(r0),len(r1))):

                # why using same counter for both? I'm assuming very similar probabilities between frames, hence order in detection.
                # I know this is naive...
                name0 = r0[cnt][0]
                predict0 = r0[cnt][1]

                name1 = r1[cnt][0]
                predict1 = r1[cnt][1]

                # this prevents calculating flow between a dog and a person ...
                isPerson = ('person' in name0)
                isPerson = isPerson and ('person' in name1)

                if  isPerson:
                    humanProb = min(predict0,predict1)

                    (pixel_list0, pixel_tuple0) = getPixelT(r0[cnt])
                    (pixel_list1, pixel_tuple1) = getPixelT(r1[cnt])

                    max_tuple = biggestTuple(pixel_tuple1,pixel_tuple0)
                    (x_min, y_min, x_max, y_max) = max_tuple
                    gray0 = cv.cvtColor(img0, cv.COLOR_BGR2GRAY)
                    gray1 = cv.cvtColor(img1, cv.COLOR_BGR2GRAY)

                    gray_cropped0 = imcrop(gray0,max_tuple)
                    gray_cropped1 = imcrop(gray1,max_tuple)

                    flow = cv.calcOpticalFlowFarneback(gray_cropped1, gray_cropped0, None, 0.5, 3, 15, 3, 5, 1.2, 0)

                    h, w =  gray_cropped1.shape[:2]

                    hi = int(h*factor)
                    upper_v = (np.mean(np.mean( flow[ 0:hi,:,],axis=0),axis=0))
                    lower_v = (np.mean(np.mean( flow[hi:  ,:,],axis=0),axis=0))

                    #probability of a wave=upper_half/(upper + lower)
                    global_flow = module((upper_v[0]+lower_v[0])/2.0,(upper_v[1]+lower_v[1])/2)
                    up_flow = module(upper_v[0],upper_v[1])
                    down_flow = module(lower_v[0],lower_v[1])

                    if global_flow>global_flow_thres:
                        waveProb =  up_flow/(up_flow+down_flow)
                    else:
                        waveProb = 0

                    peopleCounter+=1

                    mem_key_event  = "Actions/PeopleWaving/NewDetection"

                    mem_key0 = "Actions/PeopleWaving/"+"person{0:02}".format(peopleCounter)

                    mem_key = mem_key0 + "/WaveProbability"
                    memory_service.insertData(mem_key, waveProb)

                    mem_key = mem_key0 + "/BBox/Xmin"
                    memory_service.insertData(mem_key, x_min)

                    mem_key = mem_key0 + "/BBox/Ymin"
                    memory_service.insertData(mem_key, y_min)

                    mem_key = mem_key0 + "/BBox/Xmax"
                    memory_service.insertData(mem_key, x_max)

                    mem_key = mem_key0 + "/BBox/Ymax"
                    memory_service.insertData(mem_key, y_max)

                    mem_key = mem_key0 + "/timestamp"
                    memory_service.insertData(mem_key, timestampSecs)

                    isEvent = (waveProb>=flow_event_thres)
                    memory_service.raiseEvent(mem_key_event,isEvent)

                    if isEvent:
                        print "is waving me!"
                cnt+=1

        print ("-------------------------\n\n")
        time.sleep(throttleInterval)
    print actionName+" thread quit"

def init(session):
    global memory_service
    global video_service
    global imgClient
    global DarknetSRV

    camera = 0 # upper camera
    memory_service =  session.service("ALMemory")
    video_service = session.service("ALVideoDevice")
    resolution = vision_definitions.kQVGA  # kQVGA =320 * 240  ,kVGA =640x480
    colorSpace = vision_definitions.kRGBColorSpace

    name = time.strftime('imageclient_%S')
    imgClient = video_service.subscribe(name, resolution, colorSpace, 5)

    # Select camera.
    video_service.setParam(vision_definitions.kCameraSelectID, camera)

    # Services
    DarknetSRV = session.service("DarknetSRV")

    action_base.init(session, actionName, wavingThread)

def quit():
    global actionName
    print actionName+" quit"
    wavingThread.do_run = False

if __name__ == "__main__":

    app = action_base.initApp(actionName)

    init(app.session)

    #Program stays at this point until we stop it
    app.run()

    quit()

<commit_msg>ALMemory ...<commit_after>import qi
import argparse
import sys
import time
import threading
import math
import os
import threading
from naoqi import ALProxy
import vision_definitions
import cv2 as cv
import numpy as np

import action_base
from action_base import *

import conditions
from conditions import set_condition

"""
This action launches the object detection framework. In order to work, it needs darknetsrv.py running in the GPU and exposing the service.
It uses only 1 param: time between consecutive image captures/detections. The higher, the more network load I will be putting.

Upon detection, it will be publishing into ALMemory, under the address Actions/DarknetPerception/[detected object name]

Timestamp in seconds from the used image:
"Actions/DarknetPerception/"+name+"/timestamp"
Detection probability or confidence of the NN
"Actions/DarknetPerception/"+name+"/Confidence"

Bounding box data (relative to the image)
"Actions/DarknetPerception/"+name+"/BBox/Xmin"
"Actions/DarknetPerception/"+name+"/BBox/Ymin"
"Actions/DarknetPerception/"+name+"/BBox/Xmax"
"Actions/DarknetPerception/"+name+"/BBox/Ymax"

"""

actionName = "iswaving"

memory_service = None
video_service = None
DarknetSRV = None

# bounding box height ratio upper/lower chunks
factor = 1/3.0

# minimum global flow to start considering waving. Should filter small flows, as noise
global_flow_thres = 0.4

# minimum probability to trigger event
flow_event_thres = 0.5

def imcrop(img, bbox):
    x1,y1,x2,y2 = bbox
    if x1 < 0 or y1 < 0 or x2 > img.shape[1] or y2 > img.shape[0]:
        img, x1, x2, y1, y2 = pad_img_to_fit_bbox(img, x1, x2, y1, y2)

    if img.ndim==3:
        return img[y1:y2, x1:x2, :]
    else:
        return img[y1:y2, x1:x2]

def pad_img_to_fit_bbox(img, x1, x2, y1, y2):
    if img.ndim==3:
        img = np.pad(img, ((np.abs(np.minimum(0, y1)), np.maximum(y2 - img.shape[0], 0)),
               (np.abs(np.minimum(0, x1)), np.maximum(x2 - img.shape[1], 0)), (0,0)), mode="constant")
    else:
        img = np.pad(img, ((np.abs(np.minimum(0, y1)), np.maximum(y2 - img.shape[0], 0)),
                  (np.abs(np.minimum(0, x1)), np.maximum(x2 - img.shape[1], 0))), mode="constant")

    y1 += np.abs(np.minimum(0, y1))
    y2 += np.abs(np.minimum(0, y1))
    x1 += np.abs(np.minimum(0, x1))
    x2 += np.abs(np.minimum(0, x1))
    return img, x1, x2, y1, y2

def biggestTuple(tA,tB):
    (Ax_min, Ay_min, Ax_max, Ay_max )= tA
    (Bx_min, By_min, Bx_max, By_max )= tB

    tC = ( min(Ax_min,Bx_min) , min(Ay_min,By_min), max(Ax_max,Bx_max), max(Ay_max,By_max))
    return tC

def getPixelT(ri):
    x = ri[2][0]
    y = ri[2][1]
    w = ri[2][2]
    z = ri[2][3]

    x_max = int(round((2*x+w)/2))
    x_min = int(round((2*x-w)/2))
    y_min = int(round((2*y-z)/2))
    y_max = int(round((2*y+z)/2))

    pixel_list = [ x_min, y_min, x_max, y_max]
    pixel_tuple = (x_min, y_min, x_max, y_max)
    return (pixel_list, pixel_tuple)

def isCorrectImage(result):
    ans = False
    if result == None:
        print 'cannot capture.'
    elif result[6] == None:
        print 'no image data string.'
    else:
        ans = True
    return ans

def image_qi2np(dataImage):
    image = None
    if( dataImage != None ):
        image = np.reshape( np.frombuffer(dataImage[6], dtype='%iuint8' % dataImage[2]), (dataImage[1], dataImage[0], dataImage[2]))
        # image = np.fromstring(str(alImage[6]), dtype=np.uint8).reshape( alImage[1],alImage[0], dataImage[2])
    return image

def image_np2cv(npImage):
        # dirty way to use in cv2 or cv3
        if cv.__version__ == '3.3.1-dev':
            open_cv_image = cv.cvtColor(npImage, cv.COLOR_BGR2RGB)
        else:
            open_cv_image = cv.cvtColor(npImage, cv.cv.CV_BGR2RGB)
        return open_cv_image

def image_qi2cv(qiImg):
    npImg = image_qi2np(qiImg)
    cvImg = image_np2cv(npImg)
    return cvImg

def module(x,y):
    return np.sqrt((x*x)+(y*y))

def wavingThread (params):
    # This is awful....
    global actionName
    global memory_service
    global video_service
    global imgClient
    global DarknetSRV

    global factor
    global global_flow_thres
    global flow_event_thres

    t = threading.currentThread()

    paramList = params.split('_')
    sampleInterval = float(paramList[0])
    throttleInterval = float(paramList[1])

    print actionName+" thread started. Sample Period: "+ str(sampleInterval) + ", throttle Period: " + str(throttleInterval)

    while getattr(t, "do_run", True):
        # get two images with time spacing...
        isOk = False
        while not isOk:
            img0 = video_service.getImageRemote(imgClient)
            time.sleep(sampleInterval)
            img1 = video_service.getImageRemote(imgClient)
            isOk = isCorrectImage(img0) and isCorrectImage(img1)

        timestampSecs =  img1[4]
        timestampMicrosecs =  img1[5]

        # detect stuff on them
        r0 = DarknetSRV.identify(img0)
        r1 = DarknetSRV.identify(img1)

        img0 = image_qi2cv(img0)
        img1 = image_qi2cv(img1)

        if (r0 != []) and (r1 != []):
            cnt = 0
            peopleCounter = 0
            while cnt < (min(len(r0),len(r1))):

                # why using same counter for both? I'm assuming very similar probabilities between frames, hence order in detection.
                # I know this is naive...
                name0 = r0[cnt][0]
                predict0 = r0[cnt][1]

                name1 = r1[cnt][0]
                predict1 = r1[cnt][1]

                # this prevents calculating flow between a dog and a person ...
                isPerson = ('person' in name0)
                isPerson = isPerson and ('person' in name1)

                if  isPerson:
                    humanProb = min(predict0,predict1)

                    (pixel_list0, pixel_tuple0) = getPixelT(r0[cnt])
                    (pixel_list1, pixel_tuple1) = getPixelT(r1[cnt])

                    max_tuple = biggestTuple(pixel_tuple1,pixel_tuple0)
                    (x_min, y_min, x_max, y_max) = max_tuple
                    gray0 = cv.cvtColor(img0, cv.COLOR_BGR2GRAY)
                    gray1 = cv.cvtColor(img1, cv.COLOR_BGR2GRAY)

                    gray_cropped0 = imcrop(gray0,max_tuple)
                    gray_cropped1 = imcrop(gray1,max_tuple)

                    flow = cv.calcOpticalFlowFarneback(gray_cropped1, gray_cropped0, None, 0.5, 3, 15, 3, 5, 1.2, 0)

                    h, w =  gray_cropped1.shape[:2]

                    hi = int(h*factor)
                    upper_v = (np.mean(np.mean( flow[ 0:hi,:,],axis=0),axis=0))
                    lower_v = (np.mean(np.mean( flow[hi:  ,:,],axis=0),axis=0))

                    #probability of a wave=upper_half/(upper + lower)
                    global_flow = module((upper_v[0]+lower_v[0])/2.0,(upper_v[1]+lower_v[1])/2)
                    up_flow = module(upper_v[0],upper_v[1])
                    down_flow = module(lower_v[0],lower_v[1])

                    if global_flow>global_flow_thres:
                        waveProb =  up_flow/(up_flow+down_flow)
                    else:
                        waveProb = 0.0

                    peopleCounter+=1

                    mem_key_event  = "Actions/PeopleWaving/NewDetection"

                    mem_key0 = "Actions/PeopleWaving/"+"person{0:02}".format(peopleCounter)

                    mem_key = mem_key0 + "/WaveProbability"
                    memory_service.insertData(mem_key, str(waveProb))

                    mem_key = mem_key0 + "/BBox/Xmin"
                    memory_service.insertData(mem_key, str(x_min))

                    mem_key = mem_key0 + "/BBox/Ymin"
                    memory_service.insertData(mem_key, str(y_min))

                    mem_key = mem_key0 + "/BBox/Xmax"
                    memory_service.insertData(mem_key, str(x_max))

                    mem_key = mem_key0 + "/BBox/Ymax"
                    memory_service.insertData(mem_key, str(y_max))

                    mem_key = mem_key0 + "/timestamp"
                    memory_service.insertData(mem_key, str(timestampSecs))

                    isEvent = (waveProb>=flow_event_thres)
                    memory_service.raiseEvent(mem_key_event,isEvent)

                    if isEvent:
                        print "is waving me!"
                cnt+=1

        print ("-------------------------\n\n")
        time.sleep(throttleInterval)
    print actionName+" thread quit"

def init(session):
    global memory_service
    global video_service
    global imgClient
    global DarknetSRV

    camera = 0 # upper camera
    memory_service =  session.service("ALMemory")
    video_service = session.service("ALVideoDevice")
    resolution = vision_definitions.kQVGA  # kQVGA =320 * 240  ,kVGA =640x480
    colorSpace = vision_definitions.kRGBColorSpace

    name = time.strftime('imageclient_%S')
    imgClient = video_service.subscribe(name, resolution, colorSpace, 5)

    # Select camera.
    video_service.setParam(vision_definitions.kCameraSelectID, camera)

    # Services
    DarknetSRV = session.service("DarknetSRV")

    action_base.init(session, actionName, wavingThread)

def quit():
    global actionName
    print actionName+" quit"
    wavingThread.do_run = False

if __name__ == "__main__":

    app = action_base.initApp(actionName)

    init(app.session)

    #Program stays at this point until we stop it
    app.run()

    quit()

<|endoftext|>


<commit_before>import numpy as np
import sklearn.linear_model as lm

# un-comment if you have compile enet_solver
print '0'
import pyximport; pyximport.install()
import adascreen.enet_solver
# import imp
# try:
#     print '1'
#     import pyximport; pyximport.install()
#     import enet_solver
#     print '3'
#     imp.find_module('enet_solver')
#     found_enet_solver = True
# except ImportError:
#     found_enet_solver = False

# un-comment if you have installed glmnet
import imp
try:
    imp.find_module('glmnet')
    found_glmnet_solver = True
    import glmnet.glmnet as glmnet
except ImportError:
    found_glmnet_solver = False

class AbstractSolver(object):
    """ Wrapper to enable the use of various solver for ElasticNet and Lasso. """
    name = 'AbstractSolver'

    def __init__(self, name):
        self.name = name

    def init(self, lmax, lmax_x, X, y, path):
        print('{0}: nothing to initialize.'.format(self.name))

    def solve(self, start_pos, X, y, l1_reg, l2_reg, max_iter=20000, tol=1e-6):
        # return (new_coefs, iterations, dual_gap)
        # set to zero, if not applicable
        pass

    def release(self):
        print('{0}: nothing to release.'.format(self.name))

    def __str__(self):
        return '{0}'.format(self.name)

class SklearnCDSolver(AbstractSolver):
    """ Wrapper for sklearn solver for ElasticNet and Lasso. """
    def __init__(self):
        AbstractSolver.__init__(self, 'SklearnCDSolver')

    def solve(self, start_pos, X, y, l1_reg, l2_reg, max_iter=20000, tol=1e-6):
        # (coefs, dual_gap, eps) = lm.cd_fast.enet_coordinate_descent(start_pos, l1_reg, l2_reg, np.asfortranarray(X), y, max_iter, tol, False)
        # coef_, l1_reg, l2_reg, X, y, max_iter, tol, rng, random, positive)
        coefs, dual_gap, eps, n_iter_ = lm.cd_fast.enet_coordinate_descent(
            start_pos, l1_reg, l2_reg, np.asfortranarray(X), y, max_iter, tol, np.random, 0, 0)
        return coefs, 0, dual_gap

class SklearnLarsSolver(AbstractSolver):
    """ Wrapper for sklearn solver for ElasticNet and Lasso. """
    def __init__(self):
        AbstractSolver.__init__(self, 'SklearnLarsSolver')

    def solve(self, start_pos, X, y, l1_reg, l2_reg, max_iter=20000, tol=1e-6):
        lars = lm.LassoLars(alpha=l1_reg / float(y.size), max_iter=max_iter, fit_path=False, normalize=False, fit_intercept=False)
        lars.coef_ = start_pos
        lars.fit(X, y)
        return (lars.coef_[0], 0, 0)

class ActiveSetCDSolver(AbstractSolver):
    """ Wrapper for our refined sklearn solver for ElasticNet and Lasso. """
    use_active_set = 1

    def __init__(self, use_active_set=1):
        postfix = ''
        if use_active_set==0:
            postfix = ' (active set disabled)'
        AbstractSolver.__init__(self, 'ActiveSetCDSolver{0}'.format(postfix))
        self.use_active_set = use_active_set

    def solve(self, start_pos, X, y, l1_reg, l2_reg, max_iter=20000, tol=1e-6):
        (coefs, gap, eps, iters, n_evals) = enet_solver.enet_coordinate_descent(start_pos, l1_reg, l2_reg, np.asfortranarray(X), y, max_iter, tol, self.use_active_set)
        print iters
        return (coefs, iters, gap)

class GlmnetSolver(AbstractSolver):
    """ Wrapper for Fortran GLMNET Lasso. """
    lmax = 1.0

    def __init__(self):
        AbstractSolver.__init__(self, 'Glmnet')

    def init(self, lmax, lmax_x, X, y, path):
        self.lmax = lmax
        print('{0}: initialization.'.format(self.name))

    def solve(self, start_pos, X, y, l1_reg, l2_reg, max_iter=20000, tol=1e-6):
        n_lambdas, intercept_, coef_, _, _, rsquared_, lambdas, _, jerr  = \
            glmnet.elastic_net(np.asfortranarray(X), np.asfortranarray(y), \
            1.0, flmin=l1_reg/self.lmax, standardize=False)
        return (coef_[:,0], 0, 0.0)

class ProximalGradientSolver(AbstractSolver):
    """ Proximal gradient solver for elastic net. """
    def __init__(self):
        AbstractSolver.__init__(self, 'ProxGradientSolver')

    def solve(self, start_pos, X, y, l1_reg, l2_reg, max_iter=20000, tol=1e-9, debug=False):
        # X \in R^(samples x feats)
        # y \in R^samples
        max_iter = 20000
        b = y[:, np.newaxis].copy()

        # this is what we wanna solve:
        #
        # min_w 1/2 ||y - X w||^2 + l1_reg ||w||_1 + 1/2 l2_reg ||w||^2
        #
        # w \in R^(feats x 1)
        f = lambda w : 0.5*(b-X.dot(w)).T.dot(b-X.dot(w)) \
                        + l1_reg*np.sum(np.abs(w)) + 0.5*l2_reg*w.T.dot(w)
        g = lambda u : 0.5*(b-X.dot(u)).T.dot(b-X.dot(u)) \
                        + 0.5*l2_reg*u.T.dot(u)

        tol = 1e-3
        w = start_pos[:, np.newaxis]
        tk = 1.
        beta = 0.5

        converged = False
        k = 0
        while not converged and max_iter > k:
            # optimization
            grad_w = X.T.dot(X.dot(w)) + l2_reg*w - X.T.dot(b)
            while True:
                z = prox_l1_operator(tk*l1_reg, w-tk*grad_w)
                if g(z) <= g(w) + grad_w.T.dot(z-w) + (1/(2*tk))*np.sum((z-w).T.dot(z-w)):
                    w = z
                    break
                tk *= beta   # reduce step length

            # convergence check
            obj = f(w)
            # check the dual gap first
            theta = (b - X.dot(w)) / l1_reg
            dual_obj = 0.5*b.T.dot(b) - 0.5*l1_reg*l1_reg*(theta-b/l1_reg).T.dot(theta-b/l1_reg) \
                       + 0.5*l2_reg*w.T.dot(w)
            # print k, ' - ', obj, dual_obj, np.abs(obj-dual_obj)
            if k > 0 and np.abs(obj-dual_obj) < tol:
                converged = True
            k += 1

        if debug:
            # baseline CD solver for comparison
            coefs, dual_gap, eps, n_iter_ = lm.cd_fast.enet_coordinate_descent(
                start_pos, l1_reg, l2_reg, np.asfortranarray(X), b.reshape((b.size)), max_iter, tol, np.random, 0, 0)
            print k, ' - ', np.abs(f(coefs[:, np.newaxis]) - f(w))
        print k

        return w.reshape((w.size)), 0, 0.

class AccelProximalGradientSolver(AbstractSolver):
    """ Proximal gradient solver for elastic net. """
    def __init__(self):
        AbstractSolver.__init__(self, 'AccelProxGradientSolver')

    def solve(self, start_pos, X, y, l1_reg, l2_reg, max_iter=20000, tol=1e-6, debug=False):
        # X \in R^(samples x feats)
        # y \in R^samples
        #max_iter = 20000
        b = y[:, np.newaxis]

        # print tol
        #tol = 1e-6

        # this is what we wanna solve:
        #
        # min_w 1/2 ||y - X w||^2 + l1_reg ||w||_1 + 1/2 l2_reg ||w||^2
        #
        # w \in R^(feats x 1)
        f = lambda w : 0.5*(b-X.dot(w)).T.dot(b-X.dot(w)) \
                        + l1_reg*np.sum(np.abs(w)) + 0.5*l2_reg*w.T.dot(w)
        g = lambda u : 0.5*(b-X.dot(u)).T.dot(b-X.dot(u)) \
                        + 0.5*l2_reg*u.T.dot(u)

        w = start_pos[:, np.newaxis]
        w_old = w
        tk = 1.
        beta = 0.5

        converged = False
        k = 0
        while not converged and max_iter > k:
            # optimization
            y = w + (k/(k+3))*(w - w_old)
            grad_y = X.T.dot(X.dot(y)) + l2_reg*y - X.T.dot(b)
            while True:
                z = prox_l1_operator(tk*l1_reg, w-tk*grad_y)
                if g(z) <= g(y) + grad_y.T.dot(z-y) + (1/(2*tk))*np.sum((z-y).T.dot(z-y)):
                    w_old = w
                    w = z
                    break
                tk *= beta   # reduce step length

            # convergence check
            obj = f(w)
            # check the dual gap first
            theta = (b - X.dot(w)) / l1_reg
            dual_obj = 0.5*b.T.dot(b) - 0.5*l1_reg*l1_reg*(theta-b/l1_reg).T.dot(theta-b/l1_reg) \
                       + 0.5*l2_reg*w.T.dot(w)
            # print k, ' - ', obj, dual_obj, np.abs(obj-dual_obj)
            if k > 0 and np.abs(obj-dual_obj) < tol:
                converged = True
            k += 1
        print k

        if debug:
            # baseline CD solver for comparison
            coefs, dual_gap, eps, n_iter_ = lm.cd_fast.enet_coordinate_descent(
                start_pos, l1_reg, l2_reg, np.asfortranarray(X), b.reshape((b.size)), max_iter, tol, np.random, 0, 0)
            print k, ' - ', np.abs(f(coefs[:, np.newaxis]) - f(w))
        return w.reshape((w.size)), 0, 0.

def prox_l1_operator(t, u):
    u_star1 = np.hstack([ u-t*np.ones((u.size, 1)), np.zeros((u.size, 1))])
    u_star2 = np.hstack([-u-t*np.ones((u.size, 1)), np.zeros((u.size, 1))])
    z = np.max(u_star1, axis=1) - np.max(u_star2, axis=1)
    return z[:, np.newaxis]<commit_msg>setup builds cython code<commit_after>import numpy as np
import sklearn.linear_model as lm

# un-comment if you have compile enet_solver
print '0'
import pyximport; pyximport.install()
import enet_solver
# import imp
# try:
#     print '1'
#     import pyximport; pyximport.install()
#     import enet_solver
#     print '3'
#     imp.find_module('enet_solver')
#     found_enet_solver = True
# except ImportError:
#     found_enet_solver = False

# un-comment if you have installed glmnet
import imp
try:
    imp.find_module('glmnet')
    found_glmnet_solver = True
    import glmnet.glmnet as glmnet
except ImportError:
    found_glmnet_solver = False

class AbstractSolver(object):
    """ Wrapper to enable the use of various solver for ElasticNet and Lasso. """
    name = 'AbstractSolver'

    def __init__(self, name):
        self.name = name

    def init(self, lmax, lmax_x, X, y, path):
        print('{0}: nothing to initialize.'.format(self.name))

    def solve(self, start_pos, X, y, l1_reg, l2_reg, max_iter=20000, tol=1e-6):
        # return (new_coefs, iterations, dual_gap)
        # set to zero, if not applicable
        pass

    def release(self):
        print('{0}: nothing to release.'.format(self.name))

    def __str__(self):
        return '{0}'.format(self.name)

class SklearnCDSolver(AbstractSolver):
    """ Wrapper for sklearn solver for ElasticNet and Lasso. """
    def __init__(self):
        AbstractSolver.__init__(self, 'SklearnCDSolver')

    def solve(self, start_pos, X, y, l1_reg, l2_reg, max_iter=20000, tol=1e-6):
        # (coefs, dual_gap, eps) = lm.cd_fast.enet_coordinate_descent(start_pos, l1_reg, l2_reg, np.asfortranarray(X), y, max_iter, tol, False)
        # coef_, l1_reg, l2_reg, X, y, max_iter, tol, rng, random, positive)
        coefs, dual_gap, eps, n_iter_ = lm.cd_fast.enet_coordinate_descent(
            start_pos, l1_reg, l2_reg, np.asfortranarray(X), y, max_iter, tol, np.random, 0, 0)
        return coefs, 0, dual_gap

class SklearnLarsSolver(AbstractSolver):
    """ Wrapper for sklearn solver for ElasticNet and Lasso. """
    def __init__(self):
        AbstractSolver.__init__(self, 'SklearnLarsSolver')

    def solve(self, start_pos, X, y, l1_reg, l2_reg, max_iter=20000, tol=1e-6):
        lars = lm.LassoLars(alpha=l1_reg / float(y.size), max_iter=max_iter, fit_path=False, normalize=False, fit_intercept=False)
        lars.coef_ = start_pos
        lars.fit(X, y)
        return (lars.coef_[0], 0, 0)

class ActiveSetCDSolver(AbstractSolver):
    """ Wrapper for our refined sklearn solver for ElasticNet and Lasso. """
    use_active_set = 1

    def __init__(self, use_active_set=1):
        postfix = ''
        if use_active_set==0:
            postfix = ' (active set disabled)'
        AbstractSolver.__init__(self, 'ActiveSetCDSolver{0}'.format(postfix))
        self.use_active_set = use_active_set

    def solve(self, start_pos, X, y, l1_reg, l2_reg, max_iter=20000, tol=1e-6):
        (coefs, gap, eps, iters, n_evals) = enet_solver.enet_coordinate_descent(start_pos, l1_reg, l2_reg, np.asfortranarray(X), y, max_iter, tol, self.use_active_set)
        print iters
        return (coefs, iters, gap)

class GlmnetSolver(AbstractSolver):
    """ Wrapper for Fortran GLMNET Lasso. """
    lmax = 1.0

    def __init__(self):
        AbstractSolver.__init__(self, 'Glmnet')

    def init(self, lmax, lmax_x, X, y, path):
        self.lmax = lmax
        print('{0}: initialization.'.format(self.name))

    def solve(self, start_pos, X, y, l1_reg, l2_reg, max_iter=20000, tol=1e-6):
        n_lambdas, intercept_, coef_, _, _, rsquared_, lambdas, _, jerr  = \
            glmnet.elastic_net(np.asfortranarray(X), np.asfortranarray(y), \
            1.0, flmin=l1_reg/self.lmax, standardize=False)
        return (coef_[:,0], 0, 0.0)

class ProximalGradientSolver(AbstractSolver):
    """ Proximal gradient solver for elastic net. """
    def __init__(self):
        AbstractSolver.__init__(self, 'ProxGradientSolver')

    def solve(self, start_pos, X, y, l1_reg, l2_reg, max_iter=20000, tol=1e-9, debug=False):
        # X \in R^(samples x feats)
        # y \in R^samples
        max_iter = 20000
        b = y[:, np.newaxis].copy()

        # this is what we wanna solve:
        #
        # min_w 1/2 ||y - X w||^2 + l1_reg ||w||_1 + 1/2 l2_reg ||w||^2
        #
        # w \in R^(feats x 1)
        f = lambda w : 0.5*(b-X.dot(w)).T.dot(b-X.dot(w)) \
                        + l1_reg*np.sum(np.abs(w)) + 0.5*l2_reg*w.T.dot(w)
        g = lambda u : 0.5*(b-X.dot(u)).T.dot(b-X.dot(u)) \
                        + 0.5*l2_reg*u.T.dot(u)

        tol = 1e-3
        w = start_pos[:, np.newaxis]
        tk = 1.
        beta = 0.5

        converged = False
        k = 0
        while not converged and max_iter > k:
            # optimization
            grad_w = X.T.dot(X.dot(w)) + l2_reg*w - X.T.dot(b)
            while True:
                z = prox_l1_operator(tk*l1_reg, w-tk*grad_w)
                if g(z) <= g(w) + grad_w.T.dot(z-w) + (1/(2*tk))*np.sum((z-w).T.dot(z-w)):
                    w = z
                    break
                tk *= beta   # reduce step length

            # convergence check
            obj = f(w)
            # check the dual gap first
            theta = (b - X.dot(w)) / l1_reg
            dual_obj = 0.5*b.T.dot(b) - 0.5*l1_reg*l1_reg*(theta-b/l1_reg).T.dot(theta-b/l1_reg) \
                       + 0.5*l2_reg*w.T.dot(w)
            # print k, ' - ', obj, dual_obj, np.abs(obj-dual_obj)
            if k > 0 and np.abs(obj-dual_obj) < tol:
                converged = True
            k += 1

        if debug:
            # baseline CD solver for comparison
            coefs, dual_gap, eps, n_iter_ = lm.cd_fast.enet_coordinate_descent(
                start_pos, l1_reg, l2_reg, np.asfortranarray(X), b.reshape((b.size)), max_iter, tol, np.random, 0, 0)
            print k, ' - ', np.abs(f(coefs[:, np.newaxis]) - f(w))
        print k

        return w.reshape((w.size)), 0, 0.

class AccelProximalGradientSolver(AbstractSolver):
    """ Proximal gradient solver for elastic net. """
    def __init__(self):
        AbstractSolver.__init__(self, 'AccelProxGradientSolver')

    def solve(self, start_pos, X, y, l1_reg, l2_reg, max_iter=20000, tol=1e-6, debug=False):
        # X \in R^(samples x feats)
        # y \in R^samples
        #max_iter = 20000
        b = y[:, np.newaxis]

        # print tol
        #tol = 1e-6

        # this is what we wanna solve:
        #
        # min_w 1/2 ||y - X w||^2 + l1_reg ||w||_1 + 1/2 l2_reg ||w||^2
        #
        # w \in R^(feats x 1)
        f = lambda w : 0.5*(b-X.dot(w)).T.dot(b-X.dot(w)) \
                        + l1_reg*np.sum(np.abs(w)) + 0.5*l2_reg*w.T.dot(w)
        g = lambda u : 0.5*(b-X.dot(u)).T.dot(b-X.dot(u)) \
                        + 0.5*l2_reg*u.T.dot(u)

        w = start_pos[:, np.newaxis]
        w_old = w
        tk = 1.
        beta = 0.5

        converged = False
        k = 0
        while not converged and max_iter > k:
            # optimization
            y = w + (k/(k+3))*(w - w_old)
            grad_y = X.T.dot(X.dot(y)) + l2_reg*y - X.T.dot(b)
            while True:
                z = prox_l1_operator(tk*l1_reg, w-tk*grad_y)
                if g(z) <= g(y) + grad_y.T.dot(z-y) + (1/(2*tk))*np.sum((z-y).T.dot(z-y)):
                    w_old = w
                    w = z
                    break
                tk *= beta   # reduce step length

            # convergence check
            obj = f(w)
            # check the dual gap first
            theta = (b - X.dot(w)) / l1_reg
            dual_obj = 0.5*b.T.dot(b) - 0.5*l1_reg*l1_reg*(theta-b/l1_reg).T.dot(theta-b/l1_reg) \
                       + 0.5*l2_reg*w.T.dot(w)
            # print k, ' - ', obj, dual_obj, np.abs(obj-dual_obj)
            if k > 0 and np.abs(obj-dual_obj) < tol:
                converged = True
            k += 1
        print k

        if debug:
            # baseline CD solver for comparison
            coefs, dual_gap, eps, n_iter_ = lm.cd_fast.enet_coordinate_descent(
                start_pos, l1_reg, l2_reg, np.asfortranarray(X), b.reshape((b.size)), max_iter, tol, np.random, 0, 0)
            print k, ' - ', np.abs(f(coefs[:, np.newaxis]) - f(w))
        return w.reshape((w.size)), 0, 0.

def prox_l1_operator(t, u):
    u_star1 = np.hstack([ u-t*np.ones((u.size, 1)), np.zeros((u.size, 1))])
    u_star2 = np.hstack([-u-t*np.ones((u.size, 1)), np.zeros((u.size, 1))])
    z = np.max(u_star1, axis=1) - np.max(u_star2, axis=1)
    return z[:, np.newaxis]<|endoftext|>


<commit_before>from flask import Blueprint, current_app, jsonify
from flask_restful import Api
from marshmallow import ValidationError
from {{cookiecutter.app_name}}.extensions import apispec
from {{cookiecutter.app_name}}.api.resources import UserResource, UserList
from {{cookiecutter.app_name}}.api.schemas.user import UserSchema

blueprint = Blueprint("api", __name__, url_prefix="/api/v1")
api = Api(blueprint)

api.add_resource(UserResource, "/users/<int:user_id>", endpoint="user_by_id")
api.add_resource(UserList, "/users", endpoint='users')

@blueprint.before_app_first_request
def register_views():
    apispec.spec.components.schema("UserSchema", schema=UserSchema)
    apispec.spec.path(view=UserResource, app=current_app)
    apispec.spec.path(view=UserList, app=current_app)

@blueprint.errorhandler(ValidationError)
def handle_marshmallow_error(e):
    """Return json error for marshmallow validation errors.

    This will avoid having to try/catch ValidationErrors in all endpoints, returning
    correct JSON response with associated HTTP 400 Status (https://tools.ietf.org/html/rfc7231#section-6.5.1)
    """
    return jsonify(e.messages), 400
<commit_msg>(refractor) fix formatting issues<commit_after>from flask import Blueprint, current_app, jsonify
from flask_restful import Api
from marshmallow import ValidationError
from {{cookiecutter.app_name}}.extensions import apispec
from {{cookiecutter.app_name}}.api.resources import UserResource, UserList
from {{cookiecutter.app_name}}.api.schemas.user import UserSchema

blueprint = Blueprint("api", __name__, url_prefix="/api/v1")
api = Api(blueprint)

api.add_resource(UserResource, "/users/<int:user_id>", endpoint="user_by_id")
api.add_resource(UserList, "/users", endpoint="users")

@blueprint.before_app_first_request
def register_views():
    apispec.spec.components.schema("UserSchema", schema=UserSchema)
    apispec.spec.path(view=UserResource, app=current_app)
    apispec.spec.path(view=UserList, app=current_app)

@blueprint.errorhandler(ValidationError)
def handle_marshmallow_error(e):
    """Return json error for marshmallow validation errors.

    This will avoid having to try/catch ValidationErrors in all endpoints, returning
    correct JSON response with associated HTTP 400 Status (https://tools.ietf.org/html/rfc7231#section-6.5.1)
    """
    return jsonify(e.messages), 400
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2010 Tiny SPRL (<http://tiny.be>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from openerp.osv import osv, fields
from openerp.tools import html2plaintext

class note_stage(osv.osv):
    """ Category of Note """
    _name = "note.stage"
    _description = "Note Stage"
    _columns = {
        'name': fields.char('Stage Name', translate=True, required=True),
        'sequence': fields.integer('Sequence', help="Used to order the note stages"),
        'user_id': fields.many2one('res.users', 'Owner', help="Owner of the note stage.", required=True),
        'fold': fields.boolean('Folded by Default'),
    }
    _order = 'sequence asc'
    _defaults = {
        'fold': 0,
        'user_id': lambda self, cr, uid, ctx: uid,
        'sequence' : 1,
    }

class note_tag(osv.osv):
    _name = "note.tag"
    _description = "Note Tag"
    _columns = {
        'name' : fields.char('Tag Name', required=True),
    }

class note_note(osv.osv):
    """ Note """
    _name = 'note.note'
    _inherit = ['mail.thread']
    _description = "Note"

    #writing method (no modification of values)
    def name_create(self, cr, uid, name, context=None):
        rec_id = self.create(cr, uid, {'memo': name}, context=context)
        return self.name_get(cr, uid, [rec_id], context)[0]

    #read the first line (convert hml into text)
    def _get_note_first_line(self, cr, uid, ids, name="", args={}, context=None):
        res = {}
        for note in self.browse(cr, uid, ids, context=context):
            res[note.id] = (note.memo and html2plaintext(note.memo) or "").strip().replace('*','').split("\n")[0]

        return res

    def onclick_note_is_done(self, cr, uid, ids, context=None):
        return self.write(cr, uid, ids, {'open': False, 'date_done': fields.date.today()}, context=context)

    def onclick_note_not_done(self, cr, uid, ids, context=None):
        return self.write(cr, uid, ids, {'open': True}, context=context)

    #used for undisplay the follower if it's the current user
    def _get_my_current_partner(self, cr, uid, ids, name, args, context=None):
        user = self.pool.get('res.users').browse(cr, uid, uid, context=context)
        pid = user.partner_id and user.partner_id.id or False
        return dict.fromkeys(ids, pid)

    #return the default stage for the uid user
    def _get_default_stage_id(self,cr,uid,context=None):
        ids = self.pool.get('note.stage').search(cr,uid,[('user_id','=',uid)], context=context)
        return ids and ids[0] or False

    def _set_stage_per_user(self, cr, uid, id, name, value, args=None, context=None):
        note = self.browse(cr, uid, id, context=context)
        if not value: return False
        stage_ids = [value] + [stage.id for stage in note.stage_ids if stage.user_id.id != uid ]
        return self.write(cr, uid, [id], {'stage_ids': [(6, 0, set(stage_ids))]}, context=context)

    def _get_stage_per_user(self, cr, uid, ids, name, args, context=None):
        result = dict.fromkeys(ids, False)
        for record in self.browse(cr, uid, ids, context=context):
            for stage in record.stage_ids:
                if stage.user_id.id == uid:
                    result[record.id] = stage.id
        return result

    _columns = {
        'name': fields.function(_get_note_first_line,
            string='Note Summary',
            type='text', store=True),
        'memo': fields.html('Note Content'),
        'sequence': fields.integer('Sequence'),
        'stage_id': fields.function(_get_stage_per_user,
            fnct_inv=_set_stage_per_user,
            string='Stage',
            type='many2one',
            relation='note.stage'),
        'stage_ids': fields.many2many('note.stage','note_stage_rel','note_id','stage_id','Stages of Users'),
        'open': fields.boolean('Active', track_visibility='onchange'),
        'date_done': fields.date('Date done'),
        'color': fields.integer('Color Index'),
        'tag_ids' : fields.many2many('note.tag','note_tags_rel','note_id','tag_id','Tags'),
        'current_partner_id' : fields.function(_get_my_current_partner),
    }
    _defaults = {
        'open' : 1,
        'stage_id' : _get_default_stage_id,
    }
    _order = 'sequence'

    def read_group(self, cr, uid, domain, fields, groupby, offset=0, limit=None, context=None, orderby=False):
        if groupby and groupby[0]=="stage_id":

            #search all stages
            current_stage_ids = self.pool.get('note.stage').search(cr,uid,[('user_id','=',uid)], context=context)

            if current_stage_ids: #if the user have some stages

                #dict of stages: map les ids sur les noms
                stage_name = dict(self.pool.get('note.stage').name_get(cr, uid, current_stage_ids, context=context))

                result = [{ #notes by stage for stages user
                        '__context': {'group_by': groupby[1:]},
                        '__domain': domain + [('stage_ids.id', '=', current_stage_id)],
                        'stage_id': (current_stage_id, stage_name[current_stage_id]),
                        'stage_id_count': self.search(cr,uid, domain+[('stage_ids', '=', current_stage_id)], context=context, count=True)
                    } for current_stage_id in current_stage_ids]

                #note without user's stage
                nb_notes_ws = self.search(cr,uid, domain+[('stage_ids', 'not in', current_stage_ids)], context=context, count=True)
                if nb_notes_ws:
                    result += [{ #notes for unknown stage and if stage_ids is not empty
                        '__context': {'group_by': groupby[1:]},
                        '__domain': domain + [('stage_ids', 'not in', current_stage_ids)],
                        'stage_id': (0, 'Unknown'),
                        'stage_id_count':nb_notes_ws
                    }]

            else: # if stage_ids is empty

                #note without user's stage
                nb_notes_ws = self.search(cr,uid, domain, context=context, count=True)
                if nb_notes_ws:
                    result = [{ #notes for unknown stage
                        '__context': {'group_by': groupby[1:]},
                        '__domain': domain,
                        'stage_id': (0, 'Unknown'),
                        'stage_id_count':nb_notes_ws
                    }]
                else:
                    result = []
            return result

        else:
            return super(note_note, self).read_group(self, cr, uid, domain, fields, groupby,
                offset=offset, limit=limit, context=context, orderby=orderby)

#upgrade config setting page to configure pad, fancy and tags mode
class note_base_config_settings(osv.osv_memory):
    _inherit = 'base.config.settings'
    _columns = {
        'module_note_pad': fields.boolean('Use collaborative pads (etherpad)'),
        'group_note_fancy': fields.boolean('Use fancy layouts for notes', implied_group='note.group_note_fancy'),
    }

class res_users(osv.Model):
    _name = 'res.users'
    _inherit = ['res.users']
    def create(self, cr, uid, data, context=None):
        user_id = super(res_users, self).create(cr, uid, data, context=context)
        user = self.browse(cr, uid, uid, context=context)
        note_obj = self.pool.get('note.stage')
        data_obj = self.pool.get('ir.model.data')
        model_id = data_obj.get_object_reference(cr, uid, 'base', 'group_user') #Employee Group
        group_id = model_id and model_id[1] or False
        if group_id in [x.id for x in user.groups_id]:
            for note_xml_id in ['note_stage_01','note_stage_02','note_stage_03','note_stage_04']:
                data_id = data_obj._get_id(cr, uid, 'note', note_xml_id)
                stage_id  = data_obj.browse(cr, uid, data_id, context=context).res_id
                note_obj.copy(cr, uid, stage_id, default = {
                                        'user_id': user_id}, context=context)
        return user_id<commit_msg>[FIX] note: stages should not block user deletion + incomplete fields.function declaration for current_partner_id<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2010 Tiny SPRL (<http://tiny.be>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from openerp.osv import osv, fields
from openerp.tools import html2plaintext

class note_stage(osv.osv):
    """ Category of Note """
    _name = "note.stage"
    _description = "Note Stage"
    _columns = {
        'name': fields.char('Stage Name', translate=True, required=True),
        'sequence': fields.integer('Sequence', help="Used to order the note stages"),
        'user_id': fields.many2one('res.users', 'Owner', help="Owner of the note stage.", required=True, ondelete='cascade'),
        'fold': fields.boolean('Folded by Default'),
    }
    _order = 'sequence asc'
    _defaults = {
        'fold': 0,
        'user_id': lambda self, cr, uid, ctx: uid,
        'sequence' : 1,
    }

class note_tag(osv.osv):
    _name = "note.tag"
    _description = "Note Tag"
    _columns = {
        'name' : fields.char('Tag Name', required=True),
    }

class note_note(osv.osv):
    """ Note """
    _name = 'note.note'
    _inherit = ['mail.thread']
    _description = "Note"

    #writing method (no modification of values)
    def name_create(self, cr, uid, name, context=None):
        rec_id = self.create(cr, uid, {'memo': name}, context=context)
        return self.name_get(cr, uid, [rec_id], context)[0]

    #read the first line (convert hml into text)
    def _get_note_first_line(self, cr, uid, ids, name="", args={}, context=None):
        res = {}
        for note in self.browse(cr, uid, ids, context=context):
            res[note.id] = (note.memo and html2plaintext(note.memo) or "").strip().replace('*','').split("\n")[0]

        return res

    def onclick_note_is_done(self, cr, uid, ids, context=None):
        return self.write(cr, uid, ids, {'open': False, 'date_done': fields.date.today()}, context=context)

    def onclick_note_not_done(self, cr, uid, ids, context=None):
        return self.write(cr, uid, ids, {'open': True}, context=context)

    #used for undisplay the follower if it's the current user
    def _get_my_current_partner(self, cr, uid, ids, name, args, context=None):
        user = self.pool.get('res.users').browse(cr, uid, uid, context=context)
        pid = user.partner_id and user.partner_id.id or False
        return dict.fromkeys(ids, pid)

    #return the default stage for the uid user
    def _get_default_stage_id(self,cr,uid,context=None):
        ids = self.pool.get('note.stage').search(cr,uid,[('user_id','=',uid)], context=context)
        return ids and ids[0] or False

    def _set_stage_per_user(self, cr, uid, id, name, value, args=None, context=None):
        note = self.browse(cr, uid, id, context=context)
        if not value: return False
        stage_ids = [value] + [stage.id for stage in note.stage_ids if stage.user_id.id != uid ]
        return self.write(cr, uid, [id], {'stage_ids': [(6, 0, set(stage_ids))]}, context=context)

    def _get_stage_per_user(self, cr, uid, ids, name, args, context=None):
        result = dict.fromkeys(ids, False)
        for record in self.browse(cr, uid, ids, context=context):
            for stage in record.stage_ids:
                if stage.user_id.id == uid:
                    result[record.id] = stage.id
        return result

    _columns = {
        'name': fields.function(_get_note_first_line,
            string='Note Summary',
            type='text', store=True),
        'memo': fields.html('Note Content'),
        'sequence': fields.integer('Sequence'),
        'stage_id': fields.function(_get_stage_per_user,
            fnct_inv=_set_stage_per_user,
            string='Stage',
            type='many2one',
            relation='note.stage'),
        'stage_ids': fields.many2many('note.stage','note_stage_rel','note_id','stage_id','Stages of Users'),
        'open': fields.boolean('Active', track_visibility='onchange'),
        'date_done': fields.date('Date done'),
        'color': fields.integer('Color Index'),
        'tag_ids' : fields.many2many('note.tag','note_tags_rel','note_id','tag_id','Tags'),
        'current_partner_id' : fields.function(_get_my_current_partner, type="many2one", relation='res.partner', string="Owner"),
    }
    _defaults = {
        'open' : 1,
        'stage_id' : _get_default_stage_id,
    }
    _order = 'sequence'

    def read_group(self, cr, uid, domain, fields, groupby, offset=0, limit=None, context=None, orderby=False):
        if groupby and groupby[0]=="stage_id":

            #search all stages
            current_stage_ids = self.pool.get('note.stage').search(cr,uid,[('user_id','=',uid)], context=context)

            if current_stage_ids: #if the user have some stages

                #dict of stages: map les ids sur les noms
                stage_name = dict(self.pool.get('note.stage').name_get(cr, uid, current_stage_ids, context=context))

                result = [{ #notes by stage for stages user
                        '__context': {'group_by': groupby[1:]},
                        '__domain': domain + [('stage_ids.id', '=', current_stage_id)],
                        'stage_id': (current_stage_id, stage_name[current_stage_id]),
                        'stage_id_count': self.search(cr,uid, domain+[('stage_ids', '=', current_stage_id)], context=context, count=True)
                    } for current_stage_id in current_stage_ids]

                #note without user's stage
                nb_notes_ws = self.search(cr,uid, domain+[('stage_ids', 'not in', current_stage_ids)], context=context, count=True)
                if nb_notes_ws:
                    result += [{ #notes for unknown stage and if stage_ids is not empty
                        '__context': {'group_by': groupby[1:]},
                        '__domain': domain + [('stage_ids', 'not in', current_stage_ids)],
                        'stage_id': (0, 'Unknown'),
                        'stage_id_count':nb_notes_ws
                    }]

            else: # if stage_ids is empty

                #note without user's stage
                nb_notes_ws = self.search(cr,uid, domain, context=context, count=True)
                if nb_notes_ws:
                    result = [{ #notes for unknown stage
                        '__context': {'group_by': groupby[1:]},
                        '__domain': domain,
                        'stage_id': (0, 'Unknown'),
                        'stage_id_count':nb_notes_ws
                    }]
                else:
                    result = []
            return result

        else:
            return super(note_note, self).read_group(self, cr, uid, domain, fields, groupby,
                offset=offset, limit=limit, context=context, orderby=orderby)

#upgrade config setting page to configure pad, fancy and tags mode
class note_base_config_settings(osv.osv_memory):
    _inherit = 'base.config.settings'
    _columns = {
        'module_note_pad': fields.boolean('Use collaborative pads (etherpad)'),
        'group_note_fancy': fields.boolean('Use fancy layouts for notes', implied_group='note.group_note_fancy'),
    }

class res_users(osv.Model):
    _name = 'res.users'
    _inherit = ['res.users']
    def create(self, cr, uid, data, context=None):
        user_id = super(res_users, self).create(cr, uid, data, context=context)
        user = self.browse(cr, uid, uid, context=context)
        note_obj = self.pool.get('note.stage')
        data_obj = self.pool.get('ir.model.data')
        model_id = data_obj.get_object_reference(cr, uid, 'base', 'group_user') #Employee Group
        group_id = model_id and model_id[1] or False
        if group_id in [x.id for x in user.groups_id]:
            for note_xml_id in ['note_stage_01','note_stage_02','note_stage_03','note_stage_04']:
                data_id = data_obj._get_id(cr, uid, 'note', note_xml_id)
                stage_id  = data_obj.browse(cr, uid, data_id, context=context).res_id
                note_obj.copy(cr, uid, stage_id, default = {
                                        'user_id': user_id}, context=context)
        return user_id<|endoftext|>


<commit_before>#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import math

import numpy as np
import paddle.v2 as paddle
import paddle.v2.dataset.conll05 as conll05
import paddle.v2.fluid as fluid
import time

word_dict, verb_dict, label_dict = conll05.get_dict()
word_dict_len = len(word_dict)
label_dict_len = len(label_dict)
pred_len = len(verb_dict)

mark_dict_len = 2
word_dim = 32
mark_dim = 5
hidden_dim = 512
depth = 8
mix_hidden_lr = 1e-3

IS_SPARSE = True
PASS_NUM = 10
BATCH_SIZE = 20

embedding_name = 'emb'

def load_parameter(file_name, h, w):
    with open(file_name, 'rb') as f:
        f.read(16)  # skip header.
        return np.fromfile(f, dtype=np.float32).reshape(h, w)

def db_lstm(word, predicate, ctx_n2, ctx_n1, ctx_0, ctx_p1, ctx_p2, mark,
            **ignored):
    # 8 features
    predicate_embedding = fluid.layers.embedding(
        input=predicate,
        size=[pred_len, word_dim],
        dtype='float32',
        is_sparse=IS_SPARSE,
        param_attr='vemb')

    mark_embedding = fluid.layers.embedding(
        input=mark,
        size=[mark_dict_len, mark_dim],
        dtype='float32',
        is_sparse=IS_SPARSE)

    word_input = [word, ctx_n2, ctx_n1, ctx_0, ctx_p1, ctx_p2]
    emb_layers = [
        fluid.layers.embedding(
            size=[word_dict_len, word_dim],
            input=x,
            param_attr=fluid.ParamAttr(
                name=embedding_name, trainable=False)) for x in word_input
    ]
    emb_layers.append(predicate_embedding)
    emb_layers.append(mark_embedding)

    hidden_0_layers = [
        fluid.layers.fc(input=emb, size=hidden_dim) for emb in emb_layers
    ]

    hidden_0 = fluid.layers.sums(input=hidden_0_layers)

    lstm_0 = fluid.layers.dynamic_lstm(
        input=hidden_0,
        size=hidden_dim,
        candidate_activation='relu',
        gate_activation='sigmoid',
        cell_activation='sigmoid')

    # stack L-LSTM and R-LSTM with direct edges
    input_tmp = [hidden_0, lstm_0]

    for i in range(1, depth):
        mix_hidden = fluid.layers.sums(input=[
            fluid.layers.fc(input=input_tmp[0], size=hidden_dim),
            fluid.layers.fc(input=input_tmp[1], size=hidden_dim)
        ])

        lstm = fluid.layers.dynamic_lstm(
            input=mix_hidden,
            size=hidden_dim,
            candidate_activation='relu',
            gate_activation='sigmoid',
            cell_activation='sigmoid',
            is_reverse=((i % 2) == 1))

        input_tmp = [mix_hidden, lstm]

    feature_out = fluid.layers.sums(input=[
        fluid.layers.fc(input=input_tmp[0], size=label_dict_len),
        fluid.layers.fc(input=input_tmp[1], size=label_dict_len)
    ])

    return feature_out

def to_lodtensor(data, place):
    seq_lens = [len(seq) for seq in data]
    cur_len = 0
    lod = [cur_len]
    for l in seq_lens:
        cur_len += l
        lod.append(cur_len)
    flattened_data = np.concatenate(data, axis=0).astype("int64")
    flattened_data = flattened_data.reshape([len(flattened_data), 1])
    res = fluid.LoDTensor()
    res.set(flattened_data, place)
    res.set_lod([lod])
    return res

def main():
    # define network topology
    word = fluid.layers.data(
        name='word_data', shape=[1], dtype='int64', lod_level=1)
    predicate = fluid.layers.data(
        name='verb_data', shape=[1], dtype='int64', lod_level=1)
    ctx_n2 = fluid.layers.data(
        name='ctx_n2_data', shape=[1], dtype='int64', lod_level=1)
    ctx_n1 = fluid.layers.data(
        name='ctx_n1_data', shape=[1], dtype='int64', lod_level=1)
    ctx_0 = fluid.layers.data(
        name='ctx_0_data', shape=[1], dtype='int64', lod_level=1)
    ctx_p1 = fluid.layers.data(
        name='ctx_p1_data', shape=[1], dtype='int64', lod_level=1)
    ctx_p2 = fluid.layers.data(
        name='ctx_p2_data', shape=[1], dtype='int64', lod_level=1)
    mark = fluid.layers.data(
        name='mark_data', shape=[1], dtype='int64', lod_level=1)
    feature_out = db_lstm(**locals())
    target = fluid.layers.data(
        name='target', shape=[1], dtype='int64', lod_level=1)
    crf_cost = fluid.layers.linear_chain_crf(
        input=feature_out,
        label=target,
        param_attr=fluid.ParamAttr(
            name='crfw', learning_rate=mix_hidden_lr))
    avg_cost = fluid.layers.mean(x=crf_cost)

    # TODO(qiao)
    # check other optimizers and check why out will be NAN
    sgd_optimizer = fluid.optimizer.SGD(learning_rate=0.0001)
    sgd_optimizer.minimize(avg_cost)

    # TODO(qiao)
    # add dependency track and move this config before optimizer
    crf_decode = fluid.layers.crf_decoding(
        input=feature_out, param_attr=fluid.ParamAttr(name='crfw'))

    chunk_evaluator = fluid.evaluator.ChunkEvaluator(
        input=crf_decode,
        label=target,
        chunk_scheme="IOB",
        num_chunk_types=int(math.ceil((label_dict_len - 1) / 2.0)))

    train_data = paddle.batch(
        paddle.reader.shuffle(
            paddle.dataset.conll05.test(), buf_size=8192),
        batch_size=BATCH_SIZE)
    #place = fluid.CPUPlace()
    place = fluid.CUDAPlace(0)
    feeder = fluid.DataFeeder(
        feed_list=[
            word, ctx_n2, ctx_n1, ctx_0, ctx_p1, ctx_p2, predicate, mark, target
        ],
        place=place)
    exe = fluid.Executor(place)

    exe.run(fluid.default_startup_program())

    embedding_param = fluid.global_scope().find_var(embedding_name).get_tensor()
    embedding_param.set(
        load_parameter(conll05.get_embedding(), word_dict_len, word_dim), place)

    start_time = time.time()
    batch_id = 0
    for pass_id in xrange(PASS_NUM):
        chunk_evaluator.reset(exe)
        for data in train_data():
            cost, precision, recall, f1_score = exe.run(
                fluid.default_main_program(),
                feed=feeder.feed(data),
                fetch_list=[avg_cost] + chunk_evaluator.metrics)
            pass_precision, pass_recall, pass_f1_score = chunk_evaluator.eval(
                exe)

            if batch_id % 10 == 0:
                print("avg_cost:" + str(cost) + " precision:" + str(
                    precision) + " recall:" + str(recall) + " f1_score:" + str(
                        f1_score) + " pass_precision:" + str(
                            pass_precision) + " pass_recall:" + str(pass_recall)
                      + " pass_f1_score:" + str(pass_f1_score))
                if batch_id != 0:
                    print("second per batch: " + str((time.time() - start_time)
                                                     / batch_id))

            # exit early for CI
            exit(0)

            batch_id = batch_id + 1

if __name__ == '__main__':
    main()
<commit_msg>Refine test_label_semantic_roles of Fluid (#7193)<commit_after>#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import math

import numpy as np
import paddle.v2 as paddle
import paddle.v2.dataset.conll05 as conll05
import paddle.v2.fluid as fluid
import time

word_dict, verb_dict, label_dict = conll05.get_dict()
word_dict_len = len(word_dict)
label_dict_len = len(label_dict)
pred_len = len(verb_dict)

mark_dict_len = 2
word_dim = 32
mark_dim = 5
hidden_dim = 512
depth = 8
mix_hidden_lr = 1e-3

IS_SPARSE = True
PASS_NUM = 10
BATCH_SIZE = 10

embedding_name = 'emb'

def load_parameter(file_name, h, w):
    with open(file_name, 'rb') as f:
        f.read(16)  # skip header.
        return np.fromfile(f, dtype=np.float32).reshape(h, w)

def db_lstm(word, predicate, ctx_n2, ctx_n1, ctx_0, ctx_p1, ctx_p2, mark,
            **ignored):
    # 8 features
    predicate_embedding = fluid.layers.embedding(
        input=predicate,
        size=[pred_len, word_dim],
        dtype='float32',
        is_sparse=IS_SPARSE,
        param_attr='vemb')

    mark_embedding = fluid.layers.embedding(
        input=mark,
        size=[mark_dict_len, mark_dim],
        dtype='float32',
        is_sparse=IS_SPARSE)

    word_input = [word, ctx_n2, ctx_n1, ctx_0, ctx_p1, ctx_p2]
    emb_layers = [
        fluid.layers.embedding(
            size=[word_dict_len, word_dim],
            input=x,
            param_attr=fluid.ParamAttr(
                name=embedding_name, trainable=False)) for x in word_input
    ]
    emb_layers.append(predicate_embedding)
    emb_layers.append(mark_embedding)

    hidden_0_layers = [
        fluid.layers.fc(input=emb, size=hidden_dim) for emb in emb_layers
    ]

    hidden_0 = fluid.layers.sums(input=hidden_0_layers)

    lstm_0 = fluid.layers.dynamic_lstm(
        input=hidden_0,
        size=hidden_dim,
        candidate_activation='relu',
        gate_activation='sigmoid',
        cell_activation='sigmoid')

    # stack L-LSTM and R-LSTM with direct edges
    input_tmp = [hidden_0, lstm_0]

    for i in range(1, depth):
        mix_hidden = fluid.layers.sums(input=[
            fluid.layers.fc(input=input_tmp[0], size=hidden_dim),
            fluid.layers.fc(input=input_tmp[1], size=hidden_dim)
        ])

        lstm = fluid.layers.dynamic_lstm(
            input=mix_hidden,
            size=hidden_dim,
            candidate_activation='relu',
            gate_activation='sigmoid',
            cell_activation='sigmoid',
            is_reverse=((i % 2) == 1))

        input_tmp = [mix_hidden, lstm]

    feature_out = fluid.layers.sums(input=[
        fluid.layers.fc(input=input_tmp[0], size=label_dict_len),
        fluid.layers.fc(input=input_tmp[1], size=label_dict_len)
    ])

    return feature_out

def to_lodtensor(data, place):
    seq_lens = [len(seq) for seq in data]
    cur_len = 0
    lod = [cur_len]
    for l in seq_lens:
        cur_len += l
        lod.append(cur_len)
    flattened_data = np.concatenate(data, axis=0).astype("int64")
    flattened_data = flattened_data.reshape([len(flattened_data), 1])
    res = fluid.LoDTensor()
    res.set(flattened_data, place)
    res.set_lod([lod])
    return res

def main():
    # define network topology
    word = fluid.layers.data(
        name='word_data', shape=[1], dtype='int64', lod_level=1)
    predicate = fluid.layers.data(
        name='verb_data', shape=[1], dtype='int64', lod_level=1)
    ctx_n2 = fluid.layers.data(
        name='ctx_n2_data', shape=[1], dtype='int64', lod_level=1)
    ctx_n1 = fluid.layers.data(
        name='ctx_n1_data', shape=[1], dtype='int64', lod_level=1)
    ctx_0 = fluid.layers.data(
        name='ctx_0_data', shape=[1], dtype='int64', lod_level=1)
    ctx_p1 = fluid.layers.data(
        name='ctx_p1_data', shape=[1], dtype='int64', lod_level=1)
    ctx_p2 = fluid.layers.data(
        name='ctx_p2_data', shape=[1], dtype='int64', lod_level=1)
    mark = fluid.layers.data(
        name='mark_data', shape=[1], dtype='int64', lod_level=1)
    feature_out = db_lstm(**locals())
    target = fluid.layers.data(
        name='target', shape=[1], dtype='int64', lod_level=1)
    crf_cost = fluid.layers.linear_chain_crf(
        input=feature_out,
        label=target,
        param_attr=fluid.ParamAttr(
            name='crfw', learning_rate=mix_hidden_lr))
    avg_cost = fluid.layers.mean(x=crf_cost)

    # TODO(qiao)
    # check other optimizers and check why out will be NAN
    sgd_optimizer = fluid.optimizer.SGD(learning_rate=0.0001)
    sgd_optimizer.minimize(avg_cost)

    # TODO(qiao)
    # add dependency track and move this config before optimizer
    crf_decode = fluid.layers.crf_decoding(
        input=feature_out, param_attr=fluid.ParamAttr(name='crfw'))

    chunk_evaluator = fluid.evaluator.ChunkEvaluator(
        input=crf_decode,
        label=target,
        chunk_scheme="IOB",
        num_chunk_types=int(math.ceil((label_dict_len - 1) / 2.0)))

    train_data = paddle.batch(
        paddle.reader.shuffle(
            paddle.dataset.conll05.test(), buf_size=8192),
        batch_size=BATCH_SIZE)
    #place = fluid.CPUPlace()
    place = fluid.CUDAPlace(0)
    feeder = fluid.DataFeeder(
        feed_list=[
            word, ctx_n2, ctx_n1, ctx_0, ctx_p1, ctx_p2, predicate, mark, target
        ],
        place=place)
    exe = fluid.Executor(place)

    exe.run(fluid.default_startup_program())

    embedding_param = fluid.global_scope().find_var(embedding_name).get_tensor()
    embedding_param.set(
        load_parameter(conll05.get_embedding(), word_dict_len, word_dim), place)

    start_time = time.time()
    batch_id = 0
    for pass_id in xrange(PASS_NUM):
        chunk_evaluator.reset(exe)
        for data in train_data():
            cost, precision, recall, f1_score = exe.run(
                fluid.default_main_program(),
                feed=feeder.feed(data),
                fetch_list=[avg_cost] + chunk_evaluator.metrics)
            pass_precision, pass_recall, pass_f1_score = chunk_evaluator.eval(
                exe)

            if batch_id % 10 == 0:
                print("avg_cost:" + str(cost) + " precision:" + str(
                    precision) + " recall:" + str(recall) + " f1_score:" + str(
                        f1_score) + " pass_precision:" + str(
                            pass_precision) + " pass_recall:" + str(pass_recall)
                      + " pass_f1_score:" + str(pass_f1_score))
                if batch_id != 0:
                    print("second per batch: " + str((time.time() - start_time)
                                                     / batch_id))

            # exit early for CI
            exit(0)

            batch_id = batch_id + 1

if __name__ == '__main__':
    main()
<|endoftext|>