<commit_before>import datetime
import getpass
import platform
import queue
import random
import sys
import time
import threading
import traceback

import pyperclip

import addon_utils
import bpy
from bpy.app.handlers import persistent
from bpy.props import BoolProperty, EnumProperty, IntProperty, StringProperty
from bpy_extras.io_utils import ExportHelper, path_reference_mode

def id_generator():
    id = -1
    while True:
        id += 1
        yield id
ids = id_generator()

class TasksRunner(object):
    def __init__(self, keep_finished_task_timeout = 2):
        self.keep_finished_task_timeout = keep_finished_task_timeout

        self.tasks = {}
        self.on_task_changed = []

    def add_task(self, task):
        id = next(ids)
        task.tasks_runner = self
        self.tasks[id] = task

        task.run()

        if len(self.tasks) == 1:
            bpy.ops.export_scene.previz_manage_queue()

        return id

    def tick(self):
        for task in self.tasks.values():
            task.tick()
        self.remove_finished_tasks()

    def cancel(self):
        for task in [t for t in self.tasks.values() if t.is_cancelable]:
            task.cancel()

    def remove_finished_tasks(self):
        def is_timed_out(task):
            return task.status in (DONE, CANCELED) \
                   and (time.time() - task.finished_time) > self.keep_finished_task_timeout
        ids = [id for id, task in self.tasks.items() if is_timed_out(task)]
        for id in ids:
            self.remove_task(id)
            self.notify_change(None)

    @property
    def is_empty(self):
        return len(self.tasks) == 0

    def remove_task(self, task_id):
        task = self.tasks[task_id]
        if not task.is_finished:
            msg = 'Cannot remove unfinished task {!r}'.format(task.label)
            raise RuntimeError(msg)
        del self.tasks[task_id]

    def notify_change(self, task):
        for cb in self.on_task_changed:
            cb(self, task)

tasks_runner = None

IDLE = 'idle'
STARTING = 'starting'
RUNNING = 'running'
DONE = 'done'
CANCELING = 'canceling'
CANCELED = 'canceled'
ERROR = 'error'

class Task(object):
    def __init__(self):
        self.label = 'label'
        self.status = IDLE
        self.state = 'state'
        self.error = None
        self.progress = None
        self.finished_time = None
        self.tasks_runner = None

    def run(self):
        self.state = 'Running'
        self.status = RUNNING
        self.notify()

    def canceling(self):
        self.state = 'Canceling'
        self.status = CANCELING
        self.notify()

    def canceled(self):
        self.finished_time = time.time()
        self.state = 'Canceled'
        self.status = CANCELED
        self.notify()

    def done(self):
        self.finished_time = time.time()
        self.state = 'Done'
        self.status = DONE
        self.notify()

    def set_error(self, exc_info):
        self.finished_time = time.time()
        self.error = exc_info
        self.state = 'Error'
        self.status = ERROR
        self.notify()

    @property
    def is_cancelable(self):
        return hasattr(self, 'cancel')

    @property
    def is_finished(self):
        return self.status in (DONE, CANCELED, ERROR)

    def tick(self):
        pass

    def notify(self):
        self.tasks_runner.notify_change(self)

class DebugSyncTask(Task):
    def __init__(self):
        Task.__init__(self)

    def run(self):
        super().run()
        for ms in range(0, 510, 100):
            s = ms / 1000
            time.sleep(s)
            self.label = 'task {}'.format(s*2)
            self.progress = s*2
            self.notify()
        self.done()

REQUEST_CANCEL = 0
RESPOND_CANCELED = 1
TASK_DONE = 2
TASK_UPDATE = 3
TASK_ERROR = 4

class DebugAsyncTask(Task):
    def __init__(self):
        Task.__init__(self)

        self.queue_to_worker = queue.Queue()
        self.queue_to_main = queue.Queue()
        self.thread = threading.Thread(target=DebugAsyncTask.thread_run,
                                       args=(self.queue_to_worker,
                                             self.queue_to_main))

    def run(self):
        print('MAIN: Starting thread')
        self.thread.start()
        print('MAIN: Started thread')

    def cancel(self):
        self.canceling()
        self.queue_to_worker.put((REQUEST_CANCEL, None))

    @staticmethod
    def thread_run(queue_to_worker, queue_to_main):
        print('THREAD: Starting')
        try:
            for i in range(1, 11):
                while not queue_to_worker.empty():
                    msg, data = queue_to_worker.get()
                    if msg == REQUEST_CANCEL:
                        queue_to_main.put((RESPOND_CANCELED, None))
                        queue_to_worker.task_done()
                        return

                s = random.random()/2
                msg = (i, s)
                queue_to_main.put((TASK_UPDATE, msg))
                print('THREAD: Sleep {} {:.2}'.format(*msg))
                time.sleep(s)
            queue_to_main.put((TASK_DONE, None))
        except Exception as err:
            print('****** CAUGHT')
            queue_to_main.put((TASK_ERROR, sys.exc_info()))
        finally:
            print('THREAD: Stopping')

    def tick(self):
        #print('DebugAsyncTask.tick')
        while not self.queue_to_main.empty():
            msg, data = self.queue_to_main.get()
            print('msg', msg)
            print('is_finished', self.is_finished)

            if not self.is_finished:
                if msg == RESPOND_CANCELED:
                    self.finished_time = time.time()
                    self.state = 'Canceled'
                    self.status = CANCELED
                    self.notify()

                if msg == TASK_DONE:
                    self.done()

                if msg == TASK_UPDATE:
                    self.label = 'Sleep: {} {:.2}'.format(*data)
                    self.notify()

                if msg == TASK_ERROR:
                    print('Error ---------')
                    exc_info = data
                    self.set_error(exc_info)

            self.queue_to_main.task_done()

class Test(bpy.types.Operator):
    bl_idname = 'export_scene.previz_test'
    bl_label = 'Refresh Previz projects'

    def execute(self, context):
        self.report({'INFO'}, 'Previz: progress.Test')
        task = DebugAsyncTask()
        tasks_runner.add_task(task)
        return {'FINISHED'}

class CancelTask(bpy.types.Operator):
    bl_idname = 'export_scene.previz_cancel_task'
    bl_label = 'Cancel Previz task'

    task_id = IntProperty(
        name = 'Task ID',
        default = -1
    )

    def execute(self, context):
        self.report({'INFO'}, 'Previz: Cancel task {}'.format(self.task_id))
        tasks_runner.tasks[self.task_id].cancel()
        return {'FINISHED'}

class RaiseException(bpy.types.Operator):
    bl_idname = 'export_scene.previz_raise_exception'
    bl_label = 'Raise Previz exception'

    def execute(self, context):
        sadcsadc
        return {'FINISHED'}

def task2debuginfo(task):
    type, exception, tb = task.error
    d = datetime.datetime.now()
    d_utc = datetime.datetime.utcfromtimestamp(d.timestamp())
    ret = [
        '---- PREVIZ DEBUG INFO START',
        'Date:    : {}'.format(d.isoformat()),
        'Date UTC : {}'.format(d_utc.isoformat()),
        'User     : {}'.format(getpass.getuser()),
        'Blender  : {}'.format(bpy.app.version_string),
        'OS       : {}'.format(platform.platform()),
        'Python   : {}'.format(sys.version),
        'Addon    : {}'.format(''),
        'Version  : {}'.format(''),
        'Task     : {}'.format(task.label),
        'Status   : {}'.format(task.status),
        'Progress : {}'.format(task.progress),
        'Exception: {}'.format(exception.__class__.__name__),
        'Error    : {}'.format(str(exception)),
        'Traceback:',
        ''
    ]
    ret = '\n'.join(ret) + '\n'
    ret += ''.join(traceback.format_tb(tb))
    ret += '\n---- PREVIZ DEBUG INFO END\n'
    return ret

def task2report(task):
    type, exception, tb = task.error

    return '''Previz task error
Task: {}
Exception: {}
Value: {}

See the console for debug information.

The debug information has been copied to the clipboard.
Please paste it to Previz support.
'''.format(task.label, exception.__class__.__name__, exception)

class ShowTaskError(bpy.types.Operator):
    bl_idname = 'export_scene.previz_show_task_error'
    bl_label = 'Show Previz task error'

    task_id = IntProperty(
        name = 'Task ID',
        default = -1
    )

    def execute(self, context):
        task = tasks_runner.tasks[self.task_id]
        self.report({'ERROR'}, task2report(task))
        debug_info = task2debuginfo(task)
        pyperclip.copy(debug_info)
        print(debug_info)
        return {'FINISHED'}

class RemoveTask(bpy.types.Operator):
    bl_idname = 'export_scene.previz_remove_task'
    bl_label = 'Remove Previz task'

    task_id = IntProperty(
        name = 'Task ID',
        default = -1
    )

    def execute(self, context):
        self.report({'INFO'}, 'Previz: Remove task {}'.format(self.task_id))
        tasks_runner.remove_task(self.task_id)
        return {'FINISHED'}

class ManageQueue(bpy.types.Operator):
    bl_idname = 'export_scene.previz_manage_queue'
    bl_label = 'Manage Previz task queue'

    process_polling_interval = 1 # Needs to be a debug User Preferences flag

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

        self.timer = None

    def execute(self, context):
        #print('ManageQueue.execute')
        if tasks_runner.is_empty:
            self.cleanup(context)
            #print('ManageQueue.execute FINISHED')
            return {'FINISHED'}
        self.register_timer(context)
        context.window_manager.modal_handler_add(self)
        #tasks_runner.tick()
        #print('ManageQueue.execute RUNNING_MODAL')
        return {'RUNNING_MODAL'}

    def cancel(self, context):
        self.cleanup(context)

    def modal(self, context, event):
        if event.type == 'ESC':
            #print('ManageQueue.modal CANCELED')
            return {'CANCELED'}

        if event.type == 'TIMER':
            return self.handle_timer_event(context, event)

        return {'PASS_THROUGH'}

    def handle_timer_event(self, context, event):
        if tasks_runner.is_empty:
            self.cleanup(context)
            #print('ManageQueue.handle_timer_event FINISHED')
            return {'FINISHED'}
        tasks_runner.tick()
        #print('ManageQueue.handle_timer_event RUNNING_MODAL')
        return {'RUNNING_MODAL'}

    def cleanup(self, context):
        tasks_runner.cancel()
        self.unregister_timer(context)

    def register_timer(self, context):
        if self.timer is None:
            #print('ManageQueue.register_timer')
            self.timer = context.window_manager.event_timer_add(self.process_polling_interval, context.window)

    def unregister_timer(self, context):
        if self.timer is not None:
            #print('ManageQueue.unregister_timer')
            context.window_manager.event_timer_remove(self.timer)
            self.timer = None

class Panel(bpy.types.Panel):
    bl_label = "PrevizProgress"
    bl_idname = "SCENE_PT_previz_test"
    bl_space_type = "PROPERTIES"
    bl_region_type = "WINDOW"
    bl_context = "scene"

    def draw(self, context):
        self.layout.operator(
            'export_scene.previz_test',
            text='Progress test'
        )

        self.layout.operator(
            'export_scene.previz_raise_exception'
        )

        for id, task in tasks_runner.tasks.items():
            row = self.layout.row()
            row.label('{} ({})'.format(task.label, task.state))

            if task.status == ERROR:
                row.operator(
                    'export_scene.previz_show_task_error',
                    text='',
                    icon='ERROR').task_id = id

            if task.is_cancelable and not task.is_finished:
                row.operator(
                    'export_scene.previz_cancel_task',
                    text='',
                    icon='CANCEL').task_id = id

            if task.is_finished:
                row.operator(
                    'export_scene.previz_remove_task',
                    text='',
                    icon='X').task_id = id

            row.enabled = task.status != CANCELING

def register():
    bpy.utils.register_class(Test)
    bpy.utils.register_class(CancelTask)
    bpy.utils.register_class(RemoveTask)
    bpy.utils.register_class(Panel)
    bpy.utils.register_class(ManageQueue)
    bpy.utils.register_class(ShowTaskError)
    bpy.utils.register_class(RaiseException)

    global tasks_runner
    tasks_runner = TasksRunner()

    def refresh_panel(*args, **kwarsg):
        bpy.ops.wm.redraw_timer(type='DRAW_WIN_SWAP', iterations=1)
    tasks_runner.on_task_changed.append(refresh_panel)

def unregister():
    bpy.utils.unregister_class(Test)
    bpy.utils.unregister_class(CancelTask)
    bpy.utils.unregister_class(RemoveTask)
    bpy.utils.unregister_class(Panel)
    bpy.utils.unregister_class(ManageQueue)
    bpy.utils.unregister_class(ShowTaskError)
    bpy.utils.unregister_class(RaiseException)
<commit_msg>Made task message enum strings<commit_after>import datetime
import getpass
import platform
import queue
import random
import sys
import time
import threading
import traceback

import pyperclip

import addon_utils
import bpy
from bpy.app.handlers import persistent
from bpy.props import BoolProperty, EnumProperty, IntProperty, StringProperty
from bpy_extras.io_utils import ExportHelper, path_reference_mode

def id_generator():
    id = -1
    while True:
        id += 1
        yield id
ids = id_generator()

class TasksRunner(object):
    def __init__(self, keep_finished_task_timeout = 2):
        self.keep_finished_task_timeout = keep_finished_task_timeout

        self.tasks = {}
        self.on_task_changed = []

    def add_task(self, task):
        id = next(ids)
        task.tasks_runner = self
        self.tasks[id] = task

        task.run()

        if len(self.tasks) == 1:
            bpy.ops.export_scene.previz_manage_queue()

        return id

    def tick(self):
        for task in self.tasks.values():
            task.tick()
        self.remove_finished_tasks()

    def cancel(self):
        for task in [t for t in self.tasks.values() if t.is_cancelable]:
            task.cancel()

    def remove_finished_tasks(self):
        def is_timed_out(task):
            return task.status in (DONE, CANCELED) \
                   and (time.time() - task.finished_time) > self.keep_finished_task_timeout
        ids = [id for id, task in self.tasks.items() if is_timed_out(task)]
        for id in ids:
            self.remove_task(id)
            self.notify_change(None)

    @property
    def is_empty(self):
        return len(self.tasks) == 0

    def remove_task(self, task_id):
        task = self.tasks[task_id]
        if not task.is_finished:
            msg = 'Cannot remove unfinished task {!r}'.format(task.label)
            raise RuntimeError(msg)
        del self.tasks[task_id]

    def notify_change(self, task):
        for cb in self.on_task_changed:
            cb(self, task)

tasks_runner = None

IDLE = 'idle'
STARTING = 'starting'
RUNNING = 'running'
DONE = 'done'
CANCELING = 'canceling'
CANCELED = 'canceled'
ERROR = 'error'

class Task(object):
    def __init__(self):
        self.label = 'label'
        self.status = IDLE
        self.state = 'state'
        self.error = None
        self.progress = None
        self.finished_time = None
        self.tasks_runner = None

    def run(self):
        self.state = 'Running'
        self.status = RUNNING
        self.notify()

    def canceling(self):
        self.state = 'Canceling'
        self.status = CANCELING
        self.notify()

    def canceled(self):
        self.finished_time = time.time()
        self.state = 'Canceled'
        self.status = CANCELED
        self.notify()

    def done(self):
        self.finished_time = time.time()
        self.state = 'Done'
        self.status = DONE
        self.notify()

    def set_error(self, exc_info):
        self.finished_time = time.time()
        self.error = exc_info
        self.state = 'Error'
        self.status = ERROR
        self.notify()

    @property
    def is_cancelable(self):
        return hasattr(self, 'cancel')

    @property
    def is_finished(self):
        return self.status in (DONE, CANCELED, ERROR)

    def tick(self):
        pass

    def notify(self):
        self.tasks_runner.notify_change(self)

class DebugSyncTask(Task):
    def __init__(self):
        Task.__init__(self)

    def run(self):
        super().run()
        for ms in range(0, 510, 100):
            s = ms / 1000
            time.sleep(s)
            self.label = 'task {}'.format(s*2)
            self.progress = s*2
            self.notify()
        self.done()

REQUEST_CANCEL = 'REQUEST_CANCEL'
RESPOND_CANCELED = 'RESPOND_CANCELED'
TASK_DONE = 'TASK_DONE'
TASK_UPDATE = 'TASK_UPDATE'
TASK_ERROR = 'TASK_ERROR'

class DebugAsyncTask(Task):
    def __init__(self):
        Task.__init__(self)

        self.queue_to_worker = queue.Queue()
        self.queue_to_main = queue.Queue()
        self.thread = threading.Thread(target=DebugAsyncTask.thread_run,
                                       args=(self.queue_to_worker,
                                             self.queue_to_main))

    def run(self):
        print('MAIN: Starting thread')
        self.thread.start()
        print('MAIN: Started thread')

    def cancel(self):
        self.canceling()
        self.queue_to_worker.put((REQUEST_CANCEL, None))

    @staticmethod
    def thread_run(queue_to_worker, queue_to_main):
        print('THREAD: Starting')
        try:
            for i in range(1, 11):
                while not queue_to_worker.empty():
                    msg, data = queue_to_worker.get()
                    if msg == REQUEST_CANCEL:
                        queue_to_main.put((RESPOND_CANCELED, None))
                        queue_to_worker.task_done()
                        return

                s = random.random()/2
                msg = (i, s)
                queue_to_main.put((TASK_UPDATE, msg))
                print('THREAD: Sleep {} {:.2}'.format(*msg))
                time.sleep(s)
            queue_to_main.put((TASK_DONE, None))
        except Exception as err:
            print('****** CAUGHT')
            queue_to_main.put((TASK_ERROR, sys.exc_info()))
        finally:
            print('THREAD: Stopping')

    def tick(self):
        #print('DebugAsyncTask.tick')
        while not self.queue_to_main.empty():
            msg, data = self.queue_to_main.get()
            print('msg', msg)
            print('is_finished', self.is_finished)

            if not self.is_finished:
                if msg == RESPOND_CANCELED:
                    self.finished_time = time.time()
                    self.state = 'Canceled'
                    self.status = CANCELED
                    self.notify()

                if msg == TASK_DONE:
                    self.done()

                if msg == TASK_UPDATE:
                    self.label = 'Sleep: {} {:.2}'.format(*data)
                    self.notify()

                if msg == TASK_ERROR:
                    print('Error ---------')
                    exc_info = data
                    self.set_error(exc_info)

            self.queue_to_main.task_done()

class Test(bpy.types.Operator):
    bl_idname = 'export_scene.previz_test'
    bl_label = 'Refresh Previz projects'

    def execute(self, context):
        self.report({'INFO'}, 'Previz: progress.Test')
        task = DebugAsyncTask()
        tasks_runner.add_task(task)
        return {'FINISHED'}

class CancelTask(bpy.types.Operator):
    bl_idname = 'export_scene.previz_cancel_task'
    bl_label = 'Cancel Previz task'

    task_id = IntProperty(
        name = 'Task ID',
        default = -1
    )

    def execute(self, context):
        self.report({'INFO'}, 'Previz: Cancel task {}'.format(self.task_id))
        tasks_runner.tasks[self.task_id].cancel()
        return {'FINISHED'}

class RaiseException(bpy.types.Operator):
    bl_idname = 'export_scene.previz_raise_exception'
    bl_label = 'Raise Previz exception'

    def execute(self, context):
        sadcsadc
        return {'FINISHED'}

def task2debuginfo(task):
    type, exception, tb = task.error
    d = datetime.datetime.now()
    d_utc = datetime.datetime.utcfromtimestamp(d.timestamp())
    ret = [
        '---- PREVIZ DEBUG INFO START',
        'Date:    : {}'.format(d.isoformat()),
        'Date UTC : {}'.format(d_utc.isoformat()),
        'User     : {}'.format(getpass.getuser()),
        'Blender  : {}'.format(bpy.app.version_string),
        'OS       : {}'.format(platform.platform()),
        'Python   : {}'.format(sys.version),
        'Addon    : {}'.format(''),
        'Version  : {}'.format(''),
        'Task     : {}'.format(task.label),
        'Status   : {}'.format(task.status),
        'Progress : {}'.format(task.progress),
        'Exception: {}'.format(exception.__class__.__name__),
        'Error    : {}'.format(str(exception)),
        'Traceback:',
        ''
    ]
    ret = '\n'.join(ret) + '\n'
    ret += ''.join(traceback.format_tb(tb))
    ret += '\n---- PREVIZ DEBUG INFO END\n'
    return ret

def task2report(task):
    type, exception, tb = task.error

    return '''Previz task error
Task: {}
Exception: {}
Value: {}

See the console for debug information.

The debug information has been copied to the clipboard.
Please paste it to Previz support.
'''.format(task.label, exception.__class__.__name__, exception)

class ShowTaskError(bpy.types.Operator):
    bl_idname = 'export_scene.previz_show_task_error'
    bl_label = 'Show Previz task error'

    task_id = IntProperty(
        name = 'Task ID',
        default = -1
    )

    def execute(self, context):
        task = tasks_runner.tasks[self.task_id]
        self.report({'ERROR'}, task2report(task))
        debug_info = task2debuginfo(task)
        pyperclip.copy(debug_info)
        print(debug_info)
        return {'FINISHED'}

class RemoveTask(bpy.types.Operator):
    bl_idname = 'export_scene.previz_remove_task'
    bl_label = 'Remove Previz task'

    task_id = IntProperty(
        name = 'Task ID',
        default = -1
    )

    def execute(self, context):
        self.report({'INFO'}, 'Previz: Remove task {}'.format(self.task_id))
        tasks_runner.remove_task(self.task_id)
        return {'FINISHED'}

class ManageQueue(bpy.types.Operator):
    bl_idname = 'export_scene.previz_manage_queue'
    bl_label = 'Manage Previz task queue'

    process_polling_interval = 1 # Needs to be a debug User Preferences flag

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

        self.timer = None

    def execute(self, context):
        #print('ManageQueue.execute')
        if tasks_runner.is_empty:
            self.cleanup(context)
            #print('ManageQueue.execute FINISHED')
            return {'FINISHED'}
        self.register_timer(context)
        context.window_manager.modal_handler_add(self)
        #tasks_runner.tick()
        #print('ManageQueue.execute RUNNING_MODAL')
        return {'RUNNING_MODAL'}

    def cancel(self, context):
        self.cleanup(context)

    def modal(self, context, event):
        if event.type == 'ESC':
            #print('ManageQueue.modal CANCELED')
            return {'CANCELED'}

        if event.type == 'TIMER':
            return self.handle_timer_event(context, event)

        return {'PASS_THROUGH'}

    def handle_timer_event(self, context, event):
        if tasks_runner.is_empty:
            self.cleanup(context)
            #print('ManageQueue.handle_timer_event FINISHED')
            return {'FINISHED'}
        tasks_runner.tick()
        #print('ManageQueue.handle_timer_event RUNNING_MODAL')
        return {'RUNNING_MODAL'}

    def cleanup(self, context):
        tasks_runner.cancel()
        self.unregister_timer(context)

    def register_timer(self, context):
        if self.timer is None:
            #print('ManageQueue.register_timer')
            self.timer = context.window_manager.event_timer_add(self.process_polling_interval, context.window)

    def unregister_timer(self, context):
        if self.timer is not None:
            #print('ManageQueue.unregister_timer')
            context.window_manager.event_timer_remove(self.timer)
            self.timer = None

class Panel(bpy.types.Panel):
    bl_label = "PrevizProgress"
    bl_idname = "SCENE_PT_previz_test"
    bl_space_type = "PROPERTIES"
    bl_region_type = "WINDOW"
    bl_context = "scene"

    def draw(self, context):
        self.layout.operator(
            'export_scene.previz_test',
            text='Progress test'
        )

        self.layout.operator(
            'export_scene.previz_raise_exception'
        )

        for id, task in tasks_runner.tasks.items():
            row = self.layout.row()
            row.label('{} ({})'.format(task.label, task.state))

            if task.status == ERROR:
                row.operator(
                    'export_scene.previz_show_task_error',
                    text='',
                    icon='ERROR').task_id = id

            if task.is_cancelable and not task.is_finished:
                row.operator(
                    'export_scene.previz_cancel_task',
                    text='',
                    icon='CANCEL').task_id = id

            if task.is_finished:
                row.operator(
                    'export_scene.previz_remove_task',
                    text='',
                    icon='X').task_id = id

            row.enabled = task.status != CANCELING

def register():
    bpy.utils.register_class(Test)
    bpy.utils.register_class(CancelTask)
    bpy.utils.register_class(RemoveTask)
    bpy.utils.register_class(Panel)
    bpy.utils.register_class(ManageQueue)
    bpy.utils.register_class(ShowTaskError)
    bpy.utils.register_class(RaiseException)

    global tasks_runner
    tasks_runner = TasksRunner()

    def refresh_panel(*args, **kwarsg):
        bpy.ops.wm.redraw_timer(type='DRAW_WIN_SWAP', iterations=1)
    tasks_runner.on_task_changed.append(refresh_panel)

def unregister():
    bpy.utils.unregister_class(Test)
    bpy.utils.unregister_class(CancelTask)
    bpy.utils.unregister_class(RemoveTask)
    bpy.utils.unregister_class(Panel)
    bpy.utils.unregister_class(ManageQueue)
    bpy.utils.unregister_class(ShowTaskError)
    bpy.utils.unregister_class(RaiseException)
<|endoftext|>


<commit_before>import math
from itertools import chain
import operator
import bisect
import itertools
from collections import defaultdict

from sklearn import preprocessing, tree, cross_validation
import numpy as np

from base import gob, utils
from base.utils import coord_in_miles
import explore.peek

def logify(x,fudge=1):
    return math.log(x+fudge,2)

def unlogify(x,fudge=1):
    return 2**x - fudge

class NebrVect(object):
    def __init__(self,env):
        to_froms = chain.from_iterable(
            env.load('geo_ated.%02d'%x) for x in xrange(100)
        )
        self.geo_ated = {to:set(froms) for to, froms in to_froms}

        self.cheap_locals = dict(chain.from_iterable(
            env.load('cheap_locals.%02d'%x) for x in xrange(100)
        ))

    def _scaled_local(self,_id):
        return self.cheap_locals.get(_id,.3)

    @gob.mapper()
    def nebr_vect(self,user):
        mentioned = self.geo_ated.get(user['_id'],())
        for nebr in user['nebrs']:
            # I really don't like the way I did these flags.
            ated,fols,frds = [nebr['kind'] >>i & 1 for i in range(3)]
            at_back = int(nebr['_id'] in mentioned)
            flags = [ated, at_back, ated and at_back, fols, frds, fols and frds]
            logged = [logify(nebr[k]) for k in ('mdist','folc','frdc')]
            others = [
                self._scaled_local(nebr['_id']),
                int(bool(nebr['prot'])),
                logify(coord_in_miles(user['mloc'],nebr),fudge=.01),
            ]
            yield flags + logged + others

def vects_as_mat(vects):
    # convert vects to a scaled numpy array
    vects_ = np.fromiter( chain.from_iterable(vects), np.float32 )
    vects_.shape = (len(vects_)//10),10
    X = np.array(vects_[:,:-1])
    y = np.array(vects_[:,-1])
    #scaler = preprocessing.Scaler().fit(X)

    # In final system, you will want to only fit the training data.
    #return scaler.transform(X), y
    return X,y

@gob.mapper(all_items=True)
def nebr_clf(vects):
    X, y = vects_as_mat(vects)
    # FIXME: I made up the number for min_samples_leaf
    clf = tree.DecisionTreeRegressor(min_samples_leaf=500)
    clf.fit(X,y)
    yield clf

class Predictor(object):
    def predict(self,nebrs_d,vect_fit):
        "return the index of a neighbor or (eventually) a coordinate"
        raise NotImplementedError

class Omni(Predictor):
    def predict(self,nebrs_d,vect_fit):
        values = [v[-1] for v in nebrs_d['vects']]
        return values.index(min(values))

class Nearest(Predictor):
    def predict(self,nebrs_d,vect_fit):
        return np.argmin(nebrs_d['pred_dists'])

class Last(Predictor):
    def predict(self,nebrs_d,vect_fit):
        return len(nebrs_d['strange_prob'])-1

class FacebookMLE(Predictor):
    def predict(self,nebrs_d,vect_fit):
        probs = np.sum(np.log(nebrs_d['contact_prob']),axis=0)
        return np.argmax(probs+nebrs_d['strange_prob'])

class NearestMLE(Predictor):
    def __init__(self,count):
        self.count = count

    def predict(self,nebrs_d,vect_fit):
        dists = sorted(
                    enumerate(nebrs_d['pred_dists']),
                    key=operator.itemgetter(1)
                    )
        best = [d[0] for d in dists[:self.count]]
        probs_ =    (
                    nebrs_d['contact_prob'][x][y]
                    for x in xrange(len(dists))
                    for y in xrange(len(dists))
                    if x in best and y in best
                    )
        probs = np.fromiter(probs_,float)
        probs.shape = len(best),len(best)

        sums = np.sum(np.log10(probs),axis=0)
        index, prob = max(enumerate(sums), key=operator.itemgetter(1))
        return best[index]

class FriendLoc(Predictor):
    def __init__(self,env,loc_factor,tz_factor=0,force_loc=False):
        self.env = env
        self.loc_factor = loc_factor
        self.tz_factor = tz_factor
        self.force_loc = force_loc

    def predict(self,nebrs_d,vect_fit):
        self.cutoffs,self.curves = zip(*vect_fit)
        if self.force_loc and nebrs_d['gnp'] and nebrs_d['gnp']['mdist']<25:
            return len(nebrs_d['vects'])
        mat = nebrs_d['dist_mat']
        probs = np.empty_like(mat)
        for index,pred in enumerate(nebrs_d['pred_dists']):
            curve = self.curves[bisect.bisect(self.cutoffs,pred)-1]
            probs[index,:] = explore.peek.contact_curve(mat[index,:],*curve)
        total_probs = (
            np.sum(np.log(probs),axis=0) +
            nebrs_d['strange_prob'] +
            self.tz_factor*nebrs_d['tz_prob'] +
            self.loc_factor*nebrs_d['location_prob']
            )
        return np.argmax(total_probs)

class FriendLocWeights(Predictor):
    def __init__(self,env,weights,vect_fit):
        self.env = env
        self.weights = weights

    def predict(self,nebrs_d,vect_fit):
        self.cutoffs,self.curves = zip(*vect_fit)
        mat = nebrs_d['dist_mat']
        probs = np.empty_like(mat)
        for index,pred in enumerate(nebrs_d['pred_dists']):
            curve = self.curves[bisect.bisect(self.cutoffs,pred)-1]
            probs[index,:] = explore.peek.contact_curve(mat[index,:],*curve)
        prob_sum = np.sum(np.log(probs),axis=0)
        w = self.weights
        total_probs = (
            w[0]*prob_sum +
            w[1]*nebrs_d['location_prob'] +
            w[2]*nebrs_d['strange_prob']/2 +
            w[3]*nebrs_d['tz_prob']/2
            )
        return np.argmax(total_probs)

def _calc_dists(nebrs_d):
    gnp = nebrs_d['gnp']
    lats = [r['lat'] for r in nebrs_d['nebrs']]
    lngs = [r['lng'] for r in nebrs_d['nebrs']]
    all_lats = lats+[gnp['lat']] if gnp else lats
    all_lngs = lngs+[gnp['lng']] if gnp else lngs
    lat1,lat2 = np.meshgrid(all_lats,lats)
    lng1,lng2 = np.meshgrid(all_lngs,lngs)
    return utils.np_haversine(lng1,lng2,lat1,lat2)

class Predictors(object):
    def __init__(self,env):
        self.env = env
        self.classifiers = dict(
            omni=Omni(),
            nearest=Nearest(),
            last=Last(),
            backstrom=FacebookMLE(),
            friendloc_plain=FriendLoc(env,0),
            friendloc_field=FriendLoc(env,1),
            friendloc_tz=FriendLoc(env,1,1),
            friendloc_cut=FriendLoc(env,0,force_loc=True),
        )
        '''
        steps = [0,.333,1,3]
        self.classifiers = {
            vect:FriendLocWeights(env,vect)
            for vect in itertools.product(steps[1:],steps[1:],steps,steps)
            if any(vect)
        }
        '''

    def _mdist_curves(self,clump):
        data = list(self.env.load('mdist_curves.'+clump,'mp'))
        return dict(
            curve_fn = [np.poly1d(d['coeffs']) for d in data],
            cutoff = [d['cutoff'] for d in data],
            local = [d['local'] for d in data],
        )

    def _stranger_mat(self):
        mat = next(self.env.load('stranger_mat','npz'))
        # strangers is based on contacts, but prediction is based on nebrs, so
        # we scale down the values in stranger_mat
        return np.maximum(-50,mat)*.25

    def _add_location_prob(self,nebrs_d):
        gnp = nebrs_d['gnp']
        if not gnp:
            nebrs_d['location_prob'] = np.zeros_like(nebrs_d['strange_prob'])
            return

        # append 0 to include the distance to the gnp point
        dists = np.append(nebrs_d['dist_mat'][:,-1],[0])
        mdist = nebrs_d['gnp']['mdist']
        index = bisect.bisect(self.mdist_curves['cutoff'],mdist)-1
        curve = self.mdist_curves['curve_fn'][index]
        local = np.log(self.mdist_curves['local'][index])
        probs = [
            (local if dist<1 else curve(np.log(dist)))
            for dist in dists
        ]
        nebrs_d['location_prob'] = np.array(probs)

    def _stranger_prob(self,nebr):
        lng_t = explore.peek._tile(nebr['lng'])
        lat_t_ = explore.peek._tile(nebr['lat'])
        lat_t = max(-900,min(lat_t_,899))
        return self.stranger_mat[(lng_t+1800)%3600,lat_t+900]

    def _tz_prob(self,nebrs_d,nebr):
        if 'utco' not in nebrs_d:
            return 1.0/24
        delta_lng = int(nebr['lng']-nebrs_d['utco']/240)
        index = ((delta_lng+180)%360)/15
        return self.utc_offset[index]

    def prep(self,nebrs_d):
        # add fields to nebrs_d
        nebrs_d['vects'] = list(nebr_vect(nebrs_d))
        X, y = vects_as_mat(nebrs_d['vects'])
        nebrs_d['pred_dists'] = self.nebr_clf.predict(X)
        nebrs_d['dist_mat'] = _calc_dists(nebrs_d)
        nebrs_d['contact_prob'] = utils.contact_prob(nebrs_d['dist_mat'])

        s_probs = [self._stranger_prob(nebr) for nebr in nebrs_d['nebrs']]
        if nebrs_d['gnp']:
            s_probs.append(self._stranger_prob(nebrs_d['gnp']))
        nebrs_d['strange_prob'] = np.array(s_probs)

        t_probs = [self._tz_prob(nebrs_d, nebr) for nebr in nebrs_d['nebrs']]
        if nebrs_d['gnp']:
            t_probs.append(self._tz_prob(nebrs_d, nebrs_d['gnp']))
        nebrs_d['tz_prob'] = np.log(t_probs)

        self._add_location_prob(nebrs_d)

    @gob.mapper(all_items=True)
    def predictions(self, nebrs_ds, in_paths):
        results = defaultdict(list)
        self.stranger_mat = self._stranger_mat()
        clump = in_paths[0][-1]
        self.utc_offset = next(self.env.load('utc_offset.'+clump,'mp'))
        self.mdist_curves = self._mdist_curves(clump)
        self.nebr_clf = next(self.env.load('nebr_clf.'+clump,'pkl'))
        vect_fit = tuple(self.env.load('vect_fit.'+clump))

        for nebrs_d in nebrs_ds:
            if not nebrs_d['nebrs']:
                continue
            self.prep(nebrs_d)
            for key,classifier in self.classifiers.iteritems():
                index = classifier.predict(nebrs_d,vect_fit)
                if index==len(nebrs_d['vects']):
                    # the last one is the gnp one
                    dist = utils.coord_in_miles(nebrs_d['gnp'],nebrs_d['mloc'])
                else:
                    dist = unlogify(nebrs_d['vects'][index][-1],.01)
                results[key].append(dist)
        return results.iteritems()

@gob.mapper(all_items=True)
def eval_preds(preds):
    for key,vals in preds:
        yield key,sum(1 for v in vals if v<25)
<commit_msg>use new vectors in predictions<commit_after>import math
from itertools import chain
import operator
import bisect
import itertools
from collections import defaultdict

from sklearn import preprocessing, tree, cross_validation
import numpy as np

from base import gob, utils
from base.utils import coord_in_miles
import explore.peek

def logify(x,fudge=1):
    return math.log(x+fudge,2)

def unlogify(x,fudge=1):
    return 2**x - fudge

class NebrVect(object):
    def __init__(self,env):
        to_froms = chain.from_iterable(
            env.load('geo_ated.%02d'%x) for x in xrange(100)
        )
        # I wish there was a more functional way to get data from files
        self.geo_ated = {to:set(froms) for to, froms in to_froms}

        self.cheap_locals = dict(chain.from_iterable(
            env.load('cheap_locals.%02d'%x) for x in xrange(100)
        ))

    @gob.mapper()
    def nebr_vect(self,user):
        return _make_nebr_vect(user,self.geo_ated,self.cheap_locals)

def _make_nebr_vect(user,geo_ated,cheap_locals):
    mentioned = geo_ated.get(user['_id'],())
    for nebr in user['nebrs']:
        # I really don't like the way I did these flags.
        ated,fols,frds = [nebr['kind'] >>i & 1 for i in range(3)]
        at_back = int(nebr['_id'] in mentioned)
        flags = [ated, at_back, ated and at_back, fols, frds, fols and frds]
        logged = [logify(nebr[k]) for k in ('mdist','folc','frdc')]
        others = [
            cheap_locals.get(nebr['_id'],.3),
            int(bool(nebr['prot'])),
            logify(coord_in_miles(user['mloc'],nebr),fudge=.01),
        ]
        yield flags + logged + others

def vects_as_mat(vects):
    # convert vects to a scaled numpy array
    vects_ = np.fromiter( chain.from_iterable(vects), np.float32 )
    vects_.shape = (len(vects_)//12),12
    X = np.array(vects_[:,:-1])
    y = np.array(vects_[:,-1])
    #scaler = preprocessing.Scaler().fit(X)

    # In final system, you will want to only fit the training data.
    #return scaler.transform(X), y
    return X,y

@gob.mapper(all_items=True)
def nebr_clf(vects):
    X, y = vects_as_mat(vects)
    # FIXME: I made up the number for min_samples_leaf
    clf = tree.DecisionTreeRegressor(min_samples_leaf=500)
    clf.fit(X,y)
    yield clf

class Predictor(object):
    def predict(self,nebrs_d,vect_fit):
        "return the index of a neighbor or (eventually) a coordinate"
        raise NotImplementedError

class Omni(Predictor):
    def predict(self,nebrs_d,vect_fit):
        values = [v[-1] for v in nebrs_d['vects']]
        return values.index(min(values))

class Nearest(Predictor):
    def predict(self,nebrs_d,vect_fit):
        return np.argmin(nebrs_d['pred_dists'])

class Last(Predictor):
    def predict(self,nebrs_d,vect_fit):
        return len(nebrs_d['strange_prob'])-1

class FacebookMLE(Predictor):
    def predict(self,nebrs_d,vect_fit):
        probs = np.sum(np.log(nebrs_d['contact_prob']),axis=0)
        return np.argmax(probs+nebrs_d['strange_prob'])

class NearestMLE(Predictor):
    def __init__(self,count):
        self.count = count

    def predict(self,nebrs_d,vect_fit):
        dists = sorted(
                    enumerate(nebrs_d['pred_dists']),
                    key=operator.itemgetter(1)
                    )
        best = [d[0] for d in dists[:self.count]]
        probs_ =    (
                    nebrs_d['contact_prob'][x][y]
                    for x in xrange(len(dists))
                    for y in xrange(len(dists))
                    if x in best and y in best
                    )
        probs = np.fromiter(probs_,float)
        probs.shape = len(best),len(best)

        sums = np.sum(np.log10(probs),axis=0)
        index, prob = max(enumerate(sums), key=operator.itemgetter(1))
        return best[index]

class FriendLoc(Predictor):
    def __init__(self,env,loc_factor,tz_factor=0,force_loc=False):
        self.env = env
        self.loc_factor = loc_factor
        self.tz_factor = tz_factor
        self.force_loc = force_loc

    def predict(self,nebrs_d,vect_fit):
        self.cutoffs,self.curves = zip(*vect_fit)
        if self.force_loc and nebrs_d['gnp'] and nebrs_d['gnp']['mdist']<25:
            return len(nebrs_d['vects'])
        mat = nebrs_d['dist_mat']
        probs = np.empty_like(mat)
        for index,pred in enumerate(nebrs_d['pred_dists']):
            curve = self.curves[bisect.bisect(self.cutoffs,pred)-1]
            probs[index,:] = explore.peek.contact_curve(mat[index,:],*curve)
        total_probs = (
            np.sum(np.log(probs),axis=0) +
            nebrs_d['strange_prob'] +
            self.tz_factor*nebrs_d['tz_prob'] +
            self.loc_factor*nebrs_d['location_prob']
            )
        return np.argmax(total_probs)

class FriendLocWeights(Predictor):
    def __init__(self,env,weights,vect_fit):
        self.env = env
        self.weights = weights

    def predict(self,nebrs_d,vect_fit):
        self.cutoffs,self.curves = zip(*vect_fit)
        mat = nebrs_d['dist_mat']
        probs = np.empty_like(mat)
        for index,pred in enumerate(nebrs_d['pred_dists']):
            curve = self.curves[bisect.bisect(self.cutoffs,pred)-1]
            probs[index,:] = explore.peek.contact_curve(mat[index,:],*curve)
        prob_sum = np.sum(np.log(probs),axis=0)
        w = self.weights
        total_probs = (
            w[0]*prob_sum +
            w[1]*nebrs_d['location_prob'] +
            w[2]*nebrs_d['strange_prob']/2 +
            w[3]*nebrs_d['tz_prob']/2
            )
        return np.argmax(total_probs)

def _calc_dists(nebrs_d):
    gnp = nebrs_d['gnp']
    lats = [r['lat'] for r in nebrs_d['nebrs']]
    lngs = [r['lng'] for r in nebrs_d['nebrs']]
    all_lats = lats+[gnp['lat']] if gnp else lats
    all_lngs = lngs+[gnp['lng']] if gnp else lngs
    lat1,lat2 = np.meshgrid(all_lats,lats)
    lng1,lng2 = np.meshgrid(all_lngs,lngs)
    return utils.np_haversine(lng1,lng2,lat1,lat2)

class Predictors(object):
    def __init__(self,env):
        self.env = env
        self.classifiers = dict(
            omni=Omni(),
            nearest=Nearest(),
            last=Last(),
            backstrom=FacebookMLE(),
            friendloc_plain=FriendLoc(env,0),
            friendloc_field=FriendLoc(env,1),
            friendloc_tz=FriendLoc(env,1,1),
            friendloc_cut=FriendLoc(env,0,force_loc=True),
        )
        '''
        steps = [0,.333,1,3]
        self.classifiers = {
            vect:FriendLocWeights(env,vect)
            for vect in itertools.product(steps[1:],steps[1:],steps,steps)
            if any(vect)
        }
        '''

    def _mdist_curves(self,clump):
        data = list(self.env.load('mdist_curves.'+clump,'mp'))
        return dict(
            curve_fn = [np.poly1d(d['coeffs']) for d in data],
            cutoff = [d['cutoff'] for d in data],
            local = [d['local'] for d in data],
        )

    def _stranger_mat(self):
        mat = next(self.env.load('stranger_mat','npz'))
        # strangers is based on contacts, but prediction is based on nebrs, so
        # we scale down the values in stranger_mat
        return np.maximum(-50,mat)*.25

    def _add_location_prob(self,nebrs_d):
        gnp = nebrs_d['gnp']
        if not gnp:
            nebrs_d['location_prob'] = np.zeros_like(nebrs_d['strange_prob'])
            return

        # append 0 to include the distance to the gnp point
        dists = np.append(nebrs_d['dist_mat'][:,-1],[0])
        mdist = nebrs_d['gnp']['mdist']
        index = bisect.bisect(self.mdist_curves['cutoff'],mdist)-1
        curve = self.mdist_curves['curve_fn'][index]
        local = np.log(self.mdist_curves['local'][index])
        probs = [
            (local if dist<1 else curve(np.log(dist)))
            for dist in dists
        ]
        nebrs_d['location_prob'] = np.array(probs)

    def _stranger_prob(self,nebr):
        lng_t = explore.peek._tile(nebr['lng'])
        lat_t_ = explore.peek._tile(nebr['lat'])
        lat_t = max(-900,min(lat_t_,899))
        return self.stranger_mat[(lng_t+1800)%3600,lat_t+900]

    def _tz_prob(self,nebrs_d,nebr):
        if 'utco' not in nebrs_d:
            return 1.0/24
        delta_lng = int(nebr['lng']-nebrs_d['utco']/240)
        index = ((delta_lng+180)%360)/15
        return self.utc_offset[index]

    def prep(self,nebrs_d):
        # add fields to nebrs_d
        vects = _make_nebr_vect(nebrs_d,self.geo_ated,self.cheap_locals)
        nebrs_d['vects']=list(vects)
        X, y = vects_as_mat(nebrs_d['vects'])
        nebrs_d['pred_dists'] = self.nebr_clf.predict(X)
        nebrs_d['dist_mat'] = _calc_dists(nebrs_d)
        nebrs_d['contact_prob'] = utils.contact_prob(nebrs_d['dist_mat'])

        s_probs = [self._stranger_prob(nebr) for nebr in nebrs_d['nebrs']]
        if nebrs_d['gnp']:
            s_probs.append(self._stranger_prob(nebrs_d['gnp']))
        nebrs_d['strange_prob'] = np.array(s_probs)

        t_probs = [self._tz_prob(nebrs_d, nebr) for nebr in nebrs_d['nebrs']]
        if nebrs_d['gnp']:
            t_probs.append(self._tz_prob(nebrs_d, nebrs_d['gnp']))
        nebrs_d['tz_prob'] = np.log(t_probs)

        self._add_location_prob(nebrs_d)

    @gob.mapper(all_items=True)
    def predictions(self, nebrs_ds, in_paths):
        results = defaultdict(list)
        self.stranger_mat = self._stranger_mat()
        clump = in_paths[0][-1]
        self.utc_offset = next(self.env.load('utc_offset.'+clump,'mp'))
        self.mdist_curves = self._mdist_curves(clump)
        self.nebr_clf = next(self.env.load('nebr_clf.'+clump,'pkl'))
        vect_fit = tuple(self.env.load('vect_fit.'+clump))

        # This is ugly.
        to_froms = chain.from_iterable(
            self.env.load('geo_ated.%02d'%x) for x in xrange(100)
        )
        self.geo_ated = {to:set(froms) for to, froms in to_froms}

        self.cheap_locals = dict(chain.from_iterable(
            self.env.load('cheap_locals.%02d'%x) for x in xrange(100)
        ))

        for nebrs_d in nebrs_ds:
            if not nebrs_d['nebrs']:
                continue
            self.prep(nebrs_d)
            for key,classifier in self.classifiers.iteritems():
                index = classifier.predict(nebrs_d,vect_fit)
                if index==len(nebrs_d['vects']):
                    # the last one is the gnp one
                    dist = utils.coord_in_miles(nebrs_d['gnp'],nebrs_d['mloc'])
                else:
                    dist = unlogify(nebrs_d['vects'][index][-1],.01)
                results[key].append(dist)
        return results.iteritems()

@gob.mapper(all_items=True)
def eval_preds(preds):
    for key,vals in preds:
        yield key,sum(1 for v in vals if v<25)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Copyright 2015 Telefonica Investigación y Desarrollo, S.A.U

This file is part of telefonica-iot-qa-tools

iot-qa-tools is free software: you can redistribute it and/or
modify it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the License,
or (at your option) any later version.

iot-qa-tools is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
See the GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public
License along with iot-qa-tools.
If not, seehttp://www.gnu.org/licenses/.

For those usages not covered by the GNU Affero General Public License
please contact with::[iot_support@tid.es]
"""

__author__ = 'gtsa07'

# imports 3rd party libs
import requests

# Params GW
SERVER_ROOT = 'http://localhost:8002'

MeasuresType = {
    "UL": "/idas/2.0?apikey={}&ID={}",
    "UL2": "/d?k={}&i={}",
    "IoTUL2CmdResp": "/iot/d?k={}&i={}&c={}",
    "IoTUL2": "/iot/d?k={}&i={}",
    "IoTRepsol": "/iot/repsol",
    "IoTTT": "/iot/tt",
    "IoTEvadts": "/iot/evadts?apikey={}&ID={}",
    "RegLight": "/idas/2.0?apikey={}"
}

class Gw_Measures_Utils(object):
    """Constructor"""

    def __init__(self, **kwargs):
        self.server_root = kwargs.get('server_root', SERVER_ROOT)

    """General Methods"""

    """
    getUrl-> Return a correct URL to send measures for SBC
    type(string): measure format (UL/UL2)
    apikey(string): apikey service
    idDevice(string): id Device registed in SBC
    """

    def getUrl(self, url_type, apikey, idDevice, command=()):
        path = MeasuresType.get(url_type, False)
        url = self.server_root
        if path != False:
            if url_type == 'RegLight':
                url += path.format(apikey)
#            elif url_type == 'UL2CmdResp':
#                url += path.format(apikey, idDevice, command)
            else:
                url += path.format(apikey, idDevice, command)
        else:
            if apikey:
                url += url_type.format(apikey)
            elif idDevice:
                url += url_type.format(idDevice)
        return url

    """
    SendMeasure -> send measures simulating a device

    type(string): measure format (UL,UL2,etc)
    apikey(string): apikey service
    idDevice(string): id Device registed in SBC
    *measures(dict): dictionaries with the measures
    example:
    measure1={
        "timestamp": YYYY-MM-DDTHH:MM:SSZ, ->This param is optional
        "id": "8:1", ->Phenomenom ID (only for UL)
        "alias": "t", (mandatory)
        "value": 20 (mandatory)
    }
    """

    def sendMeasure(self, measure_type, apikey, idDevice, measures, field={}):
        url = self.getUrl(measure_type, apikey, idDevice)
        if field == 'getCmd':
            url += '&getCmd=1'
        print 'url: ' + url
        data = self.getMeasure(measure_type, measures, field)
        res = requests.post(url, data=data)
        return res

    """
    getMeasure -> Format a measures dictionary
    type(string): measure format (UL,UL2,etc)
    measures(list): list of dictionaries with the measures
    example:
    measures=[{
        "timestamp": YYYY-MM-DDTHH:MM:SSZ, ->This param is optional
        "id": "8:1", ->Phenomenom ID
        "alias": "t",
        "value": 20
    }]
"""

    def getMeasure(self, protocol, measures, field={}):
        result = ""
        print measures
        for measure in measures:
            # Format type UL or UL2
            if protocol == "UL":
                result += "||" + str(measure.get("timestamp", "")) + "|" + str(measure.get("id", "")) + "||" + str(
                    measure.get("alias", "")) + "|" + str(measure.get("value", "")) + "#"
            elif (protocol == "UL2") | (protocol == "IoTUL2"):
                result += str(measure.get("timestamp", "")) + "|" + str(measure.get("alias", "")) + "|" + str(
                    measure.get("value", "")) + "#"
            elif protocol == "IoTRepsol":
                result2 = {}
                result2["id"] = str(measure.get("id", ""))
                if field != "from":
                    result2["from"] = "tel:" + str(measure.get("device", "")) + ";phone-context=+34"
                result2["to"] = "tel:+123123123;phone-context=+34"
                if field != "message":
                    result2["message"] = str(measure.get("message", ""))
                if field != "timestamp":
                    result2["timestamp"] = str(measure.get("timestamp", ""))
                result = str(result2)
            else:
                result = str(measure)
        if "UL" in protocol:
            result = result.rpartition("#")[0]  # delete the last "#"
        print result
        # Replace specials words and characters
        replaces = {
            "True": "1",
            "False": "0",
            "true": "1",
            "false": "0",
            ";": "/",
            "&": "|"
        }
        if not "IoT" in protocol:
            for kreplace in replaces:
                result = result.replace(kreplace, replaces[kreplace])
        else:
            result = result.replace("\'", "\"")
        return result

    def sendRegister(self, apikey, device, asset, model, phenomena):
        url = self.getUrl('RegLight', apikey, device)
        print 'url: ' + url
        uom_id = 1
        result = "<rs>"
        result += "<id href=\"1:1\">" + device + "</id>"
        result += "<id href=\"1:8\">" + asset + "</id>"
        result += "<param name=\"ModelName\">"
        result += "<text>" + model + "</text>"
        result += "</param>"
        print phenomena
        for phenom in phenomena:
            result += "<what href=\"" + str(phenom.get("href", "")) + "\" id=\"" + str(phenom.get("alias", "")) + "\"/>"
        for phenom in phenomena:
            result += "<data name=\"" + str(phenom.get("phenom", "")) + "\" id=\"" + str(
                phenom.get("alias", "")) + "\">"
            if phenom.get("uom"):
                result += "<quan uom=\"" + str(phenom.get("uom", "")) + "\">" + str(uom_id) + "</quan>"
            else:
                result += "<text>" + str(uom_id) + "</text>"
            uom_id += 1
            result += "</data>"
        result += "</rs>"
        print result
        res = requests.post(url, data=result)
        return res

    """
    GetCommand -> get commands simulating a device

    measure_type(string): measure format (UL,UL2,etc)
    apikey(string): apikey service
    idDevice(string): id Device registed in SBC
    """

    def getCommand(self, measure_type, apikey, idDevice):
        url = self.getUrl(measure_type, apikey, idDevice)
        print 'url: ' + url
        res = requests.get(url)
        return res

    """
    SendCommandResponse -> Send command responses simulating a device

    measure_type(string): measure format (UL,UL2,etc)
    apikey(string): apikey service
    idDevice(string): id Device registed in SBC
    """

    def SendCmdResponse(self, measure_type, apikey, idDevice, command, response):
        url = self.getUrl(measure_type, apikey, idDevice, command)
        print 'url: ' + url
        data = str(command) + "|" + str(response)
        res = requests.post(url, data=data)
        return res
<commit_msg>add field ip in sendMeasure<commit_after># -*- coding: utf-8 -*-
"""
Copyright 2015 Telefonica Investigación y Desarrollo, S.A.U

This file is part of telefonica-iot-qa-tools

iot-qa-tools is free software: you can redistribute it and/or
modify it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the License,
or (at your option) any later version.

iot-qa-tools is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
See the GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public
License along with iot-qa-tools.
If not, seehttp://www.gnu.org/licenses/.

For those usages not covered by the GNU Affero General Public License
please contact with::[iot_support@tid.es]
"""

__author__ = 'gtsa07'

# imports 3rd party libs
import requests

# Params GW
SERVER_ROOT = 'http://localhost:8002'

MeasuresType = {
    "UL": "/idas/2.0?apikey={}&ID={}",
    "UL2": "/d?k={}&i={}",
    "IoTUL2CmdResp": "/iot/d?k={}&i={}&c={}",
    "IoTUL2": "/iot/d?k={}&i={}",
    "IoTRepsol": "/iot/repsol",
    "IoTTT": "/iot/tt",
    "IoTEvadts": "/iot/evadts?apikey={}&ID={}",
    "RegLight": "/idas/2.0?apikey={}"
}

class Gw_Measures_Utils(object):
    """Constructor"""

    def __init__(self, **kwargs):
        self.server_root = kwargs.get('server_root', SERVER_ROOT)

    """General Methods"""

    """
    getUrl-> Return a correct URL to send measures for SBC
    type(string): measure format (UL/UL2)
    apikey(string): apikey service
    idDevice(string): id Device registed in SBC
    """

    def getUrl(self, url_type, apikey, idDevice, command=()):
        path = MeasuresType.get(url_type, False)
        url = self.server_root
        if path != False:
            if url_type == 'RegLight':
                url += path.format(apikey)
#            elif url_type == 'UL2CmdResp':
#                url += path.format(apikey, idDevice, command)
            else:
                url += path.format(apikey, idDevice, command)
        else:
            if apikey:
                url += url_type.format(apikey)
            elif idDevice:
                url += url_type.format(idDevice)
        return url

    """
    SendMeasure -> send measures simulating a device

    type(string): measure format (UL,UL2,etc)
    apikey(string): apikey service
    idDevice(string): id Device registed in SBC
    *measures(dict): dictionaries with the measures
    example:
    measure1={
        "timestamp": YYYY-MM-DDTHH:MM:SSZ, ->This param is optional
        "id": "8:1", ->Phenomenom ID (only for UL)
        "alias": "t", (mandatory)
        "value": 20 (mandatory)
    }
    """

    def sendMeasure(self, measure_type, apikey, idDevice, measures, field={}):
        url = self.getUrl(measure_type, apikey, idDevice)
        if field == 'getCmd':
            url += '&getCmd=1'
        if "ip" in field:
            url += '&ip='
            url += field["ip"]

        print 'url: ' + url
        data = self.getMeasure(measure_type, measures, field)
        res = requests.post(url, data=data)
        return res

    """
    getMeasure -> Format a measures dictionary
    type(string): measure format (UL,UL2,etc)
    measures(list): list of dictionaries with the measures
    example:
    measures=[{
        "timestamp": YYYY-MM-DDTHH:MM:SSZ, ->This param is optional
        "id": "8:1", ->Phenomenom ID
        "alias": "t",
        "value": 20
    }]
"""

    def getMeasure(self, protocol, measures, field={}):
        result = ""
        print measures
        for measure in measures:
            # Format type UL or UL2
            if protocol == "UL":
                result += "||" + str(measure.get("timestamp", "")) + "|" + str(measure.get("id", "")) + "||" + str(
                    measure.get("alias", "")) + "|" + str(measure.get("value", "")) + "#"
            elif (protocol == "UL2") | (protocol == "IoTUL2"):
                result += str(measure.get("timestamp", "")) + "|" + str(measure.get("alias", "")) + "|" + str(
                    measure.get("value", "")) + "#"
            elif protocol == "IoTRepsol":
                result2 = {}
                result2["id"] = str(measure.get("id", ""))
                if field != "from":
                    result2["from"] = "tel:" + str(measure.get("device", "")) + ";phone-context=+34"
                result2["to"] = "tel:+123123123;phone-context=+34"
                if field != "message":
                    result2["message"] = str(measure.get("message", ""))
                if field != "timestamp":
                    result2["timestamp"] = str(measure.get("timestamp", ""))
                result = str(result2)
            else:
                result = str(measure)
        if "UL" in protocol:
            result = result.rpartition("#")[0]  # delete the last "#"
        print result
        # Replace specials words and characters
        replaces = {
            "True": "1",
            "False": "0",
            "true": "1",
            "false": "0",
            ";": "/",
            "&": "|"
        }
        if not "IoT" in protocol:
            for kreplace in replaces:
                result = result.replace(kreplace, replaces[kreplace])
        else:
            result = result.replace("\'", "\"")
        return result

    def sendRegister(self, apikey, device, asset, model, phenomena):
        url = self.getUrl('RegLight', apikey, device)
        print 'url: ' + url
        uom_id = 1
        result = "<rs>"
        result += "<id href=\"1:1\">" + device + "</id>"
        result += "<id href=\"1:8\">" + asset + "</id>"
        result += "<param name=\"ModelName\">"
        result += "<text>" + model + "</text>"
        result += "</param>"
        print phenomena
        for phenom in phenomena:
            result += "<what href=\"" + str(phenom.get("href", "")) + "\" id=\"" + str(phenom.get("alias", "")) + "\"/>"
        for phenom in phenomena:
            result += "<data name=\"" + str(phenom.get("phenom", "")) + "\" id=\"" + str(
                phenom.get("alias", "")) + "\">"
            if phenom.get("uom"):
                result += "<quan uom=\"" + str(phenom.get("uom", "")) + "\">" + str(uom_id) + "</quan>"
            else:
                result += "<text>" + str(uom_id) + "</text>"
            uom_id += 1
            result += "</data>"
        result += "</rs>"
        print result
        res = requests.post(url, data=result)
        return res

    """
    GetCommand -> get commands simulating a device

    measure_type(string): measure format (UL,UL2,etc)
    apikey(string): apikey service
    idDevice(string): id Device registed in SBC
    """

    def getCommand(self, measure_type, apikey, idDevice):
        url = self.getUrl(measure_type, apikey, idDevice)
        print 'url: ' + url
        res = requests.get(url)
        return res

    """
    SendCommandResponse -> Send command responses simulating a device

    measure_type(string): measure format (UL,UL2,etc)
    apikey(string): apikey service
    idDevice(string): id Device registed in SBC
    """

    def SendCmdResponse(self, measure_type, apikey, idDevice, command, response):
        url = self.getUrl(measure_type, apikey, idDevice, command)
        print 'url: ' + url
        data = str(command) + "|" + str(response)
        res = requests.post(url, data=data)
        return res
<|endoftext|>


<commit_before># coding: utf-8

"""
Bikes availability prediction (i.e. probability) using xgboost.
"""

import logging
import daiquiri

import numpy as np
import pandas as pd

from sklearn.model_selection import train_test_split

import xgboost as xgb

SEED = 1337
np.random.seed(SEED)

daiquiri.setup(logging.INFO)
logger = daiquiri.getLogger("prediction")

def datareader(fpath):
    """Read a CSV file ane return a DataFrame
    """
    logger.info("read the file '%s'", fpath)
    coldate = 'last_update'
    return pd.read_csv(fpath, parse_dates=[coldate])

def complete_data(df):
    """Add some columns

    - day of the week
    - hour of the day
    - minute (10 by 10)
    """
    logger.info("complete some data")
    def group_minute(value):
        if value <= 10:
            return 0
        if value <= 20:
            return 10
        if value <= 30:
            return 20
        if value <= 40:
            return 30
        if value <= 50:
            return 40
        return 50
    df = df.copy()
    df['day'] = df['ts'].apply(lambda x: x.weekday())
    df['hour'] = df['ts'].apply(lambda x: x.hour)
    minute = df['ts'].apply(lambda x: x.minute)
    df['minute'] = minute.apply(group_minute)
    return df

def cleanup(df):
    """Clean up

    - keep OPEN station
    - drop duplicates
    - rename some columns
    - drop some columns
    - drop lines when stands == bikes == 0
    """
    logger.info("cleanup processing")
    columns_to_drop = ['availability', 'status',
                       'bike_stands', 'availabilitycode']
    df = (df.copy()
          .query("status == 'OPEN'")
          .drop(columns_to_drop, axis=1)
          .drop_duplicates()
          .rename_axis({"available_bike_stands": "stands",
                        "available_bikes": "bikes",
                        "last_update": "ts",
                        "number": "station"}, axis=1)
          .query("stands > 0 and bikes > 0")) # or 'Gris' availability value...
    return df

def availability(df, threshold):
    """Set an 'availability' column according to a threshold

    if the number of bikes is less than `threshold`, the availability (of bikes,
    not stands) is low.
    """
    logger.info("set the availability level")
    df = df.copy()
    key = 'availability'
    df[key] = 'medium'
    low_mask = df['bikes'] < threshold
    high_mask = np.logical_and(np.logical_not(low_mask),
                               df['stands'] < threshold)
    df.loc[low_mask, key] = 'low'
    df.loc[high_mask, key] = 'high'
    return df

def bikes_probability(df):
    logger.info("bikes probability")
    df['probability'] = df['bikes'] / (df['bikes'] + df['stands'])
    return df

def extract_bonus_by_station(df):
    """Return a series with station id and bonus oui/non

    turn the french yes/no into 1/0
    """
    logger.info("extract the bonus for each station")
    result =  (df.groupby(["station", "bonus"])["bikes"]
               .count()
               .reset_index())
    result['bonus'] = result['bonus'].apply(lambda x: 1 if x == 'Oui' else 0)
    return result[["station", "bonus"]].set_index("station")

def time_resampling(df, freq="10T"):
    """Normalize the timeseries
    """
    logger.info("Time resampling for each station by '%s'", freq)
    df = (df.groupby("station")
          .resample(freq, on="ts")[["ts", "bikes", "stands"]]
          .mean()
          .bfill())
    return df.reset_index()

def prepare_data_for_training(df, date, freq='1H', start=None, periods=1,
                              observation='availability'):
    """Prepare data for training

    date: datetime / Timestamp
        date for the prediction
    freq: str
        the delay between the latest available data and the prediction. e.g. one hour
    start: Timestamp
        start of the history data (for training)
    periods: int
        number of predictions

    Returns 4 DataFrames: two for training, two for testing
    """
    logger.info("prepare data for training")
    logger.info("sort values (station, ts)")
    data = df.sort_values(['station', 'ts']).set_index(["ts", "station"])
    logger.info("compute the future availability at '%s'", freq)
    label = data[observation].copy()
    label.name = "future"
    label = (label.reset_index(level=1)
             .shift(-1, freq=freq)
             .reset_index()
             .set_index(["ts", "station"]))
    logger.info("merge data with the future availability")
    result = data.merge(label, left_index=True, right_index=True)
    logger.info("availability label as values")
    if observation == 'availability':
        result[observation] = result[observation].replace({"low": 0, "medium": 1, "high": 2})
        result['future'] = result['future'].replace({"low": 0, "medium": 1, "high": 2})
    result.reset_index(level=1, inplace=True)
    freq = freq.replace("T", "m")
    if start is not None:
        result = result[result.index >= start]
    cut = date - pd.Timedelta(freq)
    stop = date + periods * pd.Timedelta(freq)
    logger.info("cut date %s", cut)
    logger.info("stop date %s", stop)
    logger.info("split train and test according to a prediction date")
    train = result[result.index <= cut].copy()
    train_X = train.drop([observation, "future"], axis=1)
    train_Y = train['future'].copy()
    # time window
    mask = np.logical_and(result.index > date, result.index <= stop)
    test = result[mask].copy()
    test_X = test.drop([observation, "future"], axis=1)
    test_Y = test['future'].copy()
    return train_X, train_Y, test_X, test_Y

def fit(train_X, train_Y, test_X, test_Y):
    """Train the xgboost model

    Return the booster trained model
    """
    logger.info("fit")
    # param = {'objective': 'reg:linear'}
    param = {'objective': 'reg:logistic'}
    param['eta'] = 0.2
    param['max_depth'] = 6
    param['silent'] = 1
    param['nthread'] = 4
    # used num_class only for classification (e.g. a level of availability)
    # param = {'objective': 'multi:softmax'}
    # param['num_class'] = train_Y.nunique()
    xg_train = xgb.DMatrix(train_X, label=train_Y)
    xg_test = xgb.DMatrix(test_X, label=test_Y)
    watchlist = [(xg_train, 'train'), (xg_test, 'test')]
    num_round = 10
    bst = xgb.train(param, xg_train, num_round, watchlist)
    return bst

def prediction(bst, test_X, test_Y):
    xg_test = xgb.DMatrix(test_X, label=test_Y)
    pred = bst.predict(xg_test)
    return pred

def error_rate(bst, test_X, test_Y):
    xg_test = xgb.DMatrix(test_X, label=test_Y)
    pred = bst.predict(xg_test)
    error_rate = np.sum(pred != test_Y) / test_Y.shape[0]
    logger.info('Test error using softmax = %s', error_rate)
    return error_rate

if __name__ == '__main__':
    DATAFILE = "./data/lyon.csv"
    THRESHOLD = 3

    raw = datareader(DATAFILE)
    df_clean = cleanup(raw)
    bonus = extract_bonus_by_station(df_clean)
    df_clean = df_clean.drop("bonus", axis=1)
    # df = (df_clean.pipe(time_resampling)
    #       .pipe(complete_data)
    #       .pipe(lambda x: availability(x, THRESHOLD)))
    df = (df_clean.pipe(time_resampling)
          .pipe(complete_data)
          .pipe(bikes_probability))

    # Note: date range is are 2017-07-08 15:20:28  -  2017-09-26 14:58:45
    start = pd.Timestamp("2017-07-11") # Tuesday
    # predict_date = pd.Timestamp("2017-07-26T19:30:00") # wednesday
    predict_date = pd.Timestamp("2017-07-26T10:00:00") # wednesday
    train_X, train_Y, test_X, test_Y = prepare_data_for_training(df,
                                                                 predict_date,
                                                                 freq='30T',
                                                                 start=start,
                                                                 periods=2,
                                                                 observation='probability')
    # train_X, train_Y, test_X, test_Y = prepare_data_for_training(df, predict_date, freq='1H', start=start, periods=2)

    bst = fit(train_X, train_Y, test_X, test_Y)
    # err = error_rate(bst, test_X, test_Y)
    # print("Error rate: {}".format(err))
    pred = prediction(bst, test_X, test_Y)
    rmse = np.sqrt(np.mean((pred - test_Y)**2))
    print("RMSE: {}".format(rmse))
<commit_msg>update the prediction script<commit_after># coding: utf-8

"""
Bikes availability prediction (i.e. probability) using xgboost.
"""

import logging
import daiquiri

import numpy as np
import pandas as pd

from sklearn.model_selection import train_test_split

import xgboost as xgb

SEED = 1337
np.random.seed(SEED)

daiquiri.setup(logging.INFO)
logger = daiquiri.getLogger("prediction")

def datareader(fpath):
    """Read a CSV file ane return a DataFrame
    """
    logger.info("read the file '%s'", fpath)
    coldate = 'last_update'
    return pd.read_csv(fpath, parse_dates=[coldate])

def complete_data(df):
    """Add some columns

    - day of the week
    - hour of the day
    - minute (10 by 10)
    """
    logger.info("complete some data")
    def group_minute(value):
        if value <= 10:
            return 0
        if value <= 20:
            return 10
        if value <= 30:
            return 20
        if value <= 40:
            return 30
        if value <= 50:
            return 40
        return 50
    df = df.copy()
    df['day'] = df['ts'].apply(lambda x: x.weekday())
    df['hour'] = df['ts'].apply(lambda x: x.hour)
    minute = df['ts'].apply(lambda x: x.minute)
    df['minute'] = minute.apply(group_minute)
    return df

def cleanup(df):
    """Clean up

    - keep OPEN station
    - drop duplicates
    - rename some columns
    - drop some columns
    - drop lines when stands == bikes == 0
    """
    logger.info("cleanup processing")
    columns_to_drop = ['availability', 'status',
                       'bike_stands', 'availabilitycode']
    df = (df.copy()
          .query("status == 'OPEN'")
          .drop(columns_to_drop, axis=1)
          .drop_duplicates()
          .rename_axis({"available_bike_stands": "stands",
                        "available_bikes": "bikes",
                        "last_update": "ts",
                        "number": "station"}, axis=1)
          .query("stands > 0 and bikes > 0")) # or 'Gris' availability value...
    return df

def availability(df, threshold):
    """Set an 'availability' column according to a threshold

    if the number of bikes is less than `threshold`, the availability (of bikes,
    not stands) is low.
    """
    logger.info("set the availability level")
    df = df.copy()
    key = 'availability'
    df[key] = 'medium'
    low_mask = df['bikes'] < threshold
    high_mask = np.logical_and(np.logical_not(low_mask),
                               df['stands'] < threshold)
    df.loc[low_mask, key] = 'low'
    df.loc[high_mask, key] = 'high'
    return df

def bikes_probability(df):
    logger.info("bikes probability")
    df['probability'] = df['bikes'] / (df['bikes'] + df['stands'])
    return df

def extract_bonus_by_station(df):
    """Return a series with station id and bonus oui/non

    turn the french yes/no into 1/0
    """
    logger.info("extract the bonus for each station")
    result =  (df.groupby(["station", "bonus"])["bikes"]
               .count()
               .reset_index())
    result['bonus'] = result['bonus'].apply(lambda x: 1 if x == 'Oui' else 0)
    return result[["station", "bonus"]].set_index("station")

def time_resampling(df, freq="10T"):
    """Normalize the timeseries
    """
    logger.info("Time resampling for each station by '%s'", freq)
    df = (df.groupby("station")
          .resample(freq, on="ts")[["ts", "bikes", "stands"]]
          .mean()
          .bfill())
    return df.reset_index()

def prepare_data_for_training(df, date, freq='1H', start=None, periods=1,
                              observation='availability'):
    """Prepare data for training

    date: datetime / Timestamp
        date for the prediction
    freq: str
        the delay between the latest available data and the prediction. e.g. one hour
    start: Timestamp
        start of the history data (for training)
    periods: int
        number of predictions

    Returns 4 DataFrames: two for training, two for testing
    """
    logger.info("prepare data for training")
    logger.info("sort values (station, ts)")
    data = df.sort_values(['station', 'ts']).set_index(["ts", "station"])
    logger.info("compute the future availability at '%s'", freq)
    label = data[observation].copy()
    label.name = "future"
    label = (label.reset_index(level=1)
             .shift(-1, freq=freq)
             .reset_index()
             .set_index(["ts", "station"]))
    logger.info("merge data with the future availability")
    result = data.merge(label, left_index=True, right_index=True)
    logger.info("availability label as values")
    if observation == 'availability':
        result[observation] = result[observation].replace({"low": 0, "medium": 1, "high": 2})
        result['future'] = result['future'].replace({"low": 0, "medium": 1, "high": 2})
    result.reset_index(level=1, inplace=True)
    if start is not None:
        result = result[result.index >= start]
    cut = date - pd.Timedelta(freq.replace('T', 'm'))
    stop = date + periods * pd.Timedelta(freq.replace('T', 'm'))
    logger.info("cut date %s", cut)
    logger.info("stop date %s", stop)
    logger.info("split train and test according to a prediction date")
    train = result[result.index <= cut].copy()
    train_X = train.drop([observation, "future"], axis=1)
    train_Y = train['future'].copy()
    # time window
    mask = np.logical_and(result.index >= date, result.index <= stop)
    test = result[mask].copy()
    test_X = test.drop([observation, "future"], axis=1)
    test_Y = test['future'].copy()
    return train_X, train_Y, test_X, test_Y

def fit(train_X, train_Y, test_X, test_Y):
    """Train the xgboost model

    Return the booster trained model
    """
    logger.info("fit")
    # param = {'objective': 'reg:linear'}
    param = {'objective': 'reg:logistic'}
    param['eta'] = 0.2
    param['max_depth'] = 6
    param['silent'] = 1
    param['nthread'] = 4
    # used num_class only for classification (e.g. a level of availability)
    # param = {'objective': 'multi:softmax'}
    # param['num_class'] = train_Y.nunique()
    xg_train = xgb.DMatrix(train_X, label=train_Y)
    xg_test = xgb.DMatrix(test_X, label=test_Y)
    watchlist = [(xg_train, 'train'), (xg_test, 'test')]
    num_round = 25
    bst = xgb.train(param, xg_train, num_round, watchlist)
    return bst

def prediction(bst, test_X, test_Y):
    xg_test = xgb.DMatrix(test_X, label=test_Y)
    pred = bst.predict(xg_test)
    return pred

def error_rate(bst, test_X, test_Y):
    xg_test = xgb.DMatrix(test_X, label=test_Y)
    pred = bst.predict(xg_test)
    error_rate = np.sum(pred != test_Y) / test_Y.shape[0]
    logger.info('Test error using softmax = %s', error_rate)
    return error_rate

if __name__ == '__main__':
    DATAFILE = "./data/lyon.csv"
    THRESHOLD = 3

    raw = datareader(DATAFILE)
    df_clean = cleanup(raw)
    bonus = extract_bonus_by_station(df_clean)
    df_clean = df_clean.drop("bonus", axis=1)
    # df = (df_clean.pipe(time_resampling)
    #       .pipe(complete_data)
    #       .pipe(lambda x: availability(x, THRESHOLD)))
    df = (df_clean.pipe(time_resampling)
          .pipe(complete_data)
          .pipe(bikes_probability))

    # Note: date range is are 2017-07-08 15:20:28  -  2017-09-26 14:58:45
    start = pd.Timestamp("2017-07-11") # Tuesday
    # predict_date = pd.Timestamp("2017-07-26T19:30:00") # wednesday
    predict_date = pd.Timestamp("2017-07-26T10:00:00") # wednesday
    # predict the further 30 minutes
    freq = '30T'
    train_X, train_Y, test_X, test_Y = prepare_data_for_training(df,
                                                                 predict_date,
                                                                 freq=freq,
                                                                 start=start,
                                                                 periods=2,
                                                                 observation='probability')
    # train_X, train_Y, test_X, test_Y = prepare_data_for_training(df, predict_date, freq='1H', start=start, periods=2)

    bst = fit(train_X, train_Y, test_X, test_Y)
    # err = error_rate(bst, test_X, test_Y)
    # print("Error rate: {}".format(err))
    pred = prediction(bst, test_X, test_Y)
    rmse = np.sqrt(np.mean((pred - test_Y)**2))
    print("RMSE: {}".format(rmse))

    # put observation and prediction in a 'test' DataFrame
    test = test_X.copy()
    #obs = test_Y.to_frame()
    obs['ts_future'] = test_Y.index.shift(1, freq=freq)

    test['observation'] = test_Y.copy()
    test['ts_future'] = test_Y.index.shift(1, freq=freq)
    test['prediction'] = pred
    test['error'] = pred - test_Y
    test['relative_error'] = 100. * np.abs(pred - test_Y) / test_Y
    test['quad_error'] = (pred - test_Y)**2
    test.to_csv("prediction-freq-{}-{}.csv".format(freq, predict_date))
<|endoftext|>


<commit_before># -*- test-case-name: go.apps.http_api.tests.test_vumi_app -*-
from collections import defaultdict
import random

from twisted.internet.defer import inlineCallbacks, maybeDeferred
from twisted.web import http

from vumi.utils import http_request_full
from vumi.transports.httprpc import httprpc
from vumi import log

from go.vumitools.api import VumiApi
from go.vumitools.app_worker import GoApplicationWorker
from go.apps.http_api.resource import (StreamingResource, MessageStream,
                                        EventStream)

class StreamingClientManager(object):

    MAX_BACKLOG_SIZE = 100
    CLIENT_PREFIX = 'clients'

    def __init__(self, redis):
        self.redis = redis
        self.clients = defaultdict(list)

    def client_key(self, *args):
        return u':'.join([self.CLIENT_PREFIX] + map(unicode, args))

    def backlog_key(self, key):
        return self.client_key('backlog', key)

    @inlineCallbacks
    def flush_backlog(self, key, message_class, callback):
        backlog_key = self.backlog_key(key)
        while True:
            obj = yield self.redis.rpop(backlog_key)
            if obj is None:
                break
            yield maybeDeferred(callback, message_class.from_json(obj))

    def start(self, key, message_class, callback):
        self.clients[key].append(callback)

    def stop(self, key, callback):
        self.clients[key].remove(callback)

    def publish(self, key, msg):
        callbacks = self.clients[key]
        if callbacks:
            callback = random.choice(callbacks)
            return maybeDeferred(callback, msg)
        else:
            return self.queue_in_backlog(key, msg)

    @inlineCallbacks
    def queue_in_backlog(self, key, msg):
        backlog_key = self.backlog_key(key)
        yield self.redis.lpush(backlog_key, msg.to_json())
        yield self.redis.ltrim(backlog_key, 0, self.MAX_BACKLOG_SIZE - 1)

class StreamingHTTPWorker(GoApplicationWorker):
    """

    :param str web_path:
        The path the HTTP worker should expose the API on
    :param int web_port:
        The port the HTTP worker should open for the API
    :param str health_path:
        The path the resource should receive health checks on.
        Defaults to '/health/'
    """
    worker_name = 'http_api_worker'
    SEND_TO_TAGS = frozenset(['default'])

    def validate_config(self):
        super(StreamingHTTPWorker, self).validate_config()
        self.web_path = self.config['web_path']
        self.web_port = int(self.config['web_port'])
        self.health_path = self.config.get('health_path', '/health/')
        self.metrics_prefix = self.config['metrics_prefix']

    @inlineCallbacks
    def setup_application(self):
        yield super(StreamingHTTPWorker, self).setup_application()
        # Set these to empty dictionaries because we're not interested
        # in using any of the helper functions at this point.
        self._event_handlers = {}
        self._session_handlers = {}
        self.vumi_api = yield VumiApi.from_config_async(self.config)
        self.client_manager = StreamingClientManager(
            self.redis.sub_manager('http_api:message_cache'))

        self.webserver = self.start_web_resources([
            (StreamingResource(self), self.web_path),
            (httprpc.HttpRpcHealthResource(self), self.health_path),
        ], self.web_port)

    def stream(self, stream_class, conversation_key, message):
        # Publish the message by manually specifying the routing key
        rk = stream_class.routing_key % {
            'transport_name': self.transport_name,
            'conversation_key': conversation_key,
        }
        return self.client_manager.publish(rk, message)

    def register_client(self, key, message_class, callback):
        self.client_manager.start(key, message_class, callback)
        return self.client_manager.flush_backlog(key, message_class, callback)

    def unregister_client(self, conversation_key, callback):
        self.client_manager.stop(conversation_key, callback)

    def get_api_metadata(self, conversation, key):
        metadata = conversation.get_metadata(default={})
        http_api_metadata = metadata.get('http_api', {})
        return http_api_metadata.get(key)

    @inlineCallbacks
    def consume_user_message(self, message):
        md = self.get_go_metadata(message)
        account_key = yield md.get_account_key()
        raw_conv = yield md.get_conversation()
        if raw_conv is None:
            log.warning("Cannot find conversation for message: %r" % (
                message,))
            return

        user_api = self.get_user_api(account_key)
        conversation = user_api.wrap_conversation(raw_conv)
        push_message_url = self.get_api_metadata(conversation,
            'push_message_url')
        if push_message_url:
            resp = yield self.push(push_message_url, message)
            if resp.code != http.OK:
                log.warning('Got unexpected response code %s from %s' % (
                    resp.code, push_message_url))
        else:
            yield self.stream(MessageStream, conversation.key, message)

    @inlineCallbacks
    def consume_unknown_event(self, event):
        """
        FIXME:  We're forced to do too much hoopla when trying to link events
                back to the conversation the original message was part of.
        """
        outbound_message = yield self.find_outboundmessage_for_event(event)
        if outbound_message is None:
            log.warning('Unable to find message %s for event %s.' % (
                event['user_message_id'], event['event_id']))
        batch = yield outbound_message.batch.get()
        account_key = batch.metadata['user_account']
        user_api = self.get_user_api(account_key)
        conversations = user_api.conversation_store.conversations
        mr = conversations.index_lookup('batches', batch.key)
        [conv_key] = yield mr.get_keys()
        conversation = yield user_api.get_wrapped_conversation(conv_key)
        push_event_url = self.get_api_metadata(conversation, 'push_event_url')
        if push_event_url:
            resp = yield self.push(push_event_url, event)
            if resp.code != http.OK:
                log.warning('Got unexpected response code %s from %s' % (
                    resp.code, push_event_url))
        else:
            yield self.stream(EventStream, conversation.key, event)

    def push(self, url, vumi_message):
        data = vumi_message.to_json().encode('utf-8')
        return http_request_full(url.encode('utf-8'), data=data, headers={
            'Content-Type': 'application/json; charset=utf-8',
        })

    def get_health_response(self):
        return str(sum([len(callbacks) for callbacks in
                    self.client_manager.clients.values()]))

    @inlineCallbacks
    def teardown_application(self):
        yield super(StreamingHTTPWorker, self).teardown_application()
        yield self.webserver.loseConnection()
<commit_msg>adding process_command_start() (thanks @jerith)<commit_after># -*- test-case-name: go.apps.http_api.tests.test_vumi_app -*-
from collections import defaultdict
import random

from twisted.internet.defer import inlineCallbacks, maybeDeferred
from twisted.web import http

from vumi.utils import http_request_full
from vumi.transports.httprpc import httprpc
from vumi import log

from go.vumitools.api import VumiApi
from go.vumitools.app_worker import GoApplicationWorker
from go.apps.http_api.resource import (StreamingResource, MessageStream,
                                        EventStream)

class StreamingClientManager(object):

    MAX_BACKLOG_SIZE = 100
    CLIENT_PREFIX = 'clients'

    def __init__(self, redis):
        self.redis = redis
        self.clients = defaultdict(list)

    def client_key(self, *args):
        return u':'.join([self.CLIENT_PREFIX] + map(unicode, args))

    def backlog_key(self, key):
        return self.client_key('backlog', key)

    @inlineCallbacks
    def flush_backlog(self, key, message_class, callback):
        backlog_key = self.backlog_key(key)
        while True:
            obj = yield self.redis.rpop(backlog_key)
            if obj is None:
                break
            yield maybeDeferred(callback, message_class.from_json(obj))

    def start(self, key, message_class, callback):
        self.clients[key].append(callback)

    def stop(self, key, callback):
        self.clients[key].remove(callback)

    def publish(self, key, msg):
        callbacks = self.clients[key]
        if callbacks:
            callback = random.choice(callbacks)
            return maybeDeferred(callback, msg)
        else:
            return self.queue_in_backlog(key, msg)

    @inlineCallbacks
    def queue_in_backlog(self, key, msg):
        backlog_key = self.backlog_key(key)
        yield self.redis.lpush(backlog_key, msg.to_json())
        yield self.redis.ltrim(backlog_key, 0, self.MAX_BACKLOG_SIZE - 1)

class StreamingHTTPWorker(GoApplicationWorker):
    """

    :param str web_path:
        The path the HTTP worker should expose the API on
    :param int web_port:
        The port the HTTP worker should open for the API
    :param str health_path:
        The path the resource should receive health checks on.
        Defaults to '/health/'
    """
    worker_name = 'http_api_worker'
    SEND_TO_TAGS = frozenset(['default'])

    def validate_config(self):
        super(StreamingHTTPWorker, self).validate_config()
        self.web_path = self.config['web_path']
        self.web_port = int(self.config['web_port'])
        self.health_path = self.config.get('health_path', '/health/')
        self.metrics_prefix = self.config['metrics_prefix']

    @inlineCallbacks
    def setup_application(self):
        yield super(StreamingHTTPWorker, self).setup_application()
        # Set these to empty dictionaries because we're not interested
        # in using any of the helper functions at this point.
        self._event_handlers = {}
        self._session_handlers = {}
        self.vumi_api = yield VumiApi.from_config_async(self.config)
        self.client_manager = StreamingClientManager(
            self.redis.sub_manager('http_api:message_cache'))

        self.webserver = self.start_web_resources([
            (StreamingResource(self), self.web_path),
            (httprpc.HttpRpcHealthResource(self), self.health_path),
        ], self.web_port)

    def stream(self, stream_class, conversation_key, message):
        # Publish the message by manually specifying the routing key
        rk = stream_class.routing_key % {
            'transport_name': self.transport_name,
            'conversation_key': conversation_key,
        }
        return self.client_manager.publish(rk, message)

    def register_client(self, key, message_class, callback):
        self.client_manager.start(key, message_class, callback)
        return self.client_manager.flush_backlog(key, message_class, callback)

    def unregister_client(self, conversation_key, callback):
        self.client_manager.stop(conversation_key, callback)

    def get_api_metadata(self, conversation, key):
        metadata = conversation.get_metadata(default={})
        http_api_metadata = metadata.get('http_api', {})
        return http_api_metadata.get(key)

    def process_command_start(self, batch_id, conversation_type,
                              conversation_key, msg_options,
                              is_client_initiated, **extra_params):
        log.info("Starting HTTP API for conversation (key: %r)." %
                 (conversation_key,))

    @inlineCallbacks
    def consume_user_message(self, message):
        md = self.get_go_metadata(message)
        account_key = yield md.get_account_key()
        raw_conv = yield md.get_conversation()
        if raw_conv is None:
            log.warning("Cannot find conversation for message: %r" % (
                message,))
            return

        user_api = self.get_user_api(account_key)
        conversation = user_api.wrap_conversation(raw_conv)
        push_message_url = self.get_api_metadata(conversation,
            'push_message_url')
        if push_message_url:
            resp = yield self.push(push_message_url, message)
            if resp.code != http.OK:
                log.warning('Got unexpected response code %s from %s' % (
                    resp.code, push_message_url))
        else:
            yield self.stream(MessageStream, conversation.key, message)

    @inlineCallbacks
    def consume_unknown_event(self, event):
        """
        FIXME:  We're forced to do too much hoopla when trying to link events
                back to the conversation the original message was part of.
        """
        outbound_message = yield self.find_outboundmessage_for_event(event)
        if outbound_message is None:
            log.warning('Unable to find message %s for event %s.' % (
                event['user_message_id'], event['event_id']))
        batch = yield outbound_message.batch.get()
        account_key = batch.metadata['user_account']
        user_api = self.get_user_api(account_key)
        conversations = user_api.conversation_store.conversations
        mr = conversations.index_lookup('batches', batch.key)
        [conv_key] = yield mr.get_keys()
        conversation = yield user_api.get_wrapped_conversation(conv_key)
        push_event_url = self.get_api_metadata(conversation, 'push_event_url')
        if push_event_url:
            resp = yield self.push(push_event_url, event)
            if resp.code != http.OK:
                log.warning('Got unexpected response code %s from %s' % (
                    resp.code, push_event_url))
        else:
            yield self.stream(EventStream, conversation.key, event)

    def push(self, url, vumi_message):
        data = vumi_message.to_json().encode('utf-8')
        return http_request_full(url.encode('utf-8'), data=data, headers={
            'Content-Type': 'application/json; charset=utf-8',
        })

    def get_health_response(self):
        return str(sum([len(callbacks) for callbacks in
                    self.client_manager.clients.values()]))

    @inlineCallbacks
    def teardown_application(self):
        yield super(StreamingHTTPWorker, self).teardown_application()
        yield self.webserver.loseConnection()
<|endoftext|>