<commit_before>#  Copyright 2008-2009 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import wx
from wx import grid

from robotide.publish import RideGridCellChanged
from robotide.utils import PopupMenu, RideEventHandler

from grid import GridEditor
from contentassist import ExpandingContentAssistTextCtrl
from popupwindow import RidePopupWindow

class KeywordEditorUi(GridEditor, RideEventHandler):

    def __init__(self, parent, num_rows, num_cols):
        GridEditor.__init__(self, parent)
        self.SetRowLabelSize(25)
        self.SetColLabelSize(0)
        self.SetDefaultColSize(170)
        self.SetDefaultCellOverflow(False)
        self.CreateGrid(num_rows, num_cols)
        self.Bind(grid.EVT_GRID_LABEL_RIGHT_CLICK, self.OnLabelRightClick)
        # This makes it possible to select cell 0,0 without opening editor, issue 479
        self.SetGridCursor(self.NumberRows -1, self.NumberCols -1)

    def write_cell(self, row, col, value, update_history=True):
        previous = self.GetCellValue(row, col) \
                if (row < self.NumberRows and col < self.NumberCols) else ''
        GridEditor.write_cell(self, row, col, value, update_history)
        RideGridCellChanged(cell=(row, col), value=value, previous=previous,
                            grid=self).publish()

    def _remove_selected_rows(self):
        """If whole row(s) are selected, remove them from the grid"""
        self._update_history()
        for row in sorted(self.GetSelectedRows(), reverse=True):
            self.DeleteRows(row, 1)

    def _get_selected_rows(self):
        rows = self.GetSelectedRows()
        if not rows:
            rows = self._active_row and [self._active_row] or \
                    [self.selection.topleft.row]
        return rows

    def _toggle_underlined(self, cell):
        font = self.GetCellFont(cell.Row, cell.Col)
        font.SetUnderlined(not font.Underlined)
        self.SetCellFont(cell.Row, cell.Col, font)
        self.Refresh()

    def comment(self):
        self._do_action_on_selected_rows(self._comment_row)

    def uncomment(self):
        self._do_action_on_selected_rows(self._uncomment_row)

    def _comment_row(self, row):
        rowdata = self._get_commented_row(row)
        if len(rowdata) > self.NumberCols:
            self.AppendCols(1)
        for col, value in enumerate(rowdata):
            self.write_cell(row, col, value)

    def _get_commented_row(self, row):
        data = [ self.GetCellValue(row, col) for col in range(self.NumberCols) ]
        comment_index = self._get_comment_insertion_index(data)
        data.insert(comment_index, 'Comment')
        return self._strip_trailing_empty_cells(data)

    def _get_comment_insertion_index(self, data):
        index = 0
        while data:
            if data[0]:
                break
            index += 1
            data = data[1:]
        return index

    def _uncomment_row(self, row):
        if self._row_is_commented(row):
            for col in range(1, self.GetNumberCols()):
                value = self.GetCellValue(row, col)
                self.write_cell(row, col-1, value)
            self.write_cell(row, self.GetNumberCols()-1, '')

    def _row_is_commented(self, row):
        data = [ self.GetCellValue(row, col) for col in range(self.NumberCols) ]
        while data:
            if self._is_comment(data[0]):
                return True
            data = data[1:]
        return False

    def _is_comment(self, value):
        return value.lower().strip() == 'comment'

    def _do_action_on_selected_rows(self, action):
        for row in self._get_selected_rows():
            action(row)
        self.set_dirty()

    def OnLabelRightClick(self, event):
        self._active_row = event.GetRow()
        PopupMenu(self, ['Insert Rows', 'Delete Rows\tDel',
                         'Comment Rows\tCtrl-3', 'Uncomment Rows\tCtrl-4'])
        self._active_row = None
        event.Skip()

    def OnInsertRows(self, event):
        self.InsertRows(*self._get_insertion_position_and_row_count(event))
        self.GetParent().Sizer.Layout()
        event.Skip()

    def _get_insertion_position_and_row_count(self, event):
        if isinstance(event.EventObject, wx.Button):
            return self.GetNumberRows(), 1
        rows = self._get_selected_rows()
        return min(rows), len(rows)

    def OnDeleteRows(self, event):
        self.set_dirty()
        self._remove_selected_rows()
        self.GetParent().Sizer.Layout()
        event.Skip()

    def OnCommentRows(self, event):
        self.comment()
        event.Skip()

    def OnUncommentRows(self, event):
        self.uncomment()
        event.Skip()

class KeywordEditor(KeywordEditorUi):
    dirty = property(lambda self: self._controller.dirty)
    _no_cell = grid.GridCellCoords(-1, -1)

    def __init__(self, parent, controller, tree):
        self._keywords = controller.steps
        KeywordEditorUi.__init__(self, parent, len(self._keywords)+5, 5)
        self.SetDefaultEditor(ContentAssistCellEditor(parent.plugin))
        self._controller = controller
        # TODO: Tooltip may be smaller when the documentation is wrapped correctly
        self._tooltip = RidePopupWindow(self, (650, 400))
        self._marked_cell = None
        self._idle_mouse_cell = self._no_cell
        self._active_row = self._active_col = None
        self._make_bindings()
        self._write_keywords(self._keywords)
        self._tree = tree
        self._plugin = parent.plugin

    def _make_bindings(self):
        self.Bind(grid.EVT_GRID_EDITOR_SHOWN, self.OnEditor)
        self.Bind(wx.EVT_KEY_DOWN, self.OnKey)
        self.Bind(wx.EVT_IDLE, self.OnIdle)
        self.Bind(grid.EVT_GRID_CELL_LEFT_CLICK, self.OnCellLeftClick)
        self.Bind(grid.EVT_GRID_CELL_LEFT_DCLICK, self.OnCellLeftDClick)

    def _write_keywords(self, keywords):
        data = []
        for step in keywords:
            data.append(self._format_comments(step.as_list()))
            if hasattr(step, 'steps'):
                for s in step.steps:
                    data.append([''] + self._format_comments(s.as_list()))
        self._write_data(data, update_history=False)

    def _format_comments(self, data):
        # TODO: This should be moved to robot.model
        in_comment = False
        ret = []
        for cell in data:
            if cell.strip().startswith('#'):
                in_comment = True
            if in_comment:
                cell = cell.replace(' |', '')
            ret.append(cell)
        return ret

    def get_selected_datafile_controller(self):
        return self._controller.datafile_controller

    def OnCopy(self, event=None):
        self.copy()

    def OnCut(self, event=None):
        self.cut()
        self._save_keywords()
        self.set_dirty()

    def OnDelete(self, event=None):
        self.delete()
        self._save_keywords()
        self.set_dirty()

    def OnPaste(self, event=None):
        self.paste()
        self._save_keywords()
        self.set_dirty()

    def OnUndo(self, event=None):
        self.undo()
        self._save_keywords()
        self.set_dirty()

    def set_dirty(self):
        self._controller.mark_dirty()
        self._tree.mark_dirty(self._controller)

    def save(self):
        self.hide_tooltip()
        if self.IsCellEditControlShown():
            cell_editor = self.GetCellEditor(*self.selection.cell)
            cell_editor.EndEdit(self.selection.topleft.row,
                                self.selection.topleft.col, self)
        if self.dirty:
            self._save_keywords()

    def _save_keywords(self):
        rows = []
        for i in range(self.GetNumberRows()):
            rowdata = []
            for j in range(self.GetNumberCols()):
                cellvalue = self.GetCellValue(i, j).replace('\n', ' ')
                rowdata.append(cellvalue)
            rowdata = self._strip_trailing_empty_cells(rowdata)
            if rowdata:
                rows.append(rowdata)
        self._controller.parse_steps_from_rows(rows)

    def show_content_assist(self):
        if self.IsCellEditControlShown():
            self.GetCellEditor(*self.selection.cell).show_content_assist()

    def _calculate_position(self):
        x, y = wx.GetMousePosition()
        return x, y + 20

    def OnEditor(self, event):
        row_height = self.GetRowSize(self.selection.topleft.row)
        self.GetCellEditor(*self.selection.cell).SetHeight(row_height)
        event.Skip()

    def OnKey(self, event):
        keycode, control_down = event.GetKeyCode(), event.ControlDown()
        if keycode == wx.WXK_CONTROL and self._tooltip.IsShown():
            return
        self.hide_tooltip()
        if control_down or keycode not in [wx.WXK_RETURN, wx.WXK_BACK]:
            event.Skip()
            return
        self.DisableCellEditControl()
        if keycode == wx.WXK_RETURN:
            self.MoveCursorRight(event.ShiftDown())
        else:
            self.MoveCursorLeft(event.ShiftDown())

    def OnCellLeftClick(self, event):
        self.hide_tooltip()
        if event.ControlDown():
            if not self._navigate_to_matching_user_keyword(event.Row, event.Col):
                event.Skip()
        else:
            event.Skip()

    def OnCellLeftDClick(self, event):
        self.hide_tooltip()
        if not self._navigate_to_matching_user_keyword(event.Row, event.Col):
            event.Skip()

    def _navigate_to_matching_user_keyword(self, row, col):
        value = self.GetCellValue(row, col)
        uk = self._plugin.get_user_keyword(value)
        if uk:
            self._toggle_underlined((grid.GridCellCoords(row, col)))
            self._marked_cell = None
            self._tree.select_user_keyword_node(uk)
            return True
        return False

    def OnIdle(self, evt):
        if not self._is_active_window() or self.IsCellEditControlShown():
            self.hide_tooltip()
            return
        cell = self._cell_under_cursor()
        if cell == self._no_cell:
            return
        if not wx.GetMouseState().ControlDown():
            self.hide_tooltip()
            self._hide_link_if_necessary(cell)
            return
        self._idle_mouse_cell = cell
        self._hide_link_if_necessary(cell)
        self._show_possible_user_keyword_link(cell)
        if not self._tooltip.IsShown():
            self._show_kw_tooltip(cell)

    def _is_active_window(self):
        return self.IsShownOnScreen() and self.FindFocus()

    def _cell_under_cursor(self):
        pos = self.CalcUnscrolledPosition(self.ScreenToClient(wx.GetMousePosition()))
        return self.XYToCell(*pos)

    def _show_possible_user_keyword_link(self, cell):
        if cell == self._marked_cell:
            return
        value = self.GetCellValue(cell.Row, cell.Col)
        if not self._plugin.get_user_keyword(value):
            return
        self._toggle_underlined(cell)
        self._marked_cell = cell

    def _hide_link_if_necessary(self, cell):
        if not self._marked_cell:
            return
        if cell != self._marked_cell:
            self._toggle_underlined(self._marked_cell)
            self._marked_cell = None

    def _show_kw_tooltip(self, cell):
        value = self.GetCellValue(cell.Row, cell.Col)
        details = self._plugin.get_keyword_details(value)
        if not details:
            return
        self._tooltip.set_content(details)
        point = self.CellToRect(cell.Row, cell.Col).GetTopRight()
        point.x += self.GetRowLabelSize() + 5
        point = self.CalcScrolledPosition(point)
        self._tooltip.SetPosition(self.ClientToScreen(point))
        self._tooltip.Show()

    def hide_tooltip(self):
        if self._tooltip and self._tooltip.IsShown():
            self._tooltip.Show(False)

class ContentAssistCellEditor(grid.PyGridCellEditor):

    def __init__(self, plugin):
        grid.PyGridCellEditor.__init__(self)
        self._plugin = plugin
        self._grid = None
        self._previous_value = None

    def show_content_assist(self):
        self._tc.show_content_assist()

    def Create(self, parent, id, evthandler):
        self._tc = ExpandingContentAssistTextCtrl(parent, self._plugin)
        self._tc.Bind(wx.EVT_TEXT, self.OnText, self._tc)
        self.SetControl(self._tc)
        if evthandler:
            self._tc.PushEventHandler(evthandler)

    def SetSize(self, rect):
        self._tc.SetDimensions(rect.x, rect.y, rect.width+2, rect.height+2,
                               wx.SIZE_ALLOW_MINUS_ONE)

    def SetHeight(self, height):
        self._height = height

    def BeginEdit(self, row, col, grid):
        self._tc.SetSize((-1, self._height))
        self._original_value = self._previous_value = grid.GetCellValue(row, col)
        self._grid = grid
        self.StartingClick()

    def EndEdit(self, row, col, grid):
        if self._tc.content_assist_value():
            value = self._tc.content_assist_value()
        else:
            value = self._tc.GetValue()
        if value != self._original_value:
            grid.write_cell(row, col, value)
            grid.set_dirty()
        self._previous_value = value
        self._tc.hide()
        grid.SetFocus()
        return True

    def Reset(self):
        self._tc.SetValue(self._original_value)
        self._tc.reset()

    def StartingKey(self, event):
        key = event.GetKeyCode()
        if key is wx.WXK_DELETE or key > 255:
            self._grid.HideCellEditControl()
            return
        self._tc.SetValue(unichr(key))
        self._tc.SetFocus()
        self._tc.SetInsertionPointEnd()

    def StartingClick(self):
        self._tc.SetValue(self._original_value)
        self._tc.SelectAll()
        self._tc.SetFocus()

    def Clone(self):
        return ContentAssistCellEditor()

    def OnText(self, event):
        if self._previous_value != self._tc.GetValue() and self._grid:
            self._grid.set_dirty()
        self._previous_value = self._tc.GetValue()
        event.Skip()
<commit_msg>Automated merge with https://robotframework-ride.googlecode.com/hg/<commit_after>#  Copyright 2008-2009 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import wx
from wx import grid

from robotide.publish import RideGridCellChanged
from robotide.utils import PopupMenu, RideEventHandler

from grid import GridEditor
from contentassist import ExpandingContentAssistTextCtrl
from popupwindow import RideHtmlPopupWindow

class KeywordEditorUi(GridEditor, RideEventHandler):

    def __init__(self, parent, num_rows, num_cols):
        GridEditor.__init__(self, parent)
        self.SetRowLabelSize(25)
        self.SetColLabelSize(0)
        self.SetDefaultColSize(170)
        self.SetDefaultCellOverflow(False)
        self.CreateGrid(num_rows, num_cols)
        self.Bind(grid.EVT_GRID_LABEL_RIGHT_CLICK, self.OnLabelRightClick)
        # This makes it possible to select cell 0,0 without opening editor, issue 479
        self.SetGridCursor(self.NumberRows -1, self.NumberCols -1)

    def write_cell(self, row, col, value, update_history=True):
        previous = self.GetCellValue(row, col) \
                if (row < self.NumberRows and col < self.NumberCols) else ''
        GridEditor.write_cell(self, row, col, value, update_history)
        RideGridCellChanged(cell=(row, col), value=value, previous=previous,
                            grid=self).publish()

    def _remove_selected_rows(self):
        """If whole row(s) are selected, remove them from the grid"""
        self._update_history()
        for row in sorted(self.GetSelectedRows(), reverse=True):
            self.DeleteRows(row, 1)

    def _get_selected_rows(self):
        rows = self.GetSelectedRows()
        if not rows:
            rows = self._active_row and [self._active_row] or \
                    [self.selection.topleft.row]
        return rows

    def _toggle_underlined(self, cell):
        font = self.GetCellFont(cell.Row, cell.Col)
        font.SetUnderlined(not font.Underlined)
        self.SetCellFont(cell.Row, cell.Col, font)
        self.Refresh()

    def comment(self):
        self._do_action_on_selected_rows(self._comment_row)

    def uncomment(self):
        self._do_action_on_selected_rows(self._uncomment_row)

    def _comment_row(self, row):
        rowdata = self._get_commented_row(row)
        if len(rowdata) > self.NumberCols:
            self.AppendCols(1)
        for col, value in enumerate(rowdata):
            self.write_cell(row, col, value)

    def _get_commented_row(self, row):
        data = [ self.GetCellValue(row, col) for col in range(self.NumberCols) ]
        comment_index = self._get_comment_insertion_index(data)
        data.insert(comment_index, 'Comment')
        return self._strip_trailing_empty_cells(data)

    def _get_comment_insertion_index(self, data):
        index = 0
        while data:
            if data[0]:
                break
            index += 1
            data = data[1:]
        return index

    def _uncomment_row(self, row):
        if self._row_is_commented(row):
            for col in range(1, self.GetNumberCols()):
                value = self.GetCellValue(row, col)
                self.write_cell(row, col-1, value)
            self.write_cell(row, self.GetNumberCols()-1, '')

    def _row_is_commented(self, row):
        data = [ self.GetCellValue(row, col) for col in range(self.NumberCols) ]
        while data:
            if self._is_comment(data[0]):
                return True
            data = data[1:]
        return False

    def _is_comment(self, value):
        return value.lower().strip() == 'comment'

    def _do_action_on_selected_rows(self, action):
        for row in self._get_selected_rows():
            action(row)
        self.set_dirty()

    def OnLabelRightClick(self, event):
        self._active_row = event.GetRow()
        PopupMenu(self, ['Insert Rows', 'Delete Rows\tDel',
                         'Comment Rows\tCtrl-3', 'Uncomment Rows\tCtrl-4'])
        self._active_row = None
        event.Skip()

    def OnInsertRows(self, event):
        self.InsertRows(*self._get_insertion_position_and_row_count(event))
        self.GetParent().Sizer.Layout()
        event.Skip()

    def _get_insertion_position_and_row_count(self, event):
        if isinstance(event.EventObject, wx.Button):
            return self.GetNumberRows(), 1
        rows = self._get_selected_rows()
        return min(rows), len(rows)

    def OnDeleteRows(self, event):
        self.set_dirty()
        self._remove_selected_rows()
        self.GetParent().Sizer.Layout()
        event.Skip()

    def OnCommentRows(self, event):
        self.comment()
        event.Skip()

    def OnUncommentRows(self, event):
        self.uncomment()
        event.Skip()

class KeywordEditor(KeywordEditorUi):
    dirty = property(lambda self: self._controller.dirty)
    _no_cell = grid.GridCellCoords(-1, -1)

    def __init__(self, parent, controller, tree):
        self._keywords = controller.steps
        KeywordEditorUi.__init__(self, parent, len(self._keywords)+5, 5)
        self.SetDefaultEditor(ContentAssistCellEditor(parent.plugin))
        self._controller = controller
        # TODO: Tooltip may be smaller when the documentation is wrapped correctly
        self._tooltip = RideHtmlPopupWindow(self, (650, 400))
        self._marked_cell = None
        self._idle_mouse_cell = self._no_cell
        self._active_row = self._active_col = None
        self._make_bindings()
        self._write_keywords(self._keywords)
        self._tree = tree
        self._plugin = parent.plugin

    def _make_bindings(self):
        self.Bind(grid.EVT_GRID_EDITOR_SHOWN, self.OnEditor)
        self.Bind(wx.EVT_KEY_DOWN, self.OnKey)
        self.Bind(wx.EVT_IDLE, self.OnIdle)
        self.Bind(grid.EVT_GRID_CELL_LEFT_CLICK, self.OnCellLeftClick)
        self.Bind(grid.EVT_GRID_CELL_LEFT_DCLICK, self.OnCellLeftDClick)

    def _write_keywords(self, keywords):
        data = []
        for step in keywords:
            data.append(self._format_comments(step.as_list()))
            if hasattr(step, 'steps'):
                for s in step.steps:
                    data.append([''] + self._format_comments(s.as_list()))
        self._write_data(data, update_history=False)

    def _format_comments(self, data):
        # TODO: This should be moved to robot.model
        in_comment = False
        ret = []
        for cell in data:
            if cell.strip().startswith('#'):
                in_comment = True
            if in_comment:
                cell = cell.replace(' |', '')
            ret.append(cell)
        return ret

    def get_selected_datafile_controller(self):
        return self._controller.datafile_controller

    def OnCopy(self, event=None):
        self.copy()

    def OnCut(self, event=None):
        self.cut()
        self._save_keywords()
        self.set_dirty()

    def OnDelete(self, event=None):
        self.delete()
        self._save_keywords()
        self.set_dirty()

    def OnPaste(self, event=None):
        self.paste()
        self._save_keywords()
        self.set_dirty()

    def OnUndo(self, event=None):
        self.undo()
        self._save_keywords()
        self.set_dirty()

    def set_dirty(self):
        self._controller.mark_dirty()
        self._tree.mark_dirty(self._controller)

    def save(self):
        self.hide_tooltip()
        if self.IsCellEditControlShown():
            cell_editor = self.GetCellEditor(*self.selection.cell)
            cell_editor.EndEdit(self.selection.topleft.row,
                                self.selection.topleft.col, self)
        if self.dirty:
            self._save_keywords()

    def _save_keywords(self):
        rows = []
        for i in range(self.GetNumberRows()):
            rowdata = []
            for j in range(self.GetNumberCols()):
                cellvalue = self.GetCellValue(i, j).replace('\n', ' ')
                rowdata.append(cellvalue)
            rowdata = self._strip_trailing_empty_cells(rowdata)
            if rowdata:
                rows.append(rowdata)
        self._controller.parse_steps_from_rows(rows)

    def show_content_assist(self):
        if self.IsCellEditControlShown():
            self.GetCellEditor(*self.selection.cell).show_content_assist()

    def _calculate_position(self):
        x, y = wx.GetMousePosition()
        return x, y + 20

    def OnEditor(self, event):
        row_height = self.GetRowSize(self.selection.topleft.row)
        self.GetCellEditor(*self.selection.cell).SetHeight(row_height)
        event.Skip()

    def OnKey(self, event):
        keycode, control_down = event.GetKeyCode(), event.ControlDown()
        if keycode == wx.WXK_CONTROL and self._tooltip.IsShown():
            return
        self.hide_tooltip()
        if control_down or keycode not in [wx.WXK_RETURN, wx.WXK_BACK]:
            event.Skip()
            return
        self.DisableCellEditControl()
        if keycode == wx.WXK_RETURN:
            self.MoveCursorRight(event.ShiftDown())
        else:
            self.MoveCursorLeft(event.ShiftDown())

    def OnCellLeftClick(self, event):
        self.hide_tooltip()
        if event.ControlDown():
            if not self._navigate_to_matching_user_keyword(event.Row, event.Col):
                event.Skip()
        else:
            event.Skip()

    def OnCellLeftDClick(self, event):
        self.hide_tooltip()
        if not self._navigate_to_matching_user_keyword(event.Row, event.Col):
            event.Skip()

    def _navigate_to_matching_user_keyword(self, row, col):
        value = self.GetCellValue(row, col)
        uk = self._plugin.get_user_keyword(value)
        if uk:
            self._toggle_underlined((grid.GridCellCoords(row, col)))
            self._marked_cell = None
            self._tree.select_user_keyword_node(uk)
            return True
        return False

    def OnIdle(self, evt):
        if not self._is_active_window() or self.IsCellEditControlShown():
            self.hide_tooltip()
            return
        cell = self._cell_under_cursor()
        if cell == self._no_cell:
            return
        if not wx.GetMouseState().ControlDown():
            self.hide_tooltip()
            self._hide_link_if_necessary(cell)
            return
        self._idle_mouse_cell = cell
        self._hide_link_if_necessary(cell)
        self._show_possible_user_keyword_link(cell)
        if not self._tooltip.IsShown():
            self._show_kw_tooltip(cell)

    def _is_active_window(self):
        return self.IsShownOnScreen() and self.FindFocus()

    def _cell_under_cursor(self):
        pos = self.CalcUnscrolledPosition(self.ScreenToClient(wx.GetMousePosition()))
        return self.XYToCell(*pos)

    def _show_possible_user_keyword_link(self, cell):
        if cell == self._marked_cell:
            return
        value = self.GetCellValue(cell.Row, cell.Col)
        if not self._plugin.get_user_keyword(value):
            return
        self._toggle_underlined(cell)
        self._marked_cell = cell

    def _hide_link_if_necessary(self, cell):
        if not self._marked_cell:
            return
        if cell != self._marked_cell:
            self._toggle_underlined(self._marked_cell)
            self._marked_cell = None

    def _show_kw_tooltip(self, cell):
        value = self.GetCellValue(cell.Row, cell.Col)
        details = self._plugin.get_keyword_details(value)
        if not details:
            return
        self._tooltip.set_content(details)
        point = self.CellToRect(cell.Row, cell.Col).GetTopRight()
        point.x += self.GetRowLabelSize() + 5
        point = self.CalcScrolledPosition(point)
        self._tooltip.SetPosition(self.ClientToScreen(point))
        self._tooltip.Show()

    def hide_tooltip(self):
        if self._tooltip and self._tooltip.IsShown():
            self._tooltip.Show(False)

class ContentAssistCellEditor(grid.PyGridCellEditor):

    def __init__(self, plugin):
        grid.PyGridCellEditor.__init__(self)
        self._plugin = plugin
        self._grid = None
        self._previous_value = None

    def show_content_assist(self):
        self._tc.show_content_assist()

    def Create(self, parent, id, evthandler):
        self._tc = ExpandingContentAssistTextCtrl(parent, self._plugin)
        self._tc.Bind(wx.EVT_TEXT, self.OnText, self._tc)
        self.SetControl(self._tc)
        if evthandler:
            self._tc.PushEventHandler(evthandler)

    def SetSize(self, rect):
        self._tc.SetDimensions(rect.x, rect.y, rect.width+2, rect.height+2,
                               wx.SIZE_ALLOW_MINUS_ONE)

    def SetHeight(self, height):
        self._height = height

    def BeginEdit(self, row, col, grid):
        self._tc.SetSize((-1, self._height))
        self._original_value = self._previous_value = grid.GetCellValue(row, col)
        self._grid = grid
        self.StartingClick()

    def EndEdit(self, row, col, grid):
        if self._tc.content_assist_value():
            value = self._tc.content_assist_value()
        else:
            value = self._tc.GetValue()
        if value != self._original_value:
            grid.write_cell(row, col, value)
            grid.set_dirty()
        self._previous_value = value
        self._tc.hide()
        grid.SetFocus()
        return True

    def Reset(self):
        self._tc.SetValue(self._original_value)
        self._tc.reset()

    def StartingKey(self, event):
        key = event.GetKeyCode()
        if key is wx.WXK_DELETE or key > 255:
            self._grid.HideCellEditControl()
            return
        self._tc.SetValue(unichr(key))
        self._tc.SetFocus()
        self._tc.SetInsertionPointEnd()

    def StartingClick(self):
        self._tc.SetValue(self._original_value)
        self._tc.SelectAll()
        self._tc.SetFocus()

    def Clone(self):
        return ContentAssistCellEditor()

    def OnText(self, event):
        if self._previous_value != self._tc.GetValue() and self._grid:
            self._grid.set_dirty()
        self._previous_value = self._tc.GetValue()
        event.Skip()
<|endoftext|>


<commit_before>import math
import numpy as np
from numpy import cos, sin, pi
from gym.envs.classic_control import rendering
from utils import *

class Part():

    def __init__(self):
        self.parent = None
        self.parts = []
        self._transform = None
        self._render_transform = None
        self._apos = None
        self.set_position(0., 0., 0.)

    def set_position(self, x, y, theta):
        self._position = np.array([x, y, theta, 1])
        if len(self.parts) > 0:
            self.set_transform()
        for part in self.parts:
            part._apos = None

    def set_rotation(self, theta):
        x, y, _, _ = self._position
        self.set_position(x, y, theta)

    def translate(self, dist, angle):
        x, y, theta, _ = self._position
        self.set_position(x + dist * cos(angle), y + dist * sin(angle), theta)

    def set_transform(self):
        x, y, theta = self.get_position()
        self._transform = Part.get_transform(x, y, theta)

    def get_position(self):
        if self.parent is None:
            return self._position[:-1]
        else:
            if self._apos is None:
                pos = np.reshape(self._position, (4,1))
                self._apos = (self.parent._transform * pos)[:-1]
            return self._apos

    def part_distances(self, parts):
        '''
        Returns the distances and angles to the specified list of parts.
        '''
        distances = []
        for part in parts:
            if part != self:
                d, alpha = self.part_distance(part)
                distances.append((d, alpha, part))
        return distances

    def part_distance(self, part):
        '''
        Calculates the distance and relative angle to the specified part.
        '''
        x1, y1, th1 = self.get_position()
        x2, y2, th2 = part.get_position()
        d, alpha = self.distance(x2, y2)
        alpha2 = th2 - th1 + alpha
        bd2 = part.boundary_distance(alpha2)
        return d - bd2, alpha

    def distance(self, x, y):
        '''
        Calculates the distance and relative angle to the specified location.
        '''
        x1, y1, th1 = self.get_position()
        dx = x - x1
        dy = y - y1
        dc = math.sqrt(dx ** 2 + dy ** 2)
        alpha = canonical_angle(math.atan2(dy, dx) - th1)
        bd = self.boundary_distance(alpha)
        return dc - bd, alpha

    def boundary_distance(self, alpha):
        '''
        Calculates the part's boundary distance in the specified direction.
        '''
        return 0.0

    def visible_arc(self, angle):
        xp, yp, thp = self.get_position()
        thcs = thp - angle
        return self.boundary_distance(thcs)

    def is_collided(self, part):
        dist, angle = self.part_distance(part)
        return dist <= 0.

    def add_part(self, part, x, y, theta):
        if self._transform is None:
            self.set_transform()
        part.set_position(x, y, theta)
        part.parent = self
        self.parts.append(part)
        return part

    def get_rotation(a):
        ca = cos(a)
        sa = sin(a)
        return np.matrix([\
            [ ca, -sa, 0., 0.], \
            [ sa, ca, 0., 0.], \
            [ 0., 0., 1., a ], \
            [ 0., 0., 0., 1.]])

    def get_translation(x, y):
        return np.matrix([\
            [1., 0., 0., x ], \
            [0., 1., 0., y ], \
            [0., 0., 1., 0.], \
            [0., 0., 0., 1.]])

    def get_transform(x, y, theta, unit=1.):
        translation = Part.get_translation(x, y)
        rotation = Part.get_rotation(theta)
        transform = translation * rotation
        return transform

    def get_geometry(self):
        return []

    def render(self, viewer):

        if self._render_transform is None:
            self._render_transform = rendering.Transform()
            for g in self.get_geometry():
                viewer.add_geom(g)
                g.add_attr(self._render_transform)

        x, y, theta = self.get_position()
        self._render_transform.set_translation(x, y)
        self._render_transform.set_rotation(theta)

        for part in self.parts:
            part.render(viewer)

class RectangularPart(Part):

    def __init__(self, length, width):
        Part.__init__(self)
        self.length = length
        self.width = width
        self._diag_angle = math.atan2(width, length)

        self.front_left = self.add_part(Part(), self.length / 2., self.width / 2., 0.)
        self.front_right = self.add_part(Part(), self.length / 2., -self.width / 2., 0.)
        self.back_left = self.add_part(Part(), -self.length / 2., self.width / 2., 0.)
        self.back_right = self.add_part(Part(), -self.length / 2., -self.width / 2., 0.)
        self.corners = (self.front_left, self.front_right, self.back_left, self.back_right)

    def boundary_distance(self, alpha):
        '''
        Calculates the rectangle's boundary distance in the specified direction.
        '''
        l = self.length / 2.0
        w = self.width / 2.0
        alpha = abs(wrap(alpha, -right_angle, right_angle))
        if alpha <= self._diag_angle:
            return l / cos(alpha)
        else:
            return w / sin(alpha)

    def visible_arc(self, alpha):
        xp, yp, thp = self.get_position()
        thcs = thp - alpha
        appw = max(self.width, self.length * abs(sin(thcs))) / 2.
        return appw

<commit_msg>Don't call get_position() twice in a row.<commit_after>import math
import numpy as np
from numpy import cos, sin, pi
from gym.envs.classic_control import rendering
from utils import *

class Part():

    def __init__(self):
        self.parent = None
        self.parts = []
        self._transform = None
        self._render_transform = None
        self._apos = None
        self.set_position(0., 0., 0.)

    def set_position(self, x, y, theta):
        self._position = np.array([x, y, theta, 1])
        if len(self.parts) > 0:
            self.set_transform()
        for part in self.parts:
            part._apos = None

    def set_rotation(self, theta):
        x, y, _, _ = self._position
        self.set_position(x, y, theta)

    def translate(self, dist, angle):
        x, y, theta, _ = self._position
        self.set_position(x + dist * cos(angle), y + dist * sin(angle), theta)

    def set_transform(self):
        x, y, theta = self.get_position()
        self._transform = Part.get_transform(x, y, theta)

    def get_position(self):
        if self.parent is None:
            return self._position[:-1]
        else:
            if self._apos is None:
                pos = np.reshape(self._position, (4,1))
                self._apos = (self.parent._transform * pos)[:-1]
            return self._apos

    def part_distances(self, parts):
        '''
        Returns the distances and angles to the specified list of parts.
        '''
        distances = []
        for part in parts:
            if part != self:
                d, alpha = self.part_distance(part)
                distances.append((d, alpha, part))
        return distances

    def part_distance(self, part):
        '''
        Calculates the distance and relative angle to the specified part.
        '''
        x1, y1, th1 = self.get_position()
        x2, y2, th2 = part.get_position()
        d, alpha = self._distance(x1, y1, th1, x2, y2)
        alpha2 = th2 - th1 + alpha
        bd2 = part.boundary_distance(alpha2)
        return d - bd2, alpha

    def distance(self, x, y):
        x1, y1, th1 = self.get_position()
        return self._distance(x1, y1, th1, x, y)

    def _distance(self, x1, y1, th1, x, y):
        '''
        Calculates the distance and relative angle to the specified location.
        '''
        dx = x - x1
        dy = y - y1
        dc = math.sqrt(dx ** 2 + dy ** 2)
        alpha = canonical_angle(math.atan2(dy, dx) - th1)
        bd = self.boundary_distance(alpha)
        return dc - bd, alpha

    def boundary_distance(self, alpha): #pylint: disable=W0613
        '''
        Calculates the part's boundary distance in the specified direction.
        '''
        return 0.0

    def visible_arc(self, angle):
        _, _, thp = self.get_position()
        thcs = thp - angle
        return self.boundary_distance(thcs)

    def is_collided(self, part):
        dist, _ = self.part_distance(part)
        return dist <= 0.

    def add_part(self, part, x, y, theta):
        if self._transform is None:
            self.set_transform()
        part.set_position(x, y, theta)
        part.parent = self
        self.parts.append(part)
        return part

    def get_rotation(a):
        ca = cos(a)
        sa = sin(a)
        return np.matrix([\
            [ ca, -sa, 0., 0.], \
            [ sa, ca, 0., 0.], \
            [ 0., 0., 1., a ], \
            [ 0., 0., 0., 1.]])

    def get_translation(x, y):
        return np.matrix([\
            [1., 0., 0., x ], \
            [0., 1., 0., y ], \
            [0., 0., 1., 0.], \
            [0., 0., 0., 1.]])

    def get_transform(x, y, theta, unit=1.):
        translation = Part.get_translation(x, y)
        rotation = Part.get_rotation(theta)
        transform = translation * rotation
        return transform

    def get_geometry(self):
        return []

    def render(self, viewer):

        if self._render_transform is None:
            self._render_transform = rendering.Transform()
            for g in self.get_geometry():
                viewer.add_geom(g)
                g.add_attr(self._render_transform)

        x, y, theta = self.get_position()
        self._render_transform.set_translation(x, y)
        self._render_transform.set_rotation(theta)

        for part in self.parts:
            part.render(viewer)

class RectangularPart(Part):

    def __init__(self, length, width):
        Part.__init__(self)
        self.length = length
        self.width = width
        self._diag_angle = math.atan2(width, length)

        self.front_left = self.add_part(Part(), self.length / 2., self.width / 2., 0.)
        self.front_right = self.add_part(Part(), self.length / 2., -self.width / 2., 0.)
        self.back_left = self.add_part(Part(), -self.length / 2., self.width / 2., 0.)
        self.back_right = self.add_part(Part(), -self.length / 2., -self.width / 2., 0.)
        self.corners = (self.front_left, self.front_right, self.back_left, self.back_right)

    def boundary_distance(self, alpha):
        '''
        Calculates the rectangle's boundary distance in the specified direction.
        '''
        l = self.length / 2.0
        w = self.width / 2.0
        alpha = abs(wrap(alpha, -right_angle, right_angle))
        if alpha <= self._diag_angle:
            return l / cos(alpha)
        else:
            return w / sin(alpha)

    def visible_arc(self, angle):
        _, _, thp = self.get_position()
        thcs = thp - angle
        appw = max(self.width, self.length * abs(sin(thcs))) / 2.
        return appw
<|endoftext|>


<commit_before>'''
Created on Jun 17, 2016

@author: svanhmic
'''

import urllib
import gzip
import json
import sys
import requests
from datetime import date, timedelta, datetime
from elasticsearch import Elasticsearch
from fileinput import filename
from multiprocessing import Pool

regn = "http://regnskaber.virk.dk/10275523/ZG9rdW1lbnRsYWdlcjovLzAzLzIxLzQ3L2NkLzFkLzFlYmItNDZlMi1iNDRiLTNlMGUxZDA3ZjJkNQ.xml"
index = "http://distribution.virk.dk/offentliggoerelser/_search"
#jList = "/home/svanhmic/workspace/Python/Erhvervs/data/regnskabsdata/regnskab.json"
site = "http://distribution.virk.dk/offentliggoerelser"
dataFolderXml = "/home/svanhmic/workspace/Python/Erhvervs/data/regnskabsdata/testXML"
HEADERS_FOR_JSON = {

                    'accept': "application/json; charset=utf-8",
                    'content-type': "application/json"
                    }

#with open(jList) as data_file:
#    jData = json.load(data_file)

def getQueryData(from_date=None,to_date=None,size=None):
        # Build query
    data = {}
    data['query'] = {}
    if size is not None:
        data['size'] = size
    if from_date is not None or to_date is not None:
        range_value = {}
        if from_date is not None:
            range_value['from'] = from_date
        if to_date is not None:
                range_value['to'] = to_date
        range_value = {
            'offentliggoerelse.offentliggoerelsesTidspunkt':
            range_value
        }
        data['query']['range'] = range_value
    #print data
    return data

def openAndCollectXML(pathList):

    #print(pathList[2])
    if pathList[2] is not None:
        #Download the .gz files
        urllib.request.urlretrieve(pathList[2],filename=dataFolderXml+"/"+str(pathList[1])+str(pathList[0])+".gz")
    else:
        print("Not an adress"+str(pathList[2]))

    return str(pathList[2])

def extractDokUrl(lst):
    output = list(filter(lambda x: x['dokumentMimeType'] == "application/xml", lst))
    #print(len(output))
    try:
        return output[0]["dokumentUrl"]
    except IndexError:
        #print("No xml file")
        return None

def parseToXmlData(jData):
    '''
        takes a json file and extracts the attachments
    '''
    pool = Pool(processes=4)
    dokData = jData["hits"]["hits"]
    dokList = [(v["_source"]["cvrNummer"]
                     ,v["_source"]["regnskab"]["regnskabsperiode"]["startDato"]
                     ,extractDokUrl(v["_source"]["dokumenter"])) for v in dokData]

    #xmlDok = [openAndCollectXML(i)  for i in dokList]
    xmlDok = pool.map(openAndCollectXML,dokList)
    #for i in range(0,len(dokData)):
    #    #print dokData[i]["_source"]["cvrNummer"]
    #    dokList = dokData[i]["_source"]["dokumenter"]#[0]["dokumentUrl"]
    #    for d in dokList:
    #        if d["dokumentMimeType"] == "application/xml":
    #            x = http.urlopen("GET", d["dokumentUrl"])
    #            text_file = open(dataFolderZip+"/"+str(dokData[i]["_source"]["regnskab"]["regnskabsperiode"]["startDato"])+str(dokData[i]["_source"]["cvrNummer"])+".gz", "w+")
    #            text_file.write(str(x.data))
    #            text_file.close()

                #with gzip.open(dataFolderZip+"/"+str(dokData[i]["_source"]["regnskab"]["regnskabsperiode"]["startDato"])+str(dokData[i]["_source"]["cvrNummer"])+".gz", "rb") as f:
                #    file_content = f.read()
                #text_file = open(dataFolderXml+"/"+str(dokData[i]["_source"]["regnskab"]["regnskabsperiode"]["startDato"])+"cvr"+str(dokData[i]["_source"]["cvrNummer"])+".xml", "w+")
                #text_file.write(file_content)
                #text_file.close()
    #            xmlDok.append(d["dokumentUrl"])

    #for i in dokList:
    #    print(i)
    print("number of xml-Documents collected is: ", len(xmlDok))

if __name__ == '__main__':
    #the main method
    #usage python3 DownloadData.py startdate enddate pathToSaveFile

    if len(sys.argv) == 2:
        start_date = datetime.strptime(sys.argv[1],"%Y-%m-%d")
    elif len(sys.argv) == 3:
        start_date = datetime.strptime(sys.argv[1],"%Y-%m-%d")
        end_date = datetime.strptime(sys.argv[2],"%Y-%m-%d")
    elif len(sys.argv) == 4:
        start_date = datetime.strptime(sys.argv[1],"%Y-%m-%d")
        end_date = datetime.strptime(sys.argv[2],"%Y-%m-%d")
        global dataFolderXml
        dataFolderXml = sys.argv[3]
    else:
        start_date = date(2016,4,1)
        end_date = date(2016,4,2)

    d = start_date
    delta = timedelta(days=1)
    while d < end_date:
        sd = d.strftime("%Y-%m-%d")
        d += delta
        ed = d.strftime("%Y-%m-%d")
        jData = getQueryData(sd,ed,1000000)
        response = requests.get(index, data=json.dumps(jData),
                                     headers=HEADERS_FOR_JSON)
        response_data = response.json()
        parseToXmlData(response_data)
        print(sd)

<commit_msg>extra feature added path is commandlined<commit_after>'''
Created on Jun 17, 2016

@author: svanhmic
'''

import urllib
import os
import json
import sys
import re
import requests
from datetime import date, timedelta, datetime
from elasticsearch import Elasticsearch
from fileinput import filename
from multiprocessing import Pool

regn = "http://regnskaber.virk.dk/10275523/ZG9rdW1lbnRsYWdlcjovLzAzLzIxLzQ3L2NkLzFkLzFlYmItNDZlMi1iNDRiLTNlMGUxZDA3ZjJkNQ.xml"
index = "http://distribution.virk.dk/offentliggoerelser/_search"
#jList = "/home/svanhmic/workspace/Python/Erhvervs/data/regnskabsdata/regnskab.json"
site = "http://distribution.virk.dk/offentliggoerelser"
dataFolderXml = "/home/svanhmic/workspace/Python/Erhvervs/data/regnskabsdata/testXML"
HEADERS_FOR_JSON = {

                    'accept': "application/json; charset=utf-8",
                    'content-type': "application/json"
                    }

#with open(jList) as data_file:
#    jData = json.load(data_file)

def getQueryData(from_date=None,to_date=None,size=None):
        # Build query
    data = {}
    data['query'] = {}
    if size is not None:
        data['size'] = size
    if from_date is not None or to_date is not None:
        range_value = {}
        if from_date is not None:
            range_value['from'] = from_date
        if to_date is not None:
                range_value['to'] = to_date
        range_value = {
            'offentliggoerelse.offentliggoerelsesTidspunkt':
            range_value
        }
        data['query']['range'] = range_value
    #print data
    return data

def openAndCollectXML(pathList):

    #print(pathList[2])
    if pathList[2] is not None:
        #Download the .gz files and saving them as files: date-cvrnumber.gz
        urllib.request.urlretrieve(pathList[2],filename=dataFolderXml+"/"+str(pathList[0])+"/"+str(pathList[1])+".gz")
    else:
        print("Not an adress"+str(pathList[2]))

    return str(pathList[2])

def extractDokUrl(lst):
    output = list(filter(lambda x: x['dokumentMimeType'] == "application/xml", lst))
    #print(len(output))
    try:
        return output[0]["dokumentUrl"]
    except IndexError:
        #print("No xml file")
        return None

def parseToXmlData(jData):
    '''
        takes a json file and extracts the attachments
    '''
    pool = Pool(processes=4)
    dokData = jData["hits"]["hits"]

    #Extract offentliggørelsestidspunkt for the current day
    lf = re.match("\d{4}-\d{2}-\d{2}",dokData[0]["_source"]["offentliggoerelsesTidspunkt"])
    print(lf.group(0))
    dokList = [(lf.group(0),str(v["_source"]["regnskab"]["regnskabsperiode"]["startDato"])+str(v["_source"]["cvrNummer"])
                     ,extractDokUrl(v["_source"]["dokumenter"])) for v in dokData]

    if os.path.isdir(dataFolderXml+"/"+lf.group(0)) == False:
        os.mkdir(dataFolderXml+"/"+str(lf.group(0)))
    exsistingFiles = os.listdir(dataFolderXml+"/"+str(lf.group(0)))
    updatedDoklist = list(filter(lambda x: x[1]+".gz" not in exsistingFiles,dokList))
    #print(updatedDoklist)

    print("number in list: ",len(updatedDoklist))
    #xmlDok = [openAndCollectXML(i)  for i in dokList]

    xmlDok = pool.map(openAndCollectXML,updatedDoklist)
    print("number of xml-Documents collected is: ", len(xmlDok))
    #for i in range(0,len(dokData)):
    #    #print dokData[i]["_source"]["cvrNummer"]
    #    dokList = dokData[i]["_source"]["dokumenter"]#[0]["dokumentUrl"]
    #    for d in dokList:
    #        if d["dokumentMimeType"] == "application/xml":
    #            x = http.urlopen("GET", d["dokumentUrl"])
    #            text_file = open(dataFolderZip+"/"+str(dokData[i]["_source"]["regnskab"]["regnskabsperiode"]["startDato"])+str(dokData[i]["_source"]["cvrNummer"])+".gz", "w+")
    #            text_file.write(str(x.data))
    #            text_file.close()

                #with gzip.open(dataFolderZip+"/"+str(dokData[i]["_source"]["regnskab"]["regnskabsperiode"]["startDato"])+str(dokData[i]["_source"]["cvrNummer"])+".gz", "rb") as f:
                #    file_content = f.read()
                #text_file = open(dataFolderXml+"/"+str(dokData[i]["_source"]["regnskab"]["regnskabsperiode"]["startDato"])+"cvr"+str(dokData[i]["_source"]["cvrNummer"])+".xml", "w+")
                #text_file.write(file_content)
                #text_file.close()
    #            xmlDok.append(d["dokumentUrl"])

    #for i in dokList:
    #    print(i)

if __name__ == '__main__':
    #the main method
    #usage python3 DownloadData.py startdate enddate pathToSaveFileRootDirectory

    if len(sys.argv) == 2:
        start_date = datetime.strptime(sys.argv[1],"%Y-%m-%d")
    elif len(sys.argv) == 3:
        start_date = datetime.strptime(sys.argv[1],"%Y-%m-%d")
        end_date = datetime.strptime(sys.argv[2],"%Y-%m-%d")
    elif len(sys.argv) == 4:
        start_date = datetime.strptime(sys.argv[1],"%Y-%m-%d")
        end_date = datetime.strptime(sys.argv[2],"%Y-%m-%d")
        global dataFolderXml
        dataFolderXml = sys.argv[3]
    else:
        start_date = date(2016,4,1)
        end_date = date(2016,4,2)

    d = start_date
    delta = timedelta(days=1)
    while d < end_date:
        sd = d.strftime("%Y-%m-%d")
        d += delta
        ed = d.strftime("%Y-%m-%d")
        jData = getQueryData(sd,ed,1000000)
        response = requests.get(index, data=json.dumps(jData),
                                     headers=HEADERS_FOR_JSON)
        response_data = response.json()
        parseToXmlData(response_data)
        print(sd)

<|endoftext|>


<commit_before>from datetime import timedelta, datetime

from sqlalchemy import func
from sqlalchemy.exc import IntegrityError

from ichnaea.content.models import (
    Stat,
    STAT_TYPE,
)
from ichnaea.models import (
    Cell,
    CellMeasure,
    Wifi,
    WifiMeasure,
)
from ichnaea.tasks import (
    DatabaseTask,
    daily_task_days,
)
from ichnaea.worker import celery

def histogram_query(session, model, min_day, max_day):
    query = session.query(
        func.count(model.id)).filter(
        model.created < max_day).filter(
        model.created >= min_day)
    return query.first()[0]

def add_stat(session, name, day, value):
    stat_key = STAT_TYPE[name]
    query = session.query(Stat.value).filter(
        Stat.key == stat_key).filter(
        Stat.time == day - timedelta(days=1))
    result = query.first()
    before = 0
    if result is not None:
        before = int(result[0])

    # on duplicate key, do a no-op change
    stmt = Stat.__table__.insert(
        on_duplicate='time=time').values(
        key=stat_key, time=day, value=before + int(value))
    session.execute(stmt)

def incr_stat(session, name, incr, date=datetime.utcnow().date()):
    stat_key = STAT_TYPE[name]
    cumulative = get_curr_stat(session, name) + incr

    # on duplicate key, update existing
    stmt = Stat.__table__.insert(
        on_duplicate='value=%s' % cumulative).values(
            key=stat_key, time=date, value=cumulative)
    session.execute(stmt)

def get_curr_stat(session, name, date=datetime.utcnow().date()):
    stat_key = STAT_TYPE[name]
    query = session.query(Stat.value).filter(
        Stat.key == stat_key).filter(
        Stat.time <= date).order_by(
        Stat.time.desc())
    result = query.first()
    if result is not None:
        return int(result[0])
    else:
        return 0

@celery.task(base=DatabaseTask, bind=True)
def cell_histogram(self, ago=1):
    day, max_day = daily_task_days(ago)
    try:
        with self.db_session() as session:
            value = histogram_query(session, CellMeasure, day, max_day)
            add_stat(session, 'cell', day, value)
            session.commit()
            return 1
    except IntegrityError as exc:
        self.heka_client.raven('error')
        return 0
    except Exception as exc:  # pragma: no cover
        raise self.retry(exc=exc)

@celery.task(base=DatabaseTask, bind=True)
def wifi_histogram(self, ago=1):
    day, max_day = daily_task_days(ago)
    try:
        with self.db_session() as session:
            value = histogram_query(session, WifiMeasure, day, max_day)
            add_stat(session, 'wifi', day, value)
            session.commit()
            return 1
    except IntegrityError as exc:
        self.heka_client.raven('error')
        return 0
    except Exception as exc:  # pragma: no cover
        raise self.retry(exc=exc)

@celery.task(base=DatabaseTask, bind=True)
def unique_cell_histogram(self, ago=1):
    day, max_day = daily_task_days(ago)
    try:
        with self.db_session() as session:
            value = histogram_query(session, Cell, day, max_day)
            add_stat(session, 'unique_cell', day, value)
            session.commit()
            return 1
    except IntegrityError as exc:
        self.heka_client.raven('error')
        return 0
    except Exception as exc:  # pragma: no cover
        raise self.retry(exc=exc)

@celery.task(base=DatabaseTask, bind=True)
def unique_wifi_histogram(self, ago=1):
    day, max_day = daily_task_days(ago)
    try:
        with self.db_session() as session:
            value = histogram_query(session, Wifi, day, max_day)
            add_stat(session, 'unique_wifi', day, value)
            session.commit()
            return 1
    except IntegrityError as exc:
        self.heka_client.raven('error')
        return 0
    except Exception as exc:  # pragma: no cover
        raise self.retry(exc=exc)
<commit_msg>Avoid persistent date objects in default arguments<commit_after>from datetime import timedelta, datetime

from sqlalchemy import func
from sqlalchemy.exc import IntegrityError

from ichnaea.content.models import (
    Stat,
    STAT_TYPE,
)
from ichnaea.models import (
    Cell,
    CellMeasure,
    Wifi,
    WifiMeasure,
)
from ichnaea.tasks import (
    DatabaseTask,
    daily_task_days,
)
from ichnaea.worker import celery

def histogram_query(session, model, min_day, max_day):
    query = session.query(
        func.count(model.id)).filter(
        model.created < max_day).filter(
        model.created >= min_day)
    return query.first()[0]

def add_stat(session, name, day, value):
    stat_key = STAT_TYPE[name]
    query = session.query(Stat.value).filter(
        Stat.key == stat_key).filter(
        Stat.time == day - timedelta(days=1))
    result = query.first()
    before = 0
    if result is not None:
        before = int(result[0])

    # on duplicate key, do a no-op change
    stmt = Stat.__table__.insert(
        on_duplicate='time=time').values(
        key=stat_key, time=day, value=before + int(value))
    session.execute(stmt)

def incr_stat(session, name, incr, date=None):
    if date is None:
        date = datetime.utcnow().date()
    stat_key = STAT_TYPE[name]
    cumulative = get_curr_stat(session, name) + incr

    # on duplicate key, update existing
    stmt = Stat.__table__.insert(
        on_duplicate='value=%s' % cumulative).values(
            key=stat_key, time=date, value=cumulative)
    session.execute(stmt)

def get_curr_stat(session, name, date=None):
    if date is None:
        date = datetime.utcnow().date()
    stat_key = STAT_TYPE[name]
    query = session.query(Stat.value).filter(
        Stat.key == stat_key).filter(
        Stat.time <= date).order_by(
        Stat.time.desc())
    result = query.first()
    if result is not None:
        return int(result[0])
    else:
        return 0

@celery.task(base=DatabaseTask, bind=True)
def cell_histogram(self, ago=1):
    day, max_day = daily_task_days(ago)
    try:
        with self.db_session() as session:
            value = histogram_query(session, CellMeasure, day, max_day)
            add_stat(session, 'cell', day, value)
            session.commit()
            return 1
    except IntegrityError as exc:
        self.heka_client.raven('error')
        return 0
    except Exception as exc:  # pragma: no cover
        raise self.retry(exc=exc)

@celery.task(base=DatabaseTask, bind=True)
def wifi_histogram(self, ago=1):
    day, max_day = daily_task_days(ago)
    try:
        with self.db_session() as session:
            value = histogram_query(session, WifiMeasure, day, max_day)
            add_stat(session, 'wifi', day, value)
            session.commit()
            return 1
    except IntegrityError as exc:
        self.heka_client.raven('error')
        return 0
    except Exception as exc:  # pragma: no cover
        raise self.retry(exc=exc)

@celery.task(base=DatabaseTask, bind=True)
def unique_cell_histogram(self, ago=1):
    day, max_day = daily_task_days(ago)
    try:
        with self.db_session() as session:
            value = histogram_query(session, Cell, day, max_day)
            add_stat(session, 'unique_cell', day, value)
            session.commit()
            return 1
    except IntegrityError as exc:
        self.heka_client.raven('error')
        return 0
    except Exception as exc:  # pragma: no cover
        raise self.retry(exc=exc)

@celery.task(base=DatabaseTask, bind=True)
def unique_wifi_histogram(self, ago=1):
    day, max_day = daily_task_days(ago)
    try:
        with self.db_session() as session:
            value = histogram_query(session, Wifi, day, max_day)
            add_stat(session, 'unique_wifi', day, value)
            session.commit()
            return 1
    except IntegrityError as exc:
        self.heka_client.raven('error')
        return 0
    except Exception as exc:  # pragma: no cover
        raise self.retry(exc=exc)
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

import logging
import os
from abc import ABCMeta, abstractmethod
from argparse import ArgumentParser, Namespace
from typing import AsyncContextManager, Tuple

from idb.common import plugin
from idb.common.command import Command
from idb.common.constants import DEFAULT_DAEMON_GRPC_PORT, DEFAULT_DAEMON_HOST
from idb.common.types import IdbClient, IdbManagementClient
from idb.grpc.client import IdbClient as IdbClientGrpc
from idb.grpc.logging import log_call
from idb.grpc.management import IdbManagementClient as IdbManagementClientGrpc
from idb.utils.contextlib import asynccontextmanager

def _parse_companion_info(value: str) -> Tuple[str, int]:
    (host, port) = value.rsplit(":", 1)
    return (host, int(port))

@asynccontextmanager
async def _get_client(
    args: Namespace, logger: logging.Logger
) -> AsyncContextManager[IdbClient]:
    companion = vars(args).get("companion")
    if companion is not None:
        (host, port) = _parse_companion_info(companion)
        async with IdbClientGrpc.build(
            host=host, port=port, is_local=False, logger=logger
        ) as client:
            yield client
    else:
        async with IdbManagementClientGrpc(
            logger=logger, companion_path=args.companion_path
        ).from_udid(udid=vars(args).get("udid")) as client:
            yield client

class BaseCommand(Command, metaclass=ABCMeta):
    def __init__(self) -> None:
        super().__init__()
        # Will inherit log levels when the log level is set on the base logger in run()
        self.logger: logging.Logger = logging.getLogger(self.name)

    def add_parser_arguments(self, parser: ArgumentParser) -> None:
        parser.add_argument(
            "--log",
            dest="log_level_deprecated",
            choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
            default=None,
            help="Set the logging level. Deprecated: Please place --log before the command name",
        )
        parser.add_argument(
            "--json",
            action="store_true",
            default=False,
            help="Create json structured output",
        )

    async def run(self, args: Namespace) -> None:
        # In order to keep the argparse compatible with old invocations
        # We should use the --log after command if set, otherwise use the pre-command --log
        logging.getLogger().setLevel(args.log_level_deprecated or args.log_level)
        name = self.__class__.__name__
        self.logger.debug(f"{name} command run with: {args}")
        if args.log_level_deprecated is not None:
            self.logger.warning(
                f"Setting --log after the command is deprecated, please place it at the start of the invocation"
            )
        async with log_call(
            name=name, metadata=plugin.resolve_metadata(logger=self.logger)
        ):
            await self._run_impl(args)

    @abstractmethod
    async def _run_impl(self, args: Namespace) -> None:
        raise Exception("subclass")

# A command that vends the IdbClientBase interface.
class CompanionCommand(BaseCommand):
    def add_parser_arguments(self, parser: ArgumentParser) -> None:
        parser.add_argument(
            "--udid",
            help="Udid of target, can also be set with the IDB_UDID env var",
            default=os.environ.get("IDB_UDID"),
        )
        super().add_parser_arguments(parser)

    async def _run_impl(self, args: Namespace) -> None:
        async with _get_client(args=args, logger=self.logger) as client:
            await self.run_with_client(args=args, client=client)

    @abstractmethod
    async def run_with_client(self, args: Namespace, client: IdbClient) -> None:
        pass

# A command that vends the IdbClient interface
class ManagementCommand(BaseCommand):
    async def _run_impl(self, args: Namespace) -> None:
        await self.run_with_client(
            args=args,
            client=IdbManagementClientGrpc(
                companion_path=args.companion_path,
                logger=self.logger,
                prune_dead_companion=args.prune_dead_companion,
            ),
        )

    @abstractmethod
    async def run_with_client(
        self, args: Namespace, client: IdbManagementClient
    ) -> None:
        pass
<commit_msg>Add Companion pruning on non-management command<commit_after>#!/usr/bin/env python3
# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

import logging
import os
from abc import ABCMeta, abstractmethod
from argparse import ArgumentParser, Namespace
from typing import AsyncContextManager, Optional, Tuple

from idb.common import plugin
from idb.common.command import Command
from idb.common.constants import DEFAULT_DAEMON_GRPC_PORT, DEFAULT_DAEMON_HOST
from idb.common.types import (
    Address,
    IdbClient,
    IdbConnectionException,
    IdbManagementClient,
)
from idb.grpc.client import IdbClient as IdbClientGrpc
from idb.grpc.logging import log_call
from idb.grpc.management import IdbManagementClient as IdbManagementClientGrpc
from idb.utils.contextlib import asynccontextmanager

def _parse_companion_info(value: str) -> Tuple[str, int]:
    (host, port) = value.rsplit(":", 1)
    return (host, int(port))

def _get_management_client(
    logger: logging.Logger, args: Namespace
) -> IdbManagementClient:
    return IdbManagementClientGrpc(
        companion_path=args.companion_path,
        logger=logger,
        prune_dead_companion=args.prune_dead_companion,
    )

@asynccontextmanager
async def _get_client(
    args: Namespace, logger: logging.Logger
) -> AsyncContextManager[IdbClientGrpc]:
    companion = vars(args).get("companion")
    if companion is not None:
        (host, port) = _parse_companion_info(companion)
        async with IdbClientGrpc.build(
            host=host, port=port, is_local=False, logger=logger
        ) as client:
            yield client
    else:
        async with IdbManagementClientGrpc(
            logger=logger, companion_path=args.companion_path
        ).from_udid(udid=vars(args).get("udid")) as client:
            yield client

class BaseCommand(Command, metaclass=ABCMeta):
    def __init__(self) -> None:
        super().__init__()
        # Will inherit log levels when the log level is set on the base logger in run()
        self.logger: logging.Logger = logging.getLogger(self.name)

    def add_parser_arguments(self, parser: ArgumentParser) -> None:
        parser.add_argument(
            "--log",
            dest="log_level_deprecated",
            choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
            default=None,
            help="Set the logging level. Deprecated: Please place --log before the command name",
        )
        parser.add_argument(
            "--json",
            action="store_true",
            default=False,
            help="Create json structured output",
        )

    async def run(self, args: Namespace) -> None:
        # In order to keep the argparse compatible with old invocations
        # We should use the --log after command if set, otherwise use the pre-command --log
        logging.getLogger().setLevel(args.log_level_deprecated or args.log_level)
        name = self.__class__.__name__
        self.logger.debug(f"{name} command run with: {args}")
        if args.log_level_deprecated is not None:
            self.logger.warning(
                f"Setting --log after the command is deprecated, please place it at the start of the invocation"
            )
        async with log_call(
            name=name, metadata=plugin.resolve_metadata(logger=self.logger)
        ):
            await self._run_impl(args)

    @abstractmethod
    async def _run_impl(self, args: Namespace) -> None:
        raise Exception("subclass")

# A command that vends the IdbClientBase interface.
class CompanionCommand(BaseCommand):
    def add_parser_arguments(self, parser: ArgumentParser) -> None:
        parser.add_argument(
            "--udid",
            help="Udid of target, can also be set with the IDB_UDID env var",
            default=os.environ.get("IDB_UDID"),
        )
        super().add_parser_arguments(parser)

    async def _run_impl(self, args: Namespace) -> None:
        address: Optional[Address] = None
        try:
            async with _get_client(args=args, logger=self.logger) as client:
                address = client.address
                await self.run_with_client(args=args, client=client)
        except IdbConnectionException as ex:
            if not args.prune_dead_companion:
                raise ex
            if address is None:
                raise ex
            try:
                await _get_management_client(logger=self.logger, args=args).disconnect(
                    destination=address
                )
            finally:
                raise ex

    @abstractmethod
    async def run_with_client(self, args: Namespace, client: IdbClient) -> None:
        pass

# A command that vends the IdbClient interface
class ManagementCommand(BaseCommand):
    async def _run_impl(self, args: Namespace) -> None:
        await self.run_with_client(
            args=args, client=_get_management_client(logger=self.logger, args=args)
        )

    @abstractmethod
    async def run_with_client(
        self, args: Namespace, client: IdbManagementClient
    ) -> None:
        pass
<|endoftext|>


<commit_before>"""
Utilities for the SciTokens library
"""
import cryptography.utils
import base64

def long_from_bytes(data):
    """
    Return an integer from base64-encoded string.

    :param data: UTF-8 string containing base64-encoded data.
    :returns: Corresponding decoded integer.
    """
    return cryptography.utils.int_from_bytes(decode_base64(data.encode("ascii")), 'big')

def decode_base64(data):
    """Decode base64, padding being optional.

    :param data: Base64 data as an ASCII byte string
    :returns: The decoded byte string.

    """
    missing_padding = len(data) % 4
    if missing_padding != 0:
        data += b'='* (4 - missing_padding)

    return base64.urlsafe_b64decode(data)

def string_from_long(data):
    """
    Create a base64 encoded string for an integer
    """
    return base64.urlsafe_b64encode(cryptography.utils.int_to_bytes(data)).decode('ascii')

def bytes_from_long(data):
    """
    Create a base64 encoded bytes for an integer
    """
    return base64.urlsafe_b64encode(cryptography.utils.int_to_bytes(data))
<commit_msg>scitokens.utils: fix deprecation warning<commit_after>"""
Utilities for the SciTokens library
"""
import cryptography.utils
import base64

def long_from_bytes(data):
    """
    Return an integer from base64-encoded string.

    :param data: UTF-8 string containing base64-encoded data.
    :returns: Corresponding decoded integer.
    """
    return int.from_bytes(decode_base64(data.encode("ascii")), 'big')

def decode_base64(data):
    """Decode base64, padding being optional.

    :param data: Base64 data as an ASCII byte string
    :returns: The decoded byte string.

    """
    missing_padding = len(data) % 4
    if missing_padding != 0:
        data += b'='* (4 - missing_padding)

    return base64.urlsafe_b64decode(data)

def string_from_long(data):
    """
    Create a base64 encoded string for an integer
    """
    return base64.urlsafe_b64encode(cryptography.utils.int_to_bytes(data)).decode('ascii')

def bytes_from_long(data):
    """
    Create a base64 encoded bytes for an integer
    """
    return base64.urlsafe_b64encode(cryptography.utils.int_to_bytes(data))
<|endoftext|>


<commit_before>#!/usr/bin/env python3

"""
Created on 17 Apr 2017

@author: Bruno Beloff (bruno.beloff@southcoastscience.com)

DESCRIPTION
The function of the control_receiver utility is execute commands received over a messaging topic. In addition to
enabling secure remote management, the utility provides a secure challenge-response facility.

A typical South Coast Science device is provided with a messaging topic whose purpose is to enable bidirectional
command-and-response communications between the device and one or more remote management systems. Commands
are in the form of a specific JSON format, which is validated as follows:

* the message must identify the device as the recipient
* the digest in the incoming message matches the digest computed by the device
* the command must be listed in the device's ~/SCS/cmd/ directory, or be "?"

The digest is computed using a shared secret generated by the scs_mfr/shared_secret utility.

If validated, the control_receiver utility executes the command, then publishes a receipt message which includes:

* the command stdout
* the command stderr
* the command return code
* the original message digest
* a new message digest

Entries in ~/SCS/cmd/ are typically symbolic links to commands that are implemented elsewhere, either by the operating
system, or by South Coast Science packages.

It is the responsibility of the device administrator to mange the ~/SCS/cmd/ directory. Care should be taken to exclude
commands that:

* could cause harm to the system
* require an interactive mode
* require root privileges
* can change the contents of the ~/SCS/cmd/ directory

SYNOPSIS
control_receiver.py [-r] [-e] [-v]

EXAMPLES
/home/pi/SCS/scs_dev/src/scs_dev/aws_topic_subscriber.py -cX -s /home/pi/SCS/pipes/mqtt_control_subscription.uds | \
/home/pi/SCS/scs_dev/src/scs_dev/control_receiver.py -r -v | \
/home/pi/SCS/scs_dev/src/scs_dev/aws_topic_publisher.py -v -cX -p /home/pi/SCS/pipes/mqtt_publication.uds
FILES
~/SCS/cmd/*
~/SCS/conf/system_id.json
~/SCS/conf/shared_secret.json

DOCUMENT EXAMPLE - REQUEST
{"/orgs/south-coast-science-dev/development/device/alpha-pi-eng-000006/control":
{"tag": "bruno", "attn": "scs-ap1-6", "rec": "2018-04-04T14:41:11.872+01:00", "cmd_tokens": ["?"],
"digest": "bf682976cb45d889207bf3e3b4a6e12336859a93d7023b8454514"}}

DOCUMENT EXAMPLE - RESPONSE
{"/orgs/south-coast-science-dev/development/device/alpha-pi-eng-000006/control":
{"tag": "scs-ap1-6", "rec": "2018-04-04T13:41:59.521+00:00",
"cmd": {"cmd": "?", "params": [],
"stdout": ["[\"afe_baseline\", \"afe_calib\", \"opc_power\", \"ps\", \"schedule\", \"shared_secret\"]"],
"stderr": [], "ret": 0},
"omd": "40ef7a9c0f70033bbe21827ed25286b448a5ad3ace9b16f44f3d94da6a89ab25",
"digest": "597f8de3852f1067f52f126398777204c378e8f5d30bad6d8d99ee536"}}

SEE ALSO
scs_analysis/aws_mqtt_control
scs_analysis/osio_mqtt_control
scs_mfr/shared_secret
"""

import json
import sys
import time

from scs_core.control.command import Command
from scs_core.control.control_datum import ControlDatum
from scs_core.control.control_receipt import ControlReceipt

from scs_core.data.datetime import LocalizedDatetime
from scs_core.data.json import JSONify

from scs_core.sys.shared_secret import SharedSecret
from scs_core.sys.signalled_exit import SignalledExit
from scs_core.sys.system_id import SystemID

from scs_dev.cmd.cmd_control_receiver import CmdControlReceiver

from scs_host.sys.host import Host

# --------------------------------------------------------------------------------------------------------------------

if __name__ == '__main__':

    # ----------------------------------------------------------------------------------------------------------------
    # config...

    deferred_commands = ('shutdown', 'reboot', 'restart')

    # ----------------------------------------------------------------------------------------------------------------
    # cmd...

    cmd = CmdControlReceiver()

    if cmd.verbose:
        print("control_receiver: %s" % cmd, file=sys.stderr)

    # ------------------------------------------------------------------------------------------------------------
    # resources...

    # SystemID...
    system_id = SystemID.load(Host)

    if system_id is None:
        print("control_receiver: SystemID not available.", file=sys.stderr)
        exit(1)

    if cmd.verbose:
        print("control_receiver: %s" % system_id, file=sys.stderr)

    # SharedSecret...
    secret = SharedSecret.load(Host)

    if secret is None:
        print("control_receiver: SharedSecret not available.", file=sys.stderr)
        exit(1)

    if cmd.verbose:
        print("control_receiver: %s" % secret, file=sys.stderr)
        sys.stderr.flush()

    system_tag = system_id.message_tag()
    key = secret.key

    try:
        # ------------------------------------------------------------------------------------------------------------
        # run...

        # signal handler...
        SignalledExit.construct("control_receiver", cmd.verbose)

        for line in sys.stdin:
            # control...
            try:
                jdict = json.loads(line)
            except ValueError:
                continue

            try:
                datum = ControlDatum.construct_from_jdict(jdict)
            except TypeError:
                continue

            if datum.attn != system_tag:
                continue

            if cmd.verbose:
                print("control_receiver: %s" % datum, file=sys.stderr)
                sys.stderr.flush()

            if not datum.is_valid(key):
                print("control_receiver: invalid digest: %s" % datum, file=sys.stderr)
                sys.stderr.flush()
                continue

            if cmd.echo:
                print(JSONify.dumps(datum))
                sys.stdout.flush()

            # command...
            command = Command.construct_from_tokens(datum.cmd_tokens)

            if command.cmd is not None and not command.is_valid(Host):
                command.error("invalid command")

            # execute immediate commands...
            elif command.cmd not in deferred_commands:
                command.execute(Host)

            # receipt...
            if cmd.receipt:
                now = LocalizedDatetime.now().utc()
                receipt = ControlReceipt.construct_from_datum(datum, now, command, key)

                print(JSONify.dumps(receipt))
                sys.stdout.flush()

                if cmd.verbose:
                    print(JSONify.dumps(receipt), file=sys.stderr)
                    sys.stderr.flush()

            # execute deferred commands...
            if command.cmd in deferred_commands:
                time.sleep(10.0)                            # wait, hoping that the receipt is sent
                command.execute(Host)

    # ----------------------------------------------------------------------------------------------------------------
    # end...

    except ConnectionError as ex:
        print("control_receiver: %s" % ex, file=sys.stderr)

    except (KeyboardInterrupt, SystemExit):
        pass

    finally:
        if cmd and cmd.verbose:
            print("control_receiver: finishing", file=sys.stderr)
<commit_msg>Added timeout field to ControlDatum and Command classes<commit_after>#!/usr/bin/env python3

"""
Created on 17 Apr 2017

@author: Bruno Beloff (bruno.beloff@southcoastscience.com)

DESCRIPTION
The function of the control_receiver utility is execute commands received over a messaging topic. In addition to
enabling secure remote management, the utility provides a secure challenge-response facility.

A typical South Coast Science device is provided with a messaging topic whose purpose is to enable bidirectional
command-and-response communications between the device and one or more remote management systems. Commands
are in the form of a specific JSON format, which is validated as follows:

* the message must identify the device as the recipient
* the digest in the incoming message matches the digest computed by the device
* the command must be listed in the device's ~/SCS/cmd/ directory, or be "?"

The digest is computed using a shared secret generated by the scs_mfr/shared_secret utility.

If validated, the control_receiver utility executes the command, then publishes a receipt message which includes:

* the command stdout
* the command stderr
* the command return code
* the original message digest
* a new message digest

Entries in ~/SCS/cmd/ are typically symbolic links to commands that are implemented elsewhere, either by the operating
system, or by South Coast Science packages.

It is the responsibility of the device administrator to mange the ~/SCS/cmd/ directory. Care should be taken to exclude
commands that:

* could cause harm to the system
* require an interactive mode
* require root privileges
* can change the contents of the ~/SCS/cmd/ directory

SYNOPSIS
control_receiver.py [-r] [-e] [-v]

EXAMPLES
/home/pi/SCS/scs_dev/src/scs_dev/aws_topic_subscriber.py -cX -s /home/pi/SCS/pipes/mqtt_control_subscription.uds | \
/home/pi/SCS/scs_dev/src/scs_dev/control_receiver.py -r -v | \
/home/pi/SCS/scs_dev/src/scs_dev/aws_topic_publisher.py -v -cX -p /home/pi/SCS/pipes/mqtt_publication.uds

FILES
~/SCS/cmd/*
~/SCS/conf/system_id.json
~/SCS/conf/shared_secret.json

DOCUMENT EXAMPLE - REQUEST
{"/orgs/south-coast-science-dev/development/device/alpha-pi-eng-000006/control":
{"tag": "bruno", "attn": "scs-ap1-6", "rec": "2018-04-04T14:41:11.872+01:00", "cmd_tokens": ["?"],
"digest": "bf682976cb45d889207bf3e3b4a6e12336859a93d7023b8454514"}}

DOCUMENT EXAMPLE - RESPONSE
{"/orgs/south-coast-science-dev/development/device/alpha-pi-eng-000006/control":
{"tag": "scs-ap1-6", "rec": "2018-04-04T13:41:59.521+00:00",
"cmd": {"cmd": "?", "params": [],
"stdout": ["[\"afe_baseline\", \"afe_calib\", \"opc_power\", \"ps\", \"schedule\", \"shared_secret\"]"],
"stderr": [], "ret": 0},
"omd": "40ef7a9c0f70033bbe21827ed25286b448a5ad3ace9b16f44f3d94da6a89ab25",
"digest": "597f8de3852f1067f52f126398777204c378e8f5d30bad6d8d99ee536"}}

SEE ALSO
scs_analysis/aws_mqtt_control
scs_analysis/osio_mqtt_control
scs_mfr/shared_secret
"""

import json
import sys
import time

from scs_core.control.command import Command
from scs_core.control.control_datum import ControlDatum
from scs_core.control.control_receipt import ControlReceipt

from scs_core.data.datetime import LocalizedDatetime
from scs_core.data.json import JSONify

from scs_core.sys.shared_secret import SharedSecret
from scs_core.sys.signalled_exit import SignalledExit
from scs_core.sys.system_id import SystemID

from scs_dev.cmd.cmd_control_receiver import CmdControlReceiver

from scs_host.sys.host import Host

# --------------------------------------------------------------------------------------------------------------------

if __name__ == '__main__':

    # ----------------------------------------------------------------------------------------------------------------
    # config...

    deferred_commands = ('shutdown', 'reboot', 'restart')

    # ----------------------------------------------------------------------------------------------------------------
    # cmd...

    cmd = CmdControlReceiver()

    if cmd.verbose:
        print("control_receiver: %s" % cmd, file=sys.stderr)

    # ------------------------------------------------------------------------------------------------------------
    # resources...

    # SystemID...
    system_id = SystemID.load(Host)

    if system_id is None:
        print("control_receiver: SystemID not available.", file=sys.stderr)
        exit(1)

    if cmd.verbose:
        print("control_receiver: %s" % system_id, file=sys.stderr)

    # SharedSecret...
    secret = SharedSecret.load(Host)

    if secret is None:
        print("control_receiver: SharedSecret not available.", file=sys.stderr)
        exit(1)

    if cmd.verbose:
        print("control_receiver: %s" % secret, file=sys.stderr)
        sys.stderr.flush()

    system_tag = system_id.message_tag()
    key = secret.key

    try:
        # ------------------------------------------------------------------------------------------------------------
        # run...

        # signal handler...
        SignalledExit.construct("control_receiver", cmd.verbose)

        for line in sys.stdin:
            # control...
            try:
                jdict = json.loads(line)
            except ValueError:
                continue

            try:
                datum = ControlDatum.construct_from_jdict(jdict)
            except TypeError:
                continue

            if datum.attn != system_tag:
                continue

            if cmd.verbose:
                print("control_receiver: %s" % datum, file=sys.stderr)
                sys.stderr.flush()

            if not datum.is_valid(key):
                print("control_receiver: invalid digest: %s" % datum, file=sys.stderr)
                sys.stderr.flush()
                continue

            if cmd.echo:
                print(JSONify.dumps(datum))
                sys.stdout.flush()

            # command...
            command = Command.construct_from_tokens(datum.cmd_tokens, datum.timeout)

            if command.cmd is not None and not command.is_valid(Host):
                command.error("invalid command")

            # execute immediate commands...
            elif command.cmd not in deferred_commands:
                command.execute(Host)

            # receipt...
            if cmd.receipt:
                now = LocalizedDatetime.now().utc()
                receipt = ControlReceipt.construct_from_datum(datum, now, command, key)

                print(JSONify.dumps(receipt))
                sys.stdout.flush()

                if cmd.verbose:
                    print(JSONify.dumps(receipt), file=sys.stderr)
                    sys.stderr.flush()

            # execute deferred commands...
            if command.cmd in deferred_commands:
                time.sleep(10.0)                            # wait, hoping that the receipt is sent
                command.execute(Host)

    # ----------------------------------------------------------------------------------------------------------------
    # end...

    except ConnectionError as ex:
        print("control_receiver: %s" % ex, file=sys.stderr)

    except (KeyboardInterrupt, SystemExit):
        pass

    finally:
        if cmd and cmd.verbose:
            print("control_receiver: finishing", file=sys.stderr)
<|endoftext|>


<commit_before>VERSION = (0, 0, 3)
__version__ = '.'.join(map(str, VERSION))
<commit_msg>0.0.4<commit_after>VERSION = (0, 0, 4)
__version__ = '.'.join(map(str, VERSION))
<|endoftext|>


<commit_before>#!/usr/bin/env python

__author__ = 'Maurice Manning'
__license__ = 'Apache 2.0'

from interface.services.sa.idata_process_management_service  import BaseDataProcessManagementService

class DataProcessManagementService(BaseDataProcessManagementService):

    def create_data_process(self, data_process={}):
        """
        method docstring
        """
        # Return Value
        # ------------
        # {data_process_id: ''}
        #
        pass

    def update_data_process(self, data_process={}):
        """
        method docstring
        """
        # Return Value
        # ------------
        # {success: true}
        #
        pass

    def read_data_process(self, data_process_id=''):
        """
        method docstring
        """
        # Return Value
        # ------------
        # data_process: {}
        #
        pass

    def delete_data_process(self, data_process_id=''):
        """
        method docstring
        """
        # Return Value
        # ------------
        # {success: true}
        #
        pass

    def find_data_process(self, filters={}):
        """
        method docstring
        """
        # Return Value
        # ------------
        # data_process_list: []
        #
        pass

    def attach_process(self, process=''):
        """
        method docstring
        """
        # Return Value
        # ------------
        # {success: true}
        #
        pass
<commit_msg>change data process create command to use subscriptions<commit_after>#!/usr/bin/env python

__author__ = 'Maurice Manning'
__license__ = 'Apache 2.0'

from interface.services.sa.idata_process_management_service  import BaseDataProcessManagementService

class DataProcessManagementService(BaseDataProcessManagementService):

    def create_data_process(self, data_process={}, in_subscription_id='', out_data_product_id=''):
        """method docstring
        """
        # Return Value
        # ------------
        # {data_process_id: ''}
        #
        pass

    def update_data_process(self, data_process={}):
        """method docstring
        """
        # Return Value
        # ------------
        # {success: true}
        #
        pass

    def read_data_process(self, data_process_id=''):
        """method docstring
        """
        # Return Value
        # ------------
        # data_process: {}
        #
        pass

    def delete_data_process(self, data_process_id=''):
        """method docstring
        """
        # Return Value
        # ------------
        # {success: true}
        #
        pass

    def find_data_process(self, filters={}):
        """method docstring
        """
        # Return Value
        # ------------
        # data_process_list: []
        #
        pass

    def attach_process(self, process=''):
        """method docstring
        """
        # Return Value
        # ------------
        # {success: true}
        #
        pass<|endoftext|>


<commit_before>import datetime

from django.utils.translation import ugettext_lazy as _

from cms.plugin_base import CMSPluginBase
from cms.plugin_pool import plugin_pool
from .models import GalleryPlugin

class ImagesiftPlugin(CMSPluginBase):
    model = GalleryPlugin
    name = _('Imagesift Plugin')
    render_template = "imagesift_plugin.html"

    def date_digest(self, images):
        """
        return a list of unique dates, for all the images passed
        """
        dates = {}
        for i in images:
            dates.setdefault(i.overrideable_date().date(), None)
        return sorted(dates.keys())

    def render(self, context, instance, placeholder):
        url = context['request'].get_full_path()
        date = context['request'].GET.get('date')
        limit = instance.thumbnail_limit
        qs = instance.get_images_queryset()
        if limit:
            qs = qs[:limit]
        filtered = False
        if date:
            date = datetime.datetime.strptime(date, "%Y-%m-%d").date()
            qs = list(qs)
            qs = [i for i in qs if i.overrideable_date().date() == date]
            filtered = _('The set of images is filtered to %s' % unicode(date))

        context.update({
            'dates': [d.isoformat() for d in self.date_digest(qs)],
            'filtered':filtered,
            'images': qs,
            'instance': instance,
            'placeholder': placeholder,
            'url':url,
        })
        return context

plugin_pool.register_plugin(ImagesiftPlugin)<commit_msg>Sort returned images by date, taking into account overrides<commit_after>import datetime

from django.utils.translation import ugettext_lazy as _

from cms.plugin_base import CMSPluginBase
from cms.plugin_pool import plugin_pool
from .models import GalleryPlugin

class ImagesiftPlugin(CMSPluginBase):
    model = GalleryPlugin
    name = _('Imagesift Plugin')
    render_template = "imagesift_plugin.html"

    def date_digest(self, images):
        """
        return a list of unique dates, for all the images passed
        """
        dates = {}
        for i in images:
            dates.setdefault(i.overrideable_date().date(), None)
        return sorted(dates.keys())

    def render(self, context, instance, placeholder):
        url = context['request'].get_full_path()
        date = context['request'].GET.get('date')
        limit = instance.thumbnail_limit
        qs = instance.get_images_queryset()
        # there's no way around listing, sorry.
        qs = list(qs)

        filtered = False
        if date:
            date = datetime.datetime.strptime(date, "%Y-%m-%d").date()
            qs = [i for i in qs if i.overrideable_date().date() == date]
            filtered = _('The set of images is filtered to %s' % unicode(date))

        # sort before limit
        qs.sort(key=lambda i: i.overrideable_date())

        if limit:
            qs = qs[:limit]

        context.update({
            'dates': [d.isoformat() for d in self.date_digest(qs)],
            'filtered':filtered,
            'images': qs,
            'instance': instance,
            'placeholder': placeholder,
            'url':url,
        })
        return context

plugin_pool.register_plugin(ImagesiftPlugin)<|endoftext|>