<commit_before>import wx
from wx.glcanvas import WX_GL_RGBA, WX_GL_DOUBLEBUFFER, WX_GL_DEPTH_SIZE, WX_GL_CORE_PROFILE

from vistas.core.graphics.camera import Camera
from vistas.core.paths import get_resource_bitmap
from vistas.core.plugins.visualization import VisualizationPlugin3D
from vistas.ui.controllers.project import ProjectChangedEvent
from vistas.ui.controls.gl_canvas import GLCanvas
from vistas.ui.project import Project
from vistas.core.utils import get_platform

class ViewerPanel(wx.Panel):
    NORTH = 'north'
    EAST = 'east'
    SOUTH = 'south'
    WEST = 'west'

    def __init__(self, parent, id):
        super().__init__(parent, id)

        self.north = []
        self.east = None
        self.south = []
        self.west = None
        self.width = 1.
        self.height = 1.

        self.resizing_north = False
        self.resizing_east = False
        self.resizing_south = False
        self.resizing_west = False
        self.last_pos = None

        self.scenes = []
        self.selected_scene = None

        self.parent = parent
        self.north_resize_area = wx.Window(self, wx.ID_ANY, wx.DefaultPosition, wx.Size(-1, 2))
        self.east_resize_area = wx.Window(self, wx.ID_ANY, wx.DefaultPosition, wx.Size(2, -1))
        self.south_resize_area = wx.Window(self, wx.ID_ANY, wx.DefaultPosition, wx.Size(-1, 2))
        self.west_resize_area = wx.Window(self, wx.ID_ANY, wx.DefaultPosition, wx.Size(2, -1))

        self.north_resize_area.SetCursor(wx.Cursor(wx.CURSOR_SIZENS))
        self.east_resize_area.SetCursor(wx.Cursor(wx.CURSOR_SIZEWE))
        self.south_resize_area.SetCursor(wx.Cursor(wx.CURSOR_SIZENS))
        self.west_resize_area.SetCursor(wx.Cursor(wx.CURSOR_SIZEWE))

        self.scene_choice = wx.Choice(self, wx.ID_ANY)
        self.scene_choice.SetToolTip('Select a scene to view')

        self.legend_button = wx.BitmapButton(self, wx.ID_ANY, get_resource_bitmap('stripes.png'))
        self.geodata_button = wx.BitmapButton(self, wx.ID_ANY, get_resource_bitmap('globe.png'))

        self.legend_button.SetToolTip('Show/hide legend')
        self.geodata_button.SetToolTip('Show place names')

        self.camera = Camera()

        attrib_list = [WX_GL_RGBA, WX_GL_DOUBLEBUFFER, WX_GL_DEPTH_SIZE, 16]
        if get_platform() == 'macos':
            attrib_list += [WX_GL_CORE_PROFILE]

        self.gl_canvas = GLCanvas(
            self, wx.ID_ANY, self.camera, attrib_list=attrib_list
        )
        self.gl_canvas.Refresh()

        sizer = wx.BoxSizer(wx.VERTICAL)
        self.SetSizer(sizer)

        top_sizer = wx.BoxSizer(wx.HORIZONTAL)
        top_sizer.Add(self.scene_choice, 1, wx.RIGHT | wx.ALIGN_CENTER_VERTICAL)
        top_sizer.Add(self.legend_button, 0, wx.LEFT | wx.RIGHT | wx.ALIGN_CENTER_VERTICAL, 3)
        top_sizer.Add(self.geodata_button, 0, wx.LEFT | wx.RIGHT | wx.ALIGN_CENTER_VERTICAL, 3)

        content_sizer = wx.BoxSizer(wx.VERTICAL)
        content_sizer.Add(top_sizer, 0, wx.EXPAND | wx.ALL, 5)
        content_sizer.Add(self.gl_canvas, 1, wx.EXPAND)

        center_sizer = wx.BoxSizer(wx.HORIZONTAL)
        center_sizer.Add(self.west_resize_area, 0, wx.EXPAND)
        center_sizer.Add(content_sizer, 1, wx.EXPAND)
        center_sizer.Add(self.east_resize_area, 0, wx.EXPAND)

        sizer.Add(self.north_resize_area, 0, wx.EXPAND)
        sizer.Add(center_sizer, 1, wx.EXPAND)
        sizer.Add(self.south_resize_area, 0, wx.EXPAND)

        # Event handlers
        self.scene_choice.Bind(wx.EVT_CHOICE, self.OnSceneChoice)
        self.legend_button.Bind(wx.EVT_BUTTON, self.OnLegendLabel)
        self.geodata_button.Bind(wx.EVT_BUTTON, self.OnGeographicLabel)

        self.north_resize_area.Bind(wx.EVT_LEFT_DOWN, self.OnResizeLeftDown)
        self.north_resize_area.Bind(wx.EVT_LEFT_UP, self.OnResizeLeftUp)
        self.north_resize_area.Bind(wx.EVT_MOTION, self.OnResizeMotion)

        self.east_resize_area.Bind(wx.EVT_LEFT_DOWN, self.OnResizeLeftDown)
        self.east_resize_area.Bind(wx.EVT_LEFT_UP, self.OnResizeLeftUp)
        self.east_resize_area.Bind(wx.EVT_MOTION, self.OnResizeMotion)

        self.south_resize_area.Bind(wx.EVT_LEFT_DOWN, self.OnResizeLeftDown)
        self.south_resize_area.Bind(wx.EVT_LEFT_UP, self.OnResizeLeftUp)
        self.south_resize_area.Bind(wx.EVT_MOTION, self.OnResizeMotion)

        self.west_resize_area.Bind(wx.EVT_LEFT_DOWN, self.OnResizeLeftDown)
        self.west_resize_area.Bind(wx.EVT_LEFT_UP, self.OnResizeLeftUp)
        self.west_resize_area.Bind(wx.EVT_MOTION, self.OnResizeMotion)

        self.gl_canvas.Bind(wx.EVT_LEFT_DCLICK, self.OnCanvasDClick)
        self.gl_canvas.Bind(wx.EVT_RIGHT_DOWN, self.OnCanvasRightClick)

        # Todo: VI_EVENT_VIZPLUGIN_HAS_NEW_LEGEND event

        self.gl_canvas.camera.wireframe = False

        self.gl_canvas.SetFocus()

        self.legend_window = object()  # Todo: LegendWindow()

        self.RefreshScenes()

        # Todo: observable

    def __del__(self):
        pass  # Todo

    def SetNeighbor(self, neighbor, direction):
        if direction == self.NORTH:
            self.north_resize_area.Show()
            self.north.append(neighbor)

        elif direction == self.EAST:
            self.east_resize_area.Show()
            self.east = neighbor

        elif direction == self.SOUTH:
            self.south_resize_area.Show()
            self.south.append(neighbor)

        elif direction == self.WEST:
            self.west_resize_area.Show()
            self.west = neighbor

    def GetNeighbor(self, direction):
        if direction == self.NORTH:
            return self.north[0] if self.north else None

        elif direction == self.EAST:
            return self.east

        elif direction == self.SOUTH:
            return self.south[0] if self.south else None

        elif direction == self.WEST:
            return self.west

    def RemoveNeighbor(self, neighbor):
        self.north.remove(neighbor)

        if self.east == neighbor:
            self.east = None

        self.south.remove(neighbor)

        if self.west == neighbor:
            self.west = None

    def ResetNeighbors(self):
        self.north = []
        self.east = None
        self.south = []
        self.west = None

    def HideResizeAreas(self):
        self.north_resize_area.Hide()
        self.east_resize_area.Hide()
        self.south_resize_area.Hide()
        self.west_resize_area.Hide()

    def GetSceneChoice(self):
        return self.scene_choice.GetSelection()

    def SetSceneChoice(self, choice):
        self.scene_choice.SetSelection(choice)
        self.UpdateScene()

    def UpdateScene(self):
        for i, scene in enumerate(self.scenes):
            if i == self.scene_choice.GetSelection():
                # Todo: global interactor

                self.gl_canvas.camera.scene = scene  # Temporary until interactor code is added
                self.gl_canvas.camera_interactor.reset_position()
                self.selected_scene = scene

        self.gl_canvas.Refresh()

    def OnResizeLeftDown(self, event):
        self.resizing_north = self.resizing_east = self.resizing_south = self.resizing_west = False

        if event.GetEventObject() == self.north_resize_area:
            self.resizing_north = True
            self.last_pos = event.y

            self.north_resize_area.CaptureMouse()

        elif event.GetEventObject() == self.east_resize_area:
            self.resizing_east = True
            self.last_pos = event.x

            self.east_resize_area.CaptureMouse()

        elif event.GetEventObject() == self.south_resize_area:
            self.resizing_south = True
            self.last_pos = event.y

            self.south_resize_area.CaptureMouse()

        elif event.GetEventObject() == self.west_resize_area:
            self.resizing_west = True
            self.last_pos = event.x

            self.west_resize_area.CaptureMouse()

    def OnResizeLeftUp(self, event):
        self.resizing_north = self.resizing_east = self.resizing_south = self.resizing_west = False

        if self.north_resize_area.HasCapture():
            self.north_resize_area.ReleaseMouse()
        if self.east_resize_area.HasCapture():
            self.east_resize_area.ReleaseMouse()
        if self.south_resize_area.HasCapture():
            self.south_resize_area.ReleaseMouse()
        if self.west_resize_area.HasCapture():
            self.west_resize_area.ReleaseMouse()

    def OnResizeMotion(self, event):
        if self.resizing_north:
            diff = self.last_pos - event.y
            parent_height = self.GetParent().GetSize().GetHeight()
            prop_diff = diff / parent_height

            self.height += prop_diff

            # Update neighbors to the east
            east = self.east
            while east is not None:
                east.height += prop_diff
                east = east.east

            # Update neighbors to the west
            west = self.west
            while west is not None:
                west.height += prop_diff
                west = west.west

            for viewer in self.north:
                viewer.height -= prop_diff

        elif self.resizing_east:
            diff = event.x - self.last_pos
            parent_width = self.GetParent().GetSize().GetWidth()
            prop_diff = diff / parent_width

            self.width += prop_diff
            self.east.width -= prop_diff

        elif self.resizing_south:
            diff = event.y - self.last_pos
            parent_height = self.GetParent().GetSize().GetHeight()
            prop_diff = diff / parent_height

            self.height += prop_diff

            # Update neighbors to the east
            east = self.east
            while east is not None:
                east.height += prop_diff
                east = east.east

            # Update neighbors to the west
            west = self.west
            while west is not None:
                west.height += prop_diff
                west = west.west

            for viewer in self.south:
                viewer.height -= prop_diff

        elif self.resizing_west:
            diff = self.last_pos - event.x
            parent_width = self.GetParent().GetSize().GetWidth()
            prop_diff = diff / parent_width

            self.width += prop_diff
            self.west.width -= prop_diff

        self.GetParent().UpdateViewerSizes()

    def OnCanvasDClick(self, event):
        pass  # Todo

    def OnCanvasRightClick(self, event):
        pass  # Todo

    def OnCanvasPopupMenu(self, event):
        pass  # Todo

    def RefreshScenes(self):
        visualization_root = Project.get().visualization_root
        self.scenes = []
        self.FetchScenes(visualization_root)

        self.scene_choice.Clear()

        for i, scene in enumerate(self.scenes):
            self.scene_choice.Append(scene.name)
            if scene == self.selected_scene:
                self.scene_choice.SetSelection(i)

        # If not valid scene is currently selected and a scene exists, select it
        if self.scenes and (not self.selected_scene or self.selected_scene.name != self.scene_choice.GetLabelText()):
            self.scene_choice.SetSelection(0)
            self.scene_choice.AddPendingEvent(wx.CommandEvent(wx.wxEVT_CHOICE))

    def FetchScenes(self, root):
        for node in root.children:
            if node.is_scene:
                self.scenes.append(node.scene)
            elif node.is_folder:
                self.FetchScenes(node)

    def ProjectChanged(self, event):
        # Recenter camera when a visualization is added to the current scene
        if event.change == ProjectChangedEvent.ADDED_VISUALIZATION:
            node = event.node
            if node is not None and node.is_visualization:
                if isinstance(node.visualization, VisualizationPlugin3D):
                    parent = node.parent
                    while parent is not None and not parent.is_scene:
                        parent = parent.parent

                    if parent is not None and parent.is_scene:
                        if parent.scene == self.selected_scene:
                            self.gl_canvas.camera_interactor.reset_position()

        else:
            self.RefreshScenes()
            self.gl_canvas.Refresh()
            self.UpdateLegend()

    def OnSceneChoice(self, event):
        self.UpdateScene()

    def OnLegendLabel(self, event):
        pass  # Todo

    def OnGeographicLabel(self, event):
        pass  # Todo

    def OnVizHasNewLegend(self, event):
        pass  # Todo

    def UpdateLegend(self):
        pass  # Todo

    def ResetCameraInteractor(self):
        # Todo: observable
        self.gl_canvas.camera_interactor.reset_position()

    def Update(self):
        pass  # Todo

    def UpdateGeocoderInfo(self):
        pass  # Todo
<commit_msg>Add copy viewer to clipboard functionality<commit_after>import wx
from wx.glcanvas import WX_GL_RGBA, WX_GL_DOUBLEBUFFER, WX_GL_DEPTH_SIZE, WX_GL_CORE_PROFILE

from vistas.core.graphics.camera import Camera
from vistas.core.paths import get_resource_bitmap
from vistas.core.plugins.visualization import VisualizationPlugin3D
from vistas.ui.controllers.project import ProjectChangedEvent
from vistas.ui.controls.gl_canvas import GLCanvas
from vistas.ui.project import Project
from vistas.core.utils import get_platform

class ViewerPanel(wx.Panel):
    NORTH = 'north'
    EAST = 'east'
    SOUTH = 'south'
    WEST = 'west'

    POPUP_COPY = 1

    def __init__(self, parent, id):
        super().__init__(parent, id)

        self.north = []
        self.east = None
        self.south = []
        self.west = None
        self.width = 1.
        self.height = 1.

        self.resizing_north = False
        self.resizing_east = False
        self.resizing_south = False
        self.resizing_west = False
        self.last_pos = None

        self.scenes = []
        self.selected_scene = None

        self.parent = parent
        self.north_resize_area = wx.Window(self, wx.ID_ANY, wx.DefaultPosition, wx.Size(-1, 2))
        self.east_resize_area = wx.Window(self, wx.ID_ANY, wx.DefaultPosition, wx.Size(2, -1))
        self.south_resize_area = wx.Window(self, wx.ID_ANY, wx.DefaultPosition, wx.Size(-1, 2))
        self.west_resize_area = wx.Window(self, wx.ID_ANY, wx.DefaultPosition, wx.Size(2, -1))

        self.north_resize_area.SetCursor(wx.Cursor(wx.CURSOR_SIZENS))
        self.east_resize_area.SetCursor(wx.Cursor(wx.CURSOR_SIZEWE))
        self.south_resize_area.SetCursor(wx.Cursor(wx.CURSOR_SIZENS))
        self.west_resize_area.SetCursor(wx.Cursor(wx.CURSOR_SIZEWE))

        self.scene_choice = wx.Choice(self, wx.ID_ANY)
        self.scene_choice.SetToolTip('Select a scene to view')

        self.legend_button = wx.BitmapButton(self, wx.ID_ANY, get_resource_bitmap('stripes.png'))
        self.geodata_button = wx.BitmapButton(self, wx.ID_ANY, get_resource_bitmap('globe.png'))

        self.legend_button.SetToolTip('Show/hide legend')
        self.geodata_button.SetToolTip('Show place names')

        self.camera = Camera()

        attrib_list = [WX_GL_RGBA, WX_GL_DOUBLEBUFFER, WX_GL_DEPTH_SIZE, 16]
        if get_platform() == 'macos':
            attrib_list += [WX_GL_CORE_PROFILE]

        self.gl_canvas = GLCanvas(
            self, wx.ID_ANY, self.camera, attrib_list=attrib_list
        )
        self.gl_canvas.Refresh()

        sizer = wx.BoxSizer(wx.VERTICAL)
        self.SetSizer(sizer)

        top_sizer = wx.BoxSizer(wx.HORIZONTAL)
        top_sizer.Add(self.scene_choice, 1, wx.RIGHT | wx.ALIGN_CENTER_VERTICAL)
        top_sizer.Add(self.legend_button, 0, wx.LEFT | wx.RIGHT | wx.ALIGN_CENTER_VERTICAL, 3)
        top_sizer.Add(self.geodata_button, 0, wx.LEFT | wx.RIGHT | wx.ALIGN_CENTER_VERTICAL, 3)

        content_sizer = wx.BoxSizer(wx.VERTICAL)
        content_sizer.Add(top_sizer, 0, wx.EXPAND | wx.ALL, 5)
        content_sizer.Add(self.gl_canvas, 1, wx.EXPAND)

        center_sizer = wx.BoxSizer(wx.HORIZONTAL)
        center_sizer.Add(self.west_resize_area, 0, wx.EXPAND)
        center_sizer.Add(content_sizer, 1, wx.EXPAND)
        center_sizer.Add(self.east_resize_area, 0, wx.EXPAND)

        sizer.Add(self.north_resize_area, 0, wx.EXPAND)
        sizer.Add(center_sizer, 1, wx.EXPAND)
        sizer.Add(self.south_resize_area, 0, wx.EXPAND)

        # Event handlers
        self.scene_choice.Bind(wx.EVT_CHOICE, self.OnSceneChoice)
        self.legend_button.Bind(wx.EVT_BUTTON, self.OnLegendLabel)
        self.geodata_button.Bind(wx.EVT_BUTTON, self.OnGeographicLabel)

        self.north_resize_area.Bind(wx.EVT_LEFT_DOWN, self.OnResizeLeftDown)
        self.north_resize_area.Bind(wx.EVT_LEFT_UP, self.OnResizeLeftUp)
        self.north_resize_area.Bind(wx.EVT_MOTION, self.OnResizeMotion)

        self.east_resize_area.Bind(wx.EVT_LEFT_DOWN, self.OnResizeLeftDown)
        self.east_resize_area.Bind(wx.EVT_LEFT_UP, self.OnResizeLeftUp)
        self.east_resize_area.Bind(wx.EVT_MOTION, self.OnResizeMotion)

        self.south_resize_area.Bind(wx.EVT_LEFT_DOWN, self.OnResizeLeftDown)
        self.south_resize_area.Bind(wx.EVT_LEFT_UP, self.OnResizeLeftUp)
        self.south_resize_area.Bind(wx.EVT_MOTION, self.OnResizeMotion)

        self.west_resize_area.Bind(wx.EVT_LEFT_DOWN, self.OnResizeLeftDown)
        self.west_resize_area.Bind(wx.EVT_LEFT_UP, self.OnResizeLeftUp)
        self.west_resize_area.Bind(wx.EVT_MOTION, self.OnResizeMotion)

        self.gl_canvas.Bind(wx.EVT_LEFT_DCLICK, self.OnCanvasDClick)
        self.gl_canvas.Bind(wx.EVT_RIGHT_DOWN, self.OnCanvasRightClick)

        # Todo: VI_EVENT_VIZPLUGIN_HAS_NEW_LEGEND event

        self.gl_canvas.camera.wireframe = False

        self.gl_canvas.SetFocus()

        self.legend_window = object()  # Todo: LegendWindow()

        self.RefreshScenes()

        # Todo: observable

    def __del__(self):
        pass  # Todo

    def SetNeighbor(self, neighbor, direction):
        if direction == self.NORTH:
            self.north_resize_area.Show()
            self.north.append(neighbor)

        elif direction == self.EAST:
            self.east_resize_area.Show()
            self.east = neighbor

        elif direction == self.SOUTH:
            self.south_resize_area.Show()
            self.south.append(neighbor)

        elif direction == self.WEST:
            self.west_resize_area.Show()
            self.west = neighbor

    def GetNeighbor(self, direction):
        if direction == self.NORTH:
            return self.north[0] if self.north else None

        elif direction == self.EAST:
            return self.east

        elif direction == self.SOUTH:
            return self.south[0] if self.south else None

        elif direction == self.WEST:
            return self.west

    def RemoveNeighbor(self, neighbor):
        self.north.remove(neighbor)

        if self.east == neighbor:
            self.east = None

        self.south.remove(neighbor)

        if self.west == neighbor:
            self.west = None

    def ResetNeighbors(self):
        self.north = []
        self.east = None
        self.south = []
        self.west = None

    def HideResizeAreas(self):
        self.north_resize_area.Hide()
        self.east_resize_area.Hide()
        self.south_resize_area.Hide()
        self.west_resize_area.Hide()

    def GetSceneChoice(self):
        return self.scene_choice.GetSelection()

    def SetSceneChoice(self, choice):
        self.scene_choice.SetSelection(choice)
        self.UpdateScene()

    def UpdateScene(self):
        for i, scene in enumerate(self.scenes):
            if i == self.scene_choice.GetSelection():
                # Todo: global interactor

                self.gl_canvas.camera.scene = scene  # Temporary until interactor code is added
                self.gl_canvas.camera_interactor.reset_position()
                self.selected_scene = scene

        self.gl_canvas.Refresh()

    def OnResizeLeftDown(self, event):
        self.resizing_north = self.resizing_east = self.resizing_south = self.resizing_west = False

        if event.GetEventObject() == self.north_resize_area:
            self.resizing_north = True
            self.last_pos = event.y

            self.north_resize_area.CaptureMouse()

        elif event.GetEventObject() == self.east_resize_area:
            self.resizing_east = True
            self.last_pos = event.x

            self.east_resize_area.CaptureMouse()

        elif event.GetEventObject() == self.south_resize_area:
            self.resizing_south = True
            self.last_pos = event.y

            self.south_resize_area.CaptureMouse()

        elif event.GetEventObject() == self.west_resize_area:
            self.resizing_west = True
            self.last_pos = event.x

            self.west_resize_area.CaptureMouse()

    def OnResizeLeftUp(self, event):
        self.resizing_north = self.resizing_east = self.resizing_south = self.resizing_west = False

        if self.north_resize_area.HasCapture():
            self.north_resize_area.ReleaseMouse()
        if self.east_resize_area.HasCapture():
            self.east_resize_area.ReleaseMouse()
        if self.south_resize_area.HasCapture():
            self.south_resize_area.ReleaseMouse()
        if self.west_resize_area.HasCapture():
            self.west_resize_area.ReleaseMouse()

    def OnResizeMotion(self, event):
        if self.resizing_north:
            diff = self.last_pos - event.y
            parent_height = self.GetParent().GetSize().GetHeight()
            prop_diff = diff / parent_height

            self.height += prop_diff

            # Update neighbors to the east
            east = self.east
            while east is not None:
                east.height += prop_diff
                east = east.east

            # Update neighbors to the west
            west = self.west
            while west is not None:
                west.height += prop_diff
                west = west.west

            for viewer in self.north:
                viewer.height -= prop_diff

        elif self.resizing_east:
            diff = event.x - self.last_pos
            parent_width = self.GetParent().GetSize().GetWidth()
            prop_diff = diff / parent_width

            self.width += prop_diff
            self.east.width -= prop_diff

        elif self.resizing_south:
            diff = event.y - self.last_pos
            parent_height = self.GetParent().GetSize().GetHeight()
            prop_diff = diff / parent_height

            self.height += prop_diff

            # Update neighbors to the east
            east = self.east
            while east is not None:
                east.height += prop_diff
                east = east.east

            # Update neighbors to the west
            west = self.west
            while west is not None:
                west.height += prop_diff
                west = west.west

            for viewer in self.south:
                viewer.height -= prop_diff

        elif self.resizing_west:
            diff = self.last_pos - event.x
            parent_width = self.GetParent().GetSize().GetWidth()
            prop_diff = diff / parent_width

            self.width += prop_diff
            self.west.width -= prop_diff

        self.GetParent().UpdateViewerSizes()

    def OnCanvasDClick(self, event):
        pass  # Todo

    def OnCanvasRightClick(self, event):
        menu = wx.Menu()
        menu.Append(self.POPUP_COPY, 'Copy')
        menu.Bind(wx.EVT_MENU, self.OnCanvasPopupMenu)

        self.PopupMenu(menu)

    def OnCanvasPopupMenu(self, event):
        if event.GetId() == self.POPUP_COPY:
            if wx.TheClipboard.IsOpened():
                opened = True
            else:
                opened = wx.TheClipboard.Open()

            if opened:
                size = self.gl_canvas.GetSize()
                im = self.gl_canvas.camera.render_to_bitmap(size.x, size.y)
                wx_image = wx.EmptyImage(size.x, size.y)
                wx_image.SetData(im.convert('RGB').tobytes())
                wx.TheClipboard.SetData(wx.BitmapDataObject(wx_image.ConvertToBitmap()))

    def RefreshScenes(self):
        visualization_root = Project.get().visualization_root
        self.scenes = []
        self.FetchScenes(visualization_root)

        self.scene_choice.Clear()

        for i, scene in enumerate(self.scenes):
            self.scene_choice.Append(scene.name)
            if scene == self.selected_scene:
                self.scene_choice.SetSelection(i)

        # If not valid scene is currently selected and a scene exists, select it
        if self.scenes and (not self.selected_scene or self.selected_scene.name != self.scene_choice.GetLabelText()):
            self.scene_choice.SetSelection(0)
            self.scene_choice.AddPendingEvent(wx.CommandEvent(wx.wxEVT_CHOICE))

    def FetchScenes(self, root):
        for node in root.children:
            if node.is_scene:
                self.scenes.append(node.scene)
            elif node.is_folder:
                self.FetchScenes(node)

    def ProjectChanged(self, event):
        # Recenter camera when a visualization is added to the current scene
        if event.change == ProjectChangedEvent.ADDED_VISUALIZATION:
            node = event.node
            if node is not None and node.is_visualization:
                if isinstance(node.visualization, VisualizationPlugin3D):
                    parent = node.parent
                    while parent is not None and not parent.is_scene:
                        parent = parent.parent

                    if parent is not None and parent.is_scene:
                        if parent.scene == self.selected_scene:
                            self.gl_canvas.camera_interactor.reset_position()

        else:
            self.RefreshScenes()
            self.gl_canvas.Refresh()
            self.UpdateLegend()

    def OnSceneChoice(self, event):
        self.UpdateScene()

    def OnLegendLabel(self, event):
        pass  # Todo

    def OnGeographicLabel(self, event):
        pass  # Todo

    def OnVizHasNewLegend(self, event):
        pass  # Todo

    def UpdateLegend(self):
        pass  # Todo

    def ResetCameraInteractor(self):
        # Todo: observable
        self.gl_canvas.camera_interactor.reset_position()

    def Update(self):
        pass  # Todo

    def UpdateGeocoderInfo(self):
        pass  # Todo
<|endoftext|>


<commit_before># coding: utf8
from __future__ import unicode_literals

class IrishMorph:
    consonants = ['b', 'c', 'd', 'f', 'g', 'h', 'j', 'k', 'l', 'm', 'n', 'p', 'q', 'r', 's', 't', 'v', 'w', 'x', 'z']
    broad_vowels = ['a', 'á', 'o', 'ó', 'u', 'ú']
    slender_vowels = ['e', 'é', 'i', 'í']
    vowels = broad_vowels + slender_vowels

    def ends_dentals(word):
        if word[-1:] in ['d', 'n', 't', 's']:
            return True
        else:
            return False

    def devoice(word):
        if word[-2] == 's' and word[-1] == 'd':
            return word[:-1] + 't'
        else:
            return word

    def ends_with_vowel(word):
        return word[-1] in vowels

    def starts_with_vowel(word):
        return word[0] in vowels

    def deduplicate(word):
        if word[-2] == word[-1] and word[-1] in consonants:
            return word[:-1]
        else:
            return word

<commit_msg>bounds checks<commit_after># coding: utf8
from __future__ import unicode_literals

class IrishMorph:
    consonants = ['b', 'c', 'd', 'f', 'g', 'h', 'j', 'k', 'l', 'm', 'n', 'p', 'q', 'r', 's', 't', 'v', 'w', 'x', 'z']
    broad_vowels = ['a', 'á', 'o', 'ó', 'u', 'ú']
    slender_vowels = ['e', 'é', 'i', 'í']
    vowels = broad_vowels + slender_vowels

    def ends_dentals(word):
        if word != "" and word[-1] in ['d', 'n', 't', 's']:
            return True
        else:
            return False

    def devoice(word):
        if len(word) > 2 and word[-2] == 's' and word[-1] == 'd':
            return word[:-1] + 't'
        else:
            return word

    def ends_with_vowel(word):
        return word != "" and word[-1] in vowels

    def starts_with_vowel(word):
        return word != "" and word[0] in vowels

    def deduplicate(word):
        if len(word) > 2 and word[-2] == word[-1] and word[-1] in consonants:
            return word[:-1]
        else:
            return word

<|endoftext|>


<commit_before>import ast

from data.logic import _dimension_factory, _expr_transformer, _model
from spec.mamba import *

with description('_expr_transformer.ExprTransformer'):
  with it('instantiates'):
    expect(calling(_expr_transformer.ExprTransformer, None)).not_to(raise_error)

  with description('compile'):
    with before.each:
      self.factory = _dimension_factory._DimensionFactory()
      self.model = _model._Model(self.factory)
      self.transformer = _expr_transformer.ExprTransformer(self.model)
      self.andy, self.bob = self.name = self.factory(name=['andy', 'bob'])
      self.cherries, self.dates = self.fruit = self.factory(
          fruit=['cherries', 'dates'])

    with it('fails to visit unsupported nodes'):
      expect(calling(self.transformer.compile, ast.Await)).to(
          raise_error(TypeError))
      expect(calling(self.transformer.visit, ast.Await)).to(
          raise_error(NotImplementedError))
      expect(calling(self.transformer.generic_visit, ast.Await)).to(
          raise_error(NotImplementedError))

    with it('supports precise (2d) assignment'):
      expr = self.name['andy'].fruit == self.fruit['cherries']
      expect(self.transformer.compile(expr)).to(be_a(_model._Predicates))
<commit_msg>Look for _predicates.Predicates in expr transformer spec.<commit_after>import ast

from data.logic import _dimension_factory, _expr_transformer, _model, \
  _predicates
from spec.mamba import *

with description('_expr_transformer.ExprTransformer'):
  with it('instantiates'):
    expect(calling(_expr_transformer.ExprTransformer, None)).not_to(raise_error)

  with description('compile'):
    with before.each:
      self.factory = _dimension_factory._DimensionFactory()
      self.model = _model._Model(self.factory)
      self.transformer = _expr_transformer.ExprTransformer(self.model)
      self.andy, self.bob = self.name = self.factory(name=['andy', 'bob'])
      self.cherries, self.dates = self.fruit = self.factory(
          fruit=['cherries', 'dates'])

    with it('fails to visit unsupported nodes'):
      expect(calling(self.transformer.compile, ast.Await)).to(
          raise_error(TypeError))
      expect(calling(self.transformer.visit, ast.Await)).to(
          raise_error(NotImplementedError))
      expect(calling(self.transformer.generic_visit, ast.Await)).to(
          raise_error(NotImplementedError))

    with it('supports precise (2d) assignment'):
      expr = self.name['andy'].fruit == self.fruit['cherries']
      expect(self.transformer.compile(expr)).to(be_a(_predicates.Predicates))
<|endoftext|>


<commit_before>import numpy as np
import scipy.sparse as sp

from sklearn.base import BaseEstimator, ClusterMixin, TransformerMixin
from sklearn.cluster.k_means_ import _init_centroids, _labels_inertia

#class VonMisesFisher(BaseEstimator, ClusterMixin, TransformerMixin):
<commit_msg>Sketch of soft von mises clustering<commit_after>import numpy as np
from scipy.special import jv
#import scipy.sparse as sp

#from sklearn.base import BaseEstimator, ClusterMixin, TransformerMixin
#from sklearn.cluster.k_means_ import _init_centroids, _labels_inertia
from sklearn.cluster.k_means_ import _init_centroids
from sklearn.utils.extmath import row_norms, squared_norm

def _vmf_distribution(X, kappa, mu):
    n_examples, n_features = X.shape
    return _vmf_normalize(kappa, n_features) * np.exp(kappa * np.dot(mu, X))

def _vmf_normalize(kappa, dim):
    num = np.pow(kappa, dim/2. - 1.)
    denom = np.pow(2. * np.pi, dim/2.) * jv(kappa, dim/2. - 1.)
    if denom == 0:
        print 'TODO: raise error here'
    return num/denom

def _update_params(X, posterior):
    n_examples, n_features = X.shape
    n_clusters, n_examples = posterior.shape
    weights = np.zeros((n_clusters,))
    centers = np.zeros((n_clusters, n_examples))
    concentrations = np.zeros((n_clusters,))
    for cc in range(n_clusters):
        # update weights (alpha)
        weights[cc] = np.sum(posterior[cc, :]) / n_examples

        # update centers (mu)
        for ee in range(n_examples):
            centers[cc, ee] = X[ee, :] * posterior[cc, ee]

        # precomputes
        center_norm = np.linalg.norm(centers[cc, :])
        rbar = center_norm / (n_examples * weights[cc])

        # normalize centers
        centers[cc, :] = centers[cc, :] / center_norm

        # update concentration (kappa)
        concentrations[cc] = rbar * n_features - np.pow(rbar, 3)
        concentrations[cc] /= 1 - np.pow(rbar, 2)

    return centers, weights, concentrations

def _soft_moVMF(X, n_clusters, max_iter=300, verbose=False,
               init='k-means++', random_state=None, tol=1e-4):
    random_state = check_random_state(random_state)

    # init centers (mus)
    centers = _init_centroids(X, n_clusters, init, random_state=random_state,
                              x_squared_norms=np.ones(X.shape[1]))

    # init weights (alphas)
    weights = np.ones((n_clusters,))
    weights = weights / np.sum(weights)

    # init concentrations (kappas)
    concentrations = np.ones((n_clusters,))
    concentrations = concentrations / np.sum(concentrations)

    n_examples, n_features = np.shape(X)

    if verbose:
        print("Initialization complete")

    for iter in range(max_iter):
        centers_prev = centers.copy()

        # (expectation)

        # estimate posterior

        f = np.zeros((n_clusters, n_examples))
        for cc in range(n_clusters):
            f[cc, :] = _vmf_distribution(X, concentrations[cc], centers[cc, :])

        posterior = np.zeros((n_clusters, n_examples))
        for ee in range(n_examples):
            normalization = np.sum(f[:, ee], axis=1)
            for cc in range(n_clusters):
                posterior[cc, ee] = weights[cc] * f[cc, ee]
                posterior[cc, ee] /= normalization

        # (maximization)
        centers, weights, concentrations = _update_params(X, posterior)

        # check convergence
        tolcheck = squared_norm(centers_prev - centers)
        if tolcheck <= tol:
            if verbose:
                print("Converged at iteration %d: "
                      "center shift %e within tolerance %e"
                      % (iter, tolcheck, tol))
            break

#class VonMisesFisher(BaseEstimator, ClusterMixin, TransformerMixin):
#    def __init__(self):

<|endoftext|>


<commit_before># Kaggel Auxillary Functions
# AGC 2013
def cat_bool_maker(s):
    '''
    when used with the pandas df.series.apply() method,  it will create a boolean cat var.
    if values exist the bool will register as 1
    if nan vaues esit the bool will register as 0

    '''
    if np.isnan(s) == True:
        return 0
    else:
        return 1

def ml_formula(y, df):
    '''
    a simple function to create a formula using all available features for patsy dmatrices function.

    ins
    --
    y = a string of the variable your trying to predict

    df = the data frame your trying to make your predctions on

    outs
    ---
    a string of the desired formula
    '''
    formula = y +' ~ '
    for i, val in enumerate(df.columns):
        if i == 0 and val != y:
            formula += val
        if i != 0 and val != y:
            formula += ' + ' + val
    return formula

def progress(i, num_tasks):
    '''
    A simple textual progress bar

    Ins
    --
    i = should be an iterable value to measure the progress of your taks
    num_tasks = total number of tasks

    outs
    --
    A progress bar like [#########  ]
    '''
    import sys

    progress = "\r["

    for _ in range (0,i):
        progress += "#"

    for _ in range (i, num_tasks):
        progress += " "

    progress += "]"

    sys.stdout.write(progress)
    sys.stdout.flush()

def cat_clean(s):
    """
    Cleans categorical variables of NaNs by setting the NaN to 0 so it can be processed by a patsy dmatrices function.

    Inputs
    --
    s = cat value

    output
    --
    cat value or int 0 in place of a NaN val
    """
    import numpy as np

    if isinstance(s, str) == False and np.isnan(s) == True:
        s = 0
    return s

def predict(test_data, results, i):
    """
    Returns a NumPy array of independent variable predictions of a test file based on your regression of a train file. Built for speed

    Parameters
    --
    Test_data: should be test data you are trying to predict in a pandas dataframe
    results: should be dict of your models results wrapper and the formula used to produce it.
        ie.
        results['Model_Name'] = {[<statsmodels.regression.linear_model.RegressionResultsWrapper> , "Price ~ I(Supply, Demand)] }
    i: should be the name of your model. You can iterate through the results dict.
    --

    Returns
    --
    Predictions in a flat NumPy array.
    AGC 2013
    """
    import numpy as np
    from pandas import DataFrame
    from patsy import dmatrices

    model_params = DataFrame(results[i][0].params)
    formula = results[i][1]

    # Create reg friendly test dataframe
    yt, xt = dmatrices(formula, data=test_data, return_type='dataframe')

    # remove extraneous features for efficiency
    to_drop = list()
    to_drop[:] = [] # Empty list, in case cells are executed out of order
    for c in xt.columns:
        if c not in model_params.index:
            to_drop.append(c)
    xt = xt.drop(to_drop, axis=1)

    to_drop[:] = [] # Empty list
    for c in model_params.index :
        if c not in xt.columns:
            to_drop.append(c)
    model_params = model_params.drop(to_drop)

    # Convert to NumPy arrays for performance
    model_params = np.asarray(model_params)
    yt = np.asarray(yt)
    yt = yt.ravel()
    xt = np.asarray(xt)

    # Use our models to create predictions
    row, col = xt.shape
    model_params = model_params.ravel() # flatten array
    model_array = []

    for _ in xrange(row):
            model_array.append(model_params)
    model_array = np.asarray(model_array)

    # Multiply matrix together
    predictions = np.multiply(xt, model_array)
    predictions = np.sum(predictions, axis = 1)

    return predictions

def quater_maker(d):
    """
    Pareses dates and returns the appropriate quarter.
    --
    Parameters
    d: a python date time object
    --
    Returns:
    The quarter in a string

    AGC 2013
    """
    r = ''
    Q1 = [1, 2, 3]
    Q2 = [4, 5, 6]
    Q3 = [7, 8, 9]
    Q4 = [10, 11, 12]
    if d.month in Q1:
        r = 'Q1'
    if d.month in Q2:
        r = 'Q2'
    if d.month in Q3:
        r = 'Q3'
    if d.month in Q4:
        r = 'Q4'
    return r

      #if date is within daterange, Q1 ect. then output ex: 2011 Q1
      #manipulate months. say if  0<m4: return YYYY + Q1 ect.
      # then apply to dateline

def score_rmsle(y, df, df2, p = 0):
    '''
    ins
    --
    y =  what your trying to predict. must be a string. ie. 'SalesPrice'
    df =  your predictions
    df2 = the solutions set
    p = option to print rmsle as string; 0 = dont print, 1 = print
    outs
    --
    prints rmsle
    rmsle as a float
    '''
    import numpy as np

    prediction = np.asarray(df[y])
    actual = np.asarray(df2[y])

    rsmle = np.sqrt(np.mean(np.power(np.log(actual + 1) - np.log(prediction + 1), 2)))

    if p == 1 :
        print "rsmle: " + str(rsmle)
    return rsmle

def score_rmse(y, df, df2, p = 0 ):
    """
    ins
    --
    y =  what your trying to predict. must be a string. ie. 'SalesPrice'
    df =  your predictions
    df2 = the solutions set
    p = option to print rmsle as string; 0 = don't print, 1 = print

    outs
    --
    prints rmse
    rmse as a float
    """
    import numpy as np

    prediction = np.asarray(df[y])
    actual = np.asarray(df2[y])

    rsme = np.sqrt(np.mean(np.power((actual - prediction), 2)))

    if p == 1:
        print "rsme: " + str(rsme)

    return rsme

def unwanted_pals(x, s = .1):
    '''
    Inputs
    --
    x should be a pandas series of the results.pvalues of your model
    s = significance level, default at 90% confidence

    Outs
    --
    returns a list of columns below significance level
    '''
    dropl = list()
    for i,z in enumerate(x):
        if z>s:
             dropl.append(x.index[i])
    return dropl

def stock_price_at_date(x, ticker, lag=0):
    '''
    ins
    --
    x should be the date your looking for
    ticker should be the stock ticker
    lag should be # of days to lag stock price

    outs
    --
    stock price.
    '''
    from pandas.io.data import DataReader
    from datetime import date, timedelta

    x = (x - timedelta(days = lag))
    r = DataReader(ticker,start=x, end=x, data_source='yahoo')
    r = r.ix[0,5]
    return r

def side_by_side(*objs, **kwds):
    '''
    created by wes mickinney, it only exists here becuase I use this function all the time.
    '''
    from pandas.core.common import adjoin
    space = kwds.get('space', 4)
    reprs = [repr(obj).split('\n') for obj in objs]
    print adjoin(space, *reprs)

def describe_frame(df):
    """
    ins
    ---
    df = dataframe you want to describe

    outs
    ---
    descriptive stats on your dataframe, in dataframe.

    agc2013
    """
    from pandas import Series, DataFrame
    sum_stats = []
    for i in df.columns:
        x = Series(df[i].describe())
        x.name = i
        sum_stats.append(x)
    stats = DataFrame(sum_stats)
    stats
    return stats

def bin_residuals(resid, var, bins):
    '''
    Compute average residuals within bins of a variable.

    Returns a dataframe indexed by the bins, with the bin midpoint,
    the residual average within the bin, and the confidence interval
    bounds.

    ins
    --
    resid, var, bins

    out
    --
    bin DataFrame

    '''
    from pandas import DataFrame, qcut
    import NumPy as np
    # use scipy's binned stat method
    resid_df = DataFrame({'var': var, 'resid': resid})
    resid_df['bins'] = qcut(var, bins)
    bin_group = resid_df.groupby('bins')
    bin_df = bin_group['var', 'resid'].mean()
    bin_df['count'] = bin_group['resid'].count()
    bin_df['lower_ci'] = -2 * (bin_group['resid'].std() /
                               np.sqrt(bin_group['resid'].count()))
    bin_df['upper_ci'] =  2 * (bin_group['resid'].std() /
                               np.sqrt(bin_df['count']))
    bin_df = bin_df.sort('var')
    return(bin_df)

def plot_binned_residuals(bin_df):
    '''
    Plotted binned residual averages and confidence intervals.

    ins
    --
    bin_df ie from bin_residuals(resid, var, bins)
    outs
    --
    pretty plots
    '''
    import matplotlib as plt

    plt.plot(bin_df['var'], bin_df['resid'], '.')
    plt.plot(bin_df['var'], bin_df['lower_ci'], '-r')
    plt.plot(bin_df['var'], bin_df['upper_ci'], '-r')
    plt.axhline(0, color = 'gray', lw = .5)
<commit_msg>added dataframe_welch_ttest<commit_after># Kaggel Auxillary Functions
# AGC 2013
def dataframe_welch_ttest(df, desc_frame):
    '''
    Takes in a dataframe, and a described dataframe ( from the desribe_frame() method)

    returns

    t-statistic and p-value for each feature in a pandas dataframe
    '''
    import scipy as sp
    import numpy as np
    desc_frame['t-statistic'] = np.nan
    desc_frame['p-value'] = np.nan

    for name,item in df.iteritems():
        try :
            res = sp.stats.ttest_ind(df[name][df.sued_bool == 0].dropna(),
                                    df[name][df.sued_bool == 1].dropna(),
                                    equal_var=False
                                    )
        except:
            res = (np.nan, np.nan)

        desc_frame.ix[name,'t-statistic'], desc_frame.ix[name,'p-value'] = res

def cat_bool_maker(s):
    '''
    when used with the pandas df.series.apply() method,  it will create a boolean cat var.
    if values exist the bool will register as 1
    if nan vaues esit the bool will register as 0

    '''
    if np.isnan(s) == True:
        return 0
    else:
        return 1

def ml_formula(y, df):
    '''
    a simple function to create a formula using all available features for patsy dmatrices function.

    ins
    --
    y = a string of the variable your trying to predict

    df = the data frame your trying to make your predctions on

    outs
    ---
    a string of the desired formula
    '''
    formula = y +' ~ '
    for i, val in enumerate(df.columns):
        if i == 0 and val != y:
            formula += val
        if i != 0 and val != y:
            formula += ' + ' + val
    return formula

def progress(i, num_tasks):
    '''
    A simple textual progress bar

    Ins
    --
    i = should be an iterable value to measure the progress of your taks
    num_tasks = total number of tasks

    outs
    --
    A progress bar like [#########  ]
    '''
    import sys

    progress = "\r["

    for _ in range (0,i):
        progress += "#"

    for _ in range (i, num_tasks):
        progress += " "

    progress += "]"

    sys.stdout.write(progress)
    sys.stdout.flush()

def cat_clean(s):
    """
    Cleans categorical variables of NaNs by setting the NaN to 0 so it can be processed by a patsy dmatrices function.

    Inputs
    --
    s = cat value

    output
    --
    cat value or int 0 in place of a NaN val
    """
    import numpy as np

    if isinstance(s, str) == False and np.isnan(s) == True:
        s = 0
    return s

def predict(test_data, results, i):
    """
    Returns a NumPy array of independent variable predictions of a test file based on your regression of a train file. Built for speed

    Parameters
    --
    Test_data: should be test data you are trying to predict in a pandas dataframe
    results: should be dict of your models results wrapper and the formula used to produce it.
        ie.
        results['Model_Name'] = {[<statsmodels.regression.linear_model.RegressionResultsWrapper> , "Price ~ I(Supply, Demand)] }
    i: should be the name of your model. You can iterate through the results dict.
    --

    Returns
    --
    Predictions in a flat NumPy array.
    AGC 2013
    """
    import numpy as np
    from pandas import DataFrame
    from patsy import dmatrices

    model_params = DataFrame(results[i][0].params)
    formula = results[i][1]

    # Create reg friendly test dataframe
    yt, xt = dmatrices(formula, data=test_data, return_type='dataframe')

    # remove extraneous features for efficiency
    to_drop = list()
    to_drop[:] = [] # Empty list, in case cells are executed out of order
    for c in xt.columns:
        if c not in model_params.index:
            to_drop.append(c)
    xt = xt.drop(to_drop, axis=1)

    to_drop[:] = [] # Empty list
    for c in model_params.index :
        if c not in xt.columns:
            to_drop.append(c)
    model_params = model_params.drop(to_drop)

    # Convert to NumPy arrays for performance
    model_params = np.asarray(model_params)
    yt = np.asarray(yt)
    yt = yt.ravel()
    xt = np.asarray(xt)

    # Use our models to create predictions
    row, col = xt.shape
    model_params = model_params.ravel() # flatten array
    model_array = []

    for _ in xrange(row):
            model_array.append(model_params)
    model_array = np.asarray(model_array)

    # Multiply matrix together
    predictions = np.multiply(xt, model_array)
    predictions = np.sum(predictions, axis = 1)

    return predictions

def quater_maker(d):
    """
    Pareses dates and returns the appropriate quarter.
    --
    Parameters
    d: a python date time object
    --
    Returns:
    The quarter in a string

    AGC 2013
    """
    r = ''
    Q1 = [1, 2, 3]
    Q2 = [4, 5, 6]
    Q3 = [7, 8, 9]
    Q4 = [10, 11, 12]
    if d.month in Q1:
        r = 'Q1'
    if d.month in Q2:
        r = 'Q2'
    if d.month in Q3:
        r = 'Q3'
    if d.month in Q4:
        r = 'Q4'
    return r

      #if date is within daterange, Q1 ect. then output ex: 2011 Q1
      #manipulate months. say if  0<m4: return YYYY + Q1 ect.
      # then apply to dateline

def score_rmsle(y, df, df2, p = 0):
    '''
    ins
    --
    y =  what your trying to predict. must be a string. ie. 'SalesPrice'
    df =  your predictions
    df2 = the solutions set
    p = option to print rmsle as string; 0 = dont print, 1 = print
    outs
    --
    prints rmsle
    rmsle as a float
    '''
    import numpy as np

    prediction = np.asarray(df[y])
    actual = np.asarray(df2[y])

    rsmle = np.sqrt(np.mean(np.power(np.log(actual + 1) - np.log(prediction + 1), 2)))

    if p == 1 :
        print "rsmle: " + str(rsmle)
    return rsmle

def score_rmse(y, df, df2, p = 0 ):
    """
    ins
    --
    y =  what your trying to predict. must be a string. ie. 'SalesPrice'
    df =  your predictions
    df2 = the solutions set
    p = option to print rmsle as string; 0 = don't print, 1 = print

    outs
    --
    prints rmse
    rmse as a float
    """
    import numpy as np

    prediction = np.asarray(df[y])
    actual = np.asarray(df2[y])

    rsme = np.sqrt(np.mean(np.power((actual - prediction), 2)))

    if p == 1:
        print "rsme: " + str(rsme)

    return rsme

def unwanted_pals(x, s = .1):
    '''
    Inputs
    --
    x should be a pandas series of the results.pvalues of your model
    s = significance level, default at 90% confidence

    Outs
    --
    returns a list of columns below significance level
    '''
    dropl = list()
    for i,z in enumerate(x):
        if z>s:
             dropl.append(x.index[i])
    return dropl

def stock_price_at_date(x, ticker, lag=0):
    '''
    ins
    --
    x should be the date your looking for
    ticker should be the stock ticker
    lag should be # of days to lag stock price

    outs
    --
    stock price.
    '''
    from pandas.io.data import DataReader
    from datetime import date, timedelta

    x = (x - timedelta(days = lag))
    r = DataReader(ticker,start=x, end=x, data_source='yahoo')
    r = r.ix[0,5]
    return r

def side_by_side(*objs, **kwds):
    '''
    created by wes mickinney, it only exists here becuase I use this function all the time.
    '''
    from pandas.core.common import adjoin
    space = kwds.get('space', 4)
    reprs = [repr(obj).split('\n') for obj in objs]
    print adjoin(space, *reprs)

def describe_frame(df):
    """
    ins
    ---
    df = dataframe you want to describe

    outs
    ---
    descriptive stats on your dataframe, in dataframe.

    agc2013
    """
    from pandas import Series, DataFrame
    sum_stats = []
    for i in df.columns:
        x = Series(df[i].describe())
        x.name = i
        sum_stats.append(x)
    stats = DataFrame(sum_stats)
    stats
    return stats

def bin_residuals(resid, var, bins):
    '''
    Compute average residuals within bins of a variable.

    Returns a dataframe indexed by the bins, with the bin midpoint,
    the residual average within the bin, and the confidence interval
    bounds.

    ins
    --
    resid, var, bins

    out
    --
    bin DataFrame

    '''
    from pandas import DataFrame, qcut
    import NumPy as np
    # use scipy's binned stat method
    resid_df = DataFrame({'var': var, 'resid': resid})
    resid_df['bins'] = qcut(var, bins)
    bin_group = resid_df.groupby('bins')
    bin_df = bin_group['var', 'resid'].mean()
    bin_df['count'] = bin_group['resid'].count()
    bin_df['lower_ci'] = -2 * (bin_group['resid'].std() /
                               np.sqrt(bin_group['resid'].count()))
    bin_df['upper_ci'] =  2 * (bin_group['resid'].std() /
                               np.sqrt(bin_df['count']))
    bin_df = bin_df.sort('var')
    return(bin_df)

def plot_binned_residuals(bin_df):
    '''
    Plotted binned residual averages and confidence intervals.

    ins
    --
    bin_df ie from bin_residuals(resid, var, bins)
    outs
    --
    pretty plots
    '''
    import matplotlib as plt

    plt.plot(bin_df['var'], bin_df['resid'], '.')
    plt.plot(bin_df['var'], bin_df['lower_ci'], '-r')
    plt.plot(bin_df['var'], bin_df['upper_ci'], '-r')
    plt.axhline(0, color = 'gray', lw = .5)
<|endoftext|>


<commit_before>from library import TwitterBot, MarkovBabbler, TwitterListener
import json
import random
from datetime import datetime

def get_listener():
    """
    instantate the listener and backfill recent tweets
    """
    smw_listener = TwitterListener('smw',
                                   query='#SMWNYC OR "social media week" OR @SMWNYC OR #SMWbot OR @SmwKanye')
    try:
        smw_listener.backfill()
    except:
        pass
    return smw_listener

def get_kanye_babbler():
    f = open('data/kanye/quotes.txt','rb')
    yeezy_text = "\n".join([l for l in f.readlines()])
    f.close()
    return MarkovBabbler(yeezy_text)

def get_smw_seed_generators():
    # load text data sets
    f = open('data/smw/talks.ldjson','rb')
    talks = [json.loads(l) for l in f.readlines()]
    smw_program_text = " ".join([t['name']+" "+t['description'] for t in talks])
    smw_program_text = smw_program_text + "\n".join([l for l in f.readlines()])
    f.close()
    smw_tweet_text_all = smw_listener.get_recent_text(1000)
    smw_tweet_text_recent = smw_listener.get_recent_text(25)

    # make some markov babblers
    smw_program_babbler = MarkovBabbler(smw_program_text)
    smw_all_tweets_babbler = MarkovBabbler(smw_tweet_text_all)
    smw_recent_tweets_babbler = MarkovBabbler(smw_tweet_text_recent)
    return [smw_program_babbler,
            smw_all_tweets_babbler,
            smw_recent_tweets_babbler]

def generate_text(seed_generator, in_reply_to=None):
    try:
        if in_reply_to:
            if random.random() < .5:
                seed = MarkovBabbler(in_reply_to.text).generate_seed()
            else:
                seed = kanye_babbler.generate_seed()
        else:
            seed = seed_generator.generate_seed()
        tweet_text = kanye_babbler.generate(seed=seed)
        return tweet_text
    except:
        return generate_text(seed_generator, in_reply_to=in_reply_to)

if __name__ == '__main__':
    # instantiate the bot
    kanye = TwitterBot('production')
    kanye.generate_text = generate_text

    smw_listener = get_listener()
    seed_generators = get_smw_seed_generators()

    kanye_babbler = get_kanye_babbler()

    # reply to outstanding mentions
    mentions_not_replied_to = kanye.get_mentions_not_replied_to()
    print mentions_not_replied_to
    for mention in mentions_not_replied_to:
        text = generate_text(None, in_reply_to=mention)
        kanye.tweet(text, in_reply_to=mention)

    should_i_tweet_now = {0: 0.125,
                          1: 0.125,
                          2: 0.099999999999999992,
                          3: 0.024999999999999998,
                          4: 0.024999999999999998,
                          12: 0.14999999999999999,
                          13: 0.125,
                          14: 0.29999999999999999,
                          15: 0.35000000000000003,
                          16: 0.47499999999999998,
                          17: 0.57500000000000007,
                          18: 0.67500000000000004,
                          19: 0.55000000000000004,
                          20: 0.70000000000000007,
                          21: 0.27500000000000002,
                          22: 0.25,
                          23: 0.17500000000000002}

    if random.random() < should_i_tweet_now[datetime.now().hour]:

        seed_generator = random.choice(seed_generators)

        # reply to topical tweets
        try:
            query = """
            (#smwnyc OR "social bots" OR "social media week" OR "twitter bots" OR "twitter bot" OR "social bot" OR #SMWbots OR #SMWbot)
            """
            topical_tweets = kanye._api_client.GetSearch(query)
            for tweet in topical_tweets:
                if random.random() < .07:
                    if not kanye.should_reply(tweet):
                        text = generate_text(seed_generator, in_reply_to=tweet)
                        kanye.tweet(text, in_reply_to=tweet)

        except Exception, e:
            print str(e)
            pass

        text = generate_text(seed_generator)
        kanye.tweet(text)
    else:
        print "not going to tweet now"

<commit_msg>missing keys<commit_after>from library import TwitterBot, MarkovBabbler, TwitterListener
import json
import random
from datetime import datetime

def get_listener():
    """
    instantate the listener and backfill recent tweets
    """
    smw_listener = TwitterListener('smw',
                                   query='#SMWNYC OR "social media week" OR @SMWNYC OR #SMWbot OR @SmwKanye')
    try:
        smw_listener.backfill()
    except:
        pass
    return smw_listener

def get_kanye_babbler():
    f = open('data/kanye/quotes.txt','rb')
    yeezy_text = "\n".join([l for l in f.readlines()])
    f.close()
    return MarkovBabbler(yeezy_text)

def get_smw_seed_generators():
    # load text data sets
    f = open('data/smw/talks.ldjson','rb')
    talks = [json.loads(l) for l in f.readlines()]
    smw_program_text = " ".join([t['name']+" "+t['description'] for t in talks])
    smw_program_text = smw_program_text + "\n".join([l for l in f.readlines()])
    f.close()
    smw_tweet_text_all = smw_listener.get_recent_text(1000)
    smw_tweet_text_recent = smw_listener.get_recent_text(25)

    # make some markov babblers
    smw_program_babbler = MarkovBabbler(smw_program_text)
    smw_all_tweets_babbler = MarkovBabbler(smw_tweet_text_all)
    smw_recent_tweets_babbler = MarkovBabbler(smw_tweet_text_recent)
    return [smw_program_babbler,
            smw_all_tweets_babbler,
            smw_recent_tweets_babbler]

def generate_text(seed_generator, in_reply_to=None):
    try:
        if in_reply_to:
            if random.random() < .5:
                seed = MarkovBabbler(in_reply_to.text).generate_seed()
            else:
                seed = kanye_babbler.generate_seed()
        else:
            seed = seed_generator.generate_seed()
        tweet_text = kanye_babbler.generate(seed=seed)
        return tweet_text
    except:
        return generate_text(seed_generator, in_reply_to=in_reply_to)

if __name__ == '__main__':
    # instantiate the bot
    kanye = TwitterBot('production')
    kanye.generate_text = generate_text

    smw_listener = get_listener()
    seed_generators = get_smw_seed_generators()

    kanye_babbler = get_kanye_babbler()

    # reply to outstanding mentions
    mentions_not_replied_to = kanye.get_mentions_not_replied_to()
    print mentions_not_replied_to
    for mention in mentions_not_replied_to:
        text = generate_text(None, in_reply_to=mention)
        kanye.tweet(text, in_reply_to=mention)

    should_i_tweet_now = {0: 0.125,
                          1: 0.125,
                          2: 0.099999999999999992,
                          3: 0.024999999999999998,
                          4: 0.024999999999999998,
                          5: 0,
                          6: 0,
                          7: 0,
                          8: 0,
                          9: 0,
                          10: 0,
                          12: 0.14999999999999999,
                          13: 0.125,
                          14: 0.29999999999999999,
                          15: 0.35000000000000003,
                          16: 0.47499999999999998,
                          17: 0.57500000000000007,
                          18: 0.67500000000000004,
                          19: 0.55000000000000004,
                          20: 0.70000000000000007,
                          21: 0.27500000000000002,
                          22: 0.25,
                          23: 0.17500000000000002}

    if random.random() < should_i_tweet_now[datetime.now().hour]:

        seed_generator = random.choice(seed_generators)

        # reply to topical tweets
        try:
            query = """
            (#smwnyc OR "social bots" OR "social media week" OR "twitter bots" OR "twitter bot" OR "social bot" OR #SMWbots OR #SMWbot)
            """
            topical_tweets = kanye._api_client.GetSearch(query)
            for tweet in topical_tweets:
                if random.random() < .07:
                    if not kanye.should_reply(tweet):
                        text = generate_text(seed_generator, in_reply_to=tweet)
                        kanye.tweet(text, in_reply_to=tweet)

        except Exception, e:
            print str(e)
            pass

        text = generate_text(seed_generator)
        kanye.tweet(text)
    else:
        print "not going to tweet now"

<|endoftext|>


<commit_before>import pandas as pd
from datetime import timedelta
import matplotlib.pyplot as plt
import matplotlib

matplotlib.style.use('ggplot') # Look Pretty

#
# INFO: This dataset has call records for 10 users tracked over the course of 3 years.
# Your job is to find out where the users likely live at!

def showandtell(title=None):
  if title != None: plt.savefig(title + ".png", bbox_inches='tight', dpi=300)
  plt.show()
  # exit()

def clusterInfo(model):
  print "Cluster Analysis Inertia: ", model.inertia_
  print '------------------------------------------'
  for i in range(len(model.cluster_centers_)):
    print "\n  Cluster ", i
    print "    Centroid ", model.cluster_centers_[i]
    print "    #Samples ", (model.labels_==i).sum() # NumPy Power

# Find the cluster with the least # attached nodes
def clusterWithFewestSamples(model):
  # Ensure there's at least on cluster...
  minSamples = len(model.labels_)
  minCluster = 0
  for i in range(len(model.cluster_centers_)):
    if minSamples > (model.labels_==i).sum():
      minCluster = i
      minSamples = (model.labels_==i).sum()
  print "\n  Cluster With Fewest Samples: ", minCluster
  return (model.labels_==minCluster)

def doKMeans(data, clusters=0):
  #
  # TODO: Be sure to only feed in Lat and Lon coordinates to the KMeans algo, since none of the other
  # data is suitable for your purposes. Since both Lat and Lon are (approximately) on the same scale,
  # no feature scaling is required. Print out the centroid locations and add them onto your scatter
  # plot. Use a distinguishable marker and color.
  #
  # Hint: Make sure you fit ONLY the coordinates, and in the CORRECT order (lat first). This is part
  # of your domain expertise. Also, *YOU* need to instantiate (and return) the variable named `model`
  # here, which will be a SKLearn K-Means model for this to work.
  #
  # .. your code here ..
  return model

#
# TODO: Load up the dataset and take a peek at its head and dtypes.
# Convert the date using pd.to_datetime, and the time using pd.to_timedelta
#
# .. your code here ..

#
# TODO: Create a unique list of of the phone-number values (users) stored in the
# "In" column of the dataset, and save it to a variable called `unique_numbers`.
# Manually check through unique_numbers to ensure the order the numbers appear is
# the same order they appear (uniquely) in your dataset:
#
# .. your code here ..

#
# INFO: The locations map above should be too "busy" to really wrap your head around. This
# is where domain expertise comes into play. Your intuition tells you that people are likely
# to behave differently on weekends:
#
# On Weekdays:
#   1. People probably don't go into work
#   2. They probably sleep in late on Saturday
#   3. They probably run a bunch of random errands, since they couldn't during the week
#   4. They should be home, at least during the very late hours, e.g. 1-4 AM
#
# On Weekdays:
#   1. People probably are at work during normal working hours
#   2. They probably are at home in the early morning and during the late night
#   3. They probably spend time commuting between work and home everyday

print "\n\nExamining person: ", 0
#
# TODO: Create a slice called user1 that filters to only include dataset records where the
# "In" feature (user phone number) is equal to the first number on your unique list above
#
# .. your code here ..

#
# TODO: Alter your slice so that it includes only Weekday (Mon-Fri) values.
#
# .. your code here ..

#
# TODO: The idea is that the call was placed before 5pm. From Midnight-730a, the user is
# probably sleeping and won't call / wake up to take a call. There should be a brief time
# in the morning during their commute to work, then they'll spend the entire day at work.
# So the assumption is that most of the time is spent either at work, or in 2nd, at home.
#
# .. your code here ..

#
# TODO: Plot the Cell Towers the user connected to
#
# .. your code here ..

#
# INFO: Run K-Means with K=3 or K=4. There really should only be a two areas of concentration. If you
# notice multiple areas that are "hot" (multiple areas the usr spends a lot of time at that are FAR
# apart from one another), then increase K=5, with the goal being that all centroids except two will
# sweep up the annoying outliers and not-home, not-work travel occasions. the other two will zero in
# on the user's approximate home location and work locations. Or rather the location of the cell
# tower closest to them.....
model = doKMeans(user1, 3)

#
# INFO: Print out the mean CallTime value for the samples belonging to the cluster with the LEAST
# samples attached to it. If our logic is correct, the cluster with the MOST samples will be work.
# The cluster with the 2nd most samples will be home. And the K=3 cluster with the least samples
# should be somewhere in between the two. What time, on average, is the user in between home and
# work, between the midnight and 5pm?
midWayClusterIndices = clusterWithFewestSamples(model)
midWaySamples = user1[midWayClusterIndices]
print "    Its Waypoint Time: ", midWaySamples.CallTime.mean()

#
# Let's visualize the results!
# First draw the X's for the clusters:
ax.scatter(model.cluster_centers_[:,1], model.cluster_centers_[:,0], s=169, c='r', marker='x', alpha=0.8, linewidths=2)
#
# Then save the results:
showandtell('Weekday Calls Centroids')  # Comment this line out when you're ready to proceed
<commit_msg>Snapshot<commit_after>import pandas as pd
from datetime import timedelta
import matplotlib.pyplot as plt
import matplotlib
import numpy as np
matplotlib.style.use('ggplot') # Look Pretty

#
# INFO: This dataset has call records for 10 users tracked over the course of 3 years.
# Your job is to find out where the users likely live at!

def showandtell(title=None):
  if title != None: plt.savefig(title + ".png", bbox_inches='tight', dpi=300)
  plt.show()
  # exit()

def clusterInfo(model):
  print "Cluster Analysis Inertia: ", model.inertia_
  print '------------------------------------------'
  for i in range(len(model.cluster_centers_)):
    print "\n  Cluster ", i
    print "    Centroid ", model.cluster_centers_[i]
    print "    #Samples ", (model.labels_==i).sum() # NumPy Power

# Find the cluster with the least # attached nodes
def clusterWithFewestSamples(model):
  # Ensure there's at least on cluster...
  minSamples = len(model.labels_)
  minCluster = 0
  for i in range(len(model.cluster_centers_)):
    if minSamples > (model.labels_==i).sum():
      minCluster = i
      minSamples = (model.labels_==i).sum()
  print "\n  Cluster With Fewest Samples: ", minCluster
  return (model.labels_==minCluster)

def doKMeans(data, clusters=0):
  import sklearn
  from sklearn.cluster import KMeans

  #
  # TODO: Be sure to only feed in Lat and Lon coordinates to the KMeans algo, since none of the other
  # data is suitable for your purposes. Since both Lat and Lon are (approximately) on the same scale,
  # no feature scaling is required. Print out the centroid locations and add them onto your scatter
  # plot. Use a distinguishable marker and color.
  #
  # Hint: Make sure you fit ONLY the coordinates, and in the CORRECT order (lat first). This is part
  # of your domain expertise. Also, *YOU* need to instantiate (and return) the variable named `model`
  # here, which will be a SKLearn K-Means model for this to work.
  #
  # .. your code here ..
  data = [data['TowerLon'], data['TowerLat']]
  data = np.array(data)
  data = np.transpose(data)
  model = KMeans(n_clusters=clusters, random_state=0).fit(data)
  return model

#
# TODO: Load up the dataset and take a peek at its head and dtypes.
# Convert the date using pd.to_datetime, and the time using pd.to_timedelta
#
# .. your code here ..

df = pd.read_csv('Datasets/CDR.csv' )
#print (df.head())
#print df.describe()
#print df.dtypes

df['CallDate'] = pd.to_datetime(df.loc[:, 'CallDate'])
df['CallTime'] = pd.to_timedelta(df.loc[:, 'CallTime'])

#print df.dtypes#

#
# TODO: Create a unique list of of the phone-number values (users) stored in the
# "In" column of the dataset, and save it to a variable called `unique_numbers`.
# Manually check through unique_numbers to ensure the order the numbers appear is
# the same order they appear (uniquely) in your dataset:
#

# .. your code here ..

uninums = df['In'].unique()
print uninums
#stophere
#
# INFO: The locations map above should be too "busy" to really wrap your head around. This
# is where domain expertise comes into play. Your intuition tells you that people are likely
# to behave differently on weekends:
#
# On Weekdays:
#   1. People probably don't go into work
#   2. They probably sleep in late on Saturday
#   3. They probably run a bunch of random errands, since they couldn't during the week
#   4. They should be home, at least during the very late hours, e.g. 1-4 AM
#
# On Weekdays:
#   1. People probably are at work during normal working hours
#   2. They probably are at home in the early morning and during the late night
#   3. They probably spend time commuting between work and home everyday

print "\n\nExamining person: ", 0
#
# TODO: Create a slice called user1 that filters to only include dataset records where the
# "In" feature (user phone number) is equal to the first number on your unique list above
#
# .. your code here ..
user1 = df[df['In'] == df.loc[0,'In']]

#
# TODO: Alter your slice so that it includes only Weekday (Mon-Fri) values.
#
# .. your code here ..
user1 =  user1[user1.DOW == ('Mon'or 'Tue' or 'Wed' or 'Thu' or 'Fri')]
print user1
#
# TODO: The idea is that the call was placed before 5pm. From Midnight-730a, the user is
# probably sleeping and won't call / wake up to take a call. There should be a brief time
# in the morning during their commute to work, then they'll spend the entire day at work.
# So the assumption is that most of the time is spent either at work, or in 2nd, at home.
#
# .. your code here ..
u1 =  user1[user1.CallTime > '05:30:00']
u2 =  user1[user1.CallTime < '17:00:00']
user1 = pd.concat([u1, u2])
#
# TODO: Plot the Cell Towers the user connected to
#
# .. your code here ..
fig = plt.figure()
ax = fig.add_subplot(111)
ax.scatter(user1.TowerLon,user1.TowerLat, c='g', marker='o', alpha=0.2)
ax.set_title('Weekeday Calls (> 5:30 to 17:00)')

#
# INFO: Run K-Means with K=3 or K=4. There really should only be a two areas of concentration. If you
# notice multiple areas that are "hot" (multiple areas the usr spends a lot of time at that are FAR
# apart from one another), then increase K=5, with the goal being that all centroids except two will
# sweep up the annoying outliers and not-home, not-work travel occasions. the other two will zero in
# on the user's approximate home location and work locations. Or rather the location of the cell
# tower closest to them.....
model = doKMeans(user1, 4)
centroids = model.cluster_centers_
ax.scatter(centroids[:,0], centroids[:,1], marker='x', c='red', alpha=0.5, linewidths=3, s=169)
clusterInfo(model)
showandtell()

#
# INFO: Print out the mean CallTime value for the samples belonging to the cluster with the LEAST
# samples attached to it. If our logic is correct, the cluster with the MOST samples will be work.
# The cluster with the 2nd most samples will be home. And the K=3 cluster with the least samples
# should be somewhere in between the two. What time, on average, is the user in between home and
# work, between the midnight and 5pm?
midWayClusterIndices = clusterWithFewestSamples(model)
midWaySamples = user1[midWayClusterIndices]
print "    Its Waypoint Time: ", midWaySamples.CallTime.mean()

#
# Let's visualize the results!
# First draw the X's for the clusters:
ax.scatter(model.cluster_centers_[:,1], model.cluster_centers_[:,0], s=169, c='r', marker='x', alpha=0.8, linewidths=2)
#
# Then save the results:
showandtell('Weekday Calls Centroids')  # Comment this line out when you're ready to proceed
<|endoftext|>


<commit_before>
"""Network-related monitors for SimpleMonitor."""

import urllib
import urllib2
import re
import os
import socket
import datetime

from monitor import Monitor

class MonitorHTTP(Monitor):
    """Check an HTTP server is working right.

    We can either check that we get a 200 OK back, or we can check for a regexp match in the page.
    """

    url = ""
    regexp = None
    regexp_text = ""
    allowed_codes = []

    type = "http"

    def __init__(self, name, config_options):
        Monitor.__init__(self, name, config_options)
        try:
            url = config_options["url"]
        except:
            raise RuntimeError("Required configuration fields missing")

        if config_options.has_key("regexp"):
            regexp = config_options["regexp"]
        else:
            regexp = ""
        if config_options.has_key("allowed_codes"):
            allowed_codes = [int(x.strip()) for x in config_options["allowed_codes"].split(",")]
        else:
            allowed_codes = []

        self.url = url
        if regexp != "":
            self.regexp = re.compile(regexp)
            self.regexp_text = regexp
        self.allowed_codes = allowed_codes

    def run_test(self):
        # store the current default timeout (since it's global)
        original_timeout = socket.getdefaulttimeout()
        socket.setdefaulttimeout(5)
        start_time = datetime.datetime.now()
        end_time = None
        try:
            url_handle = urllib2.urlopen(self.url)
            end_time = datetime.datetime.now()
            load_time = end_time - start_time
            status = "200 OK"
            if hasattr(url_handle, "status"):
                if url_handle.status != "":
                    status = url_handle.status
            if status != "200 OK":
                self.record_fail("Got status '%s' instead of 200 OK" % status)
                socket.setdefaulttimeout(original_timeout)
                return False
            if self.regexp == None:
                self.record_success("%s in %0.2fs" % (status, (load_time.seconds + (load_time.microseconds / 1000000.2))))
                socket.setdefaulttimeout(original_timeout)
                return True
            else:
                for line in url_handle:
                    matches = self.regexp.search(line)
                    if matches:
                        self.record_success("%s in %0.2fs" % (status, (load_time.seconds + (load_time.microseconds / 1000000.2))))
                        socket.setdefaulttimeout(original_timeout)
                        return True
                self.record_fail("Got 200 OK but couldn't match /%s/ in page." % self.regexp_text)
                socket.setdefaulttimeout(original_timeout)
                return False
        except urllib2.HTTPError, e:
            if e.code in self.allowed_codes:
                if end_time != None:
                    load_time = end_time - start_time
                    self.record_success("%s in %0.2fs" % (status, (load_time.seconds + (load_time.microseconds / 1000000.2))))
                else:
                    self.record_success("%s" % status)
                socket.setdefaulttimeout(original_timeout)
                return True
            self.record_fail("HTTP error while opening URL: %s" % e)
            socket.setdefaulttimeout(original_timeout)
            return False
        except Exception, e:
            self.record_fail("Exception while trying to open url: %s" % e)
            socket.setdefaulttimeout(original_timeout)
            return False

    def describe(self):
        """Explains what we do."""
        if self.regexp == None:
            message = "Checking that accessing %s returns HTTP/200 OK" % self.url
        else:
            message = "Checking that accessing %s returns HTTP/200 OK and that /%s/ matches the page" % (self.url, self.regexp_text)
        return message

    def get_params(self):
        return (self.url, self.regexp_text, self.allowed_codes)

class MonitorTCP(Monitor):
    """TCP port monitor"""

    host = ""
    port = ""
    type = "tcp"

    def __init__(self, name, config_options):
        """Constructor"""
        Monitor.__init__(self, name, config_options)
        try:
            host = config_options["host"]
            port = int(config_options["port"])
        except:
            raise RuntimeError("Required configuration fields missing")

        if host == "":
            raise RuntimeError("missing hostname")
        if port == "" or port <= 0:
            raise RuntimeError("missing or invalid port number")
        self.host = host
        self.port = port

    def run_test(self):
        """Check the port is open on the remote host"""
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        try:
            s.settimeout(5.0)
            s.connect((self.host, self.port))
        except:
            self.record_fail()
            return False
        s.close()
        self.record_success()
        return True

    def describe(self):
        """Explains what this instance is checking"""
        return "checking for open tcp socket on %s:%d" % (self.host, self.port)

    def get_params(self):
        return (self.host, self.port)

class MonitorHost(Monitor):
    """Ping a host to make sure it's up"""

    host = ""
    ping_command = ""
    ping_regexp = ""
    type = "host"
    time_regexp = ""

    def __init__(self, name, config_options):
        """
        Note: We use -w/-t on Windows/POSIX to limit the amount of time we wait to 5 seconds.
        This is to stop ping holding things up too much. A machine that can't ping back in <5s is
        a machine in trouble anyway, so should probably count as a failure.
        """
        Monitor.__init__(self, name, config_options)
        if self.is_windows(allow_cygwin=True):
            self.ping_command = "ping -n 1 -w 5000 %s"
            self.ping_regexp = "Reply from "
            self.time_regexp = "Average = (?P<ms>\d+)ms"
        else:
            self.ping_command = "ping -c1 -t5 %s 2> /dev/null"
            self.ping_regexp = "bytes from"
            #XXX this regexp is only for freebsd at the moment; not sure about other platforms
            #XXX looks like Linux uses this format too
            self.time_regexp = "min/avg/max/stddev = [\d.]+/(?P<ms>[\d.]+)/"
        try:
            host = config_options["host"]
        except:
            raise RuntimeError("Required configuration fields missing")
        if host == "":
            raise RuntimeError("missing hostname")
        self.host = host

    def run_test(self):
        r = re.compile(self.ping_regexp)
        r2 = re.compile(self.time_regexp)
        success = False
        pingtime = 0.0
        try:
            process_handle = os.popen(self.ping_command % self.host)
            for line in process_handle:
                matches = r.search(line)
                if matches:
                    success = True
                else:
                    matches = r2.search(line)
                    if matches:
                        pingtime = matches.group("ms")
        except Exception, e:
            self.record_fail(e)
            pass
        if success:
            if pingtime > 0:
                self.record_success("%sms" % pingtime)
            else:
                self.record_success()
            return True
        self.record_fail()
        return False

    def describe(self):
        """Explains what this instance is checking"""
        return "checking host %s is pingable" % self.host

    def get_params(self):
        return (self.host, )

<commit_msg>add ping_ttl setting for linux/unix hosts Allows "remote" testing from long network places<commit_after>
"""Network-related monitors for SimpleMonitor."""

import urllib
import urllib2
import re
import os
import socket
import datetime

from monitor import Monitor

class MonitorHTTP(Monitor):
    """Check an HTTP server is working right.

    We can either check that we get a 200 OK back, or we can check for a regexp match in the page.
    """

    url = ""
    regexp = None
    regexp_text = ""
    allowed_codes = []

    type = "http"

    def __init__(self, name, config_options):
        Monitor.__init__(self, name, config_options)
        try:
            url = config_options["url"]
        except:
            raise RuntimeError("Required configuration fields missing")

        if config_options.has_key("regexp"):
            regexp = config_options["regexp"]
        else:
            regexp = ""
        if config_options.has_key("allowed_codes"):
            allowed_codes = [int(x.strip()) for x in config_options["allowed_codes"].split(",")]
        else:
            allowed_codes = []

        self.url = url
        if regexp != "":
            self.regexp = re.compile(regexp)
            self.regexp_text = regexp
        self.allowed_codes = allowed_codes

    def run_test(self):
        # store the current default timeout (since it's global)
        original_timeout = socket.getdefaulttimeout()
        socket.setdefaulttimeout(5)
        start_time = datetime.datetime.now()
        end_time = None
        try:
            url_handle = urllib2.urlopen(self.url)
            end_time = datetime.datetime.now()
            load_time = end_time - start_time
            status = "200 OK"
            if hasattr(url_handle, "status"):
                if url_handle.status != "":
                    status = url_handle.status
            if status != "200 OK":
                self.record_fail("Got status '%s' instead of 200 OK" % status)
                socket.setdefaulttimeout(original_timeout)
                return False
            if self.regexp == None:
                self.record_success("%s in %0.2fs" % (status, (load_time.seconds + (load_time.microseconds / 1000000.2))))
                socket.setdefaulttimeout(original_timeout)
                return True
            else:
                for line in url_handle:
                    matches = self.regexp.search(line)
                    if matches:
                        self.record_success("%s in %0.2fs" % (status, (load_time.seconds + (load_time.microseconds / 1000000.2))))
                        socket.setdefaulttimeout(original_timeout)
                        return True
                self.record_fail("Got 200 OK but couldn't match /%s/ in page." % self.regexp_text)
                socket.setdefaulttimeout(original_timeout)
                return False
        except urllib2.HTTPError, e:
            if e.code in self.allowed_codes:
                if end_time != None:
                    load_time = end_time - start_time
                    self.record_success("%s in %0.2fs" % (status, (load_time.seconds + (load_time.microseconds / 1000000.2))))
                else:
                    self.record_success("%s" % status)
                socket.setdefaulttimeout(original_timeout)
                return True
            self.record_fail("HTTP error while opening URL: %s" % e)
            socket.setdefaulttimeout(original_timeout)
            return False
        except Exception, e:
            self.record_fail("Exception while trying to open url: %s" % e)
            socket.setdefaulttimeout(original_timeout)
            return False

    def describe(self):
        """Explains what we do."""
        if self.regexp == None:
            message = "Checking that accessing %s returns HTTP/200 OK" % self.url
        else:
            message = "Checking that accessing %s returns HTTP/200 OK and that /%s/ matches the page" % (self.url, self.regexp_text)
        return message

    def get_params(self):
        return (self.url, self.regexp_text, self.allowed_codes)

class MonitorTCP(Monitor):
    """TCP port monitor"""

    host = ""
    port = ""
    type = "tcp"

    def __init__(self, name, config_options):
        """Constructor"""
        Monitor.__init__(self, name, config_options)
        try:
            host = config_options["host"]
            port = int(config_options["port"])
        except:
            raise RuntimeError("Required configuration fields missing")

        if host == "":
            raise RuntimeError("missing hostname")
        if port == "" or port <= 0:
            raise RuntimeError("missing or invalid port number")
        self.host = host
        self.port = port

    def run_test(self):
        """Check the port is open on the remote host"""
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        try:
            s.settimeout(5.0)
            s.connect((self.host, self.port))
        except:
            self.record_fail()
            return False
        s.close()
        self.record_success()
        return True

    def describe(self):
        """Explains what this instance is checking"""
        return "checking for open tcp socket on %s:%d" % (self.host, self.port)

    def get_params(self):
        return (self.host, self.port)

class MonitorHost(Monitor):
    """Ping a host to make sure it's up"""

    host = ""
    ping_command = ""
    ping_regexp = ""
    type = "host"
    time_regexp = ""

    def __init__(self, name, config_options):
        """
        Note: We use -w/-t on Windows/POSIX to limit the amount of time we wait to 5 seconds.
        This is to stop ping holding things up too much. A machine that can't ping back in <5s is
        a machine in trouble anyway, so should probably count as a failure.
        """
        Monitor.__init__(self, name, config_options)
        if self.is_windows(allow_cygwin=True):
            self.ping_command = "ping -n 1 -w 5000 %s"
            self.ping_regexp = "Reply from "
            self.time_regexp = "Average = (?P<ms>\d+)ms"
        else:
            try:
                ping_ttl = config_options["ping_ttl"]
            except:
                ping_ttl = "5"
            self.ping_command = "ping -c1 -t"+ ping_ttl + " %s 2> /dev/null"
            self.ping_regexp = "bytes from"
            #XXX this regexp is only for freebsd at the moment; not sure about other platforms
            #XXX looks like Linux uses this format too
            self.time_regexp = "min/avg/max/stddev = [\d.]+/(?P<ms>[\d.]+)/"
        try:
            host = config_options["host"]
        except:
            raise RuntimeError("Required configuration fields missing")
        if host == "":
            raise RuntimeError("missing hostname")
        self.host = host

    def run_test(self):
        r = re.compile(self.ping_regexp)
        r2 = re.compile(self.time_regexp)
        success = False
        pingtime = 0.0
        try:
            process_handle = os.popen(self.ping_command % self.host)
            for line in process_handle:
                matches = r.search(line)
                if matches:
                    success = True
                else:
                    matches = r2.search(line)
                    if matches:
                        pingtime = matches.group("ms")
        except Exception, e:
            self.record_fail(e)
            pass
        if success:
            if pingtime > 0:
                self.record_success("%sms" % pingtime)
            else:
                self.record_success()
            return True
        self.record_fail()
        return False

    def describe(self):
        """Explains what this instance is checking"""
        return "checking host %s is pingable" % self.host

    def get_params(self):
        return (self.host, )

<|endoftext|>