<commit_before>#!/usr/bin/env python
"""
.. module:: notebook
   :synopsis: The Notebook class and related classes.
"""

import glob
import os
import uuid

from xml.etree import ElementTree

from PyQt4 import QtCore, QtGui

import notebook_chooser_ui as notebookChooserUi

import logger
import misc
import note

@logger.logFn
def chooseNotebook(notebook_mvc, a_notebook = None):
    """
    Prompts the user to choose a NotebookStandardItem from either:
      (1) The currently selected NotebookStandardItems or
      (2) All of the the NotebookStandardItems if none are selected.

    If only one NotebookStandardItem is selected that will be returned
    immediately without opening a dialog.

    Returns the choosen NotebookStandardItem (or None).
    """
    if a_notebook is None:
        notebooks = notebook_mvc.getSelectedNotebooks()
    else:
        notebooks = notebook_mvc.getAllNotebooks()
        notebooks.remove(a_notebook)

    if (len(notebooks) == 0):
        notebooks = notebook_mvc.getAllNotebooks()
    if (len(notebooks) == 1):
        return notebooks[0]
    else:
        dialog = NotebookChooser(notebooks)
        if dialog.exec_():
            return dialog.choosen
        else:
            return None

class NotebookChooser(QtGui.QDialog):
    """
    Dialog for choosing a NotebookStandardItem from a list of NotebookStandardItems.
    """
    @logger.logFn
    def __init__(self, notebook_items = [], parent = None):
        QtGui.QDialog.__init__(self, parent)
        self.choosen = None

        self.ui = notebookChooserUi.Ui_Dialog()
        self.ui.setupUi(self)
        self.setWindowTitle("Notebook Chooser")

        self.model = QtGui.QStandardItemModel()
        self.proxy_model = QtGui.QSortFilterProxyModel()
        self.proxy_model.setSourceModel(self.model)
        self.ui.chooserListView.setModel(self.proxy_model)

        #
        # Hm, apparently we can't use the same items in two different models,
        # so we have to do some fiddling..
        #
        for nb in notebook_items:
            item = QtGui.QStandardItem(nb.getName())
            item.notebook_item = nb
            self.model.appendRow(item)

        self.proxy_model.sort(0)

        self.ui.chooserListView.clicked.connect(self.handleClick)
        self.ui.chooserListView.doubleClicked.connect(self.handleDoubleClick)

    @logger.logFn
    def handleClick(self, index):
        source_index = self.proxy_model.mapToSource(index)
        self.choosen = self.model.itemFromIndex(source_index).notebook_item

    @logger.logFn
    def handleDoubleClick(self, index):
        self.handleClick(index)
        self.accept()

class NotebookListViewDelegate(QtGui.QStyledItemDelegate):
    """
    A custom look for each notebook item.
    """
    def __init__(self, model, proxy_model):
        QtGui.QStyledItemDelegate.__init__(self)
        self.model = model
        self.proxy_model = proxy_model

    def itemFromProxyIndex(self, proxy_index):
        source_index = self.proxy_model.mapToSource(proxy_index)
        return self.model.itemFromIndex(source_index)

    def paint(self, painter, option, index):
        notebook = self.itemFromProxyIndex(index)

        # Draw correct background.
        style = option.widget.style()
        style.drawControl(QtGui.QStyle.CE_ItemViewItem, option, painter, option.widget)

        # Draw text.
        #if notebook.getUnpushed():
        #    painter.setPen(QtGui.QColor(100,0,0))
        #else:
        #    painter.setPen(QtGui.QColor(0,100,0))
        if notebook.getUnpushed():
            painter.drawText(option.rect, QtCore.Qt.AlignLeft, " " + notebook.getName() + "*")
        else:
            painter.drawText(option.rect, QtCore.Qt.AlignLeft, " " + notebook.getName())
        painter.drawText(option.rect, QtCore.Qt.AlignRight, "(" + str(notebook.getNumberNotes()) + " notes) ")

class NotebookMVC(QtGui.QListView):
    """
    Encapsulates a list view specialized for notebooks and it's associated model.
    """
    addNewNote = QtCore.pyqtSignal(object)
    addNewNotebook = QtCore.pyqtSignal()
    selectedNotebooksChanged = QtCore.pyqtSignal(list)

    @logger.logFn
    def __init__(self, parent = None):
        QtGui.QListView.__init__(self, parent)
        self.right_clicked = None

        self.setSelectionMode(QtGui.QAbstractItemView.ExtendedSelection)

        # Context menu
        self.addNoteAction = QtGui.QAction(self.tr("Add Note"), self)
        self.addNoteAction.triggered.connect(self.handleAddNewNote)
        self.addNotebookAction = QtGui.QAction(self.tr("New Notebook"), self)
        self.addNotebookAction.triggered.connect(self.handleAddNewNotebook)
        self.deleteAction = QtGui.QAction(self.tr("Delete Notebook"), self)
        self.deleteAction.triggered.connect(self.handleDelete)
        self.renameNotebookAction = QtGui.QAction(self.tr("Rename"), self)
        self.renameNotebookAction.triggered.connect(self.handleRenameNotebook)
        self.syncNotebookAction = QtGui.QAction(self.tr("Sync"), self)
        self.syncNotebookAction.triggered.connect(self.handleSyncNotebook)

        self.nb_popup_menu = QtGui.QMenu(self)
        self.nb_popup_menu.addAction(self.addNoteAction)
        self.nb_popup_menu.addAction(self.addNotebookAction)
        self.nb_popup_menu.addAction(self.deleteAction)
        self.nb_popup_menu.addAction(self.renameNotebookAction)
        self.nb_popup_menu.addAction(self.syncNotebookAction)
        self.no_nb_popup_menu = QtGui.QMenu(self)
        self.no_nb_popup_menu.addAction(self.addNotebookAction)

        # Notebook model
        self.notebook_model = NotebookStandardItemModel(self)
        self.notebook_proxy_model = NotebookSortFilterProxyModel(self)
        self.notebook_proxy_model.setSourceModel(self.notebook_model)
        self.setModel(self.notebook_proxy_model)

        # Rendering
        self.setItemDelegate(NotebookListViewDelegate(self.notebook_model, self.notebook_proxy_model))

        # Get selection changes.
        self.selectionModel().selectionChanged.connect(self.handleSelectionChange)

    @logger.logFn
    def addNotebook(self, directory, notebook_name, username, email):
        nb = NotebookStandardItem(directory)
        nb.createWithName(notebook_name, username, email)
        self.notebook_model.appendRow(nb)
        self.notebook_proxy_model.sort(0)

    @logger.logFn
    def clearNotebooks(self):
        self.notebook_model.clear()

    @logger.logFn
    def getAllNotebooks(self):
        all_notebooks = []
        for row in range(self.notebook_model.rowCount()):
            index = self.notebook_model.index(row, 0)
            all_notebooks.append(self.notebook_model.itemFromIndex(index))
        return all_notebooks

    @logger.logFn
    def getSelectedNotebooks(self):
        selected_notebooks = []
        for index in self.selectedIndexes():
            selected_notebooks.append(self.notebookFromProxyIndex(index))
        return selected_notebooks

    @logger.logFn
    def handleAddNewNote(self, boolean):
        self.addNewNote.emit(self.notebookFromProxyIndex(self.right_clicked))

    @logger.logFn
    def handleAddNewNotebook(self, boolean):
        self.addNewNotebook.emit()

    @logger.logFn
    def handleDelete(self, boolean):
        notebook = self.notebookFromProxyIndex(self.right_clicked)
        notebook_name = notebook.getName()

        reply = QtGui.QMessageBox.question(self,
                                           "Warning!",
                                           "Really delete notebook '" + notebook_name + "'?",
                                           QtGui.QMessageBox.Yes,
                                           QtGui.QMessageBox.No)
        if (reply == QtGui.QMessageBox.Yes):
            source_index = self.notebook_proxy_model.mapToSource(self.right_clicked)
            self.notebook_model.removeRow(source_index.row())

            # This just renames the notebook.xml file to deleted.xml so that
            # it won't be found the next time the program is started.
            notebook.deleteNotebook()

    @logger.logFn
    def handleRenameNotebook(self, boolean):
        notebook = self.notebookFromProxyIndex(self.right_clicked)
        notebook_name = notebook.getName()
        [new_name, ok] = QtGui.QInputDialog.getText(self,
                                                    'Rename Notebook',
                                                    'Enter a new name:',
                                                    text = notebook_name)
        if ok:
            notebook.rename(new_name)
            self.notebook_proxy_model.sort(0)

    @logger.logFn
    def handleSelectionChange(self, new_item_selection, old_item_selection):
        selected_notebooks = self.getSelectedNotebooks()
        if (len(selected_notebooks) > 0):
            self.selectedNotebooksChanged.emit(selected_notebooks)

    @logger.logFn
    def handleSyncNotebook(self, boolean):
        notebook = self.notebookFromProxyIndex(self.right_clicked)
        misc.gitSync(notebook.getDirectory())
        notebook.setUnpushed(False)

    @logger.logFn
    def loadNotebooks(self, directory):
        self.clearNotebooks()

        for nb_id in map(lambda(x): x[len(directory) + 3:], glob.glob(directory + "nb_*")):
            nb = NotebookStandardItem(directory)
            if nb.loadWithUUID(nb_id):
                self.notebook_model.appendRow(nb)

        self.notebook_proxy_model.sort(0)

    @logger.logFn
    def mousePressEvent(self, event):
        if (event.button() == QtCore.Qt.RightButton):
            self.right_clicked = self.indexAt(event.pos())
            if (self.right_clicked.row() > -1):
                self.nb_popup_menu.exec_(event.globalPos())
            else:
                self.no_nb_popup_menu.exec_(event.globalPos())
        else:
            QtGui.QListView.mousePressEvent(self, event)

    @logger.logFn
    def notebookFromProxyIndex(self, proxy_index):
        source_index = self.notebook_proxy_model.mapToSource(proxy_index)
        return self.notebook_model.itemFromIndex(source_index)

class NotebookSortFilterProxyModel(QtGui.QSortFilterProxyModel):
    """
    Sort notebooks.
    """

class NotebookStandardItem(QtGui.QStandardItem):
    """
    A single notebook in the notebook listview model.
    """
    @logger.logFn
    def __init__(self, directory):
        """
        After creating the item you need to enter the appropriate initial
        data using one of createWithName() or loadWithUUID.
        """
        QtGui.QStandardItem.__init__(self, "NA")

        self.directory = directory + "nb_"
        self.git_log = []
        self.has_unpushed = False
        self.name = None
        self.number_notes = 0
        self.number_unsaved = 0
        self.uuid = None

    @logger.logFn
    def createWithName(self, notebook_name, username, email):
        self.name = notebook_name
        self.uuid = str(uuid.uuid1())
        self.setText(self.name)

        self.directory += self.uuid
        os.makedirs(self.directory)

        xml = ElementTree.Element("notebook")
        name_xml = ElementTree.SubElement(xml, "name")
        name_xml.text = self.name

        self.directory += "/"
        misc.pSaveXML(self.directory + "notebook.xml", xml)

        # Create a new git repository for this notebook.
        misc.gitInit(self.directory, username, email)

        # Commit the notebook name.
        misc.gitAddCommit(self.directory, self.directory + "notebook.xml", "add notebook.")

        self.has_unpushed = True
        self.setToolTip(self.directory)

    @logger.logFn
    def deleteNotebook(self):
        os.rename(self.directory + "notebook.xml", self.directory + "deleted.xml")

    @logger.logFn
    def decNumberUnsaved(self):
        self.number_unsaved -= 1
        if (self.number_unsaved == 0):
            self.setForeground(QtGui.QBrush(QtGui.QColor(0,0,0)))

    @logger.logFn
    def getDirectory(self):
        return self.directory

    @logger.logFn
    def getName(self):
        return self.name

    @logger.logFn
    def getNoteVersions(self, note_filename):
        versions = []
        for commit in self.git_log:
            if (commit[2] == note_filename):
                versions.append(commit[0])
        return list(reversed(versions))

    def getNumberNotes(self):
        return self.number_notes

    @logger.logFn
    def getUnpushed(self):
        return self.has_unpushed

    @logger.logFn
    def incNumberNotes(self, inc):
        self.number_notes += inc
        self.emitDataChanged()

    @logger.logFn
    def incNumberUnsaved(self):
        self.number_unsaved += 1
        self.setForeground(QtGui.QBrush(QtGui.QColor(100,0,0)))

    @logger.logFn
    def loadWithUUID(self, notebook_uuid):
        self.uuid = notebook_uuid
        self.directory = self.directory + self.uuid + "/"
        if os.path.exists(self.directory + "notebook.xml"):
            xml = ElementTree.parse(self.directory + "notebook.xml").getroot()
            self.name = xml.find("name").text
            self.setText(self.name)

            self.git_log = misc.gitGetLog(self.directory)
            self.has_unpushed = misc.gitHasUnpushed(self.directory)
            self.setToolTip(self.directory)

            return True
        else:
            return False

    @logger.logFn
    def rename(self, new_name):
        self.name = str(new_name)
        self.setText(new_name)

        # Update the XML and commit.
        xml = ElementTree.Element("notebook")
        name_xml = ElementTree.SubElement(xml, "name")
        name_xml.text = self.name

        misc.pSaveXML(self.directory + "notebook.xml", xml)
        misc.gitAddCommit(self.directory, self.directory + "notebook.xml", "rename notebook.")

        self.setUnpushed()

    @logger.logFn
    def setNumberNotes(self, number_notes):
        self.number_notes = number_notes

    @logger.logFn
    def setUnpushed(self, unpushed = True):
        self.has_unpushed = unpushed
        self.emitDataChanged()

class NotebookStandardItemModel(QtGui.QStandardItemModel):
    """
    The notebook listview model.
    """

<commit_msg>Add a function to copy the notebook directory name to the clipboard. This makes it easier to create remote repositories.<commit_after>#!/usr/bin/env python
"""
.. module:: notebook
   :synopsis: The Notebook class and related classes.
"""

import glob
import os
import uuid

from xml.etree import ElementTree

from PyQt4 import QtCore, QtGui

import notebook_chooser_ui as notebookChooserUi

import logger
import misc
import note

@logger.logFn
def chooseNotebook(notebook_mvc, a_notebook = None):
    """
    Prompts the user to choose a NotebookStandardItem from either:
      (1) The currently selected NotebookStandardItems or
      (2) All of the the NotebookStandardItems if none are selected.

    If only one NotebookStandardItem is selected that will be returned
    immediately without opening a dialog.

    Returns the choosen NotebookStandardItem (or None).
    """
    if a_notebook is None:
        notebooks = notebook_mvc.getSelectedNotebooks()
    else:
        notebooks = notebook_mvc.getAllNotebooks()
        notebooks.remove(a_notebook)

    if (len(notebooks) == 0):
        notebooks = notebook_mvc.getAllNotebooks()
    if (len(notebooks) == 1):
        return notebooks[0]
    else:
        dialog = NotebookChooser(notebooks)
        if dialog.exec_():
            return dialog.choosen
        else:
            return None

class NotebookChooser(QtGui.QDialog):
    """
    Dialog for choosing a NotebookStandardItem from a list of NotebookStandardItems.
    """
    @logger.logFn
    def __init__(self, notebook_items = [], parent = None):
        QtGui.QDialog.__init__(self, parent)
        self.choosen = None

        self.ui = notebookChooserUi.Ui_Dialog()
        self.ui.setupUi(self)
        self.setWindowTitle("Notebook Chooser")

        self.model = QtGui.QStandardItemModel()
        self.proxy_model = QtGui.QSortFilterProxyModel()
        self.proxy_model.setSourceModel(self.model)
        self.ui.chooserListView.setModel(self.proxy_model)

        #
        # Hm, apparently we can't use the same items in two different models,
        # so we have to do some fiddling..
        #
        for nb in notebook_items:
            item = QtGui.QStandardItem(nb.getName())
            item.notebook_item = nb
            self.model.appendRow(item)

        self.proxy_model.sort(0)

        self.ui.chooserListView.clicked.connect(self.handleClick)
        self.ui.chooserListView.doubleClicked.connect(self.handleDoubleClick)

    @logger.logFn
    def handleClick(self, index):
        source_index = self.proxy_model.mapToSource(index)
        self.choosen = self.model.itemFromIndex(source_index).notebook_item

    @logger.logFn
    def handleDoubleClick(self, index):
        self.handleClick(index)
        self.accept()

class NotebookListViewDelegate(QtGui.QStyledItemDelegate):
    """
    A custom look for each notebook item.
    """
    def __init__(self, model, proxy_model):
        QtGui.QStyledItemDelegate.__init__(self)
        self.model = model
        self.proxy_model = proxy_model

    def itemFromProxyIndex(self, proxy_index):
        source_index = self.proxy_model.mapToSource(proxy_index)
        return self.model.itemFromIndex(source_index)

    def paint(self, painter, option, index):
        notebook = self.itemFromProxyIndex(index)

        # Draw correct background.
        style = option.widget.style()
        style.drawControl(QtGui.QStyle.CE_ItemViewItem, option, painter, option.widget)

        # Draw text.
        #if notebook.getUnpushed():
        #    painter.setPen(QtGui.QColor(100,0,0))
        #else:
        #    painter.setPen(QtGui.QColor(0,100,0))
        if notebook.getUnpushed():
            painter.drawText(option.rect, QtCore.Qt.AlignLeft, " " + notebook.getName() + "*")
        else:
            painter.drawText(option.rect, QtCore.Qt.AlignLeft, " " + notebook.getName())
        painter.drawText(option.rect, QtCore.Qt.AlignRight, "(" + str(notebook.getNumberNotes()) + " notes) ")

class NotebookMVC(QtGui.QListView):
    """
    Encapsulates a list view specialized for notebooks and it's associated model.
    """
    addNewNote = QtCore.pyqtSignal(object)
    addNewNotebook = QtCore.pyqtSignal()
    selectedNotebooksChanged = QtCore.pyqtSignal(list)

    @logger.logFn
    def __init__(self, parent = None):
        QtGui.QListView.__init__(self, parent)
        self.right_clicked = None

        self.setSelectionMode(QtGui.QAbstractItemView.ExtendedSelection)

        # Context menu
        self.addNoteAction = QtGui.QAction(self.tr("Add Note"), self)
        self.addNoteAction.triggered.connect(self.handleAddNewNote)
        self.addNotebookAction = QtGui.QAction(self.tr("New Notebook"), self)
        self.addNotebookAction.triggered.connect(self.handleAddNewNotebook)
        self.copyRepoNameAction = QtGui.QAction(self.tr("Copy Directory Name to Clipboard"), self)
        self.copyRepoNameAction.triggered.connect(self.handleCopyRepoName)
        self.deleteAction = QtGui.QAction(self.tr("Delete Notebook"), self)
        self.deleteAction.triggered.connect(self.handleDelete)
        self.renameNotebookAction = QtGui.QAction(self.tr("Rename"), self)
        self.renameNotebookAction.triggered.connect(self.handleRenameNotebook)
        self.syncNotebookAction = QtGui.QAction(self.tr("Sync"), self)
        self.syncNotebookAction.triggered.connect(self.handleSyncNotebook)

        self.nb_popup_menu = QtGui.QMenu(self)
        self.nb_popup_menu.addAction(self.addNoteAction)
        self.nb_popup_menu.addAction(self.addNotebookAction)
        self.nb_popup_menu.addAction(self.copyRepoNameAction)
        self.nb_popup_menu.addAction(self.deleteAction)
        self.nb_popup_menu.addAction(self.renameNotebookAction)
        self.nb_popup_menu.addAction(self.syncNotebookAction)
        self.no_nb_popup_menu = QtGui.QMenu(self)
        self.no_nb_popup_menu.addAction(self.addNotebookAction)

        # Notebook model
        self.notebook_model = NotebookStandardItemModel(self)
        self.notebook_proxy_model = NotebookSortFilterProxyModel(self)
        self.notebook_proxy_model.setSourceModel(self.notebook_model)
        self.setModel(self.notebook_proxy_model)

        # Rendering
        self.setItemDelegate(NotebookListViewDelegate(self.notebook_model, self.notebook_proxy_model))

        # Get selection changes.
        self.selectionModel().selectionChanged.connect(self.handleSelectionChange)

    @logger.logFn
    def addNotebook(self, directory, notebook_name, username, email):
        nb = NotebookStandardItem(directory)
        nb.createWithName(notebook_name, username, email)
        self.notebook_model.appendRow(nb)
        self.notebook_proxy_model.sort(0)

    @logger.logFn
    def clearNotebooks(self):
        self.notebook_model.clear()

    @logger.logFn
    def getAllNotebooks(self):
        all_notebooks = []
        for row in range(self.notebook_model.rowCount()):
            index = self.notebook_model.index(row, 0)
            all_notebooks.append(self.notebook_model.itemFromIndex(index))
        return all_notebooks

    @logger.logFn
    def getSelectedNotebooks(self):
        selected_notebooks = []
        for index in self.selectedIndexes():
            selected_notebooks.append(self.notebookFromProxyIndex(index))
        return selected_notebooks

    @logger.logFn
    def handleAddNewNote(self, boolean):
        self.addNewNote.emit(self.notebookFromProxyIndex(self.right_clicked))

    @logger.logFn
    def handleAddNewNotebook(self, boolean):
        self.addNewNotebook.emit()

    @logger.logFn
    def handleCopyRepoName(self, boolean):
        clipboard = QtGui.QApplication.clipboard()
        a_notebook = self.notebookFromProxyIndex(self.right_clicked)
        clipboard.setText(os.path.basename(a_notebook.getDirectory()[:-1]))

    @logger.logFn
    def handleDelete(self, boolean):
        notebook = self.notebookFromProxyIndex(self.right_clicked)
        notebook_name = notebook.getName()

        reply = QtGui.QMessageBox.question(self,
                                           "Warning!",
                                           "Really delete notebook '" + notebook_name + "'?",
                                           QtGui.QMessageBox.Yes,
                                           QtGui.QMessageBox.No)
        if (reply == QtGui.QMessageBox.Yes):
            source_index = self.notebook_proxy_model.mapToSource(self.right_clicked)
            self.notebook_model.removeRow(source_index.row())

            # This just renames the notebook.xml file to deleted.xml so that
            # it won't be found the next time the program is started.
            notebook.deleteNotebook()

    @logger.logFn
    def handleRenameNotebook(self, boolean):
        notebook = self.notebookFromProxyIndex(self.right_clicked)
        notebook_name = notebook.getName()
        [new_name, ok] = QtGui.QInputDialog.getText(self,
                                                    'Rename Notebook',
                                                    'Enter a new name:',
                                                    text = notebook_name)
        if ok:
            notebook.rename(new_name)
            self.notebook_proxy_model.sort(0)

    @logger.logFn
    def handleSelectionChange(self, new_item_selection, old_item_selection):
        selected_notebooks = self.getSelectedNotebooks()
        if (len(selected_notebooks) > 0):
            self.selectedNotebooksChanged.emit(selected_notebooks)

    @logger.logFn
    def handleSyncNotebook(self, boolean):
        notebook = self.notebookFromProxyIndex(self.right_clicked)
        misc.gitSync(notebook.getDirectory())
        notebook.setUnpushed(False)

    @logger.logFn
    def loadNotebooks(self, directory):
        self.clearNotebooks()

        for nb_id in map(lambda(x): x[len(directory) + 3:], glob.glob(directory + "nb_*")):
            nb = NotebookStandardItem(directory)
            if nb.loadWithUUID(nb_id):
                self.notebook_model.appendRow(nb)

        self.notebook_proxy_model.sort(0)

    @logger.logFn
    def mousePressEvent(self, event):
        if (event.button() == QtCore.Qt.RightButton):
            self.right_clicked = self.indexAt(event.pos())
            if (self.right_clicked.row() > -1):
                self.nb_popup_menu.exec_(event.globalPos())
            else:
                self.no_nb_popup_menu.exec_(event.globalPos())
        else:
            QtGui.QListView.mousePressEvent(self, event)

    @logger.logFn
    def notebookFromProxyIndex(self, proxy_index):
        source_index = self.notebook_proxy_model.mapToSource(proxy_index)
        return self.notebook_model.itemFromIndex(source_index)

class NotebookSortFilterProxyModel(QtGui.QSortFilterProxyModel):
    """
    Sort notebooks.
    """

class NotebookStandardItem(QtGui.QStandardItem):
    """
    A single notebook in the notebook listview model.
    """
    @logger.logFn
    def __init__(self, directory):
        """
        After creating the item you need to enter the appropriate initial
        data using one of createWithName() or loadWithUUID.
        """
        QtGui.QStandardItem.__init__(self, "NA")

        self.directory = directory + "nb_"
        self.git_log = []
        self.has_unpushed = False
        self.name = None
        self.number_notes = 0
        self.number_unsaved = 0
        self.uuid = None

    @logger.logFn
    def createWithName(self, notebook_name, username, email):
        self.name = notebook_name
        self.uuid = str(uuid.uuid1())
        self.setText(self.name)

        self.directory += self.uuid
        os.makedirs(self.directory)

        xml = ElementTree.Element("notebook")
        name_xml = ElementTree.SubElement(xml, "name")
        name_xml.text = self.name

        self.directory += "/"
        misc.pSaveXML(self.directory + "notebook.xml", xml)

        # Create a new git repository for this notebook.
        misc.gitInit(self.directory, username, email)

        # Commit the notebook name.
        misc.gitAddCommit(self.directory, self.directory + "notebook.xml", "add notebook.")

        self.has_unpushed = True
        self.setToolTip(self.directory)

    @logger.logFn
    def deleteNotebook(self):
        os.rename(self.directory + "notebook.xml", self.directory + "deleted.xml")

    @logger.logFn
    def decNumberUnsaved(self):
        self.number_unsaved -= 1
        if (self.number_unsaved == 0):
            self.setForeground(QtGui.QBrush(QtGui.QColor(0,0,0)))

    @logger.logFn
    def getDirectory(self):
        return self.directory

    @logger.logFn
    def getName(self):
        return self.name

    @logger.logFn
    def getNoteVersions(self, note_filename):
        versions = []
        for commit in self.git_log:
            if (commit[2] == note_filename):
                versions.append(commit[0])
        return list(reversed(versions))

    def getNumberNotes(self):
        return self.number_notes

    @logger.logFn
    def getUnpushed(self):
        return self.has_unpushed

    @logger.logFn
    def incNumberNotes(self, inc):
        self.number_notes += inc
        self.emitDataChanged()

    @logger.logFn
    def incNumberUnsaved(self):
        self.number_unsaved += 1
        self.setForeground(QtGui.QBrush(QtGui.QColor(100,0,0)))

    @logger.logFn
    def loadWithUUID(self, notebook_uuid):
        self.uuid = notebook_uuid
        self.directory = self.directory + self.uuid + "/"
        if os.path.exists(self.directory + "notebook.xml"):
            xml = ElementTree.parse(self.directory + "notebook.xml").getroot()
            self.name = xml.find("name").text
            self.setText(self.name)

            self.git_log = misc.gitGetLog(self.directory)
            self.has_unpushed = misc.gitHasUnpushed(self.directory)
            self.setToolTip(self.directory)

            return True
        else:
            return False

    @logger.logFn
    def rename(self, new_name):
        self.name = str(new_name)
        self.setText(new_name)

        # Update the XML and commit.
        xml = ElementTree.Element("notebook")
        name_xml = ElementTree.SubElement(xml, "name")
        name_xml.text = self.name

        misc.pSaveXML(self.directory + "notebook.xml", xml)
        misc.gitAddCommit(self.directory, self.directory + "notebook.xml", "rename notebook.")

        self.setUnpushed()

    @logger.logFn
    def setNumberNotes(self, number_notes):
        self.number_notes = number_notes

    @logger.logFn
    def setUnpushed(self, unpushed = True):
        self.has_unpushed = unpushed
        self.emitDataChanged()

class NotebookStandardItemModel(QtGui.QStandardItemModel):
    """
    The notebook listview model.
    """

<|endoftext|>


<commit_before>from __future__ import absolute_import, unicode_literals

from django.db import models

class MenuItemManager(models.Manager):
    ''' App-specific manager overrides '''

    def for_display(self):
        return self.filter(
            models.Q(link_page__isnull=True) |
            models.Q(link_page__live=True) &
            models.Q(link_page__expired=False) &
            models.Q(link_page__show_in_menus=True)
        ).select_related('link_page')

    def page_links(self):
        return self.filter(link_page__isnull=False)

    def page_links_for_display(self):
        return self.page_links().filter(
            link_page__live=True,
            link_page__expired=False,
            link_page__show_in_menus=True)
<commit_msg>Don't prefetch pages in `for_display()`<commit_after>from __future__ import absolute_import, unicode_literals

from django.db import models

class MenuItemManager(models.Manager):
    ''' App-specific manager overrides '''

    def for_display(self):
        return self.filter(
            models.Q(link_page__isnull=True) |
            models.Q(link_page__live=True) &
            models.Q(link_page__expired=False) &
            models.Q(link_page__show_in_menus=True)
        )

    def page_links(self):
        return self.filter(link_page__isnull=False)

    def page_links_for_display(self):
        return self.page_links().filter(
            link_page__live=True,
            link_page__expired=False,
            link_page__show_in_menus=True)
<|endoftext|>


<commit_before>#!/usr/local/bin/python2.7

'''
Cluster Doctor Fabric File

Check configuration & status of all nodes in a cluster.
'''

__author__    = 'Dongjoon Hyun (dongjoon@apache.org)'
__license__   = 'Apache License'
__version__   = '0.1'

from fabric.api import *

env.roledefs = {}
nn = []
for i in range(95,99):
	nn.append('50.1.100.'+str(i))
env.roledefs['nn'] = nn
dn = []
for i in range(101,131):
	dn.append('50.1.100.'+str(i))
env.roledefs['dn'] = dn

print "\n\n", "-"*80, "\n\n# of IPs: ", len(nn + dn)

env.warn_only = True
env.skip_bad_hosts = True
env.colorize_errors = True
env.use_ssh_config = True
env.user = 'root'
env.key_filename = '~/.ssh/GPU.pem'
env.timeout = 3
env.parallel = True
output['status'] = False
output['stdout'] = True
output['warnings'] = False
output['running'] = False

@roles('nn','dn')
def os():
	'''
	OS Version: CentOS release 6.5
	'''
	cmd = "/usr/bin/lsb_release -r | cut -d '\t' -f 2"
	expected = '6.5'
	run('''[ "`%(cmd)s`" != "%(expected)s" ] && (echo "Expected: %(expected)s, Actual: `%(cmd)s`")''' % locals())

@roles('nn','dn')
def os_info():
	'''
	Display OS Version
	'''
	cmd = '/usr/bin/lsb_release -r'
	run(cmd)

@roles('nn','dn')
def kernel():
	'''
	Kernel Version: 2.6.32-431.el6.x86_64
	'''
	cmd = 'uname -r | tr -d [:space:]'
	expected = '2.6.32-431.el6.x86_64'
	run('''[ "`%(cmd)s`" != "%(expected)s" ] && (echo "Expected: %(expected)s, Actual: `%(cmd)s`")''' % locals())

@roles('nn','dn')
def selinux():
	'''
	SELINUX : disabled
	'''
	cmd = "cat /etc/sysconfig/selinux | grep 'SELINUX=disabled'"
	expected = 'SELINUX=disabled'
	run('''[ "`%(cmd)s`" != "%(expected)s" ] && (echo "Expected: %(expected)s, Actual: `%(cmd)s`")''' % locals())

@roles('nn','dn')
def sys():
	'''
	Sys Parameter: /proc/sys/kernel/hung_task_timeout_secs == 0
	'''
	cmd = 'cat /proc/sys/kernel/hung_task_timeout_secs'
	expected = '0'
	run('''[ "`%(cmd)s`" != "%(expected)s" ] && (echo "Expected: %(expected)s, Actual: `%(cmd)s`")''' % locals())

	cmd = 'grep hung_task_timeout_secs /etc/rc.local | wc -l'
	expected = '2'
	run('''[ "`%(cmd)s`" != "%(expected)s" ] && (echo "Expected: %(expected)s, Actual: `%(cmd)s`")''' % locals())

@roles('nn','dn')
def os_hugepage_enabled():
	'''
	OS Parameter: /sys/kernel/mm/redhat_transparent_hugepage/enabled == always [never]
	'''
	cmd = 'cat /sys/kernel/mm/redhat_transparent_hugepage/enabled'
	expected = 'always madvise [never]'
	run('''[ "`%(cmd)s`" != "%(expected)s" ] && (echo "Expected: %(expected)s, Actual: `%(cmd)s`")''' % locals())

	cmd = 'grep enabled /etc/rc.local | wc -l'
	expected = '2'
	run('''[ "`%(cmd)s`" != "%(expected)s" ] && (echo "Expected: %(expected)s, Actual: `%(cmd)s`")''' % locals())

@roles('nn','dn')
def os_hugepage_defrag():
	'''
	OS Parameter: cat /sys/kernel/mm/redhat_transparent_hugepage/defrag
	'''
	cmd = 'cat /sys/kernel/mm/redhat_transparent_hugepage/defrag'
	expected = 'always madvise [never]'
	run('''[ "`%(cmd)s`" != "%(expected)s" ] && (echo "Expected: %(expected)s, Actual: `%(cmd)s`")''' % locals())

	cmd = 'grep defrag /etc/rc.local | wc -l'
	expected = '2'
	run('''[ "`%(cmd)s`" != "%(expected)s" ] && (echo "Expected: %(expected)s, Actual: `%(cmd)s`")''' % locals())

@roles('nn','dn')
def date():
	'''
	System Time: time gap < 1 minute
	'''
	from datetime import datetime
	cmd = 'date +%Y-%m-%dT%H:%M'
	expected = datetime.now().isoformat()[0:16]
	run('''[ "`%(cmd)s`" != "%(expected)s" ] && (echo "Expected: %(expected)s, Actual: `%(cmd)s`")''' % locals())

@roles('nn','dn')
def service_gmond():
	'''
	gmond Installation: chkconfig --list | grep gmond
	'''
	cmd = 'chkconfig --list|grep gmond | wc -l'
	expected = '1'
	run('''[ "`%(cmd)s`" != "%(expected)s" ] && (echo "Expected: %(expected)s, Actual: `%(cmd)s`")''' % locals())

@roles('nn','dn')
def service_gmond_on():
	'''
	gmond Running: telnet $host 8649
	'''
	cmd = 'telnet %s 8649 | grep Analysis | wc -l' % (env.host)
	expected = '1'
	run('''[ "`%(cmd)s`" != "%(expected)s" ] && (echo "Expected: %(expected)s, Actual: `%(cmd)s`")''' % locals())

@roles('nn','dn')
def disk():
	'''
	#, Usage of Disks (TODO): remaining space > 20%
	'''
	run("df -a -h | awk '{print $5}'")

@roles('nn','dn')
def disk_info():
	'''
	Display Disk Info
	'''
	cmd = "fdisk -l 2> /dev/null | grep '^Disk /dev/sd' | awk '{sum+=$3} END {print total,sum/1024}'"
	run(cmd)

	cmd = "fdisk -l 2> /dev/null | grep '^Disk /dev/sd' | awk '{print $2,$3}' | sort"
	run(cmd)

@roles('dn')
def disk_size():
	'''
	Size: /data* should be '1171964464'
	'''
	cmd = "df | grep data | awk '{print $2}' | grep -v 1171964464"
	run(cmd)

@roles('dn')
def disk_ext4():
	'''
	noatime: /data* should have 'ext4'
	'''
	run("grep '/data' /etc/fstab | grep -v ext4")

@roles('dn')
def disk_noatime():
	'''
	noatime: /data* should have 'noatime'
	'''
	run("grep '/data' /etc/fstab | grep -v noatime")

@roles('nn','dn')
def hadoop():
	'''
	Hadoop: /usr/lib/hadoop
	'''
	cmd = '[ -d /usr/lib/hadoop ] && echo 1'
	expected = '1'
	run('''[ "`%(cmd)s`" != "%(expected)s" ] && (echo "Expected: %(expected)s, Actual: `%(cmd)s`")''' % locals())

@roles('nn','dn')
def sensors():
	'''
	Sensors : Chip `Intel digital thermal sensor' (confidence: 9)
		  Chip `IPMI BMC KCS' (confidence: 8)
	'''
	cmd = "sensors | awk '{print $3}' | grep '+[6-9]'"
	run(cmd)

@roles('nn','dn')
def fan():
	'''
	Fan Speed
	'''
	cmd = "ipmi-sensors | grep Fan | grep RPM"
	run(cmd)

@roles('nn','dn')
def netstat():
	'''
	Task Tracker CLOSE_WAIT Bug Check
	'''
	cmd = "netstat -n | grep CLOSE_WAIT | wc -l | grep '[0-9][0-9][0-9][0-9]'"
	run(cmd)

@roles('nn','dn')
def overruns():
	'''
	Network Packket Overrun Check
	'''
	cmd = "ifconfig bond0 | grep overruns | grep -v 'overruns:0'"
	run(cmd)

@roles('nn','dn')
def errors():
	'''
	Network Packket Overrun Check
	'''
	cmd = "ifconfig bond0 | grep errors | grep -v 'errors:0'"
	run(cmd)

@roles('nn','dn')
def mtu():
	'''
	Network MTU Check
	'''
	cmd = "ifconfig bond0 | grep MTU | grep -v 1500"
	run(cmd)

@roles('nn','dn')
def serial_info():
	'''
	Display Serial Number
	'''
	cmd = "dmidecode | grep 'Serial Number:' | head -n 1 | awk '{print $3}'"
	run(cmd)

@roles('nn','dn')
def model_info():
	'''
	Display HW Model Number
	'''
	cmd = "dmidecode | grep 'Product Name: ' | head -n 1 | awk '{print $3,$4,$5,$6}'"
	run(cmd)

@roles('nn','dn')
def cpu_info():
	'''
	Display CPU
	'''
	cmd = "dmidecode | grep 'CPU' | grep 'Version'"
	run(cmd)

@roles('nn','dn')
def ram_info():
	'''
	Display RAM
	'''
	cmd = "dmidecode | grep 'Size: ' | grep -v 'Range' | grep MB | awk '{print $2}' | awk '{sum+=$1} END {print sum/1024}'"
	run(cmd)

	cmd = "dmidecode | grep 'Size: ' | grep -v 'Range' | grep MB | awk '{print $2}' | wc -l"
	run(cmd)

	cmd = "dmidecode | grep 'Size: ' | grep -v 'Range' | grep MB | awk '{print $2}'"
	run(cmd)

@roles('nn','dn')
def log():
	'''
	Job Tracker & Task Tracker Log
	'''
	cmd = "df /log/hadoop 2> /dev/null | tail -n 1 | awk '{print $5}' | grep '[5-9][0-9]%'"
	run(cmd)

@roles('nn','dn')
def log_mapred():
	'''
	Non-DFS Used Data Check
	'''
	cmd = "du -s -c -h /data*/mapred/local/userlogs | tail -n 1"
	run(cmd)

@roles('nn','dn')
def height():
	'''
	Height
	'''
	cmd = "dmidecode | grep Height | awk '{print $2}'"
	run(cmd)

@roles('nn','dn')
def uptime():
	'''
	Uptime
	'''
	cmd = "uptime"
	run(cmd)

@roles('nn','dn')
def hostname():
	'''
	Hostname
	'''
	cmd = "hostname"
	run(cmd)

@roles('dn')
def gpu():
	'''
	GPU: 84:00.0 3D controller: NVIDIA Corporation GK210GL [Tesla K80] (rev a1)
	'''
	cmd = "lspci | grep -i nvidia | tr -d '[:space:]'"
	expected = '84:00.03Dcontroller:NVIDIACorporationGK210GL[TeslaK80](reva1)' + '85:00.03Dcontroller:NVIDIACorporationGK210GL[TeslaK80](reva1)'
	run('''[ "`%(cmd)s`" != "%(expected)s" ] && (echo "Expected: %(expected)s, Actual: `%(cmd)s`")''' % locals())

@roles('dn')
def gpu_uuid_info():
	'''
	GPU UUID
	'''
	cmd = "nvidia-smi -L"
	run(cmd)

@roles('dn')
def r():
	'''
	R Version: R version 3.1.3 (2015-03-09) -- "Smooth Sidewalk"
	'''
	cmd = "/usr/bin/R --version | head -n 1 | cut -d ' ' -f 3"
	expected = '3.1.3'
	run('''[ "`%(cmd)s`" != "%(expected)s" ] && (echo "Expected: %(expected)s, Actual: `%(cmd)s`")''' % locals())

@roles('dn')
def sge_execd():
	'''
	Sun Grid Engine Daemon
	'''
	cmd = "ps aux | grep sge_execd | grep hmi | wc -l"
	expected = '1'
	run('''[ "`%(cmd)s`" != "%(expected)s" ] && (echo "Expected: %(expected)s, Actual: `%(cmd)s`")''' % locals())

@roles('dn')
def hmount():
	'''
	Check /hdfs/ mounting
	'''
	cmd = "ls /hdfs"
	run(cmd)

@roles('nn','dn')
def alive():
	'''
	Check server are alive
	'''
	cmd = "echo alive"
	run(cmd)

@roles('dn')
def gpu_temp():
	'''
	GPU Temperature
	'''
	cmd = "nvidia-smi -q -d TEMPERATURE | grep 'GPU Current Temp'"
	run(cmd)
<commit_msg>add some arguments to get max value for sensors and fan<commit_after>#!/usr/local/bin/python2.7

'''
Cluster Doctor Fabric File

Check configuration & status of all nodes in a cluster.
'''

__author__    = 'Dongjoon Hyun (dongjoon@apache.org)'
__license__   = 'Apache License'
__version__   = '0.1'

from fabric.api import *

env.roledefs = {}
nn = []
for i in range(95,99):
	nn.append('50.1.100.'+str(i))
env.roledefs['nn'] = nn
dn = []
for i in range(101,131):
	dn.append('50.1.100.'+str(i))
env.roledefs['dn'] = dn

print "\n\n", "-"*80, "\n\n# of IPs: ", len(nn + dn)

env.warn_only = True
env.skip_bad_hosts = True
env.colorize_errors = True
env.use_ssh_config = True
env.user = 'root'
env.key_filename = '~/.ssh/GPU.pem'
env.timeout = 3
env.parallel = True
output['status'] = False
output['stdout'] = True
output['warnings'] = False
output['running'] = False

@roles('nn','dn')
def os():
	'''
	OS Version: CentOS release 6.5
	'''
	cmd = "/usr/bin/lsb_release -r | cut -d '\t' -f 2"
	expected = '6.5'
	run('''[ "`%(cmd)s`" != "%(expected)s" ] && (echo "Expected: %(expected)s, Actual: `%(cmd)s`")''' % locals())

@roles('nn','dn')
def os_info():
	'''
	Display OS Version
	'''
	cmd = '/usr/bin/lsb_release -r'
	run(cmd)

@roles('nn','dn')
def kernel():
	'''
	Kernel Version: 2.6.32-431.el6.x86_64
	'''
	cmd = 'uname -r | tr -d [:space:]'
	expected = '2.6.32-431.el6.x86_64'
	run('''[ "`%(cmd)s`" != "%(expected)s" ] && (echo "Expected: %(expected)s, Actual: `%(cmd)s`")''' % locals())

@roles('nn','dn')
def selinux():
	'''
	SELINUX : disabled
	'''
	cmd = "cat /etc/sysconfig/selinux | grep 'SELINUX=disabled'"
	expected = 'SELINUX=disabled'
	run('''[ "`%(cmd)s`" != "%(expected)s" ] && (echo "Expected: %(expected)s, Actual: `%(cmd)s`")''' % locals())

@roles('nn','dn')
def sys():
	'''
	Sys Parameter: /proc/sys/kernel/hung_task_timeout_secs == 0
	'''
	cmd = 'cat /proc/sys/kernel/hung_task_timeout_secs'
	expected = '0'
	run('''[ "`%(cmd)s`" != "%(expected)s" ] && (echo "Expected: %(expected)s, Actual: `%(cmd)s`")''' % locals())

	cmd = 'grep hung_task_timeout_secs /etc/rc.local | wc -l'
	expected = '2'
	run('''[ "`%(cmd)s`" != "%(expected)s" ] && (echo "Expected: %(expected)s, Actual: `%(cmd)s`")''' % locals())

@roles('nn','dn')
def os_hugepage_enabled():
	'''
	OS Parameter: /sys/kernel/mm/redhat_transparent_hugepage/enabled == always [never]
	'''
	cmd = 'cat /sys/kernel/mm/redhat_transparent_hugepage/enabled'
	expected = 'always madvise [never]'
	run('''[ "`%(cmd)s`" != "%(expected)s" ] && (echo "Expected: %(expected)s, Actual: `%(cmd)s`")''' % locals())

	cmd = 'grep enabled /etc/rc.local | wc -l'
	expected = '2'
	run('''[ "`%(cmd)s`" != "%(expected)s" ] && (echo "Expected: %(expected)s, Actual: `%(cmd)s`")''' % locals())

@roles('nn','dn')
def os_hugepage_defrag():
	'''
	OS Parameter: cat /sys/kernel/mm/redhat_transparent_hugepage/defrag
	'''
	cmd = 'cat /sys/kernel/mm/redhat_transparent_hugepage/defrag'
	expected = 'always madvise [never]'
	run('''[ "`%(cmd)s`" != "%(expected)s" ] && (echo "Expected: %(expected)s, Actual: `%(cmd)s`")''' % locals())

	cmd = 'grep defrag /etc/rc.local | wc -l'
	expected = '2'
	run('''[ "`%(cmd)s`" != "%(expected)s" ] && (echo "Expected: %(expected)s, Actual: `%(cmd)s`")''' % locals())

@roles('nn','dn')
def date():
	'''
	System Time: time gap < 1 minute
	'''
	from datetime import datetime
	cmd = 'date +%Y-%m-%dT%H:%M'
	expected = datetime.now().isoformat()[0:16]
	run('''[ "`%(cmd)s`" != "%(expected)s" ] && (echo "Expected: %(expected)s, Actual: `%(cmd)s`")''' % locals())

@roles('nn','dn')
def service_gmond():
	'''
	gmond Installation: chkconfig --list | grep gmond
	'''
	cmd = 'chkconfig --list|grep gmond | wc -l'
	expected = '1'
	run('''[ "`%(cmd)s`" != "%(expected)s" ] && (echo "Expected: %(expected)s, Actual: `%(cmd)s`")''' % locals())

@roles('nn','dn')
def service_gmond_on():
	'''
	gmond Running: telnet $host 8649
	'''
	cmd = 'telnet %s 8649 | grep Analysis | wc -l' % (env.host)
	expected = '1'
	run('''[ "`%(cmd)s`" != "%(expected)s" ] && (echo "Expected: %(expected)s, Actual: `%(cmd)s`")''' % locals())

@roles('nn','dn')
def disk():
	'''
	#, Usage of Disks (TODO): remaining space > 20%
	'''
	run("df -a -h | awk '{print $5}'")

@roles('nn','dn')
def disk_info():
	'''
	Display Disk Info
	'''
	cmd = "fdisk -l 2> /dev/null | grep '^Disk /dev/sd' | awk '{sum+=$3} END {print total,sum/1024}'"
	run(cmd)

	cmd = "fdisk -l 2> /dev/null | grep '^Disk /dev/sd' | awk '{print $2,$3}' | sort"
	run(cmd)

@roles('dn')
def disk_size():
	'''
	Size: /data* should be '1171964464'
	'''
	cmd = "df | grep data | awk '{print $2}' | grep -v 1171964464"
	run(cmd)

@roles('dn')
def disk_ext4():
	'''
	noatime: /data* should have 'ext4'
	'''
	run("grep '/data' /etc/fstab | grep -v ext4")

@roles('dn')
def disk_noatime():
	'''
	noatime: /data* should have 'noatime'
	'''
	run("grep '/data' /etc/fstab | grep -v noatime")

@roles('nn','dn')
def hadoop():
	'''
	Hadoop: /usr/lib/hadoop
	'''
	cmd = '[ -d /usr/lib/hadoop ] && echo 1'
	expected = '1'
	run('''[ "`%(cmd)s`" != "%(expected)s" ] && (echo "Expected: %(expected)s, Actual: `%(cmd)s`")''' % locals())

@roles('nn','dn')
def sensors():
	'''
	Sensors : Chip `Intel digital thermal sensor' (confidence: 9)
		  Chip `IPMI BMC KCS' (confidence: 8)
	'''
	cmd = "sensors | awk '{print $3}' | grep '+[0-9]' | sort -r | head -1"
	run(cmd)

@roles('nn','dn')
def fan():
	'''
	Fan Speed
	'''
	cmd = "ipmi-sensors | grep Fan | grep RPM | awk -F '|' '{print $4}' | sort -r | head -1"
	run(cmd)

@roles('nn','dn')
def netstat():
	'''
	Task Tracker CLOSE_WAIT Bug Check
	'''
	cmd = "netstat -n | grep CLOSE_WAIT | wc -l | grep '[0-9][0-9][0-9][0-9]'"
	run(cmd)

@roles('nn','dn')
def overruns():
	'''
	Network Packket Overrun Check
	'''
	cmd = "ifconfig bond0 | grep overruns | grep -v 'overruns:0'"
	run(cmd)

@roles('nn','dn')
def errors():
	'''
	Network Packket Overrun Check
	'''
	cmd = "ifconfig bond0 | grep errors | grep -v 'errors:0'"
	run(cmd)

@roles('nn','dn')
def mtu():
	'''
	Network MTU Check
	'''
	cmd = "ifconfig bond0 | grep MTU | grep -v 1500"
	run(cmd)

@roles('nn','dn')
def serial_info():
	'''
	Display Serial Number
	'''
	cmd = "dmidecode | grep 'Serial Number:' | head -n 1 | awk '{print $3}'"
	run(cmd)

@roles('nn','dn')
def model_info():
	'''
	Display HW Model Number
	'''
	cmd = "dmidecode | grep 'Product Name: ' | head -n 1 | awk '{print $3,$4,$5,$6}'"
	run(cmd)

@roles('nn','dn')
def cpu_info():
	'''
	Display CPU
	'''
	cmd = "dmidecode | grep 'CPU' | grep 'Version'"
	run(cmd)

@roles('nn','dn')
def ram_info():
	'''
	Display RAM
	'''
	cmd = "dmidecode | grep 'Size: ' | grep -v 'Range' | grep MB | awk '{print $2}' | awk '{sum+=$1} END {print sum/1024}'"
	run(cmd)

	cmd = "dmidecode | grep 'Size: ' | grep -v 'Range' | grep MB | awk '{print $2}' | wc -l"
	run(cmd)

	cmd = "dmidecode | grep 'Size: ' | grep -v 'Range' | grep MB | awk '{print $2}'"
	run(cmd)

@roles('nn','dn')
def log():
	'''
	Job Tracker & Task Tracker Log
	'''
	cmd = "df /log/hadoop 2> /dev/null | tail -n 1 | awk '{print $5}' | grep '[5-9][0-9]%'"
	run(cmd)

@roles('nn','dn')
def log_mapred():
	'''
	Non-DFS Used Data Check
	'''
	cmd = "du -s -c -h /data*/mapred/local/userlogs | tail -n 1"
	run(cmd)

@roles('nn','dn')
def height():
	'''
	Height
	'''
	cmd = "dmidecode | grep Height | awk '{print $2}'"
	run(cmd)

@roles('nn','dn')
def uptime():
	'''
	Uptime
	'''
	cmd = "uptime"
	run(cmd)

@roles('nn','dn')
def hostname():
	'''
	Hostname
	'''
	cmd = "hostname"
	run(cmd)

@roles('dn')
def gpu():
	'''
	GPU: 84:00.0 3D controller: NVIDIA Corporation GK210GL [Tesla K80] (rev a1)
	'''
	cmd = "lspci | grep -i nvidia | tr -d '[:space:]'"
	expected = '84:00.03Dcontroller:NVIDIACorporationGK210GL[TeslaK80](reva1)' + '85:00.03Dcontroller:NVIDIACorporationGK210GL[TeslaK80](reva1)'
	run('''[ "`%(cmd)s`" != "%(expected)s" ] && (echo "Expected: %(expected)s, Actual: `%(cmd)s`")''' % locals())

@roles('dn')
def gpu_uuid_info():
	'''
	GPU UUID
	'''
	cmd = "nvidia-smi -L"
	run(cmd)

@roles('dn')
def r():
	'''
	R Version: R version 3.1.3 (2015-03-09) -- "Smooth Sidewalk"
	'''
	cmd = "/usr/bin/R --version | head -n 1 | cut -d ' ' -f 3"
	expected = '3.1.3'
	run('''[ "`%(cmd)s`" != "%(expected)s" ] && (echo "Expected: %(expected)s, Actual: `%(cmd)s`")''' % locals())

@roles('dn')
def sge_execd():
	'''
	Sun Grid Engine Daemon
	'''
	cmd = "ps aux | grep sge_execd | grep hmi | wc -l"
	expected = '1'
	run('''[ "`%(cmd)s`" != "%(expected)s" ] && (echo "Expected: %(expected)s, Actual: `%(cmd)s`")''' % locals())

@roles('dn')
def hmount():
	'''
	Check /hdfs/ mounting
	'''
	cmd = "ls /hdfs"
	run(cmd)

@roles('nn','dn')
def alive():
	'''
	Check server are alive
	'''
	cmd = "echo alive"
	run(cmd)

@roles('dn')
def gpu_temp():
	'''
	GPU Temperature
	'''
	cmd = "nvidia-smi -q -d TEMPERATURE | grep 'GPU Current Temp'"
	run(cmd)
<|endoftext|>


<commit_before>""" Implementation of a protocol based on the SWIM protocol described in
http://www.cs.cornell.edu/~asdas/research/dsn02-SWIM.pdf ("the paper") """

# Disable 'Too many instance attributes'             pylint: disable=R0902

import random
import itertools
import swimmsg
import swimprotocol

class Membership(object):
    """  Each member of the distributed process group
    should instantiate a single instance of this class.

    This class is responsible for:
     -  maintaining the membership of this local process in the distributed
        process group
     -  maintaining an up-to-date local view of the membership of the
        distributed process group
     -  detecting failures in remote members
     -  disseminating information about joined/left/failed members
    """

    def __init__(
            self,
            member_id,
            expected_remote_members,
            messagerouter,
            enable_infection_dissemination = True
        ):
        self.member_id = member_id
        self.messagerouter = messagerouter
        self.incarnation_number = 0 # Futures: should persist
        self.expected_remote_members = expected_remote_members
        self.alive_remote_members = []
        self.last_received_message = None
        self.received_messages = 0
        self.messagerouter.register_for_messages(self.member_id, self)
        self.nodes_to_ping = None
        self.enable_infection_dissemination = enable_infection_dissemination
        self._update_incarnation()

    def _update_incarnation(self):
        """We need to update our incarnation"""
        self.incarnation_number += 1

    def __str__(self):
        return "Membership(member_id=%s)" % self.member_id

    def start(self):
        """Prepare to check the liveness of remote members"""
        for remote_member in self.expected_remote_members:
            remote_member.start(self)

    def tick(self, time_now):
        """Time is advancing - we should check up on remote nodes
        (time_now should be some sort of monotonic time)"""
        new_node_to_ping = self._select_node_to_ping()
        if new_node_to_ping.is_currently_being_checked():
            # We've already got a ping in progress for this node. Let's
            # wait for that to succeed/fail.
            pass
        else:
            new_node_to_ping.begin_checking_for_failure(time_now)

        # prod each node to see if it needs to change state/time out etc.
        for node in self.expected_remote_members:
            node.on_tick(time_now)

    def _select_node_to_ping(self):
        """Select a node to ping using randomised round-robin as per
        section 4.3 of the paper.
        (helps to provide time bounded strong completeness)"""
        if self.nodes_to_ping is None:
            random.shuffle(self.expected_remote_members)
            self.nodes_to_ping = itertools.cycle(self.expected_remote_members)
        return self.nodes_to_ping.next()

    def select_nodes_to_ping_req(self, node_id_to_ping):
        """Returns K nodes for the failure detection subgroup """
        nodes_to_ping_req = []
        if self.nodes_to_ping is not None:
            aux = list(self.expected_remote_members)
            random.shuffle(aux)
            nodes_to_ping_req = [ node for node in aux
                                  if node.remote_member_id != node_id_to_ping ]
        return nodes_to_ping_req[: swimprotocol.SWIM.K ]

    def broadcast_message(self, message):
        """ Broadcast a message to all known members """
        for member in self.expected_remote_members:
            self.send_message_to_member_id(message, member.remote_member_id)

    def get_piggyback_data_to_send(self):
        """Construct piggyback data for infection style dissemination
        inspired by section 4.1 of the paper"""
        # Futures: Limit the number of nodes we will report on (to give bounded
        # message size/load as opposed to fastest possible dissemination)

        alive_nodes = [(self.member_id, self.incarnation_number)]
        dead_nodes = []
        for member in self.expected_remote_members:
            if member.state == "alive":
                alive_nodes.append((
                    member.remote_member_id,
                    member.incarnation_number
                ))
            elif member.state == "dead":
                dead_nodes.append((
                    member.remote_member_id, member.incarnation_number
                ))
        piggyback_data = {
            "Alive"  : alive_nodes,
            "Dead"  : dead_nodes,
        }
        return piggyback_data

    def send_message_to_member_id(self, message, remote_member_id):
        """ Send a message to a specific member"""
        if self.enable_infection_dissemination:
            # Infection style dissemination is enabled. Let's inject
            # piggyback data <here>
            piggyback_data = self.get_piggyback_data_to_send()
            # assert message.piggyback_data is None
            message.piggyback_data = piggyback_data

        self.messagerouter.send_message_to(
            remote_member_id,
            message,
            self.member_id
        )

    def _logical_sender_from_id(self, from_sender):
        """Returns the RemoteMember from expected_remote_members with the
        id matching 'from_sender' if one can be found; None otherwise"""
        logical_from_sender = None
        for member in self.expected_remote_members:
            if member.remote_member_id == from_sender:
                logical_from_sender = member
                break
        return logical_from_sender

    def on_incoming_message(self, message, from_sender_id):
        """We've received a message from the sender identified by
        'from_sender_id# """
        self.last_received_message = message
        self.received_messages = self.received_messages + 1
        logical_from_sender = self._logical_sender_from_id(from_sender_id)
        if logical_from_sender is None:
            # Could be a message sent by recently added or
            # removed node; log & ignore.
            print "Warning: %s got message from unknown sender '%s':%s" % (
                self.member_id, from_sender_id, message
            )
        else:
            logical_from_sender.handle_incoming_message(message)

    def member_indirectly_reachable(
            self,
            member_id,
            reachable_from_member_id,
            message
        ):
        """We know that a node 'member_id' that wasn't directly reachable
        has been reached indirectly by node 'reachable_from_member_id' on
        our behalf. We must update internal state accordingly. """
        assert message.message_name in ['ping_req_ack']
        # print "%s can reach %s (whereas we=%s cannot) " % (
        #     reachable_from_member_id,
        #     member_id,
        #     self.member_id
        # )
        alive_member = self._logical_sender_from_id(member_id)
        if alive_member is None:
            # Could be a message sent by recently added or removed
            # node; log & ignore.
            print "Warning: %s got member_indirectly_reachable for "\
            "unknown member '%s' thanks to '%s' : %s" % (
                self.member_id,
                alive_member,
                reachable_from_member_id,
                message
            )
        else:
            alive_member.handle_incoming_message(message)

class FailureDetectionTransaction(object):
    """ Class to handle failure detection """
    def __init__(self, time_now, owner, remote_member_id):
        self.start_time = time_now
        self.owner = owner
        self.remote_member_id = remote_member_id
        self.ack_received = False
        self.ping_req_ack_received = False
        self.response_timeout = 2
        self.state = "idle"

    def start(self):
        """Start checking the remote node for failure"""
        self.state = "ping_sent"
        self.owner.send_ping()

    def on_tick(self, time_now):
        """Time is marching on - do we need to time any operations out?
        (time_now should be some sort of monotonic time)"""
        # print "FDT %s on_tick start_time=%s time_now=%s state=%s" % (
        # self.remote_member_id,
        # self.start_time,
        # time_now,
        # self.state
        # )
        if self.state == "ping_sent":
            if time_now > self.start_time + self.response_timeout:
                # Direct ping has failed; let's try indirect ping (ping_req)
                self.state = "ping_req_sent"
                self.owner.send_ping_reqs()
        elif self.state == "ping_req_sent":
            if time_now > self.start_time + (self.response_timeout * 2):
                # print "FailureDetectionTransaction not heard back "\
                #      "regarding %s; assuming failure" % self.remote_member_id
                self.state = "failure_detected"
                self.owner.node_failed()

    def on_ack(self):
        """We pinged a node ourselves and it responded directly to us"""
        self.state = "alive"
        self.owner.node_alive()

    def on_ping_req_ack(self):
        """We have received a ping_req_ack - so we know that a node we
        sent a ping_req to managed, on our behalf, to ping the node
        we're checking."""
        self.state = "alive"
        self.owner.node_alive()

class RemoteMember(object):
    """  Represents a remote member of the distributed process group;
    handles  messages communicated between this node and the remote
    member and maintains 'state' about the remote member's liveness. """
    def __init__(self, remote_member_id):
        """
        """
        self.remote_member_id = remote_member_id
        self.incarnation_number = 0
        self.state = "unknown"
        self.failure_detection_transaction = None
        self.membership = None

    def __str__(self):
        return 'RemoteMember(remote_member_id=%s, state=%s)' % (
            self.remote_member_id, self.state
        )

    def start(self, membership):
        """Prepare to become operational"""
        self.membership = membership

    def on_tick(self, time_now):
        """Time is marching forwards - do we need to time-out any
        operations etc.?"""
        if self.failure_detection_transaction is not None:
            self.failure_detection_transaction.on_tick(time_now)
        else:
            # We're not in the midst of a ping transaction; we
            # need do nothing.
            pass

    def is_currently_being_checked(self):
        """Are we already in the middle of checking the liveness
        of this node?"""
        return self.failure_detection_transaction is not None

    def begin_checking_for_failure(self, time_now):
        """Let's check the liveness of this node"""
        assert self.membership is not None
        assert self.failure_detection_transaction is None
        self.failure_detection_transaction = FailureDetectionTransaction(
            time_now,
            self,
            self.remote_member_id
        )
        self.failure_detection_transaction.start()

    def node_alive(self):
        """This node appears to be alive"""
        self.state = "alive"
        self.failure_detection_transaction = None

    def node_failed(self):
        """This node appears to be failed/unreachable"""
        self.state = "dead"
        self.failure_detection_transaction = None

    def send_ping(self):
        """ Send a ping (along with some piggyback data) to the remote
        member tracked by this object """
        ping_msg = swimmsg.ping()
        self.membership.send_message_to_member_id(
            ping_msg,
            self.remote_member_id
        )

    def send_ping_reqs(self):
        """Our owner wishes to request that this remote member sends
        a ping (along with some piggyback data)to another
        remote member"""
        ping_req_msg = swimmsg.ping_req(
            self.membership.member_id,
            self.remote_member_id
        )
        members_to_ping_req = self.membership.select_nodes_to_ping_req(
            self.remote_member_id
        )
        for member in members_to_ping_req:
            self.membership.send_message_to_member_id(
                ping_req_msg,
                member.remote_member_id
            )

    def handle_incoming_message(self, message):
        """We've received a message from the wire"""
        # FUTURE: if we are "dead" should we treat receipt of a
        # message from a node as meaning that node is alive again?
        if (self.failure_detection_transaction is not None
            and message.message_name in ['ack', 'ping_req_ack'] ):
            if message.message_name == 'ack':
                self.failure_detection_transaction.on_ack()
            elif message.message_name == 'ping_req_ack':
                self.failure_detection_transaction.on_ping_req_ack()
        else:
            if message.message_name == 'ping':
                # Always respond to a ping with an ack
                self.membership.send_message_to_member_id(
                    swimmsg.ack(meta_data=message.meta_data),
                    self.remote_member_id
                )
            elif message.message_name == 'ping_req':
                # On receipt of a ping_req, attempt to ping the node in question
                member_id_to_ping = message.meta_data.get(
                    "member_id_to_ping",
                    None
                )
                self.membership.send_message_to_member_id(
                    swimmsg.ping(meta_data=message.meta_data),
                    member_id_to_ping
                )
            elif message.message_name == 'ack':
                # if this is an ack for a ping sent in response to a
                # ping_req, we need to send a ping_req_ack to the
                # original requester.
                if message.meta_data is not None:
                    requested_by_member_id = message.meta_data.get(
                        "requested_by_member_id",
                        None
                    )
                    member_id_to_ping = message.meta_data.get(
                        "member_id_to_ping",
                        None
                    )
                    if requested_by_member_id and member_id_to_ping:
                        # Send the ping_req_ack to the member to sent
                        # the ping_req
                        self.membership.send_message_to_member_id(
                            swimmsg.ping_req_ack(
                                requested_by_member_id,
                                member_id_to_ping
                            ),
                            requested_by_member_id
                        )
            elif message.message_name == 'ping_req_ack':
                requested_by_member_id = message.meta_data.get(
                    "requested_by_member_id",
                    None
                )
                member_id_to_ping = message.meta_data.get(
                    "member_id_to_ping",
                    None
                )
                if (requested_by_member_id == self.membership.member_id and
                    member_id_to_ping != self.remote_member_id ):
                    if member_id_to_ping is not None:
                        # We've got a ping_req_ack that is of interest
                        # to one of our in-process peer objects - bounce
                        # this over, via our owner
                        self.membership.member_indirectly_reachable(
                            member_id_to_ping,
                            self.remote_member_id,
                            message
                        )
<commit_msg>Rename function<commit_after>""" Implementation of a protocol based on the SWIM protocol described in
http://www.cs.cornell.edu/~asdas/research/dsn02-SWIM.pdf ("the paper") """

# Disable 'Too many instance attributes'             pylint: disable=R0902

import random
import itertools
import swimmsg
import swimprotocol

class Membership(object):
    """  Each member of the distributed process group
    should instantiate a single instance of this class.

    This class is responsible for:
     -  maintaining the membership of this local process in the distributed
        process group
     -  maintaining an up-to-date local view of the membership of the
        distributed process group
     -  detecting failures in remote members
     -  disseminating information about joined/left/failed members
    """

    def __init__(
            self,
            member_id,
            expected_remote_members,
            messagerouter,
            enable_infection_dissemination = True
        ):
        self.member_id = member_id
        self.messagerouter = messagerouter
        self.incarnation_number = 0 # Futures: should persist
        self.expected_remote_members = expected_remote_members
        self.alive_remote_members = []
        self.last_received_message = None
        self.received_messages = 0
        self.messagerouter.register_for_messages(self.member_id, self)
        self.nodes_to_ping = None
        self.enable_infection_dissemination = enable_infection_dissemination
        self._update_incarnation()

    def _update_incarnation(self):
        """We need to update our incarnation"""
        self.incarnation_number += 1

    def __str__(self):
        return "Membership(member_id=%s)" % self.member_id

    def start(self):
        """Prepare to check the liveness of remote members"""
        for remote_member in self.expected_remote_members:
            remote_member.start(self)

    def tick(self, time_now):
        """Time is advancing - we should check up on remote nodes
        (time_now should be some sort of monotonic time)"""
        new_node_to_ping = self._select_node_to_ping()
        if new_node_to_ping.is_currently_being_checked():
            # We've already got a ping in progress for this node. Let's
            # wait for that to succeed/fail.
            pass
        else:
            new_node_to_ping.begin_checking_for_failure(time_now)

        # prod each node to see if it needs to change state/time out etc.
        for node in self.expected_remote_members:
            node.on_tick(time_now)

    def _select_node_to_ping(self):
        """Select a node to ping using randomised round-robin as per
        section 4.3 of the paper.
        (helps to provide time bounded strong completeness)"""
        if self.nodes_to_ping is None:
            random.shuffle(self.expected_remote_members)
            self.nodes_to_ping = itertools.cycle(self.expected_remote_members)
        return self.nodes_to_ping.next()

    def select_nodes_to_ping_req(self, node_id_to_ping):
        """Returns K nodes for the failure detection subgroup """
        nodes_to_ping_req = []
        if self.nodes_to_ping is not None:
            aux = list(self.expected_remote_members)
            random.shuffle(aux)
            nodes_to_ping_req = [ node for node in aux
                                  if node.remote_member_id != node_id_to_ping ]
        return nodes_to_ping_req[: swimprotocol.SWIM.K ]

    def broadcast_message(self, message):
        """ Broadcast a message to all known members """
        for member in self.expected_remote_members:
            self.send_message_to_member_id(message, member.remote_member_id)

    def get_piggyback_data_to_send(self):
        """Construct piggyback data for infection style dissemination
        inspired by section 4.1 of the paper"""
        # Futures: Limit the number of nodes we will report on (to give bounded
        # message size/load as opposed to fastest possible dissemination)

        alive_nodes = [(self.member_id, self.incarnation_number)]
        dead_nodes = []
        for member in self.expected_remote_members:
            if member.state == "alive":
                alive_nodes.append((
                    member.remote_member_id,
                    member.incarnation_number
                ))
            elif member.state == "dead":
                dead_nodes.append((
                    member.remote_member_id, member.incarnation_number
                ))
        piggyback_data = {
            "Alive"  : alive_nodes,
            "Dead"  : dead_nodes,
        }
        return piggyback_data

    def send_message_to_member_id(self, message, remote_member_id):
        """ Send a message to a specific member"""
        if self.enable_infection_dissemination:
            # Infection style dissemination is enabled. Let's inject
            # piggyback data <here>
            piggyback_data = self.get_piggyback_data_to_send()
            # assert message.piggyback_data is None
            message.piggyback_data = piggyback_data

        self.messagerouter.send_message_to(
            remote_member_id,
            message,
            self.member_id
        )

    def _remote_member_from_id(self, member_id):
        """Returns the RemoteMember from expected_remote_members with the
        id matching 'from_sender' if one can be found; None otherwise"""
        logical_from_sender = None
        for member in self.expected_remote_members:
            if member.remote_member_id == member_id:
                logical_from_sender = member
                break
        return logical_from_sender

    def on_incoming_message(self, message, from_sender_id):
        """We've received a message from the sender identified by
        'from_sender_id# """
        self.last_received_message = message
        self.received_messages = self.received_messages + 1
        logical_from_sender = self._remote_member_from_id(from_sender_id)
        if logical_from_sender is None:
            # Could be a message sent by recently added or
            # removed node; log & ignore.
            print "Warning: %s got message from unknown sender '%s':%s" % (
                self.member_id, from_sender_id, message
            )
        else:
            logical_from_sender.handle_incoming_message(message)

    def member_indirectly_reachable(
            self,
            member_id,
            reachable_from_member_id,
            message
        ):
        """We know that a node 'member_id' that wasn't directly reachable
        has been reached indirectly by node 'reachable_from_member_id' on
        our behalf. We must update internal state accordingly. """
        assert message.message_name in ['ping_req_ack']
        # print "%s can reach %s (whereas we=%s cannot) " % (
        #     reachable_from_member_id,
        #     member_id,
        #     self.member_id
        # )
        alive_member = self._remote_member_from_id(member_id)
        if alive_member is None:
            # Could be a message sent by recently added or removed
            # node; log & ignore.
            print "Warning: %s got member_indirectly_reachable for "\
            "unknown member '%s' thanks to '%s' : %s" % (
                self.member_id,
                alive_member,
                reachable_from_member_id,
                message
            )
        else:
            alive_member.handle_incoming_message(message)

class FailureDetectionTransaction(object):
    """ Class to handle failure detection """
    def __init__(self, time_now, owner, remote_member_id):
        self.start_time = time_now
        self.owner = owner
        self.remote_member_id = remote_member_id
        self.ack_received = False
        self.ping_req_ack_received = False
        self.response_timeout = 2
        self.state = "idle"

    def start(self):
        """Start checking the remote node for failure"""
        self.state = "ping_sent"
        self.owner.send_ping()

    def on_tick(self, time_now):
        """Time is marching on - do we need to time any operations out?
        (time_now should be some sort of monotonic time)"""
        # print "FDT %s on_tick start_time=%s time_now=%s state=%s" % (
        # self.remote_member_id,
        # self.start_time,
        # time_now,
        # self.state
        # )
        if self.state == "ping_sent":
            if time_now > self.start_time + self.response_timeout:
                # Direct ping has failed; let's try indirect ping (ping_req)
                self.state = "ping_req_sent"
                self.owner.send_ping_reqs()
        elif self.state == "ping_req_sent":
            if time_now > self.start_time + (self.response_timeout * 2):
                # print "FailureDetectionTransaction not heard back "\
                #      "regarding %s; assuming failure" % self.remote_member_id
                self.state = "failure_detected"
                self.owner.node_failed()

    def on_ack(self):
        """We pinged a node ourselves and it responded directly to us"""
        self.state = "alive"
        self.owner.node_alive()

    def on_ping_req_ack(self):
        """We have received a ping_req_ack - so we know that a node we
        sent a ping_req to managed, on our behalf, to ping the node
        we're checking."""
        self.state = "alive"
        self.owner.node_alive()

class RemoteMember(object):
    """  Represents a remote member of the distributed process group;
    handles  messages communicated between this node and the remote
    member and maintains 'state' about the remote member's liveness. """
    def __init__(self, remote_member_id):
        """
        """
        self.remote_member_id = remote_member_id
        self.incarnation_number = 0
        self.state = "unknown"
        self.failure_detection_transaction = None
        self.membership = None

    def __str__(self):
        return 'RemoteMember(remote_member_id=%s, state=%s)' % (
            self.remote_member_id, self.state
        )

    def start(self, membership):
        """Prepare to become operational"""
        self.membership = membership

    def on_tick(self, time_now):
        """Time is marching forwards - do we need to time-out any
        operations etc.?"""
        if self.failure_detection_transaction is not None:
            self.failure_detection_transaction.on_tick(time_now)
        else:
            # We're not in the midst of a ping transaction; we
            # need do nothing.
            pass

    def is_currently_being_checked(self):
        """Are we already in the middle of checking the liveness
        of this node?"""
        return self.failure_detection_transaction is not None

    def begin_checking_for_failure(self, time_now):
        """Let's check the liveness of this node"""
        assert self.membership is not None
        assert self.failure_detection_transaction is None
        self.failure_detection_transaction = FailureDetectionTransaction(
            time_now,
            self,
            self.remote_member_id
        )
        self.failure_detection_transaction.start()

    def node_alive(self):
        """This node appears to be alive"""
        self.state = "alive"
        self.failure_detection_transaction = None

    def node_failed(self):
        """This node appears to be failed/unreachable"""
        self.state = "dead"
        self.failure_detection_transaction = None

    def send_ping(self):
        """ Send a ping (along with some piggyback data) to the remote
        member tracked by this object """
        ping_msg = swimmsg.ping()
        self.membership.send_message_to_member_id(
            ping_msg,
            self.remote_member_id
        )

    def send_ping_reqs(self):
        """Our owner wishes to request that this remote member sends
        a ping (along with some piggyback data)to another
        remote member"""
        ping_req_msg = swimmsg.ping_req(
            self.membership.member_id,
            self.remote_member_id
        )
        members_to_ping_req = self.membership.select_nodes_to_ping_req(
            self.remote_member_id
        )
        for member in members_to_ping_req:
            self.membership.send_message_to_member_id(
                ping_req_msg,
                member.remote_member_id
            )

    def handle_incoming_message(self, message):
        """We've received a message from the wire"""
        # FUTURE: if we are "dead" should we treat receipt of a
        # message from a node as meaning that node is alive again?
        if (self.failure_detection_transaction is not None
            and message.message_name in ['ack', 'ping_req_ack'] ):
            if message.message_name == 'ack':
                self.failure_detection_transaction.on_ack()
            elif message.message_name == 'ping_req_ack':
                self.failure_detection_transaction.on_ping_req_ack()
        else:
            if message.message_name == 'ping':
                # Always respond to a ping with an ack
                self.membership.send_message_to_member_id(
                    swimmsg.ack(meta_data=message.meta_data),
                    self.remote_member_id
                )
            elif message.message_name == 'ping_req':
                # On receipt of a ping_req, attempt to ping the node in question
                member_id_to_ping = message.meta_data.get(
                    "member_id_to_ping",
                    None
                )
                self.membership.send_message_to_member_id(
                    swimmsg.ping(meta_data=message.meta_data),
                    member_id_to_ping
                )
            elif message.message_name == 'ack':
                # if this is an ack for a ping sent in response to a
                # ping_req, we need to send a ping_req_ack to the
                # original requester.
                if message.meta_data is not None:
                    requested_by_member_id = message.meta_data.get(
                        "requested_by_member_id",
                        None
                    )
                    member_id_to_ping = message.meta_data.get(
                        "member_id_to_ping",
                        None
                    )
                    if requested_by_member_id and member_id_to_ping:
                        # Send the ping_req_ack to the member to sent
                        # the ping_req
                        self.membership.send_message_to_member_id(
                            swimmsg.ping_req_ack(
                                requested_by_member_id,
                                member_id_to_ping
                            ),
                            requested_by_member_id
                        )
            elif message.message_name == 'ping_req_ack':
                requested_by_member_id = message.meta_data.get(
                    "requested_by_member_id",
                    None
                )
                member_id_to_ping = message.meta_data.get(
                    "member_id_to_ping",
                    None
                )
                if (requested_by_member_id == self.membership.member_id and
                    member_id_to_ping != self.remote_member_id ):
                    if member_id_to_ping is not None:
                        # We've got a ping_req_ack that is of interest
                        # to one of our in-process peer objects - bounce
                        # this over, via our owner
                        self.membership.member_indirectly_reachable(
                            member_id_to_ping,
                            self.remote_member_id,
                            message
                        )
<|endoftext|>


<commit_before># This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

from flask import Flask
from werkzeug.contrib.cache import MemcachedCache

app=Flask(__name__)
app.config.from_object('config')

# set up a memcached Werkzeug cache, prefixing each key, with adjustable timeout
cache=MemcachedCache(servers=app.config['MEMCACHED_SERVERS'], key_prefix=app.config['MEMCACHED_KEYPREFIX'], default_timeout=app.config['MEMCACHED_TIMEOUT'])

import wahgwan_http.views
<commit_msg>config.py location<commit_after># This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

from flask import Flask
from werkzeug.contrib.cache import MemcachedCache

app=Flask(__name__)
app.config.from_pyfile('config.py')

# set up a memcached Werkzeug cache, prefixing each key, with adjustable timeout
cache=MemcachedCache(servers=app.config['MEMCACHED_SERVERS'], key_prefix=app.config['MEMCACHED_KEYPREFIX'], default_timeout=app.config['MEMCACHED_TIMEOUT'])

import wahgwan_http.views
<|endoftext|>


<commit_before>import io
import json
import random
import re

import aiohttp
import datetime
import discord
from bs4 import BeautifulSoup
from redbot.core import commands

DATE_RE = r"^([0-9][0-9]|19[0-9][0-9]|20[0-9][0-9])(\.|-|\/)([1-9]|0[1-9]|1[0-2])(\.|-|\/)([1-9]|0[1-9]|1[0-9]|2[0-9]|3[0-1])$"

class Comics(commands.Cog):
    """Print random comics from popular sources"""

    def __init__(self, bot):
        self.bot = bot
        self.session = aiohttp.ClientSession()

    def cog_unload(self):
        self.bot.loop.create_task(self.session.close())

    async def red_delete_data_for_user(self, **kwargs):
        """Nothing to delete."""
        return

    @commands.command()
    async def ohno(self, ctx):
        """Webcomic Name"""

        url = "http://webcomicname.com/random"

        async with ctx.typing():
            async with self.session.get(url) as response:
                soup = BeautifulSoup(await response.text(), "html.parser")

            img_url = soup.find(property='og:image')['content']

            async with self.session.get(img_url) as response:
                img = io.BytesIO(await response.read())

            await ctx.send(file=discord.File(img, 'ohno.png'))

    @commands.command()
    async def smbc(self, ctx):
        """Saturday Morning Breakfast Cereal"""

        url = "http://www.smbc-comics.com/comic/archive"

        async with ctx.typing():
            async with self.session.get(url, headers={'Connection': 'keep-alive'}) as response:
                soup = BeautifulSoup(await response.text(), "html.parser")

            all_comics = soup.find('select', attrs = {'name': 'comic'})
            all_comics_url_stubs = [option['value'] for option in all_comics.findChildren()]

            random_comic = random.choice(all_comics_url_stubs)
            comic_url = f"http://www.smbc-comics.com/{random_comic}"

            async with self.session.get(comic_url, headers={'Connection': 'keep-alive'}) as resp:
                soup = BeautifulSoup(await resp.text(), "html.parser")
                img_url = soup.find(property='og:image')['content']
                extra_url = soup.find(id='aftercomic').img['src']

            async with self.session.get(img_url) as response:
                img = io.BytesIO(await response.read())

            await ctx.send(file=discord.File(img, 'smbc.png'))

    @commands.command()
    async def pbf(self, ctx):
        """The Perry Bible Fellowship"""

        url = "http://pbfcomics.com/random"

        async with ctx.typing():
            async with self.session.get(url) as response:
                soup = BeautifulSoup(await response.text(), "html.parser")

            img_url = soup.find(property='og:image')['content']

            async with self.session.get(img_url) as response:
                img = io.BytesIO(await response.read())

            await ctx.send(file=discord.File(img, 'pbf.png'))

    @commands.command()
    async def cah(self, ctx):
        """Cyanide and Happiness"""

        url = "http://explosm.net/comics/random"

        async with ctx.typing():
            async with self.session.get(url) as response:
                soup = BeautifulSoup(await response.text(), "html.parser")

            img_url = soup.find(property='og:image')['content']

            async with self.session.get(img_url) as response:
                img = io.BytesIO(await response.read())

            await ctx.send(file=discord.File(img, 'cah.png'))

    @commands.command()
    async def xkcd(self, ctx):
        """XKCD"""

        url = "https://c.xkcd.com/random/comic/"
        phrase = r"Image URL \(for hotlinking\/embedding\)\:.*"

        async with ctx.typing():
            async with self.session.get(url) as response:
                soup = BeautifulSoup(await response.text(), "html.parser")

            img_url = soup.find(string=re.compile(phrase))
            img_url = 'https://' + img_url.split('https://')[1].rstrip()

            async with self.session.get(img_url) as response:
                img = io.BytesIO(await response.read())

            await ctx.send(file=discord.File(img, 'xkcd.png'))

    @commands.command()
    async def mrls(self, ctx):
        """Mr. Lovenstein"""

        url = "http://www.mrlovenstein.com/shuffle"

        async with ctx.typing():
            async with self.session.get(url) as response:
                soup = BeautifulSoup(await response.text(), "html.parser")

            img_url = f"http://www.mrlovenstein.com{soup.find(id='comic_main_image')['src']}"

            async with self.session.get(img_url) as response:
                img = io.BytesIO(await response.read())

            await ctx.send(file=discord.File(img, 'mrls.png'))

    @commands.command()
    async def chainsaw(self, ctx):
        """Chainsawsuit"""

        url = "http://chainsawsuit.com/comic/random/?random&nocache=1"

        async with ctx.typing():
            async with self.session.get(url) as response:
                soup = BeautifulSoup(await response.text(), "html.parser")

            img_url = soup.find(property='og:image')['content']

            async with self.session.get(img_url) as response:
                img = io.BytesIO(await response.read())

            await ctx.send(file=discord.File(img, 'chainsawsuit.png'))

    @commands.command()
    async def sarah(self, ctx):
        """Sarah's Scribbles"""

        url = "http://www.gocomics.com/random/sarahs-scribbles"

        async with ctx.typing():
            async with self.session.get(url) as response:
                soup = BeautifulSoup(await response.text(), "html.parser")

            img_url = soup.find(property='og:image')['content']

            async with self.session.get(img_url) as response:
                img = io.BytesIO(await response.read())

            await ctx.send(file=discord.File(img, 'sarahsscribbles.png'))

    @commands.command()
    async def dilbert(self, ctx, date: str = None):
        """Dilbert

        Random, or specify a date in YYYY-MM-DD format (2020-01-15).
        Examples:
        \t`[p]dilbert`\t\tFetches random comic
        \t`[p]dilbert 2020-1-15`\tFetches comic for Jan 15, 2020
        """
        if date:
            date_match = re.match(DATE_RE, date)
            if not date_match:
                return await ctx.send("That doesn't seem like a valid date. Try a format like `2020-01-15`.")
            date = date_match[0]
        else:
            start_date = datetime.date(1989, 4, 16)
            end_date = datetime.datetime.today().date()
            time_between_dates = end_date - start_date
            days_between_dates = time_between_dates.days
            random_number_of_days = random.randrange(days_between_dates)
            date = start_date + datetime.timedelta(days=random_number_of_days)

        url = f"https://dilbert.com/strip/{date}"

        async with ctx.typing():
            async with self.session.get(url) as response:
                html = await response.text()
                soup = BeautifulSoup(html, "html.parser")
                script_tag = soup.find("script", type="application/ld+json")
                tag = script_tag.string.lstrip().rstrip()
                tag = tag.replace("\n", " ").replace("\r", " ")
                tag = re.sub(" +", " ", tag)
                try:
                    comic_info = json.loads(tag)
                except json.decoder.JSONDecodeError:
                    return await ctx.send("I can't read that comic page.")

            async with self.session.get(comic_info["image"]) as response:
                img = io.BytesIO(await response.read())

            await ctx.send(file=discord.File(img, f"dilbert-{date}.png"))
<commit_msg>[Comics] Check for attach files perm & fix<commit_after>import io
import json
import random
import re

import aiohttp
import datetime
import discord
from bs4 import BeautifulSoup
from redbot.core import commands

DATE_RE = r"^([0-9][0-9]|19[0-9][0-9]|20[0-9][0-9])(\.|-|\/)([1-9]|0[1-9]|1[0-2])(\.|-|\/)([1-9]|0[1-9]|1[0-9]|2[0-9]|3[0-1])$"

class Comics(commands.Cog):
    """Print random comics from popular sources"""

    def __init__(self, bot):
        self.bot = bot
        self.session = aiohttp.ClientSession()

    def cog_unload(self):
        self.bot.loop.create_task(self.session.close())

    async def red_delete_data_for_user(self, **kwargs):
        """Nothing to delete."""
        return

    @commands.bot_has_permissions(attach_files=True)
    @commands.command()
    async def ohno(self, ctx):
        """Webcomic Name"""

        url = "http://webcomicname.com/random"

        async with ctx.typing():
            async with self.session.get(url) as response:
                soup = BeautifulSoup(await response.text(), "html.parser")

            img_url = soup.find(property='og:image')['content']

            async with self.session.get(img_url) as response:
                img = io.BytesIO(await response.read())

            await ctx.send(file=discord.File(img, 'ohno.png'))

    @commands.bot_has_permissions(attach_files=True)
    @commands.command()
    async def smbc(self, ctx):
        """Saturday Morning Breakfast Cereal"""

        url = "http://www.smbc-comics.com/comic/archive"

        async with ctx.typing():
            async with self.session.get(url, headers={'Connection': 'keep-alive'}) as response:
                soup = BeautifulSoup(await response.text(), "html.parser")

            all_comics = soup.find('select', attrs = {'name': 'comic'})
            all_comics_url_stubs = [option['value'] for option in all_comics.findChildren()]

            random_comic = random.choice(all_comics_url_stubs)
            comic_url = f"http://www.smbc-comics.com/{random_comic}"

            async with self.session.get(comic_url, headers={'Connection': 'keep-alive'}) as resp:
                soup = BeautifulSoup(await resp.text(), "html.parser")
                img_url = soup.find(property='og:image')['content']
                extra_url = soup.find(id='aftercomic').img['src']

            async with self.session.get(img_url) as response:
                img = io.BytesIO(await response.read())

            await ctx.send(file=discord.File(img, 'smbc.png'))

    @commands.bot_has_permissions(attach_files=True)
    @commands.command()
    async def pbf(self, ctx):
        """The Perry Bible Fellowship"""

        url = "http://pbfcomics.com/random"

        async with ctx.typing():
            async with self.session.get(url) as response:
                soup = BeautifulSoup(await response.text(), "html.parser")

            img_url = soup.find(property='og:image')['content']

            async with self.session.get(img_url) as response:
                img = io.BytesIO(await response.read())

            await ctx.send(file=discord.File(img, 'pbf.png'))

    @commands.bot_has_permissions(attach_files=True)
    @commands.command()
    async def cah(self, ctx):
        """Cyanide and Happiness"""

        url = "http://explosm.net/comics/random"

        async with ctx.typing():
            async with self.session.get(url) as response:
                soup = BeautifulSoup(await response.text(), "html.parser")

            img_url = soup.find(property='og:image')['content']

            async with self.session.get(img_url) as response:
                img = io.BytesIO(await response.read())

            await ctx.send(file=discord.File(img, 'cah.png'))

    @commands.bot_has_permissions(attach_files=True)
    @commands.command()
    async def xkcd(self, ctx):
        """XKCD"""

        url = "https://c.xkcd.com/random/comic/"
        phrase = r"Image URL \(for hotlinking\/embedding\)\:.*"

        async with ctx.typing():
            async with self.session.get(url) as response:
                soup = BeautifulSoup(await response.text(), "html.parser")

            img_url = soup.find(string=re.compile(phrase))
            img_url = 'https://' + img_url.split('https://')[1].rstrip()

            async with self.session.get(img_url) as response:
                img = io.BytesIO(await response.read())

            await ctx.send(file=discord.File(img, 'xkcd.png'))

    @commands.bot_has_permissions(attach_files=True)
    @commands.command()
    async def mrls(self, ctx):
        """Mr. Lovenstein"""

        url = "http://www.mrlovenstein.com/shuffle"

        async with ctx.typing():
            async with self.session.get(url) as response:
                soup = BeautifulSoup(await response.text(), "html.parser")

            img_url = f"http://www.mrlovenstein.com{soup.find(id='comic_main_image')['src']}"

            async with self.session.get(img_url) as response:
                img = io.BytesIO(await response.read())

            await ctx.send(file=discord.File(img, 'mrls.png'))

    @commands.bot_has_permissions(attach_files=True)
    @commands.command()
    async def chainsaw(self, ctx):
        """Chainsawsuit"""

        url = "http://chainsawsuit.com/comic/random/?random&nocache=1"

        async with ctx.typing():
            async with self.session.get(url) as response:
                soup = BeautifulSoup(await response.text(), "html.parser")

            img_url = soup.find(property='og:image')['content']

            async with self.session.get(img_url) as response:
                img = io.BytesIO(await response.read())

            await ctx.send(file=discord.File(img, 'chainsawsuit.png'))

    @commands.bot_has_permissions(attach_files=True)
    @commands.command()
    async def sarah(self, ctx):
        """Sarah's Scribbles"""

        url = "http://www.gocomics.com/random/sarahs-scribbles"

        async with ctx.typing():
            async with self.session.get(url) as response:
                soup = BeautifulSoup(await response.text(), "html.parser")

            img_url = soup.find(property='og:image')['content']

            async with self.session.get(img_url) as response:
                img = io.BytesIO(await response.read())

            await ctx.send(file=discord.File(img, 'sarahsscribbles.png'))

    @commands.bot_has_permissions(attach_files=True)
    @commands.command()
    async def dilbert(self, ctx, date: str = None):
        """Dilbert

        Random, or specify a date in YYYY-MM-DD format (2020-01-15).
        Examples:
        \t`[p]dilbert`\t\tFetches random comic
        \t`[p]dilbert 2020-1-15`\tFetches comic for Jan 15, 2020
        """
        if date:
            date_match = re.match(DATE_RE, date)
            if not date_match:
                return await ctx.send("That doesn't seem like a valid date. Try a format like `2020-01-15`.")
            date = date_match[0]
        else:
            start_date = datetime.date(1989, 4, 16)
            end_date = datetime.datetime.today().date()
            time_between_dates = end_date - start_date
            days_between_dates = time_between_dates.days
            random_number_of_days = random.randrange(days_between_dates)
            date = start_date + datetime.timedelta(days=random_number_of_days)

        url = f"https://dilbert.com/strip/{date}"

        async with ctx.typing():
            async with self.session.get(url) as response:
                html = await response.text()
                soup = BeautifulSoup(html, "html.parser")
                script_tag = soup.find("script", type="application/ld+json")
                tag = script_tag.string.lstrip().rstrip()
                tag = tag.replace("\n", " ").replace("\r", " ")
                tag = re.sub(" +", " ", tag)
                try:
                    comic_info = json.loads(tag)
                except json.decoder.JSONDecodeError:
                    return await ctx.send("I can't read that comic page.")

            image_url = comic_info.get("image", None)

            if image_url:
                async with self.session.get(comic_info["image"]) as response:
                    img = io.BytesIO(await response.read())

                return await ctx.send(file=discord.File(img, f"dilbert-{date}.png"))

            return await ctx.send("I can't read that comic page.")

<|endoftext|>