<commit_before># -*- coding: utf-8 -*-

"""

  Copyright (c) 2012, Davyd McColl; 2013, 2014 Jaime Soffer

   All rights reserved.

   Redistribution and use in source and binary forms, with or without
   modification, are permitted provided that the following conditions are met:

   Redistributions of source code must retain the above copyright notice, this
   list of conditions and the following disclaimer.

   Redistributions in binary form must reproduce the above copyright notice,
   this list of conditions and the following disclaimer in the documentation
   and/or other materials provided with the distribution.

   Neither the name of the involved organizations nor the names of its
   contributors may be used to endorse or promote products derived from this
   software without specific prior written permission.

   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
   THE POSSIBILITY OF SUCH DAMAGE.

"""

from PyQt4.QtGui import (QWidget, QProgressBar, QGridLayout,
                         QFrame, QLabel, QLineEdit, QCompleter, QKeyEvent,
                         QPalette, QColor, QToolTip)
from PyQt4.QtWebKit import QWebPage, QWebSettings
from PyQt4.QtCore import Qt, QEvent

from PyQt4.Qt import QApplication

from functools import partial

# local
from eilat.WebView import WebView
from eilat.DatabaseLog import DatabaseLogLite
from eilat.libeilat import set_shortcuts, notify
from eilat.global_store import mainwin, clipboard

class WebTab(QWidget):
    """ Cada tab contiene una página web """
    def __init__(self, parent=None):
        super(WebTab, self).__init__(parent)

        self.current = {
            'title': "[EMPTY]",
            'address': ""
        }

        # address bar
        self.address_bar = AddressBar(parent=self)

        # webkit (the actual "web engine")
        self.webkit = WebView(parent=self)
        self.webkit.set_prefix.connect(self.address_bar.set_model)

        # small label displaying instance ID and pending tab operations

        info_label = QLabel(parent=self)
        info_label.setText('?')

        self.webkit.webkit_info.connect(info_label.setText)

        def update_address(qurl):
            """ The 'connect' gives a QUrl and setText receives a string;
            can't just connect setText

            Required because a 3XX HTTP redirection will change the address,
            and without updating, the address bar will be left stale

            """
            self.current['address'] = qurl.toString()
            self.address_bar.setText(self.current['address'])

        self.webkit.urlChanged.connect(update_address)
        self.webkit.loadStarted.connect(self.load_started)
        self.webkit.loadFinished.connect(self.load_finished)
        self.webkit.titleChanged.connect(self.save_title)
        self.webkit.loadProgress.connect(self.load_progress)

        def fill_notifier(message, request):
            """ sends a message to be displayed by the notifier, and starts a
            timer to hide it after eight seconds

            """
            notify(message + " " + request.url().toString())

        self.webkit.page().downloadRequested.connect(
            partial(fill_notifier, "download"))
        self.webkit.page().unsupportedContent.connect(
            partial(fill_notifier, "unsupported"))

        # input area for access-key navigation

        self.nav_bar = NavigateInput(parent=self)
        self.nav_bar.editingFinished.connect(self.webkit.clear_labels)

        self.nav_bar.textEdited.connect(self.webkit.akeynav)
        self.webkit.nonvalid_tag.connect(self.nav_bar.clear)

        # 'corner' message and notification label, not on timer, smaller

        self.message_label = MessageLabel(self.webkit)

        def handle_hovered(link, title, content):
            """ When hovered, if ALT is pressed, show message label;
            hide otherwise

            """

            if ((QApplication.keyboardModifiers() & Qt.AltModifier) and
                    (link or title or content)):
                # ugly hack to ensure proper resizing; find a better way?
                self.message_label.hide()
                self.message_label.setText(
                    link + " " + title + " " + content)
                self.message_label.show()
            else:
                self.message_label.hide()

        self.webkit.page().linkHovered.connect(handle_hovered)

        def handle_signaled(title):
            """ We received a string through a signal; display it on
            the message label

            """

            #if title:
            self.message_label.hide()
            self.message_label.setText(title)
            self.message_label.show()

        self.webkit.display_title.connect(handle_signaled)

        self.webkit.loadStarted.connect(self.message_label.hide)
        self.webkit.page().scrollRequested.connect(self.message_label.hide)
        self.webkit.hide_overlay.connect(self.message_label.hide)

        # progress bar
        self.pbar = QProgressBar(self)

        self.pbar.setRange(0, 100)
        self.pbar.setTextVisible(False)
        self.pbar.setVisible(False)
        self.pbar.setMaximumHeight(7)

        # search in page
        self.search_frame = SearchFrame(parent=self)

        self.webkit.link_selected.connect(self.on_link_hovered)

        self.search_frame.search_line.textChanged.connect(self.do_search)

        # layout
        grid = QGridLayout(self)
        grid.setSpacing(0)
        grid.setVerticalSpacing(0)
        grid.setContentsMargins(0, 0, 0, 0)
        grid.setRowStretch(1, 1)

        grid.addWidget(info_label, 0, 0, 1, 1)
        grid.addWidget(self.address_bar, 0, 1, 1, 1)
        grid.addWidget(self.nav_bar, 0, 2, 1, 1)

        grid.addWidget(self.webkit, 1, 0, 1, 3)

        grid.addWidget(self.search_frame, 2, 0, 1, 3)
        grid.addWidget(self.pbar, 3, 0, 1, 3)

        def show_search():
            """ One-time callback for QShortcut """
            self.search_frame.setVisible(True)
            self.search_frame.search_line.setFocus()

        def hide_search():
            """ One-time callback for QShortcut """
            self.search_frame.setVisible(False)
            self.webkit.findText("")
            self.webkit.setFocus()

        def navigate_completion(key=Qt.Key_Down):
            """ Sends an "arrow press" to the completion popup to navigate
            results.

            Not the best way to do this. It would be better to find out what
            function is being called by that arrow press.

            """
            event = QKeyEvent(QEvent.KeyPress, key, Qt.NoModifier)

            self.address_bar.completer().popup().keyPressEvent(event)

        def reset_addressbar(store=False):
            """ Restore the address bar to its original address and color (it
            could have changed because of a hover event).

            Optionally, store the original address in the clipboard.

            """

            palette = self.address_bar.palette()
            palette.setColor(QPalette.Text, QColor(0, 0, 0))
            self.address_bar.setPalette(palette)

            if self.current['address']:
                self.address_bar.setText(self.current['address'])

            if store:
                clipboard(self.current['address'])

        self.webkit.loadStarted.connect(reset_addressbar)

        set_shortcuts([
            # search
            ("G", self.webkit, show_search),
            ("Escape", self.search_frame, hide_search),
            ("Return", self.search_frame, self.do_search),
            ("Ctrl+J", self.search_frame, self.do_search),
            # go to page
            ("Ctrl+J", self.address_bar, partial(
                self.webkit.navigate, self.address_bar.text)),
            ("Return", self.address_bar, partial(
                self.webkit.navigate, self.address_bar.text)),
            # address bar interaction
            ("Ctrl+L", self.webkit, self.address_bar.setFocus),
            ("Escape", self.address_bar, self.webkit.setFocus),
            ("Ctrl+I", self.address_bar, navigate_completion),
            ("Ctrl+P", self.address_bar, partial(
                navigate_completion, Qt.Key_Up)),
            # in-page element navigation
            ("Ñ", self, self.enter_nav),
            (";", self, self.enter_nav),
            ("Ctrl+Ñ", self, partial(self.enter_nav, target="titles")),
            # toggle
            ("Q", self.webkit, self.toggle_script),
            # clipboard
            ("E", self, partial(reset_addressbar, store=True))
            ])

    def enter_nav(self, target="links"):
        """ A request for access-key navigation was received; display
        link labels and go to the input area

        """

        self.webkit.make_labels(target)
        if self.webkit.map_tags:
            self.nav_bar.show()
            self.nav_bar.setFocus()

    def toggle_script(self):
        """ Activa o desactiva javascript, y notifica cambiando el color
        del address bar

        Callback for shortcut action
        """

        javascript_on = self.webkit.settings().testAttribute(
            QWebSettings.JavascriptEnabled)

        if javascript_on:
            self.webkit.settings().setAttribute(
                QWebSettings.JavascriptEnabled, False)
            self.address_bar.set_color()
        else:
            self.webkit.settings().setAttribute(
                QWebSettings.JavascriptEnabled, True)
            self.address_bar.set_color((230, 230, 255))

    def load_progress(self, val):
        """ Callback for connection """
        if self.pbar.isVisible():
            self.pbar.setValue(val)
            self.set_title("{}% {}".format(val, self.current['title']))

    # connect (en constructor)
    def load_started(self):
        """ Callback for connection """

        self.address_bar.completer().popup().close()
        self.search_frame.setVisible(False)

        self.pbar.setValue(0)
        self.pbar.setVisible(True)

    # connect (en constructor)
    def load_finished(self, success):
        """ Callback for connection """

        self.webkit.navlist = []

        self.pbar.setVisible(False)
        self.set_title(self.current['title'])

        if self.address_bar.hasFocus():
            self.webkit.setFocus()

        if not success:
            notify("[F]")
            print("loadFinished: failed")

    # connect (en constructor)
    def on_link_hovered(self, link):
        """ A link has been selected (no relation with legacy 'link hovered').
        Display the href (if there's no href, it's '') on the (pseudo) status
        bar.

        """

        # change the address bar's color to point out that we're in a
        # pseudo status bar, not the regular address bar
        palette = self.address_bar.palette()
        palette.setColor(QPalette.Text, QColor(127, 127, 255))
        self.address_bar.setPalette(palette)

        self.address_bar.setText(link)

        # if used, overwrites the clipboard every time navigation is performed,
        # either access-key or spatial
        # clipboard(link)

    # connect (en constructor)

    def save_title(self, title):
        """ Store a recently changed title, and display it """
        self.current['title'] = title
        self.set_title(title)

    def set_title(self, title):
        """ Go upwards to the main window's tab widget and set this
        tab's title
        """

        if title is None:
            title = "[NO TITLE]"

        mainwin().tab_widget.setTabText(
            mainwin().tab_widget.indexOf(self),
            title[:40])

    # connection in constructor and action
    def do_search(self, search=None):
        """ Find text on the currently loaded web page. If no text
        is provided, it's extracted from the search widget.

        """
        if search is None:
            search = self.search_frame.search_line.text()
        self.webkit.findText(search, QWebPage.FindWrapsAroundDocument)

class SearchFrame(QFrame):
    """ A frame with a label and a text entry. The text is provided to
    the application upwards to perform in-page search.

    """

    def __init__(self, parent=None):
        super(SearchFrame, self).__init__(parent)

        self.search_grid = QGridLayout(self)
        self.search_grid.setSpacing(0)
        self.search_grid.setContentsMargins(0, 0, 0, 0)
        self.label = QLabel("Find in page:")
        self.search_line = QLineEdit()
        self.search_grid.addWidget(self.label, 0, 0)
        self.search_grid.addWidget(self.search_line, 0, 1)
        self.setVisible(False)

        set_shortcuts([
            ("Ctrl+H", self, self.search_line.backspace),
            ])

class AddressBar(QLineEdit):
    """ A command line of sorts; receives addresses, search terms or
    app-defined commands

    """

    def __init__(self, parent=None):
        super(AddressBar, self).__init__(parent)

        self.database = DatabaseLogLite()
        self.__completer = None

        # This first assignation is to allow a blank tab, who hasn't chosen yet
        # an instance, to display completions for all sites
        self.set_model(None)

        self.set_color()

        set_shortcuts([
            ("Ctrl+H", self, self.backspace),
            # do not create a new tab when on the address bar;
            # popup related trouble
            ("Ctrl+T", self, lambda: None),
            ("Ctrl+Shift+T", self, lambda: None)
        ])

    def focus_in_event(self, event):
        """ Reset the address bar's color if entering; because the
        pseudo-status-bar color is not valid anymore

        """

        palette = self.palette()
        palette.setColor(QPalette.Text, QColor(0, 0, 0))
        self.setPalette(palette)
        QLineEdit.focusInEvent(self, event)

    def set_model(self, prefix):
        """ Update the completion model when the prefix is known. Has to
        be done through an instance variable because of a bug (will not
        complete the line edit otherwise)

        """
        self.__completer = QCompleter(self.database.model(prefix), self)
        self.setCompleter(self.__completer)

    def set_color(self, rgb=(255, 255, 255)):
        """ Sets the background color of the address bar """
        palette = self.palette()
        (red, green, blue) = rgb
        palette.setColor(QPalette.Base, QColor(red, green, blue))
        self.setPalette(palette)

    # Clean reimplement for Qt
    # pylint: disable=C0103
    focusInEvent = focus_in_event
    # pylint: enable=C0103

class NavigateInput(QLineEdit):
    """ When access-key navigation starts, jump to a line edit, where it's
    easier to input the label name than intercepting keystrokes inside
    the web view

    """

    def __init__(self, parent=None):
        super(NavigateInput, self).__init__(parent)

        set_shortcuts([
            ("Ctrl+H", self, self.backspace),
            ("Escape", self, self.exit)
        ])

        self.hide()

    def exit(self):
        """ We are (no matter how) done with the input area; clear and hide """

        self.clear()
        self.hide()

    def focus_out_event(self, _):
        """ If the line edit loses focus (either by regular means, or because
        it has been hidden by 'exit'), notify the webkit

        """

        # it's ok to call twice if the event was result of calling 'exit', the
        # line edit is hidden now and will not send another event
        self.exit()
        self.editingFinished.emit()

    # Clean reimplement for Qt
    # pylint: disable=C0103
    focusOutEvent = focus_out_event
    # pylint: enable=C0103

class MessageLabel(QLabel):
    """ A label to be displayed on the top left corner of the webview;
    it performs most of the functions of a status bar

    """

    def __init__(self, parent=None):
        super(MessageLabel, self).__init__(parent)

        palette = QToolTip.palette()
        color = QColor(Qt.yellow)
        palette.setColor(QPalette.Window, color)

        self.setPalette(palette)
        self.setAutoFillBackground(True)
        self.setFrameStyle(QFrame.Box | QFrame.Plain)

        self.setWordWrap(True)

        self.hide()

<commit_msg>report which url was it where loadFinished failed<commit_after># -*- coding: utf-8 -*-

"""

  Copyright (c) 2012, Davyd McColl; 2013, 2014 Jaime Soffer

   All rights reserved.

   Redistribution and use in source and binary forms, with or without
   modification, are permitted provided that the following conditions are met:

   Redistributions of source code must retain the above copyright notice, this
   list of conditions and the following disclaimer.

   Redistributions in binary form must reproduce the above copyright notice,
   this list of conditions and the following disclaimer in the documentation
   and/or other materials provided with the distribution.

   Neither the name of the involved organizations nor the names of its
   contributors may be used to endorse or promote products derived from this
   software without specific prior written permission.

   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
   THE POSSIBILITY OF SUCH DAMAGE.

"""

from PyQt4.QtGui import (QWidget, QProgressBar, QGridLayout,
                         QFrame, QLabel, QLineEdit, QCompleter, QKeyEvent,
                         QPalette, QColor, QToolTip)
from PyQt4.QtWebKit import QWebPage, QWebSettings
from PyQt4.QtCore import Qt, QEvent

from PyQt4.Qt import QApplication

from functools import partial

# local
from eilat.WebView import WebView
from eilat.DatabaseLog import DatabaseLogLite
from eilat.libeilat import set_shortcuts, notify
from eilat.global_store import mainwin, clipboard

class WebTab(QWidget):
    """ Cada tab contiene una página web """
    def __init__(self, parent=None):
        super(WebTab, self).__init__(parent)

        self.current = {
            'title': "[EMPTY]",
            'address': ""
        }

        # address bar
        self.address_bar = AddressBar(parent=self)

        # webkit (the actual "web engine")
        self.webkit = WebView(parent=self)
        self.webkit.set_prefix.connect(self.address_bar.set_model)

        # small label displaying instance ID and pending tab operations

        info_label = QLabel(parent=self)
        info_label.setText('?')

        self.webkit.webkit_info.connect(info_label.setText)

        def update_address(qurl):
            """ The 'connect' gives a QUrl and setText receives a string;
            can't just connect setText

            Required because a 3XX HTTP redirection will change the address,
            and without updating, the address bar will be left stale

            """
            self.current['address'] = qurl.toString()
            self.address_bar.setText(self.current['address'])

        self.webkit.urlChanged.connect(update_address)
        self.webkit.loadStarted.connect(self.load_started)
        self.webkit.loadFinished.connect(self.load_finished)
        self.webkit.titleChanged.connect(self.save_title)
        self.webkit.loadProgress.connect(self.load_progress)

        def fill_notifier(message, request):
            """ sends a message to be displayed by the notifier, and starts a
            timer to hide it after eight seconds

            """
            notify(message + " " + request.url().toString())

        self.webkit.page().downloadRequested.connect(
            partial(fill_notifier, "download"))
        self.webkit.page().unsupportedContent.connect(
            partial(fill_notifier, "unsupported"))

        # input area for access-key navigation

        self.nav_bar = NavigateInput(parent=self)
        self.nav_bar.editingFinished.connect(self.webkit.clear_labels)

        self.nav_bar.textEdited.connect(self.webkit.akeynav)
        self.webkit.nonvalid_tag.connect(self.nav_bar.clear)

        # 'corner' message and notification label, not on timer, smaller

        self.message_label = MessageLabel(self.webkit)

        def handle_hovered(link, title, content):
            """ When hovered, if ALT is pressed, show message label;
            hide otherwise

            """

            if ((QApplication.keyboardModifiers() & Qt.AltModifier) and
                    (link or title or content)):
                # ugly hack to ensure proper resizing; find a better way?
                self.message_label.hide()
                self.message_label.setText(
                    link + " " + title + " " + content)
                self.message_label.show()
            else:
                self.message_label.hide()

        self.webkit.page().linkHovered.connect(handle_hovered)

        def handle_signaled(title):
            """ We received a string through a signal; display it on
            the message label

            """

            #if title:
            self.message_label.hide()
            self.message_label.setText(title)
            self.message_label.show()

        self.webkit.display_title.connect(handle_signaled)

        self.webkit.loadStarted.connect(self.message_label.hide)
        self.webkit.page().scrollRequested.connect(self.message_label.hide)
        self.webkit.hide_overlay.connect(self.message_label.hide)

        # progress bar
        self.pbar = QProgressBar(self)

        self.pbar.setRange(0, 100)
        self.pbar.setTextVisible(False)
        self.pbar.setVisible(False)
        self.pbar.setMaximumHeight(7)

        # search in page
        self.search_frame = SearchFrame(parent=self)

        self.webkit.link_selected.connect(self.on_link_hovered)

        self.search_frame.search_line.textChanged.connect(self.do_search)

        # layout
        grid = QGridLayout(self)
        grid.setSpacing(0)
        grid.setVerticalSpacing(0)
        grid.setContentsMargins(0, 0, 0, 0)
        grid.setRowStretch(1, 1)

        grid.addWidget(info_label, 0, 0, 1, 1)
        grid.addWidget(self.address_bar, 0, 1, 1, 1)
        grid.addWidget(self.nav_bar, 0, 2, 1, 1)

        grid.addWidget(self.webkit, 1, 0, 1, 3)

        grid.addWidget(self.search_frame, 2, 0, 1, 3)
        grid.addWidget(self.pbar, 3, 0, 1, 3)

        def show_search():
            """ One-time callback for QShortcut """
            self.search_frame.setVisible(True)
            self.search_frame.search_line.setFocus()

        def hide_search():
            """ One-time callback for QShortcut """
            self.search_frame.setVisible(False)
            self.webkit.findText("")
            self.webkit.setFocus()

        def navigate_completion(key=Qt.Key_Down):
            """ Sends an "arrow press" to the completion popup to navigate
            results.

            Not the best way to do this. It would be better to find out what
            function is being called by that arrow press.

            """
            event = QKeyEvent(QEvent.KeyPress, key, Qt.NoModifier)

            self.address_bar.completer().popup().keyPressEvent(event)

        def reset_addressbar(store=False):
            """ Restore the address bar to its original address and color (it
            could have changed because of a hover event).

            Optionally, store the original address in the clipboard.

            """

            palette = self.address_bar.palette()
            palette.setColor(QPalette.Text, QColor(0, 0, 0))
            self.address_bar.setPalette(palette)

            if self.current['address']:
                self.address_bar.setText(self.current['address'])

            if store:
                clipboard(self.current['address'])

        self.webkit.loadStarted.connect(reset_addressbar)

        set_shortcuts([
            # search
            ("G", self.webkit, show_search),
            ("Escape", self.search_frame, hide_search),
            ("Return", self.search_frame, self.do_search),
            ("Ctrl+J", self.search_frame, self.do_search),
            # go to page
            ("Ctrl+J", self.address_bar, partial(
                self.webkit.navigate, self.address_bar.text)),
            ("Return", self.address_bar, partial(
                self.webkit.navigate, self.address_bar.text)),
            # address bar interaction
            ("Ctrl+L", self.webkit, self.address_bar.setFocus),
            ("Escape", self.address_bar, self.webkit.setFocus),
            ("Ctrl+I", self.address_bar, navigate_completion),
            ("Ctrl+P", self.address_bar, partial(
                navigate_completion, Qt.Key_Up)),
            # in-page element navigation
            ("Ñ", self, self.enter_nav),
            (";", self, self.enter_nav),
            ("Ctrl+Ñ", self, partial(self.enter_nav, target="titles")),
            # toggle
            ("Q", self.webkit, self.toggle_script),
            # clipboard
            ("E", self, partial(reset_addressbar, store=True))
            ])

    def enter_nav(self, target="links"):
        """ A request for access-key navigation was received; display
        link labels and go to the input area

        """

        self.webkit.make_labels(target)
        if self.webkit.map_tags:
            self.nav_bar.show()
            self.nav_bar.setFocus()

    def toggle_script(self):
        """ Activa o desactiva javascript, y notifica cambiando el color
        del address bar

        Callback for shortcut action
        """

        javascript_on = self.webkit.settings().testAttribute(
            QWebSettings.JavascriptEnabled)

        if javascript_on:
            self.webkit.settings().setAttribute(
                QWebSettings.JavascriptEnabled, False)
            self.address_bar.set_color()
        else:
            self.webkit.settings().setAttribute(
                QWebSettings.JavascriptEnabled, True)
            self.address_bar.set_color((230, 230, 255))

    def load_progress(self, val):
        """ Callback for connection """
        if self.pbar.isVisible():
            self.pbar.setValue(val)
            self.set_title("{}% {}".format(val, self.current['title']))

    # connect (en constructor)
    def load_started(self):
        """ Callback for connection """

        self.address_bar.completer().popup().close()
        self.search_frame.setVisible(False)

        self.pbar.setValue(0)
        self.pbar.setVisible(True)

    # connect (en constructor)
    def load_finished(self, success):
        """ Callback for connection """

        self.webkit.navlist = []

        self.pbar.setVisible(False)
        self.set_title(self.current['title'])

        if self.address_bar.hasFocus():
            self.webkit.setFocus()

        if not success:
            notify("[F]")
            print("loadFinished: failed", self.webkit.url())

    # connect (en constructor)
    def on_link_hovered(self, link):
        """ A link has been selected (no relation with legacy 'link hovered').
        Display the href (if there's no href, it's '') on the (pseudo) status
        bar.

        """

        # change the address bar's color to point out that we're in a
        # pseudo status bar, not the regular address bar
        palette = self.address_bar.palette()
        palette.setColor(QPalette.Text, QColor(127, 127, 255))
        self.address_bar.setPalette(palette)

        self.address_bar.setText(link)

        # if used, overwrites the clipboard every time navigation is performed,
        # either access-key or spatial
        # clipboard(link)

    # connect (en constructor)

    def save_title(self, title):
        """ Store a recently changed title, and display it """
        self.current['title'] = title
        self.set_title(title)

    def set_title(self, title):
        """ Go upwards to the main window's tab widget and set this
        tab's title
        """

        if title is None:
            title = "[NO TITLE]"

        mainwin().tab_widget.setTabText(
            mainwin().tab_widget.indexOf(self),
            title[:40])

    # connection in constructor and action
    def do_search(self, search=None):
        """ Find text on the currently loaded web page. If no text
        is provided, it's extracted from the search widget.

        """
        if search is None:
            search = self.search_frame.search_line.text()
        self.webkit.findText(search, QWebPage.FindWrapsAroundDocument)

class SearchFrame(QFrame):
    """ A frame with a label and a text entry. The text is provided to
    the application upwards to perform in-page search.

    """

    def __init__(self, parent=None):
        super(SearchFrame, self).__init__(parent)

        self.search_grid = QGridLayout(self)
        self.search_grid.setSpacing(0)
        self.search_grid.setContentsMargins(0, 0, 0, 0)
        self.label = QLabel("Find in page:")
        self.search_line = QLineEdit()
        self.search_grid.addWidget(self.label, 0, 0)
        self.search_grid.addWidget(self.search_line, 0, 1)
        self.setVisible(False)

        set_shortcuts([
            ("Ctrl+H", self, self.search_line.backspace),
            ])

class AddressBar(QLineEdit):
    """ A command line of sorts; receives addresses, search terms or
    app-defined commands

    """

    def __init__(self, parent=None):
        super(AddressBar, self).__init__(parent)

        self.database = DatabaseLogLite()
        self.__completer = None

        # This first assignation is to allow a blank tab, who hasn't chosen yet
        # an instance, to display completions for all sites
        self.set_model(None)

        self.set_color()

        set_shortcuts([
            ("Ctrl+H", self, self.backspace),
            # do not create a new tab when on the address bar;
            # popup related trouble
            ("Ctrl+T", self, lambda: None),
            ("Ctrl+Shift+T", self, lambda: None)
        ])

    def focus_in_event(self, event):
        """ Reset the address bar's color if entering; because the
        pseudo-status-bar color is not valid anymore

        """

        palette = self.palette()
        palette.setColor(QPalette.Text, QColor(0, 0, 0))
        self.setPalette(palette)
        QLineEdit.focusInEvent(self, event)

    def set_model(self, prefix):
        """ Update the completion model when the prefix is known. Has to
        be done through an instance variable because of a bug (will not
        complete the line edit otherwise)

        """
        self.__completer = QCompleter(self.database.model(prefix), self)
        self.setCompleter(self.__completer)

    def set_color(self, rgb=(255, 255, 255)):
        """ Sets the background color of the address bar """
        palette = self.palette()
        (red, green, blue) = rgb
        palette.setColor(QPalette.Base, QColor(red, green, blue))
        self.setPalette(palette)

    # Clean reimplement for Qt
    # pylint: disable=C0103
    focusInEvent = focus_in_event
    # pylint: enable=C0103

class NavigateInput(QLineEdit):
    """ When access-key navigation starts, jump to a line edit, where it's
    easier to input the label name than intercepting keystrokes inside
    the web view

    """

    def __init__(self, parent=None):
        super(NavigateInput, self).__init__(parent)

        set_shortcuts([
            ("Ctrl+H", self, self.backspace),
            ("Escape", self, self.exit)
        ])

        self.hide()

    def exit(self):
        """ We are (no matter how) done with the input area; clear and hide """

        self.clear()
        self.hide()

    def focus_out_event(self, _):
        """ If the line edit loses focus (either by regular means, or because
        it has been hidden by 'exit'), notify the webkit

        """

        # it's ok to call twice if the event was result of calling 'exit', the
        # line edit is hidden now and will not send another event
        self.exit()
        self.editingFinished.emit()

    # Clean reimplement for Qt
    # pylint: disable=C0103
    focusOutEvent = focus_out_event
    # pylint: enable=C0103

class MessageLabel(QLabel):
    """ A label to be displayed on the top left corner of the webview;
    it performs most of the functions of a status bar

    """

    def __init__(self, parent=None):
        super(MessageLabel, self).__init__(parent)

        palette = QToolTip.palette()
        color = QColor(Qt.yellow)
        palette.setColor(QPalette.Window, color)

        self.setPalette(palette)
        self.setAutoFillBackground(True)
        self.setFrameStyle(QFrame.Box | QFrame.Plain)

        self.setWordWrap(True)

        self.hide()

<|endoftext|>


<commit_before>from .pandas_source import PandasDatasource
from .sqlalchemy_source import SqlAlchemyDatasource
from .spark_source import SparkDFDatasource

from .filesystem_path_generator import SubdirReaderGenerator, GlobReaderGenerator<commit_msg>Streamline datasource imports<commit_after>from .datasource import  Datasource
from .pandas_source import PandasDatasource
from .sqlalchemy_source import SqlAlchemyDatasource
from .spark_source import SparkDFDatasource
from .dbt_source import DBTDatasource

from .filesystem_path_generator import SubdirReaderGenerator, GlobReaderGenerator
<|endoftext|>


<commit_before>#!/usr/bin/env python

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

from builtins import str

from future import standard_library
standard_library.install_aliases()

import dns.exception
import dns.resolver
import logging
import os
import requests
import sys
import time

from tld import get_tld

# Enable verified HTTPS requests on older Pythons
# http://urllib3.readthedocs.org/en/latest/security.html
if sys.version_info[0] == 2:
    try:
        requests.packages.urllib3.contrib.pyopenssl.inject_into_urllib3()
    except AttributeError:
        # see https://github.com/certbot/certbot/issues/1883
        import urllib3.contrib.pyopenssl
        urllib3.contrib.pyopenssl.inject_into_urllib3()

logger = logging.getLogger(__name__)
logger.addHandler(logging.StreamHandler())

if os.environ.get('CF_DEBUG'):
    logger.setLevel(logging.DEBUG)
else:
    logger.setLevel(logging.INFO)

try:
    CF_HEADERS = {
        'X-Auth-Email': os.environ['CF_EMAIL'],
        'X-Auth-Key'  : os.environ['CF_KEY'],
        'Content-Type': 'application/json',
    }
except KeyError:
    logger.error(" + Unable to locate Cloudflare credentials in environment!")
    sys.exit(1)

try:
    dns_servers = os.environ['CF_DNS_SERVERS']
    dns_servers = dns_servers.split()
except KeyError:
    dns_servers = False

def _has_dns_propagated(name, token):
    try:
        if dns_servers:
            custom_resolver = dns.resolver.Resolver()
            custom_resolver.nameservers = dns_servers
            dns_response = custom_resolver.query(name, 'TXT')
        else:
            dns_response = dns.resolver.query(name, 'TXT')

        for rdata in dns_response:
            if token in [b.decode('utf-8') for b in rdata.strings]:
                return True

    except dns.exception.DNSException as e:
        logger.debug(" + {0}. Retrying query...".format(e))

# https://api.cloudflare.com/#zone-list-zones
def _get_zone_id(domain):
    tld = get_tld('http://' + domain)
    url = "https://api.cloudflare.com/client/v4/zones?name={0}".format(tld)
    r = requests.get(url, headers=CF_HEADERS)
    r.raise_for_status()
    return r.json()['result'][0]['id']

# https://api.cloudflare.com/#dns-records-for-a-zone-dns-record-details
def _get_txt_record_id(zone_id, name, token):
    url = "https://api.cloudflare.com/client/v4/zones/{0}/dns_records?type=TXT&name={1}&content={2}".format(zone_id, name, token)
    r = requests.get(url, headers=CF_HEADERS)
    r.raise_for_status()
    try:
        record_id = r.json()['result'][0]['id']
    except IndexError:
        logger.debug(" + Unable to locate record named {0}".format(name))
        return

    return record_id

# https://api.cloudflare.com/#dns-records-for-a-zone-create-dns-record
def create_txt_record(args):
    domain, token = args[0], args[2]
    logger.debug(' + Creating TXT record: {0} => {1}'.format(domain, token))
    zone_id = _get_zone_id(domain)
    name = "{0}.{1}".format('_acme-challenge', domain)

    record_id = _get_txt_record_id(zone_id, name, token)
    if record_id:
        logger.debug(" + TXT record exists, skipping creation.")
        return

    url = "https://api.cloudflare.com/client/v4/zones/{0}/dns_records".format(zone_id)
    payload = {
        'type': 'TXT',
        'name': name,
        'content': token,
        'ttl': 1,
    }
    r = requests.post(url, headers=CF_HEADERS, json=payload)
    r.raise_for_status()
    record_id = r.json()['result']['id']
    logger.debug(" + TXT record created, ID: {0}".format(record_id))

# https://api.cloudflare.com/#dns-records-for-a-zone-delete-dns-record
def delete_txt_record(args):
    domain, token = args[0], args[2]
    if not domain:
        logger.info(" + http_request() error in letsencrypt.sh?")
        return

    zone_id = _get_zone_id(domain)
    name = "{0}.{1}".format('_acme-challenge', domain)
    record_id = _get_txt_record_id(zone_id, name, token)

    logger.debug(" + Deleting TXT record name: {0}".format(name))
    url = "https://api.cloudflare.com/client/v4/zones/{0}/dns_records/{1}".format(zone_id, record_id)
    r = requests.delete(url, headers=CF_HEADERS)
    r.raise_for_status()

def deploy_cert(args):
    domain, privkey_pem, cert_pem, fullchain_pem, chain_pem, timestamp = args
    logger.info(' + ssl_certificate: {0}'.format(fullchain_pem))
    logger.info(' + ssl_certificate_key: {0}'.format(privkey_pem))
    return

def unchanged_cert(args):
    return

def invalid_challenge(args):
    return

def create_all_txt_records(args):
    X = 3
    for i in range(0, len(args), X):
        create_txt_record(args[i:i+X])
    # give it 10 seconds to settle down and avoid nxdomain caching
    logger.info(" + Settling down for 10s...")
    time.sleep(10)
    for i in range(0, len(args), X):
        domain, token = args[i], args[i+2]
        name = "{0}.{1}".format('_acme-challenge', domain)
        while(_has_dns_propagated(name, token) == False):
            logger.info(" + DNS not propagated, waiting 30s...")
            time.sleep(30)

def delete_all_txt_records(args):
    X = 3
    for i in range(0, len(args), X):
        delete_txt_record(args[i:i+X])

def main(argv):
    ops = {
        'deploy_challenge': create_all_txt_records,
        'clean_challenge' : delete_all_txt_records,
        'deploy_cert'     : deploy_cert,
        'unchanged_cert'  : unchanged_cert,
        'invalid_challenge': invalid_challenge,
    }
    logger.info(" + CloudFlare hook executing: {0}".format(argv[0]))
    ops[argv[0]](argv[1:])

if __name__ == '__main__':
    main(sys.argv[1:])
<commit_msg>Optimize debug output<commit_after>#!/usr/bin/env python

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

from builtins import str

from future import standard_library
standard_library.install_aliases()

import dns.exception
import dns.resolver
import logging
import os
import requests
import sys
import time

from tld import get_tld

# Enable verified HTTPS requests on older Pythons
# http://urllib3.readthedocs.org/en/latest/security.html
if sys.version_info[0] == 2:
    try:
        requests.packages.urllib3.contrib.pyopenssl.inject_into_urllib3()
    except AttributeError:
        # see https://github.com/certbot/certbot/issues/1883
        import urllib3.contrib.pyopenssl
        urllib3.contrib.pyopenssl.inject_into_urllib3()

logger = logging.getLogger(__name__)
logger.addHandler(logging.StreamHandler())

if os.environ.get('CF_DEBUG'):
    logger.setLevel(logging.DEBUG)
else:
    logger.setLevel(logging.INFO)

try:
    CF_HEADERS = {
        'X-Auth-Email': os.environ['CF_EMAIL'],
        'X-Auth-Key'  : os.environ['CF_KEY'],
        'Content-Type': 'application/json',
    }
except KeyError:
    logger.error(" + Unable to locate Cloudflare credentials in environment!")
    sys.exit(1)

try:
    dns_servers = os.environ['CF_DNS_SERVERS']
    dns_servers = dns_servers.split()
except KeyError:
    dns_servers = False

def _has_dns_propagated(name, token):
    try:
        if dns_servers:
            custom_resolver = dns.resolver.Resolver()
            custom_resolver.nameservers = dns_servers
            dns_response = custom_resolver.query(name, 'TXT')
        else:
            dns_response = dns.resolver.query(name, 'TXT')

        for rdata in dns_response:
            if token in [b.decode('utf-8') for b in rdata.strings]:
                return True

    except dns.exception.DNSException as e:
        logger.debug(" + {0}. Retrying query...".format(e))

# https://api.cloudflare.com/#zone-list-zones
def _get_zone_id(domain):
    tld = get_tld('http://' + domain)
    url = "https://api.cloudflare.com/client/v4/zones?name={0}".format(tld)
    r = requests.get(url, headers=CF_HEADERS)
    r.raise_for_status()
    return r.json()['result'][0]['id']

# https://api.cloudflare.com/#dns-records-for-a-zone-dns-record-details
def _get_txt_record_id(zone_id, name, token):
    url = "https://api.cloudflare.com/client/v4/zones/{0}/dns_records?type=TXT&name={1}&content={2}".format(zone_id, name, token)
    r = requests.get(url, headers=CF_HEADERS)
    r.raise_for_status()
    try:
        record_id = r.json()['result'][0]['id']
    except IndexError:
        logger.debug(" + Unable to locate record named {0}".format(name))
        return

    return record_id

# https://api.cloudflare.com/#dns-records-for-a-zone-create-dns-record
def create_txt_record(args):
    domain, challenge, token = args
    logger.debug(' + Creating TXT record: {0} => {1}'.format(domain, token))
    logger.debug(' + Challenge: {0}'.format(challenge))
    zone_id = _get_zone_id(domain)
    name = "{0}.{1}".format('_acme-challenge', domain)

    record_id = _get_txt_record_id(zone_id, name, token)
    if record_id:
        logger.debug(" + TXT record exists, skipping creation.")
        return

    url = "https://api.cloudflare.com/client/v4/zones/{0}/dns_records".format(zone_id)
    payload = {
        'type': 'TXT',
        'name': name,
        'content': token,
        'ttl': 1,
    }
    r = requests.post(url, headers=CF_HEADERS, json=payload)
    r.raise_for_status()
    record_id = r.json()['result']['id']
    logger.debug(" + TXT record created, CFID: {0}".format(record_id))

# https://api.cloudflare.com/#dns-records-for-a-zone-delete-dns-record
def delete_txt_record(args):
    domain, token = args[0], args[2]
    if not domain:
        logger.info(" + http_request() error in letsencrypt.sh?")
        return

    zone_id = _get_zone_id(domain)
    name = "{0}.{1}".format('_acme-challenge', domain)
    record_id = _get_txt_record_id(zone_id, name, token)

    if record_id:
        url = "https://api.cloudflare.com/client/v4/zones/{0}/dns_records/{1}".format(zone_id, record_id)
        r = requests.delete(url, headers=CF_HEADERS)
        r.raise_for_status()
        logger.debug(" + Deleted TXT {0}, CFID {1}".format(name, record_id))
    else:
        logger.debug(" + No TXT {0} with token {1}".format(name, token))

def deploy_cert(args):
    domain, privkey_pem, cert_pem, fullchain_pem, chain_pem, timestamp = args
    logger.debug(' + ssl_certificate: {0}'.format(fullchain_pem))
    logger.debug(' + ssl_certificate_key: {0}'.format(privkey_pem))
    return

def unchanged_cert(args):
    return

def invalid_challenge(args):
    domain, result = args
    logger.debug(' + invalid_challenge for {0}'.format(domain))
    logger.debug(' + Full error: {0}'.format(result))
    return

def create_all_txt_records(args):
    X = 3
    for i in range(0, len(args), X):
        create_txt_record(args[i:i+X])
    # give it 10 seconds to settle down and avoid nxdomain caching
    logger.info(" + Settling down for 10s...")
    time.sleep(10)
    for i in range(0, len(args), X):
        domain, token = args[i], args[i+2]
        name = "{0}.{1}".format('_acme-challenge', domain)
        while(_has_dns_propagated(name, token) == False):
            logger.info(" + DNS not propagated, waiting 30s...")
            time.sleep(30)

def delete_all_txt_records(args):
    X = 3
    for i in range(0, len(args), X):
        delete_txt_record(args[i:i+X])

def main(argv):
    ops = {
        'deploy_challenge': create_all_txt_records,
        'clean_challenge' : delete_all_txt_records,
        'deploy_cert'     : deploy_cert,
        'unchanged_cert'  : unchanged_cert,
        'invalid_challenge': invalid_challenge,
    }
    logger.info(" + CloudFlare hook executing: {0}".format(argv[0]))
    ops[argv[0]](argv[1:])

if __name__ == '__main__':
    main(sys.argv[1:])
<|endoftext|>


<commit_before>#import numpy as np

import model_sim as model
import trec_utils as trec
from dl_text.metrics import eval_metric
from dl_text import dl

glove_fname = 'D:/workspace/Trec_QA-master/data/Glove/glove.6B.50d.txt'

################### DEFINING MODEL AND PREDICTION FILE ###################

lrmodel = model.cnn_sim_ft
model_name = lrmodel.func_name

################### DEFINING HYPERPARAMETERS ###################

dimx = 50
dimy = 50
dimft = 44
batch_size = 70
vocab_size = 10000
embedding_dim = 50
nb_filter = 120
filter_length = (50,4)
depth = 1
nb_epoch = 4

ques, ans, label_train, train_len, test_len,\
         wordVec_model, res_fname, pred_fname, feat_train, feat_test = trec.load_trec(model_name, glove_fname)
data_l , data_r, embedding_matrix = dl.process_data(ques, ans,
                                                 wordVec_model,dimx=dimx,
                                                 dimy=dimy,vocab_size=vocab_size,
                                                 embedding_dim=embedding_dim)

X_train_l,X_test_l,X_dev_l,X_train_r,X_test_r,X_dev_r = trec.prepare_train_test(data_l,data_r,
                                                                           train_len,test_len)

if model_name == 'cnn_sim_ft':
    lrmodel = lrmodel(embedding_matrix, dimx=dimx, dimy=dimy, dimft=dimft, nb_filter = nb_filter,
                      embedding_dim = embedding_dim,filter_length = filter_length, vocab_size = vocab_size,
                      depth = depth)

    print '\n',model_name,'model built \n'
    lrmodel.fit([X_train_l, X_train_r,feat_train],label_train,batch_size=batch_size,nb_epoch=nb_epoch,verbose=2)
    map_val, mrr_val = eval_metric(lrmodel, X_test_l, X_test_r, res_fname, pred_fname, feat_test=feat_test)

else:
    lrmodel = lrmodel(embedding_matrix, dimx=dimx, dimy=dimy, nb_filter = nb_filter, embedding_dim = embedding_dim,
                      filter_length = filter_length, vocab_size = vocab_size, depth = depth)

    print '\n', model_name,'model built \n'
    lrmodel.fit([X_train_l, X_train_r],label_train,batch_size=batch_size,nb_epoch=nb_epoch,verbose=2)
    map_val, mrr_val = eval_metric(lrmodel, X_test_l, X_test_r, res_fname, pred_fname)

print 'MAP : ',map_val,' MRR : ',mrr_val
<commit_msg>Update main.py<commit_after># --**deeplean-ai.com**--
# --**dl-lab**--

#import numpy as np

import model_sim as model
import trec_utils as trec
from dl_text.metrics import eval_metric
from dl_text import dl

glove_fname = 'D:/workspace/Trec_QA-master/data/Glove/glove.6B.50d.txt'

################### DEFINING MODEL AND PREDICTION FILE ###################

lrmodel = model.cnn_sim_ft
model_name = lrmodel.func_name

################### DEFINING HYPERPARAMETERS ###################

dimx = 50
dimy = 50
dimft = 44
batch_size = 70
vocab_size = 10000
embedding_dim = 50
nb_filter = 120
filter_length = (50,4)
depth = 1
nb_epoch = 4

ques, ans, label_train, train_len, test_len,\
         wordVec_model, res_fname, pred_fname, feat_train, feat_test = trec.load_trec(model_name, glove_fname)
data_l , data_r, embedding_matrix = dl.process_data(ques, ans,
                                                 wordVec_model,dimx=dimx,
                                                 dimy=dimy,vocab_size=vocab_size,
                                                 embedding_dim=embedding_dim)

X_train_l,X_test_l,X_dev_l,X_train_r,X_test_r,X_dev_r = trec.prepare_train_test(data_l,data_r,
                                                                           train_len,test_len)

if model_name == 'cnn_sim_ft':
    lrmodel = lrmodel(embedding_matrix, dimx=dimx, dimy=dimy, dimft=dimft, nb_filter = nb_filter,
                      embedding_dim = embedding_dim,filter_length = filter_length, vocab_size = vocab_size,
                      depth = depth)

    print '\n',model_name,'model built \n'
    lrmodel.fit([X_train_l, X_train_r,feat_train],label_train,batch_size=batch_size,nb_epoch=nb_epoch,verbose=2)
    map_val, mrr_val = eval_metric(lrmodel, X_test_l, X_test_r, res_fname, pred_fname, feat_test=feat_test)

else:
    lrmodel = lrmodel(embedding_matrix, dimx=dimx, dimy=dimy, nb_filter = nb_filter, embedding_dim = embedding_dim,
                      filter_length = filter_length, vocab_size = vocab_size, depth = depth)

    print '\n', model_name,'model built \n'
    lrmodel.fit([X_train_l, X_train_r],label_train,batch_size=batch_size,nb_epoch=nb_epoch,verbose=2)
    map_val, mrr_val = eval_metric(lrmodel, X_test_l, X_test_r, res_fname, pred_fname)

print 'MAP : ',map_val,' MRR : ',mrr_val
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Django settings for basic pinax project.

import os.path
import datetime
import time

from django.utils.http import http_date

import djcelery

djcelery.setup_loader()

PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir))

DEBUG = False
TEMPLATE_DEBUG = True

SERVE_MEDIA = DEBUG
SERVE_LESS = DEBUG

# django-compressor is turned off by default due to deployment overhead for
# most users. See <URL> for more information
COMPRESS = False

INTERNAL_IPS = [
    "127.0.0.1",
]

DATABASES = {
    "default": {
        "ENGINE": "django.db.backends.postgresql_psycopg2",
        "NAME": "crate",
    }
}

TIME_ZONE = "UTC"
LANGUAGE_CODE = "en-us"

USE_I18N = True
USE_L10N = True
USE_TZ = True

MEDIA_ROOT = os.path.join(PROJECT_ROOT, "site_media", "media")
MEDIA_URL = "/site_media/media/"

STATIC_ROOT = os.path.join(PROJECT_ROOT, "site_media", "static")
STATIC_URL = "/site_media/static/"

STATICFILES_DIRS = [
    os.path.join(PROJECT_ROOT, "static"),
]

STATICFILES_FINDERS = [
    "staticfiles.finders.FileSystemFinder",
    "staticfiles.finders.AppDirectoriesFinder",
    "staticfiles.finders.LegacyAppDirectoriesFinder",
    "compressor.finders.CompressorFinder",
]

COMPRESS_OUTPUT_DIR = "cache"

SECRET_KEY = ")ys)^6l18seiq)veen^41ox%l@6#o-2t*9-8n5$7$f+qi*x03$"

TEMPLATE_LOADERS = [
    "django.template.loaders.filesystem.Loader",
    "django.template.loaders.app_directories.Loader",
]

MIDDLEWARE_CLASSES = [
    "django.middleware.common.CommonMiddleware",
    "django_hosts.middleware.HostsMiddleware",
    "django.contrib.sessions.middleware.SessionMiddleware",
    "django.middleware.csrf.CsrfViewMiddleware",
    "django.contrib.auth.middleware.AuthenticationMiddleware",
    "django_openid.consumer.SessionConsumer",
    "django.contrib.messages.middleware.MessageMiddleware",
    "django.middleware.clickjacking.XFrameOptionsMiddleware",

    "pagination.middleware.PaginationMiddleware",

    "pinax.apps.account.middleware.LocaleMiddleware",
    "pinax.middleware.security.HideSensistiveFieldsMiddleware",
]

ROOT_URLCONF = "crate_project.urls"
ROOT_HOSTCONF = "crate_project.hosts"

DEFAULT_HOST = "default"

WSGI_APPLICATION = "crate_project.wsgi.application"

TEMPLATE_DIRS = [
    os.path.join(PROJECT_ROOT, "templates"),
]

TEMPLATE_CONTEXT_PROCESSORS = [
    "django.contrib.auth.context_processors.auth",
    "django.core.context_processors.debug",
    "django.core.context_processors.i18n",
    "django.core.context_processors.media",
    "django.core.context_processors.request",
    "django.contrib.messages.context_processors.messages",

    "staticfiles.context_processors.static",

    "pinax.core.context_processors.pinax_settings",

    "pinax.apps.account.context_processors.account",

    # "notification.context_processors.notification",
    "announcements.context_processors.site_wide_announcements",
]

INSTALLED_APPS = [
    # Django
    "django.contrib.admin",
    "django.contrib.auth",
    "django.contrib.contenttypes",
    "django.contrib.sessions",
    "django.contrib.sites",
    "django.contrib.messages",
    "django.contrib.humanize",
    "django.contrib.markup",

    "pinax.templatetags",

    # theme
    "pinax_theme_bootstrap",

    # external (Pinax)
    # "notification",  # must be first
    "staticfiles",
    "pagination",
    "compressor",
    "django_openid",
    "timezones",
    "emailconfirmation",
    "announcements",
    "idios",
    "metron",

    # Pinax
    "pinax.apps.account",
    "pinax.apps.signup_codes",

    # external (Project)
    "south",
    "djcelery",
    "django_hosts",
    "haystack",
    "storages",
    "intercom",
    "celery_haystack",

    # project
    "about",
    "profiles",
    "packages",
    "pypi",
    "search",
    "crate",
    "privatebeta",
]

FIXTURE_DIRS = [
    os.path.join(PROJECT_ROOT, "fixtures"),
]

MESSAGE_STORAGE = "django.contrib.messages.storage.session.SessionStorage"

ABSOLUTE_URL_OVERRIDES = {
    "auth.user": lambda o: "/profiles/profile/%s/" % o.username,
}

AUTH_PROFILE_MODULE = "profiles.Profile"
NOTIFICATION_LANGUAGE_MODULE = "account.Account"

ACCOUNT_OPEN_SIGNUP = False
ACCOUNT_USE_OPENID = False
ACCOUNT_REQUIRED_EMAIL = True
ACCOUNT_EMAIL_VERIFICATION = True
ACCOUNT_EMAIL_AUTHENTICATION = False
ACCOUNT_UNIQUE_EMAIL = EMAIL_CONFIRMATION_UNIQUE_EMAIL = True

AUTHENTICATION_BACKENDS = [
    "pinax.apps.account.auth_backends.AuthenticationBackend",
]

LOGIN_URL = "/account/login/"
LOGIN_REDIRECT_URLNAME = "what_next"
LOGOUT_REDIRECT_URLNAME = "search"

EMAIL_CONFIRMATION_DAYS = 2
EMAIL_DEBUG = DEBUG

DEBUG_TOOLBAR_CONFIG = {
    "INTERCEPT_REDIRECTS": False,
}

CELERY_SEND_TASK_ERROR_EMAILS = True
CELERY_DISABLE_RATE_LIMITS = True
CELERY_TASK_PUBLISH_RETRY = True

CELERY_TASK_RESULT_EXPIRES = 7 * 24 * 60 * 60  # 7 Days
CELERYD_TASK_TIME_LIMIT = 120
CELERYD_TASK_SOFT_TIME_LIMIT = 30

CELERYD_HIJACK_ROOT_LOGGER = False

CELERYBEAT_SCHEDULER = "djcelery.schedulers.DatabaseScheduler"

HAYSTACK_SEARCH_RESULTS_PER_PAGE = 15

AWS_QUERYSTRING_AUTH = False
AWS_S3_SECURE_URLS = False

AWS_HEADERS = {
    "Expires": lambda: http_date(time.mktime((datetime.datetime.now() + datetime.timedelta(days=365)).timetuple())),
    "Cache-Control": "max-age=31556926",
}

METRON_SETTINGS = {
    "mixpanel": {3: "a280fbfb3039709d8a5ac15834a16e5e"},
    # "google": {3: ""},
    "gauges": {3: "4f1e4cd0613f5d7003000002"}
}
<commit_msg>switch to using whole page cache for anonymous users<commit_after># -*- coding: utf-8 -*-
# Django settings for basic pinax project.

import os.path
import datetime
import time

from django.utils.http import http_date

import djcelery

djcelery.setup_loader()

PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir))

DEBUG = False
TEMPLATE_DEBUG = True

SERVE_MEDIA = DEBUG
SERVE_LESS = DEBUG

# django-compressor is turned off by default due to deployment overhead for
# most users. See <URL> for more information
COMPRESS = False

INTERNAL_IPS = [
    "127.0.0.1",
]

DATABASES = {
    "default": {
        "ENGINE": "django.db.backends.postgresql_psycopg2",
        "NAME": "crate",
    }
}

TIME_ZONE = "UTC"
LANGUAGE_CODE = "en-us"

USE_I18N = True
USE_L10N = True
USE_TZ = True

MEDIA_ROOT = os.path.join(PROJECT_ROOT, "site_media", "media")
MEDIA_URL = "/site_media/media/"

STATIC_ROOT = os.path.join(PROJECT_ROOT, "site_media", "static")
STATIC_URL = "/site_media/static/"

STATICFILES_DIRS = [
    os.path.join(PROJECT_ROOT, "static"),
]

STATICFILES_FINDERS = [
    "staticfiles.finders.FileSystemFinder",
    "staticfiles.finders.AppDirectoriesFinder",
    "staticfiles.finders.LegacyAppDirectoriesFinder",
    "compressor.finders.CompressorFinder",
]

COMPRESS_OUTPUT_DIR = "cache"

SECRET_KEY = ")ys)^6l18seiq)veen^41ox%l@6#o-2t*9-8n5$7$f+qi*x03$"

TEMPLATE_LOADERS = [
    "django.template.loaders.filesystem.Loader",
    "django.template.loaders.app_directories.Loader",
]

MIDDLEWARE_CLASSES = [
    "django.middleware.cache.UpdateCacheMiddleware",
    "django.middleware.common.CommonMiddleware",
    "django_hosts.middleware.HostsMiddleware",
    "django.contrib.sessions.middleware.SessionMiddleware",
    "django.middleware.csrf.CsrfViewMiddleware",
    "django.contrib.auth.middleware.AuthenticationMiddleware",
    "django_openid.consumer.SessionConsumer",
    "django.contrib.messages.middleware.MessageMiddleware",
    "django.middleware.clickjacking.XFrameOptionsMiddleware",

    "pagination.middleware.PaginationMiddleware",

    "pinax.apps.account.middleware.LocaleMiddleware",
    "pinax.middleware.security.HideSensistiveFieldsMiddleware",
    "django.middleware.cache.FetchFromCacheMiddleware",
]

ROOT_URLCONF = "crate_project.urls"
ROOT_HOSTCONF = "crate_project.hosts"

DEFAULT_HOST = "default"

WSGI_APPLICATION = "crate_project.wsgi.application"

TEMPLATE_DIRS = [
    os.path.join(PROJECT_ROOT, "templates"),
]

TEMPLATE_CONTEXT_PROCESSORS = [
    "django.contrib.auth.context_processors.auth",
    "django.core.context_processors.debug",
    "django.core.context_processors.i18n",
    "django.core.context_processors.media",
    "django.core.context_processors.request",
    "django.contrib.messages.context_processors.messages",

    "staticfiles.context_processors.static",

    "pinax.core.context_processors.pinax_settings",

    "pinax.apps.account.context_processors.account",

    # "notification.context_processors.notification",
    "announcements.context_processors.site_wide_announcements",
]

INSTALLED_APPS = [
    # Django
    "django.contrib.admin",
    "django.contrib.auth",
    "django.contrib.contenttypes",
    "django.contrib.sessions",
    "django.contrib.sites",
    "django.contrib.messages",
    "django.contrib.humanize",
    "django.contrib.markup",

    "pinax.templatetags",

    # theme
    "pinax_theme_bootstrap",

    # external (Pinax)
    # "notification",  # must be first
    "staticfiles",
    "pagination",
    "compressor",
    "django_openid",
    "timezones",
    "emailconfirmation",
    "announcements",
    "idios",
    "metron",

    # Pinax
    "pinax.apps.account",
    "pinax.apps.signup_codes",

    # external (Project)
    "south",
    "djcelery",
    "django_hosts",
    "haystack",
    "storages",
    "intercom",
    "celery_haystack",

    # project
    "about",
    "profiles",
    "packages",
    "pypi",
    "search",
    "crate",
    "privatebeta",
]

CACHE_MIDDLEWARE_ANONYMOUS_ONLY = True

FIXTURE_DIRS = [
    os.path.join(PROJECT_ROOT, "fixtures"),
]

MESSAGE_STORAGE = "django.contrib.messages.storage.session.SessionStorage"

ABSOLUTE_URL_OVERRIDES = {
    "auth.user": lambda o: "/profiles/profile/%s/" % o.username,
}

AUTH_PROFILE_MODULE = "profiles.Profile"
NOTIFICATION_LANGUAGE_MODULE = "account.Account"

ACCOUNT_OPEN_SIGNUP = False
ACCOUNT_USE_OPENID = False
ACCOUNT_REQUIRED_EMAIL = True
ACCOUNT_EMAIL_VERIFICATION = True
ACCOUNT_EMAIL_AUTHENTICATION = False
ACCOUNT_UNIQUE_EMAIL = EMAIL_CONFIRMATION_UNIQUE_EMAIL = True

AUTHENTICATION_BACKENDS = [
    "pinax.apps.account.auth_backends.AuthenticationBackend",
]

LOGIN_URL = "/account/login/"
LOGIN_REDIRECT_URLNAME = "what_next"
LOGOUT_REDIRECT_URLNAME = "search"

EMAIL_CONFIRMATION_DAYS = 2
EMAIL_DEBUG = DEBUG

DEBUG_TOOLBAR_CONFIG = {
    "INTERCEPT_REDIRECTS": False,
}

CELERY_SEND_TASK_ERROR_EMAILS = True
CELERY_DISABLE_RATE_LIMITS = True
CELERY_TASK_PUBLISH_RETRY = True

CELERY_TASK_RESULT_EXPIRES = 7 * 24 * 60 * 60  # 7 Days
CELERYD_TASK_TIME_LIMIT = 120
CELERYD_TASK_SOFT_TIME_LIMIT = 30

CELERYD_HIJACK_ROOT_LOGGER = False

CELERYBEAT_SCHEDULER = "djcelery.schedulers.DatabaseScheduler"

HAYSTACK_SEARCH_RESULTS_PER_PAGE = 15

AWS_QUERYSTRING_AUTH = False
AWS_S3_SECURE_URLS = False

AWS_HEADERS = {
    "Expires": lambda: http_date(time.mktime((datetime.datetime.now() + datetime.timedelta(days=365)).timetuple())),
    "Cache-Control": "max-age=31556926",
}

METRON_SETTINGS = {
    "mixpanel": {3: "a280fbfb3039709d8a5ac15834a16e5e"},
    # "google": {3: ""},
    "gauges": {3: "4f1e4cd0613f5d7003000002"}
}
<|endoftext|>


<commit_before>import click
from subprocess import call

def load_rules():
    with open("C:\Windows\System32\drivers\etc\hosts") as file:
        contents = file.readlines();

        rules = []
        for line in contents:
            if not line.startswith("#") and not line.strip() == "":
                print(parts)
                rule = {
                        "ip" : parts[0].strip(),
                        "domain" : parts[1].strip()
                        }
                rules.append(rule)

        return rules

def save_rules(rules):
    with open("C:\Windows\System32\drivers\etc\hosts", "w") as file:
        for rule in rules:
            file.write("{}\t{}\n".format(rule["ip"], rule["domain"]))

def add_rule(rules, ip, domain):
    rules.append({
        "ip" : ip,
        "domain" : domain
        })
    return rules

@click.group()
def rules():
    """Command line utility script to add/remove/view DNS rules in the hosts file"""
    pass

@click.command()
def purge():
    """Purges all rules from the host file. Use with caution."""
    rules = load_rules()
    for rule in rules:
        print("Purging {} - {}".format(rule['ip'], rule['domain']))
    rules = []
    save_rules(rules)

@click.command()
@click.argument("domain")
@click.argument("ip")
@click.option("--www/--no-www", default=False, help="Adds a second rule that prepends 'www.' to the given domain")
def add(domain, ip, www):
    """Add a rule to the hosts file"""
    rules = load_rules()
    add_rule(rules, ip, domain)
    if www:
        add_rule(rules, ip, "www." + domain)
    save_rules(rules)

def flush_dns():
    call(["ipconfig", "/flushdns"])

@click.command()
@click.argument("domain")
@click.option("--www/--no-www", default=False, help="Remove a 'www' version of the domain, if it exists")
def remove(domain, www):
    """Remove a rule in the hosts file that has the given domain"""
    rules = load_rules()
    slated_for_removal = []
    for rule in rules:
        if rule['domain'] == domain:
            print("Removing {} - {}".format(rule['ip'], rule['domain']))
            slated_for_removal.append(rule)
        if www and rule['domain'] == "www." + domain:
            print("Removing {} - {}".format(rule['ip'], rule['domain']))
            slated_for_removal.append(rule)
    for slated in slated_for_removal:
        rules.remove(slated)
    save_rules(rules)

@click.command("list")
def list_rules():
    """View all of the rules in the host file"""
    print("{:<30}{:<30}".format("IP Address", "Domain"))
    for rule in load_rules():
        print("{:<30}{:<30}".format(rule['ip'], rule['domain']))

rules.add_command(add)
rules.add_command(remove)
rules.add_command(list_rules)
rules.add_command(purge)

if __name__ == "__main__":
    rules()
<commit_msg>Actually changed string split method<commit_after>import click
from subprocess import call

def load_rules():
    with open("C:\Windows\System32\drivers\etc\hosts") as file:
        contents = file.readlines();

        rules = []
        for line in contents:
            if not line.startswith("#") and not line.strip() == "":
                parts = line.split()
                rule = {
                        "ip" : parts[0].strip(),
                        "domain" : parts[1].strip()
                        }
                rules.append(rule)

        return rules

def save_rules(rules):
    with open("C:\Windows\System32\drivers\etc\hosts", "w") as file:
        for rule in rules:
            file.write("{}\t{}\n".format(rule["ip"], rule["domain"]))

def add_rule(rules, ip, domain):
    rules.append({
        "ip" : ip,
        "domain" : domain
        })
    return rules

@click.group()
def rules():
    """Command line utility script to add/remove/view DNS rules in the hosts file"""
    pass

@click.command()
def purge():
    """Purges all rules from the host file. Use with caution."""
    rules = load_rules()
    for rule in rules:
        print("Purging {} - {}".format(rule['ip'], rule['domain']))
    rules = []
    save_rules(rules)

@click.command()
@click.argument("domain")
@click.argument("ip")
@click.option("--www/--no-www", default=False, help="Adds a second rule that prepends 'www.' to the given domain")
def add(domain, ip, www):
    """Add a rule to the hosts file"""
    rules = load_rules()
    add_rule(rules, ip, domain)
    if www:
        add_rule(rules, ip, "www." + domain)
    save_rules(rules)

def flush_dns():
    call(["ipconfig", "/flushdns"])

@click.command()
@click.argument("domain")
@click.option("--www/--no-www", default=False, help="Remove a 'www' version of the domain, if it exists")
def remove(domain, www):
    """Remove a rule in the hosts file that has the given domain"""
    rules = load_rules()
    slated_for_removal = []
    for rule in rules:
        if rule['domain'] == domain:
            print("Removing {} - {}".format(rule['ip'], rule['domain']))
            slated_for_removal.append(rule)
        if www and rule['domain'] == "www." + domain:
            print("Removing {} - {}".format(rule['ip'], rule['domain']))
            slated_for_removal.append(rule)
    for slated in slated_for_removal:
        rules.remove(slated)
    save_rules(rules)

@click.command("list")
def list_rules():
    """View all of the rules in the host file"""
    print("{:<30}{:<30}".format("IP Address", "Domain"))
    for rule in load_rules():
        print("{:<30}{:<30}".format(rule['ip'], rule['domain']))

rules.add_command(add)
rules.add_command(remove)
rules.add_command(list_rules)
rules.add_command(purge)

if __name__ == "__main__":
    rules()
<|endoftext|>


<commit_before>#
# Typofinder for domain typo discovery
#
# Released as open source by NCC Group Plc - http://www.nccgroup.com/
#
# Developed by Ollie Whitehouse, ollie dot whitehouse at nccgroup dot com
#
# http://www.github.com/nccgroup/typofinder
#
# Released under AGPL see LICENSE for more information#
#

import argparse
import sys
import time
import socket
import http.server
import urllib
import re
from os import curdir, sep
from socketserver import ThreadingMixIn
from dns.resolver import NoNameservers
import typogen
import hostinfo
import safebrowsing

HOST_NAME = ''      # leave like this for all
PORT_NUMBER = 801   # this will be fine

_hostinfo = hostinfo.hostinfo()

def handleHost(sHostname, http_handler, bMX, bTypo):

    if bMX:
        http_handler.output("--- [host] MX Host ")
    elif bTypo:
        http_handler.output("[host] Typo Host ")
    else:
        http_handler.output("[host] Host ")

    http_handler.output(sHostname)

    try:
        IPv4 = _hostinfo.getIPv4(sHostname)
    except:
        IPv4 = None

    try:
       IPv6 = _hostinfo.getIPv6(sHostname)
    except:
       IPv6 = None

    if (IPv4 is not None or IPv6 is not None) and not bMX:
        http_handler.output(" " + safebrowsing.safebrowsingquery(sHostname))

    http_handler.output("
")

    if IPv4 is None and IPv6 is None:
       if bMX:
           http_handler.output("---")
       http_handler.output("--- [host] No IPv6 or IPv4 address
")

    if IPv4 is not None:
        for hostData in IPv4:
            if bMX:
                http_handler.output("---")
            http_handler.output("--- [host IPv4] A: " + hostData.address + " from " + sHostname + " ")
            #print(_hostinfo.getGeoImagebyIP(hostData.address))
            #print(_hostinfo.getGeoImagebyHostname(sHostname))
            strFlag = _hostinfo.getGeoImagebyIP(hostData.address)
            http_handler.output(strFlag + "
")

    if IPv6 is not None:
        for hostData in IPv6:
            if bMX:
                http_handler.output("---")
            http_handler.output("--- [host IPv6] AAAA: " +hostData.address + " from " + sHostname + " ")
            #print(_hostinfo.getGeoImagebyIP(hostData.address))
            #_hostinfo.getGeoImagebyIP(hostData.address)
            strFlag = _hostinfo.getGeoImagebyIPv6(hostData.address)
            http_handler.output(strFlag + "
")

    if not bMX:
        try:
            IPMX = _hostinfo.getMX(sHostname)
        except NoNameservers:
            IPMX = None

        if IPMX is not None:
            for hostData in IPMX:
                #print(hostData.exchange)
                http_handler.output("--- [host MX] for " + sHostname + " is " + str(hostData.exchange).strip(".") + "
")
                handleHost(str(hostData.exchange).strip("."),http_handler,True, bTypo)
        else:
            http_handler.output("--- [host] No MX records
")

    # www v4
    try:
        IPWWW = _hostinfo.getWWW(sHostname)
    except:
        IPWWW = None

    if IPWWW is not None:
        for hostData in IPWWW:
            if not bMX:
                http_handler.output("--- [www. IPv4] A: " +hostData.address + " from " + sHostname + " ")
                strFlag = _hostinfo.getGeoImagebyIP(hostData.address)
                http_handler.output(strFlag + "
")

    # www v6
    try:
        IPWWW = _hostinfo.getWWWv6(sHostname)
    except:
        IPWWW = None

    if IPWWW is not None:
        for hostData in IPWWW:
            if not bMX:
                http_handler.output("--- [www. IPv6] A: " +hostData.address + " from " + sHostname + " ")
                strFlag = _hostinfo.getGeoImagebyIPv6(hostData.address)
                http_handler.output(strFlag + "
")

    # webmail v4
    try:
        IPWebMail= _hostinfo.getWEBMail(sHostname)
    except:
        IPWebMail = None

    if IPWebMail is not None:
        for hostData in IPWebMail:
            if not bMX:
                http_handler.output("--- [webmail. IPv4] A: " +hostData.address + " from " + sHostname + " ")
                strFlag = _hostinfo.getGeoImagebyIP(hostData.address)
                http_handler.output(strFlag + "
")

    # webmail v6
    try:
        IPWebMail= _hostinfo.getWEBMailv6(sHostname)
    except:
        IPWebMail = None

    if IPWebMail is not None:
        for hostData in IPWebMail:
            if not bMX:
                http_handler.output("--- [webmail. IPv6] A: " +hostData.address + " from " + sHostname + " ")
                strFlag = _hostinfo.getGeoImagebyIPv6(hostData.address)
                http_handler.output(strFlag + "
")

    # m v4
    try:
        IPM= _hostinfo.getM(sHostname)
    except:
        IPM = None

    if IPM is not None:
        for hostData in IPM:
            if not bMX:
                http_handler.output("--- [m. IPv4] A: " +hostData.address + " from " + sHostname + " ")
                strFlag = _hostinfo.getGeoImagebyIP(hostData.address)
                http_handler.output(strFlag + "
")

    # m v6
    try:
        IPM= _hostinfo.getMv6(sHostname)
    except:
        IPM = None

    if IPM is not None:
        for hostData in IPM:
            if not bMX:
                http_handler.output("--- [m. IPv6] A: " +hostData.address + " from " + sHostname + " ")
                strFlag = _hostinfo.getGeoImagebyIPv6(hostData.address)
                http_handler.output(strFlag + "
")

    # if we're not a typo (i.e. we're the base domain) then mutate
    if bTypo == False and bMX == False:
        # this could be a different country if you supplied the map
        lstTypos = typogen.typogen.generatetypos(sHostname,"GB")
        if lstTypos is not None:
            for strTypoHost in lstTypos:
                handleHost(strTypoHost,http_handler,False,True)

    # WHOIS TODO
    #domain = whois.query('zemes.com')
    #print(domain)

    return

class MyHandler(http.server.BaseHTTPRequestHandler):

    def output(self, outputString):
        self.wfile.write(outputString.encode('utf-8'))

    def do_HEAD(self):
        self.send_response(200)
        self.send_header("Content-type", "text/html")
        self.end_headers()

    def do_POST(self):

        self.send_response(200)
        self.send_header("Content-type", "text/html")
        self.end_headers()
        self.output("")
        self.output("<title>NCC Typo Finder Results</title>")
        self.output("<link rel=\"stylesheet\" type=\"text/css\" href=\"/style.css\">")
        self.output("")
        self.output("Released under AGPL by NCC Group - source available here
")

        length = int(self.headers['Content-Length'])
        post_data = urllib.parse.parse_qs(self.rfile.read(length).decode('utf-8'))

        #for key, value in post_data.items() :
        #    print (key, value)

        strHost = str(post_data['host'])[2:-2]
        if re.match('^[a-zA-Z0-9.-]+$',strHost):
            handleHost(strHost,self,False,False)
        #self.output(strHost + "
")

        return

    def do_GET(self):
        """Respond to a GET request."""

        try:
            if self.path.endswith("/"):
                f = open(curdir + sep + "index.html")
                self.send_response(200)
                self.send_header('Content-type','text/html')
                self.end_headers()
                self.output(f.read())
                f.close()
                return
            elif self.path.endswith(".html") and self.path.find("..") != 0:
                f = open(curdir + sep + self.path)
                self.send_response(200)
                self.send_header('Content-type','text/html')
                self.end_headers()
                self.wfile.write(f.read())
                f.close()
                return
            elif self.path.endswith(".css") and self.path.find("..") != 0:
                f = open(curdir + sep + self.path)
                self.send_response(200)
                self.send_header('Content-type','text/css')
                self.end_headers()
                self.wfile.write(bytes(f.read(), 'UTF-8'))
                f.close()
                return
            elif self.path.endswith(".png") and self.path.find("..") != 0:
                f = open(curdir + sep + self.path, "rb")
                self.send_response(200)
                self.send_header('Content-type','image/png')
                self.end_headers()
                self.wfile.write(f.read())
                f.close()
                return
            else:
               self.send_error(404,'File Not Found: %s' % self.path)

        except IOError:
            self.send_error(404,'File Not Found: %s' % self.path)

class MultiThreadedHTTPServer(ThreadingMixIn, http.server.HTTPServer):
    pass

if __name__ == '__main__':

    print ("[i] NCC Group domain typofinder - https://github.com/nccgroup")

    parser = argparse.ArgumentParser()
    parser.add_argument('-p','--port', help='Port to listen on',required=False)
    parser.add_argument('-a','--address', help='hostname / IP address to bind to',required=False)
    parser.add_argument('-k','--key',help='Google SafeBrowsing API key', required=False)
    args = parser.parse_args()

    if(args.port != None):
        try:
            PORT_NUMBER = int(args.port)
        except ValueError:
            print("[!] Port number needs to be an integer - defaulting back to 801")
            pass

    if(args.address != None):
        HOST_NAME = args.address

    try:
        httpd = MultiThreadedHTTPServer((HOST_NAME, PORT_NUMBER), MyHandler)
    except socket.gaierror:
        print("[!] Supplied address invalid! exiting!")
        sys.exit()

    print ("[i]", time.asctime(), " Server Starts - %s:%s" % (HOST_NAME, PORT_NUMBER))
    try:
        httpd.serve_forever()
    except KeyboardInterrupt:
        pass

    httpd.server_close()
    print ("[i]", time.asctime(), " Server Stops - %s:%s" % (HOST_NAME, PORT_NUMBER))
<commit_msg>more error handling<commit_after>#
# Typofinder for domain typo discovery
#
# Released as open source by NCC Group Plc - http://www.nccgroup.com/
#
# Developed by Ollie Whitehouse, ollie dot whitehouse at nccgroup dot com
#
# http://www.github.com/nccgroup/typofinder
#
# Released under AGPL see LICENSE for more information#
#

import argparse
import sys
import time
import socket
import http.server
import urllib
import re
from os import curdir, sep
from socketserver import ThreadingMixIn
from dns.resolver import NoNameservers
import typogen
import hostinfo
import safebrowsing

HOST_NAME = ''      # leave like this for all
PORT_NUMBER = 801   # this will be fine

_hostinfo = hostinfo.hostinfo()

def handleHost(sHostname, http_handler, bMX, bTypo):

    if bMX:
        http_handler.output("--- [host] MX Host ")
    elif bTypo:
        http_handler.output("[host] Typo Host ")
    else:
        http_handler.output("[host] Host ")

    http_handler.output(sHostname)

    try:
        IPv4 = _hostinfo.getIPv4(sHostname)
    except:
        IPv4 = None

    try:
       IPv6 = _hostinfo.getIPv6(sHostname)
    except:
       IPv6 = None

    if (IPv4 is not None or IPv6 is not None) and not bMX:
        http_handler.output(" " + safebrowsing.safebrowsingquery(sHostname))

    http_handler.output("
")

    if IPv4 is None and IPv6 is None:
       if bMX:
           http_handler.output("---")
       http_handler.output("--- [host] No IPv6 or IPv4 address
")

    if IPv4 is not None:
        for hostData in IPv4:
            if bMX:
                http_handler.output("---")
            http_handler.output("--- [host IPv4] A: " + hostData.address + " from " + sHostname + " ")
            #print(_hostinfo.getGeoImagebyIP(hostData.address))
            #print(_hostinfo.getGeoImagebyHostname(sHostname))
            strFlag = _hostinfo.getGeoImagebyIP(hostData.address)
            http_handler.output(strFlag + "
")

    if IPv6 is not None:
        for hostData in IPv6:
            if bMX:
                http_handler.output("---")
            http_handler.output("--- [host IPv6] AAAA: " +hostData.address + " from " + sHostname + " ")
            #print(_hostinfo.getGeoImagebyIP(hostData.address))
            #_hostinfo.getGeoImagebyIP(hostData.address)
            strFlag = _hostinfo.getGeoImagebyIPv6(hostData.address)
            http_handler.output(strFlag + "
")

    if not bMX:
        try:
            IPMX = _hostinfo.getMX(sHostname)
        except NoNameservers:
            IPMX = None

        if IPMX is not None:
            for hostData in IPMX:
                #print(hostData.exchange)
                http_handler.output("--- [host MX] for " + sHostname + " is " + str(hostData.exchange).strip(".") + "
")
                handleHost(str(hostData.exchange).strip("."),http_handler,True, bTypo)
        else:
            http_handler.output("--- [host] No MX records
")

    # www v4
    try:
        IPWWW = _hostinfo.getWWW(sHostname)
    except:
        IPWWW = None

    if IPWWW is not None:
        for hostData in IPWWW:
            if not bMX:
                http_handler.output("--- [www. IPv4] A: " +hostData.address + " from " + sHostname + " ")
                strFlag = _hostinfo.getGeoImagebyIP(hostData.address)
                http_handler.output(strFlag + "
")

    # www v6
    try:
        IPWWW = _hostinfo.getWWWv6(sHostname)
    except:
        IPWWW = None

    if IPWWW is not None:
        for hostData in IPWWW:
            if not bMX:
                http_handler.output("--- [www. IPv6] A: " +hostData.address + " from " + sHostname + " ")
                strFlag = _hostinfo.getGeoImagebyIPv6(hostData.address)
                http_handler.output(strFlag + "
")

    # webmail v4
    try:
        IPWebMail= _hostinfo.getWEBMail(sHostname)
    except:
        IPWebMail = None

    if IPWebMail is not None:
        for hostData in IPWebMail:
            if not bMX:
                http_handler.output("--- [webmail. IPv4] A: " +hostData.address + " from " + sHostname + " ")
                strFlag = _hostinfo.getGeoImagebyIP(hostData.address)
                http_handler.output(strFlag + "
")

    # webmail v6
    try:
        IPWebMail= _hostinfo.getWEBMailv6(sHostname)
    except:
        IPWebMail = None

    if IPWebMail is not None:
        for hostData in IPWebMail:
            if not bMX:
                http_handler.output("--- [webmail. IPv6] A: " +hostData.address + " from " + sHostname + " ")
                strFlag = _hostinfo.getGeoImagebyIPv6(hostData.address)
                http_handler.output(strFlag + "
")

    # m v4
    try:
        IPM= _hostinfo.getM(sHostname)
    except:
        IPM = None

    if IPM is not None:
        for hostData in IPM:
            if not bMX:
                http_handler.output("--- [m. IPv4] A: " +hostData.address + " from " + sHostname + " ")
                strFlag = _hostinfo.getGeoImagebyIP(hostData.address)
                http_handler.output(strFlag + "
")

    # m v6
    try:
        IPM= _hostinfo.getMv6(sHostname)
    except:
        IPM = None

    if IPM is not None:
        for hostData in IPM:
            if not bMX:
                http_handler.output("--- [m. IPv6] A: " +hostData.address + " from " + sHostname + " ")
                strFlag = _hostinfo.getGeoImagebyIPv6(hostData.address)
                http_handler.output(strFlag + "
")

    # if we're not a typo (i.e. we're the base domain) then mutate
    if bTypo == False and bMX == False:
        # this could be a different country if you supplied the map
        lstTypos = typogen.typogen.generatetypos(sHostname,"GB")
        if lstTypos is not None:
            for strTypoHost in lstTypos:
                handleHost(strTypoHost,http_handler,False,True)

    # WHOIS TODO
    #domain = whois.query('zemes.com')
    #print(domain)

    return

class MyHandler(http.server.BaseHTTPRequestHandler):

    def output(self, outputString):
        self.wfile.write(outputString.encode('utf-8'))

    def do_HEAD(self):
        self.send_response(200)
        self.send_header("Content-type", "text/html")
        self.end_headers()

    def do_POST(self):

        try:
            self.send_response(200)
            self.send_header("Content-type", "text/html")
            self.end_headers()
            self.output("")
            self.output("<title>NCC Typo Finder Results</title>")
            self.output("<link rel=\"stylesheet\" type=\"text/css\" href=\"/style.css\">")
            self.output("")
            self.output("Released under AGPL by NCC Group - source available here
")

            length = int(self.headers['Content-Length'])
            post_data = urllib.parse.parse_qs(self.rfile.read(length).decode('utf-8'))

            #for key, value in post_data.items() :
            #    print (key, value)

            strHost = str(post_data['host'])[2:-2]
            if re.match('^[a-zA-Z0-9.-]+$',strHost):
                handleHost(strHost,self,False,False)
            #self.output(strHost + "
")
        except:
            pass

        return

    def do_GET(self):
        """Respond to a GET request."""

        try:
            if self.path.endswith("/"):
                f = open(curdir + sep + "index.html")
                self.send_response(200)
                self.send_header('Content-type','text/html')
                self.end_headers()
                self.output(f.read())
                f.close()
                return
            elif self.path.endswith(".html") and self.path.find("..") != 0:
                f = open(curdir + sep + self.path)
                self.send_response(200)
                self.send_header('Content-type','text/html')
                self.end_headers()
                self.wfile.write(f.read())
                f.close()
                return
            elif self.path.endswith(".css") and self.path.find("..") != 0:
                f = open(curdir + sep + self.path)
                self.send_response(200)
                self.send_header('Content-type','text/css')
                self.end_headers()
                self.wfile.write(bytes(f.read(), 'UTF-8'))
                f.close()
                return
            elif self.path.endswith(".png") and self.path.find("..") != 0:
                f = open(curdir + sep + self.path, "rb")
                self.send_response(200)
                self.send_header('Content-type','image/png')
                self.end_headers()
                self.wfile.write(f.read())
                f.close()
                return
            else:
               self.send_error(404,'[!] File Not Found: %s' % self.path)

        except IOError:
            self.send_error(404,'[!] File Not Found: %s' % self.path)

        except:
            pass

class MultiThreadedHTTPServer(ThreadingMixIn, http.server.HTTPServer):
    pass

if __name__ == '__main__':

    print ("[i] NCC Group domain typofinder - https://github.com/nccgroup")

    parser = argparse.ArgumentParser()
    parser.add_argument('-p','--port', help='Port to listen on',required=False)
    parser.add_argument('-a','--address', help='hostname / IP address to bind to',required=False)
    parser.add_argument('-k','--key',help='Google SafeBrowsing API key', required=False)
    args = parser.parse_args()

    if(args.port != None):
        try:
            PORT_NUMBER = int(args.port)
        except ValueError:
            print("[!] Port number needs to be an integer - defaulting back to 801")
            pass

    if(args.address != None):
        HOST_NAME = args.address

    try:
        httpd = MultiThreadedHTTPServer((HOST_NAME, PORT_NUMBER), MyHandler)
    except socket.gaierror:
        print("[!] Supplied address invalid! exiting!")
        sys.exit()

    print ("[i]", time.asctime(), " Server Starts - %s:%s" % (HOST_NAME, PORT_NUMBER))
    try:
        httpd.serve_forever()
    except KeyboardInterrupt:
        pass

    httpd.server_close()
    print ("[i]", time.asctime(), " Server Stops - %s:%s" % (HOST_NAME, PORT_NUMBER))
<|endoftext|>