<commit_before>import time

from PyQt5.QtCore import QObject, pyqtSlot

from anki import notes
from anki.hooks import addHook, wrap
from anki.notes import Note
from anki.sound import clearAudioQueue
from aqt import addcards, editcurrent
from aqt import mw
from aqt.addcards import AddCards
from aqt.editcurrent import EditCurrent
from aqt.reviewer import Reviewer
from aqt.utils import showInfo, showWarning, tooltip

from ir.importer import Importer
from ir.settings import SettingsManager
from ir.schedule import Scheduler
from ir.util import (addMenuItem, disableOutdated, getField, getInput, isIrCard,
                     setField, viewingIrText)
from ir.view import ViewManager

class ReadingManager():
    def __init__(self):
        self.quickKeyActions = []
        self.controlsLoaded = False

        addHook('profileLoaded', self.onProfileLoaded)
        addHook('reset', self.restoreView)
        addHook('showQuestion', self.restoreView)

    def onProfileLoaded(self):
        mw.settingsManager = SettingsManager()
        mw.viewManager = ViewManager()
        self.scheduler = Scheduler()
        self.importer = Importer(mw.settingsManager.settings)

        self.settings = mw.settingsManager.settings
        mw.viewManager.settings = mw.settingsManager.settings
        self.scheduler.settings = mw.settingsManager.settings

        self.addModel()
        disableOutdated()

        if not self.controlsLoaded:
            mw.settingsManager.addMenuItem()
            self.scheduler.addMenuItem()
            addMenuItem('Read',
                        'Import Webpage',
                        self.importer.importWebpage,
                        'Alt+3')
            addMenuItem('Read',
                        'Import Feed',
                        self.importer.importFeed,
                        'Alt+4')
            mw.viewManager.addMenuItems()
            mw.viewManager.addShortcuts()
            self.controlsLoaded = True

        mw.viewManager.resetZoom('deckBrowser')

        mw.moveToState = wrap(mw.moveToState, self.setShortcuts)

    def setShortcuts(self, state, *args):
        if state == 'review':
            mw.setStateShortcuts([
                (mw.settingsManager.settings['extractKey'].lower(),
                 self.extract),
                (mw.settingsManager.settings['highlightKey'].lower(),
                 self.highlightText),
                (mw.settingsManager.settings['removeKey'].lower(),
                 self.removeText)])

    def addModel(self):
        "Only adds model if no model with the same name is present"
        col = mw.col
        mm = col.models
        iread_model = mm.byName(self.settings['modelName'])
        if not iread_model:
            iread_model = mm.new(self.settings['modelName'])
            model_field = mm.newField(self.settings['titleField'])
            mm.addField(iread_model, model_field)
            text_field = mm.newField(self.settings['textField'])
            mm.addField(iread_model, text_field)
            source_field = mm.newField(self.settings['sourceField'])
            source_field['sticky'] = True
            mm.addField(iread_model, source_field)

            t = mm.newTemplate('IR Card')
            t['qfmt'] = '
{{%s}}
' % (self.settings['textField'])
            t['afmt'] = 'When do you want to see this card again?'

            mm.addTemplate(iread_model, t)
            mm.add(iread_model)
            return iread_model
        else:
            fmap = mm.fieldMap(iread_model)
            title_ord, title_field = fmap[self.settings['titleField']]
            text_ord, text_field = fmap[self.settings['textField']]
            source_ord, source_field = fmap[self.settings['sourceField']]
            source_field['sticky'] = True

    def extract(self):
        if not mw.web.selectedText():
            showInfo(_('Please select some text to extract.'))
            return

        if self.settings['plainText']:
            mw.web.evalWithCallback('getPlainText()', self.createExtractNote)
        else:
            mw.web.evalWithCallback('getHtmlText()', self.createExtractNote)

    def createExtractNote(self, text):
        self.highlightText(self.settings['extractBgColor'],
                           self.settings['extractTextColor'])

        currentCard = mw.reviewer.card
        currentNote = currentCard.note()
        model = mw.col.models.byName(self.settings['modelName'])
        newNote = Note(mw.col, model)
        newNote.tags = currentNote.tags

        setField(newNote, self.settings['textField'], text)
        setField(newNote,
                 self.settings['sourceField'],
                 getField(currentNote, self.settings['sourceField']))

        if self.settings['editSource']:
            EditCurrent(mw)

        if self.settings['extractDeck']:
            did = mw.col.decks.byName(self.settings['extractDeck'])['id']
        else:
            did = currentCard.did

        if self.settings['editExtract']:
            addCards = AddCards(mw)
            addCards.editor.setNote(newNote)
            deckName = mw.col.decks.get(did)['name']
            addCards.deckChooser.deck.setText(deckName)
            addCards.modelChooser.models.setText(self.settings['modelName'])
        else:
            title = getInput('Extract Text', 'Title')
            setField(newNote, self.settings['titleField'], title)
            newNote.model()['did'] = did
            mw.col.addNote(newNote)

    def restoreView(self):
        if viewingIrText():
            cid = str(mw.reviewer.card.id)
            if cid not in self.settings['zoom']:
                self.settings['zoom'][cid] = 1

            if cid not in self.settings['scroll']:
                self.settings['scroll'][cid] = 0

            mw.viewManager.setZoom()
            mw.viewManager.setScroll()
            self.restoreHighlighting()

            def storePageInfo(pageInfo):
                (mw.viewManager.viewportHeight,
                 mw.viewManager.pageBottom) = pageInfo

            mw.web.evalWithCallback(
                '[window.innerHeight, document.body.scrollHeight];',
                storePageInfo)

    def restoreHighlighting(self):
        initJavaScript()
        mw.web.eval('restoreHighlighting()')

    def highlightText(self, bgColor=None, textColor=None):
        if not bgColor:
            bgColor = self.settings['highlightBgColor']
        if not textColor:
            textColor = self.settings['highlightTextColor']

        identifier = str(int(time.time() * 10))
        script = "markRange('%s', '%s', '%s');" % (identifier,
                                                   bgColor,
                                                   textColor)
        script += "highlight('%s', '%s');" % (bgColor, textColor)
        mw.web.eval(script)
        self.saveText()

    def saveText(self):
        def callback(text):
            if text:
                note = mw.reviewer.card.note()
                note['Text'] = text
                note.flush()
                self.restoreView()

        mw.web.evalWithCallback(
            'document.getElementsByClassName("ir-text")[0].innerHTML;',
            callback)

    def removeText(self):
        mw.web.eval('removeText()')
        self.saveText()

    def htmlUpdated(self):
        curNote = mw.reviewer.card.note()
        curNote['Text'] = mw.web.page().mainFrame().toHtml()
        curNote.flush()
        mw.web.setHtml(curNote['Text'])
        self.restoreView()

    def quickAdd(self, quickKey):
        if not viewingIrText():
            return

        self.currentQuickKey = quickKey

        if quickKey['plainText']:
            mw.web.evalWithCallback('getPlainText()', self.createNote)
        else:
            mw.web.evalWithCallback('getHtmlText()', self.createNote)

    def createNote(self, selectedText):
        self.highlightText(self.currentQuickKey['bgColor'],
                           self.currentQuickKey['textColor'])

        newModel = mw.col.models.byName(self.currentQuickKey['modelName'])
        newNote = notes.Note(mw.col, newModel)
        setField(newNote, self.currentQuickKey['fieldName'], selectedText)

        card = mw.reviewer.card
        currentNote = card.note()
        tags = currentNote.stringTags()
        # Sets tags for the note, but still have to set them in the editor
        #   if show dialog (see below)
        newNote.setTagsFromStr(tags)

        for f in newModel['flds']:
            if self.settings['sourceField'] == f['name']:
                setField(newNote,
                         self.settings['sourceField'],
                         getField(currentNote, self.settings['sourceField']))

        if self.currentQuickKey['editExtract']:
            addCards = addcards.AddCards(mw)
            addCards.editor.setNote(newNote)
            if newNote.stringTags():
                addCards.editor.tags.setText(newNote.stringTags().strip())
            addCards.modelChooser.models.setText(
                self.currentQuickKey['modelName'])
            addCards.deckChooser.deck.setText(
                self.currentQuickKey['deckName'])
        else:
            deckId = mw.col.decks.byName(self.currentQuickKey['deckName'])['id']
            newNote.model()['did'] = deckId
            ret = newNote.dupeOrEmpty()
            if ret == 1:
                showWarning(_(
                    'The first field is empty.'),
                    help='AddItems#AddError')
                return
            cards = mw.col.addNote(newNote)
            if not cards:
                showWarning(_('''\
                    The input you have provided would make an empty \
                    question on all cards.'''), help='AddItems')
                return

            clearAudioQueue()
            mw.col.autosave()
            tooltip(_('Added'))

        if self.currentQuickKey['editSource']:
            self.editCurrent = editcurrent.EditCurrent(mw)

class IREJavaScriptCallback(QObject):
    @pyqtSlot(str)
    def htmlUpdated(self, context):
        mw.readingManager.htmlUpdated()

def initJavaScript():
    javaScript = """
    function highlight(bgColor, textColor) {
        if (window.getSelection) {
            var range, sel = window.getSelection();

            if (sel.rangeCount && sel.getRangeAt) {
                range = sel.getRangeAt(0);
            }

            document.designMode = "on";
            if (range) {
                sel.removeAllRanges();
                sel.addRange(range);
            }

            document.execCommand("foreColor", false, textColor);
            document.execCommand("hiliteColor", false, bgColor);

            document.designMode = "off";
            sel.removeAllRanges();
        }
    }

    function unhighlight(identifier) {
        var startNode, endNode;
        startNode = document.getElementById('s' + identifier);
        endNode = document.getElementById('e' + identifier);
        if (startNode) {
            range = document.createRange();
            range.setStartAfter(startNode);
            range.setEndBefore(endNode);
            sel = window.getSelection();
            sel.removeAllRanges();
            sel.addRange(range);
            highlight('white', 'black');
            startNode.parentNode.removeChild(startNode);
            endNode.parentNode.removeChild(endNode);
            pyCallback.htmlUpdated('');
        }
    }

    function markRange(identifier, bgColor, textColor) {
        var range, sel = window.getSelection();
        if (sel.rangeCount && sel.getRangeAt) {
            range = sel.getRangeAt(0);
            var startNode = document.createElement('span');
            startNode.setAttribute('id', ('s' + identifier));
            startNode.setAttribute('ir-bg-color', bgColor);
            startNode.setAttribute('ir-text-color', textColor);
            range.insertNode(startNode);
            var endNode = document.createElement('span');
            endNode.setAttribute('id', ('e' + identifier));
            // editHighlightLink = document.createElement('a');
            // editHighlightLink.setAttribute('href','javascript:');
            // var tmp = ('unhighlight(' + identifier + '); return false;');
            // editHighlightLink.setAttribute('onclick', tmp);
            // sub = document.createElement('sub');
            // sub.appendChild(document.createTextNode('#'));
            // editHighlightLink.appendChild(sub);
            // endNode.appendChild(editHighlightLink);
            range.collapse(false);
            range.insertNode(endNode);
            range.setStartAfter(startNode);
            range.setEndBefore(endNode);
            sel.removeAllRanges();
            sel.addRange(range);
        }
    }

    function selectMarkedRange(identifier) {
        var startNode, endNode, range, sel;
        startNode = document.getElementById('s' + identifier);
        endNode = document.getElementById('e' + identifier);
        if (startNode && endNode) {
            range = document.createRange();
            range.setStartAfter(startNode);
            range.setEndBefore(endNode);
            sel = window.getSelection();
            sel.removeAllRanges();
            sel.addRange(range);
        }
    }

    function restoreHighlighting() {
        var startNodesXPathResult = document.evaluate(
            '//*[@ir-bg-color]', document, null, XPathResult.ANY_TYPE, null);
        var sNodes = new Array();
        var startNode = startNodesXPathResult.iterateNext();
        while(startNode) {
            sNodes.push(startNode);
            startNode = startNodesXPathResult.iterateNext();
        }
        var id;
        for (var i=0; i < sNodes.length; i++) {
            startNode = sNodes[i];
            id = startNode.id.substring(1);
            selectMarkedRange(id);
            highlight(startNode.getAttribute('ir-bg-color'),
                      startNode.getAttribute('ir-text-color'))
        }
    }

    function removeText() {
        var range, sel = window.getSelection();
        if (sel.rangeCount && sel.getRangeAt) {
            range = sel.getRangeAt(0);
            var startNode = document.createElement('span');
            range.insertNode(startNode);
            var endNode = document.createElement('span');
            range.collapse(false);
            range.insertNode(endNode);
            range.setStartAfter(startNode);
            range.setEndBefore(endNode);
            sel.addRange(range);
            range.deleteContents();
        }
    }

    function getPlainText() {
        return window.getSelection().toString();
    }

    function getHtmlText() {
        var selection = window.getSelection();
        var range = selection.getRangeAt(0);
        var div = document.createElement('div');
        div.appendChild(range.cloneContents());
        return div.innerHTML;
    }
    """
    mw.web.eval(javaScript)

def answerButtonList(self, _old):
    if isIrCard():
        l = ((1, _("Soon")),)
        cnt = mw.col.sched.answerButtons(self.card)
        if cnt == 2:
            return l + ((2, _("Later")),)
        elif cnt == 3:
            return l + ((2, _("Later")), (3, _("Custom")))
        else:
            return l + ((2, _("Later")), (3, _("MuchLater")), (4, _("Custom")))
    else:
        return _old(self)

def answerCard(self, ease, _old):
    # Get the card before scheduler kicks in, else you are looking at a
    #   different card or NONE (which gives error)
    card = self.card

    _old(self, ease)

    if isIrCard():
        mw.readingManager.scheduler.scheduleCard(card, ease)

def buttonTime(self, i, _old):
    if isIrCard():
        return '

'
    else:
        return _old(self, i)

Reviewer._answerButtonList = wrap(Reviewer._answerButtonList,
                                  answerButtonList,
                                  'around')

Reviewer._answerCard = wrap(Reviewer._answerCard, answerCard, 'around')
Reviewer._buttonTime = wrap(Reviewer._buttonTime, buttonTime, 'around')
<commit_msg>Use addHook instead of wrap for state changes<commit_after>import time

from PyQt5.QtCore import QObject, pyqtSlot

from anki import notes
from anki.hooks import addHook, wrap
from anki.notes import Note
from anki.sound import clearAudioQueue
from aqt import addcards, editcurrent
from aqt import mw
from aqt.addcards import AddCards
from aqt.editcurrent import EditCurrent
from aqt.reviewer import Reviewer
from aqt.utils import showInfo, showWarning, tooltip

from ir.importer import Importer
from ir.settings import SettingsManager
from ir.schedule import Scheduler
from ir.util import (addMenuItem, disableOutdated, getField, getInput, isIrCard,
                     setField, viewingIrText)
from ir.view import ViewManager

class ReadingManager():
    def __init__(self):
        self.quickKeyActions = []
        self.controlsLoaded = False

        addHook('profileLoaded', self.onProfileLoaded)
        addHook('reset', self.restoreView)
        addHook('showQuestion', self.restoreView)

    def onProfileLoaded(self):
        mw.settingsManager = SettingsManager()
        mw.viewManager = ViewManager()
        self.scheduler = Scheduler()
        self.importer = Importer(mw.settingsManager.settings)

        self.settings = mw.settingsManager.settings
        mw.viewManager.settings = mw.settingsManager.settings
        self.scheduler.settings = mw.settingsManager.settings

        self.addModel()
        disableOutdated()

        if not self.controlsLoaded:
            mw.settingsManager.addMenuItem()
            self.scheduler.addMenuItem()
            addMenuItem('Read',
                        'Import Webpage',
                        self.importer.importWebpage,
                        'Alt+3')
            addMenuItem('Read',
                        'Import Feed',
                        self.importer.importFeed,
                        'Alt+4')
            mw.viewManager.addMenuItems()
            mw.viewManager.addShortcuts()
            self.controlsLoaded = True

        mw.viewManager.resetZoom('deckBrowser')
        addHook('afterStateChange', self.setShortcuts)

    def setShortcuts(self, state, *args):
        if state == 'review':
            mw.setStateShortcuts([
                (mw.settingsManager.settings['extractKey'].lower(),
                 self.extract),
                (mw.settingsManager.settings['highlightKey'].lower(),
                 self.highlightText),
                (mw.settingsManager.settings['removeKey'].lower(),
                 self.removeText)])

    def addModel(self):
        "Only adds model if no model with the same name is present"
        col = mw.col
        mm = col.models
        iread_model = mm.byName(self.settings['modelName'])
        if not iread_model:
            iread_model = mm.new(self.settings['modelName'])
            model_field = mm.newField(self.settings['titleField'])
            mm.addField(iread_model, model_field)
            text_field = mm.newField(self.settings['textField'])
            mm.addField(iread_model, text_field)
            source_field = mm.newField(self.settings['sourceField'])
            source_field['sticky'] = True
            mm.addField(iread_model, source_field)

            t = mm.newTemplate('IR Card')
            t['qfmt'] = '
{{%s}}
' % (self.settings['textField'])
            t['afmt'] = 'When do you want to see this card again?'

            mm.addTemplate(iread_model, t)
            mm.add(iread_model)
            return iread_model
        else:
            fmap = mm.fieldMap(iread_model)
            title_ord, title_field = fmap[self.settings['titleField']]
            text_ord, text_field = fmap[self.settings['textField']]
            source_ord, source_field = fmap[self.settings['sourceField']]
            source_field['sticky'] = True

    def extract(self):
        if not mw.web.selectedText():
            showInfo(_('Please select some text to extract.'))
            return

        if self.settings['plainText']:
            mw.web.evalWithCallback('getPlainText()', self.createExtractNote)
        else:
            mw.web.evalWithCallback('getHtmlText()', self.createExtractNote)

    def createExtractNote(self, text):
        self.highlightText(self.settings['extractBgColor'],
                           self.settings['extractTextColor'])

        currentCard = mw.reviewer.card
        currentNote = currentCard.note()
        model = mw.col.models.byName(self.settings['modelName'])
        newNote = Note(mw.col, model)
        newNote.tags = currentNote.tags

        setField(newNote, self.settings['textField'], text)
        setField(newNote,
                 self.settings['sourceField'],
                 getField(currentNote, self.settings['sourceField']))

        if self.settings['editSource']:
            EditCurrent(mw)

        if self.settings['extractDeck']:
            did = mw.col.decks.byName(self.settings['extractDeck'])['id']
        else:
            did = currentCard.did

        if self.settings['editExtract']:
            addCards = AddCards(mw)
            addCards.editor.setNote(newNote)
            deckName = mw.col.decks.get(did)['name']
            addCards.deckChooser.deck.setText(deckName)
            addCards.modelChooser.models.setText(self.settings['modelName'])
        else:
            title = getInput('Extract Text', 'Title')
            setField(newNote, self.settings['titleField'], title)
            newNote.model()['did'] = did
            mw.col.addNote(newNote)

    def restoreView(self):
        if viewingIrText():
            cid = str(mw.reviewer.card.id)
            if cid not in self.settings['zoom']:
                self.settings['zoom'][cid] = 1

            if cid not in self.settings['scroll']:
                self.settings['scroll'][cid] = 0

            mw.viewManager.setZoom()
            mw.viewManager.setScroll()
            self.restoreHighlighting()

            def storePageInfo(pageInfo):
                (mw.viewManager.viewportHeight,
                 mw.viewManager.pageBottom) = pageInfo

            mw.web.evalWithCallback(
                '[window.innerHeight, document.body.scrollHeight];',
                storePageInfo)

    def restoreHighlighting(self):
        initJavaScript()
        mw.web.eval('restoreHighlighting()')

    def highlightText(self, bgColor=None, textColor=None):
        if not bgColor:
            bgColor = self.settings['highlightBgColor']
        if not textColor:
            textColor = self.settings['highlightTextColor']

        identifier = str(int(time.time() * 10))
        script = "markRange('%s', '%s', '%s');" % (identifier,
                                                   bgColor,
                                                   textColor)
        script += "highlight('%s', '%s');" % (bgColor, textColor)
        mw.web.eval(script)
        self.saveText()

    def saveText(self):
        def callback(text):
            if text:
                note = mw.reviewer.card.note()
                note['Text'] = text
                note.flush()
                self.restoreView()

        mw.web.evalWithCallback(
            'document.getElementsByClassName("ir-text")[0].innerHTML;',
            callback)

    def removeText(self):
        mw.web.eval('removeText()')
        self.saveText()

    def htmlUpdated(self):
        curNote = mw.reviewer.card.note()
        curNote['Text'] = mw.web.page().mainFrame().toHtml()
        curNote.flush()
        mw.web.setHtml(curNote['Text'])
        self.restoreView()

    def quickAdd(self, quickKey):
        if not viewingIrText():
            return

        self.currentQuickKey = quickKey

        if quickKey['plainText']:
            mw.web.evalWithCallback('getPlainText()', self.createNote)
        else:
            mw.web.evalWithCallback('getHtmlText()', self.createNote)

    def createNote(self, selectedText):
        self.highlightText(self.currentQuickKey['bgColor'],
                           self.currentQuickKey['textColor'])

        newModel = mw.col.models.byName(self.currentQuickKey['modelName'])
        newNote = notes.Note(mw.col, newModel)
        setField(newNote, self.currentQuickKey['fieldName'], selectedText)

        card = mw.reviewer.card
        currentNote = card.note()
        tags = currentNote.stringTags()
        # Sets tags for the note, but still have to set them in the editor
        #   if show dialog (see below)
        newNote.setTagsFromStr(tags)

        for f in newModel['flds']:
            if self.settings['sourceField'] == f['name']:
                setField(newNote,
                         self.settings['sourceField'],
                         getField(currentNote, self.settings['sourceField']))

        if self.currentQuickKey['editExtract']:
            addCards = addcards.AddCards(mw)
            addCards.editor.setNote(newNote)
            if newNote.stringTags():
                addCards.editor.tags.setText(newNote.stringTags().strip())
            addCards.modelChooser.models.setText(
                self.currentQuickKey['modelName'])
            addCards.deckChooser.deck.setText(
                self.currentQuickKey['deckName'])
        else:
            deckId = mw.col.decks.byName(self.currentQuickKey['deckName'])['id']
            newNote.model()['did'] = deckId
            ret = newNote.dupeOrEmpty()
            if ret == 1:
                showWarning(_(
                    'The first field is empty.'),
                    help='AddItems#AddError')
                return
            cards = mw.col.addNote(newNote)
            if not cards:
                showWarning(_('''\
                    The input you have provided would make an empty \
                    question on all cards.'''), help='AddItems')
                return

            clearAudioQueue()
            mw.col.autosave()
            tooltip(_('Added'))

        if self.currentQuickKey['editSource']:
            self.editCurrent = editcurrent.EditCurrent(mw)

class IREJavaScriptCallback(QObject):
    @pyqtSlot(str)
    def htmlUpdated(self, context):
        mw.readingManager.htmlUpdated()

def initJavaScript():
    javaScript = """
    function highlight(bgColor, textColor) {
        if (window.getSelection) {
            var range, sel = window.getSelection();

            if (sel.rangeCount && sel.getRangeAt) {
                range = sel.getRangeAt(0);
            }

            document.designMode = "on";
            if (range) {
                sel.removeAllRanges();
                sel.addRange(range);
            }

            document.execCommand("foreColor", false, textColor);
            document.execCommand("hiliteColor", false, bgColor);

            document.designMode = "off";
            sel.removeAllRanges();
        }
    }

    function unhighlight(identifier) {
        var startNode, endNode;
        startNode = document.getElementById('s' + identifier);
        endNode = document.getElementById('e' + identifier);
        if (startNode) {
            range = document.createRange();
            range.setStartAfter(startNode);
            range.setEndBefore(endNode);
            sel = window.getSelection();
            sel.removeAllRanges();
            sel.addRange(range);
            highlight('white', 'black');
            startNode.parentNode.removeChild(startNode);
            endNode.parentNode.removeChild(endNode);
            pyCallback.htmlUpdated('');
        }
    }

    function markRange(identifier, bgColor, textColor) {
        var range, sel = window.getSelection();
        if (sel.rangeCount && sel.getRangeAt) {
            range = sel.getRangeAt(0);
            var startNode = document.createElement('span');
            startNode.setAttribute('id', ('s' + identifier));
            startNode.setAttribute('ir-bg-color', bgColor);
            startNode.setAttribute('ir-text-color', textColor);
            range.insertNode(startNode);
            var endNode = document.createElement('span');
            endNode.setAttribute('id', ('e' + identifier));
            // editHighlightLink = document.createElement('a');
            // editHighlightLink.setAttribute('href','javascript:');
            // var tmp = ('unhighlight(' + identifier + '); return false;');
            // editHighlightLink.setAttribute('onclick', tmp);
            // sub = document.createElement('sub');
            // sub.appendChild(document.createTextNode('#'));
            // editHighlightLink.appendChild(sub);
            // endNode.appendChild(editHighlightLink);
            range.collapse(false);
            range.insertNode(endNode);
            range.setStartAfter(startNode);
            range.setEndBefore(endNode);
            sel.removeAllRanges();
            sel.addRange(range);
        }
    }

    function selectMarkedRange(identifier) {
        var startNode, endNode, range, sel;
        startNode = document.getElementById('s' + identifier);
        endNode = document.getElementById('e' + identifier);
        if (startNode && endNode) {
            range = document.createRange();
            range.setStartAfter(startNode);
            range.setEndBefore(endNode);
            sel = window.getSelection();
            sel.removeAllRanges();
            sel.addRange(range);
        }
    }

    function restoreHighlighting() {
        var startNodesXPathResult = document.evaluate(
            '//*[@ir-bg-color]', document, null, XPathResult.ANY_TYPE, null);
        var sNodes = new Array();
        var startNode = startNodesXPathResult.iterateNext();
        while(startNode) {
            sNodes.push(startNode);
            startNode = startNodesXPathResult.iterateNext();
        }
        var id;
        for (var i=0; i < sNodes.length; i++) {
            startNode = sNodes[i];
            id = startNode.id.substring(1);
            selectMarkedRange(id);
            highlight(startNode.getAttribute('ir-bg-color'),
                      startNode.getAttribute('ir-text-color'))
        }
    }

    function removeText() {
        var range, sel = window.getSelection();
        if (sel.rangeCount && sel.getRangeAt) {
            range = sel.getRangeAt(0);
            var startNode = document.createElement('span');
            range.insertNode(startNode);
            var endNode = document.createElement('span');
            range.collapse(false);
            range.insertNode(endNode);
            range.setStartAfter(startNode);
            range.setEndBefore(endNode);
            sel.addRange(range);
            range.deleteContents();
        }
    }

    function getPlainText() {
        return window.getSelection().toString();
    }

    function getHtmlText() {
        var selection = window.getSelection();
        var range = selection.getRangeAt(0);
        var div = document.createElement('div');
        div.appendChild(range.cloneContents());
        return div.innerHTML;
    }
    """
    mw.web.eval(javaScript)

def answerButtonList(self, _old):
    if isIrCard():
        l = ((1, _("Soon")),)
        cnt = mw.col.sched.answerButtons(self.card)
        if cnt == 2:
            return l + ((2, _("Later")),)
        elif cnt == 3:
            return l + ((2, _("Later")), (3, _("Custom")))
        else:
            return l + ((2, _("Later")), (3, _("MuchLater")), (4, _("Custom")))
    else:
        return _old(self)

def answerCard(self, ease, _old):
    # Get the card before scheduler kicks in, else you are looking at a
    #   different card or NONE (which gives error)
    card = self.card

    _old(self, ease)

    if isIrCard():
        mw.readingManager.scheduler.scheduleCard(card, ease)

def buttonTime(self, i, _old):
    if isIrCard():
        return '

'
    else:
        return _old(self, i)

Reviewer._answerButtonList = wrap(Reviewer._answerButtonList,
                                  answerButtonList,
                                  'around')

Reviewer._answerCard = wrap(Reviewer._answerCard, answerCard, 'around')
Reviewer._buttonTime = wrap(Reviewer._buttonTime, buttonTime, 'around')
<|endoftext|>


<commit_before>#  Copyright 2008-2010 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import sys

if __name__ == '__main__':
    sys.stderr.write("Use 'runner' or 'rebot' for executing.\n")
    sys.exit(252)  # 252 == DATA_ERROR

if 'pythonpathsetter' not in sys.modules:
    import pythonpathsetter
from output import Output, CommandLineMonitor, LOGGER
from conf import RobotSettings, RebotSettings
from running import TestSuite, STOP_SIGNAL_MONITOR
from serializing import RobotTestOutput, RebotTestOutput, SplitIndexTestOutput
from errors import (DataError, Information, INFO_PRINTED, DATA_ERROR,
                    STOPPED_BY_USER, FRAMEWORK_ERROR)
from variables import init_global_variables
from version import get_version, get_full_version
import utils

__version__ = get_version()

def run_from_cli(args, usage):
    LOGGER.info(get_full_version('Robot Framework'))
    _run_or_rebot_from_cli(run, args, usage, pythonpath='pythonpath')

def rebot_from_cli(args, usage):
    LOGGER.info(get_full_version('Rebot'))
    _run_or_rebot_from_cli(run_rebot, args, usage)

def _run_or_rebot_from_cli(method, cliargs, usage, **argparser_config):
    LOGGER.register_file_logger()
    ap = utils.ArgumentParser(usage, get_full_version())
    try:
        options, datasources = \
            ap.parse_args(cliargs, argfile='argumentfile', unescape='escape',
                          help='help', version='version', check_args=True,
                          **argparser_config)
    except Information, msg:
        _exit(INFO_PRINTED, utils.unic(msg))
    except DataError, err:
        _exit(DATA_ERROR, utils.unic(err))

    LOGGER.info('Data sources: %s' % utils.seq2str(datasources))
    try:
        suite = method(*datasources, **options)
    except DataError, err:
        _exit(DATA_ERROR, unicode(err))
    except (KeyboardInterrupt, SystemExit):
        _exit(STOPPED_BY_USER, 'Execution stopped by user.')
    except:
        error, details = utils.get_error_details()
        _exit(FRAMEWORK_ERROR, 'Unexpected error: %s' % error, details)
    else:
        _exit(_failed_critical_test_count(suite))

def _failed_critical_test_count(suite):
    rc = suite.critical_stats.failed
    if rc > 250:
        rc = 250
    return rc

def run(*datasources, **options):
    """Executes given Robot data sources with given options.

    Data sources are paths to files and directories, similarly as when running
    pybot/jybot from command line. Options are given as keywords arguments and
    their names are same as long command line options without hyphens.

    Examples:
    run('/path/to/tests.html')
    run('/path/to/tests.html', '/path/to/tests2.html', log='mylog.html')

    Equivalent command line usage:
    pybot /path/to/tests.html
    pybot --log mylog.html /path/to/tests.html /path/to/tests2.html
    """
    STOP_SIGNAL_MONITOR.start()
    settings = RobotSettings(options)
    LOGGER.register_console_logger(settings['MonitorWidth'],
                                   settings['MonitorColors'])
    output = Output(settings)
    init_global_variables(settings)
    suite = TestSuite(datasources, settings)
    suite.run(output)
    LOGGER.info("Tests execution ended. Statistics:\n%s"
                % suite.get_stat_message())
    testoutput = RobotTestOutput(suite, settings)
    output.close(suite)
    if settings.is_rebot_needed():
        datasources, settings = settings.get_rebot_datasources_and_settings()
        if settings['SplitOutputs'] > 0:
            testoutput = SplitIndexTestOutput(suite, datasources[0], settings)
        else:
            testoutput = RebotTestOutput(datasources, settings)
        testoutput.serialize(settings)
    LOGGER.close()
    return suite

def run_rebot(*datasources, **options):
    """Creates reports/logs from given Robot output files with given options.

    Given input files are paths to Robot output files similarly as when running
    rebot from command line. Options are given as keywords arguments and
    their names are same as long command line options without hyphens.

    Examples:
    run_rebot('/path/to/output.xml')
    run_rebot('/path/out1.xml', '/path/out2.xml', report='myrep.html', log='NONE')

    Equivalent command line usage:
    rebot /path/to/output.xml
    rebot --report myrep.html --log NONE /path/out1.xml /path/out2.xml
    """
    settings = RebotSettings(options)
    LOGGER.register_console_logger(colors=settings['MonitorColors'])
    LOGGER.disable_message_cache()
    testoutput = RebotTestOutput(datasources, settings)
    testoutput.serialize(settings, generator='Rebot')
    LOGGER.close()
    return testoutput.suite

def _exit(rc, message=None, details=None):
    """Exits with given rc or rc from given output. Reports possible error.

    Exit code is the number of failed critical tests or error number.
      0       - Tests executed and all critical tests passed
      1-250   - Tests executed but returned number of critical tests failed
                (250 means 250 or more failures)
      251     - Help or version info was printed
      252     - Invalid test data or command line arguments
      253     - Execution stopped by user
      255     - Internal and unexpected error occurred in the framework itself
    """
    if rc == INFO_PRINTED:
        print message
    else:
        if rc == DATA_ERROR:
            message += '\n\nTry --help for usage information.'
        LOGGER.error(message)
        if details:
            LOGGER.info(details)
    sys.exit(rc)
<commit_msg>encode --help/--version output before printing, issue 637<commit_after>#  Copyright 2008-2010 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import sys

if __name__ == '__main__':
    sys.stderr.write("Use 'runner' or 'rebot' for executing.\n")
    sys.exit(252)  # 252 == DATA_ERROR

if 'pythonpathsetter' not in sys.modules:
    import pythonpathsetter
from output import Output, CommandLineMonitor, LOGGER
from conf import RobotSettings, RebotSettings
from running import TestSuite, STOP_SIGNAL_MONITOR
from serializing import RobotTestOutput, RebotTestOutput, SplitIndexTestOutput
from errors import (DataError, Information, INFO_PRINTED, DATA_ERROR,
                    STOPPED_BY_USER, FRAMEWORK_ERROR)
from variables import init_global_variables
from version import get_version, get_full_version
import utils

__version__ = get_version()

def run_from_cli(args, usage):
    LOGGER.info(get_full_version('Robot Framework'))
    _run_or_rebot_from_cli(run, args, usage, pythonpath='pythonpath')

def rebot_from_cli(args, usage):
    LOGGER.info(get_full_version('Rebot'))
    _run_or_rebot_from_cli(run_rebot, args, usage)

def _run_or_rebot_from_cli(method, cliargs, usage, **argparser_config):
    LOGGER.register_file_logger()
    ap = utils.ArgumentParser(usage, get_full_version())
    try:
        options, datasources = \
            ap.parse_args(cliargs, argfile='argumentfile', unescape='escape',
                          help='help', version='version', check_args=True,
                          **argparser_config)
    except Information, msg:
        _exit(INFO_PRINTED, utils.unic(msg))
    except DataError, err:
        _exit(DATA_ERROR, utils.unic(err))

    LOGGER.info('Data sources: %s' % utils.seq2str(datasources))
    try:
        suite = method(*datasources, **options)
    except DataError, err:
        _exit(DATA_ERROR, unicode(err))
    except (KeyboardInterrupt, SystemExit):
        _exit(STOPPED_BY_USER, 'Execution stopped by user.')
    except:
        error, details = utils.get_error_details()
        _exit(FRAMEWORK_ERROR, 'Unexpected error: %s' % error, details)
    else:
        _exit(_failed_critical_test_count(suite))

def _failed_critical_test_count(suite):
    rc = suite.critical_stats.failed
    if rc > 250:
        rc = 250
    return rc

def run(*datasources, **options):
    """Executes given Robot data sources with given options.

    Data sources are paths to files and directories, similarly as when running
    pybot/jybot from command line. Options are given as keywords arguments and
    their names are same as long command line options without hyphens.

    Examples:
    run('/path/to/tests.html')
    run('/path/to/tests.html', '/path/to/tests2.html', log='mylog.html')

    Equivalent command line usage:
    pybot /path/to/tests.html
    pybot --log mylog.html /path/to/tests.html /path/to/tests2.html
    """
    STOP_SIGNAL_MONITOR.start()
    settings = RobotSettings(options)
    LOGGER.register_console_logger(settings['MonitorWidth'],
                                   settings['MonitorColors'])
    output = Output(settings)
    init_global_variables(settings)
    suite = TestSuite(datasources, settings)
    suite.run(output)
    LOGGER.info("Tests execution ended. Statistics:\n%s"
                % suite.get_stat_message())
    testoutput = RobotTestOutput(suite, settings)
    output.close(suite)
    if settings.is_rebot_needed():
        datasources, settings = settings.get_rebot_datasources_and_settings()
        if settings['SplitOutputs'] > 0:
            testoutput = SplitIndexTestOutput(suite, datasources[0], settings)
        else:
            testoutput = RebotTestOutput(datasources, settings)
        testoutput.serialize(settings)
    LOGGER.close()
    return suite

def run_rebot(*datasources, **options):
    """Creates reports/logs from given Robot output files with given options.

    Given input files are paths to Robot output files similarly as when running
    rebot from command line. Options are given as keywords arguments and
    their names are same as long command line options without hyphens.

    Examples:
    run_rebot('/path/to/output.xml')
    run_rebot('/path/out1.xml', '/path/out2.xml', report='myrep.html', log='NONE')

    Equivalent command line usage:
    rebot /path/to/output.xml
    rebot --report myrep.html --log NONE /path/out1.xml /path/out2.xml
    """
    settings = RebotSettings(options)
    LOGGER.register_console_logger(colors=settings['MonitorColors'])
    LOGGER.disable_message_cache()
    testoutput = RebotTestOutput(datasources, settings)
    testoutput.serialize(settings, generator='Rebot')
    LOGGER.close()
    return testoutput.suite

def _exit(rc, message=None, details=None):
    """Exits with given rc or rc from given output. Reports possible error.

    Exit code is the number of failed critical tests or error number.
      0       - Tests executed and all critical tests passed
      1-250   - Tests executed but returned number of critical tests failed
                (250 means 250 or more failures)
      251     - Help or version info was printed
      252     - Invalid test data or command line arguments
      253     - Execution stopped by user
      255     - Internal and unexpected error occurred in the framework itself
    """
    if rc == INFO_PRINTED:
        print utils.encode_output(message)
    else:
        if rc == DATA_ERROR:
            message += '\n\nTry --help for usage information.'
        LOGGER.error(message)
        if details:
            LOGGER.info(details)
    sys.exit(rc)
<|endoftext|>


<commit_before>import json
import logging
import os
import sys
import time

import networkx as nx
import py2neo
import requests
from networkx.readwrite import GraphMLReader
from networkx.readwrite import json_graph
from pyparsing import ParseException
from requests_file import FileAdapter

from .exceptions import PyBelWarning, PyBelError
from .manager.namespace_cache import DefinitionCacheManager
from .parser.parse_bel import BelParser
from .parser.parse_metadata import MetadataParser
from .parser.utils import split_file_to_annotations_and_definitions, subdict_matches
from .utils import flatten, flatten_graph_data, expand_dict

from collections import defaultdict

__all__ = ['BELGraph', 'from_url', 'from_path', 'from_pickle',
           'from_graphml', 'to_graphml', 'to_json', 'to_neo4j', 'to_pickle', 'from_json']

log = logging.getLogger('pybel')

PYBEL_CONTEXT_TAG = 'pybel_context'

def from_url(url, **kwargs):
    """Loads a BEL graph from a URL resource

    :param url: a valid URL pointing to a BEL resource
    :type url: str
    :param \**kwargs: keyword arguments to pass to :py:meth:`BELGraph`
    :return: a parsed BEL graph
    :rtype: BELGraph
    """
    log.info('Loading from url: %s', url)

    session = requests.session()
    if url.startswith('file://'):
        session.mount('file://', FileAdapter())
    response = session.get(url)
    response.raise_for_status()

    lines = (line.decode('utf-8') for line in response.iter_lines())

    return BELGraph(lines=lines, **kwargs)

def from_path(path, **kwargs):
    """Loads a BEL graph from a file resource

    :param path: a file path
    :type path: str
    :param \**kwargs: keyword arguments to pass to :py:meth:`BELGraph`
    :return: a parsed BEL graph
    :rtype: BELGraph"""

    log.info('Loading from path: %s', path)
    with open(os.path.expanduser(path)) as f:
        return BELGraph(lines=f, **kwargs)

def from_database(connection):
    """Loads a BEL graph from a database

    :param connection: The string form of the URL is :code:`dialect[+driver]://user:password@host/dbname[?key=value..]`,
                       where dialect is a database name such as mysql, oracle, postgresql, etc., and driver the name
                       of a DBAPI, such as psycopg2, pyodbc, cx_oracle, etc. Alternatively, the URL can be an instance
                       of URL.
    :type connection: str
    :return: a BEL graph loaded from the database
    :rtype: BELGraph
    """
    raise NotImplementedError("Can't load from from database: {}".format(connection))

class BELGraph(nx.MultiDiGraph):
    """An extension of a NetworkX MultiDiGraph to hold a BEL graph."""

    def __init__(self, lines=None, context=None, lenient=False, definition_cache_manager=None, log_stream=None,
                 *attrs, **kwargs):
        """Parses a BEL file from an iterable of strings. This can be a file, file-like, or list of strings.

        :param lines: iterable over lines of BEL data file
        :param context: disease context string
        :type context: str
        :param lenient: if true, allow naked namespaces
        :type lenient: bool
        :param definition_cache_manager: database connection string to namespace namspace_cache, pre-built namespace namspace_cache manager,
                    or True to use the default
        :type definition_cache_manager: str or pybel.mangager.NamespaceCache or bool
        :param log_stream: a stream to write debug logging to
        :param \*attrs: arguments to pass to :py:meth:`networkx.MultiDiGraph`
        :param \**kwargs: keyword arguments to pass to :py:meth:`networkx.MultiDiGraph`
        """
        nx.MultiDiGraph.__init__(self, *attrs, **kwargs)

        self.last_parse_errors = defaultdict(int)

        if lines is not None:
            self.parse_lines(lines, context, lenient, definition_cache_manager, log_stream)

    def parse_lines(self, lines, context=None, lenient=False, definition_cache_manager=None, log_stream=None):
        """Parses an iterable of lines into this graph

        :param lines: iterable over lines of BEL data file
        :param context: disease context string
        :type context: str
        :param lenient: if true, allow naked namespaces
        :type lenient: bool
        :param definition_cache_manager: database connection string to namespace namspace_cache, pre-built namespace namspace_cache manager,
                    or True to use the default
        :type definition_cache_manager: str or pybel.mangager.NamespaceCache or bool
        :param log_stream: a stream to write debug logging to
        """
        if log_stream is not None:
            sh = logging.StreamHandler(stream=log_stream)
            sh.setLevel(5)
            sh.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s'))
            log.addHandler(sh)

        self.context = context

        docs, defs, states = split_file_to_annotations_and_definitions(lines)

        self.graph['document_lines'] = docs
        self.graph['definition_lines'] = defs

        if isinstance(definition_cache_manager, DefinitionCacheManager):
            self.metadata_parser = MetadataParser(definition_cache_manager=definition_cache_manager)
        elif isinstance(definition_cache_manager, str):
            self.metadata_parser = MetadataParser(
                definition_cache_manager=DefinitionCacheManager(conn=definition_cache_manager))
        else:
            self.metadata_parser = MetadataParser(definition_cache_manager=DefinitionCacheManager())

        self.parse_document(docs)

        self.parse_definitions(defs)

        self.bel_parser = BelParser(graph=self,
                                    valid_namespaces=self.metadata_parser.namespace_dict,
                                    valid_annotations=self.metadata_parser.annotations_dict,
                                    lenient=lenient)

        self.parse_statements(states)

        log.info('Network has %d nodes and %d edges', self.number_of_nodes(), self.number_of_edges())

    def parse_document(self, document_metadata):
        t = time.time()

        for line_number, line in document_metadata:
            try:
                self.metadata_parser.parseString(line)
            except:
                log.error('Line %07d - failed: %s', line_number, line)

        self.graph['document_metadata'] = self.metadata_parser.document_metadata

        log.info('Finished parsing document section in %.02f seconds', time.time() - t)

    def parse_definitions(self, definitions):
        t = time.time()

        for line_number, line in definitions:
            try:
                self.metadata_parser.parseString(line)
            except PyBelError as e:
                log.critical('Line %07d - %s', line_number, line)
                raise e
            except ParseException as e:
                log.error('Line %07d - invalid statement: %s', line_number, line)
            except PyBelWarning as e:
                log.warning('Line %07d - %s: %s', line_number, e, line)
            except:
                exc_type, exc_value, exc_traceback = sys.exc_info()
                log.error('Line %07d - general failure: %s - %s: %s', line_number, line, exc_type, exc_value)
                log.debug('Traceback: %s', exc_traceback)

        self.graph['namespace_owl'] = self.metadata_parser.namespace_owl_dict
        self.graph['namespace_url'] = self.metadata_parser.namespace_url_dict
        self.graph['namespace_list'] = {e: self.metadata_parser.namespace_dict[e] for e in
                                        self.metadata_parser.namespace_list_list}
        self.graph['annotation_url'] = self.metadata_parser.annotation_url_dict
        self.graph['annotation_list'] = {e: self.metadata_parser.annotations_dict[e] for e in
                                         self.metadata_parser.annotation_list_list}

        log.info('Finished parsing definitions section in %.02f seconds', time.time() - t)

    def parse_statements(self, statements):
        t = time.time()

        self.bel_parser.language.streamline()
        log.info('Finished streamlining BEL parser in %.02fs', time.time() - t)

        t = time.time()

        self.last_parse_errors = defaultdict(int)

        for line_number, line in statements:
            try:
                self.bel_parser.parseString(line)
            except PyBelError as e:
                log.critical('Line %07d - %s', line_number, line)
                raise e
            except ParseException as e:
                log.error('Line %07d - general parser failure: %s', line_number, line)
                self.last_parse_errors['parse_exception'] += 1
            except PyBelWarning as e:
                log.warning('Line %07d - %s: %s', line_number, e, line)
                self.last_parse_errors[e.code] += 1
            except:
                exc_type, exc_value, exc_traceback = sys.exc_info()
                log.error('Line %07d - general failure: %s - %s: %s', line_number, line, exc_type, exc_value)
                self.last_parse_errors['general'] += 1

        log.info('Finished parsing statements section in %.02f seconds with %s warnings', time.time() - t,
                 dict(self.last_parse_errors))

    def edges_iter(self, nbunch=None, data=False, keys=False, default=None, **kwargs):
        """Allows for filtering by checking keyword arguments are a subdictionary of each edges' data. See :py:meth:`networkx.MultiDiGraph.edges_iter`"""
        for u, v, k, d in nx.MultiDiGraph.edges_iter(self, nbunch=nbunch, data=True, keys=True, default=default):
            if not subdict_matches(d, kwargs):
                continue
            elif keys and data:
                yield u, v, k, d
            elif data:
                yield u, v, d
            elif keys:
                yield u, v, k
            else:
                yield u, v

    def nodes_iter(self, data=False, **kwargs):
        """Allows for filtering by checking keyword arguments are a subdictionary of each nodes' data. See :py:meth:`networkx.MultiDiGraph.edges_iter`"""
        for n, d in nx.MultiDiGraph.nodes_iter(self, data=True):
            if not subdict_matches(d, kwargs):
                continue
            elif data:
                yield n, d
            else:
                yield n

    @property
    def document(self):
        return self.graph['document_metadata']

    @property
    def namespace_url(self):
        return self.graph['namespace_url']

    @property
    def namespace_owl(self):
        return self.graph['namespace_owl']

    @property
    def namespace_list(self):
        return self.graph['namespace_list']

    @property
    def annotation_url(self):
        return self.graph['annotation_url']

    @property
    def annotation_list(self):
        return self.graph['annotation_list']

def to_neo4j(graph, neo_graph, context=None):
    """Uploads a BEL graph to Neo4J graph database using `py2neo`

    :param graph: a BEL Graph
    :type graph: BELGraph
    :param neo_graph: a py2neo graph object, Refer to the `py2neo documentation <http://py2neo.org/v3/database.html#the-graph>`_ for how to build this object.
    :type neo_graph: py2neo.Graph
    :param context: a disease context to allow for multiple disease models in one neo4j instance. Each edge will be assigned an attribute :code:`pybel_context` with this value
    :type context: str
    """

    if context is not None:
        graph.context = context

    tx = neo_graph.begin()

    node_map = {}
    for i, (node, data) in enumerate(graph.nodes(data=True)):
        node_type = data['type']
        attrs = {k: v for k, v in data.items() if k != 'type'}

        if 'name' in data:
            attrs['value'] = data['name']

        node_map[node] = py2neo.Node(node_type, cname=str(node), cnum=str(i), **attrs)

        tx.create(node_map[node])

    for u, v, data in graph.edges(data=True):
        neo_u = node_map[u]
        neo_v = node_map[v]
        rel_type = data['relation']
        attrs = flatten(data)
        if graph.context is not None:
            attrs[PYBEL_CONTEXT_TAG] = str(graph.context)
        rel = py2neo.Relationship(neo_u, rel_type, neo_v, **attrs)
        tx.create(rel)

    tx.commit()

def to_pickle(graph, output):
    """Writes this graph to a pickle object with nx.write_gpickle

    Cast as a nx.MultiDiGraph before outputting because the pickle serializer can't handle the PyParsing elements
    within the BELGraph class.

    :param graph: a BEL graph
    :type graph: BELGraph
    :param output: a file or filename to write to
    """
    nx.write_gpickle(nx.MultiDiGraph(graph), output)

def from_pickle(path):
    """Reads a graph from a gpickle file

    :param path: File or filename to write. Filenames ending in .gz or .bz2 will be uncompressed.
    :type path: file or list
    :rtype: BELGraph
    """
    return BELGraph(data=nx.read_gpickle(path))

def to_json(graph, output):
    """Writes this graph to a node-link JSON object

    :param graph: a BEL graph
    :type graph: BELGraph
    :param output: a write-supporting filelike object
    """
    # FIXME
    data = json_graph.node_link_data(graph)
    json.dump(data, output, ensure_ascii=False)

def from_json(path):
    """Reads graph from node-link JSON Object"""
    with open(os.path.expanduser(path)) as f:
        data = json.load(f)

    for i, node in enumerate(data['nodes']):
        data['nodes'][i]['id'] = tuple(node['id'])

    g = json_graph.node_link_graph(data, directed=True, multigraph=True)
    return BELGraph(data=g)

def to_graphml(graph, output):
    """Writes this graph to GraphML file. Use .graphml extension so Cytoscape can recognize it

    :param graph: a BEL graph
    :type graph: BELGraph
    :param output: a file or filelike object
    """

    g = nx.MultiDiGraph()

    for node, data in graph.nodes(data=True):
        g.add_node(node, json=json.dumps(data))

    for u, v, key, data in graph.edges(data=True, keys=True):
        g.add_edge(u, v, key=key, attr_dict=flatten(data))

    nx.write_graphml(g, output)

def from_graphml(path):
    """Reads a graph from a graphml file

    :param path: File or filename to write
    :type path: file or str
    :rtype: networkx.MultiDiGraph
    """
    reader = GraphMLReader(node_type=str)
    reader.multigraph = True
    g = list(reader(path=path))[0]
    g = expand_edges(g)
    for n in g:
        g.node[n] = json.loads(g.node[n]['json'])
    return g

def to_csv(graph, output):
    """Writes graph to edge list csv

    :param graph: a BEL graph
    :type graph: BELGraph
    :param output: a file or filelike object
    """
    nx.write_edgelist(flatten_graph_data(graph), output, data=True)

def expand_edges(graph):
    """Returns a new graph with expanded edge data dictionaries

    :param graph: nx.MultiDiGraph
    :type graph: BELGraph
    :rtype: BELGraph
    """

    g = BELGraph()

    for node, data in graph.nodes(data=True):
        g.add_node(node, data)

    for u, v, key, data in graph.edges(data=True, keys=True):
        g.add_edge(u, v, key=key, attr_dict=expand_dict(data))

    return g
<commit_msg>Update logging for graph parsing<commit_after>import json
import logging
import os
import sys
import time
from collections import Counter
from collections import defaultdict

import networkx as nx
import py2neo
import requests
from networkx.readwrite import GraphMLReader
from networkx.readwrite import json_graph
from pyparsing import ParseException
from requests_file import FileAdapter

from .exceptions import PyBelWarning, PyBelError
from .manager.namespace_cache import DefinitionCacheManager
from .parser.parse_bel import BelParser
from .parser.parse_metadata import MetadataParser
from .parser.utils import split_file_to_annotations_and_definitions, subdict_matches
from .utils import flatten, flatten_graph_data, expand_dict

__all__ = ['BELGraph', 'from_url', 'from_path', 'from_pickle',
           'from_graphml', 'to_graphml', 'to_json', 'to_neo4j', 'to_pickle', 'from_json']

log = logging.getLogger('pybel')

PYBEL_CONTEXT_TAG = 'pybel_context'

def from_url(url, **kwargs):
    """Loads a BEL graph from a URL resource

    :param url: a valid URL pointing to a BEL resource
    :type url: str
    :param \**kwargs: keyword arguments to pass to :py:meth:`BELGraph`
    :return: a parsed BEL graph
    :rtype: BELGraph
    """
    log.info('Loading from url: %s', url)

    session = requests.session()
    if url.startswith('file://'):
        session.mount('file://', FileAdapter())
    response = session.get(url)
    response.raise_for_status()

    lines = (line.decode('utf-8') for line in response.iter_lines())

    return BELGraph(lines=lines, **kwargs)

def from_path(path, **kwargs):
    """Loads a BEL graph from a file resource

    :param path: a file path
    :type path: str
    :param \**kwargs: keyword arguments to pass to :py:meth:`BELGraph`
    :return: a parsed BEL graph
    :rtype: BELGraph"""

    log.info('Loading from path: %s', path)
    with open(os.path.expanduser(path)) as f:
        return BELGraph(lines=f, **kwargs)

def from_database(connection):
    """Loads a BEL graph from a database

    :param connection: The string form of the URL is :code:`dialect[+driver]://user:password@host/dbname[?key=value..]`,
                       where dialect is a database name such as mysql, oracle, postgresql, etc., and driver the name
                       of a DBAPI, such as psycopg2, pyodbc, cx_oracle, etc. Alternatively, the URL can be an instance
                       of URL.
    :type connection: str
    :return: a BEL graph loaded from the database
    :rtype: BELGraph
    """
    raise NotImplementedError("Can't load from from database: {}".format(connection))

class BELGraph(nx.MultiDiGraph):
    """An extension of a NetworkX MultiDiGraph to hold a BEL graph."""

    def __init__(self, lines=None, context=None, lenient=False, definition_cache_manager=None, log_stream=None,
                 *attrs, **kwargs):
        """Parses a BEL file from an iterable of strings. This can be a file, file-like, or list of strings.

        :param lines: iterable over lines of BEL data file
        :param context: disease context string
        :type context: str
        :param lenient: if true, allow naked namespaces
        :type lenient: bool
        :param definition_cache_manager: database connection string to namespace namspace_cache, pre-built namespace namspace_cache manager,
                    or True to use the default
        :type definition_cache_manager: str or pybel.mangager.NamespaceCache or bool
        :param log_stream: a stream to write debug logging to
        :param \*attrs: arguments to pass to :py:meth:`networkx.MultiDiGraph`
        :param \**kwargs: keyword arguments to pass to :py:meth:`networkx.MultiDiGraph`
        """
        nx.MultiDiGraph.__init__(self, *attrs, **kwargs)

        self.last_parse_errors = defaultdict(int)

        if lines is not None:
            self.parse_lines(lines, context, lenient, definition_cache_manager, log_stream)

    def parse_lines(self, lines, context=None, lenient=False, definition_cache_manager=None, log_stream=None):
        """Parses an iterable of lines into this graph

        :param lines: iterable over lines of BEL data file
        :param context: disease context string
        :type context: str
        :param lenient: if true, allow naked namespaces
        :type lenient: bool
        :param definition_cache_manager: database connection string to namespace namspace_cache, pre-built namespace namspace_cache manager,
                    or True to use the default
        :type definition_cache_manager: str or pybel.mangager.NamespaceCache or bool
        :param log_stream: a stream to write debug logging to
        """
        if log_stream is not None:
            sh = logging.StreamHandler(stream=log_stream)
            sh.setLevel(5)
            sh.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s'))
            log.addHandler(sh)

        self.context = context

        docs, defs, states = split_file_to_annotations_and_definitions(lines)

        self.graph['document_lines'] = docs
        self.graph['definition_lines'] = defs

        if isinstance(definition_cache_manager, DefinitionCacheManager):
            self.metadata_parser = MetadataParser(definition_cache_manager=definition_cache_manager)
        elif isinstance(definition_cache_manager, str):
            self.metadata_parser = MetadataParser(
                definition_cache_manager=DefinitionCacheManager(conn=definition_cache_manager))
        else:
            self.metadata_parser = MetadataParser(definition_cache_manager=DefinitionCacheManager())

        self.parse_document(docs)

        self.parse_definitions(defs)

        self.bel_parser = BelParser(graph=self,
                                    valid_namespaces=self.metadata_parser.namespace_dict,
                                    valid_annotations=self.metadata_parser.annotations_dict,
                                    lenient=lenient)

        self.parse_statements(states)

        log.info('Network has %d nodes and %d edges', self.number_of_nodes(), self.number_of_edges())
        for fn, count in sorted(Counter(d['type'] for n, d in self.nodes_iter(data=True)).items()):
            log.debug(' %s: %d', fn, count)

    def parse_document(self, document_metadata):
        t = time.time()

        for line_number, line in document_metadata:
            try:
                self.metadata_parser.parseString(line)
            except:
                log.error('Line %07d - failed: %s', line_number, line)

        self.graph['document_metadata'] = self.metadata_parser.document_metadata

        log.info('Finished parsing document section in %.02f seconds', time.time() - t)

    def parse_definitions(self, definitions):
        t = time.time()

        for line_number, line in definitions:
            try:
                self.metadata_parser.parseString(line)
            except PyBelError as e:
                log.critical('Line %07d - %s', line_number, line)
                raise e
            except ParseException as e:
                log.error('Line %07d - invalid statement: %s', line_number, line)
            except PyBelWarning as e:
                log.warning('Line %07d - %s: %s', line_number, e, line)
            except:
                exc_type, exc_value, exc_traceback = sys.exc_info()
                log.error('Line %07d - general failure: %s - %s: %s', line_number, line, exc_type, exc_value)
                log.debug('Traceback: %s', exc_traceback)

        self.graph['namespace_owl'] = self.metadata_parser.namespace_owl_dict
        self.graph['namespace_url'] = self.metadata_parser.namespace_url_dict
        self.graph['namespace_list'] = {e: self.metadata_parser.namespace_dict[e] for e in
                                        self.metadata_parser.namespace_list_list}
        self.graph['annotation_url'] = self.metadata_parser.annotation_url_dict
        self.graph['annotation_list'] = {e: self.metadata_parser.annotations_dict[e] for e in
                                         self.metadata_parser.annotation_list_list}

        log.info('Finished parsing definitions section in %.02f seconds', time.time() - t)

    def parse_statements(self, statements):
        t = time.time()

        self.bel_parser.language.streamline()
        log.info('Finished streamlining BEL parser in %.02fs', time.time() - t)

        t = time.time()

        self.last_parse_errors = defaultdict(int)

        for line_number, line in statements:
            try:
                self.bel_parser.parseString(line)
            except PyBelError as e:
                log.critical('Line %07d - %s', line_number, line)
                raise e
            except ParseException as e:
                log.error('Line %07d - general parser failure: %s', line_number, line)
                self.last_parse_errors['parse_exception'] += 1
            except PyBelWarning as e:
                log.warning('Line %07d - %s: %s', line_number, e, line)
                self.last_parse_errors[e.code] += 1
            except:
                exc_type, exc_value, exc_traceback = sys.exc_info()
                log.error('Line %07d - general failure: %s - %s: %s', line_number, line, exc_type, exc_value)
                self.last_parse_errors['general'] += 1

        log.info('Finished parsing statements section in %.02f seconds with %s warnings', time.time() - t,
                 dict(self.last_parse_errors))

    def edges_iter(self, nbunch=None, data=False, keys=False, default=None, **kwargs):
        """Allows for filtering by checking keyword arguments are a subdictionary of each edges' data. See :py:meth:`networkx.MultiDiGraph.edges_iter`"""
        for u, v, k, d in nx.MultiDiGraph.edges_iter(self, nbunch=nbunch, data=True, keys=True, default=default):
            if not subdict_matches(d, kwargs):
                continue
            elif keys and data:
                yield u, v, k, d
            elif data:
                yield u, v, d
            elif keys:
                yield u, v, k
            else:
                yield u, v

    def nodes_iter(self, data=False, **kwargs):
        """Allows for filtering by checking keyword arguments are a subdictionary of each nodes' data. See :py:meth:`networkx.MultiDiGraph.edges_iter`"""
        for n, d in nx.MultiDiGraph.nodes_iter(self, data=True):
            if not subdict_matches(d, kwargs):
                continue
            elif data:
                yield n, d
            else:
                yield n

    @property
    def document(self):
        return self.graph['document_metadata']

    @property
    def namespace_url(self):
        return self.graph['namespace_url']

    @property
    def namespace_owl(self):
        return self.graph['namespace_owl']

    @property
    def namespace_list(self):
        return self.graph['namespace_list']

    @property
    def annotation_url(self):
        return self.graph['annotation_url']

    @property
    def annotation_list(self):
        return self.graph['annotation_list']

def to_neo4j(graph, neo_graph, context=None):
    """Uploads a BEL graph to Neo4J graph database using `py2neo`

    :param graph: a BEL Graph
    :type graph: BELGraph
    :param neo_graph: a py2neo graph object, Refer to the `py2neo documentation <http://py2neo.org/v3/database.html#the-graph>`_ for how to build this object.
    :type neo_graph: py2neo.Graph
    :param context: a disease context to allow for multiple disease models in one neo4j instance. Each edge will be assigned an attribute :code:`pybel_context` with this value
    :type context: str
    """

    if context is not None:
        graph.context = context

    tx = neo_graph.begin()

    node_map = {}
    for i, (node, data) in enumerate(graph.nodes(data=True)):
        node_type = data['type']
        attrs = {k: v for k, v in data.items() if k != 'type'}

        if 'name' in data:
            attrs['value'] = data['name']

        node_map[node] = py2neo.Node(node_type, cname=str(node), cnum=str(i), **attrs)

        tx.create(node_map[node])

    for u, v, data in graph.edges(data=True):
        neo_u = node_map[u]
        neo_v = node_map[v]
        rel_type = data['relation']
        attrs = flatten(data)
        if graph.context is not None:
            attrs[PYBEL_CONTEXT_TAG] = str(graph.context)
        rel = py2neo.Relationship(neo_u, rel_type, neo_v, **attrs)
        tx.create(rel)

    tx.commit()

def to_pickle(graph, output):
    """Writes this graph to a pickle object with nx.write_gpickle

    Cast as a nx.MultiDiGraph before outputting because the pickle serializer can't handle the PyParsing elements
    within the BELGraph class.

    :param graph: a BEL graph
    :type graph: BELGraph
    :param output: a file or filename to write to
    """
    nx.write_gpickle(nx.MultiDiGraph(graph), output)

def from_pickle(path):
    """Reads a graph from a gpickle file

    :param path: File or filename to write. Filenames ending in .gz or .bz2 will be uncompressed.
    :type path: file or list
    :rtype: BELGraph
    """
    return BELGraph(data=nx.read_gpickle(path))

def to_json(graph, output):
    """Writes this graph to a node-link JSON object

    :param graph: a BEL graph
    :type graph: BELGraph
    :param output: a write-supporting filelike object
    """
    # FIXME
    data = json_graph.node_link_data(graph)
    json.dump(data, output, ensure_ascii=False)

def from_json(path):
    """Reads graph from node-link JSON Object"""
    with open(os.path.expanduser(path)) as f:
        data = json.load(f)

    for i, node in enumerate(data['nodes']):
        data['nodes'][i]['id'] = tuple(node['id'])

    g = json_graph.node_link_graph(data, directed=True, multigraph=True)
    return BELGraph(data=g)

def to_graphml(graph, output):
    """Writes this graph to GraphML file. Use .graphml extension so Cytoscape can recognize it

    :param graph: a BEL graph
    :type graph: BELGraph
    :param output: a file or filelike object
    """

    g = nx.MultiDiGraph()

    for node, data in graph.nodes(data=True):
        g.add_node(node, json=json.dumps(data))

    for u, v, key, data in graph.edges(data=True, keys=True):
        g.add_edge(u, v, key=key, attr_dict=flatten(data))

    nx.write_graphml(g, output)

def from_graphml(path):
    """Reads a graph from a graphml file

    :param path: File or filename to write
    :type path: file or str
    :rtype: networkx.MultiDiGraph
    """
    reader = GraphMLReader(node_type=str)
    reader.multigraph = True
    g = list(reader(path=path))[0]
    g = expand_edges(g)
    for n in g:
        g.node[n] = json.loads(g.node[n]['json'])
    return g

def to_csv(graph, output):
    """Writes graph to edge list csv

    :param graph: a BEL graph
    :type graph: BELGraph
    :param output: a file or filelike object
    """
    nx.write_edgelist(flatten_graph_data(graph), output, data=True)

def expand_edges(graph):
    """Returns a new graph with expanded edge data dictionaries

    :param graph: nx.MultiDiGraph
    :type graph: BELGraph
    :rtype: BELGraph
    """

    g = BELGraph()

    for node, data in graph.nodes(data=True):
        g.add_node(node, data)

    for u, v, key, data in graph.edges(data=True, keys=True):
        g.add_edge(u, v, key=key, attr_dict=expand_dict(data))

    return g
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import json
import logging
import pickle
from configparser import ConfigParser
from datetime import datetime

import hashlib
import networkx as nx
import os
import requests
import requests.exceptions
from collections import defaultdict, MutableMapping
from pkg_resources import get_distribution
from requests.compat import urlparse
from requests_file import FileAdapter
from six import string_types

from .constants import CITATION_ENTRIES, BELFRAMEWORK_DOMAIN, OPENBEL_DOMAIN

log = logging.getLogger(__name__)

def download(url):
    """Uses requests to download an URL, maybe from a file"""
    session = requests.Session()
    session.mount('file://', FileAdapter())

    if url.startswith(BELFRAMEWORK_DOMAIN):
        log.warning('%s has expired', BELFRAMEWORK_DOMAIN)
        url = url.replace(BELFRAMEWORK_DOMAIN, OPENBEL_DOMAIN)

    try:
        res = session.get(url)
    except requests.exceptions.ConnectionError as e:
        raise e

    res.raise_for_status()
    return res

def parse_bel_resource(lines):
    """Parses a BEL config (BELNS, BELANNO, or BELEQ) file from the given line iterator over the file

    :param iter[str] lines: An iterable over the lines in a BEL config file
    :return: A config-style dictionary representing the BEL config file
    :rtype: dict
    """
    lines = list(lines)

    value_line = 1 + max(i for i, line in enumerate(lines) if '[Values]' == line.strip())

    metadata_config = ConfigParser(strict=False)
    metadata_config.optionxform = lambda option: option
    metadata_config.read_file(lines[:value_line])

    delimiter = metadata_config['Processing']['DelimiterString']

    value_dict = {}
    for line in lines[value_line:]:
        sline = line.rsplit(delimiter, 1)
        key = sline[0].strip()

        value_dict[key] = sline[1].strip() if len(sline) == 2 else None

    res = {}
    res.update({k: dict(v) for k, v in metadata_config.items()})
    res['Values'] = value_dict

    return res

def is_url(s):
    """Checks if a string is a valid URL

    :param str s: An input string
    :return: Is the string a valid URL?
    :rtype: bool
    """
    return urlparse(s).scheme != ""

def get_bel_resource(location):
    """Loads/downloads and parses a config file from the given url or file path

    :param str location: The URL or file path to a BELNS, BELANNO, or BELEQ file to download and parse
    :return: A config-style dictionary representing the BEL config file
    :rtype: dict
    """

    if is_url(location):
        res = download(location)
        lines = list(line.decode('utf-8', errors='ignore').strip() for line in res.iter_lines())
    else:
        with open(os.path.expanduser(location)) as f:
            lines = list(f)

    try:
        result = parse_bel_resource(lines)
    except ValueError:
        log.error('No [Values] section found in %s', location)
        raise ValueError('No [Values] section found in {}'.format(location))

    if not result['Values']:
        raise ValueError('Downloaded empty file: {}'.format(location))

    return result

def expand_dict(flat_dict, sep='_'):
    """Expands a flattened dictionary

    :param dict flat_dict: a nested dictionary that has been flattened so the keys are composite
    :param str sep: the separator between concatenated keys
    """
    res = {}
    rdict = defaultdict(list)

    for flat_key, value in flat_dict.items():
        key = flat_key.split(sep, 1)
        if 1 == len(key):
            res[key[0]] = value
        else:
            rdict[key[0]].append((key[1:], value))

    for k, v in rdict.items():
        res[k] = expand_dict({ik: iv for (ik,), iv in v})

    return res

def flatten_dict(d, parent_key='', sep='_'):
    """Flattens a nested dictionary.

    :param d: A nested dictionary
    :type d: dict or MutableMapping
    :param str parent_key: The parent's key. This is a value for tail recursion, so don't set it yourself.
    :param str sep: The separator used between dictionary levels

    .. seealso:: http://stackoverflow.com/a/6027615
    """
    items = []
    for k, v in d.items():
        new_key = parent_key + sep + k if parent_key else k
        if isinstance(v, (dict, MutableMapping)):
            items.extend(flatten_dict(v, new_key, sep=sep).items())
        elif isinstance(v, (set, list)):
            items.append((new_key, ','.join(v)))
        else:
            items.append((new_key, v))
    return dict(items)

def flatten_graph_data(graph):
    """Returns a new graph with flattened edge data dictionaries.

    :param nx.MultiDiGraph graph: A graph with nested edge data dictionaries
    :return: A graph with flattened edge data dictionaries
    :rtype: nx.MultiDiGraph
    """
    g = nx.MultiDiGraph(**graph.graph)

    for node, data in graph.nodes(data=True):
        g.add_node(node, data)

    for u, v, key, data in graph.edges(data=True, keys=True):
        g.add_edge(u, v, key=key, attr_dict=flatten_dict(data))

    return g

def list2tuple(l):
    """Recursively converts a nested list to a nested tuple"""
    if isinstance(l, list):
        return tuple(list2tuple(e) for e in l)
    else:
        return l

def get_version():
    """Gets the current PyBEL version

    :return: The current PyBEL version
    :rtype: str
    """
    return get_distribution('pybel').version

def tokenize_version(version_string):
    """Tokenizes a version string to a tuple. Truncates qualifiers like ``-dev``.

    :param str version_string: A version string
    :return: A tuple representing the version string
    :rtype: tuple

    >>> tokenize_version('0.1.2-dev')
    (0, 1, 2)

    """
    return tuple(map(int, version_string.split('.')[0:3]))

def citation_dict_to_tuple(citation):
    """Convert the ``d[CITATION]`` entry in an edge data dictionary to a tuple"""
    if all(x in citation for x in CITATION_ENTRIES):
        return tuple(citation[x] for x in CITATION_ENTRIES)

    if all(x in citation for x in CITATION_ENTRIES[3:5]):
        return tuple(citation[x] for x in CITATION_ENTRIES[:5])

    if all(x in citation for x in CITATION_ENTRIES[3:4]):
        return tuple(citation[x] for x in CITATION_ENTRIES[:4])

    return tuple(citation[x] for x in CITATION_ENTRIES[:3])

def flatten_citation(citation):
    """Flattens a citation dict, from the ``d[CITATION]`` entry in an edge data dictionary"""
    return ','.join('"{}"'.format(e) for e in citation_dict_to_tuple(citation))

def ensure_quotes(s):
    """Quote a string that isn't solely alphanumeric

    :param str s: a string
    :rtype: str
    """
    return '"{}"'.format(s) if not s.isalnum() else s

CREATION_DATE_FMT = '%Y-%m-%dT%H:%M:%S'
PUBLISHED_DATE_FMT = '%Y-%m-%d'
PUBLISHED_DATE_FMT_2 = '%d:%m:%Y %H:%M'

def valid_date(s):
    """Checks that a string represents a valid date in ISO 8601 format YYYY-MM-DD

    :type s: str
    :rtype: bool
    """
    try:
        datetime.strptime(s, '%Y-%m-%d')
        return True
    except ValueError:
        return False

def valid_date_version(s):
    """Checks that the string is a valid date versions string"""
    try:
        datetime.strptime(s, '%Y%m%d')
        return True
    except ValueError:
        return False

def parse_datetime(s):
    """Tries to parse a datetime object from a standard datetime format or date format

    :param str s: A string representing a date or datetime
    :return: A parsed date object
    :rtype: datetime.date
    """
    try:
        dt = datetime.strptime(s, CREATION_DATE_FMT)
        return dt
    except:
        try:
            dt = datetime.strptime(s, PUBLISHED_DATE_FMT)
            return dt
        except:
            try:
                dt = datetime.strptime(s, PUBLISHED_DATE_FMT_2)
                return dt
            except:
                raise ValueError('Incorrect datetime format for {}'.format(s))

def hash_node(node):
    """Converts a PyBEL node tuple to a hash

    :param tuple node: A BEL node
    :return: A hashed version of the node tuple using md5 hash of the binary pickle dump
    :rtype: str
    """
    return hashlib.md5(pickle.dumps(node)).hexdigest()

def hash_edge(u, v, k, d):
    """Converts an edge tuple to a hash

    :return: A hashed version of the edge tuple using md5 hash of the binary pickle dump of u, v, and the json dump of d
    :rtype: str
    """
    edge_tuple = (u, v, json.dumps(d, ensure_ascii=False, sort_keys=True))
    return hashlib.md5(pickle.dumps(edge_tuple)).hexdigest()

def subdict_matches(target, query, partial_match=True):
    """Checks if all the keys in the query dict are in the target dict, and that their values match

    1. Checks that all keys in the query dict are in the target dict
    2. Matches the values of the keys in the query dict
        a. If the value is a string, then must match exactly
        b. If the value is a set/list/tuple, then will match any of them
        c. If the value is a dict, then recursively check if that subdict matches

    :param dict target: The dictionary to search
    :param dict query: A query dict with keys to match
    :param bool partial_match: Should the query values be used as partial or exact matches? Defaults to :code:`True`.
    :return: if all keys in b are in target_dict and their values match
    :rtype: bool
    """
    for k, v in query.items():
        if k not in target:
            return False
        elif not isinstance(v, (int, string_types, list, set, dict, tuple)):
            raise ValueError('invalid value: {}'.format(v))
        elif isinstance(v, (int, string_types)) and target[k] != v:
            return False
        elif isinstance(v, dict):
            if partial_match:
                if not isinstance(target[k], dict):
                    return False
                elif not subdict_matches(target[k], v, partial_match):
                    return False
            elif not partial_match and target[k] != v:
                return False
        elif isinstance(v, (list, set, tuple)) and target[k] not in v:
            return False

    return True
<commit_msg>Switch md5 to sha1<commit_after># -*- coding: utf-8 -*-

import json
import logging
import pickle
from configparser import ConfigParser
from datetime import datetime

import hashlib
import networkx as nx
import os
import requests
import requests.exceptions
from collections import defaultdict, MutableMapping
from pkg_resources import get_distribution
from requests.compat import urlparse
from requests_file import FileAdapter
from six import string_types

from .constants import CITATION_ENTRIES, BELFRAMEWORK_DOMAIN, OPENBEL_DOMAIN

log = logging.getLogger(__name__)

def download(url):
    """Uses requests to download an URL, maybe from a file"""
    session = requests.Session()
    session.mount('file://', FileAdapter())

    if url.startswith(BELFRAMEWORK_DOMAIN):
        log.warning('%s has expired', BELFRAMEWORK_DOMAIN)
        url = url.replace(BELFRAMEWORK_DOMAIN, OPENBEL_DOMAIN)

    try:
        res = session.get(url)
    except requests.exceptions.ConnectionError as e:
        raise e

    res.raise_for_status()
    return res

def parse_bel_resource(lines):
    """Parses a BEL config (BELNS, BELANNO, or BELEQ) file from the given line iterator over the file

    :param iter[str] lines: An iterable over the lines in a BEL config file
    :return: A config-style dictionary representing the BEL config file
    :rtype: dict
    """
    lines = list(lines)

    value_line = 1 + max(i for i, line in enumerate(lines) if '[Values]' == line.strip())

    metadata_config = ConfigParser(strict=False)
    metadata_config.optionxform = lambda option: option
    metadata_config.read_file(lines[:value_line])

    delimiter = metadata_config['Processing']['DelimiterString']

    value_dict = {}
    for line in lines[value_line:]:
        sline = line.rsplit(delimiter, 1)
        key = sline[0].strip()

        value_dict[key] = sline[1].strip() if len(sline) == 2 else None

    res = {}
    res.update({k: dict(v) for k, v in metadata_config.items()})
    res['Values'] = value_dict

    return res

def is_url(s):
    """Checks if a string is a valid URL

    :param str s: An input string
    :return: Is the string a valid URL?
    :rtype: bool
    """
    return urlparse(s).scheme != ""

def get_bel_resource(location):
    """Loads/downloads and parses a config file from the given url or file path

    :param str location: The URL or file path to a BELNS, BELANNO, or BELEQ file to download and parse
    :return: A config-style dictionary representing the BEL config file
    :rtype: dict
    """

    if is_url(location):
        res = download(location)
        lines = list(line.decode('utf-8', errors='ignore').strip() for line in res.iter_lines())
    else:
        with open(os.path.expanduser(location)) as f:
            lines = list(f)

    try:
        result = parse_bel_resource(lines)
    except ValueError:
        log.error('No [Values] section found in %s', location)
        raise ValueError('No [Values] section found in {}'.format(location))

    if not result['Values']:
        raise ValueError('Downloaded empty file: {}'.format(location))

    return result

def expand_dict(flat_dict, sep='_'):
    """Expands a flattened dictionary

    :param dict flat_dict: a nested dictionary that has been flattened so the keys are composite
    :param str sep: the separator between concatenated keys
    """
    res = {}
    rdict = defaultdict(list)

    for flat_key, value in flat_dict.items():
        key = flat_key.split(sep, 1)
        if 1 == len(key):
            res[key[0]] = value
        else:
            rdict[key[0]].append((key[1:], value))

    for k, v in rdict.items():
        res[k] = expand_dict({ik: iv for (ik,), iv in v})

    return res

def flatten_dict(d, parent_key='', sep='_'):
    """Flattens a nested dictionary.

    :param d: A nested dictionary
    :type d: dict or MutableMapping
    :param str parent_key: The parent's key. This is a value for tail recursion, so don't set it yourself.
    :param str sep: The separator used between dictionary levels

    .. seealso:: http://stackoverflow.com/a/6027615
    """
    items = []
    for k, v in d.items():
        new_key = parent_key + sep + k if parent_key else k
        if isinstance(v, (dict, MutableMapping)):
            items.extend(flatten_dict(v, new_key, sep=sep).items())
        elif isinstance(v, (set, list)):
            items.append((new_key, ','.join(v)))
        else:
            items.append((new_key, v))
    return dict(items)

def flatten_graph_data(graph):
    """Returns a new graph with flattened edge data dictionaries.

    :param nx.MultiDiGraph graph: A graph with nested edge data dictionaries
    :return: A graph with flattened edge data dictionaries
    :rtype: nx.MultiDiGraph
    """
    g = nx.MultiDiGraph(**graph.graph)

    for node, data in graph.nodes(data=True):
        g.add_node(node, data)

    for u, v, key, data in graph.edges(data=True, keys=True):
        g.add_edge(u, v, key=key, attr_dict=flatten_dict(data))

    return g

def list2tuple(l):
    """Recursively converts a nested list to a nested tuple"""
    if isinstance(l, list):
        return tuple(list2tuple(e) for e in l)
    else:
        return l

def get_version():
    """Gets the current PyBEL version

    :return: The current PyBEL version
    :rtype: str
    """
    return get_distribution('pybel').version

def tokenize_version(version_string):
    """Tokenizes a version string to a tuple. Truncates qualifiers like ``-dev``.

    :param str version_string: A version string
    :return: A tuple representing the version string
    :rtype: tuple

    >>> tokenize_version('0.1.2-dev')
    (0, 1, 2)

    """
    return tuple(map(int, version_string.split('.')[0:3]))

def citation_dict_to_tuple(citation):
    """Convert the ``d[CITATION]`` entry in an edge data dictionary to a tuple"""
    if all(x in citation for x in CITATION_ENTRIES):
        return tuple(citation[x] for x in CITATION_ENTRIES)

    if all(x in citation for x in CITATION_ENTRIES[3:5]):
        return tuple(citation[x] for x in CITATION_ENTRIES[:5])

    if all(x in citation for x in CITATION_ENTRIES[3:4]):
        return tuple(citation[x] for x in CITATION_ENTRIES[:4])

    return tuple(citation[x] for x in CITATION_ENTRIES[:3])

def flatten_citation(citation):
    """Flattens a citation dict, from the ``d[CITATION]`` entry in an edge data dictionary"""
    return ','.join('"{}"'.format(e) for e in citation_dict_to_tuple(citation))

def ensure_quotes(s):
    """Quote a string that isn't solely alphanumeric

    :param str s: a string
    :rtype: str
    """
    return '"{}"'.format(s) if not s.isalnum() else s

CREATION_DATE_FMT = '%Y-%m-%dT%H:%M:%S'
PUBLISHED_DATE_FMT = '%Y-%m-%d'
PUBLISHED_DATE_FMT_2 = '%d:%m:%Y %H:%M'

def valid_date(s):
    """Checks that a string represents a valid date in ISO 8601 format YYYY-MM-DD

    :type s: str
    :rtype: bool
    """
    try:
        datetime.strptime(s, '%Y-%m-%d')
        return True
    except ValueError:
        return False

def valid_date_version(s):
    """Checks that the string is a valid date versions string"""
    try:
        datetime.strptime(s, '%Y%m%d')
        return True
    except ValueError:
        return False

def parse_datetime(s):
    """Tries to parse a datetime object from a standard datetime format or date format

    :param str s: A string representing a date or datetime
    :return: A parsed date object
    :rtype: datetime.date
    """
    try:
        dt = datetime.strptime(s, CREATION_DATE_FMT)
        return dt
    except:
        try:
            dt = datetime.strptime(s, PUBLISHED_DATE_FMT)
            return dt
        except:
            try:
                dt = datetime.strptime(s, PUBLISHED_DATE_FMT_2)
                return dt
            except:
                raise ValueError('Incorrect datetime format for {}'.format(s))

def hash_node(node):
    """Converts a PyBEL node tuple to a hash

    :param tuple node: A BEL node
    :return: A hashed version of the node tuple using md5 hash of the binary pickle dump
    :rtype: str
    """
    return hashlib.sha1(pickle.dumps(node)).hexdigest()

def hash_edge(u, v, k, d):
    """Converts an edge tuple to a hash

    :return: A hashed version of the edge tuple using md5 hash of the binary pickle dump of u, v, and the json dump of d
    :rtype: str
    """
    edge_tuple = (u, v, json.dumps(d, ensure_ascii=False, sort_keys=True))
    return hashlib.sha1(pickle.dumps(edge_tuple)).hexdigest()

def subdict_matches(target, query, partial_match=True):
    """Checks if all the keys in the query dict are in the target dict, and that their values match

    1. Checks that all keys in the query dict are in the target dict
    2. Matches the values of the keys in the query dict
        a. If the value is a string, then must match exactly
        b. If the value is a set/list/tuple, then will match any of them
        c. If the value is a dict, then recursively check if that subdict matches

    :param dict target: The dictionary to search
    :param dict query: A query dict with keys to match
    :param bool partial_match: Should the query values be used as partial or exact matches? Defaults to :code:`True`.
    :return: if all keys in b are in target_dict and their values match
    :rtype: bool
    """
    for k, v in query.items():
        if k not in target:
            return False
        elif not isinstance(v, (int, string_types, list, set, dict, tuple)):
            raise ValueError('invalid value: {}'.format(v))
        elif isinstance(v, (int, string_types)) and target[k] != v:
            return False
        elif isinstance(v, dict):
            if partial_match:
                if not isinstance(target[k], dict):
                    return False
                elif not subdict_matches(target[k], v, partial_match):
                    return False
            elif not partial_match and target[k] != v:
                return False
        elif isinstance(v, (list, set, tuple)) and target[k] not in v:
            return False

    return True
<|endoftext|>