<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from math import log
import pickle

class DecisionTree():

  def __init__(self, name=None, dataSet=None, labels=None, infilename=None, outfilename=None):
    if infilename:
      self.isModelExists_ = True
      self.infilename_ = infilename
      tree, labels = self.grabTree(infilename)
      self.outfilename_ = outfilename
      self.tree_ = tree
      self.labels_ = labels
      self.dataSet_ = None
      self.name_ = name if name else "test"
    else:
      self.isModelExists_ = False
      self.infilename_ = infilename
      self.outfilename_ = outfilename
      self.tree_ = None
      self.labels_ = labels
      self.dataSet_ = dataSet
      self.name_ = name if name else "test"

  def __del__(self):
    if not self.outfilename_:
      if not self.isModelExists_ and self.name_:
        filename = "../models/%s.model" % self.name_
        self.storeTree(filename)
    else:
      self.storeTree(self.outfilename_)

  def grabTree(self, filename):
    fr = open(filename)
    data = pickle.load(fr)
    fr.close()
    return (data["tree"], data["labels"])

  def storeTree(self, filename):
    fw = open(filename, 'w')
    data = {}
    data["tree"] = self.tree_
    data["labels"] = self.labels_
    pickle.dump(data, fw)
    fw.close()

  def calcShannonEnt(self, dataSet):
    numEntries = len(dataSet)
    labelCounts = {}
    for featVec in dataSet: #the the number of unique elements and their occurance
      currentLabel = featVec[-1]
      if currentLabel not in labelCounts.keys():
        labelCounts[currentLabel] = 0
      labelCounts[currentLabel] += 1
    shannonEnt = 0.0
    for key in labelCounts:
      prob = float(labelCounts[key])/numEntries
      shannonEnt -= prob * log(prob,2) #log base 2
    return shannonEnt

  def splitDataSet(self, dataSet, axis, value):
    retDataSet = []
    for featVec in dataSet:
      if featVec[axis] == value:
        reducedFeatVec = featVec[:axis]
        reducedFeatVec.extend(featVec[axis+1:])
        retDataSet.append(reducedFeatVec)
    return retDataSet

  def chooseBestFeatureToSplit(self, dataSet):
    numFeatures = len(dataSet[0]) - 1
    baseEntropy = self.calcShannonEnt(dataSet)
    bestInfoGain = 0.0; bestFeature = -1
    for i in range(numFeatures):
      featList = [example[i] for example in dataSet]
      uniqueVals = set(featList)
      newEntropy = 0.0
      for value in uniqueVals:
        subDataSet = self.splitDataSet(dataSet, i, value)
        prob = len(subDataSet)/float(len(dataSet))
        newEntropy += prob * self.calcShannonEnt(subDataSet)
      infoGain = baseEntropy - newEntropy
      if (infoGain > bestInfoGain):
        bestInfoGain = infoGain
        bestFeature = i
    return bestFeature

  def majorityCnt(self, classList):
    classCount = {}
    for vote in classList:
      if vote not in classCount.keys(): classCount[vote] = 0
      classCount[vote] += 1
    sortedClassCount = sorted(classCount.iteritems(), \
        key=operator.itemgetter(1), reverse=True)
    return sortedClassCount[0][0]

  def createTree(self):
    import copy
    if self.dataSet_ and self.labels_:
      tmpLabels = copy.deepcopy(self.labels_)
      self.tree_ = self._createTree(self.dataSet_, tmpLabels)

  def _createTree(self, dataSet, labels):
    classList = [example[-1] for example in dataSet]
    if classList.count(classList[0]) == len(classList):
      return classList[0]
    if len(dataSet[0]) == 1:
      return self.majorityCnt(classList)
    bestFeat = self.chooseBestFeatureToSplit(dataSet)
    bestFeatLabel = labels[bestFeat]
    myTree = {bestFeatLabel:{}}
    del(labels[bestFeat])
    featValues = [example[bestFeat] for example in dataSet]
    uniqueVals = set(featValues)
    for value in uniqueVals:
      subLabels = labels[:]
      myTree[bestFeatLabel][value] = self._createTree(self.splitDataSet(dataSet, bestFeat, value), subLabels)
    return myTree

  def classify(self, testVec):
    if self.tree_ and self.labels_:
      return self._classify(self.tree_, self.labels_, testVec)
    else:
      print "please create tree first"
      return None

  def _classify(self, inputTree, featLabels, testVec):
    firstStr = inputTree.keys()[0]
    secondDict = inputTree[firstStr]
    featIndex = featLabels.index(firstStr)
    key = testVec[featIndex]
    valueOfFeat = secondDict[key]
    if isinstance(valueOfFeat, dict):
      classLabel = self._classify(valueOfFeat, featLabels, testVec)
    else: classLabel = valueOfFeat
    return classLabel

if __name__ == "__main__":
  dataSet = [
      [1, 1, 'yes'],
      [1, 1, 'yes'],
      [1, 0, 'no'],
      [0, 1, 'no'],
      [0, 1, 'no']
      ]
  labels = ['no surfacing','flippers']
  tree = DecisionTree("test", dataSet, labels)
  tree.createTree()
  print tree.classify([0,0])
  #print tree.calcShannonEnt(dataSet)
  #print tree.splitDataSet(dataSet, 0, 1)
  #print tree.splitDataSet(dataSet, 0, 0)
  #print tree.chooseBestFeatureToSplit(dataSet)
  #print tree.createTree(dataSet, labels)

  tree1 = DecisionTree(infilename="../models/test.model")
  print tree.classify([1,1])

<commit_msg>add files<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from math import log
import simplejson as json

class DecisionTree():

  def __init__(self, name=None, dataSet=None, labels=None, infilename=None, outfilename=None):
    if infilename:
      self.isModelExists_ = True
      self.infilename_ = infilename
      tree, labels = self.grabTree(infilename)
      self.outfilename_ = outfilename
      self.tree_ = tree
      self.labels_ = labels
      self.dataSet_ = None
      self.name_ = name if name else "test"
    else:
      self.isModelExists_ = False
      self.infilename_ = infilename
      self.outfilename_ = outfilename
      self.tree_ = None
      self.labels_ = labels
      self.dataSet_ = dataSet
      self.name_ = name if name else "test"

  def __del__(self):
    if not self.outfilename_:
      if not self.isModelExists_ and self.name_:
        filename = "../models/%s.model" % self.name_
        self.storeTree(filename)
    else:
      self.storeTree(self.outfilename_)

  def grabTree(self, filename):
    fr = open(filename)
    data = json.loads(fr.read())
    fr.close()
    return (data["tree"], data["labels"])

  def storeTree(self, filename):
    fw = open(filename, 'w')
    data = {}
    data["tree"] = self.tree_
    data["labels"] = self.labels_
    out = json.dumps(data, ensure_ascii=False)
    fw.write(out)
    fw.close()

  def calcShannonEnt(self, dataSet):
    numEntries = len(dataSet)
    labelCounts = {}
    for featVec in dataSet: #the the number of unique elements and their occurance
      currentLabel = featVec[-1]
      if currentLabel not in labelCounts.keys():
        labelCounts[currentLabel] = 0
      labelCounts[currentLabel] += 1
    shannonEnt = 0.0
    for key in labelCounts:
      prob = float(labelCounts[key])/numEntries
      shannonEnt -= prob * log(prob,2) #log base 2
    return shannonEnt

  def splitDataSet(self, dataSet, axis, value):
    retDataSet = []
    for featVec in dataSet:
      if featVec[axis] == value:
        reducedFeatVec = featVec[:axis]
        reducedFeatVec.extend(featVec[axis+1:])
        retDataSet.append(reducedFeatVec)
    return retDataSet

  def chooseBestFeatureToSplit(self, dataSet):
    numFeatures = len(dataSet[0]) - 1
    baseEntropy = self.calcShannonEnt(dataSet)
    bestInfoGain = 0.0; bestFeature = -1
    for i in range(numFeatures):
      featList = [example[i] for example in dataSet]
      uniqueVals = set(featList)
      newEntropy = 0.0
      for value in uniqueVals:
        subDataSet = self.splitDataSet(dataSet, i, value)
        prob = len(subDataSet)/float(len(dataSet))
        newEntropy += prob * self.calcShannonEnt(subDataSet)
      infoGain = baseEntropy - newEntropy
      if (infoGain > bestInfoGain):
        bestInfoGain = infoGain
        bestFeature = i
    return bestFeature

  def majorityCnt(self, classList):
    classCount = {}
    for vote in classList:
      if vote not in classCount.keys(): classCount[vote] = 0
      classCount[vote] += 1
    sortedClassCount = sorted(classCount.iteritems(), \
        key=operator.itemgetter(1), reverse=True)
    return sortedClassCount[0][0]

  def createTree(self):
    import copy
    if self.dataSet_ and self.labels_:
      tmpLabels = copy.deepcopy(self.labels_)
      self.tree_ = self._createTree(self.dataSet_, tmpLabels)

  def _createTree(self, dataSet, labels):
    classList = [example[-1] for example in dataSet]
    if classList.count(classList[0]) == len(classList):
      return classList[0]
    if len(dataSet[0]) == 1:
      return self.majorityCnt(classList)
    bestFeat = self.chooseBestFeatureToSplit(dataSet)
    bestFeatLabel = labels[bestFeat]
    myTree = {bestFeatLabel:{}}
    del(labels[bestFeat])
    featValues = [example[bestFeat] for example in dataSet]
    uniqueVals = set(featValues)
    for value in uniqueVals:
      subLabels = labels[:]
      myTree[bestFeatLabel][str(value)] = self._createTree(self.splitDataSet(dataSet, bestFeat, value), subLabels)
    return myTree

  def classify(self, testVec):
    if self.tree_ and self.labels_:
      return self._classify(self.tree_, self.labels_, testVec)
    else:
      print "please create tree first"
      return None

  def _classify(self, inputTree, featLabels, testVec):
    firstStr = inputTree.keys()[0]
    secondDict = inputTree[firstStr]
    featIndex = featLabels.index(firstStr)
    key = testVec[featIndex]
    valueOfFeat = secondDict[str(key)]
    if isinstance(valueOfFeat, dict):
      classLabel = self._classify(valueOfFeat, featLabels, testVec)
    else: classLabel = valueOfFeat
    return classLabel

if __name__ == "__main__":
  dataSet = [
      [1, 1, 'yes'],
      [1, 1, 'yes'],
      [1, 0, 'no'],
      [0, 1, 'no'],
      [0, 1, 'no']
      ]
  labels = ['no surfacing','flippers']
  tree = DecisionTree("test", dataSet, labels)
  tree.createTree()
  print tree.classify([0,0])
  #print tree.calcShannonEnt(dataSet)
  #print tree.splitDataSet(dataSet, 0, 1)
  #print tree.splitDataSet(dataSet, 0, 0)
  #print tree.chooseBestFeatureToSplit(dataSet)
  #print tree.createTree(dataSet, labels)
  del tree
  tree1 = DecisionTree(infilename="../models/test.model")
  print tree1.classify([1,1])
  print tree1.classify([0,1])

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 3 of the License,
    or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
    See the GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program; if not, see <http://www.gnu.org/licenses/>.

    @author: mkaay
"""

from PyQt4.QtCore import *
from PyQt4.QtGui import *

from time import sleep, time

class OverviewModel(QAbstractListModel):
    PackageName = 10
    Progress = 11
    PartsFinished = 12
    Parts = 13
    ETA = 14
    Speed = 15

    def __init__(self, view, connector):
        QAbstractListModel.__init__(self)

        self.packages = []

    def queueChanged(self): #dirty..
        self.beginResetModel()

        self.packages = []

        def partsFinished(p):
            f = 0
            for c in p.children:
                if c.data["status"] == 0:
                    f += 1
            return f

        d = self.queue._data
        for p in d:
            info = {
                OverviewModel.PackageName: p.data["name"],
                OverviewModel.Progress: self.queue.getProgress(p),
                OverviewModel.PartsFinished: partsFinished(p),
                OverviewModel.Parts: len(p.children),
                OverviewModel.ETA: "n/a",
                OverviewModel.Speed: self.queue.getSpeed(p),
            }

            self.packages.append(info)

        self.endResetModel()

    def headerData(self, section, orientation, role=Qt.DisplayRole):
        return QVariant(_("Package"))

    def rowCount(self, parent=QModelIndex()):
        return len(self.packages)

    def data(self, index, role=Qt.DisplayRole):
        if role in [OverviewModel.PackageName, OverviewModel.Progress, OverviewModel.PartsFinished, OverviewModel.Parts, OverviewModel.ETA, OverviewModel.Speed]:
            return QVariant(self.packages[index.row()][role])
        return QVariant()

class OverviewView(QListView):
    def __init__(self, connector):
        QListView.__init__(self)
        self.setModel(OverviewModel(self, connector))

        self.setAlternatingRowColors(True)
        self.delegate = OverviewDelegate(self)
        self.setItemDelegate(self.delegate)

class OverviewDelegate(QItemDelegate):
    def __init__(self, parent):
        QItemDelegate.__init__(self, parent)
        self.parent = parent
        self.model = parent.model()

    def paint(self, painter, option, index):
        option.rect.setHeight(59+16)
        option.rect.setWidth(self.parent.width())

        #if option.state & QStyle.State_Selected:
        #    painter.fillRect(option.rect, option.palette.color(QPalette.Highlight))

        packagename = index.data(OverviewModel.PackageName).toString()
        partsf = index.data(OverviewModel.PartsFinished).toString()
        parts = index.data(OverviewModel.Parts).toString()
        eta = index.data(OverviewModel.ETA).toString()
        speed = index.data(OverviewModel.Speed).toString() or 0
        progress = int(index.data(OverviewModel.Progress).toString())

        statusline = QString(_("Parts: %s/%s          ETA: %s          Speed: %s kb/s" % (partsf, parts, eta, speed)))

        f = painter.font()
        f.setPointSize(12)
        f.setBold(True)
        painter.setFont(f)

        r = option.rect.adjusted(4, 4, -4, -4)
        painter.drawText(r.left(), r.top(), r.width(), r.height(), Qt.AlignTop | Qt.AlignLeft, packagename)
        newr = painter.boundingRect(r.left(), r.top(), r.width(), r.height(), Qt.AlignTop | Qt.AlignLeft, packagename)

        f.setPointSize(11)
        f.setBold(False)
        painter.setFont(f)

        painter.drawText(r.left(), newr.bottom()+4, r.width(), r.height(), Qt.AlignTop | Qt.AlignLeft, statusline)
        newr = painter.boundingRect(r.left(), newr.bottom()+2, r.width(), r.height(), Qt.AlignTop | Qt.AlignLeft, statusline)
        newr.setTop(newr.bottom()+8)
        newr.setBottom(newr.top()+20)
        newr.setRight(self.parent.width()-5)

        f.setPointSize(10)
        painter.setFont(f)

        opts = QStyleOptionProgressBarV2()
        opts.maximum = 100
        opts.minimum = 0
        opts.progress = progress
        opts.rect = newr
        opts.textVisible = True
        opts.textAlignment = Qt.AlignCenter
        opts.text = QString.number(opts.progress) + "%"
        QApplication.style().drawControl(QStyle.CE_ProgressBar, opts, painter)

    def sizeHint(self, option, index):
        return QSize(self.parent.width(), 59+16)
<commit_msg>better gui overview<commit_after># -*- coding: utf-8 -*-
"""
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 3 of the License,
    or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
    See the GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program; if not, see <http://www.gnu.org/licenses/>.

    @author: mkaay
"""

from PyQt4.QtCore import *
from PyQt4.QtGui import *

from time import sleep, time

class OverviewModel(QAbstractListModel):
    PackageName = 10
    Progress = 11
    PartsFinished = 12
    Parts = 13
    ETA = 14
    Speed = 15
    CurrentSize = 16
    MaxSize = 17

    def __init__(self, view, connector):
        QAbstractListModel.__init__(self)

        self.packages = []

    def queueChanged(self): #dirty..
        self.beginResetModel()

        self.packages = []

        def partsFinished(p):
            f = 0
            for c in p.children:
                if c.data["status"] == 0:
                    f += 1
            return f

        def maxSize(p):
            s = 0
            for c in p.children:
                s += c.data["size"]
            return s

        d = self.queue._data
        for p in d:
            progress = self.queue.getProgress(p)
            maxsize = maxSize(p)
            speed = self.queue.getSpeed(p)
            if speed:
                eta = (maxsize - (maxsize * (progress/100.0)))/1024/speed
            else:
                eta = 0
            info = {
                OverviewModel.PackageName: p.data["name"],
                OverviewModel.Progress: progress,
                OverviewModel.PartsFinished: partsFinished(p),
                OverviewModel.Parts: len(p.children),
                OverviewModel.ETA: int(eta),
                OverviewModel.Speed: speed,
                OverviewModel.CurrentSize: maxsize * (progress/100.0),
                OverviewModel.MaxSize: maxsize,
            }

            self.packages.append(info)

        self.endResetModel()

    def headerData(self, section, orientation, role=Qt.DisplayRole):
        return QVariant(_("Package"))

    def rowCount(self, parent=QModelIndex()):
        return len(self.packages)

    def data(self, index, role=Qt.DisplayRole):
        if role in [OverviewModel.PackageName, OverviewModel.Progress, OverviewModel.PartsFinished, OverviewModel.Parts, OverviewModel.ETA, OverviewModel.Speed, OverviewModel.CurrentSize, OverviewModel.MaxSize]:
            return QVariant(self.packages[index.row()][role])
        return QVariant()

class OverviewView(QListView):
    def __init__(self, connector):
        QListView.__init__(self)
        self.setModel(OverviewModel(self, connector))

        self.setAlternatingRowColors(True)
        self.delegate = OverviewDelegate(self)
        self.setItemDelegate(self.delegate)

class OverviewDelegate(QItemDelegate):
    def __init__(self, parent):
        QItemDelegate.__init__(self, parent)
        self.parent = parent
        self.model = parent.model()

    def paint(self, painter, option, index):
        option.rect.setHeight(59+16)
        option.rect.setWidth(self.parent.width())

        #if option.state & QStyle.State_Selected:
        #    painter.fillRect(option.rect, option.palette.color(QPalette.Highlight))

        packagename = index.data(OverviewModel.PackageName).toString()
        partsf = index.data(OverviewModel.PartsFinished).toString()
        parts = index.data(OverviewModel.Parts).toString()
        eta = int(index.data(OverviewModel.ETA).toString())
        speed = index.data(OverviewModel.Speed).toString() or 0
        progress = int(index.data(OverviewModel.Progress).toString())
        currentSize = int(index.data(OverviewModel.CurrentSize).toString())
        maxSize = int(index.data(OverviewModel.MaxSize).toString())

        def formatEta(seconds):
            if seconds <= 0: return ""
            hours, seconds = divmod(seconds, 3600)
            minutes, seconds = divmod(seconds, 60)
            return _("ETA: %.2i:%.2i:%.2i") % (hours, minutes, seconds)

        statusline = QString(_("Parts: %s/%s" % (partsf, parts)))
        if partsf == parts:
            speedline = _("Finished")
        else:
            speedline = QString(formatEta(eta) + "     " + _("Speed: %s kb/s") % speed)

        def formatSize(size):
            kbytes = size/1024
            if kbytes/1024/1024 < 1024:
                if kbytes/1024 < 1024:
                    if kbytes <= 1:
                        size = "1 KiB"
                    else:
                        size = "%s KiB" % kbytes
                else:
                    size = "%s MiB" % round(kbytes/1024, 2)
            else:
                size = "%s GiB" % round(kbytes/1024/1024, 2)
            return size

        if progress == 100:
            sizeline = QString(_("Size: %s") % formatSize(maxSize))
        else:
            sizeline = QString(_("Size: %s/%s") % (formatSize(currentSize), formatSize(maxSize)))

        f = painter.font()
        f.setPointSize(12)
        f.setBold(True)
        painter.setFont(f)

        r = option.rect.adjusted(4, 4, -4, -4)
        painter.drawText(r.left(), r.top(), r.width(), r.height(), Qt.AlignTop | Qt.AlignLeft, packagename)
        newr = painter.boundingRect(r.left(), r.top(), r.width(), r.height(), Qt.AlignTop | Qt.AlignLeft, packagename)

        f.setPointSize(10)
        f.setBold(False)
        painter.setFont(f)

        painter.drawText(r.left(), newr.bottom()+5, r.width(), r.height(), Qt.AlignTop | Qt.AlignLeft, statusline)
        painter.drawText(r.left(), newr.bottom()+5, r.width(), r.height(), Qt.AlignTop | Qt.AlignHCenter, sizeline)
        painter.drawText(r.left(), newr.bottom()+5, r.width(), r.height(), Qt.AlignTop | Qt.AlignRight, speedline)
        newr = painter.boundingRect(r.left(), newr.bottom()+2, r.width(), r.height(), Qt.AlignTop | Qt.AlignLeft, statusline)
        newr.setTop(newr.bottom()+8)
        newr.setBottom(newr.top()+20)
        newr.setRight(self.parent.width()-5)

        f.setPointSize(10)
        painter.setFont(f)

        opts = QStyleOptionProgressBarV2()
        opts.maximum = 100
        opts.minimum = 0
        opts.progress = progress
        opts.rect = newr
        opts.textVisible = True
        opts.textAlignment = Qt.AlignCenter
        opts.text = QString.number(opts.progress) + "%"
        QApplication.style().drawControl(QStyle.CE_ProgressBar, opts, painter)

    def sizeHint(self, option, index):
        return QSize(self.parent.width(), 59+16)
<|endoftext|>


<commit_before>#!/usr/bin/env vpython
#
# Copyright (C) 2011 Google Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#         * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#         * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#         * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""Creates a grd file for packaging the inspector files."""

from __future__ import with_statement
from os import path

import errno
import os
import shlex
import shutil
import sys
from xml.dom import minidom

kDevToolsResourcePrefix = 'IDR_DEVTOOLS_'
kGrdTemplate = '''<?xml version="1.0" encoding="UTF-8"?>
<grit latest_public_release="0" current_release="1"
      output_all_resource_defines="false">
  <outputs>
    <output filename="grit/devtools_resources.h" type="rc_header">
      <emit emit_type='prepend'></emit>
    </output>
    <output filename="grit/devtools_resources_map.cc" type="resource_file_map_source" />
    <output filename="grit/devtools_resources_map.h" type="resource_map_header" />

    <output filename="devtools_resources.pak" type="data_package" />
  </outputs>
  <release seq="1">
    <includes>
      <include name="COMPRESSED_PROTOCOL_JSON" file="${protocol_file}" use_base_dir="false" compress="brotli" type="BINDATA" skip_in_resource_map = "true"/>
    </includes>
  </release>
</grit>
'''

class ParsedArgs:

    def __init__(self, file_list, relative_path_dirs, image_dirs, output_filename):
        self.file_list = file_list
        file_list_file = open(file_list, 'r')
        file_list_contents = file_list_file.read();
        self.source_files = shlex.split(file_list_contents)
        self.relative_path_dirs = relative_path_dirs
        self.image_dirs = image_dirs
        self.output_filename = output_filename

def parse_args(argv):
    # The arguments are of the format:
    #   --file_list <input_file_list>
    #   --relative_path_dirs [ <directory> ]*
    #   --images [ <image_dirs> ]*
    #   --output <output_file>
    file_list_position = argv.index('--file_list')
    relative_path_dirs_position = argv.index('--relative_path_dirs')
    images_position = argv.index('--images')
    output_position = argv.index('--output')
    file_list = argv[file_list_position + 1]
    relative_path_dirs = argv[relative_path_dirs_position + 1:images_position]
    image_dirs = argv[images_position + 1:output_position]
    return ParsedArgs(file_list, relative_path_dirs, image_dirs, argv[output_position + 1])

def make_name_from_filename(filename):
    return (filename.replace('/', '_').replace('\\', '_').replace('-', '_').replace('.', '_')).upper()

def add_file_to_grd(grd_doc, relative_filename):
    includes_node = grd_doc.getElementsByTagName('includes')[0]
    includes_node.appendChild(grd_doc.createTextNode('\n      '))

    new_include_node = grd_doc.createElement('include')
    new_include_node.setAttribute('name', make_name_from_filename(relative_filename))
    new_include_node.setAttribute('file', relative_filename)
    new_include_node.setAttribute('type', 'BINDATA')
    ext = os.path.splitext(relative_filename)[1]
    if ext in ['.css', '.html', '.js', '.svg']:
        new_include_node.setAttribute('compress', 'brotli')
    includes_node.appendChild(new_include_node)

def build_relative_filename(relative_path_dirs, filename):
    for relative_path_dir in relative_path_dirs:
        index = filename.find(relative_path_dir)
        if index == 0:
            return filename[len(relative_path_dir) + 1:]
    return path.basename(filename)

def main(argv):
    parsed_args = parse_args(argv[1:])

    doc = minidom.parseString(kGrdTemplate)
    output_directory = path.dirname(parsed_args.output_filename)

    try:
        os.makedirs(path.join(output_directory, 'Images'))
    except OSError as e:
        if e.errno != errno.EEXIST:
            raise e

    written_filenames = set()
    for filename in parsed_args.source_files:
        relative_filename = build_relative_filename(parsed_args.relative_path_dirs, filename)
        # Avoid writing duplicate relative filenames.
        if relative_filename in written_filenames:
            continue
        written_filenames.add(relative_filename)
        target_dir = path.join(output_directory, path.dirname(relative_filename))
        if not path.exists(target_dir):
            os.makedirs(target_dir)
        shutil.copy(filename, target_dir)
        add_file_to_grd(doc, relative_filename)

    for dirname in parsed_args.image_dirs:
        for filename in sorted(os.listdir(dirname)):
            if not filename.endswith('.png') and not filename.endswith('.gif') and not filename.endswith('.svg'):
                continue
            shutil.copy(path.join(dirname, filename), path.join(output_directory, 'Images'))
            add_file_to_grd(doc, path.join('Images', filename))

    with open(parsed_args.output_filename, 'wb') as output_file:
        output_file.write(doc.toxml(encoding='UTF-8'))

if __name__ == '__main__':
    sys.exit(main(sys.argv))
<commit_msg>Include .avif and .webp images in *.grd files<commit_after>#!/usr/bin/env vpython
#
# Copyright (C) 2011 Google Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#         * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#         * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#         * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""Creates a grd file for packaging the inspector files."""

from __future__ import with_statement
from os import path

import errno
import os
import shlex
import shutil
import sys
from xml.dom import minidom

kDevToolsResourcePrefix = 'IDR_DEVTOOLS_'
kGrdTemplate = '''<?xml version="1.0" encoding="UTF-8"?>
<grit latest_public_release="0" current_release="1"
      output_all_resource_defines="false">
  <outputs>
    <output filename="grit/devtools_resources.h" type="rc_header">
      <emit emit_type='prepend'></emit>
    </output>
    <output filename="grit/devtools_resources_map.cc" type="resource_file_map_source" />
    <output filename="grit/devtools_resources_map.h" type="resource_map_header" />

    <output filename="devtools_resources.pak" type="data_package" />
  </outputs>
  <release seq="1">
    <includes>
      <include name="COMPRESSED_PROTOCOL_JSON" file="${protocol_file}" use_base_dir="false" compress="brotli" type="BINDATA" skip_in_resource_map = "true"/>
    </includes>
  </release>
</grit>
'''

class ParsedArgs:

    def __init__(self, file_list, relative_path_dirs, image_dirs, output_filename):
        self.file_list = file_list
        file_list_file = open(file_list, 'r')
        file_list_contents = file_list_file.read();
        self.source_files = shlex.split(file_list_contents)
        self.relative_path_dirs = relative_path_dirs
        self.image_dirs = image_dirs
        self.output_filename = output_filename

def parse_args(argv):
    # The arguments are of the format:
    #   --file_list <input_file_list>
    #   --relative_path_dirs [ <directory> ]*
    #   --images [ <image_dirs> ]*
    #   --output <output_file>
    file_list_position = argv.index('--file_list')
    relative_path_dirs_position = argv.index('--relative_path_dirs')
    images_position = argv.index('--images')
    output_position = argv.index('--output')
    file_list = argv[file_list_position + 1]
    relative_path_dirs = argv[relative_path_dirs_position + 1:images_position]
    image_dirs = argv[images_position + 1:output_position]
    return ParsedArgs(file_list, relative_path_dirs, image_dirs, argv[output_position + 1])

def make_name_from_filename(filename):
    return (filename.replace('/', '_').replace('\\', '_').replace('-', '_').replace('.', '_')).upper()

def add_file_to_grd(grd_doc, relative_filename):
    includes_node = grd_doc.getElementsByTagName('includes')[0]
    includes_node.appendChild(grd_doc.createTextNode('\n      '))

    new_include_node = grd_doc.createElement('include')
    new_include_node.setAttribute('name', make_name_from_filename(relative_filename))
    new_include_node.setAttribute('file', relative_filename)
    new_include_node.setAttribute('type', 'BINDATA')
    ext = os.path.splitext(relative_filename)[1]
    if ext in ['.css', '.html', '.js', '.svg']:
        new_include_node.setAttribute('compress', 'brotli')
    includes_node.appendChild(new_include_node)

def build_relative_filename(relative_path_dirs, filename):
    for relative_path_dir in relative_path_dirs:
        index = filename.find(relative_path_dir)
        if index == 0:
            return filename[len(relative_path_dir) + 1:]
    return path.basename(filename)

def main(argv):
    parsed_args = parse_args(argv[1:])

    doc = minidom.parseString(kGrdTemplate)
    output_directory = path.dirname(parsed_args.output_filename)

    try:
        os.makedirs(path.join(output_directory, 'Images'))
    except OSError as e:
        if e.errno != errno.EEXIST:
            raise e

    written_filenames = set()
    for filename in parsed_args.source_files:
        relative_filename = build_relative_filename(parsed_args.relative_path_dirs, filename)
        # Avoid writing duplicate relative filenames.
        if relative_filename in written_filenames:
            continue
        written_filenames.add(relative_filename)
        target_dir = path.join(output_directory, path.dirname(relative_filename))
        if not path.exists(target_dir):
            os.makedirs(target_dir)
        shutil.copy(filename, target_dir)
        add_file_to_grd(doc, relative_filename)

    for dirname in parsed_args.image_dirs:
        for filename in sorted(os.listdir(dirname)):
            if not filename.endswith('.gif') and not filename.endswith(
                    '.png') and not filename.endswith(
                        '.svg') and not filename.endswith(
                            'avif') and not filename.endswith('webp'):
                continue
            shutil.copy(path.join(dirname, filename), path.join(output_directory, 'Images'))
            add_file_to_grd(doc, path.join('Images', filename))

    with open(parsed_args.output_filename, 'wb') as output_file:
        output_file.write(doc.toxml(encoding='UTF-8'))

if __name__ == '__main__':
    sys.exit(main(sys.argv))
<|endoftext|>


<commit_before>"""Tests of various renderers"""

import pytest

import altair as alt

@pytest.fixture
def chart():
    return alt.Chart('data.csv').mark_point()

def test_colab_renderer_embed_options(chart):
    """Test that embed_options in renderer metadata are correctly manifest in html"""
    with alt.renderers.enable('colab', embed_options=dict(actions=False)):
        bundle = chart._repr_mimebundle_(None, None)
        html = bundle['text/html']
        assert ('embedOpt = {"actions": false, "mode": "vega-lite"}' in html or
                'embedOpt = {"mode": "vega-lite", "actions": false}' in html)

    with alt.renderers.enable('colab', embed_options=dict(actions=True)):
        bundle = chart._repr_mimebundle_(None, None)
        html = bundle['text/html']
        assert ('embedOpt = {"actions": true, "mode": "vega-lite"}' in html or
                'embedOpt = {"mode": "vega-lite", "actions": true}' in html)

def test_default_renderer_embed_options(chart, renderer='default'):
    # check that metadata is passed appropriately
    mimetype = alt.display.VEGALITE_MIME_TYPE
    spec = chart.to_dict()
    with alt.renderers.enable(renderer, embed_options=dict(actions=False)):
        bundle, metadata = chart._repr_mimebundle_(None, None)
        assert set(bundle.keys()) == {mimetype, 'text/plain'}
        assert bundle[mimetype] == spec
        assert metadata == {mimetype: {'embed_options': {'actions': False}}}

    # Sanity check: no metadata specified
    with alt.renderers.enable(renderer):
        bundle, metadata = chart._repr_mimebundle_(None, None)
        assert bundle[mimetype] == spec
        assert metadata == {}

def test_json_renderer_embed_options(chart, renderer='json'):
    """Test that embed_options in renderer metadata are correctly manifest in html"""
    mimetype = 'application/json'
    spec = chart.to_dict()
    with alt.renderers.enable('json', option='foo'):
        bundle, metadata = chart._repr_mimebundle_(None, None)
        assert set(bundle.keys()) == {mimetype, 'text/plain'}
        assert bundle[mimetype] == spec
        assert metadata == {mimetype: {'option': 'foo'}}

    # Sanity check: no options specified
    with alt.renderers.enable(renderer):
        bundle, metadata = chart._repr_mimebundle_(None, None)
        assert bundle[mimetype] == spec
        assert metadata == {}
<commit_msg>TST: add tests of set_embed_options<commit_after>"""Tests of various renderers"""

import pytest

import altair as alt

@pytest.fixture
def chart():
    return alt.Chart('data.csv').mark_point()

def test_colab_renderer_embed_options(chart):
    """Test that embed_options in renderer metadata are correctly manifest in html"""
    def assert_actions_true(chart):
        bundle = chart._repr_mimebundle_(None, None)
        html = bundle['text/html']
        assert ('embedOpt = {"actions": true, "mode": "vega-lite"}' in html or
                'embedOpt = {"mode": "vega-lite", "actions": true}' in html)

    def assert_actions_false(chart):
        bundle = chart._repr_mimebundle_(None, None)
        html = bundle['text/html']
        assert ('embedOpt = {"actions": false, "mode": "vega-lite"}' in html or
                'embedOpt = {"mode": "vega-lite", "actions": false}' in html)

    with alt.renderers.enable('colab', embed_options=dict(actions=False)):
        assert_actions_false(chart)

    with alt.renderers.enable('colab'):
        with alt.renderers.enable(embed_options=dict(actions=True)):
            assert_actions_true(chart)

        with alt.renderers.set_embed_options(actions=False):
            assert_actions_false(chart)

        with alt.renderers.set_embed_options(actions=True):
            assert_actions_true(chart)

def test_default_renderer_embed_options(chart, renderer='default'):
    # check that metadata is passed appropriately
    mimetype = alt.display.VEGALITE_MIME_TYPE
    spec = chart.to_dict()
    with alt.renderers.enable(renderer, embed_options=dict(actions=False)):
        bundle, metadata = chart._repr_mimebundle_(None, None)
        assert set(bundle.keys()) == {mimetype, 'text/plain'}
        assert bundle[mimetype] == spec
        assert metadata == {mimetype: {'embed_options': {'actions': False}}}

    # Sanity check: no metadata specified
    with alt.renderers.enable(renderer):
        bundle, metadata = chart._repr_mimebundle_(None, None)
        assert bundle[mimetype] == spec
        assert metadata == {}

def test_json_renderer_embed_options(chart, renderer='json'):
    """Test that embed_options in renderer metadata are correctly manifest in html"""
    mimetype = 'application/json'
    spec = chart.to_dict()
    with alt.renderers.enable('json', option='foo'):
        bundle, metadata = chart._repr_mimebundle_(None, None)
        assert set(bundle.keys()) == {mimetype, 'text/plain'}
        assert bundle[mimetype] == spec
        assert metadata == {mimetype: {'option': 'foo'}}

    # Sanity check: no options specified
    with alt.renderers.enable(renderer):
        bundle, metadata = chart._repr_mimebundle_(None, None)
        assert bundle[mimetype] == spec
        assert metadata == {}
<|endoftext|>


<commit_before># mqttseed application command
from django.core.management.base import BaseCommand
from django.conf import settings

from ambulances.mqttupdate import UpdateClient

from ambulances.models import Ambulance, Hospital, HospitalEquipment

from django.utils.six import BytesIO
from rest_framework.parsers import JSONParser
from rest_framework.renderers import JSONRenderer

class Client(UpdateClient):

    # The callback for when the client receives a CONNACK
    # response from the server.
    def on_connect(self, client, userdata, flags, rc):

        # is connected?
        if not super().on_connect(client, userdata, flags, rc):
            return False

        # initialize pubset
        self.pubset = set()
        self.can_disconnect = False

        # Seed hospitals
        self.seed_hospital_data(client)
        self.seed_hospital_equipment_data(client)
        self.seed_hospital_metadata(client)

        # Seed ambulances
        self.seed_ambulance_data(client)

        # Seed calls
        # self.seed_calls(client)

        # Good to disconnect
        self.can_disconnect = True

    def publish(self, topic, message, *vargs, **kwargs):

        # increment pubcount then publish
        result = self.client.publish(topic, message, *vargs, **kwargs)
        self.pubset.add(result.mid)

        # echo if verbosity > 0
        if self.verbosity > 0:
            print("  {}: {}".format(topic, message))

    def seed_ambulance_data(self, client):

        if self.verbosity > 0:
            self.stdout.write(self.style.SUCCESS(">> Seeding ambulance data"))

        # seeding ambulances
        for obj in Ambulance.objects.all():
            self.update_ambulance(obj)

        if self.verbosity > 0:
            self.stdout.write(self.style.SUCCESS("<< Done seeding ambulance data"))

    def seed_hospital_data(self, client):

        if self.verbosity > 0:
            self.stdout.write(self.style.SUCCESS(">> Seeding hospital data"))

        # seeding hospital
        for obj in Hospital.objects.all():
            self.update_hospital(obj)

        if self.verbosity > 0:
            self.stdout.write(self.style.SUCCESS("<< Done seeding hospital data"))

    def seed_hospital_equipment_data(self, client):

        if self.verbosity > 0:
            self.stdout.write(self.style.SUCCESS(">> Seeding hospital equipment data"))

        # seeding hospital
        for obj in HospitalEquipment.objects.all():
            self.update_hospital_equipment(obj)

        if self.verbosity > 0:
            self.stdout.write(self.style.SUCCESS("<< Done seeding hospital equipment data"))

    def seed_hospital_metadata(self, client):
        if self.verbosity > 0:
            self.stdout.write(self.style.SUCCESS(">> Seeding hospital metadata"))

        # seeding hospital metadata
        for hospital in Hospital.objects.all():
            self.update_hospital_metadata(hospital)

            if self.verbosity > 0:
                self.stdout.write("   Hospital metadata for hospital {}".format(hospital.id))

        if self.verbosity > 0:
            self.stdout.write(self.style.SUCCESS("<< Done seeding hospital metadata"))

    # Message publish callback
    def on_publish(self, client, userdata, mid):

        # make sure all is published before disconnecting
        self.pubset.remove(mid)
        if len(self.pubset) == 0 and self.can_disconnect:
            self.disconnect()

class Command(BaseCommand):
    help = 'Seed the mqtt broker'

    def handle(self, *args, **options):

        import os

        broker = {
            'USERNAME': '',
            'PASSWORD': '',
            'HOST': '127.0.0.1',
            'PORT': 1883,
            'KEEPALIVE': 60,
            'CLIENT_ID': 'django',
            'CLEAN_SESSION': True
        }
        broker.update(settings.MQTT)
        broker['CLIENT_ID'] = 'mqttseed_' + str(os.getpid())

        client = Client(broker, self.stdout, self.style,
                        verbosity = options['verbosity'])

        try:
            client.loop_forever()

        except KeyboardInterrupt:
            pass

        finally:
            client.disconnect()
<commit_msg>mqttseed<commit_after># mqttseed application command
from django.core.management.base import BaseCommand
from django.conf import settings

from ambulances.mqttupdate import UpdateClient

from ambulances.models import Ambulance, Hospital, HospitalEquipment

from django.utils.six import BytesIO
from rest_framework.parsers import JSONParser
from rest_framework.renderers import JSONRenderer

class Client(UpdateClient):

    # The callback for when the client receives a CONNACK
    # response from the server.
    def on_connect(self, client, userdata, flags, rc):

        # is connected?
        if not super().on_connect(client, userdata, flags, rc):
            return False

        # initialize pubset
        self.pubset = set()
        self.can_disconnect = False

        # Seed hospitals
        self.seed_hospital_data(client)
        self.seed_hospital_equipment_data(client)
        self.seed_hospital_metadata(client)

        # Seed ambulances
        self.seed_ambulance_data(client)

        # Seed calls
        # self.seed_calls(client)

        # Good to disconnect
        self.can_disconnect = True

    def publish(self, topic, message, *vargs, **kwargs):

        # increment pubcount then publish
        result = self.client.publish(topic, message, *vargs, **kwargs)
        self.pubset.add(result.mid)

        # echo if verbosity > 0
        if self.verbosity > 0:
            self.stdout.write("   {}: {}".format(topic, message))

    def seed_ambulance_data(self, client):

        if self.verbosity > 0:
            self.stdout.write(self.style.SUCCESS(">> Seeding ambulance data"))

        # seeding ambulances
        for obj in Ambulance.objects.all():
            self.update_ambulance(obj)

        if self.verbosity > 0:
            self.stdout.write(self.style.SUCCESS("<< Done seeding ambulance data"))

    def seed_hospital_data(self, client):

        if self.verbosity > 0:
            self.stdout.write(self.style.SUCCESS(">> Seeding hospital data"))

        # seeding hospital
        for obj in Hospital.objects.all():
            self.update_hospital(obj)

        if self.verbosity > 0:
            self.stdout.write(self.style.SUCCESS("<< Done seeding hospital data"))

    def seed_hospital_equipment_data(self, client):

        if self.verbosity > 0:
            self.stdout.write(self.style.SUCCESS(">> Seeding hospital equipment data"))

        # seeding hospital
        for obj in HospitalEquipment.objects.all():
            self.update_hospital_equipment(obj)

        if self.verbosity > 0:
            self.stdout.write(self.style.SUCCESS("<< Done seeding hospital equipment data"))

    def seed_hospital_metadata(self, client):
        if self.verbosity > 0:
            self.stdout.write(self.style.SUCCESS(">> Seeding hospital metadata"))

        # seeding hospital metadata
        for hospital in Hospital.objects.all():
            self.update_hospital_metadata(hospital)

            if self.verbosity > 0:
                self.stdout.write("   Hospital metadata for hospital {}".format(hospital.id))

        if self.verbosity > 0:
            self.stdout.write(self.style.SUCCESS("<< Done seeding hospital metadata"))

    # Message publish callback
    def on_publish(self, client, userdata, mid):

        # make sure all is published before disconnecting
        self.pubset.remove(mid)
        if len(self.pubset) == 0 and self.can_disconnect:
            self.disconnect()

class Command(BaseCommand):
    help = 'Seed the mqtt broker'

    def handle(self, *args, **options):

        import os

        broker = {
            'USERNAME': '',
            'PASSWORD': '',
            'HOST': '127.0.0.1',
            'PORT': 1883,
            'KEEPALIVE': 60,
            'CLIENT_ID': 'django',
            'CLEAN_SESSION': True
        }
        broker.update(settings.MQTT)
        broker['CLIENT_ID'] = 'mqttseed_' + str(os.getpid())

        client = Client(broker, self.stdout, self.style,
                        verbosity = options['verbosity'])

        try:
            client.loop_forever()

        except KeyboardInterrupt:
            pass

        finally:
            client.disconnect()
<|endoftext|>


<commit_before>import numpy as np

import torch

from syft.frameworks.torch.differential_privacy import pate

np.random.seed(0)

def test_base_dataset():

    num_teachers, num_examples, num_labels = (100, 50, 10)
    preds = (np.random.rand(num_teachers, num_examples) * num_labels).astype(int)  # fake preds

    indices = (np.random.rand(num_examples) * num_labels).astype(int)  # true answers

    preds[:, 0:10] *= 0

    data_dep_eps, data_ind_eps = pate.perform_analysis(
        teacher_preds=preds, indices=indices, noise_eps=0.1, delta=1e-5
    )

    assert data_dep_eps < data_ind_eps

def test_base_dataset_torch():

    num_teachers, num_examples, num_labels = (100, 50, 10)
    preds = (np.random.rand(num_teachers, num_examples) * num_labels).astype(int)  # fake preds

    indices = (np.random.rand(num_examples) * num_labels).astype(int)  # true answers

    preds[:, 0:10] *= 0

    data_dep_eps, data_ind_eps = pate.perform_analysis_torch(
        preds, indices, noise_eps=0.1, delta=1e-5
    )

    assert data_dep_eps < data_ind_eps
<commit_msg>Added test to check if torch implementation matches reference numpy implementation<commit_after>import numpy as np

import torch

from syft.frameworks.torch.differential_privacy import pate

np.random.seed(0)

def test_base_dataset():

    num_teachers, num_examples, num_labels = (100, 50, 10)
    preds = (np.random.rand(num_teachers, num_examples) * num_labels).astype(int)  # fake preds

    indices = (np.random.rand(num_examples) * num_labels).astype(int)  # true answers

    preds[:, 0:10] *= 0

    data_dep_eps, data_ind_eps = pate.perform_analysis(
        teacher_preds=preds, indices=indices, noise_eps=0.1, delta=1e-5
    )

    assert data_dep_eps < data_ind_eps

def test_base_dataset_torch():

    num_teachers, num_examples, num_labels = (100, 50, 10)
    preds = (np.random.rand(num_teachers, num_examples) * num_labels).astype(int)  # fake preds

    indices = (np.random.rand(num_examples) * num_labels).astype(int)  # true answers

    preds[:, 0:10] *= 0

    data_dep_eps, data_ind_eps = pate.perform_analysis_torch(
        preds, indices, noise_eps=0.1, delta=1e-5
    )

    assert data_dep_eps < data_ind_eps

def test_torch_ref_match():

    # Verify if the torch implementation values match the original Numpy implementation.

    num_teachers, num_examples, num_labels = (100, 50, 10)
    preds = (np.random.rand(num_teachers, num_examples) * num_labels).astype(int)  # fake preds

    indices = (np.random.rand(num_examples) * num_labels).astype(int)  # true answers

    preds[:, 0:10] *= 0

    data_dep_eps, data_ind_eps = pate.perform_analysis_torch(
        preds, indices, noise_eps=0.1, delta=1e-5
    )

    data_dep_eps_ref, data_ind_eps_ref = pate.perform_analysis(
        preds, indices, noise_eps=0.1, delta=1e-5
    )

    assert torch.isclose(data_dep_eps, torch.tensor(data_dep_eps_ref.item()))
    assert torch.isclose(data_ind_eps, torch.tensor(data_ind_eps_ref.item()))
<|endoftext|>


<commit_before>"""
Provides usable args and kwargs from inspect.getcallargs.

For Python 3.3 and above, this module is unnecessary and can be achieved using
features from PEP 362:

    http://www.python.org/dev/peps/pep-0362/

For example, to override a parameter of some function:

    >>> import inspect
    >>> def func(a, b=1, c=2, d=3):
    ...    return a, b, c, d
    ...
    >>> def override_c(*args, **kwargs):
    ...    sig = inspect.signature(override)
    ...    ba = sig.bind(*args, **kwargs)
    ...    ba['c'] = 10
    ...    return func(*ba.args, *ba.kwargs)
    ...
    >>> override_c(0, c=3)
    (0, 1, 10, 3)

Also useful:

    http://www.python.org/dev/peps/pep-3102/

"""

import sys
import inspect

from inspect import getcallargs

try:
    from inspect import getfullargspec
except ImportError:
    # Python 2.X
    from collections import namedtuple
    from inspect import getargspec

    FullArgSpec = namedtuple('FullArgSpec',
    'args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, annotations')

    def getfullargspec(f):
        args, varargs, varkw, defaults = getargspec(f)
        kwonlyargs = []
        kwonlydefaults = None
        annotations = getattr(f, '__annotations__', {})
        return FullArgSpec(args, varargs, varkw, defaults,
                           kwonlyargs, kwonlydefaults, annotations)

def bindcallargs_leq32(_fUnCtIoN_, *args, **kwargs):
    """Binds arguments and keyword arguments to a function or method.

    Returns a tuple (bargs, bkwargs) suitable for manipulation and passing
    to the specified function.

    `bargs` consists of the bound args, varargs, and kwonlyargs from
    getfullargspec. `bkwargs` consists of the bound varkw from getfullargspec.
    Both can be used in a call to the specified function.  Any default
    parameter values are included in the output.

    Examples
    --------
    >>> def func(a, b=3, *args, **kwargs):
    ...    pass

    >>> bindcallargs(func, 5)
    ((5, 3), {})

    >>> bindcallargs(func, 5, 4, 3, 2, 1, hello='there')
    ((5, 4, 3, 2, 1), {'hello': 'there'})

    >>> args, kwargs = bindcallargs(func, 5)
    >>> kwargs['b'] = 5 # overwrite default value for b
    >>> func(*args, **kwargs)

    """
    # It is necessary to choose an unlikely variable name for the function.
    # The reason is that any kwarg by the same name will cause a TypeError
    # due to multiple values being passed for that argument name.
    func = _fUnCtIoN_

    callargs = getcallargs(func, *args, **kwargs)
    spec = getfullargspec(func)

    # Construct all args and varargs and use them in bargs
    bargs = [callargs[arg] for arg in spec.args]
    if spec.varargs is not None:
        bargs.extend(callargs[spec.varargs])
    bargs = tuple(bargs)

    # Start with kwonlyargs.
    bkwargs = dict((kwonlyarg, callargs[kwonlyarg]) for kwonlyarg in spec.kwonlyargs)
    # Add in kwonlydefaults for unspecified kwonlyargs only.
    if spec.kwonlydefaults is not None:
        bkwargs.update(dict([(k, v) for k, v in spec.kwonlydefaults.items()
                             if k not in bkwargs]))
    # Add in varkw.
    if spec.varkw is not None:
        bkwargs.update(callargs[spec.varkw])

    return bargs, bkwargs

def bindcallargs_geq33(_fUnCtIoN_, *args, **kwargs):
    # Should match functionality of bindcallargs_32 for Python > 3.3.
    sig = inspect.signature(_fUnCtIoN_)
    ba = sig.bind(*args, **kwargs)
    # Add in all default values
    for param in sig.parameters.values():
        if param.name not in ba.arguments:
            ba.arguments[param.name] = param.default
    return ba.args, ba.kwargs

if sys.version_info[0:2] < (3,3): # pragma: no cover
    bindcallargs = bindcallargs_leq32
else: # pragma: no cover
    bindcallargs = bindcallargs_geq33
<commit_msg>mark some bindargs stuff as no cover, and switch to dict comprehensions since we don't support 2.6<commit_after>"""
Provides usable args and kwargs from inspect.getcallargs.

For Python 3.3 and above, this module is unnecessary and can be achieved using
features from PEP 362:

    http://www.python.org/dev/peps/pep-0362/

For example, to override a parameter of some function:

    >>> import inspect
    >>> def func(a, b=1, c=2, d=3):
    ...    return a, b, c, d
    ...
    >>> def override_c(*args, **kwargs):
    ...    sig = inspect.signature(override)
    ...    ba = sig.bind(*args, **kwargs)
    ...    ba['c'] = 10
    ...    return func(*ba.args, *ba.kwargs)
    ...
    >>> override_c(0, c=3)
    (0, 1, 10, 3)

Also useful:

    http://www.python.org/dev/peps/pep-3102/

"""

import sys
import inspect

from inspect import getcallargs

try:
    from inspect import getfullargspec
except ImportError:
    # Python 2.X
    from collections import namedtuple
    from inspect import getargspec

    FullArgSpec = namedtuple('FullArgSpec',
    'args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, annotations')

    def getfullargspec(f):
        args, varargs, varkw, defaults = getargspec(f)
        kwonlyargs = []
        kwonlydefaults = None
        annotations = getattr(f, '__annotations__', {})
        return FullArgSpec(args, varargs, varkw, defaults,
                           kwonlyargs, kwonlydefaults, annotations)

def bindcallargs_leq32(_fUnCtIoN_, *args, **kwargs):
    """Binds arguments and keyword arguments to a function or method.

    Returns a tuple (bargs, bkwargs) suitable for manipulation and passing
    to the specified function.

    `bargs` consists of the bound args, varargs, and kwonlyargs from
    getfullargspec. `bkwargs` consists of the bound varkw from getfullargspec.
    Both can be used in a call to the specified function.  Any default
    parameter values are included in the output.

    Examples
    --------
    >>> def func(a, b=3, *args, **kwargs):
    ...    pass

    >>> bindcallargs(func, 5)
    ((5, 3), {})

    >>> bindcallargs(func, 5, 4, 3, 2, 1, hello='there')
    ((5, 4, 3, 2, 1), {'hello': 'there'})

    >>> args, kwargs = bindcallargs(func, 5)
    >>> kwargs['b'] = 5 # overwrite default value for b
    >>> func(*args, **kwargs)

    """
    # It is necessary to choose an unlikely variable name for the function.
    # The reason is that any kwarg by the same name will cause a TypeError
    # due to multiple values being passed for that argument name.
    func = _fUnCtIoN_

    callargs = getcallargs(func, *args, **kwargs)
    spec = getfullargspec(func)

    # Construct all args and varargs and use them in bargs
    bargs = [callargs[arg] for arg in spec.args]
    if spec.varargs is not None:
        bargs.extend(callargs[spec.varargs])
    bargs = tuple(bargs)

    # Start with kwonlyargs.
    bkwargs = {kwonlyarg: callargs[kwonlyarg] for kwonlyarg in spec.kwonlyargs}
    # Add in kwonlydefaults for unspecified kwonlyargs only.
    # Since keyword only arguements aren't allowed in python2, and we
    # don't support python 3.0, 3.1, 3.2, this should never be executed:
    if spec.kwonlydefaults is not None:  # praga: no cover
        bkwargs.update({k: v for k, v in spec.kwonlydefaults.items()
                             if k not in bkwargs})
    # Add in varkw.
    if spec.varkw is not None:
        bkwargs.update(callargs[spec.varkw])

    return bargs, bkwargs

def bindcallargs_geq33(_fUnCtIoN_, *args, **kwargs):
    # Should match functionality of bindcallargs_32 for Python > 3.3.
    sig = inspect.signature(_fUnCtIoN_)
    ba = sig.bind(*args, **kwargs)
    # Add in all default values
    for param in sig.parameters.values():
        if param.name not in ba.arguments:
            ba.arguments[param.name] = param.default
    return ba.args, ba.kwargs

if sys.version_info[0:2] < (3,3): # pragma: no cover
    bindcallargs = bindcallargs_leq32
else: # pragma: no cover
    bindcallargs = bindcallargs_geq33
<|endoftext|>


<commit_before># -*- coding:utf-8 -*-

"""
Select specific simulations from a HDF5 file and plot the granule figures and
scatter plot to compare the simulations.

"""

import matplotlib.pyplot as plt
import tables

import h5manager as h5m
from plotting import granule_pop_figure
from analysis import fftmax

DB_FILENAME = "data/db30x30_two_glom_beta_new_ps_interco_strength0_1_interco_rate0_1.h5"
DB = tables.openFile(DB_FILENAME)

# Get all simulation data
ATTRS = (('paramset', '_v_attrs', 'Common', 'inter_conn_rate', 0, 1),
         ('paramset', '_v_attrs', 'Common', 'inter_conn_strength', 0, 1),
         ('paramset', 'arrays', 'times'),
         ('paramset', '_v_attrs', 'Common', 'simu_dt'),
         ('results', 's_granule'),
         ('results', 's_syn_self'),
         ('results', '_v_attrs'),
         ('results', 'spikes_it'),
)
ALL_SIMU_ATTRS = h5m.get_all_attrs(DB, ATTRS)

SET_INTERCO_RATE = list(set(s[0] for s in ALL_SIMU_ATTRS))
SET_INTERCO_RATE.sort()
SET_INTERCO_STRENGTH = list(set(s[1] for s in ALL_SIMU_ATTRS))
SET_INTERCO_STRENGTH.sort()

# Some useful functions
def deq(a, b, delta):
    """Delta equality"""
    return abs(a - b) < delta

class SignalRepack:
    def __init__(self, values, times):
        self.values = values
        self.times = times

# Get the specific simulations
def get_interco(simu, interco_rate, interco_strength):
    return deq(simu[0], interco_rate, 0.02) and deq(simu[1], interco_strength, 0.02)

SELECTED_RATES = [28]
SELECTED_STRENGTH = [0, 10, 20, 25]
for rate in SELECTED_RATES:
    for strength in SELECTED_STRENGTH:
        INTERCO_RATE = SET_INTERCO_RATE[rate]
        INTERCO_STRENGTH = SET_INTERCO_STRENGTH[strength]
        GOOD_SIMUS = filter(lambda x: get_interco(x, INTERCO_RATE, INTERCO_STRENGTH), ALL_SIMU_ATTRS)

        REDO_FFTMAX = []
        for simu in GOOD_SIMUS:
            gr_s = simu[4].read()
            gr_s_syn_self = simu[5].read()
            times = simu[2].read()
            dt = float(simu[3])
            granule_pop_figure(gr_s, gr_s_syn_self, times, dt)

            signal = SignalRepack(gr_s, times)
            REDO_FFTMAX.append(fftmax(signal, 2, dt))

            spikes_it = simu[7].read()
            plt.figure()
            plt.title("rate: "+str(rate)+", strength: "+str(strength))
            plt.plot(spikes_it[1], spikes_it[0], ' .', mew=0)

        print 'rate:', rate, 'strength:', strength, REDO_FFTMAX
plt.show()

# Finally close the DB
DB.close()
<commit_msg>Fix bugs that selected multiple simu instead of 1<commit_after># -*- coding:utf-8 -*-

"""
Select specific simulations from a HDF5 file and plot the granule figures and
scatter plot to compare the simulations.

"""

import matplotlib.pyplot as plt
import tables
from numpy import allclose

import h5manager as h5m
from plotting import granule_pop_figure
from analysis import fftmax

DB_FILENAME = "data/db30x30_beta_homeostasis.h5"
DB = tables.openFile(DB_FILENAME)

# Get all simulation data
ATTRS = (('paramset', '_v_attrs', 'Common', 'inter_conn_rate', 0, 1),
         ('paramset', '_v_attrs', 'Common', 'inter_conn_strength', 0, 1),
         ('paramset', 'arrays', 'times'),
         ('paramset', '_v_attrs', 'Common', 'simu_dt'),
         ('results', 's_granule'),
         ('results', 's_syn_self'),
         ('results', '_v_attrs'),
         ('results', 'spikes_it'),
)
ALL_SIMU_ATTRS = h5m.get_all_attrs(DB, ATTRS)
ALL_SIMU_ATTRS.sort()

SET_INTERCO_RATE = list(set(s[0] for s in ALL_SIMU_ATTRS))
SET_INTERCO_RATE.sort()
SET_INTERCO_STRENGTH = list(set(s[1] for s in ALL_SIMU_ATTRS))
SET_INTERCO_STRENGTH.sort()

# Some useful functions
def deq(a, b, delta):
    """Delta equality"""
    return abs(a - b) < delta

class SignalRepack:
    def __init__(self, values, times):
        self.values = values
        self.times = times

# Get the specific simulations
def get_interco(simu, interco_rate, interco_strength):
    return deq(simu[0], interco_rate, 0.02) and deq(simu[1], interco_strength, 0.02)

SELECTED_RATES = [20, 25]
SELECTED_STRENGTH = [6]
REDO_FFTMAX = []
for rate in SELECTED_RATES:
    for strength in SELECTED_STRENGTH:
        interco_rate = SET_INTERCO_RATE[rate]
        interco_strength = SET_INTERCO_STRENGTH[strength]
        filtered_simu = filter(lambda x: allclose(x[0], interco_rate) \
                                         and allclose(x[1], interco_strength),
                               ALL_SIMU_ATTRS)
        if len(filtered_simu) > 1:
            print "Warning: more than one simulation satisfy your criteria,",
            print "selecting the first one."
        simu = filtered_simu[0]

        gr_s = simu[4].read()
        gr_s_syn_self = simu[5].read()
        times = simu[2].read()
        dt = float(simu[3])
        granule_pop_figure(gr_s, gr_s_syn_self, times, dt)

        signal = SignalRepack(gr_s, times)
        REDO_FFTMAX.append(fftmax(signal, 2, dt))

        spikes_it = simu[7].read()
        plt.figure()
        plt.title("rate: "+str(rate)+", strength: "+str(strength))
        plt.plot(spikes_it[1], spikes_it[0], ' .', mew=0)

        print 'rate:', rate, 'strength:', strength, REDO_FFTMAX
plt.show()

# Finally close the DB
DB.close()
<|endoftext|>


<commit_before>"""Make a gridded analysis of p01d_12z based on obs."""
import sys
import datetime

import numpy as np
from metpy.units import units as mpunits
from metpy.units import masked_array
from metpy.gridding.interpolation import inverse_distance
from pandas.io.sql import read_sql
from pyiem.iemre import get_daily_ncname, daily_offset
from pyiem.util import ncopen, get_dbconn

def generic_gridder(day, nc, df, idx):
    """
    Generic gridding algorithm for easy variables
    """
    domain = nc.variables['hasdata'][:, :]
    xi, yi = np.meshgrid(nc.variables['lon'][:], nc.variables['lat'][:])
    res = np.ones(xi.shape) * np.nan
    # set a sentinel of where we won't be estimating
    res = np.where(domain > 0, res, -9999)
    # do our gridding
    grid = inverse_distance(df['lon'].values, df['lat'].values,
                            df[idx].values, xi, yi, 1.5)
    # replace nan values in res with whatever now is in grid
    res = np.where(np.isnan(res), grid, res)
    # Do we still have missing values?
    if np.isnan(res).any():
        # very aggressive with search radius
        grid = inverse_distance(df['lon'].values, df['lat'].values,
                                df[idx].values, xi, yi, 5.5)
        # replace nan values in res with whatever now is in grid
        res = np.where(np.isnan(res), grid, res)
    # replace sentinel back to np.nan
    res = np.where(res == -9999, np.nan, res)
    print(("%s %s rows for %s column min:%.3f max:%.3f"
           ) % (day, len(df.index), idx, np.nanmin(res), np.nanmax(res)))
    return masked_array(np.ma.array(res, mask=np.isnan(res)), mpunits('inch'))

def main(argv):
    """Do work please"""
    day = datetime.date(int(argv[1]), int(argv[2]), int(argv[3]))
    pgconn = get_dbconn('coop')
    df = read_sql("""
        SELECT a.precip, st_x(t.geom) as lon, st_y(t.geom) as lat
        from alldata a JOIN stations t ON (a.station = t.id)
        WHERE a.day = %s and t.network ~* 'CLIMATE' and
        substr(a.station,3,4) != '0000' and substr(station,3,1) != 'C'
        and precip >= 0 and precip < 50
    """, pgconn, params=(day, ))
    nc = ncopen(get_daily_ncname(day.year), 'a')
    res = generic_gridder(day, nc, df, 'precip')
    if res is not None:
        offset = daily_offset(day)
        nc.variables['p01d_12z'][offset] = res.to(mpunits('mm')).magnitude
    nc.close()

if __name__ == '__main__':
    main(sys.argv)
<commit_msg>more p01d thrashing<commit_after>"""Make a gridded analysis of p01d_12z based on obs."""
import sys
import datetime

import numpy as np
from metpy.units import units as mpunits
from metpy.units import masked_array
from metpy.gridding.interpolation import inverse_distance
from pandas.io.sql import read_sql
from pyiem.iemre import get_daily_ncname, daily_offset
from pyiem.util import ncopen, get_dbconn

def generic_gridder(day, nc, df, idx):
    """
    Generic gridding algorithm for easy variables
    """
    domain = nc.variables['hasdata'][:, :]
    xi, yi = np.meshgrid(nc.variables['lon'][:], nc.variables['lat'][:])
    res = np.ones(xi.shape) * np.nan
    # set a sentinel of where we won't be estimating
    res = np.where(domain > 0, res, -9999)
    # do our gridding
    grid = inverse_distance(df['lon'].values, df['lat'].values,
                            df[idx].values, xi, yi, 1.5)
    # replace nan values in res with whatever now is in grid
    res = np.where(np.isnan(res), grid, res)
    # replace sentinel back to np.nan
    res = np.where(res == -9999, np.nan, res)
    print(("%s %s rows for %s column min:%.3f max:%.3f"
           ) % (day, len(df.index), idx, np.nanmin(res), np.nanmax(res)))
    return masked_array(np.ma.array(res, mask=np.isnan(res)), mpunits('inch'))

def main(argv):
    """Do work please"""
    day = datetime.date(int(argv[1]), int(argv[2]), int(argv[3]))
    pgconn = get_dbconn('coop')
    df = read_sql("""
        SELECT a.precip, st_x(t.geom) as lon, st_y(t.geom) as lat
        from alldata a JOIN stations t ON (a.station = t.id)
        WHERE a.day = %s and t.network ~* 'CLIMATE' and
        substr(a.station,3,4) != '0000' and substr(station,3,1) != 'C'
        and precip >= 0 and precip < 50
    """, pgconn, params=(day, ))
    nc = ncopen(get_daily_ncname(day.year), 'a')
    res = generic_gridder(day, nc, df, 'precip')
    if res is not None:
        offset = daily_offset(day)
        nc.variables['p01d_12z'][offset] = res.to(mpunits('mm')).magnitude
    nc.close()

if __name__ == '__main__':
    main(sys.argv)
<|endoftext|>


<commit_before>import requests

from bs4 import BeautifulSoup, Tag

from django.conf import settings

def get_dropbox_paper_soup(paper_id):
    """
        Connects to dropbox API and export the Paper's content into memory
    """
    url = "https://content.dropboxapi.com/2/files/export"
    headers = {
        "Authorization": "Bearer {}".format(settings.DROPBOX_ACCESS_TOKEN),
        "Dropbox-API-Arg": "{{\"path\":\"{}\"}}".format(paper_id)
    }
    paper = requests.post(url, headers=headers)
    soup = BeautifulSoup(paper.content, 'html.parser')
    return soup

def extract_diventi_content(mention_link, diventi_universale_soup):
    """
        Extract the paragraph between two titles in Diventi Universale.
        It connects via direct link to the desired title.
    """
    mentioned_uuid = mention_link.replace('#:uid=', '###')
    mentioned_uuid = mentioned_uuid.replace('#:h2=', '###')
    mentioned_uuid = mentioned_uuid.replace('&h2=', '###')
    mentioned_uuid = mentioned_uuid.split("###")[1]
    diventi_title = diventi_universale_soup.select_one('[data-usually-unique-id="{}"]'.format(mentioned_uuid))
    if diventi_title:
        diventi_title = diventi_title.parent
        if diventi_title.h1:
            diventi_title.h1.extract()
        if diventi_title.h2:
            diventi_title.h2.extract()
        while 1:
            diventi_paragraph = diventi_title.next_sibling
            if diventi_paragraph:
                if diventi_paragraph.h1 or diventi_paragraph.h2:
                    break
                else:
                    diventi_title.append(diventi_paragraph)
            else:
                break
        return diventi_title
    else:
        return ''

def adjust_paper_visual_styles(paper_soup):
    """
        Adds vertical spacing to tables.
    """
    tables = paper_soup.find_all('table')
    for table in tables:
        table['class'] = 'my-2'
        table['style'] = ''
    table_columns = paper_soup.find_all('td')
    for td in table_columns:
        td['style'] = ''
    return paper_soup

def render_paper_images_by_direct_url(paper_soup):
    """
        Substitutes direct links in tables with the rendered image.
        Image links should be hosted in dropbox and written with "raw=1" parameter.
    """
    links = paper_soup.find_all('a')
    for link_tag in links:
        image_link = link_tag.get('href')
        whitelist_links = [
            '.png?raw=1',
            '.jpg?raw=1',
        ]
        if [link for link in whitelist_links if(link in image_link)]:
            image_soup = BeautifulSoup('', 'html.parser')
            image_tag = image_soup.new_tag('img')
            image_tag['src'] = 'data:image/png;base64,R0lGODlhAQABAAD/ACwAAAAAAQABAAACADs='
            image_tag['data-src'] = image_link
            image_tag['class'] = 'img-fluid rounded mx-auto'
            is_table_link = link_tag.find_parent('table')
            if is_table_link:
                image_tag['class'] = 'rounded mx-auto'
            else:
                image_tag['class'] = 'w-75 d-block p-2 rounded mx-auto'
            link_tag = link_tag.find_parent('span')
            link_tag.replace_with(image_tag)
    return paper_soup

def render_paper_code_blocks(paper_soup):
    """
        Substitutes code blocks with quick's style alerts.
    """
    codes = paper_soup.find_all('code')
    for code_tag in codes:
        code_content = code_tag.findChildren()
        alert_soup = BeautifulSoup('', 'html.parser')
        alert_tag = alert_soup.new_tag('div')
        alert_tag['role'] = 'alert'
        alert_tag['class'] = 'alert alert-outline-primary my-1'
        for child in code_content:
            alert_tag.append(child)
        code_tag = code_tag.find_parent('div')
        code_tag.replace_with(alert_tag)
    return paper_soup

def remove_dropbox_placeholders(paper_soup):
    """
        Removes dropbox image previews from the document.
    """
    images = paper_soup.find_all('img')
    blacklist_links = [
        '2753.png',
        '?dl=0',
    ]
    for image_tag in images:
        image_target = image_tag.get('data-src')
        if image_target and [link for link in blacklist_links if(link in image_target)]:
            image_tag.extract()
    links = paper_soup.find_all('a')
    for link_tag in links:
        link_target = link_tag.get('data-target-href')
        if link_target and [link for link in blacklist_links if(link in link_target)]:
            link_tag.extract()
    dropbox_galleries = paper_soup.find_all('span', class_='gallery')
    for gallery_tag in dropbox_galleries:
        gallery_tag.extract()
    return paper_soup

<commit_msg>Ebooks: updated code blocks  visual style.<commit_after>import requests

from bs4 import BeautifulSoup, Tag

from django.conf import settings

def get_dropbox_paper_soup(paper_id):
    """
        Connects to dropbox API and export the Paper's content into memory
    """
    url = "https://content.dropboxapi.com/2/files/export"
    headers = {
        "Authorization": "Bearer {}".format(settings.DROPBOX_ACCESS_TOKEN),
        "Dropbox-API-Arg": "{{\"path\":\"{}\"}}".format(paper_id)
    }
    paper = requests.post(url, headers=headers)
    soup = BeautifulSoup(paper.content, 'html.parser')
    return soup

def extract_diventi_content(mention_link, diventi_universale_soup):
    """
        Extract the paragraph between two titles in Diventi Universale.
        It connects via direct link to the desired title.
    """
    mentioned_uuid = mention_link.replace('#:uid=', '###')
    mentioned_uuid = mentioned_uuid.replace('#:h2=', '###')
    mentioned_uuid = mentioned_uuid.replace('&h2=', '###')
    mentioned_uuid = mentioned_uuid.split("###")[1]
    diventi_title = diventi_universale_soup.select_one('[data-usually-unique-id="{}"]'.format(mentioned_uuid))
    if diventi_title:
        diventi_title = diventi_title.parent
        if diventi_title.h1:
            diventi_title.h1.extract()
        if diventi_title.h2:
            diventi_title.h2.extract()
        while 1:
            diventi_paragraph = diventi_title.next_sibling
            if diventi_paragraph:
                if diventi_paragraph.h1 or diventi_paragraph.h2:
                    break
                else:
                    diventi_title.append(diventi_paragraph)
            else:
                break
        return diventi_title
    else:
        return ''

def adjust_paper_visual_styles(paper_soup):
    """
        Adds vertical spacing to tables.
    """
    tables = paper_soup.find_all('table')
    for table in tables:
        table['class'] = 'my-2'
        table['style'] = ''
    table_columns = paper_soup.find_all('td')
    for td in table_columns:
        td['style'] = ''
    return paper_soup

def render_paper_images_by_direct_url(paper_soup):
    """
        Substitutes direct links in tables with the rendered image.
        Image links should be hosted in dropbox and written with "raw=1" parameter.
    """
    links = paper_soup.find_all('a')
    for link_tag in links:
        image_link = link_tag.get('href')
        whitelist_links = [
            '.png?raw=1',
            '.jpg?raw=1',
        ]
        if [link for link in whitelist_links if(link in image_link)]:
            image_soup = BeautifulSoup('', 'html.parser')
            image_tag = image_soup.new_tag('img')
            image_tag['src'] = 'data:image/png;base64,R0lGODlhAQABAAD/ACwAAAAAAQABAAACADs='
            image_tag['data-src'] = image_link
            image_tag['class'] = 'img-fluid rounded mx-auto'
            is_table_link = link_tag.find_parent('table')
            if is_table_link:
                image_tag['class'] = 'rounded mx-auto'
            else:
                image_tag['class'] = 'w-75 d-block p-2 rounded mx-auto'
            link_tag = link_tag.find_parent('span')
            link_tag.replace_with(image_tag)
    return paper_soup

def render_paper_code_blocks(paper_soup):
    """
        Substitutes code blocks with quick's style alerts.
    """
    codes = paper_soup.find_all('code')
    for code_tag in codes:
        code_content = code_tag.findChildren()
        alert_soup = BeautifulSoup('', 'html.parser')
        alert_tag = alert_soup.new_tag('div')
        alert_tag['role'] = 'alert'
        alert_tag['class'] = 'alert bg-translucent-secondary my-0 py-1'
        for child in code_content:
            if child.string:
                alert_tag.append(child.string)
        code_tag = code_tag.find_parent('div')
        code_tag.replace_with(alert_tag)
    return paper_soup

def remove_dropbox_placeholders(paper_soup):
    """
        Removes dropbox image previews from the document.
    """
    images = paper_soup.find_all('img')
    blacklist_links = [
        '2753.png',
        '?dl=0',
    ]
    for image_tag in images:
        image_target = image_tag.get('data-src')
        if image_target and [link for link in blacklist_links if(link in image_target)]:
            image_tag.extract()
    links = paper_soup.find_all('a')
    for link_tag in links:
        link_target = link_tag.get('data-target-href')
        if link_target and [link for link in blacklist_links if(link in link_target)]:
            link_tag.extract()
    dropbox_galleries = paper_soup.find_all('span', class_='gallery')
    for gallery_tag in dropbox_galleries:
        gallery_tag.extract()
    return paper_soup

<|endoftext|>