<commit_before># -*- coding: utf-8 -*-

from PySide import QtCore, QtGui
import os
from gr.pygr import *
try:
    from PySide import shiboken
except ImportError:
    import shiboken

from util.logger import Logger
import sys
from statistics.rdf import RDF
import numpy as np
import os

logger = Logger("gui.rdf_widget")
logger.setstream("default", sys.stdout, Logger.DEBUG)

class GrPlotWidget(QtGui.QWidget) :
    def __init__(self, *args) :
        QtGui.QWidget.__init__(self)

        self.init_gui(self)

        os.environ["GKS_WSTYPE"] = "381"
        os.environ["GKS_DOUBLE_BUF"] = "True"

        #self.connect(self.DrawButton, QtCore.SIGNAL("clicked()"), self.draw)
        #self.connect(self.QuitButton, QtCore.SIGNAL("clicked()"), self.quit)
        self.w = 500
        self.h = 500
        self.sizex = 1
        self.sizey = 1
        self.xvalues = None
        self.yvalues = None
        self.title = None
        self.datapoints = None

    def init_gui(self, form) :
        form.resize(QtCore.QSize(500, 500).expandedTo(form.minimumSizeHint()))

        #self.DrawButton = QtGui.QPushButton(form)
        #self.DrawButton.setText("Draw")
        #self.DrawButton.setGeometry(QtCore.QRect(290, 5, 100, 25))
        #self.DrawButton.setObjectName("draw")

        #self.QuitButton = QtGui.QPushButton(form)
        #self.QuitButton.setText("Quit")
        #self.QuitButton.setGeometry(QtCore.QRect(395, 5, 100, 25))
        #self.QuitButton.setObjectName("quit")

        QtCore.QMetaObject.connectSlotsByName(form)

    def quit(self) :
        gr.emergencyclosegks()
        self.close()

    def setdata(self, xvalues, yvalues, title, datapoints=None):
        self.xvalues = xvalues
        self.yvalues = yvalues
        self.title = title
        self.datapoints = datapoints

    def draw(self) :
        self.setStyleSheet("background-color:white;");

        #x = range(0, 128)
        #y = range(0, 128)
        #z = readfile(os.path.join(os.path.dirname(os.path.realpath(__file__)),
        #                          "kws.dat"), separator='$')
        #zrange = max(z) - min(z)
        #h = [min(z) + i * 0.025 * zrange for i in range(0, 40)]

        if not self.xvalues is None:
            rangex = (self.xvalues.min(), self.xvalues.max())
        else:
            rangex = (0, 10)
        if not self.yvalues is None:
            rangey = gr.adjustrange(self.yvalues.min(), self.yvalues.max())
        else:
            rangey = (0, 4)

        gr.clearws()
        mwidth  = self.w * 2.54 / self.logicalDpiX() / 100
        mheight = self.h * 2.54 / self.logicalDpiY() / 100
        gr.setwsviewport(0, mwidth, 0, mheight)
        gr.setwswindow(0, self.sizex, 0, self.sizey)
        gr.setviewport(0.075 * self.sizex, 0.95 * self.sizex, 0.075 * self.sizey, 0.95 * self.sizey)
        gr.setwindow(rangex[0], rangex[1], rangey[0], rangey[1])
        gr.setcharheight(0.012)

        gr.setfillintstyle(1)
        gr.setfillcolorind(0)
        gr.fillrect(rangex[0], rangex[1], rangey[0], rangey[1])

        if not self.xvalues is None and not self.yvalues is None:
            gr.setlinecolorind(2)
            gr.polyline(self.xvalues, self.yvalues)
        else:
            gr.text(0.4, 0.45, "no elements selected")

        ## rug plot
        #if not self.datapoints is None:
        #    gr.setmarkertype(gr.MARKERTYPE_SOLID_TRI_UP)
        #    gr.setmarkercolorind(2)
        #    gr.setmarkersize(1.0)
        #    gr.polymarker(self.datapoints, np.zeros_like(self.datapoints))

        gr.setlinecolorind(1)
        gr.axes(0.2, 0.2, rangex[0], rangey[0], 5, 5, 0.0075)
        gr.axes(0.2, 0.2, rangex[1], rangey[1], -5, -5, -0.0075)

        if not self.title is None:
            gr.text(0.75, 0.65, self.title)
        self.update()

    def resizeEvent(self, event):
        self.w = event.size().width()
        self.h = event.size().height()
        if self.w > self.h:
            self.sizex = 1
            self.sizey = float(self.h)/self.w
        else:
            self.sizex = float(self.w)/self.h
            self.sizey = 1
        self.draw()

    def paintEvent(self, ev) :
        self.painter = QtGui.QPainter()
        self.painter.begin(self)
        os.environ['GKSconid'] = "%x!%x" % (long(shiboken.getCppPointer(self)[0]), long(shiboken.getCppPointer(self.painter)[0]))
        gr.updatews()
        self.painter.end()

class RDFWidget(QtGui.QWidget):

    def __init__(self, parent):
        QtGui.QWidget.__init__(self, parent)

        self.control = parent.control
        self.results = None
        self.rdf = None

        self.init_gui()

    def init_gui(self):
        vbox = QtGui.QVBoxLayout()
        grid = QtGui.QGridLayout()
        self.gr_widget = GrPlotWidget()

        self.datasetlabel = QtGui.QLabel("No data loaded. Press 'Refresh'.", self)
        self.datasetlabel.setAlignment(QtCore.Qt.AlignHCenter)

        elembox = QtGui.QHBoxLayout()
        elembox.addWidget(QtGui.QLabel("Elements:", self), 0, 0)
        self.elem1 = QtGui.QComboBox(self)
        elembox.addWidget(self.elem1, 0, 1)
        self.elem2 = QtGui.QComboBox(self)
        elembox.addWidget(self.elem2, 0, 2)
        grid.addLayout(elembox, 0, 0)

        rangebox = QtGui.QHBoxLayout()
        rangebox.addWidget(QtGui.QLabel("Plot range:", self))
        self.range1 = QtGui.QLineEdit("0", self)
        rangebox.addWidget(self.range1)
        rangebox.addWidget(QtGui.QLabel("-", self))
        self.range2 = QtGui.QLineEdit("8", self)
        rangebox.addWidget(self.range2)
        grid.addLayout(rangebox, 0, 1)

        cutoffbox = QtGui.QHBoxLayout()
        cutoffbox.addWidget(QtGui.QLabel("Cutoff:", self))
        self.cutoff = QtGui.QLineEdit("12", self)
        cutoffbox.addWidget(self.cutoff)
        cutoffbox.addWidget(QtGui.QLabel("Bandwidth:", self))
        self.bandwidth = QtGui.QLineEdit("", self)
        cutoffbox.addWidget(self.bandwidth)
        grid.addLayout(cutoffbox, 0, 2)

        self.plotbutton = QtGui.QPushButton("Plot", self)
        grid.addWidget(self.plotbutton, 1, 0)
        self.connect(self.plotbutton, QtCore.SIGNAL("clicked()"), self.draw)

        self.exportbutton = QtGui.QPushButton("Save Image", self)
        grid.addWidget(self.exportbutton, 1, 1)
        self.connect(self.exportbutton, QtCore.SIGNAL("clicked()"), self.export)

        self.refreshbutton = QtGui.QPushButton("Refresh Data", self)
        grid.addWidget(self.refreshbutton, 1, 2)
        self.connect(self.refreshbutton, QtCore.SIGNAL("clicked()"), self.refresh)

        vbox.addWidget(self.gr_widget, stretch=1)
        vbox.addWidget(self.datasetlabel, stretch=0)
        vbox.addLayout(grid)
        self.setLayout(vbox)
        self.show()

    def keyPressEvent(self, event):
        if event.key() == QtCore.Qt.Key_Return \
                or event.key() == QtCore.Qt.Key_Enter:
            self.draw()

    def draw(self):
        xvalues = None
        yvalues = None
        title = None
        datapoints = None
        if self.rdf is None:
            self.refresh()
        if not self.rdf is None:
            elem1 = str(self.elem1.currentText())
            elem2 = str(self.elem2.currentText())
            range1 = float(str(self.range1.text()))
            range2 = float(str(self.range2.text()))
            cutoff = str(self.cutoff.text())
            if len(cutoff) > 0 and float(cutoff) > 0:
                cutoff = float(cutoff)
            else:
                cutoff = None
            bandwidth = str(self.bandwidth.text())
            if len(bandwidth) > 0 and float(bandwidth) > 0:
                bandwidth = float(bandwidth)
            else:
                bandwidth = None
            f = self.rdf.rdf(elem1, elem2, cutoff=cutoff, h=bandwidth)
            if not f is None:
                xvalues = np.linspace(range1, range2, 400)
                yvalues = f(xvalues)
                title = "{} - {}".format(elem1, elem2)
                datapoints = f.f.x

        self.gr_widget.setdata(xvalues, yvalues, title, datapoints)
        self.gr_widget.draw()

    def export(self):
        extensions = (".eps", ".ps", ".pdf", ".png", ".bmp", ".jpg", ".jpeg", ".png", ".tiff", ".fig", ".svg", ".wmf")
        qtext = "*" + " *".join(extensions)
        filepath, _ = QtGui.QFileDialog.getSaveFileName(self, "Save Image", \
                ".", "Image Files ({})".format(qtext))
        if len(filepath) == 0:
            return
        gr.beginprint(filepath)
        self.draw()
        gr.endprint()

    def refresh(self):
        results = self.control.results
        if not results is None:
            results = results[-1][-1]
            if self.results != results or self.rdf is None:
                self.results = results
                self.rdf = RDF(results)
                e = np.unique(results.atoms.elements).tolist()
                if len(results.domains.centers) > 0 and not "cav" in e:
                    e.append("cav")
                self.elem1.clear()
                self.elem1.addItems(e)
                self.elem2.clear()
                self.elem2.addItems(e)
                self.gr_widget.setdata(None, None, None)
                self.gr_widget.draw()
            self.datasetlabel.setText(str(results))
        else:
            self.datasetlabel.setText("")
            self.elem1.clear()
            self.elem2.clear()
<commit_msg>fix bug in rdf_widget with no domains<commit_after># -*- coding: utf-8 -*-

from PySide import QtCore, QtGui
import os
from gr.pygr import *
try:
    from PySide import shiboken
except ImportError:
    import shiboken

from util.logger import Logger
import sys
from statistics.rdf import RDF
import numpy as np
import os

logger = Logger("gui.rdf_widget")
logger.setstream("default", sys.stdout, Logger.DEBUG)

class GrPlotWidget(QtGui.QWidget) :
    def __init__(self, *args) :
        QtGui.QWidget.__init__(self)

        self.init_gui(self)

        os.environ["GKS_WSTYPE"] = "381"
        os.environ["GKS_DOUBLE_BUF"] = "True"

        #self.connect(self.DrawButton, QtCore.SIGNAL("clicked()"), self.draw)
        #self.connect(self.QuitButton, QtCore.SIGNAL("clicked()"), self.quit)
        self.w = 500
        self.h = 500
        self.sizex = 1
        self.sizey = 1
        self.xvalues = None
        self.yvalues = None
        self.title = None
        self.datapoints = None

    def init_gui(self, form) :
        form.resize(QtCore.QSize(500, 500).expandedTo(form.minimumSizeHint()))

        #self.DrawButton = QtGui.QPushButton(form)
        #self.DrawButton.setText("Draw")
        #self.DrawButton.setGeometry(QtCore.QRect(290, 5, 100, 25))
        #self.DrawButton.setObjectName("draw")

        #self.QuitButton = QtGui.QPushButton(form)
        #self.QuitButton.setText("Quit")
        #self.QuitButton.setGeometry(QtCore.QRect(395, 5, 100, 25))
        #self.QuitButton.setObjectName("quit")

        QtCore.QMetaObject.connectSlotsByName(form)

    def quit(self) :
        gr.emergencyclosegks()
        self.close()

    def setdata(self, xvalues, yvalues, title, datapoints=None):
        self.xvalues = xvalues
        self.yvalues = yvalues
        self.title = title
        self.datapoints = datapoints

    def draw(self) :
        self.setStyleSheet("background-color:white;");

        #x = range(0, 128)
        #y = range(0, 128)
        #z = readfile(os.path.join(os.path.dirname(os.path.realpath(__file__)),
        #                          "kws.dat"), separator='$')
        #zrange = max(z) - min(z)
        #h = [min(z) + i * 0.025 * zrange for i in range(0, 40)]

        if not self.xvalues is None:
            rangex = (self.xvalues.min(), self.xvalues.max())
        else:
            rangex = (0, 10)
        if not self.yvalues is None:
            rangey = gr.adjustrange(self.yvalues.min(), self.yvalues.max())
        else:
            rangey = (0, 4)

        gr.clearws()
        mwidth  = self.w * 2.54 / self.logicalDpiX() / 100
        mheight = self.h * 2.54 / self.logicalDpiY() / 100
        gr.setwsviewport(0, mwidth, 0, mheight)
        gr.setwswindow(0, self.sizex, 0, self.sizey)
        gr.setviewport(0.075 * self.sizex, 0.95 * self.sizex, 0.075 * self.sizey, 0.95 * self.sizey)
        gr.setwindow(rangex[0], rangex[1], rangey[0], rangey[1])
        gr.setcharheight(0.012)

        gr.setfillintstyle(1)
        gr.setfillcolorind(0)
        gr.fillrect(rangex[0], rangex[1], rangey[0], rangey[1])

        if not self.xvalues is None and not self.yvalues is None:
            gr.setlinecolorind(2)
            gr.polyline(self.xvalues, self.yvalues)
        else:
            gr.text(0.4, 0.45, "no elements selected")

        ## rug plot
        #if not self.datapoints is None:
        #    gr.setmarkertype(gr.MARKERTYPE_SOLID_TRI_UP)
        #    gr.setmarkercolorind(2)
        #    gr.setmarkersize(1.0)
        #    gr.polymarker(self.datapoints, np.zeros_like(self.datapoints))

        gr.setlinecolorind(1)
        gr.axes(0.2, 0.2, rangex[0], rangey[0], 5, 5, 0.0075)
        gr.axes(0.2, 0.2, rangex[1], rangey[1], -5, -5, -0.0075)

        if not self.title is None:
            gr.text(0.75, 0.65, self.title)
        self.update()

    def resizeEvent(self, event):
        self.w = event.size().width()
        self.h = event.size().height()
        if self.w > self.h:
            self.sizex = 1
            self.sizey = float(self.h)/self.w
        else:
            self.sizex = float(self.w)/self.h
            self.sizey = 1
        self.draw()

    def paintEvent(self, ev) :
        self.painter = QtGui.QPainter()
        self.painter.begin(self)
        os.environ['GKSconid'] = "%x!%x" % (long(shiboken.getCppPointer(self)[0]), long(shiboken.getCppPointer(self.painter)[0]))
        gr.updatews()
        self.painter.end()

class RDFWidget(QtGui.QWidget):

    def __init__(self, parent):
        QtGui.QWidget.__init__(self, parent)

        self.control = parent.control
        self.results = None
        self.rdf = None

        self.init_gui()

    def init_gui(self):
        vbox = QtGui.QVBoxLayout()
        grid = QtGui.QGridLayout()
        self.gr_widget = GrPlotWidget()

        self.datasetlabel = QtGui.QLabel("No data loaded. Press 'Refresh'.", self)
        self.datasetlabel.setAlignment(QtCore.Qt.AlignHCenter)

        elembox = QtGui.QHBoxLayout()
        elembox.addWidget(QtGui.QLabel("Elements:", self), 0, 0)
        self.elem1 = QtGui.QComboBox(self)
        elembox.addWidget(self.elem1, 0, 1)
        self.elem2 = QtGui.QComboBox(self)
        elembox.addWidget(self.elem2, 0, 2)
        grid.addLayout(elembox, 0, 0)

        rangebox = QtGui.QHBoxLayout()
        rangebox.addWidget(QtGui.QLabel("Plot range:", self))
        self.range1 = QtGui.QLineEdit("0", self)
        rangebox.addWidget(self.range1)
        rangebox.addWidget(QtGui.QLabel("-", self))
        self.range2 = QtGui.QLineEdit("8", self)
        rangebox.addWidget(self.range2)
        grid.addLayout(rangebox, 0, 1)

        cutoffbox = QtGui.QHBoxLayout()
        cutoffbox.addWidget(QtGui.QLabel("Cutoff:", self))
        self.cutoff = QtGui.QLineEdit("12", self)
        cutoffbox.addWidget(self.cutoff)
        cutoffbox.addWidget(QtGui.QLabel("Bandwidth:", self))
        self.bandwidth = QtGui.QLineEdit("", self)
        cutoffbox.addWidget(self.bandwidth)
        grid.addLayout(cutoffbox, 0, 2)

        self.plotbutton = QtGui.QPushButton("Plot", self)
        grid.addWidget(self.plotbutton, 1, 0)
        self.connect(self.plotbutton, QtCore.SIGNAL("clicked()"), self.draw)

        self.exportbutton = QtGui.QPushButton("Save Image", self)
        grid.addWidget(self.exportbutton, 1, 1)
        self.connect(self.exportbutton, QtCore.SIGNAL("clicked()"), self.export)

        self.refreshbutton = QtGui.QPushButton("Refresh Data", self)
        grid.addWidget(self.refreshbutton, 1, 2)
        self.connect(self.refreshbutton, QtCore.SIGNAL("clicked()"), self.refresh)

        vbox.addWidget(self.gr_widget, stretch=1)
        vbox.addWidget(self.datasetlabel, stretch=0)
        vbox.addLayout(grid)
        self.setLayout(vbox)
        self.show()

    def keyPressEvent(self, event):
        if event.key() == QtCore.Qt.Key_Return \
                or event.key() == QtCore.Qt.Key_Enter:
            self.draw()

    def draw(self):
        xvalues = None
        yvalues = None
        title = None
        datapoints = None
        if self.rdf is None:
            self.refresh()
        if not self.rdf is None:
            elem1 = str(self.elem1.currentText())
            elem2 = str(self.elem2.currentText())
            range1 = float(str(self.range1.text()))
            range2 = float(str(self.range2.text()))
            cutoff = str(self.cutoff.text())
            if len(cutoff) > 0 and float(cutoff) > 0:
                cutoff = float(cutoff)
            else:
                cutoff = None
            bandwidth = str(self.bandwidth.text())
            if len(bandwidth) > 0 and float(bandwidth) > 0:
                bandwidth = float(bandwidth)
            else:
                bandwidth = None
            f = self.rdf.rdf(elem1, elem2, cutoff=cutoff, h=bandwidth)
            if not f is None:
                xvalues = np.linspace(range1, range2, 400)
                yvalues = f(xvalues)
                title = "{} - {}".format(elem1, elem2)
                datapoints = f.f.x

        self.gr_widget.setdata(xvalues, yvalues, title, datapoints)
        self.gr_widget.draw()

    def export(self):
        extensions = (".eps", ".ps", ".pdf", ".png", ".bmp", ".jpg", ".jpeg", ".png", ".tiff", ".fig", ".svg", ".wmf")
        qtext = "*" + " *".join(extensions)
        filepath, _ = QtGui.QFileDialog.getSaveFileName(self, "Save Image", \
                ".", "Image Files ({})".format(qtext))
        if len(filepath) == 0:
            return
        gr.beginprint(filepath)
        self.draw()
        gr.endprint()

    def refresh(self):
        results = self.control.results
        if not results is None:
            results = results[-1][-1]
            if self.results != results or self.rdf is None:
                self.results = results
                self.rdf = RDF(results)
                e = np.unique(results.atoms.elements).tolist()
                if not results.domains is None \
                        and len(results.domains.centers) > 0 \
                        and not "cav" in e:
                    e.append("cav")
                self.elem1.clear()
                self.elem1.addItems(e)
                self.elem2.clear()
                self.elem2.addItems(e)
                self.gr_widget.setdata(None, None, None)
                self.gr_widget.draw()
            self.datasetlabel.setText(str(results))
        else:
            self.datasetlabel.setText("")
            self.elem1.clear()
            self.elem2.clear()
<|endoftext|>


<commit_before># Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Classes for polling a git repository via the gitiles web interface.

The advantage of using gitiles is that a local clone is not needed."""

import datetime
import json
import os
import re
import traceback
import urllib
from urlparse import urlparse

import sqlalchemy as sa
import buildbot.status.web.base as base
from buildbot.status.web.console import RevisionComparator
from buildbot.changes.base import PollingChangeSource
from twisted.internet import defer
from twisted.python import log

from common.gerrit_agent import GerritAgent

LOG_TEMPLATE = '%s/+log/%s?format=JSON&n=%d'
DETAIL_TEMPLATE = '%s/+/%s?format=JSON'

def time_to_datetime(tm):
  tm_parts = tm.split()
  # Time stamps from gitiles sometimes have a UTC offset (e.g., -0800), and
  # sometimes not.  time.strptime() cannot parse UTC offsets, so if one is
  # present, strip it out and parse manually.
  timezone = None
  if len(tm_parts) == 6:
    tm = ' '.join(tm_parts[:-1])
    timezone = tm_parts[-1]
  dt = datetime.datetime.strptime(tm, "%a %b %d %H:%M:%S %Y")
  if timezone:
    m = re.match(r'([+-])(\d\d):?(\d\d)?', timezone)
    assert m, 'Could not parse time zone information from "%s"' % timezone
    timezone_delta = datetime.timedelta(
        hours=int(m.group(2)), minutes=int(m.group(3) or '0'))
    if m.group(1) == '-':
      dt += timezone_delta
    else:
      dt -= timezone_delta
  return dt

class Revision(object):
  """Commit wrapper that can be compared to other commits through a comparator.

  (See GitilesRevisionComparator.comparableRevision)
  """
  def __init__(self, comparator, commit):
    self.comparator = comparator
    self.commit = commit

  def __cmp__(self, other):
    assert self.comparator.initialized, "The comparator must be initialized"
    return self.comparator.tagcmp(self.commit, other.commit)

  def __repr__(self):
    return self.commit

class GitilesRevisionComparator(RevisionComparator):
  """Tracks the commit order of tags in a git repository."""

  def __init__(self):
    super(GitilesRevisionComparator, self).__init__()
    self.sha1_lookup = {}
    self.initialized = False
    self.initLock = defer.DeferredLock()

  @defer.inlineCallbacks
  def initialize(self, db):
    yield self.initLock.acquire()
    if self.initialized:
      self.initLock.release()
      return
    def db_thread_main(conn):
      changes_tbl = db.model.changes
      q = changes_tbl.select(order_by=[sa.asc(changes_tbl.c.changeid)])
      rp = conn.execute(q)
      for row in rp:
        self.addRevision(row.revision)
    try:
      yield db.pool.do(db_thread_main)
      log.msg(
          'GitilesRevisionComparator: Finished initializing revision history')
      self.initialized = True
    finally:
      self.initLock.release()

  def comparableRevision(self, commit):
    assert self.initialized, "The comparator must be initialized"
    return Revision(self, commit)

  def addRevision(self, revision):
    if revision in self.sha1_lookup:
      return
    idx = len(self.sha1_lookup)
    self.sha1_lookup[revision] = idx

  def tagcmp(self, x, y):
    return cmp(self.sha1_lookup[x], self.sha1_lookup[y])

  def isValidRevision(self, revision):
    return revision in self.sha1_lookup

  def isRevisionEarlier(self, first_change, second_change):
    return self.tagcmp(first_change.revision, second_change.revision) < 0

  def getSortingKey(self):
    return lambda c: self.sha1_lookup.__getitem__(c.revision)

class GitilesPoller(PollingChangeSource):
  """Polls a git repository using the gitiles web interface. """

  git_svn_id_re = re.compile(r'^git-svn-id: (.*)@([0-9]+) [0-9a-fA-F\-]*$')
  re_pattern_type = type(re.compile(''))

  def __init__(
      self, repo_url, branches=None, pollInterval=30, category=None,
      project=None, revlinktmpl=None, agent=None, svn_mode=False,
      svn_branch=None, change_filter=None, comparator=None):
    """Args:

    repo_url: URL of the gitiles service to be polled.
    branches: List of strings and/or compiled regular expressions, specifying
        the branches to be polled.
    pollInterval: Number of seconds between polling operations.
    category: Category to be applied to generated Change objects.
    project: Project to be applied to generated Change objects.
    revlinktmpl: String template, taking a single 'revision' parameter, used to
        generate a web link to a revision.
    agent: A GerritAgent object used to make requests to the gitiles service.
    svn_mode: When polling a mirror of an svn repository, create changes using
        the svn revision number.
    svn_branch: When svn_mode=True, this is used to determine the svn branch
        name for each change.  It can be either a static string, or a function
        that takes (gitiles_commit_json, git_branch) as arguments and returns
        a static string.
    comparator: A GitilesRevisionComparator object, or None.  This is used to
        share a single comparator between multiple pollers.
    """
    u = urlparse(repo_url)
    self.repo_url = repo_url
    self.repo_host = u.netloc
    self.repo_path = urllib.quote(u.path)
    if branches is None:
      branches = ['master']
    elif isinstance(branches, basestring):
      branches = [branches]
    self.branches = []
    for b in branches:
      if not isinstance(b, self.re_pattern_type):
        b = b.lstrip('/')
        if not b.startswith('refs/'):
          b = 'refs/heads/' + b
      self.branches.append(b)
    self.branch_heads = {}
    self.pollInterval = pollInterval
    self.category = category
    if project is None:
      project = os.path.basename(repo_url)
      if project.endswith('.git'):
        project = project[:-4]
    self.project = project
    self.revlinktmpl = revlinktmpl or '%s/+/%%s' % repo_url
    self.svn_mode = svn_mode
    self.svn_branch = svn_branch
    if svn_mode and not svn_branch:
      self.svn_branch = project
    if agent is None:
      agent = GerritAgent('%s://%s' % (u.scheme, u.netloc), read_only=True)
    self.agent = agent
    self.dry_run = os.environ.get('POLLER_DRY_RUN')
    self.change_filter = change_filter
    self.comparator = comparator or GitilesRevisionComparator()

  @defer.inlineCallbacks
  def startService(self):
    # Initialize revision comparator with revisions from all changes
    # known to buildbot.
    yield self.comparator.initialize(self.master.db)

    # Get the head commit for each branch being polled.
    branches = yield self._get_branches()
    for branch, branch_head in branches.iteritems():
      log.msg('GitilesPoller: Initial revision for branch %s is %s' % (
          branch, branch_head))
      self.branch_heads[branch] = branch_head

    PollingChangeSource.startService(self)

  @defer.inlineCallbacks
  def _get_branches(self):
    result = {}
    if self.dry_run:
      refs = []
    elif any([isinstance(b, self.re_pattern_type) for b in self.branches]):
      refs = ['refs']
    else:
      refs = self.branches

    refs_json = {}
    for ref in refs:
      path = DETAIL_TEMPLATE % (self.repo_path, ref)
      more_refs_json = yield self.agent.request('GET', path, retry=5)
      refs_json.update(more_refs_json)

    for ref, ref_head in refs_json.iteritems():
      for branch in self.branches:
        if (ref == branch or
            (isinstance(branch, self.re_pattern_type) and
             (branch.match(ref) or
              (ref.startswith('refs/heads/') and branch.match(ref[11:]))))):
          result[ref] = ref_head['value']
          break
    deleted_branches = []
    for branch in self.branch_heads:
      if branch not in result:
        deleted_branches.append(branch)
    for branch in deleted_branches:
      log.msg('GitilesPoller: Deleting branch head for %s' % (branch,))
      del self.branch_heads[branch]
    defer.returnValue(result)

  def _create_change(self, commit_json, branch):
    """Send a new Change object to the buildbot master."""
    if not commit_json:
      return
    if self.change_filter and not self.change_filter(commit_json, branch):
      return
    commit_branch = branch.rpartition('/')[2]
    if callable(self.svn_branch):
      commit_branch = self.svn_branch(commit_json, branch)
    elif self.svn_branch:
      commit_branch = self.svn_branch
    commit_author = commit_json['author']['email']
    commit_tm = time_to_datetime(commit_json['committer']['time'])
    commit_files = []
    if 'tree_diff' in commit_json:
      for diff_entry in commit_json['tree_diff']:
        path = diff_entry.get('new_path')
        if path == '/dev/null' and 'old_path' in diff_entry:
          path = diff_entry['old_path']
        if path:
          commit_files.append(path)
    commit_msg = commit_json['message']
    repo_url = self.repo_url
    revision = commit_json['commit']
    properties = {'git_revision': revision}
    if self.svn_mode:
      revision = None
      for line in reversed(commit_msg.splitlines()):
        m = self.git_svn_id_re.match(line)
        if m:
          repo_url = m.group(1)
          revision = m.group(2)
          break
      if revision is None:
        log.err(
            'GitilesPoller: Could not parse svn revision out of commit message '
            'for commit %s in %s' % (commit_json['commit'], self.repo_url))
        return None
      self.comparator.addRevision(revision)
    revlink = ''
    if self.revlinktmpl and revision:
      revlink = self.revlinktmpl % revision
    return self.master.addChange(
        author=commit_author,
        revision=revision,
        files=commit_files,
        comments=commit_msg,
        when_timestamp=commit_tm,
        branch=commit_branch,
        category=self.category,
        project=self.project,
        properties=properties,
        repository=repo_url,
        revlink=revlink)

  @defer.inlineCallbacks
  def _fetch_new_commits(self, branch, since):
    """Query gitiles for all commits on 'branch' more recent than 'since'."""
    result = []
    log_json = {'next': branch}
    while 'next' in log_json:
      log_spec = '%s..%s' % (since, log_json['next'])
      path = LOG_TEMPLATE % (self.repo_path, log_spec, 100)
      if self.dry_run:
        log_json = {}
      else:
        log_json = yield self.agent.request('GET', path, retry=5)
      if log_json.get('log'):
        result.extend(log_json['log'])
    result.reverse()
    defer.returnValue(result)

  @staticmethod
  def _collate_commits(a, b):
    """Shuffle together two lists of commits.

    The result will be sorted by commit time, while guaranteeing that there are
    no inversions compared to the argument lists."""
    a = [(c, time_to_datetime(c[0]['committer']['time'])) for c in a]
    b = [(c, time_to_datetime(c[0]['committer']['time'])) for c in b]
    result = []
    while a and b:
      if a[-1][1] > b[-1][1]:
        result.append(a.pop()[0])
      else:
        result.append(b.pop()[0])
    while a:
      result.append(a.pop()[0])
    while b:
      result.append(b.pop()[0])
    result.reverse()
    return result

  @defer.inlineCallbacks
  def poll(self):
    all_commits = []
    branches = yield self._get_branches()
    for branch, branch_head in branches.iteritems():
      try:
        branch_commits = None
        if branch not in self.branch_heads:
          # New branch: trigger a build based on the current head.
          log.msg('GitilesPoller: Discovered new branch %s.' % branch)
          branch_commits = yield self._fetch_new_commits(
              branch, branch + '~')
        elif self.branch_heads[branch] != branch_head:
          branch_commits = yield self._fetch_new_commits(
              branch, self.branch_heads[branch])
        if branch_commits:
          self.branch_heads[branch] = branch_commits[-1]['commit']
          branch_commits = [(c, branch) for c in branch_commits]
          all_commits = self._collate_commits(all_commits, branch_commits)
      except Exception:
        msg = ('GitilesPoller: Error while fetching logs for branch %s:\n%s' %
                   (branch, traceback.format_exc()))
        log.err(msg)
    for commit in all_commits:
      commit, branch = commit
      if not self.svn_mode:
        self.comparator.addRevision(commit['commit'])
      try:
        path = DETAIL_TEMPLATE % (self.repo_path, commit['commit'])
        if not self.dry_run:
          detail = yield self.agent.request('GET', path, retry=5)
          yield self._create_change(detail, branch)
      except Exception:
        msg = ('GitilesPoller: Error while processing revision %s '
               'on branch %s:\n%s' % (
                   commit['commit'], branch, traceback.format_exc()))
        log.err(msg)
    if not all_commits:
      log.msg('GitilesPoller: No new commits.')

  def describe(self):
    status = self.__class__.__name__
    if not self.master:
      status += ' [STOPPED - check log]'
    return '%s repo_url=%s' % (status, self.repo_url)

class GitilesStatus(base.HtmlResource):
  """This provides, in JSON, data about the specified GitilesPoller."""
  contentType = "application/json"

  def __init__(self, poller):
    self.poller = poller
    super(GitilesStatus, self).__init__()

  def content(self, request, ctx):
    data = {
        'branch_heads': self.poller.branch_heads,
    }
    return json.dumps(data)
<commit_msg>Revert of Don't fetch all refs from gitiles if it can be avoided. (patchset #2 id:20001 of https://codereview.chromium.org/778433002/)<commit_after># Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Classes for polling a git repository via the gitiles web interface.

The advantage of using gitiles is that a local clone is not needed."""

import datetime
import json
import os
import re
import traceback
import urllib
from urlparse import urlparse

import sqlalchemy as sa
import buildbot.status.web.base as base
from buildbot.status.web.console import RevisionComparator
from buildbot.changes.base import PollingChangeSource
from twisted.internet import defer
from twisted.python import log

from common.gerrit_agent import GerritAgent

LOG_TEMPLATE = '%s/+log/%s?format=JSON&n=%d'
REFS_TEMPLATE = '%s/+refs?format=JSON'
REVISION_DETAIL_TEMPLATE = '%s/+/%s?format=JSON'

def time_to_datetime(tm):
  tm_parts = tm.split()
  # Time stamps from gitiles sometimes have a UTC offset (e.g., -0800), and
  # sometimes not.  time.strptime() cannot parse UTC offsets, so if one is
  # present, strip it out and parse manually.
  timezone = None
  if len(tm_parts) == 6:
    tm = ' '.join(tm_parts[:-1])
    timezone = tm_parts[-1]
  dt = datetime.datetime.strptime(tm, "%a %b %d %H:%M:%S %Y")
  if timezone:
    m = re.match(r'([+-])(\d\d):?(\d\d)?', timezone)
    assert m, 'Could not parse time zone information from "%s"' % timezone
    timezone_delta = datetime.timedelta(
        hours=int(m.group(2)), minutes=int(m.group(3) or '0'))
    if m.group(1) == '-':
      dt += timezone_delta
    else:
      dt -= timezone_delta
  return dt

class Revision(object):
  """Commit wrapper that can be compared to other commits through a comparator.

  (See GitilesRevisionComparator.comparableRevision)
  """
  def __init__(self, comparator, commit):
    self.comparator = comparator
    self.commit = commit

  def __cmp__(self, other):
    assert self.comparator.initialized, "The comparator must be initialized"
    return self.comparator.tagcmp(self.commit, other.commit)

  def __repr__(self):
    return self.commit

class GitilesRevisionComparator(RevisionComparator):
  """Tracks the commit order of tags in a git repository."""

  def __init__(self):
    super(GitilesRevisionComparator, self).__init__()
    self.sha1_lookup = {}
    self.initialized = False
    self.initLock = defer.DeferredLock()

  @defer.inlineCallbacks
  def initialize(self, db):
    yield self.initLock.acquire()
    if self.initialized:
      self.initLock.release()
      return
    def db_thread_main(conn):
      changes_tbl = db.model.changes
      q = changes_tbl.select(order_by=[sa.asc(changes_tbl.c.changeid)])
      rp = conn.execute(q)
      for row in rp:
        self.addRevision(row.revision)
    try:
      yield db.pool.do(db_thread_main)
      log.msg(
          'GitilesRevisionComparator: Finished initializing revision history')
      self.initialized = True
    finally:
      self.initLock.release()

  def comparableRevision(self, commit):
    assert self.initialized, "The comparator must be initialized"
    return Revision(self, commit)

  def addRevision(self, revision):
    if revision in self.sha1_lookup:
      return
    idx = len(self.sha1_lookup)
    self.sha1_lookup[revision] = idx

  def tagcmp(self, x, y):
    return cmp(self.sha1_lookup[x], self.sha1_lookup[y])

  def isValidRevision(self, revision):
    return revision in self.sha1_lookup

  def isRevisionEarlier(self, first_change, second_change):
    return self.tagcmp(first_change.revision, second_change.revision) < 0

  def getSortingKey(self):
    return lambda c: self.sha1_lookup.__getitem__(c.revision)

class GitilesPoller(PollingChangeSource):
  """Polls a git repository using the gitiles web interface. """

  git_svn_id_re = re.compile(r'^git-svn-id: (.*)@([0-9]+) [0-9a-fA-F\-]*$')
  re_pattern_type = type(re.compile(''))

  def __init__(
      self, repo_url, branches=None, pollInterval=30, category=None,
      project=None, revlinktmpl=None, agent=None, svn_mode=False,
      svn_branch=None, change_filter=None, comparator=None):
    """Args:

    repo_url: URL of the gitiles service to be polled.
    branches: List of strings and/or compiled regular expressions, specifying
        the branches to be polled.
    pollInterval: Number of seconds between polling operations.
    category: Category to be applied to generated Change objects.
    project: Project to be applied to generated Change objects.
    revlinktmpl: String template, taking a single 'revision' parameter, used to
        generate a web link to a revision.
    agent: A GerritAgent object used to make requests to the gitiles service.
    svn_mode: When polling a mirror of an svn repository, create changes using
        the svn revision number.
    svn_branch: When svn_mode=True, this is used to determine the svn branch
        name for each change.  It can be either a static string, or a function
        that takes (gitiles_commit_json, git_branch) as arguments and returns
        a static string.
    comparator: A GitilesRevisionComparator object, or None.  This is used to
        share a single comparator between multiple pollers.
    """
    u = urlparse(repo_url)
    self.repo_url = repo_url
    self.repo_host = u.netloc
    self.repo_path = urllib.quote(u.path)
    if branches is None:
      branches = ['master']
    elif isinstance(branches, basestring):
      branches = [branches]
    self.branches = []
    for b in branches:
      if not isinstance(b, self.re_pattern_type):
        b = b.lstrip('/')
        if not b.startswith('refs/'):
          b = 'refs/heads/' + b
      self.branches.append(b)
    self.branch_heads = {}
    self.pollInterval = pollInterval
    self.category = category
    if project is None:
      project = os.path.basename(repo_url)
      if project.endswith('.git'):
        project = project[:-4]
    self.project = project
    self.revlinktmpl = revlinktmpl or '%s/+/%%s' % repo_url
    self.svn_mode = svn_mode
    self.svn_branch = svn_branch
    if svn_mode and not svn_branch:
      self.svn_branch = project
    if agent is None:
      agent = GerritAgent('%s://%s' % (u.scheme, u.netloc), read_only=True)
    self.agent = agent
    self.dry_run = os.environ.get('POLLER_DRY_RUN')
    self.change_filter = change_filter
    self.comparator = comparator or GitilesRevisionComparator()

  @defer.inlineCallbacks
  def startService(self):
    # Initialize revision comparator with revisions from all changes
    # known to buildbot.
    yield self.comparator.initialize(self.master.db)

    # Get the head commit for each branch being polled.
    branches = yield self._get_branches()
    for branch, branch_head in branches.iteritems():
      log.msg('GitilesPoller: Initial revision for branch %s is %s' % (
          branch, branch_head))
      self.branch_heads[branch] = branch_head

    PollingChangeSource.startService(self)

  @defer.inlineCallbacks
  def _get_branches(self):
    result = {}
    path = REFS_TEMPLATE % (self.repo_path,)
    if self.dry_run:
      refs_json = {}
    else:
      refs_json = yield self.agent.request('GET', path, retry=5)
    for ref, ref_head in refs_json.iteritems():
      for branch in self.branches:
        if (ref == branch or
            (isinstance(branch, self.re_pattern_type) and
             (branch.match(ref) or
              (ref.startswith('refs/heads/') and branch.match(ref[11:]))))):
          result[ref] = ref_head['value']
          break
    deleted_branches = []
    for branch in self.branch_heads:
      if branch not in result:
        deleted_branches.append(branch)
    for branch in deleted_branches:
      log.msg('GitilesPoller: Deleting branch head for %s' % (branch,))
      del self.branch_heads[branch]
    defer.returnValue(result)

  def _create_change(self, commit_json, branch):
    """Send a new Change object to the buildbot master."""
    if not commit_json:
      return
    if self.change_filter and not self.change_filter(commit_json, branch):
      return
    commit_branch = branch.rpartition('/')[2]
    if callable(self.svn_branch):
      commit_branch = self.svn_branch(commit_json, branch)
    elif self.svn_branch:
      commit_branch = self.svn_branch
    commit_author = commit_json['author']['email']
    commit_tm = time_to_datetime(commit_json['committer']['time'])
    commit_files = []
    if 'tree_diff' in commit_json:
      for diff_entry in commit_json['tree_diff']:
        path = diff_entry.get('new_path')
        if path == '/dev/null' and 'old_path' in diff_entry:
          path = diff_entry['old_path']
        if path:
          commit_files.append(path)
    commit_msg = commit_json['message']
    repo_url = self.repo_url
    revision = commit_json['commit']
    properties = {'git_revision': revision}
    if self.svn_mode:
      revision = None
      for line in reversed(commit_msg.splitlines()):
        m = self.git_svn_id_re.match(line)
        if m:
          repo_url = m.group(1)
          revision = m.group(2)
          break
      if revision is None:
        log.err(
            'GitilesPoller: Could not parse svn revision out of commit message '
            'for commit %s in %s' % (commit_json['commit'], self.repo_url))
        return None
      self.comparator.addRevision(revision)
    revlink = ''
    if self.revlinktmpl and revision:
      revlink = self.revlinktmpl % revision
    return self.master.addChange(
        author=commit_author,
        revision=revision,
        files=commit_files,
        comments=commit_msg,
        when_timestamp=commit_tm,
        branch=commit_branch,
        category=self.category,
        project=self.project,
        properties=properties,
        repository=repo_url,
        revlink=revlink)

  @defer.inlineCallbacks
  def _fetch_new_commits(self, branch, since):
    """Query gitiles for all commits on 'branch' more recent than 'since'."""
    result = []
    log_json = {'next': branch}
    while 'next' in log_json:
      log_spec = '%s..%s' % (since, log_json['next'])
      path = LOG_TEMPLATE % (self.repo_path, log_spec, 100)
      if self.dry_run:
        log_json = {}
      else:
        log_json = yield self.agent.request('GET', path, retry=5)
      if log_json.get('log'):
        result.extend(log_json['log'])
    result.reverse()
    defer.returnValue(result)

  @staticmethod
  def _collate_commits(a, b):
    """Shuffle together two lists of commits.

    The result will be sorted by commit time, while guaranteeing that there are
    no inversions compared to the argument lists."""
    a = [(c, time_to_datetime(c[0]['committer']['time'])) for c in a]
    b = [(c, time_to_datetime(c[0]['committer']['time'])) for c in b]
    result = []
    while a and b:
      if a[-1][1] > b[-1][1]:
        result.append(a.pop()[0])
      else:
        result.append(b.pop()[0])
    while a:
      result.append(a.pop()[0])
    while b:
      result.append(b.pop()[0])
    result.reverse()
    return result

  @defer.inlineCallbacks
  def poll(self):
    all_commits = []
    branches = yield self._get_branches()
    for branch, branch_head in branches.iteritems():
      try:
        branch_commits = None
        if branch not in self.branch_heads:
          # New branch: trigger a build based on the current head.
          log.msg('GitilesPoller: Discovered new branch %s.' % branch)
          branch_commits = yield self._fetch_new_commits(
              branch, branch + '~')
        elif self.branch_heads[branch] != branch_head:
          branch_commits = yield self._fetch_new_commits(
              branch, self.branch_heads[branch])
        if branch_commits:
          self.branch_heads[branch] = branch_commits[-1]['commit']
          branch_commits = [(c, branch) for c in branch_commits]
          all_commits = self._collate_commits(all_commits, branch_commits)
      except Exception:
        msg = ('GitilesPoller: Error while fetching logs for branch %s:\n%s' %
                   (branch, traceback.format_exc()))
        log.err(msg)
    for commit in all_commits:
      commit, branch = commit
      if not self.svn_mode:
        self.comparator.addRevision(commit['commit'])
      try:
        path = REVISION_DETAIL_TEMPLATE % (self.repo_path, commit['commit'])
        if not self.dry_run:
          detail = yield self.agent.request('GET', path, retry=5)
          yield self._create_change(detail, branch)
      except Exception:
        msg = ('GitilesPoller: Error while processing revision %s '
               'on branch %s:\n%s' % (
                   commit['commit'], branch, traceback.format_exc()))
        log.err(msg)
    if not all_commits:
      log.msg('GitilesPoller: No new commits.')

  def describe(self):
    status = self.__class__.__name__
    if not self.master:
      status += ' [STOPPED - check log]'
    return '%s repo_url=%s' % (status, self.repo_url)

class GitilesStatus(base.HtmlResource):
  """This provides, in JSON, data about the specified GitilesPoller."""
  contentType = "application/json"

  def __init__(self, poller):
    self.poller = poller
    super(GitilesStatus, self).__init__()

  def content(self, request, ctx):
    data = {
        'branch_heads': self.poller.branch_heads,
    }
    return json.dumps(data)
<|endoftext|>


<commit_before># Copyright 2019 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This file is autogenerated by copybara, please do not edit.

buildtools_version = "f2aed9ee205d62d45c55cfabbfd26342f8526862"
buildtools_sha256 = "7a182df18df1debabd9e36ae07c8edfa1378b8424a04561b674d933b965372b3"
<commit_msg>Update dependency bazelbuild/buildtools to latest version<commit_after># Copyright 2019 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This file is autogenerated by copybara, please do not edit.

buildtools_version = "2b0a1af1a898f92183b9a30872525d9f8332a805"
buildtools_sha256 = "f6b9b5da74854afb2a68d74a9a3d20c5a1d5b8c11046d24644960049d77121bb"
<|endoftext|>


<commit_before>import sys
from os.path import split

from PyQt5.QtCore import QSettings, QPoint, QSize, pyqtSlot
from PyQt5.QtWidgets import QApplication, QFileDialog, QFontDialog, QMainWindow

from .AboutDialog import AboutDialog
from .MenuBar import MenuBar
from .TabBar import TabBar

class Editor(QMainWindow):
    def __init__(self):
        super().__init__()
        self.readWindowSettings()

        self.aboutDialog = AboutDialog()

        self.menuBar = MenuBar()
        self.setMenuBar(self.menuBar)

        self.tabBar = TabBar()
        self.readTabBarSettings()
        self.setCentralWidget(self.tabBar)

        self.changeFont(self.font())

        self.statusBar = self.statusBar()
        # self.updateStatusBarText()

        self.configureSignals()

        self.show()

    def readWindowSettings(self):
        settings = QSettings("msklosak", "EasyEdit")

        settings.beginGroup("Editor")
        self.resize(settings.value("size", QSize(600, 800)))
        self.move(settings.value("pos", QPoint(0, 0)))
        self.setFont(settings.value("font", self.font()))
        settings.endGroup()

    def writeWindowSettings(self):
        settings = QSettings("msklosak", "EasyEdit")

        settings.beginGroup("Editor")
        settings.setValue("size", self.size())
        settings.setValue("pos", self.pos())
        settings.setValue("font", self.font())
        settings.endGroup()

    def readTabBarSettings(self):
        settings = QSettings("msklosak", "EasyEdit")
        settings.beginGroup("Tab Bar")

        savedTabs = settings.value("openedTabs")
        tabLexers = settings.value("tabLexers")
        if savedTabs is None:
            self.tabBar.openTab()
        else:
            for i in range(len(savedTabs)):
                if savedTabs[i] != "Untitled":
                    self.tabBar.openTab()
                    self.openFile(savedTabs[i])
                    self.tabBar.widget(i).changeLexer(tabLexers[i])

        self.tabBar.setCurrentIndex(int(settings.value("currentTab", 0)))

        self.updateWindowTitle()

        settings.endGroup()

    def writeTabBarSettings(self):
        openTabs = []
        tabLexers = []

        for i in range(self.tabBar.count()):
            openTabs.append(self.tabBar.widget(i).filePath)
            tabLexers.append(self.tabBar.widget(i).currentLanguage)

        settings = QSettings("msklosak", "EasyEdit")
        settings.beginGroup("Tab Bar")
        settings.setValue("openedTabs", openTabs)
        settings.setValue("tabLexers", tabLexers)
        settings.setValue("currentTab", self.tabBar.currentIndex())
        settings.endGroup()

    def closeEvent(self, event):
        self.writeTabBarSettings()
        self.writeWindowSettings()

        while self.tabBar.count() > 0:
            self.tabBar.closeTab(0)

    def configureSignals(self):
        # FILE TAB
        self.menuBar.newFile.connect(self.tabBar.openTab)
        self.menuBar.openFile.connect(self.openFileDialog)
        self.menuBar.saveFile.connect(self.saveFile)
        self.menuBar.saveFileAs.connect(self.saveFile)

        # EDIT TAB
        self.menuBar.undoEdit.connect(self.tabBar.currentWidget().undo)
        self.menuBar.redoEdit.connect(self.tabBar.currentWidget().redo)
        self.menuBar.cutText.connect(self.tabBar.currentWidget().cut)
        self.menuBar.copyText.connect(self.tabBar.currentWidget().copy)
        self.menuBar.pasteText.connect(self.tabBar.currentWidget().paste)

        # SETTINGS TAB
        self.menuBar.changeFont.connect(self.changeFontDialog)

        # HELP TAB
        self.menuBar.openAboutDialog.connect(lambda: self.aboutDialog.exec_())

        # TAB BAR
        self.tabBar.currentChanged.connect(self.tabChanged)

        # TEXT AREA
        self.menuBar.changeLanguage.connect(self.tabBar.currentWidget().changeLexer)
        # self.tabBar.currentWidget().cursorPositionChanged.connect(self.updateStatusBarText)

    def changeFont(self, newFont):
        self.setFont(newFont)

        for i in range(self.tabBar.count()):
            self.tabBar.widget(i).updateFont(newFont)

    def changeFontDialog(self):
        font = QFontDialog().getFont()[0]

        self.changeFont(font)

    def openFile(self, fileName):
        with open(fileName, 'r') as file:
            self.tabBar.currentWidget().setText(file.read())

        shortenedFileName = split(fileName)[1]

        self.tabBar.currentWidget().filePath = fileName
        self.tabBar.setTabText(self.tabBar.currentIndex(), shortenedFileName)
        self.tabBar.currentWidget().changeMarginWidth()
        self.updateWindowTitle()

    def openFileDialog(self):
        fileName = QFileDialog.getOpenFileName(self, "Open File")[0]

        if fileName != "":
            self.openFile(fileName)

    def saveFile(self):
        if self.tabBar.currentWidget().filePath != "Untitled":
            fileName = self.tabBar.currentWidget().filePath
        else:
            fileName = QFileDialog.getSaveFileName(self, "Save File", None, "Text Files (*.txt);;All Files (*)")[0]

            if fileName != "":
                shortenedFileName = split(fileName)[1]

                self.tabBar.setTabText(self.tabBar.currentIndex(), shortenedFileName)
                self.updateWindowTitle()

        if fileName != "":
            text = self.tabBar.currentWidget().text()

            with open(fileName, 'w') as file:
                file.write(text)

        self.changeFont(self.font())

    def tabChanged(self):
        if self.tabBar.count() > 1:
            # self.tabBar.currentWidget().cursorPositionChanged.connect(self.updateStatusBarText)

            self.updateWindowTitle()

    def updateWindowTitle(self):
        self.setWindowTitle(self.tabBar.tabText(self.tabBar.currentIndex()) + " - EasyEdit")

    @pyqtSlot(int, int)
    def updateStatusBarText(self, line, column):
        self.statusBar.showMessage("Line {}, Column {}".format(line, column))
<commit_msg>Remove unneeded imports from Editor<commit_after>from os.path import split

from PyQt5.QtCore import QSettings, QPoint, QSize, pyqtSlot
from PyQt5.QtWidgets import QFileDialog, QFontDialog, QMainWindow

from .AboutDialog import AboutDialog
from .MenuBar import MenuBar
from .TabBar import TabBar

class Editor(QMainWindow):
    def __init__(self):
        super().__init__()
        self.readWindowSettings()

        self.aboutDialog = AboutDialog()

        self.menuBar = MenuBar()
        self.setMenuBar(self.menuBar)

        self.tabBar = TabBar()
        self.readTabBarSettings()
        self.setCentralWidget(self.tabBar)

        self.changeFont(self.font())

        self.statusBar = self.statusBar()
        # self.updateStatusBarText()

        self.configureSignals()

        self.show()

    def readWindowSettings(self):
        settings = QSettings("msklosak", "EasyEdit")

        settings.beginGroup("Editor")
        self.resize(settings.value("size", QSize(600, 800)))
        self.move(settings.value("pos", QPoint(0, 0)))
        self.setFont(settings.value("font", self.font()))
        settings.endGroup()

    def writeWindowSettings(self):
        settings = QSettings("msklosak", "EasyEdit")

        settings.beginGroup("Editor")
        settings.setValue("size", self.size())
        settings.setValue("pos", self.pos())
        settings.setValue("font", self.font())
        settings.endGroup()

    def readTabBarSettings(self):
        settings = QSettings("msklosak", "EasyEdit")
        settings.beginGroup("Tab Bar")

        savedTabs = settings.value("openedTabs")
        tabLexers = settings.value("tabLexers")
        if savedTabs is None:
            self.tabBar.openTab()
        else:
            for i in range(len(savedTabs)):
                if savedTabs[i] != "Untitled":
                    self.tabBar.openTab()
                    self.openFile(savedTabs[i])
                    self.tabBar.widget(i).changeLexer(tabLexers[i])

        self.tabBar.setCurrentIndex(int(settings.value("currentTab", 0)))

        self.updateWindowTitle()

        settings.endGroup()

    def writeTabBarSettings(self):
        openTabs = []
        tabLexers = []

        for i in range(self.tabBar.count()):
            openTabs.append(self.tabBar.widget(i).filePath)
            tabLexers.append(self.tabBar.widget(i).currentLanguage)

        settings = QSettings("msklosak", "EasyEdit")
        settings.beginGroup("Tab Bar")
        settings.setValue("openedTabs", openTabs)
        settings.setValue("tabLexers", tabLexers)
        settings.setValue("currentTab", self.tabBar.currentIndex())
        settings.endGroup()

    def closeEvent(self, event):
        self.writeTabBarSettings()
        self.writeWindowSettings()

        while self.tabBar.count() > 0:
            self.tabBar.closeTab(0)

    def configureSignals(self):
        # FILE TAB
        self.menuBar.newFile.connect(self.tabBar.openTab)
        self.menuBar.openFile.connect(self.openFileDialog)
        self.menuBar.saveFile.connect(self.saveFile)
        self.menuBar.saveFileAs.connect(self.saveFile)

        # EDIT TAB
        self.menuBar.undoEdit.connect(self.tabBar.currentWidget().undo)
        self.menuBar.redoEdit.connect(self.tabBar.currentWidget().redo)
        self.menuBar.cutText.connect(self.tabBar.currentWidget().cut)
        self.menuBar.copyText.connect(self.tabBar.currentWidget().copy)
        self.menuBar.pasteText.connect(self.tabBar.currentWidget().paste)

        # SETTINGS TAB
        self.menuBar.changeFont.connect(self.changeFontDialog)

        # HELP TAB
        self.menuBar.openAboutDialog.connect(lambda: self.aboutDialog.exec_())

        # TAB BAR
        self.tabBar.currentChanged.connect(self.tabChanged)

        # TEXT AREA
        self.menuBar.changeLanguage.connect(self.tabBar.currentWidget().changeLexer)
        # self.tabBar.currentWidget().cursorPositionChanged.connect(self.updateStatusBarText)

    def changeFont(self, newFont):
        self.setFont(newFont)

        for i in range(self.tabBar.count()):
            self.tabBar.widget(i).updateFont(newFont)

    def changeFontDialog(self):
        font = QFontDialog().getFont()[0]

        self.changeFont(font)

    def openFile(self, fileName):
        with open(fileName, 'r') as file:
            self.tabBar.currentWidget().setText(file.read())

        shortenedFileName = split(fileName)[1]

        self.tabBar.currentWidget().filePath = fileName
        self.tabBar.setTabText(self.tabBar.currentIndex(), shortenedFileName)
        self.tabBar.currentWidget().changeMarginWidth()
        self.updateWindowTitle()

    def openFileDialog(self):
        fileName = QFileDialog.getOpenFileName(self, "Open File")[0]

        if fileName != "":
            self.openFile(fileName)

    def saveFile(self):
        if self.tabBar.currentWidget().filePath != "Untitled":
            fileName = self.tabBar.currentWidget().filePath
        else:
            fileName = QFileDialog.getSaveFileName(self, "Save File", None, "Text Files (*.txt);;All Files (*)")[0]

            if fileName != "":
                shortenedFileName = split(fileName)[1]

                self.tabBar.setTabText(self.tabBar.currentIndex(), shortenedFileName)
                self.updateWindowTitle()

        if fileName != "":
            text = self.tabBar.currentWidget().text()

            with open(fileName, 'w') as file:
                file.write(text)

        self.changeFont(self.font())

    def tabChanged(self):
        if self.tabBar.count() > 1:
            # self.tabBar.currentWidget().cursorPositionChanged.connect(self.updateStatusBarText)

            self.updateWindowTitle()

    def updateWindowTitle(self):
        self.setWindowTitle(self.tabBar.tabText(self.tabBar.currentIndex()) + " - EasyEdit")

    @pyqtSlot(int, int)
    def updateStatusBarText(self, line, column):
        self.statusBar.showMessage("Line {}, Column {}".format(line, column))
<|endoftext|>


<commit_before>from __future__ import absolute_import
import numpy as np
from . import utils
from . import nmf
from . import mdl
from . import compression

def bicluster(array, n=None, share_elements=True, comp_level=None):
    if n is None:
        n_iters = array.shape[1]
        online_mdl = mdl.OnlineMDL()
    else:
        n_iters = n

    downdater = utils.Downdater(array)

    bic_list = []
    total_codelength = []
    for _ in range(n_iters):
        if downdater.array.nnz == 0:
            break

        u, v = single_bicluster(downdater.array, comp_level=comp_level)

        if u.nnz <= 1 or v.nnz <= 1:
            break

        bic_list.append((u, v))

        idx_v = utils.find(v)[1]
        downdater.remove_columns(idx_v)
        if not share_elements:
            idx_u = utils.find(u)[0]
            downdater.remove_rows(idx_u)

        if n is None:
            cl = online_mdl.add_rank1_approximation(downdater.array, u, v)
            total_codelength.append(cl)

    if n is None and bic_list:
        cut_point = np.argmin(np.array(total_codelength))
        bic_list = bic_list[:cut_point+1]

    return bic_list

def single_bicluster(array, comp_level=None):
    if comp_level is not None:
        array_nmf = compression.compress_columns(array, comp_level)
    else:
        array_nmf = array
    u, v = nmf.nmf_robust_multiplicative(array_nmf, 1)
    u = utils.binarize(u)

    idx_u = utils.find(u)[0]
    array_crop, u_crop, v_init = crop_left(array, idx_u, comp_level=comp_level)
    v = nmf.nmf_robust_admm(array_crop, u_init=u_crop, v_init=v_init,
                            update='right')
    v = utils.binarize(v)

    idx_v = utils.find(v)[1]
    array_crop, u_init, v_crop = crop_right(array, idx_v, comp_level=comp_level)
    u = nmf.nmf_robust_admm(array_crop, u_init=u_init, v_init=v_crop,
                            update='left')
    u = utils.binarize(u)
    return u, v

def crop_left(array, idx_u, comp_level=None):
    array_crop = utils.sparse(array[idx_u, :])

    if comp_level is not None and comp_level < array_crop.shape[0]:
        array_crop = compression.compress_rows(array_crop, comp_level)

    u_crop = utils.sparse(np.ones((array_crop.shape[0], 1)))
    v_init = u_crop.T.dot(array_crop)
    arr_sum = array_crop.sum(axis=0)
    v_init[arr_sum > 0] /= arr_sum[arr_sum > 0]
    return array_crop, u_crop, v_init

def crop_right(array, idx_v, comp_level=None):
    array_crop = utils.sparse(array[:, idx_v])

    if comp_level is not None and comp_level < array_crop.shape[1]:
        array_crop = compression.compress_columns(array_crop, comp_level)

    v_crop = utils.sparse(np.ones((1, array_crop.shape[1])))
    u_init = array_crop.dot(v_crop.T)
    arr_sum = array_crop.sum(axis=1)
    u_init[arr_sum > 0] /= arr_sum[arr_sum > 0]
    return array_crop, u_init, v_crop
<commit_msg>Code cleanup<commit_after>from __future__ import absolute_import
import numpy as np
from . import utils
from . import nmf
from . import mdl
from . import compression

def bicluster(array, n=None, share_elements=True, comp_level=None):
    if n is None:
        n_iters = array.shape[1]
        online_mdl = mdl.OnlineMDL()
    else:
        n_iters = n

    downdater = utils.Downdater(array)

    bic_list = []
    total_codelength = []
    for _ in range(n_iters):
        if downdater.array.nnz == 0:
            break

        u, v = single_bicluster(downdater.array, comp_level=comp_level)

        if u.nnz <= 1 or v.nnz <= 1:
            break

        bic_list.append((u, v))

        idx_v = utils.find(v)[1]
        downdater.remove_columns(idx_v)
        if not share_elements:
            idx_u = utils.find(u)[0]
            downdater.remove_rows(idx_u)

        if n is None:
            cl = online_mdl.add_rank1_approximation(downdater.array, u, v)
            total_codelength.append(cl)

    if n is None and bic_list:
        cut_point = np.argmin(np.array(total_codelength))
        bic_list = bic_list[:cut_point+1]

    return bic_list

def single_bicluster(array, comp_level=None):
    if comp_level is not None:
        array_nmf = compression.compress_columns(array, comp_level)
    else:
        array_nmf = array

    u, _ = nmf.nmf_robust_multiplicative(array_nmf, 1)
    u = utils.binarize(u)

    if u.nnz == 0:
        return u, utils.sparse((1, array.shape[1]))

    v = update_right(array, u, comp_level=comp_level)

    if v.nnz == 0:
        return u, v

    u = update_left(array, v, comp_level=comp_level)

    return u, v

def update_right(array, u, comp_level=None):
    idx_u = utils.find(u)[0]
    array_crop = utils.sparse(array[idx_u, :])

    if comp_level is not None and comp_level < array_crop.shape[0]:
        array_crop = compression.compress_rows(array_crop, comp_level)

    u_crop = utils.sparse(np.ones((array_crop.shape[0], 1)))
    v_init = u_crop.T.dot(array_crop)
    v_init = (v_init / array_crop.shape[0]).rint()
    v_init = utils.binarize(v_init)

    v = nmf.nmf_robust_admm(array_crop, u_init=u_crop, v_init=v_init,
                            update='right')
    return utils.binarize(v)

def update_left(array, v, comp_level=None):
    idx_v = utils.find(v)[1]
    array_crop = utils.sparse(array[:, idx_v])

    if comp_level is not None and comp_level < array_crop.shape[1]:
        array_crop = compression.compress_columns(array_crop, comp_level)

    v_crop = utils.sparse(np.ones((1, array_crop.shape[1])))
    u_init = array_crop.dot(v_crop.T)
    u_init = (u_init / array_crop.shape[1]).rint()
    u_init = utils.binarize(u_init)

    u = nmf.nmf_robust_admm(array_crop, u_init=u_init, v_init=v_crop,
                            update='left')
    return utils.binarize(u)
<|endoftext|>


<commit_before>#!/usr/bin/env python3
#
# Copyright (c) 2017 kotoko <kotoko@users.noreply.github.com>
#

import argparse
import sys
import os
import tempfile
import subprocess
import timeit
import textwrap
import gettext

__version__ = "0.6.3"
app = "program-tester"

gettext.bindtextdomain(
		app,
		os.path.join(
			os.path.dirname(os.path.realpath(__file__)),
			"l10n"
		)
	)
gettext.textdomain(app)
_ = gettext.gettext

class Colors(object):
	green = "\033[1m\033[92m"
	yellow = "\033[1m\033[93m"
	red = "\033[1m\033[91m"
	reset = "\033[1m\033[0m"
	ok = green
	completed = yellow
	wrong = red
	error = red

	@classmethod
	def turn_off(cls):
		cls.green = ""
		cls.yellow = ""
		cls.red = ""
		cls.reset = ""
		cls.ok = ""
		cls.completed = ""
		cls.wrong = ""
		cls.error = ""

class Options(object):
	program = ""
	tests_folder = ""
	tests_list = []
	force_colors = 0
	show_time = 1
	show_comparision = 1
	show_summary = 1
	show_test_ok = 1
	show_test_wrong = 1
	show_test_completed = 1
	show_test_error = 1

class Results(object):
	def __init__(self):
		self.tests_ok = 0
		self.tests_wrong = 0
		self.tests_completed = 0
		self.tests_error = 0

	def add_ok(self):
		self.tests_ok += 1

	def add_wrong(self):
		self.tests_wrong += 1

	def add_completed(self):
		self.tests_completed += 1

	def add_error(self):
		self.tests_error += 1

	def get_ok(self):
		return self.tests_ok

	def get_wrong(self):
		return self.tests_wrong

	def get_completed(self):
		return self.tests_completed

	def get_error(self):
		return self.tests_error

# See: http://stackoverflow.com/a/32974697
class MultilineFormatter(argparse.HelpFormatter):
	def _fill_text(self, text, width, indent):
		text = self._whitespace_matcher.sub(" ", text).strip()
		paragraphs = text.split("|n ")
		multiline_text = ""

		for paragraph in paragraphs:
			formatted_paragraph = textwrap.fill(
				paragraph,
				width,
				initial_indent=indent,
				subsequent_indent=indent
			) + "\n"

			multiline_text = multiline_text + formatted_paragraph

		return multiline_text

def check_terminal():
	if not Options.force_colors:
		if not sys.stdout.isatty():
			Colors.turn_off()

def read_arguments():
	parser = argparse.ArgumentParser(
		__file__,
		add_help=False,
		formatter_class=MultilineFormatter,
		description=_("Program tester. Script runs program on multiple tests and checks \
		program's outputs. Possible responses: OK, COMPLETED, WRONG ANSWER, RUNTIME ERROR. |n |n \
		OK - program's completed with correct output |n \
		COMPLETED - program's completed, but there is missing *.out file |n \
		WRONG ANSWER- program's completed, but the output is different than *.out file |n \
		RUNTIME ERROR - program's completed with return code other than 0")
	)

	parser.add_argument(
		"-h",
		"--help",
		action="help",
		default=argparse.SUPPRESS,
		help=_("show help message")
	)

	parser.add_argument(
		"PROGRAM",
		type=str,
		help=_("path to executable binary")
	)

	parser.add_argument(
		"TESTS",
		type=str,
		help=_("path to folder with tests (contains files *.in and *.out)")
	)

	parser.add_argument(
		"--test",
		type=str,
		action="append",
		help=_("test's name without suffix .in; program is running only on target test; \
		parameter can be specified multiple times")
	)

	parser.add_argument(
		"--portable",
		action="store_true",
		help=_("alias for -TC")
	)

	parser.add_argument(
		"--quiet",
		action="store_true",
		help=_("alias for -O")
	)

	parser.add_argument(
		"-T",
		"--no-time",
		action="store_true",
		help=_("do not show execution time")
	)

	parser.add_argument(
		"-C",
		"--no-compare",
		action="store_true",
		help=_("do not show comparison of program's output and correct answer")
	)

	parser.add_argument(
		"-O",
		"--no-ok",
		action="store_true",
		help=_("do not show tests passed correctly")
	)

	parser.add_argument(
		"-E",
		"--no-error",
		action="store_true",
		help=_("do not show tests passed incorrectly")
	)

	color = parser.add_mutually_exclusive_group()

	color.add_argument(
		"--color",
		action="store_true",
		help=_("force color even when stdout is not a tty")
	)

	color.add_argument(
		"--no-color",
		action="store_true",
		help=_("turn off color")
	)

	parser.add_argument(
		"--no-summary",
		action="store_true",
		help=_("do not show summary")
	)

	parser.add_argument(
		"-V",
		"--version",
		action="version",
		help=_("show script's version number"),
		version=__version__
	)

	return parser.parse_args()

def parse_arguments(arg):
	Options.program = os.path.abspath(arg.PROGRAM)
	Options.tests_folder = os.path.abspath(arg.TESTS)

	if arg.test:
		Options.tests_list = arg.test

	if arg.portable:
		Options.show_time = 0
		Options.show_comparision = 0

	if arg.quiet:
		Options.show_test_ok = 0
		Options.show_test_completed = 0

	if arg.no_time:
		Options.show_time = 0

	if arg.no_compare:
		Options.show_comparision = 0

	if arg.no_ok:
		Options.show_test_ok = 0
		Options.show_test_completed = 0

	if arg.no_error:
		Options.show_test_error = 0
		Options.show_test_wrong = 0

	if arg.color:
		Options.force_colors = 1

	if arg.no_color:
		Colors.turn_off()

	if arg.no_summary:
		Options.show_summary = 0

def check_files():
	if not os.path.isfile(Options.program):
		print(_("Executable binary does not exist") + ":\n" + Options.program + "\n")
		raise FileNotFoundError

	if not os.path.exists(Options.tests_folder):
		print(_("Folder with tests does not exist") + ":\n" + Options.tests_folder + "\n")
		raise FileNotFoundError

def print_tests_summary(results):
	print("\n\n-----")

	print(_("Ok") + ": " + Colors.ok + str(results.get_ok()) + Colors.reset)

	if results.get_completed() > 0:
		print(_("Completed") + ": " + Colors.completed + str(results.get_completed()) + Colors.reset)

	print(_("Wrong answer") + ": " + Colors.wrong + str(results.get_wrong()) + Colors.reset)

	print(_("Runtime error") + ": " + Colors.error + str(results.get_error()) + Colors.reset)

def print_time(time):
	if Options.show_time:
		print(_("time") + ": {:.2f}\n".format(time))

def print_test_result(test_name, status, time=-1, comparison=""):
	separator = ":\t"
	prefix = _("Test") + " "

	# ok
	if status == 0:
		if Options.show_test_ok:
			print(prefix + test_name + separator + Colors.ok + _("OK") + Colors.reset)
			print_time(time)
	# wrong answer
	elif status == 1:
		if Options.show_test_wrong:
			print(prefix + test_name + separator + Colors.wrong + _("WRONG ANSWER") + Colors.reset)
			if Options.show_comparision:
				print(comparison)
				print("(" + _("program's output") + "  |  " + _("correct answer") + ")")
			print_time(time)
	# completed
	elif status == 2:
		if Options.show_test_completed:
			print(prefix + test_name + separator + Colors.completed + _("COMPLETED") + Colors.reset)
			print_time(time)
	# runtime error
	elif status == 3:
		if Options.show_test_error:
			print(prefix + test_name + separator + Colors.error + _("RUNTIME ERROR") + Colors.reset)
			print_time(time)
	else:
		pass

def make_prefix(text, length):
	text_list = text.split("\n", 1)
	text = text_list[0]
	if len(text) <= length:
		if len(text_list) > 1:
			if len(text) <= length-3:
				return "%s..." % (text[:length])
			else:
				return "%s..." % (text[:length - 3])
		else:
			return text
	else:
		return "%s..." % (text[:length - 3])

def run_test(test_name, test_in, test_out, results):
	with open(test_in, "rt") as file_in, tempfile.SpooledTemporaryFile(mode="r+t") as file_out:
		process = subprocess.Popen(
			Options.program,
			stdin=file_in,
			stdout=file_out,
			stderr=subprocess.DEVNULL,
			shell=False
		)
		process.wait()

		# TODO: run program only once
		if Options.show_time:
			start = timeit.default_timer()
			try:
				time = timeit.timeit(
					stmt="subprocess.check_call(PROGRAM, stdin=file_in, stdout=subprocess.DEVNULL,\
					stderr=subprocess.DEVNULL, shell=False)",
					setup="import subprocess; import os; PROGRAM='" + Options.program
					+ "'; file_in = open( '" + test_in + "' , 'r');",
					number=1
				)
			except subprocess.CalledProcessError:
				time = timeit.default_timer() - start
		else:
			time = 0
		# ##### #

		if process.returncode != 0:
			results.add_error()

			print_test_result(test_name, 3, time)
		else:
			try:
				with open(test_out, "rt") as file_answer:
					file_out.seek(0)
					if file_answer.read().strip() == file_out.read().strip():
						results.add_ok()

						print_test_result(test_name, 0, time)
					else:
						results.add_wrong()

						file_answer.seek(0)
						file_out.seek(0)

						# Hardcoded numbers 100 and 25. Hmm...
						answer = file_answer.read(100).strip()
						out = file_out.read(100).strip()

						comparison = make_prefix(out.strip(), 25) \
							+ "  |  " \
							+ make_prefix(answer.strip(), 25)

						print_test_result(test_name, 1, time, comparison)
			except OSError:
				results.add_completed()

				print_test_result(test_name, 2, time)

def run_tests():
	results = Results()
	tests = {}

	if Options.tests_list:
		for file in os.listdir(Options.tests_folder):
			for name in Options.tests_list:
				if file.lower().endswith(".in") and name == os.path.splitext(file)[0]:
					file = os.path.join(Options.tests_folder, file)
					tests[name] = (file, "")

		for file in os.listdir(Options.tests_folder):
			for name in Options.tests_list:
				if file.lower().endswith(".out") and name == os.path.splitext(file)[0]:
					if name in tests:
						test = tests[name]
						file = os.path.join(Options.tests_folder, file)
						tests[name] = (test[0], file)
	else:
		for file in os.listdir(Options.tests_folder):
			if file.lower().endswith(".in"):
				name = os.path.splitext(file)[0]
				file_in = os.path.join(Options.tests_folder, file)
				tests[name] = (file_in, "")

		for file in os.listdir(Options.tests_folder):
			if file.lower().endswith(".out"):
				name = os.path.splitext(file)[0]
				if name in tests:
					test = tests[name]
					file_out = os.path.join(Options.tests_folder, file)
					tests[name] = (test[0], file_out)

	for (test_name, test_files) in sorted(tests.items()):
		run_test(test_name, test_files[0], test_files[1], results)

	if Options.show_summary:
		print_tests_summary(results)

def main():
	arguments = read_arguments()
	parse_arguments(arguments)
	check_terminal()
	check_files()
	run_tests()

if __name__ == "__main__":
	# Minimum python version:
	# 3.0
	# 3.2 - PEP 389, module argparse
	# 3.3 - PEP 3151, OSError and IOError means the same
	# 3.3 - subprocess, new constant DEVNULL
	minimum_version = (3, 3)
	if sys.version_info >= minimum_version:
		main()
	else:
		print(
			_("Python's version") + ":  "
			+ str(sys.version_info[0]) + "." + str(sys.version_info[1])
		)
		print(
			_("Required python's version") + ":  >="
			+ str(minimum_version[0]) + "." + str(minimum_version[1])
		)

	raise SystemExit
<commit_msg>Version bump to 0.7<commit_after>#!/usr/bin/env python3
#
# Copyright (c) 2017 kotoko <kotoko@users.noreply.github.com>
#

import argparse
import sys
import os
import tempfile
import subprocess
import timeit
import textwrap
import gettext

__version__ = "0.7"
app = "program-tester"

gettext.bindtextdomain(
		app,
		os.path.join(
			os.path.dirname(os.path.realpath(__file__)),
			"l10n"
		)
	)
gettext.textdomain(app)
_ = gettext.gettext

class Colors(object):
	green = "\033[1m\033[92m"
	yellow = "\033[1m\033[93m"
	red = "\033[1m\033[91m"
	reset = "\033[1m\033[0m"
	ok = green
	completed = yellow
	wrong = red
	error = red

	@classmethod
	def turn_off(cls):
		cls.green = ""
		cls.yellow = ""
		cls.red = ""
		cls.reset = ""
		cls.ok = ""
		cls.completed = ""
		cls.wrong = ""
		cls.error = ""

class Options(object):
	program = ""
	tests_folder = ""
	tests_list = []
	force_colors = 0
	show_time = 1
	show_comparision = 1
	show_summary = 1
	show_test_ok = 1
	show_test_wrong = 1
	show_test_completed = 1
	show_test_error = 1

class Results(object):
	def __init__(self):
		self.tests_ok = 0
		self.tests_wrong = 0
		self.tests_completed = 0
		self.tests_error = 0

	def add_ok(self):
		self.tests_ok += 1

	def add_wrong(self):
		self.tests_wrong += 1

	def add_completed(self):
		self.tests_completed += 1

	def add_error(self):
		self.tests_error += 1

	def get_ok(self):
		return self.tests_ok

	def get_wrong(self):
		return self.tests_wrong

	def get_completed(self):
		return self.tests_completed

	def get_error(self):
		return self.tests_error

# See: http://stackoverflow.com/a/32974697
class MultilineFormatter(argparse.HelpFormatter):
	def _fill_text(self, text, width, indent):
		text = self._whitespace_matcher.sub(" ", text).strip()
		paragraphs = text.split("|n ")
		multiline_text = ""

		for paragraph in paragraphs:
			formatted_paragraph = textwrap.fill(
				paragraph,
				width,
				initial_indent=indent,
				subsequent_indent=indent
			) + "\n"

			multiline_text = multiline_text + formatted_paragraph

		return multiline_text

def check_terminal():
	if not Options.force_colors:
		if not sys.stdout.isatty():
			Colors.turn_off()

def read_arguments():
	parser = argparse.ArgumentParser(
		__file__,
		add_help=False,
		formatter_class=MultilineFormatter,
		description=_("Program tester. Script runs program on multiple tests and checks \
		program's outputs. Possible responses: OK, COMPLETED, WRONG ANSWER, RUNTIME ERROR. |n |n \
		OK - program's completed with correct output |n \
		COMPLETED - program's completed, but there is missing *.out file |n \
		WRONG ANSWER- program's completed, but the output is different than *.out file |n \
		RUNTIME ERROR - program's completed with return code other than 0")
	)

	parser.add_argument(
		"-h",
		"--help",
		action="help",
		default=argparse.SUPPRESS,
		help=_("show help message")
	)

	parser.add_argument(
		"PROGRAM",
		type=str,
		help=_("path to executable binary")
	)

	parser.add_argument(
		"TESTS",
		type=str,
		help=_("path to folder with tests (contains files *.in and *.out)")
	)

	parser.add_argument(
		"--test",
		type=str,
		action="append",
		help=_("test's name without suffix .in; program is running only on target test; \
		parameter can be specified multiple times")
	)

	parser.add_argument(
		"--portable",
		action="store_true",
		help=_("alias for -TC")
	)

	parser.add_argument(
		"--quiet",
		action="store_true",
		help=_("alias for -O")
	)

	parser.add_argument(
		"-T",
		"--no-time",
		action="store_true",
		help=_("do not show execution time")
	)

	parser.add_argument(
		"-C",
		"--no-compare",
		action="store_true",
		help=_("do not show comparison of program's output and correct answer")
	)

	parser.add_argument(
		"-O",
		"--no-ok",
		action="store_true",
		help=_("do not show tests passed correctly")
	)

	parser.add_argument(
		"-E",
		"--no-error",
		action="store_true",
		help=_("do not show tests passed incorrectly")
	)

	color = parser.add_mutually_exclusive_group()

	color.add_argument(
		"--color",
		action="store_true",
		help=_("force color even when stdout is not a tty")
	)

	color.add_argument(
		"--no-color",
		action="store_true",
		help=_("turn off color")
	)

	parser.add_argument(
		"--no-summary",
		action="store_true",
		help=_("do not show summary")
	)

	parser.add_argument(
		"-V",
		"--version",
		action="version",
		help=_("show script's version number"),
		version=__version__
	)

	return parser.parse_args()

def parse_arguments(arg):
	Options.program = os.path.abspath(arg.PROGRAM)
	Options.tests_folder = os.path.abspath(arg.TESTS)

	if arg.test:
		Options.tests_list = arg.test

	if arg.portable:
		Options.show_time = 0
		Options.show_comparision = 0

	if arg.quiet:
		Options.show_test_ok = 0
		Options.show_test_completed = 0

	if arg.no_time:
		Options.show_time = 0

	if arg.no_compare:
		Options.show_comparision = 0

	if arg.no_ok:
		Options.show_test_ok = 0
		Options.show_test_completed = 0

	if arg.no_error:
		Options.show_test_error = 0
		Options.show_test_wrong = 0

	if arg.color:
		Options.force_colors = 1

	if arg.no_color:
		Colors.turn_off()

	if arg.no_summary:
		Options.show_summary = 0

def check_files():
	if not os.path.isfile(Options.program):
		print(_("Executable binary does not exist") + ":\n" + Options.program + "\n")
		raise FileNotFoundError

	if not os.path.exists(Options.tests_folder):
		print(_("Folder with tests does not exist") + ":\n" + Options.tests_folder + "\n")
		raise FileNotFoundError

def print_tests_summary(results):
	print("\n\n-----")

	print(_("Ok") + ": " + Colors.ok + str(results.get_ok()) + Colors.reset)

	if results.get_completed() > 0:
		print(_("Completed") + ": " + Colors.completed + str(results.get_completed()) + Colors.reset)

	print(_("Wrong answer") + ": " + Colors.wrong + str(results.get_wrong()) + Colors.reset)

	print(_("Runtime error") + ": " + Colors.error + str(results.get_error()) + Colors.reset)

def print_time(time):
	if Options.show_time:
		print(_("time") + ": {:.2f}\n".format(time))

def print_test_result(test_name, status, time=-1, comparison=""):
	separator = ":\t"
	prefix = _("Test") + " "

	# ok
	if status == 0:
		if Options.show_test_ok:
			print(prefix + test_name + separator + Colors.ok + _("OK") + Colors.reset)
			print_time(time)
	# wrong answer
	elif status == 1:
		if Options.show_test_wrong:
			print(prefix + test_name + separator + Colors.wrong + _("WRONG ANSWER") + Colors.reset)
			if Options.show_comparision:
				print(comparison)
				print("(" + _("program's output") + "  |  " + _("correct answer") + ")")
			print_time(time)
	# completed
	elif status == 2:
		if Options.show_test_completed:
			print(prefix + test_name + separator + Colors.completed + _("COMPLETED") + Colors.reset)
			print_time(time)
	# runtime error
	elif status == 3:
		if Options.show_test_error:
			print(prefix + test_name + separator + Colors.error + _("RUNTIME ERROR") + Colors.reset)
			print_time(time)
	else:
		pass

def make_prefix(text, length):
	text_list = text.split("\n", 1)
	text = text_list[0]
	if len(text) <= length:
		if len(text_list) > 1:
			if len(text) <= length-3:
				return "%s..." % (text[:length])
			else:
				return "%s..." % (text[:length - 3])
		else:
			return text
	else:
		return "%s..." % (text[:length - 3])

def run_test(test_name, test_in, test_out, results):
	with open(test_in, "rt") as file_in, tempfile.SpooledTemporaryFile(mode="r+t") as file_out:
		process = subprocess.Popen(
			Options.program,
			stdin=file_in,
			stdout=file_out,
			stderr=subprocess.DEVNULL,
			shell=False
		)
		process.wait()

		# TODO: run program only once
		if Options.show_time:
			start = timeit.default_timer()
			try:
				time = timeit.timeit(
					stmt="subprocess.check_call(PROGRAM, stdin=file_in, stdout=subprocess.DEVNULL,\
					stderr=subprocess.DEVNULL, shell=False)",
					setup="import subprocess; import os; PROGRAM='" + Options.program
					+ "'; file_in = open( '" + test_in + "' , 'r');",
					number=1
				)
			except subprocess.CalledProcessError:
				time = timeit.default_timer() - start
		else:
			time = 0
		# ##### #

		if process.returncode != 0:
			results.add_error()

			print_test_result(test_name, 3, time)
		else:
			try:
				with open(test_out, "rt") as file_answer:
					file_out.seek(0)
					if file_answer.read().strip() == file_out.read().strip():
						results.add_ok()

						print_test_result(test_name, 0, time)
					else:
						results.add_wrong()

						file_answer.seek(0)
						file_out.seek(0)

						# Hardcoded numbers 100 and 25. Hmm...
						answer = file_answer.read(100).strip()
						out = file_out.read(100).strip()

						comparison = make_prefix(out.strip(), 25) \
							+ "  |  " \
							+ make_prefix(answer.strip(), 25)

						print_test_result(test_name, 1, time, comparison)
			except OSError:
				results.add_completed()

				print_test_result(test_name, 2, time)

def run_tests():
	results = Results()
	tests = {}

	if Options.tests_list:
		for file in os.listdir(Options.tests_folder):
			for name in Options.tests_list:
				if file.lower().endswith(".in") and name == os.path.splitext(file)[0]:
					file = os.path.join(Options.tests_folder, file)
					tests[name] = (file, "")

		for file in os.listdir(Options.tests_folder):
			for name in Options.tests_list:
				if file.lower().endswith(".out") and name == os.path.splitext(file)[0]:
					if name in tests:
						test = tests[name]
						file = os.path.join(Options.tests_folder, file)
						tests[name] = (test[0], file)
	else:
		for file in os.listdir(Options.tests_folder):
			if file.lower().endswith(".in"):
				name = os.path.splitext(file)[0]
				file_in = os.path.join(Options.tests_folder, file)
				tests[name] = (file_in, "")

		for file in os.listdir(Options.tests_folder):
			if file.lower().endswith(".out"):
				name = os.path.splitext(file)[0]
				if name in tests:
					test = tests[name]
					file_out = os.path.join(Options.tests_folder, file)
					tests[name] = (test[0], file_out)

	for (test_name, test_files) in sorted(tests.items()):
		run_test(test_name, test_files[0], test_files[1], results)

	if Options.show_summary:
		print_tests_summary(results)

def main():
	arguments = read_arguments()
	parse_arguments(arguments)
	check_terminal()
	check_files()
	run_tests()

if __name__ == "__main__":
	# Minimum python version:
	# 3.0
	# 3.2 - PEP 389, module argparse
	# 3.3 - PEP 3151, OSError and IOError means the same
	# 3.3 - subprocess, new constant DEVNULL
	minimum_version = (3, 3)
	if sys.version_info >= minimum_version:
		main()
	else:
		print(
			_("Python's version") + ":  "
			+ str(sys.version_info[0]) + "." + str(sys.version_info[1])
		)
		print(
			_("Required python's version") + ":  >="
			+ str(minimum_version[0]) + "." + str(minimum_version[1])
		)

	raise SystemExit
<|endoftext|>


<commit_before>"""
CPOL Level 1b main production line.

@title: CPOL_PROD_1b
@author: Valentin Louf <valentin.louf@monash.edu>
@institution: Bureau of Meteorology
@date: 1/03/2019
@version: 1

.. autosummary::
    :toctree: generated/

    timeout_handler
    chunks
    production_line_manager
    production_line_multiproc
    main
"""
# Python Standard Library
import os
import sys
import glob
import signal
import argparse
import datetime
import warnings
import traceback

from multiprocessing import Pool

class TimeoutException(Exception):   # Custom exception class
    pass

def timeout_handler(signum, frame):   # Custom signal handler
    raise TimeoutException

def chunks(l, n):
    """
    Yield successive n-sized chunks from l.
    From http://stackoverflow.com/a/312464
    """
    for i in range(0, len(l), n):
        yield l[i:i + n]

def main(infile):
    """
    It calls the production line and manages it. Buffer function that is used
    to catch any problem with the processing line without screwing the whole
    multiprocessing stuff.

    Parameters:
    ===========
    infile: str
        Name of the input radar file.
    outpath: str
        Path for saving output data.
    """
    import cpol_processing
    # SIGALRM is unix only.
    signal.signal(signal.SIGALRM, timeout_handler)
    signal.alarm(TIME_BEFORE_DEATH)
    try:
        cpol_processing.process_and_save(infile, OUTPATH, sound_dir=SOUND_DIR)
    except TimeoutException:
        print(f'Process deadlock for file {infile}. Killing it.')
        return None
    except Exception:
        traceback.print_exc()
        return None
    else:
        signal.alarm(0)

    return None

if __name__ == '__main__':
    """
    Global variables definition.
    """
    # Main global variables (Path directories).
    INPATH = "/g/data/hj10/cpol_level_1a/ppi/"
    OUTPATH = "/g/data/hj10/cpol_level_1b/"
    SOUND_DIR = "/g/data2/rr5/CPOL_radar/DARWIN_radiosonde"
    LOG_FILE_PATH = "/short/kl02/vhl548/"
    TIME_BEFORE_DEATH = 240

    # Parse arguments
    parser_description = "Processing of radar data from level 1a to level 1b."
    parser = argparse.ArgumentParser(description=parser_description)
    parser.add_argument(
        '-s',
        '--start-date',
        dest='start_date',
        default=None,
        type=str,
        help='Starting date.',
        required=True)
    parser.add_argument(
        '-e',
        '--end-date',
        dest='end_date',
        default=None,
        type=str,
        help='Ending date.',
        required=True)

    args = parser.parse_args()
    START_DATE = args.start_date
    END_DATE = args.end_date
    try:
        start = datetime.datetime.strptime(START_DATE, "%Y%m%d")
        end = datetime.datetime.strptime(END_DATE, "%Y%m%d")
        if start > end:
            raise ValueError('End date older than start date.')
        date_range = [start + datetime.timedelta(days=x) for x in range(0, (end - start).days + 1, )]
    except ValueError:
        print("Invalid dates.")
        sys.exit()

    for day in date_range:
        input_dir = os.path.join(INPATH, str(day.year), day.strftime("%Y%m%d"), "*.*")
        flist = sorted(glob.glob(input_dir))
        if len(flist) == 0:
            print('No file found for {}.'.format(day.strftime("%Y-%b-%d")))
            continue
        print(f'{len(flist)} files found for ' + day.strftime("%Y-%b-%d"))

        with warnings.catch_warnings():
            warnings.simplefilter('ignore')
            for flist_chunk in chunks(flist, 16):
                with Pool(len(flist_chunk)) as pool:
                    pool.map(main, flist_chunk)
<commit_msg>Use spawn multiprocessing get_context<commit_after>"""
CPOL Level 1b main production line.

@title: CPOL_PROD_1b
@author: Valentin Louf <valentin.louf@monash.edu>
@institution: Bureau of Meteorology
@date: 1/03/2019
@version: 1

.. autosummary::
    :toctree: generated/

    timeout_handler
    chunks
    production_line_manager
    production_line_multiproc
    main
"""
# Python Standard Library
import os
import sys
import glob
import argparse
import datetime
import warnings
import traceback

from multiprocessing import get_context

def chunks(l, n):
    """
    Yield successive n-sized chunks from l.
    From http://stackoverflow.com/a/312464
    """
    for i in range(0, len(l), n):
        yield l[i:i + n]

def main(infile):
    """
    It calls the production line and manages it. Buffer function that is used
    to catch any problem with the processing line without screwing the whole
    multiprocessing stuff.

    Parameters:
    ===========
    infile: str
        Name of the input radar file.
    outpath: str
        Path for saving output data.
    """
    import cpol_processing
    # SIGALRM is unix only.
    try:
        cpol_processing.process_and_save(infile, OUTPATH, sound_dir=SOUND_DIR)
    except Exception:
        traceback.print_exc()
        return None

    return None

if __name__ == '__main__':
    """
    Global variables definition.
    """
    # Main global variables (Path directories).
    INPATH = "/g/data/hj10/cpol_level_1a/ppi/"
    OUTPATH = "/g/data/hj10/cpol_level_1b/"
    SOUND_DIR = "/g/data2/rr5/CPOL_radar/DARWIN_radiosonde"
    LOG_FILE_PATH = "/short/kl02/vhl548/"

    # Parse arguments
    parser_description = "Processing of radar data from level 1a to level 1b."
    parser = argparse.ArgumentParser(description=parser_description)
    parser.add_argument(
        '-s',
        '--start-date',
        dest='start_date',
        default=None,
        type=str,
        help='Starting date.',
        required=True)
    parser.add_argument(
        '-e',
        '--end-date',
        dest='end_date',
        default=None,
        type=str,
        help='Ending date.',
        required=True)

    args = parser.parse_args()
    START_DATE = args.start_date
    END_DATE = args.end_date
    try:
        start = datetime.datetime.strptime(START_DATE, "%Y%m%d")
        end = datetime.datetime.strptime(END_DATE, "%Y%m%d")
        if start > end:
            raise ValueError('End date older than start date.')
        date_range = [start + datetime.timedelta(days=x) for x in range(0, (end - start).days + 1, )]
    except ValueError:
        print("Invalid dates.")
        sys.exit()

    for day in date_range:
        input_dir = os.path.join(INPATH, str(day.year), day.strftime("%Y%m%d"), "*.*")
        flist = sorted(glob.glob(input_dir))
        if len(flist) == 0:
            print('No file found for {}.'.format(day.strftime("%Y-%b-%d")))
            continue
        print(f'{len(flist)} files found for ' + day.strftime("%Y-%b-%d"))

        with warnings.catch_warnings():
            warnings.simplefilter('ignore')
            for flist_chunk in chunks(flist, 16):
                with get_context("spawn").Pool(len(flist_chunk)) as pool:
                    pool.map(main, flist_chunk)
<|endoftext|>


<commit_before># Copyright 2019 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This file is autogenerated by copybara, please do not edit.

buildtools_version = "a9f46b2bb3de812fce9f5fe59b29e75d95750aed"
buildtools_sha256 = "44c80ed9ca8e77b96fb80cbc53d6d7f6a0de93550adbf70da70b538ba2bf72b5"
<commit_msg>Update dependency bazelbuild/buildtools to latest version<commit_after># Copyright 2019 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This file is autogenerated by copybara, please do not edit.

buildtools_version = "c802c3b06ba674e8a76d04c0677d153ab9f660c9"
buildtools_sha256 = "3a2d2890d6a2948adb3040ee555483eeb483f73b839376c0fac60443e9a3d5b2"
<|endoftext|>