<commit_before># This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

"""Index supersearch fields data into elasticsearch.

This script creates a first set of data to be used by Super Search as the list
of fields it exposes to users, as well as to generate the elasticsearch
mapping for processed and raw crashes.
"""

import json
import os

from configman import Namespace
from configman.converters import class_converter

from socorro.app import generic_app

class SetupSuperSearchApp(generic_app.App):
    """Index supersearch fields data into elasticsearch. """

    app_name = 'setup-supersearch'
    app_version = '1.0'
    app_description = __doc__

    required_config = Namespace()

    required_config.add_option(
        'supersearch_fields_file',
        default=os.path.join(
            os.path.dirname(os.path.abspath(__file__)),
            'data',
            'supersearch_fields.json'
        ),
    )

    required_config.namespace('elasticsearch')
    required_config.elasticsearch.add_option(
        'elasticsearch_class',
        default='socorro.external.es.connection_context.'
                'ConnectionContext',
        from_string_converter=class_converter,
    )
    required_config.elasticsearch.add_option(
        'index_creator_class',
        default='socorro.external.es.crashstorage.'
                'ESCrashStorage',
        from_string_converter=class_converter,
    )

    def main(self):
        # Create the socorro index in elasticsearch.
        index_creator = self.config.elasticsearch.index_creator_class(
            self.config.elasticsearch
        )
        index_creator.create_index('socorro', None)

        # Load the initial data set.
        data_file = open(self.config.supersearch_fields_file, 'r')
        all_fields = json.loads(data_file.read())

        # Index the data.
        es_connection = index_creator.es
        # XXX ADRIAN: How should this be rewritten now that the old
        # pyelasticsearch is gone as socorro.external.elasticsearch disappears.
        es_connection.bulk_index(
            index='socorro',
            doc_type='supersearch_fields',
            docs=all_fields.values(),
            id_field='name',
        )

        # Verify data was correctly inserted.
        es_connection.refresh()
        total_indexed = es_connection.count(
            '*',
            index='socorro',
            doc_type='supersearch_fields',
        )['count']
        total_expected = len(all_fields)

        if total_expected != total_indexed:
            indexed_fields = es_connection.search(
                '*',
                index='socorro',
                doc_type='supersearch_fields',
                size=total_indexed,
            )
            indexed_fields = [x['_id'] for x in indexed_fields['hits']['hits']]

            self.config.logger.error(
                'The SuperSearch fields data was not correctly indexed, '
                '%s fields are missing from the database. Missing fields: %s',
                total_expected - total_indexed,
                list(set(all_fields.keys()) - set(indexed_fields))
            )

if __name__ == '__main__':
    generic_app.main(SetupSuperSearchApp)
<commit_msg>Fixes bug 1192012 - Rewrote setup_supersearch_app.py file to use ES1.4 style bulk indexing. r=peterbe<commit_after># This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

"""Index supersearch fields data into elasticsearch.

This script creates a first set of data to be used by Super Search as the list
of fields it exposes to users, as well as to generate the elasticsearch
mapping for processed and raw crashes.
"""

import json
import os
from elasticsearch.helpers import bulk

from configman import Namespace
from configman.converters import class_converter

from socorro.app import generic_app

class SetupSuperSearchApp(generic_app.App):
    """Index supersearch fields data into elasticsearch. """

    app_name = 'setup-supersearch'
    app_version = '1.0'
    app_description = __doc__

    required_config = Namespace()

    required_config.add_option(
        'supersearch_fields_file',
        default=os.path.join(
            os.path.dirname(os.path.abspath(__file__)),
            'data',
            'supersearch_fields.json'
        ),
    )

    required_config.namespace('elasticsearch')
    required_config.elasticsearch.add_option(
        'elasticsearch_class',
        default='socorro.external.es.connection_context.ConnectionContext',
        from_string_converter=class_converter,
    )
    required_config.elasticsearch.add_option(
        'index_creator_class',
        default='socorro.external.es.index_creator.IndexCreator',
        from_string_converter=class_converter,
    )

    def main(self):
        es_index = 'socorro'

        es_context = self.config.elasticsearch.elasticsearch_class(
            config=self.config.elasticsearch
        )
        index_creator = self.config.elasticsearch.index_creator_class(
            self.config.elasticsearch
        )
        index_client = index_creator.get_index_client()

        with es_context() as conn:
            es_connection = conn

        # Create the socorro index in elasticsearch.
        index_creator.create_index(es_index, None)

        # Load the initial data set.
        data_file = open(self.config.supersearch_fields_file, 'r')
        all_fields = json.loads(data_file.read())

        # Index the data.
        actions = []
        for name, field in all_fields.iteritems():
            action = {
                '_index': es_index,
                '_type': 'supersearch_fields',
                '_id': name,
                '_source': field,
            }
            actions.append(action)

        bulk(
            client=es_connection,
            actions=actions,
        )

        # Verify data was correctly inserted.
        index_client.refresh(index=[es_index])
        total_indexed = es_connection.count(
            index=es_index,
            doc_type='supersearch_fields',
        )['count']
        total_expected = len(all_fields)

        if total_expected > total_indexed:
            indexed_fields = es_connection.search(
                index=es_index,
                doc_type='supersearch_fields',
                size=total_indexed,
            )
            indexed_fields = [x['_id'] for x in indexed_fields['hits']['hits']]

            self.config.logger.error(
                'The SuperSearch fields data was not correctly indexed, '
                '%s fields are missing from the database. Missing fields: %s',
                total_expected - total_indexed,
                list(set(all_fields.keys()) - set(indexed_fields))
            )

if __name__ == '__main__':
    generic_app.main(SetupSuperSearchApp)
<|endoftext|>


<commit_before># Copyright (c) 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Functions for discovering and clearing the build directory."""

import os
import sys

def IsFileNewerThanFile(file_a, file_b):
  """Returns True if file_a's mtime is newer than file_b's."""
  def getmtime(f):
    try:
      return os.path.getmtime(f)
    except os.error:
      return 0
  return getmtime(file_a) >= getmtime(file_b)

def AreNinjaFilesNewerThanXcodeFiles(src_dir=None):
  """Returns True if the generated ninja files are newer than the generated
  xcode files.

  Parameters:
    src_dir: The path to the src directory.  If None, it's assumed to be
             at src/ relative to the current working directory.
  """
  src_dir = src_dir or 'src'
  ninja_path = os.path.join(src_dir, 'out', 'Release', 'build.ninja')
  xcode_path = os.path.join(
      src_dir, 'build', 'all.xcodeproj', 'project.pbxproj')
  return IsFileNewerThanFile(ninja_path, xcode_path)

def AreNinjaFilesNewerThanMSVSFiles(src_dir=None):
  """Returns True if the generated ninja files are newer than the generated
  msvs files.

  Parameters:
    src_dir: The path to the src directory.  If None, it's assumed to be
             at src/ relative to the current working directory.
  """
  src_dir = src_dir or 'src'
  ninja_path = os.path.join(src_dir, 'out', 'Release', 'build.ninja')
  msvs_path = os.path.join(src_dir, 'build', 'all.sln')
  return IsFileNewerThanFile(ninja_path, msvs_path)

def GetBuildOutputDirectory(src_dir=None, cros_board=None):
  """Returns the path to the build directory, relative to the checkout root.

  Assumes that the current working directory is the checkout root.
  """
  # src_dir is only needed for compiling v8, which uses compile.py (but no other
  # of the build scripts), but its source root isn't "src" -- crbug.com/315004
  if src_dir is None:
    src_dir = 'src'

  if sys.platform.startswith('linux'):
    out_dirname = 'out'
    if cros_board:
      # Simple chrome workflow output (e.g., "out_x86-generic")
      out_dirname += '_%s' % (cros_board,)
    return os.path.join(src_dir, out_dirname)
  assert not cros_board, "'cros_board' not supported on this platform"

  if sys.platform == 'darwin':
    if AreNinjaFilesNewerThanXcodeFiles(src_dir):
      return os.path.join(src_dir, 'out')
    return os.path.join(src_dir, 'xcodebuild')

  if sys.platform == 'cygwin' or sys.platform.startswith('win'):
    if AreNinjaFilesNewerThanMSVSFiles(src_dir):
      return os.path.join(src_dir, 'out')
    return os.path.join(src_dir, 'build')

  raise NotImplementedError('Unexpected platform %s' % sys.platform)

def RmtreeExceptNinjaOrGomaFiles(build_output_dir):
  """Recursively removes everything but ninja files from a build directory."""
  for root, _, files in os.walk(build_output_dir, topdown=False):
    for f in files:
      # For .manifest in particular, gyp windows ninja generates manifest
      # files at generation time but clobber nukes at the beginning of
      # compile, so make sure not to delete those generated files, otherwise
      # compile will fail.
      if (f.endswith('.ninja') or f.endswith('.manifest') or
          f == 'args.gn' or f == 'mb_type' or f == 'runtime_deps' or
          f.startswith('msvc') or  # VS runtime DLLs.
          f.startswith('pgort') or  # VS PGO runtime DLL.
          f.startswith('vccorlib') or  # VS2015 core libraries.
          f.startswith('vcruntime') or  # VS2015 runtime libraries.
          f in ('gyp-mac-tool', 'gyp-win-tool',
                'environment.x86', 'environment.x64')):
        continue
      # Keep goma related files.
      if f == '.goma_deps':
        continue
      os.unlink(os.path.join(root, f))
    # Delete the directory if empty; this works because the walk is bottom-up.
    try:
      os.rmdir(root)
    except OSError, e:
      if e.errno in (39, 41, 66):
        # If the directory isn't empty, ignore it.
        # On Windows, os.rmdir will raise WindowsError with winerror 145,
        # which e.errno is 41.
        # On Linux, e.errno is 39.
        pass
      else:
        raise
<commit_msg>Fix CrWin7Goma(clbr) build.<commit_after># Copyright (c) 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Functions for discovering and clearing the build directory."""

import os
import sys

def IsFileNewerThanFile(file_a, file_b):
  """Returns True if file_a's mtime is newer than file_b's."""
  def getmtime(f):
    try:
      return os.path.getmtime(f)
    except os.error:
      return 0
  return getmtime(file_a) >= getmtime(file_b)

def AreNinjaFilesNewerThanXcodeFiles(src_dir=None):
  """Returns True if the generated ninja files are newer than the generated
  xcode files.

  Parameters:
    src_dir: The path to the src directory.  If None, it's assumed to be
             at src/ relative to the current working directory.
  """
  src_dir = src_dir or 'src'
  ninja_path = os.path.join(src_dir, 'out', 'Release', 'build.ninja')
  xcode_path = os.path.join(
      src_dir, 'build', 'all.xcodeproj', 'project.pbxproj')
  return IsFileNewerThanFile(ninja_path, xcode_path)

def AreNinjaFilesNewerThanMSVSFiles(src_dir=None):
  """Returns True if the generated ninja files are newer than the generated
  msvs files.

  Parameters:
    src_dir: The path to the src directory.  If None, it's assumed to be
             at src/ relative to the current working directory.
  """
  src_dir = src_dir or 'src'
  ninja_path = os.path.join(src_dir, 'out', 'Release', 'build.ninja')
  msvs_path = os.path.join(src_dir, 'build', 'all.sln')
  return IsFileNewerThanFile(ninja_path, msvs_path)

def GetBuildOutputDirectory(src_dir=None, cros_board=None):
  """Returns the path to the build directory, relative to the checkout root.

  Assumes that the current working directory is the checkout root.
  """
  # src_dir is only needed for compiling v8, which uses compile.py (but no other
  # of the build scripts), but its source root isn't "src" -- crbug.com/315004
  if src_dir is None:
    src_dir = 'src'

  if sys.platform.startswith('linux'):
    out_dirname = 'out'
    if cros_board:
      # Simple chrome workflow output (e.g., "out_x86-generic")
      out_dirname += '_%s' % (cros_board,)
    return os.path.join(src_dir, out_dirname)
  assert not cros_board, "'cros_board' not supported on this platform"

  if sys.platform == 'darwin':
    if AreNinjaFilesNewerThanXcodeFiles(src_dir):
      return os.path.join(src_dir, 'out')
    return os.path.join(src_dir, 'xcodebuild')

  if sys.platform == 'cygwin' or sys.platform.startswith('win'):
    if AreNinjaFilesNewerThanMSVSFiles(src_dir):
      return os.path.join(src_dir, 'out')
    return os.path.join(src_dir, 'build')

  raise NotImplementedError('Unexpected platform %s' % sys.platform)

def RmtreeExceptNinjaOrGomaFiles(build_output_dir):
  """Recursively removes everything but ninja files from a build directory."""
  for root, _, files in os.walk(build_output_dir, topdown=False):
    for f in files:
      # For .manifest in particular, gyp windows ninja generates manifest
      # files at generation time but clobber nukes at the beginning of
      # compile, so make sure not to delete those generated files, otherwise
      # compile will fail.
      if (f.endswith('.ninja') or f.endswith('.manifest') or
          f == 'args.gn' or f == 'mb_type' or f == 'runtime_deps' or
          f.startswith('msvc') or  # VS runtime DLLs.
          f.startswith('pgort') or  # VS PGO runtime DLL.
          f.startswith('vccorlib') or  # VS2015 core libraries.
          f.startswith('vcruntime') or  # VS2015 runtime libraries.
          f.startswith('ucrtbase') or  #  Win runtime DLLs for VS.
          f.startswith('api-ms-win') or  # Win runtime DLLs for VS.
          f in ('gyp-mac-tool', 'gyp-win-tool',
                'environment.x86', 'environment.x64')):
        continue
      # Keep goma related files.
      if f == '.goma_deps':
        continue
      os.unlink(os.path.join(root, f))
    # Delete the directory if empty; this works because the walk is bottom-up.
    try:
      os.rmdir(root)
    except OSError, e:
      if e.errno in (39, 41, 66):
        # If the directory isn't empty, ignore it.
        # On Windows, os.rmdir will raise WindowsError with winerror 145,
        # which e.errno is 41.
        # On Linux, e.errno is 39.
        pass
      else:
        raise
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from qtpy import QtGui, QtCore, QtWidgets, uic
import os

from .. import global_vars as g
from .terminal import ipython_terminal
from .script_namespace import getnamespace
from .syntax import PythonHighlighter
from ..utils.misc import save_file_gui, open_file_gui, load_ui

MESSAGE_TIME = 2000
try:
    __IPYTHON__
except NameError:
    INSIDE_IPYTHON = False
else:
    INSIDE_IPYTHON = True

def qstr2str(string):
    string=str(string)
    return string.replace(u'\u2029','\n').strip()

class Editor(QtWidgets.QPlainTextEdit):
    def __init__(self, scriptfile = ''):
        super(Editor, self).__init__()
        self.highlight = PythonHighlighter(self.document())
        self.scriptfile = ''
        if scriptfile != '':
            self.open_file(scriptfile)
        self.installEventFilter(self)

    @staticmethod
    def fromWindow(window):
        if window is None:
            g.alert("In order to load a script from a window, you need to have a window selected.")
            return None
        editor = Editor()
        editor.setPlainText('\n'.join(window.commands))
        return editor

    def open_file(self, scriptfile):
        self.scriptfile = scriptfile
        try:
            script = open(scriptfile, 'r').read()
        except FileNotFoundError as e:
            print("Failed to read %s: %s" % (scriptfile, e))
            return
        self.setPlainText(script)
        ScriptEditor.gui.statusBar().showMessage('{} opened.'.format(os.path.basename(self.scriptfile)), MESSAGE_TIME)

    def save_as(self):
        filename = save_file_gui('Save script', ScriptEditor.most_recent_script(), '*.py')
        if filename == '':
            ScriptEditor.gui.statusBar().showMessage('Save cancelled',  MESSAGE_TIME)
            return False
        self.scriptfile = filename
        self.save()
        return True

    def save(self):
        if self.scriptfile == '':
            return self.save_as()
        f = open(self.scriptfile, 'w')
        command=qstr2str(self.toPlainText())
        f.write(command)
        f.close()
        ScriptEditor.add_recent_file(self.scriptfile)
        ScriptEditor.gui.statusBar().showMessage('{} saved.'.format(os.path.basename(self.scriptfile)), MESSAGE_TIME)
        return True

    def eventFilter(self, source, event):
        if (event.type()==QtGui.QKeyEvent.KeyPress and event.key() == QtCore.Qt.Key_Tab):
            event.accept()
            self.insertPlainText('    ')
            return True
        else:
            event.ignore()
        return False

class ScriptEditor(QtWidgets.QMainWindow):
    '''
    QMainWindow for editing and running user scripts. Comprised of a tabbed text editor and console.
    '''
    def __init__(self, parent=None, ):
        super(ScriptEditor, self).__init__(parent)
        load_ui('ipythonWidget.ui', self, directory=os.path.dirname(__file__))
        text = """Predefined Libraries:
    scipy and numpy (np)
Useful variables:
    g.m.windows - list of windows.
    g.currentWindow - the selected window
    g.currentWindow.rois -list of rois in that window
    - roi.getTrace() gets an roi trace as an array
    clear() to clear the console
    reset() to reset all global and local variables """

        self.terminal = ipython_terminal(banner=text, **getnamespace())
        self.terminal.namespace.update({'clear': self.terminal.clear, 'reset': self.resetNamespace})
        layout = QtWidgets.QGridLayout()
        self.terminalWidget.setLayout(layout)
        layout.addWidget(self.terminal)
        self.addEditor()
        self.saveButton.clicked.connect(self.saveCurrentScript)
        self.runButton.clicked.connect(self.runScript)
        self.runSelectedButton.clicked.connect(self.runSelected)
        self.actionNew_Script.triggered.connect(lambda f: self.addEditor())
        self.actionFrom_File.triggered.connect(lambda f: ScriptEditor.importScript())
        self.actionFrom_Window.triggered.connect(lambda : self.addEditor(Editor.fromWindow(g.currentWindow)))
        self.actionSave_Script.triggered.connect(self.saveCurrentScript)
        self.menuRecentScripts.aboutToShow.connect(self.open_scripts)
        self.actionChangeFontSize.triggered.connect(self.changeFontSize)
        #self.eventeater = ScriptEventEater(self)
        self.setAcceptDrops(True)
        #self.installEventFilter(self.eventeater)
        self.scriptTabs.tabCloseRequested.connect(self.closeTab)
        self.setWindowTitle('Script Editor')

    def resetNamespace(self):
        self.terminal.shell.reset()
        self.terminal.shell.user_ns.update(getnamespace())
        self.terminal.namespace.update({'clear': self.terminal.clear, 'reset': self.resetNamespace})

    def changeFontSize(self):
        val, ok = QtWidgets.QInputDialog.getInt(self, "Change Font Size", "Font Size", \
            value=ScriptEditor.gui.centralWidget().font().pointSize(), min = 8, max = 30, step = 1)
        if ok:
            font = ScriptEditor.gui.centralWidget().font()
            font.setPointSize(val)
            ScriptEditor.gui.centralWidget().setFont(font)

    def dragEnterEvent(self, event):
        if event.mimeData().hasUrls():
            event.accept()
        else:
            event.ignore()

    def dropEvent(self, event):
        for url in event.mimeData().urls():
            filename=url.toString()
            filename=filename.split('file:///')[1]
            print('filename={}'.format(filename))
            ScriptEditor.importScript(filename)

        event.accept()

    def open_scripts(self):
        self.menuRecentScripts.clear()
        def makeFun(script):
            return lambda: ScriptEditor.importScript(script)
        if len(g.settings['recent_scripts']) == 0:
            action = QtWidgets.QAction("No Recent Scripts", self)
            action.setEnabled(False)
            self.menuRecentScripts.addAction(action)
        else:
            for filename in g.settings['recent_scripts']:
                if not os.path.exists(filename):
                    g.settings['recent_scripts'].remove(filename)
                    continue
                action = QtWidgets.QAction(filename, self, triggered=makeFun(filename))
                self.menuRecentScripts.addAction(action)

    def closeTab(self, index):
        self.scriptTabs.removeTab(index)

    def saveCurrentScript(self):
        cw = self.currentTab()
        if cw == None:
            return
        if cw.save():
            self.scriptTabs.setTabText(self.scriptTabs.currentIndex(), os.path.basename(cw.scriptfile))

    def currentTab(self):
        return self.scriptTabs.currentWidget()

    @staticmethod
    def most_recent_script():
        return g.settings['recent_scripts'][-1] if len(g.settings['recent_scripts']) > 0 else ''

    @staticmethod
    def add_recent_file(filename):
        filename = os.path.abspath(filename)
        if not os.path.exists(filename):
            return
        if filename in g.settings['recent_scripts']:
            g.settings['recent_scripts'].remove(filename)
        g.settings['recent_scripts'].insert(0, filename)
        if len(g.settings['recent_scripts']) > 8:
            g.settings['recent_scripts'] = g.settings['recent_scripts'][:-1]
        g.settings.save()

    @staticmethod
    def importScript(scriptfile = ''):
        if not hasattr(ScriptEditor, 'gui') or not ScriptEditor.gui.isVisible():
            ScriptEditor.show()
        if scriptfile == '':
            prompt = "Open script"
            directory = os.path.dirname(ScriptEditor.most_recent_script())
            filetypes = '*.py'
            scriptfile = open_file_gui(prompt, directory, filetypes)
            if scriptfile is None:
                return None
        if hasattr(ScriptEditor, 'gui'):
            editor = Editor(scriptfile)
            ScriptEditor.add_recent_file(scriptfile)
            ScriptEditor.gui.addEditor(editor)

    def addEditor(self, editor=None):
        self.setUpdatesEnabled(False)
        if editor is None:
            editor = Editor()
        if editor.scriptfile == '':
            name = 'New Script'
        else:
            name = editor.scriptfile
        editor.keyPressEvent = lambda ev: self.editorKeyPressEvent(editor, ev)
        self.scriptTabs.insertTab(0, editor, os.path.basename(name))
        self.scriptTabs.setCurrentIndex(0)
        self.setUpdatesEnabled(True)
        return editor

    def keyPressEvent(self, ev):
        if ev.modifiers() == QtCore.Qt.ControlModifier:
            if ev.key() == QtCore.Qt.Key_N:
                self.addEditor()
                ev.accept()
            elif ev.key() == QtCore.Qt.Key_S:
                self.saveCurrentScript()

    def editorKeyPressEvent(self, editor, ev):
        if ev.key() == QtCore.Qt.Key_F9:
            self.runSelected()
            ev.accept()
        else:
            Editor.keyPressEvent(editor, ev)

    def runScript(self):
        if self.currentTab() == None:
            return
        command=qstr2str(self.currentTab().toPlainText())
        if command:
            self.terminal.execute(command)

    def runSelected(self):
        if self.currentTab() == None:
            return
        cursor=self.currentTab().textCursor()
        command=cursor.selectedText()
        self.command=command
        command=qstr2str(command)
        if command:
            self.terminal.execute(command)

    @staticmethod
    def show():
        if 'PYCHARM_HOSTED' in os.environ:
            g.alert('You cannot run the script editor from within PyCharm.')
        elif INSIDE_IPYTHON:
            g.alert('You cannot run the script editor because flika is already running inside IPython.')
        else:
            if not hasattr(ScriptEditor, 'gui'):
                ScriptEditor.gui = ScriptEditor()
            elif ScriptEditor.gui.isVisible():
                return
            QtWidgets.QMainWindow.show(ScriptEditor.gui)

            ui = ScriptEditor.gui
            geom = ui.geometry()
            geom = (geom.x(), geom.y(), geom.width(), geom.height())
            settings = {'geometry': geom, 'sizes': ui.splitter.sizes()}
            if 'script_editor_settings' in g.settings:
                settings.update(g.settings['script_editor_settings'])
            ui.setGeometry(*settings['geometry'])
            ui.splitter.setSizes(settings['sizes'])

    def closeEvent(self, ev):
        geom = self.geometry()
        geom = (geom.x(), geom.y(), geom.width(), geom.height())
        g.settings['script_editor_settings'] = {'geometry': geom, 'sizes': self.splitter.sizes()}

    @staticmethod
    def close():
        if hasattr(ScriptEditor, 'gui'):
            QtWidgets.QMainWindow.close(ScriptEditor.gui)

if __name__ == '__main__':
    app = QtWidgets.QApplication([])
    ScriptEditor.show()
    app.exec_()
    <commit_msg>fixed save function on script editor<commit_after># -*- coding: utf-8 -*-
from qtpy import QtGui, QtCore, QtWidgets, uic
import os

from .. import global_vars as g
from .terminal import ipython_terminal
from .script_namespace import getnamespace
from .syntax import PythonHighlighter
from ..utils.misc import save_file_gui, open_file_gui, load_ui

MESSAGE_TIME = 2000
try:
    __IPYTHON__
except NameError:
    INSIDE_IPYTHON = False
else:
    INSIDE_IPYTHON = True

def qstr2str(string):
    string=str(string)
    return string.replace(u'\u2029','\n').strip()

class Editor(QtWidgets.QPlainTextEdit):
    def __init__(self, scriptfile = ''):
        super(Editor, self).__init__()
        self.highlight = PythonHighlighter(self.document())
        self.scriptfile = ''
        if scriptfile != '':
            self.open_file(scriptfile)
        self.installEventFilter(self)

    @staticmethod
    def fromWindow(window):
        if window is None:
            g.alert("In order to load a script from a window, you need to have a window selected.")
            return None
        editor = Editor()
        editor.setPlainText('\n'.join(window.commands))
        return editor

    def open_file(self, scriptfile):
        self.scriptfile = scriptfile
        try:
            script = open(scriptfile, 'r').read()
        except FileNotFoundError as e:
            print("Failed to read %s: %s" % (scriptfile, e))
            return
        self.setPlainText(script)
        ScriptEditor.gui.statusBar().showMessage('{} opened.'.format(os.path.basename(self.scriptfile)), MESSAGE_TIME)

    def save_as(self):
        filename = save_file_gui('Save script', ScriptEditor.most_recent_script(), '*.py')
        if filename == '':
            ScriptEditor.gui.statusBar().showMessage('Save cancelled',  MESSAGE_TIME)
            return False
        self.scriptfile = filename
        self.save()
        return True

    def save(self):
        if not self.scriptfile:
            return self.save_as()
        f = open(self.scriptfile, 'w')
        command=qstr2str(self.toPlainText())
        f.write(command)
        f.close()
        ScriptEditor.add_recent_file(self.scriptfile)
        ScriptEditor.gui.statusBar().showMessage('{} saved.'.format(os.path.basename(self.scriptfile)), MESSAGE_TIME)
        return True

    def eventFilter(self, source, event):
        if (event.type()==QtGui.QKeyEvent.KeyPress and event.key() == QtCore.Qt.Key_Tab):
            event.accept()
            self.insertPlainText('    ')
            return True
        else:
            event.ignore()
        return False

class ScriptEditor(QtWidgets.QMainWindow):
    '''
    QMainWindow for editing and running user scripts. Comprised of a tabbed text editor and console.
    '''
    def __init__(self, parent=None, ):
        super(ScriptEditor, self).__init__(parent)
        load_ui('ipythonWidget.ui', self, directory=os.path.dirname(__file__))
        text = """Predefined Libraries:
    scipy and numpy (np)
Useful variables:
    g.m.windows - list of windows.
    g.currentWindow - the selected window
    g.currentWindow.rois -list of rois in that window
    - roi.getTrace() gets an roi trace as an array
    clear() to clear the console
    reset() to reset all global and local variables """

        self.terminal = ipython_terminal(banner=text, **getnamespace())
        self.terminal.namespace.update({'clear': self.terminal.clear, 'reset': self.resetNamespace})
        layout = QtWidgets.QGridLayout()
        self.terminalWidget.setLayout(layout)
        layout.addWidget(self.terminal)
        self.addEditor()
        self.saveButton.clicked.connect(self.saveCurrentScript)
        self.runButton.clicked.connect(self.runScript)
        self.runSelectedButton.clicked.connect(self.runSelected)
        self.actionNew_Script.triggered.connect(lambda f: self.addEditor())
        self.actionFrom_File.triggered.connect(lambda f: ScriptEditor.importScript())
        self.actionFrom_Window.triggered.connect(lambda : self.addEditor(Editor.fromWindow(g.currentWindow)))
        self.actionSave_Script.triggered.connect(self.saveCurrentScript)
        self.menuRecentScripts.aboutToShow.connect(self.open_scripts)
        self.actionChangeFontSize.triggered.connect(self.changeFontSize)
        #self.eventeater = ScriptEventEater(self)
        self.setAcceptDrops(True)
        #self.installEventFilter(self.eventeater)
        self.scriptTabs.tabCloseRequested.connect(self.closeTab)
        self.setWindowTitle('Script Editor')

    def resetNamespace(self):
        self.terminal.shell.reset()
        self.terminal.shell.user_ns.update(getnamespace())
        self.terminal.namespace.update({'clear': self.terminal.clear, 'reset': self.resetNamespace})

    def changeFontSize(self):
        val, ok = QtWidgets.QInputDialog.getInt(self, "Change Font Size", "Font Size", \
            value=ScriptEditor.gui.centralWidget().font().pointSize(), min = 8, max = 30, step = 1)
        if ok:
            font = ScriptEditor.gui.centralWidget().font()
            font.setPointSize(val)
            ScriptEditor.gui.centralWidget().setFont(font)

    def dragEnterEvent(self, event):
        if event.mimeData().hasUrls():
            event.accept()
        else:
            event.ignore()

    def dropEvent(self, event):
        for url in event.mimeData().urls():
            filename=url.toString()
            filename=filename.split('file:///')[1]
            print('filename={}'.format(filename))
            ScriptEditor.importScript(filename)

        event.accept()

    def open_scripts(self):
        self.menuRecentScripts.clear()
        def makeFun(script):
            return lambda: ScriptEditor.importScript(script)
        if len(g.settings['recent_scripts']) == 0:
            action = QtWidgets.QAction("No Recent Scripts", self)
            action.setEnabled(False)
            self.menuRecentScripts.addAction(action)
        else:
            for filename in g.settings['recent_scripts']:
                if not os.path.exists(filename):
                    g.settings['recent_scripts'].remove(filename)
                    continue
                action = QtWidgets.QAction(filename, self, triggered=makeFun(filename))
                self.menuRecentScripts.addAction(action)

    def closeTab(self, index):
        self.scriptTabs.removeTab(index)

    def saveCurrentScript(self):
        cw = self.currentTab()
        if cw == None:
            return
        if cw.save():
            self.scriptTabs.setTabText(self.scriptTabs.currentIndex(), os.path.basename(cw.scriptfile))

    def currentTab(self):
        return self.scriptTabs.currentWidget()

    @staticmethod
    def most_recent_script():
        return g.settings['recent_scripts'][-1] if len(g.settings['recent_scripts']) > 0 else ''

    @staticmethod
    def add_recent_file(filename):
        filename = os.path.abspath(filename)
        if not os.path.exists(filename):
            return
        if filename in g.settings['recent_scripts']:
            g.settings['recent_scripts'].remove(filename)
        g.settings['recent_scripts'].insert(0, filename)
        if len(g.settings['recent_scripts']) > 8:
            g.settings['recent_scripts'] = g.settings['recent_scripts'][:-1]
        g.settings.save()

    @staticmethod
    def importScript(scriptfile = ''):
        if not hasattr(ScriptEditor, 'gui') or not ScriptEditor.gui.isVisible():
            ScriptEditor.show()
        if scriptfile == '':
            prompt = "Open script"
            directory = os.path.dirname(ScriptEditor.most_recent_script())
            filetypes = '*.py'
            scriptfile = open_file_gui(prompt, directory, filetypes)
            if scriptfile is None:
                return None
        if hasattr(ScriptEditor, 'gui'):
            editor = Editor(scriptfile)
            ScriptEditor.add_recent_file(scriptfile)
            ScriptEditor.gui.addEditor(editor)

    def addEditor(self, editor=None):
        self.setUpdatesEnabled(False)
        if editor is None:
            editor = Editor()
        if editor.scriptfile == '':
            name = 'New Script'
        else:
            name = editor.scriptfile
        editor.keyPressEvent = lambda ev: self.editorKeyPressEvent(editor, ev)
        self.scriptTabs.insertTab(0, editor, os.path.basename(name))
        self.scriptTabs.setCurrentIndex(0)
        self.setUpdatesEnabled(True)
        return editor

    def keyPressEvent(self, ev):
        if ev.modifiers() == QtCore.Qt.ControlModifier:
            if ev.key() == QtCore.Qt.Key_N:
                self.addEditor()
                ev.accept()
            elif ev.key() == QtCore.Qt.Key_S:
                self.saveCurrentScript()

    def editorKeyPressEvent(self, editor, ev):
        if ev.key() == QtCore.Qt.Key_F9:
            self.runSelected()
            ev.accept()
        else:
            Editor.keyPressEvent(editor, ev)

    def runScript(self):
        if self.currentTab() == None:
            return
        command=qstr2str(self.currentTab().toPlainText())
        if command:
            self.terminal.execute(command)

    def runSelected(self):
        if self.currentTab() == None:
            return
        cursor=self.currentTab().textCursor()
        command=cursor.selectedText()
        self.command=command
        command=qstr2str(command)
        if command:
            self.terminal.execute(command)

    @staticmethod
    def show():
        if 'PYCHARM_HOSTED' in os.environ:
            g.alert('You cannot run the script editor from within PyCharm.')
        elif INSIDE_IPYTHON:
            g.alert('You cannot run the script editor because flika is already running inside IPython.')
        else:
            if not hasattr(ScriptEditor, 'gui'):
                ScriptEditor.gui = ScriptEditor()
            elif ScriptEditor.gui.isVisible():
                return
            QtWidgets.QMainWindow.show(ScriptEditor.gui)

            ui = ScriptEditor.gui
            geom = ui.geometry()
            geom = (geom.x(), geom.y(), geom.width(), geom.height())
            settings = {'geometry': geom, 'sizes': ui.splitter.sizes()}
            if 'script_editor_settings' in g.settings:
                settings.update(g.settings['script_editor_settings'])
            ui.setGeometry(*settings['geometry'])
            ui.splitter.setSizes(settings['sizes'])

    def closeEvent(self, ev):
        geom = self.geometry()
        geom = (geom.x(), geom.y(), geom.width(), geom.height())
        g.settings['script_editor_settings'] = {'geometry': geom, 'sizes': self.splitter.sizes()}

    @staticmethod
    def close():
        if hasattr(ScriptEditor, 'gui'):
            QtWidgets.QMainWindow.close(ScriptEditor.gui)

if __name__ == '__main__':
    app = QtWidgets.QApplication([])
    ScriptEditor.show()
    app.exec_()
    <|endoftext|>


<commit_before>#!/usr/bin/python

import sys
import os
from os import listdir
from os.path import isfile, join
import time
import json
import datetime
from pytz import timezone
from dateutil import parser

"""
This script will summary failed tests results and determine if any of them may be intermittents.  For
tests that are determined to be intermittents, a dictionary structure will be generated to store information
about the intermittents.

Currently, a simple threshold test is used to determine if a test is intermittent.  If the failure count of
any test exceed the threshold, we will label it as intermittent.  In particular, the following information
will be stored for each intermittent failure:
        "JenkinsJobName"
        "BuildID"
        "Timestamp"
        "GitHash"
        "TestCategory"
        "NodeName"
        "FailureMessages"
        "FailureCount"

"""

# --------------------------------------------------------------------
# Main program
# --------------------------------------------------------------------

g_test_root_dir = os.path.dirname(os.path.realpath(__file__)) # directory where we are running out code from
g_threshold_failure = 0
g_summary_dict_name = ''
g_file_start = []

g_summary_dict_intermittents = dict()
g_summary_dict_all = dict()

def init_intermittents_dict(init_dict):
    """
    initialize the fields of dictionary storing failed tests.
    :return:
    """
    init_dict["TestName"] = []
    init_dict["TestInfo"] = []

def usage():
    """
    Print USAGE help.
    """
    print("")
    print("Usage:  ")
    print("python summarizeINtermittents threshold Filename_for_dict AWS_path Failed_PyUnits_summary_dict_from ....")
    print("- threshold is an integer for which a failed test is labeled intermittent if its number of "
          "failure exceeds it.")
    print("- Filename_for_dict is a string denoting the name of the dictionary that will store the final intermittents.")
    print("- Failed_PyUnits_summary_dict_from is a string denoting the beginning of pickle files that contains"
          "")
    print("- ... denotes extra strings that represent the beginning of pickle files that you want us to summarize"
          "for you.")

def summarizeFailedRuns():
    """
    This function will look at the local directory and pick out files that have the correct start name and
    summarize the results into one giant dict.

    :return: None
    """
    global g_summary_dict_all

    onlyFiles = [x for x in listdir(g_test_root_dir) if isfile(join(g_test_root_dir, x))]   # grab files

    for f in onlyFiles:
        for fileStart in g_file_start:
            if (fileStart in f) and (os.path.getsize(f) > 10):  # found the file containing failed tests
                fFullPath = os.path.join(g_test_root_dir, f)
                try:
                    temp_dict = json.load(open(fFullPath,'r'))

                    # scrape through temp_dict and see if we need to add the test to intermittents
                    for ind in range(len(temp_dict["TestName"])):
                        addFailedTests(g_summary_dict_all, temp_dict, ind)
                except:
                    continue
                break

def addFailedTests(summary_dict, temp_dict, index):
    testName = temp_dict["TestName"][index]
    testNameList = summary_dict["TestName"]
    # check if new intermittents or old ones
    if testName in testNameList:
        testIndex =testNameList.index(testName) # update the test
        updateFailedTestInfo(summary_dict, temp_dict["TestInfo"][index], testIndex, testName, False)
    else:    # new intermittent uncovered
        summary_dict["TestName"].append(testName)
        updateFailedTestInfo(summary_dict, temp_dict["TestInfo"][index], len(summary_dict["TestName"])-1, testName, True)

def updateFailedTestInfo(summary_dict, one_test_info, testIndex, testName, newTest):
    """
    For each test, a dictionary structure will be built to record the various info about that test's failure
    information.  In particular, for each failed tests, there will be a dictionary associated with that test
    stored in the field "TestInfo" of g_faiiled_tests_info_dict.  The following fields are included:
        "JenkinsJobName": job name
        "BuildID"
        "Timestamp": in seconds
        "GitHash"
        "TestCategory": JUnit, PyUnit, RUnit or HadoopPyUnit, HadoopRUnit
        "NodeName": name of machine that the job was run on
        "FailureCount": integer counting number of times this particular test has failed.  An intermittent can be
          determined as any test with FailureCount >= 2.
        "FailureMessages": contains failure messages for the test
    :return: a new dict for that test
    """
    if newTest: # setup the dict structure to store the new data
        summary_dict["TestInfo"].append(dict())
        summary_dict["TestInfo"][testIndex]["JenkinsJobName"]=[]
        summary_dict["TestInfo"][testIndex]["BuildID"]=[]
        summary_dict["TestInfo"][testIndex]["Timestamp"]=[]
        summary_dict["TestInfo"][testIndex]["GitHash"]=[]
        summary_dict["TestInfo"][testIndex]["TestCategory"]=[]
        summary_dict["TestInfo"][testIndex]["NodeName"]=[]
        summary_dict["TestInfo"][testIndex]["FailureCount"]=0
        summary_dict["TestInfo"][testIndex]["TestName"] = testName     # add test name

    summary_dict["TestInfo"][testIndex]["JenkinsJobName"].extend(one_test_info["JenkinsJobName"])
    summary_dict["TestInfo"][testIndex]["BuildID"].extend(one_test_info["BuildID"])
    summary_dict["TestInfo"][testIndex]["Timestamp"].extend(one_test_info["Timestamp"])
    summary_dict["TestInfo"][testIndex]["GitHash"].extend(one_test_info["GitHash"])
    summary_dict["TestInfo"][testIndex]["TestCategory"].extend(one_test_info["TestCategory"])
    summary_dict["TestInfo"][testIndex]["NodeName"].extend(one_test_info["NodeName"])
    summary_dict["TestInfo"][testIndex]["FailureCount"] += one_test_info["FailureCount"]

def extractPrintSaveIntermittens():
    """
    This function will print out the intermittents onto the screen for casual viewing.  It will also print out
    where the giant summary dictionary is going to be stored.

    :return: None
    """
    # extract intermittents from collected failed tests
    global g_summary_dict_intermittents

    localtz = time.tzname[0]

    for ind in range(len(g_summary_dict_all["TestName"])):
        if g_summary_dict_all["TestInfo"][ind]["FailureCount"] >= g_threshold_failure:
            addFailedTests(g_summary_dict_intermittents, g_summary_dict_all, ind)

    for ind in range(len(g_summary_dict_intermittents["TestName"])):
        testName = g_summary_dict_intermittents["TestName"][ind]
        numberFailure = g_summary_dict_intermittents["TestInfo"][ind]["FailureCount"]
        firstFailedTS  = parser.parse(time.ctime(min(g_summary_dict_intermittents["TestInfo"][ind]["Timestamp"]))+ ' '+localtz)
        firstFailedStr = firstFailedTS.strftime("%a %b %d %H:%M:%S %Y %Z")
        recentFail = parser.parse(time.ctime(max(g_summary_dict_intermittents["TestInfo"][ind]["Timestamp"]))+ ' '+localtz)
        recentFailStr = recentFail.strftime("%a %b %d %H:%M:%S %Y %Z")
        print("Intermittent: {0} last failed at {1} and has failed {2} times since "
              "{3}.".format(testName, recentFailStr, numberFailure, firstFailedStr))
    # save dict in file
    if len(g_summary_dict_intermittents["TestName"]) > 0:
        json.dump(g_summary_dict_intermittents, open(g_summary_dict_name, 'w'))

def main(argv):
    """
    Main program.  Expect script name plus  inputs in the following order:
    - This script name
    1. threshold: integer that will denote when a failed test will be declared an intermittent
    2. string denote filename of where our final dict structure will be stored.
    3. string that denote the beginning of a file containing failed tests info.
    4. Optional strings that denote the beginning of a file containing failed tests info.

    @return: none
    """
    global g_script_name
    global g_test_root_dir
    global g_threshold_failure
    global g_file_start
    global g_summary_dict_name
    global g_summary_dict_all
    global g_summary_dict_intermittents

    if len(argv) < 5:
        print "Wrong call.  Not enough arguments.\n"
        usage()
        sys.exit(1)
    else:   # we may be in business
        g_threshold_failure = int(argv[1])
        g_summary_dict_name = os.path.join(g_test_root_dir, argv[2])

        for ind in range(3, len(argv)):
            g_file_start.append(argv[ind])

        init_intermittents_dict(g_summary_dict_all)
        init_intermittents_dict(g_summary_dict_intermittents)
        summarizeFailedRuns()
        extractPrintSaveIntermittens()

if __name__ == "__main__":
    main(sys.argv)
<commit_msg>PUBDEV-4110-intermittents.  Will store intermittent failures as a .csv file on s3 for Nikhil. (#989)<commit_after>#!/usr/bin/python

import sys
import os
from os import listdir
from os.path import isfile, join
import time
import json
import datetime
from pytz import timezone
from dateutil import parser

"""
This script will summary failed tests results and determine if any of them may be intermittents.  For
tests that are determined to be intermittents, a dictionary structure will be generated to store information
about the intermittents.

Currently, a simple threshold test is used to determine if a test is intermittent.  If the failure count of
any test exceed the threshold, we will label it as intermittent.  In particular, the following information
will be stored for each intermittent failure:
        "JenkinsJobName"
        "BuildID"
        "Timestamp"
        "GitHash"
        "TestCategory"
        "NodeName"
        "FailureMessages"
        "FailureCount"

"""

# --------------------------------------------------------------------
# Main program
# --------------------------------------------------------------------

g_test_root_dir = os.path.dirname(os.path.realpath(__file__)) # directory where we are running out code from
g_threshold_failure = 0
g_summary_dict_name = ''
g_summary_csv_filename = ''
g_file_start = []

g_summary_dict_intermittents = dict()
g_summary_dict_all = dict()

def init_intermittents_dict(init_dict):
    """
    initialize the fields of dictionary storing failed tests.
    :return:
    """
    init_dict["TestName"] = []
    init_dict["TestInfo"] = []

def usage():
    """
    Print USAGE help.
    """
    print("")
    print("Usage:  ")
    print("python summarizeINtermittents threshold Filename_for_dict AWS_path Failed_PyUnits_summary_dict_from ....")
    print("- threshold is an integer for which a failed test is labeled intermittent if its number of "
          "failure exceeds it.")
    print("- Filename_for_dict is a string denoting the name of the dictionary that will store the final intermittents.")
    print("- Failed_PyUnits_summary_dict_from is a string denoting the beginning of pickle files that contains"
          "")
    print("- ... denotes extra strings that represent the beginning of pickle files that you want us to summarize"
          "for you.")

def summarizeFailedRuns():
    """
    This function will look at the local directory and pick out files that have the correct start name and
    summarize the results into one giant dict.

    :return: None
    """
    global g_summary_dict_all

    onlyFiles = [x for x in listdir(g_test_root_dir) if isfile(join(g_test_root_dir, x))]   # grab files

    for f in onlyFiles:
        for fileStart in g_file_start:
            if (fileStart in f) and (os.path.getsize(f) > 10):  # found the file containing failed tests
                fFullPath = os.path.join(g_test_root_dir, f)
                try:
                    temp_dict = json.load(open(fFullPath,'r'))

                    # scrape through temp_dict and see if we need to add the test to intermittents
                    for ind in range(len(temp_dict["TestName"])):
                        addFailedTests(g_summary_dict_all, temp_dict, ind)
                except:
                    continue
                break

def addFailedTests(summary_dict, temp_dict, index):
    testName = temp_dict["TestName"][index]
    testNameList = summary_dict["TestName"]
    # check if new intermittents or old ones
    if testName in testNameList:
        testIndex =testNameList.index(testName) # update the test
        updateFailedTestInfo(summary_dict, temp_dict["TestInfo"][index], testIndex, testName, False)
    else:    # new intermittent uncovered
        summary_dict["TestName"].append(testName)
        updateFailedTestInfo(summary_dict, temp_dict["TestInfo"][index], len(summary_dict["TestName"])-1, testName, True)

def updateFailedTestInfo(summary_dict, one_test_info, testIndex, testName, newTest):
    """
    For each test, a dictionary structure will be built to record the various info about that test's failure
    information.  In particular, for each failed tests, there will be a dictionary associated with that test
    stored in the field "TestInfo" of g_faiiled_tests_info_dict.  The following fields are included:
        "JenkinsJobName": job name
        "BuildID"
        "Timestamp": in seconds
        "GitHash"
        "TestCategory": JUnit, PyUnit, RUnit or HadoopPyUnit, HadoopRUnit
        "NodeName": name of machine that the job was run on
        "FailureCount": integer counting number of times this particular test has failed.  An intermittent can be
          determined as any test with FailureCount >= 2.
        "FailureMessages": contains failure messages for the test
    :return: a new dict for that test
    """
    if newTest: # setup the dict structure to store the new data
        summary_dict["TestInfo"].append(dict())
        summary_dict["TestInfo"][testIndex]["JenkinsJobName"]=[]
        summary_dict["TestInfo"][testIndex]["BuildID"]=[]
        summary_dict["TestInfo"][testIndex]["Timestamp"]=[]
        summary_dict["TestInfo"][testIndex]["GitHash"]=[]
        summary_dict["TestInfo"][testIndex]["TestCategory"]=[]
        summary_dict["TestInfo"][testIndex]["NodeName"]=[]
        summary_dict["TestInfo"][testIndex]["FailureCount"]=0
        summary_dict["TestInfo"][testIndex]["TestName"] = testName     # add test name

    summary_dict["TestInfo"][testIndex]["JenkinsJobName"].extend(one_test_info["JenkinsJobName"])
    summary_dict["TestInfo"][testIndex]["BuildID"].extend(one_test_info["BuildID"])
    summary_dict["TestInfo"][testIndex]["Timestamp"].extend(one_test_info["Timestamp"])
    summary_dict["TestInfo"][testIndex]["GitHash"].extend(one_test_info["GitHash"])
    summary_dict["TestInfo"][testIndex]["TestCategory"].extend(one_test_info["TestCategory"])
    summary_dict["TestInfo"][testIndex]["NodeName"].extend(one_test_info["NodeName"])
    summary_dict["TestInfo"][testIndex]["FailureCount"] += one_test_info["FailureCount"]

def extractPrintSaveIntermittens():
    """
    This function will print out the intermittents onto the screen for casual viewing.  It will also print out
    where the giant summary dictionary is going to be stored.

    :return: None
    """
    # extract intermittents from collected failed tests
    global g_summary_dict_intermittents

    localtz = time.tzname[0]

    for ind in range(len(g_summary_dict_all["TestName"])):
        if g_summary_dict_all["TestInfo"][ind]["FailureCount"] >= g_threshold_failure:
            addFailedTests(g_summary_dict_intermittents, g_summary_dict_all, ind)

    # save dict in file
    if len(g_summary_dict_intermittents["TestName"]) > 0:
        json.dump(g_summary_dict_intermittents, open(g_summary_dict_name, 'w'))

        with open(g_summary_csv_filename, 'w') as summaryFile:
            for ind in range(len(g_summary_dict_intermittents["TestName"])):
                testName = g_summary_dict_intermittents["TestName"][ind]
                numberFailure = g_summary_dict_intermittents["TestInfo"][ind]["FailureCount"]
                firstFailedTS  = parser.parse(time.ctime(min(g_summary_dict_intermittents["TestInfo"][ind]["Timestamp"]))+
                                              ' '+localtz)
                firstFailedStr = firstFailedTS.strftime("%a %b %d %H:%M:%S %Y %Z")
                recentFail = parser.parse(time.ctime(max(g_summary_dict_intermittents["TestInfo"][ind]["Timestamp"]))+
                                          ' '+localtz)
                recentFailStr = recentFail.strftime("%a %b %d %H:%M:%S %Y %Z")
                eachTest = "{0}, {1}, {2}, {3}\n".format(testName, recentFailStr, numberFailure,
                                                       g_summary_dict_intermittents["TestInfo"][ind]["TestCategory"][0])
                summaryFile.write(eachTest)
                print("Intermittent: {0}, Last failed: {1}, Failed {2} times since "
                      "{3}".format(testName, recentFailStr, numberFailure, firstFailedStr))

def main(argv):
    """
    Main program.  Expect script name plus  inputs in the following order:
    - This script name
    1. threshold: integer that will denote when a failed test will be declared an intermittent
    2. string denote filename of where our final dict structure will be stored.
    3. string that denote the beginning of a file containing failed tests info.
    4. Optional strings that denote the beginning of a file containing failed tests info.

    @return: none
    """
    global g_script_name
    global g_test_root_dir
    global g_threshold_failure
    global g_file_start
    global g_summary_dict_name
    global g_summary_dict_all
    global g_summary_dict_intermittents
    global g_summary_csv_filename

    if len(argv) < 5:
        print "Wrong call.  Not enough arguments.\n"
        usage()
        sys.exit(1)
    else:   # we may be in business
        g_threshold_failure = int(argv[1])
        g_summary_dict_name = os.path.join(g_test_root_dir, argv[2])
        g_summary_csv_filename = g_summary_dict_name+".csv"

        for ind in range(3, len(argv)):
            g_file_start.append(argv[ind])

        init_intermittents_dict(g_summary_dict_all)
        init_intermittents_dict(g_summary_dict_intermittents)
        summarizeFailedRuns()
        extractPrintSaveIntermittens()

if __name__ == "__main__":
    main(sys.argv)
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Standard packages
import sys
import argparse

# Third-party packages
from toil.job import Job

# Package methods
from ddb import configuration
from ddb_ngsflow import gatk
from ddb_ngsflow import annotation
from ddb_ngsflow import pipeline
from ddb_ngsflow.align import bwa
from ddb_ngsflow.variation import variation
from ddb_ngsflow.variation import haplotypecaller

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('-s', '--samples_file', help="Input configuration file for samples")
    parser.add_argument('-c', '--configuration', help="Configuration file for various settings")
    Job.Runner.addToilOptions(parser)
    args = parser.parse_args()
    args.logLevel = "INFO"

    sys.stdout.write("Parsing configuration data\n")
    config = configuration.configure_runtime(args.configuration)

    sys.stdout.write("Parsing sample data\n")
    samples = configuration.configure_samples(args.samples_file, config)

    # Workflow Graph definition. The following workflow definition should create a valid Directed Acyclic Graph (DAG)
    root_job = Job.wrapJobFn(pipeline.spawn_batch_jobs, cores=1)

    # Per sample jobs
    for sample in samples:
        # Alignment and Refinement Stages
        align_job = Job.wrapJobFn(bwa.run_bwa_mem, config, sample, samples,
                                  cores=int(config['bwa']['num_cores']),
                                  memory="{}G".format(config['bwa']['max_mem']))

        add_job = Job.wrapJobFn(gatk.add_or_replace_readgroups, config, sample, align_job.rv(),
                                cores=1,
                                memory="{}G".format(config['picard-add']['max_mem']))

        dedup_job = Job.wrapJobFn(gatk.mark_duplicates, config, sample, add_job.rv(),
                                  cores=int(config['picard-dedup']['num_cores']),
                                  memory="{}G".format(config['picard-dedup']['max_mem']))

        creator_job = Job.wrapJobFn(gatk.realign_target_creator, config, sample, dedup_job.rv(),
                                    cores=int(config['gatk-realign']['num_cores']),
                                    memory="{}G".format(config['gatk-realign']['max_mem']))

        realign_job = Job.wrapJobFn(gatk.realign_indels, config, sample, add_job.rv(), creator_job.rv(),
                                    cores=1,
                                    memory="{}G".format(config['gatk-realign']['max_mem']))

        recal_job = Job.wrapJobFn(gatk.recalibrator, config, sample, realign_job.rv(),
                                  cores=int(config['gatk-recal']['num_cores']),
                                  memory="{}G".format(config['gatk-recal']['max_mem']))
        # Variant Calling
        haplotypecaller_job = Job.wrapJobFn(haplotypecaller.haplotypecaller_single, config, sample, samples,
                                            recal_job.rv(),
                                            cores=1,
                                            memory="{}G".format(config['gatk-haplotypecaller']['max_mem']))

        # Create workflow from created jobs
        root_job.addChild(align_job)
        align_job.addChild(add_job)
        add_job.addChild(dedup_job)
        dedup_job.addChild(creator_job)
        creator_job.addChild(realign_job)
        realign_job.addChild(recal_job)
        recal_job.addChild(haplotypecaller_job)

    # Need to filter for on target only results somewhere as well
    joint_call_job = Job.wrapJobFn(haplotypecaller.joint_variant_calling, config, config['project'], samples,
                                   cores=int(config['gatk-jointgenotyper']['num_cores']),
                                   memory="{}G".format(config['gatk-jointgenotyper']['max_mem']))

    normalization_job = Job.wrapJobFn(variation.vt_normalization, config, config['project'], "hc",
                                      "{}.haplotypecaller.vcf".format(config['project']),
                                      cores=1,
                                      memory="{}G".format(config['gatk']['max_mem']))

    sample_inputs = list()
    for sample in samples:
        sample = "{}.recalibrated.sorted.bam".format(sample)
        sample_inputs.append(sample)
    sample_bam_string = " -I ".join(sample_inputs)

    gatk_annotate_job = Job.wrapJobFn(gatk.annotate_vcf, config, config['project'], normalization_job.rv(), sample_bam_string,
                                      cores=int(config['gatk-annotate']['num_cores']),
                                      memory="{}G".format(config['gatk-annotate']['max_mem']))

    gatk_filter_job = Job.wrapJobFn(gatk.filter_variants, config, config['project'], gatk_annotate_job.rv(),
                                    cores=1,
                                    memory="{}G".format(config['gatk-filter']['max_mem']))

    snpeff_job = Job.wrapJobFn(annotation.snpeff, config, config['project'], gatk_filter_job.rv(),
                               cores=int(config['snpeff']['num_cores']),
                               memory="{}G".format(config['snpeff']['max_mem']))

    root_job.addFollowOn(joint_call_job)
    joint_call_job.addChild(normalization_job)
    normalization_job.addChild(gatk_annotate_job)
    gatk_annotate_job.addChild(gatk_filter_job)
    gatk_filter_job.addChild(snpeff_job)

    # Start workflow execution
    Job.Runner.startToil(root_job, args)
<commit_msg>tweak for rerunning<commit_after>#!/usr/bin/env python

# Standard packages
import sys
import argparse

# Third-party packages
from toil.job import Job

# Package methods
from ddb import configuration
from ddb_ngsflow import gatk
from ddb_ngsflow import annotation
from ddb_ngsflow import pipeline
from ddb_ngsflow.align import bwa
from ddb_ngsflow.variation import variation
from ddb_ngsflow.variation import haplotypecaller

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('-s', '--samples_file', help="Input configuration file for samples")
    parser.add_argument('-c', '--configuration', help="Configuration file for various settings")
    Job.Runner.addToilOptions(parser)
    args = parser.parse_args()
    args.logLevel = "INFO"

    sys.stdout.write("Parsing configuration data\n")
    config = configuration.configure_runtime(args.configuration)

    sys.stdout.write("Parsing sample data\n")
    samples = configuration.configure_samples(args.samples_file, config)

    # Workflow Graph definition. The following workflow definition should create a valid Directed Acyclic Graph (DAG)
    root_job = Job.wrapJobFn(pipeline.spawn_batch_jobs, cores=1)

    # Per sample jobs
    for sample in samples:
        # Alignment and Refinement Stages
        align_job = Job.wrapJobFn(bwa.run_bwa_mem, config, sample, samples,
                                  cores=int(config['bwa']['num_cores']),
                                  memory="{}G".format(config['bwa']['max_mem']))

        add_job = Job.wrapJobFn(gatk.add_or_replace_readgroups, config, sample, align_job.rv(),
                                cores=1,
                                memory="{}G".format(config['picard-add']['max_mem']))

        dedup_job = Job.wrapJobFn(gatk.mark_duplicates, config, sample, add_job.rv(),
                                  cores=int(config['picard-dedup']['num_cores']),
                                  memory="{}G".format(config['picard-dedup']['max_mem']))

        creator_job = Job.wrapJobFn(gatk.realign_target_creator, config, sample, dedup_job.rv(),
                                    cores=int(config['gatk-realign']['num_cores']),
                                    memory="{}G".format(config['gatk-realign']['max_mem']))

        realign_job = Job.wrapJobFn(gatk.realign_indels, config, sample, add_job.rv(), creator_job.rv(),
                                    cores=1,
                                    memory="{}G".format(config['gatk-realign']['max_mem']))

        recal_job = Job.wrapJobFn(gatk.recalibrator, config, sample, realign_job.rv(),
                                  cores=int(config['gatk-recal']['num_cores']),
                                  memory="{}G".format(config['gatk-recal']['max_mem']))
        # Variant Calling
        haplotypecaller_job = Job.wrapJobFn(haplotypecaller.haplotypecaller_single, config, sample, samples,
                                            recal_job.rv(),
                                            cores=1,
                                            memory="{}G".format(config['gatk-haplotypecaller']['max_mem']))

        # Create workflow from created jobs
        # root_job.addChild(align_job)
        # align_job.addChild(add_job)
        # add_job.addChild(dedup_job)
        # dedup_job.addChild(creator_job)
        # creator_job.addChild(realign_job)
        # realign_job.addChild(recal_job)
        # recal_job.addChild(haplotypecaller_job)

    # Need to filter for on target only results somewhere as well
    joint_call_job = Job.wrapJobFn(haplotypecaller.joint_variant_calling, config, config['project'], samples,
                                   cores=int(config['gatk-jointgenotyper']['num_cores']),
                                   memory="{}G".format(config['gatk-jointgenotyper']['max_mem']))

    normalization_job = Job.wrapJobFn(variation.vt_normalization, config, config['project'], "hc",
                                      "{}.haplotypecaller.vcf".format(config['project']),
                                      cores=1,
                                      memory="{}G".format(config['gatk']['max_mem']))

    sample_inputs = list()
    for sample in samples:
        sample = "{}.recalibrated.sorted.bam".format(sample)
        sample_inputs.append(sample)
    sample_bam_string = " -I ".join(sample_inputs)

    gatk_annotate_job = Job.wrapJobFn(gatk.annotate_vcf, config, config['project'], normalization_job.rv(), sample_bam_string,
                                      cores=int(config['gatk-annotate']['num_cores']),
                                      memory="{}G".format(config['gatk-annotate']['max_mem']))

    gatk_filter_job = Job.wrapJobFn(gatk.filter_variants, config, config['project'], gatk_annotate_job.rv(),
                                    cores=1,
                                    memory="{}G".format(config['gatk-filter']['max_mem']))

    snpeff_job = Job.wrapJobFn(annotation.snpeff, config, config['project'], gatk_filter_job.rv(),
                               cores=int(config['snpeff']['num_cores']),
                               memory="{}G".format(config['snpeff']['max_mem']))

    # root_job.addFollowOn(joint_call_job)
    root_job.addChild(joint_call_job)
    joint_call_job.addChild(normalization_job)
    normalization_job.addChild(gatk_annotate_job)
    gatk_annotate_job.addChild(gatk_filter_job)
    gatk_filter_job.addChild(snpeff_job)

    # Start workflow execution
    Job.Runner.startToil(root_job, args)
<|endoftext|>


<commit_before>#!/usr/bin/env python
import sendgrid
import log

@log.make_loggable
class Mailer(object):
    def __init__(self, cfg):
        self.token = cfg.token
        self.client = sendgrid.SendGridClient(self.token)

    def mail(self, mail_to, mail_from, subject, body, headers={}):
        message = sendgrid.Mail()

        message.add_to(mail_to)
        message.set_from(mail_from)
        message.set_subject(subject)
        message.set_html(body)

        self.client.send(message)
<commit_msg>Use newer sendgrid api<commit_after>#!/usr/bin/env python
import sendgrid
from sendgrid.helpers.mail import *
import log

@log.make_loggable
class Mailer(object):
    def __init__(self, cfg):
        self.token = cfg.token
        self.sg = sendgrid.SendGridAPIClient(apikey=self.token)

    def mail(self, mail_to, mail_from, subject, body, headers={}):
        message = sendgrid.Mail()

        message.add_to(mail_to)
        message.set_from(mail_from)
        message.set_subject(subject)
        message.set_html(body)

        self.client.send(message)

        content = Content("text/html", body)
        mail = Mail(Email(mail_from), subject, Email(mail_to), content)

        return self.sg.client.mail.send.post(request_body=mail.get())

@log.make_loggable
class Formatter(object):
    def __init__(self, template):
        pass

    def format(self, data):
        pass
<|endoftext|>


<commit_before>#! /usr/bin/env python

import wikipedia
import json
from subprocess import call
from PIL import Image
import requests
from io import BytesIO

freeciv_wiki_doc = {};

def fix_tech(tech_name):
  if (tech_name.startswith("?")):
    tech_name = tech_name.partition(':')[2];

  if (tech_name == "Advanced Flight"): tech_name = "Aeronautics";
  if (tech_name == "AWACS"): tech_name = "Boeing E-3 Sentry";
  if (tech_name == "Rocketry"): tech_name = "Rocket";
  if (tech_name == "Stealth"): tech_name = "Stealth technology";
  if (tech_name == "Tactics"): tech_name = "Military tactics";
  if (tech_name == "The Republic"): tech_name = "Repubic";
  if (tech_name == "The Corporation"): tech_name = "Corporation";
  if (tech_name == "The Wheel"): tech_name = "Wheel";
  if (tech_name == "Archers"): tech_name = "Archery";
  if (tech_name == "Partisan"): tech_name = "Partisan military";
  if (tech_name == "Horsemen"): tech_name = "Equestrianism";
  if (tech_name == "Fighter"): tech_name = "Fighter aircraft";
  if (tech_name == "Carrier"): tech_name = "Aircraft carrier";
  if (tech_name == "Legion"): tech_name = "Roman legion";
  if (tech_name == "Magellan's Expedition"): tech_name = "Ferdinand Magellan";
  if (tech_name == "Nuclear"): tech_name = "Nuclear weapon";
  if (tech_name == "Caravan"): tech_name = "Camel train";
  if (tech_name == "Aqueduct"): tech_name = "Aqueduct water supply";
  if (tech_name == "Barracks II"): tech_name = "Barracks";
  if (tech_name == "Barracks III"): tech_name = "Barracks";
  if (tech_name == "Mfg. Plant"): tech_name = "Factory";
  if (tech_name == "Sewer System"): tech_name = "Sanitary_sewer";
  if (tech_name == "Space Structural"): tech_name = "Spacecraft";
  if (tech_name == "A.Smith's Trading Co."): tech_name = "Adam Smith";
  if (tech_name == "Colossus"): tech_name = "Colossus of Rhodes";
  if (tech_name == "Michelangelo's Chapel"): tech_name = "Sistine Chapel";
  if (tech_name == "Shakespeare's Theater"): tech_name = "William Shakespeare";
  if (tech_name == "Coinage"): tech_name = "Coining (mint)";
  if (tech_name == "SDI Defense"): tech_name = "Strategic Defense Initiative";
  if (tech_name == "Coastal Defense"): tech_name = "Coastal defence and fortification";
  if (tech_name == "Copernicus' Observatory"): tech_name = "Space observatory";
  if (tech_name == "Mech. Inf."): tech_name = "Mechanized infantry";
  if (tech_name == "Sun Tzu's War Academy"): tech_name = "Sun Tzu";
  if (tech_name == "Mobile Warfare"): tech_name = "Maneuver warfare";
  if (tech_name == "Leonardo's Workshop"): tech_name = "Leonardo da Vinci";
  if (tech_name == "SETI Program"): tech_name = "Search for extraterrestrial intelligence";
  if (tech_name == "J.S. Bach's Cathedral"): tech_name = "Johann Sebastian Bach";
  if (tech_name == "Marco Polo's Embassy"): tech_name = "Marco Polo";
  if (tech_name == "Mass Transit"): tech_name = "Public transport";
  if (tech_name == "Spy"): tech_name = "Espionage";
  if (tech_name == "Space Component"): tech_name = "Spacecraft propulsion";
  if (tech_name == "Space Module"): tech_name = "Life support system";
  if (tech_name == "Space Structural"): tech_name = "Spacecraft design";
  return tech_name;

def validate_image(image_url):
  return ((".png" in image_url.lower() or ".jpg" in image_url.lower())
		  and not "Ambox" in image_url
		  and not "Berthabenzportrait" in image_url
		  and not "Great_wall_of_china-mutianyu_3" in image_url
		  and not "Chevalier" in image_url
		  and not "Nuvola_apps_ksysv" in image_url
		  and not "mile_Levassor" in image_url
		  and not "Place_de_la_R" in image_url
		  and not "Elizabeth" in image_url
                  and "Writing_systems_worldwide" not in image_url);

def download_wiki_page(tech_name):
  image_width = 500;
  max_height = 450;

  print(tech_name + " -> " + fix_tech(tech_name));
  page = wikipedia.page(fix_tech(tech_name), auto_suggest=True, redirect=True);

  image = None;
  # FIXME: page.images seems to be in random order, so we'll get a random image from Wikipedia.
  for i in range(len(page.images)):
    if validate_image(page.images[i]):
      image = page.images[i];
      break;

  if image != None:
    response = requests.get(image)
    img = Image.open(BytesIO(response.content))
    wpercent = (image_width/float(img.size[0]))
    hsize = int((float(img.size[1])*float(wpercent)))
    if (hsize > max_height):
      hsize = max_height;
    img = img.resize((image_width,hsize), Image.ANTIALIAS)
    img.convert('RGB').save("../freeciv-web/src/main/webapp/images/wiki/" + page.title + ".jpg");
    image = page.title + ".jpg";

  freeciv_wiki_doc[tech_name] = {"title" : page.title, "summary" : page.summary, "image" : image};

# FIXME: extract item names from the other supported rulesets too. An item
# name that don't appear in classic may still appear in another supported
# ruleset.

f = open('../freeciv/freeciv/data/classic/techs.ruleset')
lines = f.readlines()
f.close()

techs = []
for line in lines:
  if line.startswith("name"):
    tech_line = line.split("\"");
    techs.append(tech_line[1]);

f = open('../freeciv/freeciv/data/classic/units.ruleset')
lines = f.readlines()
f.close()

for line in lines:
  if line.startswith("name"):
    tech_line = line.split("\"");
    if ("unitclass" in tech_line[1]): continue;
    result_tech = tech_line[1].replace("?unit:", "");
    techs.append(result_tech);

f = open('../freeciv/freeciv/data/classic/buildings.ruleset')
lines = f.readlines()
f.close()

for line in lines:
  if line.startswith("name"):
    tech_line = line.split("\"");
    techs.append(tech_line[1]);

f = open('../freeciv/freeciv/data/classic/governments.ruleset')
lines = f.readlines()
f.close()

for line in lines:
  if line.startswith("name"):
    tech_line = line.split("\"");
    techs.append(tech_line[1]);

for tech in techs:
  download_wiki_page(tech);

f = open('../freeciv-web/src/main/webapp/javascript/freeciv-wiki-doc.js' ,'w');
f.write("var freeciv_wiki_docs = "
        + json.dumps(freeciv_wiki_doc, sort_keys=True, indent=2) + ";\n");
f.close();
print("Please verify manually that the images from Wikipedia are suited for the game and players of all ages.");
print("Downloading tech summaries from Wikipedia complete!");
<commit_msg>Remove redundant string qualifier removal.<commit_after>#! /usr/bin/env python

import wikipedia
import json
from subprocess import call
from PIL import Image
import requests
from io import BytesIO

freeciv_wiki_doc = {};

def fix_tech(tech_name):
  if (tech_name.startswith("?")):
    tech_name = tech_name.partition(':')[2];

  if (tech_name == "Advanced Flight"): tech_name = "Aeronautics";
  if (tech_name == "AWACS"): tech_name = "Boeing E-3 Sentry";
  if (tech_name == "Rocketry"): tech_name = "Rocket";
  if (tech_name == "Stealth"): tech_name = "Stealth technology";
  if (tech_name == "Tactics"): tech_name = "Military tactics";
  if (tech_name == "The Republic"): tech_name = "Repubic";
  if (tech_name == "The Corporation"): tech_name = "Corporation";
  if (tech_name == "The Wheel"): tech_name = "Wheel";
  if (tech_name == "Archers"): tech_name = "Archery";
  if (tech_name == "Partisan"): tech_name = "Partisan military";
  if (tech_name == "Horsemen"): tech_name = "Equestrianism";
  if (tech_name == "Fighter"): tech_name = "Fighter aircraft";
  if (tech_name == "Carrier"): tech_name = "Aircraft carrier";
  if (tech_name == "Legion"): tech_name = "Roman legion";
  if (tech_name == "Magellan's Expedition"): tech_name = "Ferdinand Magellan";
  if (tech_name == "Nuclear"): tech_name = "Nuclear weapon";
  if (tech_name == "Caravan"): tech_name = "Camel train";
  if (tech_name == "Aqueduct"): tech_name = "Aqueduct water supply";
  if (tech_name == "Barracks II"): tech_name = "Barracks";
  if (tech_name == "Barracks III"): tech_name = "Barracks";
  if (tech_name == "Mfg. Plant"): tech_name = "Factory";
  if (tech_name == "Sewer System"): tech_name = "Sanitary_sewer";
  if (tech_name == "Space Structural"): tech_name = "Spacecraft";
  if (tech_name == "A.Smith's Trading Co."): tech_name = "Adam Smith";
  if (tech_name == "Colossus"): tech_name = "Colossus of Rhodes";
  if (tech_name == "Michelangelo's Chapel"): tech_name = "Sistine Chapel";
  if (tech_name == "Shakespeare's Theater"): tech_name = "William Shakespeare";
  if (tech_name == "Coinage"): tech_name = "Coining (mint)";
  if (tech_name == "SDI Defense"): tech_name = "Strategic Defense Initiative";
  if (tech_name == "Coastal Defense"): tech_name = "Coastal defence and fortification";
  if (tech_name == "Copernicus' Observatory"): tech_name = "Space observatory";
  if (tech_name == "Mech. Inf."): tech_name = "Mechanized infantry";
  if (tech_name == "Sun Tzu's War Academy"): tech_name = "Sun Tzu";
  if (tech_name == "Mobile Warfare"): tech_name = "Maneuver warfare";
  if (tech_name == "Leonardo's Workshop"): tech_name = "Leonardo da Vinci";
  if (tech_name == "SETI Program"): tech_name = "Search for extraterrestrial intelligence";
  if (tech_name == "J.S. Bach's Cathedral"): tech_name = "Johann Sebastian Bach";
  if (tech_name == "Marco Polo's Embassy"): tech_name = "Marco Polo";
  if (tech_name == "Mass Transit"): tech_name = "Public transport";
  if (tech_name == "Spy"): tech_name = "Espionage";
  if (tech_name == "Space Component"): tech_name = "Spacecraft propulsion";
  if (tech_name == "Space Module"): tech_name = "Life support system";
  if (tech_name == "Space Structural"): tech_name = "Spacecraft design";
  return tech_name;

def validate_image(image_url):
  return ((".png" in image_url.lower() or ".jpg" in image_url.lower())
		  and not "Ambox" in image_url
		  and not "Berthabenzportrait" in image_url
		  and not "Great_wall_of_china-mutianyu_3" in image_url
		  and not "Chevalier" in image_url
		  and not "Nuvola_apps_ksysv" in image_url
		  and not "mile_Levassor" in image_url
		  and not "Place_de_la_R" in image_url
		  and not "Elizabeth" in image_url
                  and "Writing_systems_worldwide" not in image_url);

def download_wiki_page(tech_name):
  image_width = 500;
  max_height = 450;

  print(tech_name + " -> " + fix_tech(tech_name));
  page = wikipedia.page(fix_tech(tech_name), auto_suggest=True, redirect=True);

  image = None;
  # FIXME: page.images seems to be in random order, so we'll get a random image from Wikipedia.
  for i in range(len(page.images)):
    if validate_image(page.images[i]):
      image = page.images[i];
      break;

  if image != None:
    response = requests.get(image)
    img = Image.open(BytesIO(response.content))
    wpercent = (image_width/float(img.size[0]))
    hsize = int((float(img.size[1])*float(wpercent)))
    if (hsize > max_height):
      hsize = max_height;
    img = img.resize((image_width,hsize), Image.ANTIALIAS)
    img.convert('RGB').save("../freeciv-web/src/main/webapp/images/wiki/" + page.title + ".jpg");
    image = page.title + ".jpg";

  freeciv_wiki_doc[tech_name] = {"title" : page.title, "summary" : page.summary, "image" : image};

# FIXME: extract item names from the other supported rulesets too. An item
# name that don't appear in classic may still appear in another supported
# ruleset.

f = open('../freeciv/freeciv/data/classic/techs.ruleset')
lines = f.readlines()
f.close()

techs = []
for line in lines:
  if line.startswith("name"):
    tech_line = line.split("\"");
    techs.append(tech_line[1]);

f = open('../freeciv/freeciv/data/classic/units.ruleset')
lines = f.readlines()
f.close()

for line in lines:
  if line.startswith("name"):
    tech_line = line.split("\"");
    if ("unitclass" in tech_line[1]): continue;
    techs.append(tech_line[1]);

f = open('../freeciv/freeciv/data/classic/buildings.ruleset')
lines = f.readlines()
f.close()

for line in lines:
  if line.startswith("name"):
    tech_line = line.split("\"");
    techs.append(tech_line[1]);

f = open('../freeciv/freeciv/data/classic/governments.ruleset')
lines = f.readlines()
f.close()

for line in lines:
  if line.startswith("name"):
    tech_line = line.split("\"");
    techs.append(tech_line[1]);

for tech in techs:
  download_wiki_page(tech);

f = open('../freeciv-web/src/main/webapp/javascript/freeciv-wiki-doc.js' ,'w');
f.write("var freeciv_wiki_docs = "
        + json.dumps(freeciv_wiki_doc, sort_keys=True, indent=2) + ";\n");
f.close();
print("Please verify manually that the images from Wikipedia are suited for the game and players of all ages.");
print("Downloading tech summaries from Wikipedia complete!");
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright 2018 Telefonica Investigacion y Desarrollo, S.A.U
#
# This file is part of Orion Context Broker.
#
# Orion Context Broker is free software: you can redistribute it and/or
# modify it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# Orion Context Broker is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
# General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/.
#
# For those usages not covered by this license please contact with
# iot_support at tid dot es

import requests
import sys
requests.packages.urllib3.disable_warnings()

__author__ = 'fermin'

##############################################################
# BEGIN of the configuration part (don't touch above this line ;)

cb_endpoint = 'http://localhost:1026'

headers = {
    'fiware-service': 'service',
    'fiware-servicepath': '/subservice',
    'x-auth-token': 'token'
}

page_size = 500

# END of the configuration part (don't touch below this line ;)
##############################################################

def get_ngsiv1_location(entity):
    """
    Search for NGSIv1-style location, i.e. a "location" metadata in some of the attributes of the given entity.

    FIXME: currently metadata location doesn't appear in NGSIv2 results. This method could be pretty useless
    until https://github.com/telefonicaid/fiware-orion/issues/3122 gets solved, but we leave here for the future
    in any case.

    :param entity: the entity in which search location
    :return: the attribute which holds location, 'None' if no one was found
    """
    for field in entity:
        if field == 'id' or field == 'type':
            continue
        for m in entity[field]['metadata']:
            if m == 'location':
                return field
    return None

def get_ngsiv2_location(entity):
    """
    Search for NGSIv2-style location, i.e. a "geo:point" attribute in the given entity.

    :param entity: the entity in which search location
    :return: the attribute which holds location, 'None' if no one was found
    """
    for field in entity:
        if field == 'id' or field == 'type':
            continue
        if entity[field]['type'] == 'geo:point':
            return field
    return None

def process_page(entities):
    """
    Process a page of entities

    :param entities: page of entities (a list) to be processed
    """
    for entity in entities:
        loc_attr = get_ngsiv2_location(entity)
        if loc_attr is None:
            loc_attr = get_ngsiv1_location(entity)
        if loc_attr is not None:
            print '%s: %s' % (entity['id'], entity[loc_attr]['value'])

### Main program starts here ###

# Get first bunch of data
res = requests.get('%s/v2/entities?offset=0&limit=%s&options=count' % (cb_endpoint, str(page_size)),
                   headers=headers, verify=False)
if res.status_code != 200:
        print 'Error getting entities (%d): %s' % (res.status_code, res.json())
        sys.exit(1)

# Get count
count = int(res.headers['fiware-total-count'])

# Process the response
process_page(res.json())

# Number of batches
pages = count / page_size
for i in range(0, pages):
    offset = (i + 1) * 10
    res = requests.get('%s/v2/entities?offset=%s&limit=%s' % (cb_endpoint, str(offset), str(page_size)),
                       headers=headers, verify=False)
    if res.status_code != 200:
        print 'Error getting entities (%d): %s' % (res.status_code, res.json())
        sys.exit(1)
    process_page(res.json())

print 'Hints:'
print '* use "awk -F \': \' \'{print $2}\'" filter if you want to grab just the coordinates'
print '* copy-paste output to https://www.darrinward.com/lat-long so see where you entities are on a map'<commit_msg>FIX issue raised in PR<commit_after>#!/usr/bin/env python
# Copyright 2018 Telefonica Investigacion y Desarrollo, S.A.U
#
# This file is part of Orion Context Broker.
#
# Orion Context Broker is free software: you can redistribute it and/or
# modify it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# Orion Context Broker is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
# General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/.
#
# For those usages not covered by this license please contact with
# iot_support at tid dot es

import requests
import sys
requests.packages.urllib3.disable_warnings()

__author__ = 'fermin'

##############################################################
# BEGIN of the configuration part (don't touch above this line ;)

cb_endpoint = 'http://localhost:1026'

headers = {
    'fiware-service': 'service',
    'fiware-servicepath': '/subservice',
    'x-auth-token': 'token'
}

page_size = 500

# END of the configuration part (don't touch below this line ;)
##############################################################

def get_ngsiv1_location(entity):
    """
    Search for NGSIv1-style location, i.e. a "location" metadata in some of the attributes of the given entity.

    FIXME: currently metadata location doesn't appear in NGSIv2 results. This method could be pretty useless
    until https://github.com/telefonicaid/fiware-orion/issues/3122 gets solved, but we leave here for the future
    in any case.

    :param entity: the entity in which search location
    :return: the attribute which holds location, 'None' if no one was found
    """
    for field in entity:
        if field == 'id' or field == 'type':
            continue
        for m in entity[field]['metadata']:
            if m == 'location':
                return field
    return None

def get_ngsiv2_location(entity):
    """
    Search for NGSIv2-style location, i.e. a "geo:point" attribute in the given entity.

    :param entity: the entity in which search location
    :return: the attribute which holds location, 'None' if no one was found
    """
    for field in entity:
        if field == 'id' or field == 'type':
            continue
        if entity[field]['type'] == 'geo:point':
            return field
    return None

def process_page(entities):
    """
    Process a page of entities

    :param entities: page of entities (a list) to be processed
    """
    for entity in entities:
        loc_attr = get_ngsiv2_location(entity)
        if loc_attr is None:
            loc_attr = get_ngsiv1_location(entity)
        if loc_attr is not None:
            print '%s: %s' % (entity['id'], entity[loc_attr]['value'])

### Main program starts here ###

# Get first bunch of data
res = requests.get('%s/v2/entities?offset=0&limit=%s&options=count' % (cb_endpoint, str(page_size)),
                   headers=headers, verify=False)
if res.status_code != 200:
        print 'Error getting entities (%d): %s' % (res.status_code, res.json())
        sys.exit(1)

# Get count
count = int(res.headers['fiware-total-count'])

# Process the response
process_page(res.json())

# Number of batches
pages = count / page_size
for i in range(0, pages):
    offset = (i + 1) * 10
    res = requests.get('%s/v2/entities?offset=%s&limit=%s' % (cb_endpoint, str(offset), str(page_size)),
                       headers=headers, verify=False)
    if res.status_code != 200:
        print 'Error getting entities (%d): %s' % (res.status_code, res.json())
        sys.exit(1)
    process_page(res.json())

print 'Hints:'
print '* use "awk -F \': \' \'{print $2}\'" filter if you want to grab just the coordinates'
print '* copy-paste output to https://www.darrinward.com/lat-long so see where you entities are on a map'
<|endoftext|>


<commit_before>#!/usr/local/bin/python

import argparse
import re
import pprint

pp = pprint.PrettyPrinter(indent=2)

parser = argparse.ArgumentParser(description="Process Fortigate configuration \
	and create communication matrix excel sheet.")
parser.add_argument('-f', action='store',
                   metavar='<configuration-file>',
                   help='path to configuration file',
                   required=True)

args = parser.parse_args()
CONFIGFILE = vars(args)['f']
print ("Parsing Configuration File: %s" % CONFIGFILE)

try:
	fullconfigstr = open(CONFIGFILE, 'r').read()
except:
	print("Error reading config file: %s" % CONFIGFILE)

fullconfiglines = fullconfigstr.splitlines()

allpolicies = fullconfiglines[fullconfiglines.index('config firewall policy')+1:]
allpolicies = allpolicies[:allpolicies.index('end')]

policydict = dict()

for line in allpolicies:
	try:
		if line.strip().startswith('edit'):
			policyid = re.match(r'edit (\d*)', line.strip()).groups()[0]
			policydict[policyid] = dict()
		elif line.strip() != 'next' and line.strip().startswith('set'):
			key, val = re.match(r'^set (\S*) (.+)$', line.strip()).groups()
			policydict[policyid][key] = val
	except:
		print "Error on line: %s" % line
		raise

print ("Total policies: %d" % len(policydict.keys()))

for pid in policydict.keys():
	for key in policydict[pid]:
		if key == 'service' or key == 'srcaddr' or key == 'dstaddr':
			policydict[pid][key] = policydict[pid][key].replace('" "', '","').replace('"', '')
			if key == 'service':
				policydict[pid][key] = policydict[pid][key].replace('_', '-')
			policydict[pid][key] = policydict[pid][key].split(',')
	pp.pprint(policydict[pid])
<commit_msg>Output to xlsx implemented.<commit_after>#!/usr/local/bin/python

import argparse
import re
import pprint
import openpyxl

pp = pprint.PrettyPrinter(indent=2)

parser = argparse.ArgumentParser(description="Process Fortigate configuration \
	and create communication matrix excel sheet.")
parser.add_argument('-f', action='store',
                   metavar='<configuration-file>',
                   help='path to configuration file',
                   required=True)

parser.add_argument('-o', action='store',
					metavar='<output-file>.xls',
					help='path to configuration file',
					required=False)

args = parser.parse_args()
CONFIGFILE = vars(args)['f']
print ("Parsing Configuration File: %s" % CONFIGFILE)

try:
	fullconfigstr = open(CONFIGFILE, 'r').read()
except:
	print("Error reading config file: %s" % CONFIGFILE)

fullconfiglines = fullconfigstr.splitlines()

allpolicies = fullconfiglines[fullconfiglines.index('config firewall policy')+1:]
allpolicies = allpolicies[:allpolicies.index('end')]

policydict = dict()

for line in allpolicies:
	try:
		if line.strip().startswith('edit'):
			policyid = re.match(r'edit (\d*)', line.strip()).groups()[0]
			policydict[policyid] = dict()
		elif line.strip() != 'next' and line.strip().startswith('set'):
			key, val = re.match(r'^set (\S*) (.+)$', line.strip()).groups()
			policydict[policyid][key] = val
	except:
		print "Error on line: %s" % line
		raise

print ("Total policies: %d" % len(policydict.keys()))

if 'o' in vars(args):
    wb = openpyxl.Workbook()
    sheet = wb.get_active_sheet()
    sheet.title = "Policies"
    row = 2
    columns = list()
    columns.append('id')
    for pid in policydict.keys():
        for key in policydict[pid]:
            if not key in columns:
                columns.append(key)
            if key == 'service' or key == 'srcaddr' or key == 'dstaddr':
                policydict[pid][key] = policydict[pid][key].replace('" "', '","').replace('"', '')
                if key == 'service':
                    policydict[pid][key] = policydict[pid][key].replace('_', '-')
                policydict[pid][key] = policydict[pid][key].split(',')

    for pid in policydict.keys():
        sheet.cell(row=row, column=columns.index('id') + 1).value = pid
        for key in policydict[pid]:
            if type(policydict[pid][key]) == type(list()):
                val = ',\n'.join(policydict[pid][key])
                sheet.cell(row=row, column=columns.index(key)+1).value = val
            else:
                sheet.cell(row=row, column=columns.index(key)+1).value = policydict[pid][key]
        row += 1
        pp.pprint(policydict[pid])
    # Write the header
    for item in columns:
        sheet.cell(row=1, column=columns.index(item) + 1).value = item
    wb.save(vars(args)['o'])<|endoftext|>


<commit_before>from django import forms

from fortuitus.feditor.models import TestCase

class TestCaseForm(BootstrapFormMixin, forms.ModelForm):
    class Meta:
        model = TestCase
        exclude = ('project',)<commit_msg>Fix NameError<commit_after>from django import forms

from fortuitus.feditor.models import TestCase

class TestCaseForm(forms.ModelForm):
    class Meta:
        model = TestCase
        exclude = ('project',)
<|endoftext|>


<commit_before>from twisted.trial import unittest

import x256

class Testx256(unittest.TestCase):
    """
    Test class for x256 module.
    """

    def setUp(self):
        self.rgb = [220, 40, 150]
        self.xcolor = 162
        self.hex = 'DC2896'
        self.aprox_hex = 'D7087'
        self.aprox_rgb = [215, 0, 135]

    def test_from_rgb(self):
        color = x256.from_rgb(self.rgb)
        self.assertEqual(self.xcolor, color)
        color = x256.from_rgb(self.rgb[0], self.rgb[1], self.rgb[2])
        self.assertEqual(self.xcolor, color)

    def test_from_rgb(self):
        color = x256.from_hex(self.hex)
        self.assertEqual(self.xcolor, color)

    def test_to_rgb(self):
        color = x256.to_rgb(self.xcolor)
        self.assertEqual(self.aprox_rgb, color)

    def test_to_hex(self):
        color = x256.to_hex(self.xcolor)
        self.assertEqual(self.aprox_hex, color)<commit_msg>Fix some bugs in tests<commit_after>from twisted.trial import unittest

from x256 import x256

class Testx256(unittest.TestCase):
    """
    Test class for x256 module.
    """

    def setUp(self):
        self.rgb = [220, 40, 150]
        self.xcolor = 162
        self.hex = 'DC2896'
        self.aprox_hex = 'D7087'
        self.aprox_rgb = [215, 0, 135]

    def test_from_rgb(self):
        color = x256.from_rgb(self.rgb)
        self.assertEqual(self.xcolor, color)
        color = x256.from_rgb(self.rgb[0], self.rgb[1], self.rgb[2])
        self.assertEqual(self.xcolor, color)

    def test_from_hex(self):
        color = x256.from_hex(self.hex)
        self.assertEqual(self.xcolor, color)

    def test_to_rgb(self):
        color = x256.to_rgb(self.xcolor)
        self.assertEqual(self.aprox_rgb, color)

    def test_to_hex(self):
        color = x256.to_hex(self.xcolor)
        self.assertEqual(self.aprox_hex, color)
<|endoftext|>