<commit_before>#!/usr/bin/env python
# encoding: utf-8
"""Main application class of vimiv."""

import os
import sys
from gi import require_version
require_version('Gtk', '3.0')
from gi.repository import Gtk, Gdk, Gio, GLib
from vimiv.configparser import parse_dirs, parse_config
from vimiv.image import Image
from vimiv.fileactions import FileExtras, populate
from vimiv.library import Library
from vimiv.thumbnail import Thumbnail
from vimiv.manipulate import Manipulate
from vimiv.statusbar import Statusbar
from vimiv.commandline import CommandLine
from vimiv.commands import Commands
from vimiv.completions import VimivComplete
from vimiv.slideshow import Slideshow
from vimiv.keyhandler import KeyHandler
from vimiv.tags import TagHandler
from vimiv.mark import Mark
from vimiv.information import Information
from vimiv.window import Window

class Vimiv(Gtk.Application):
    """Main vimiv application class inheriting from Gtk.Application.

    Attributes:
        settings: Settings from configfiles to use.
        paths: List of paths for images.
        index: Current position in paths.
        widgets: Dictionary of vimiv widgets.
            widgets[widget-name] = Gtk.Widget
        directory: Directory in which configfiles and data are stored.
        commands: Dictionary of commands.
            commands[command-name] = [function, *args]
        aliases: Dicionary of aliases to commands.
            aliases[alias-name] = command-name
        functions: Dictionary of functions. Includes all commands and additional
            functions that cannot be called from the commandline.
        screensize: Available screensize.
    """

    def __init__(self, application_id="org.vimiv"):
        """Create the Gtk.Application and connect the activate signal.

        Args:
            application_id: The ID used to register vimiv. Default: org.vimiv.
        """
        # Create directory structure and get settings
        parse_dirs()
        Gtk.Application.__init__(self, application_id=application_id)
        self.set_flags(Gio.ApplicationFlags.HANDLES_OPEN)
        self.connect("activate", self.activate_vimiv)
        self.settings = parse_config()
        self.paths = []
        self.index = 0
        self.widgets = {}
        self.directory = os.path.expanduser("~/.vimiv")
        self.commands = {}
        self.aliases = {}
        self.functions = {}
        # Sceen and window size TODO put somewhere else
        screen = Gdk.Screen()
        self.screensize = [screen.width(), screen.height()]
        # Set up all commandline options
        self.init_commandline_options()

    # Any different number of arguments will fail
    # pylint: disable=arguments-differ
    def do_open(self, files, n_files, hint):
        """Open files by populating self.paths and self.index.

        Args:
            files: List of Gio.Files.
            n_files: Number of files.
            hint: Special string for user, always empty in vimiv.
        """
        filenames = [fil.get_path() for fil in files]
        # Move to the directory of the first argument
        if os.path.isdir(filenames[0]):
            working_directory = filenames[0]
        else:
            working_directory = os.path.dirname(filenames[0])
        if os.path.exists(working_directory):
            os.chdir(working_directory)
        # Populate list of images
        recursive = self.settings["GENERAL"]["recursive"]
        shuffle = self.settings["GENERAL"]["shuffle"]
        self.paths, self.index = populate(filenames, recursive, shuffle)

        # Activate vimiv after opening files
        self.activate_vimiv(self)

    # Any different number of arguments will fail
    # pylint: disable=arguments-differ
    def do_handle_local_options(self, options):
        """Handle commandline arguments.

        Args:
            command_line:
        """
        if options.contains("version"):
            information = Information()
            print(information.get_version())
            return 0  # To exit
        if options.contains("bar"):
            self.settings["GENERAL"]["display_bar"] = True
        elif options.contains("no-bar"):
            self.settings["GENERAL"]["display_bar"] = False
        if options.contains("library"):
            self.settings["LIBRARY"]["show_library"] = True
        elif options.contains("no-library"):
            self.settings["LIBRARY"]["show_library"] = False
        if options.contains("fullscreen"):
            self.settings["GENERAL"]["start_fullscreen"] = True
        elif options.contains("no-fullscreen"):
            self.settings["GENERAL"]["start_fullscreen"] = False
        if options.contains("shuffle"):
            self.settings["GENERAL"]["shuffle"] = True
        elif options.contains("no-shuffle"):
            self.settings["GENERAL"]["shuffle"] = False
        if options.contains("recursive"):
            self.settings["GENERAL"]["recursive"] = True
        elif options.contains("no-recursive"):
            self.settings["GENERAL"]["recursive"] = False
        if options.contains("slideshow"):
            self.settings["GENERAL"]["start_slideshow"] = True
        if options.contains("start-from-desktop"):
            os.chdir(self.settings["LIBRARY"]["desktop_start_dir"])
        elif not sys.stdin.isatty():
            tty_paths = []
            for line in sys.stdin:
                tty_paths.append(line.rstrip("\n"))
            self.paths, self.index = populate(tty_paths)
        if options.contains("slideshow-delay"):
            delay = options.lookup_value("slideshow-delay").unpack()
            self.settings["GENERAL"]["slideshow_delay"] = delay
        if options.contains("geometry"):
            geometry = options.lookup_value("geometry").unpack()
            self.settings["GENERAL"]["geometry"] = geometry
        return -1  # To continue

    def activate_vimiv(self, app):
        """Starting point for the vimiv application.

        Args:
            app: The application itself.
        """
        self.init_widgets()
        self.create_window_structure()
        app.add_window(self["window"])
        # Show everything and then hide whatever needs to be hidden
        self["window"].show_all()
        self["manipulate"].scrolled_win.hide()
        self["commandline"].grid.hide()
        # Statusbar depending on setting
        if self["statusbar"].hidden:
            self["statusbar"].bar.hide()
        # Try to generate imagelist recursively from the current directory if
        # recursive is given and not paths exist
        if self.settings["GENERAL"]["recursive"] and not self.paths:
            shuffle = self.settings["GENERAL"]["shuffle"]
            self.paths, self.index = populate([os.getcwd()], True, shuffle)
        # Show the image if an imagelist exists
        if self.paths:
            self["image"].move_index(True, False, 0)
            # Show library at the beginning?
            if not self["library"].show_at_start:
                self["library"].grid.hide()
            self["image"].scrolled_win.grab_focus()
            # Start in slideshow mode?
            if self["slideshow"].at_start:
                self["slideshow"].toggle()
        else:
            # Slideshow without paths makes no sense
            self["slideshow"].running = False
            self["library"].reload(os.getcwd())
            if self["library"].expand:
                self["image"].scrolled_win.hide()
            self["statusbar"].err_message(
                "No valid paths, opening library viewer")

    def set_paths(self, paths, index):
        """Set self.paths, self.index to paths, index."""
        self.paths = paths
        self.index = index
        self.do_open(self.paths, len(self.paths), 0)

    def init_widgets(self):
        """Create all the other widgets and add them to the class."""
        self["keyhandler"] = KeyHandler(self, self.settings)
        self["commandline"] = CommandLine(self, self.settings)
        self["tags"] = TagHandler(self)
        self["mark"] = Mark(self, self.settings)
        self["fileextras"] = FileExtras(self)
        self["statusbar"] = Statusbar(self, self.settings)
        self["completions"] = VimivComplete(self)
        self["slideshow"] = Slideshow(self, self.settings)
        self["image"] = Image(self, self.settings)
        self["library"] = Library(self, self.settings)
        self["thumbnail"] = Thumbnail(self, self.settings)
        self["manipulate"] = Manipulate(self, self.settings)
        self["information"] = Information()
        self["window"] = Window(self, self.settings)
        Commands(self, self.settings)

    def create_window_structure(self):
        """Generate the structure of all the widgets and add it to the window.

        There is a large Gtk.Grid() to organize the widgets packed into a
        Gtk.Overlay(). The commandline is added as overlay.
        """
        main_grid = Gtk.Grid()
        main_grid.attach(self["library"].grid, 0, 0, 1, 1)
        main_grid.attach(self["image"].scrolled_win, 1, 0, 1, 1)
        main_grid.attach(self["manipulate"].scrolled_win, 0, 1, 2, 1)
        main_grid.attach(self["statusbar"].bar, 0, 2, 2, 1)

        # Overlay contains grid mainly and adds commandline as floating
        overlay = Gtk.Overlay()
        overlay.add(main_grid)
        overlay.add_overlay(self["commandline"].grid)
        self["window"].add(overlay)

    def quit_wrapper(self, force=False):
        """Quit the applications, print marked files and save history.

        Args:
            force: If True quit even if an image was edited.
        """
        for image in self["mark"].marked:
            print(image)
        # Check if image has been edited
        if self["image"].check_for_edit(force):
            return
        # Save the history
        histfile = os.path.expanduser("~/.vimiv/history")
        histfile = open(histfile, 'w')
        for cmd in self["commandline"].history:
            cmd += "\n"
            histfile.write(cmd)
        histfile.close()
        # Call Gtk.Application.quit()
        self.quit()

    def get_pos(self, get_filename=False, force_widget=""):
        """Get the current position in the focused widget.

        get_filename: If True return filename instead of position.
        force_widget: String to force getting the position of a widget

        Return:
            Current position as Int or filename.
        """
        # Find widget to work on
        if force_widget:
            focused_widget = force_widget
        elif self["image"].scrolled_win.is_focus():
            focused_widget = "im"
        elif self["library"].treeview.is_focus():
            focused_widget = "lib"
        elif self["thumbnail"].iconview.is_focus():
            focused_widget = "thu"

        # Get position and name
        if focused_widget != "im":
            if focused_widget == "lib":
                path = self["library"].treeview.get_cursor()[0]
                filelist = self["library"].files
            elif focused_widget == "thu":
                path = self["thumbnail"].iconview.get_cursor()[1]
                filelist = self.paths
            try:
                position = path.get_indices()[0]
                if get_filename:
                    return filelist[position]
                else:
                    return position
            except:
                if get_filename:
                    return ""
                else:
                    return 0
        else:
            if get_filename:
                return self.paths[self.index]
            else:
                return self.index

    def init_commandline_options(self):
        """Add all possible commandline options."""
        def add_option(name, short, help_str, flags=GLib.OptionFlags.NONE,
                       arg=GLib.OptionArg.NONE, value=None):
            """Add a single option.

            Args:
                name: Long name of the option, called via --name.
                short: Short name of the option, called via -s.
                help_str: Displayed help text when calling --help.
                flags: GLib options.
                arg: Type of the value to give after option.
                value: Name of the value to give after the option.
            """
            if short:
                short = ord(short)
            self.add_main_option(name, short, flags, arg, help_str, value)
        add_option("bar", "b", "Display statusbar")
        add_option("no-bar", "B", "Hide statusbar")
        add_option("library", "l", "Display library")
        add_option("no-library", "L", "Hide library")
        add_option("fullscreen", "f", "Start fullscreen")
        add_option("no-fullscreen", "F", "Do not start fullscreen")
        add_option("shuffle", "s", "Shuffle filelist")
        add_option("no-shuffle", "S", "Do not shuffle filelist")
        add_option("recursive", "r", "Search directory recursively for images")
        add_option("no-recursive", "R",
                   "Do not search directory recursively for images")
        add_option("version", "v", "Print version information and exit")
        add_option("slideshow", 0, "Start slideshow at startup")
        add_option("start-from-desktop", 0,
                   "Start using the desktop_start_dir as path")
        add_option("slideshow-delay", 0, "Set slideshow delay",
                   arg=GLib.OptionArg.DOUBLE, value="delay")
        add_option("geometry", "g", "Set the starting geometry",
                   arg=GLib.OptionArg.STRING, value="GEOMETRY")

    def __getitem__(self, name):
        """Convenience method to access widgets via self[name].

        Args:
            name: Name of the widget to access.
        Return:
            The widget if it exists. None else.
        """
        if name in self.widgets:
            return self.widgets[name]
        else:
            return None

    def __setitem__(self, name, widget):
        """Convenience method to access widgets via self[name].

        Args:
            name: Name of the widget to add.
            item: The actual widget to which name will refer.
        """
        self.widgets[name] = widget

    def __contains__(self, name):
        """Return True if a widget called name exists."""
        return name in self.widgets
<commit_msg>Remove unused methods in app.py<commit_after>#!/usr/bin/env python
# encoding: utf-8
"""Main application class of vimiv."""

import os
import sys
from gi import require_version
require_version('Gtk', '3.0')
from gi.repository import Gtk, Gdk, Gio, GLib
from vimiv.configparser import parse_dirs, parse_config
from vimiv.image import Image
from vimiv.fileactions import FileExtras, populate
from vimiv.library import Library
from vimiv.thumbnail import Thumbnail
from vimiv.manipulate import Manipulate
from vimiv.statusbar import Statusbar
from vimiv.commandline import CommandLine
from vimiv.commands import Commands
from vimiv.completions import VimivComplete
from vimiv.slideshow import Slideshow
from vimiv.keyhandler import KeyHandler
from vimiv.tags import TagHandler
from vimiv.mark import Mark
from vimiv.information import Information
from vimiv.window import Window

class Vimiv(Gtk.Application):
    """Main vimiv application class inheriting from Gtk.Application.

    Attributes:
        settings: Settings from configfiles to use.
        paths: List of paths for images.
        index: Current position in paths.
        widgets: Dictionary of vimiv widgets.
            widgets[widget-name] = Gtk.Widget
        directory: Directory in which configfiles and data are stored.
        commands: Dictionary of commands.
            commands[command-name] = [function, *args]
        aliases: Dicionary of aliases to commands.
            aliases[alias-name] = command-name
        functions: Dictionary of functions. Includes all commands and additional
            functions that cannot be called from the commandline.
        screensize: Available screensize.
    """

    def __init__(self, application_id="org.vimiv"):
        """Create the Gtk.Application and connect the activate signal.

        Args:
            application_id: The ID used to register vimiv. Default: org.vimiv.
        """
        # Create directory structure and get settings
        parse_dirs()
        Gtk.Application.__init__(self, application_id=application_id)
        self.set_flags(Gio.ApplicationFlags.HANDLES_OPEN)
        self.connect("activate", self.activate_vimiv)
        self.settings = parse_config()
        self.paths = []
        self.index = 0
        self.widgets = {}
        self.directory = os.path.expanduser("~/.vimiv")
        self.commands = {}
        self.aliases = {}
        self.functions = {}
        # Sceen and window size TODO put somewhere else
        screen = Gdk.Screen()
        self.screensize = [screen.width(), screen.height()]
        # Set up all commandline options
        self.init_commandline_options()

    # Any different number of arguments will fail
    # pylint: disable=arguments-differ
    def do_open(self, files, n_files, hint):
        """Open files by populating self.paths and self.index.

        Args:
            files: List of Gio.Files.
            n_files: Number of files.
            hint: Special string for user, always empty in vimiv.
        """
        filenames = [fil.get_path() for fil in files]
        # Move to the directory of the first argument
        if os.path.isdir(filenames[0]):
            working_directory = filenames[0]
        else:
            working_directory = os.path.dirname(filenames[0])
        if os.path.exists(working_directory):
            os.chdir(working_directory)
        # Populate list of images
        recursive = self.settings["GENERAL"]["recursive"]
        shuffle = self.settings["GENERAL"]["shuffle"]
        self.paths, self.index = populate(filenames, recursive, shuffle)

        # Activate vimiv after opening files
        self.activate_vimiv(self)

    # Any different number of arguments will fail
    # pylint: disable=arguments-differ
    def do_handle_local_options(self, options):
        """Handle commandline arguments.

        Args:
            command_line:
        """
        if options.contains("version"):
            information = Information()
            print(information.get_version())
            return 0  # To exit
        if options.contains("bar"):
            self.settings["GENERAL"]["display_bar"] = True
        elif options.contains("no-bar"):
            self.settings["GENERAL"]["display_bar"] = False
        if options.contains("library"):
            self.settings["LIBRARY"]["show_library"] = True
        elif options.contains("no-library"):
            self.settings["LIBRARY"]["show_library"] = False
        if options.contains("fullscreen"):
            self.settings["GENERAL"]["start_fullscreen"] = True
        elif options.contains("no-fullscreen"):
            self.settings["GENERAL"]["start_fullscreen"] = False
        if options.contains("shuffle"):
            self.settings["GENERAL"]["shuffle"] = True
        elif options.contains("no-shuffle"):
            self.settings["GENERAL"]["shuffle"] = False
        if options.contains("recursive"):
            self.settings["GENERAL"]["recursive"] = True
        elif options.contains("no-recursive"):
            self.settings["GENERAL"]["recursive"] = False
        if options.contains("slideshow"):
            self.settings["GENERAL"]["start_slideshow"] = True
        if options.contains("start-from-desktop"):
            os.chdir(self.settings["LIBRARY"]["desktop_start_dir"])
        elif not sys.stdin.isatty():
            tty_paths = []
            for line in sys.stdin:
                tty_paths.append(line.rstrip("\n"))
            self.paths, self.index = populate(tty_paths)
        if options.contains("slideshow-delay"):
            delay = options.lookup_value("slideshow-delay").unpack()
            self.settings["GENERAL"]["slideshow_delay"] = delay
        if options.contains("geometry"):
            geometry = options.lookup_value("geometry").unpack()
            self.settings["GENERAL"]["geometry"] = geometry
        return -1  # To continue

    def activate_vimiv(self, app):
        """Starting point for the vimiv application.

        Args:
            app: The application itself.
        """
        self.init_widgets()
        self.create_window_structure()
        app.add_window(self["window"])
        # Show everything and then hide whatever needs to be hidden
        self["window"].show_all()
        self["manipulate"].scrolled_win.hide()
        self["commandline"].grid.hide()
        # Statusbar depending on setting
        if self["statusbar"].hidden:
            self["statusbar"].bar.hide()
        # Try to generate imagelist recursively from the current directory if
        # recursive is given and not paths exist
        if self.settings["GENERAL"]["recursive"] and not self.paths:
            shuffle = self.settings["GENERAL"]["shuffle"]
            self.paths, self.index = populate([os.getcwd()], True, shuffle)
        # Show the image if an imagelist exists
        if self.paths:
            self["image"].move_index(True, False, 0)
            # Show library at the beginning?
            if not self["library"].show_at_start:
                self["library"].grid.hide()
            self["image"].scrolled_win.grab_focus()
            # Start in slideshow mode?
            if self["slideshow"].at_start:
                self["slideshow"].toggle()
        else:
            # Slideshow without paths makes no sense
            self["slideshow"].running = False
            self["library"].reload(os.getcwd())
            if self["library"].expand:
                self["image"].scrolled_win.hide()
            self["statusbar"].err_message(
                "No valid paths, opening library viewer")

    def init_widgets(self):
        """Create all the other widgets and add them to the class."""
        self["keyhandler"] = KeyHandler(self, self.settings)
        self["commandline"] = CommandLine(self, self.settings)
        self["tags"] = TagHandler(self)
        self["mark"] = Mark(self, self.settings)
        self["fileextras"] = FileExtras(self)
        self["statusbar"] = Statusbar(self, self.settings)
        self["completions"] = VimivComplete(self)
        self["slideshow"] = Slideshow(self, self.settings)
        self["image"] = Image(self, self.settings)
        self["library"] = Library(self, self.settings)
        self["thumbnail"] = Thumbnail(self, self.settings)
        self["manipulate"] = Manipulate(self, self.settings)
        self["information"] = Information()
        self["window"] = Window(self, self.settings)
        Commands(self, self.settings)

    def create_window_structure(self):
        """Generate the structure of all the widgets and add it to the window.

        There is a large Gtk.Grid() to organize the widgets packed into a
        Gtk.Overlay(). The commandline is added as overlay.
        """
        main_grid = Gtk.Grid()
        main_grid.attach(self["library"].grid, 0, 0, 1, 1)
        main_grid.attach(self["image"].scrolled_win, 1, 0, 1, 1)
        main_grid.attach(self["manipulate"].scrolled_win, 0, 1, 2, 1)
        main_grid.attach(self["statusbar"].bar, 0, 2, 2, 1)

        # Overlay contains grid mainly and adds commandline as floating
        overlay = Gtk.Overlay()
        overlay.add(main_grid)
        overlay.add_overlay(self["commandline"].grid)
        self["window"].add(overlay)

    def quit_wrapper(self, force=False):
        """Quit the applications, print marked files and save history.

        Args:
            force: If True quit even if an image was edited.
        """
        for image in self["mark"].marked:
            print(image)
        # Check if image has been edited
        if self["image"].check_for_edit(force):
            return
        # Save the history
        histfile = os.path.expanduser("~/.vimiv/history")
        histfile = open(histfile, 'w')
        for cmd in self["commandline"].history:
            cmd += "\n"
            histfile.write(cmd)
        histfile.close()
        # Call Gtk.Application.quit()
        self.quit()

    def get_pos(self, get_filename=False, force_widget=""):
        """Get the current position in the focused widget.

        get_filename: If True return filename instead of position.
        force_widget: String to force getting the position of a widget

        Return:
            Current position as Int or filename.
        """
        # Find widget to work on
        if force_widget:
            focused_widget = force_widget
        elif self["image"].scrolled_win.is_focus():
            focused_widget = "im"
        elif self["library"].treeview.is_focus():
            focused_widget = "lib"
        elif self["thumbnail"].iconview.is_focus():
            focused_widget = "thu"

        # Get position and name
        if focused_widget != "im":
            if focused_widget == "lib":
                path = self["library"].treeview.get_cursor()[0]
                filelist = self["library"].files
            elif focused_widget == "thu":
                path = self["thumbnail"].iconview.get_cursor()[1]
                filelist = self.paths
            try:
                position = path.get_indices()[0]
                if get_filename:
                    return filelist[position]
                else:
                    return position
            except:
                if get_filename:
                    return ""
                else:
                    return 0
        else:
            if get_filename:
                return self.paths[self.index]
            else:
                return self.index

    def init_commandline_options(self):
        """Add all possible commandline options."""
        def add_option(name, short, help_str, flags=GLib.OptionFlags.NONE,
                       arg=GLib.OptionArg.NONE, value=None):
            """Add a single option.

            Args:
                name: Long name of the option, called via --name.
                short: Short name of the option, called via -s.
                help_str: Displayed help text when calling --help.
                flags: GLib options.
                arg: Type of the value to give after option.
                value: Name of the value to give after the option.
            """
            if short:
                short = ord(short)
            self.add_main_option(name, short, flags, arg, help_str, value)
        add_option("bar", "b", "Display statusbar")
        add_option("no-bar", "B", "Hide statusbar")
        add_option("library", "l", "Display library")
        add_option("no-library", "L", "Hide library")
        add_option("fullscreen", "f", "Start fullscreen")
        add_option("no-fullscreen", "F", "Do not start fullscreen")
        add_option("shuffle", "s", "Shuffle filelist")
        add_option("no-shuffle", "S", "Do not shuffle filelist")
        add_option("recursive", "r", "Search directory recursively for images")
        add_option("no-recursive", "R",
                   "Do not search directory recursively for images")
        add_option("version", "v", "Print version information and exit")
        add_option("slideshow", 0, "Start slideshow at startup")
        add_option("start-from-desktop", 0,
                   "Start using the desktop_start_dir as path")
        add_option("slideshow-delay", 0, "Set slideshow delay",
                   arg=GLib.OptionArg.DOUBLE, value="delay")
        add_option("geometry", "g", "Set the starting geometry",
                   arg=GLib.OptionArg.STRING, value="GEOMETRY")

    def __getitem__(self, name):
        """Convenience method to access widgets via self[name].

        Args:
            name: Name of the widget to access.
        Return:
            The widget if it exists. None else.
        """
        if name in self.widgets:
            return self.widgets[name]
        else:
            return None

    def __setitem__(self, name, widget):
        """Convenience method to access widgets via self[name].

        Args:
            name: Name of the widget to add.
            item: The actual widget to which name will refer.
        """
        self.widgets[name] = widget
<|endoftext|>


<commit_before>#Grader called by pyxserver_wsgi.py
#Loads a grader file, which is a dict containing the prompt of the question,
#a feature extractor object, and a trained model.
#Extracts features and runs trained model on the submission to produce a final score.
#Correctness determined by ratio of score to max possible score.
#Requires aspell to be installed and added to the path.

import sys
import pickle
import os
import numpy
import logging

base_path = os.path.dirname(__file__)
sys.path.append(base_path)

from essay_set import EssaySet

#Imports needed to unpickle grader data
import feature_extractor
import sklearn.ensemble

log = logging.getLogger(__name__)

def grade(grader_path,submission,sandbox=None):
    log.debug("Grader path: {0}\n Submission: {1}".format(grader_path,submission))
    results = {'errors': [],'tests': [],'correct': False,'score': 0, 'feedback' : []}

    #Try to find and load the model file

    try:
        grader_data=pickle.load(file(grader_path,"r"))
    except:
        results['errors'].append("Could not find a valid model file.")
    grader_set=EssaySet(type="test")

    #Try to add essays to essay set object
    try:
        grader_set.add_essay(str(submission),0)
        grader_set.update_prompt(str(grader_data['prompt']))
    except:
        results['errors'].append("Essay could not be added to essay set:{0}".format(submission))

    #Try to extract features from submission and assign score via the model
    try:
        grader_feats=grader_data['extractor'].gen_feats(grader_set)
        results['feedback']=grader_data['extractor'].gen_feedback(grader_set)
        results['score']=int(grader_data['model'].predict(grader_feats)[0])
    except :
        results['errors'].append("Could not extract features and score essay.")

    #Determine maximum score and correctness of response
    max_score=numpy.max(grader_data['model'].classes_)
    if results['score']/float(max_score) >= .66:
        results['correct']=True
    else:
        results['correct']=False
    return results

<commit_msg>Alter feedback handling to return feedback as an html string based on a template.<commit_after>#Grader called by pyxserver_wsgi.py
#Loads a grader file, which is a dict containing the prompt of the question,
#a feature extractor object, and a trained model.
#Extracts features and runs trained model on the submission to produce a final score.
#Correctness determined by ratio of score to max possible score.
#Requires aspell to be installed and added to the path.

import sys
import pickle
import os
import numpy
import logging

base_path = os.path.dirname(__file__)
sys.path.append(base_path)

from essay_set import EssaySet

#Imports needed to unpickle grader data
import feature_extractor
import sklearn.ensemble

log = logging.getLogger(__name__)

feedback_template = u"""

Feedback

    {topicality}

      {spelling}

        {grammar}

        {markup_text}

"""

def grade(grader_path,submission,sandbox=None):
    log.debug("Grader path: {0}\n Submission: {1}".format(grader_path,submission))
    results = {'errors': [],'tests': [],'correct': False,'score': 0, 'feedback' : ""}

    #Try to find and load the model file

    try:
        grader_data=pickle.load(file(grader_path,"r"))
    except:
        results['errors'].append("Could not find a valid model file.")
    grader_set=EssaySet(type="test")

    #Try to add essays to essay set object
    try:
        grader_set.add_essay(str(submission),0)
        grader_set.update_prompt(str(grader_data['prompt']))
    except:
        results['errors'].append("Essay could not be added to essay set:{0}".format(submission))

    #Try to extract features from submission and assign score via the model
    try:
        grader_feats=grader_data['extractor'].gen_feats(grader_set)
        feedback=grader_data['extractor'].gen_feedback(grader_set)
        results['score']=int(grader_data['model'].predict(grader_feats)[0])
    except :
        results['errors'].append("Could not extract features and score essay.")

    #Determine maximum score and correctness of response
    max_score=numpy.max(grader_data['model'].classes_)
    if results['score']/float(max_score) >= .66:
        results['correct']=True
    else:
        results['correct']=False

    #Add feedback template to results
    results["feedback"]=feedback_template.format(topicality=feedback['topicality'],
        spelling=feedback['spelling'],grammar=feedback['grammar'],markup_text=feedback['markup_text'])

    return results

<|endoftext|>


<commit_before>import glob
import os

paths = ['src/*.cpp', 'src/kademlia/*.cpp', 'include/libtorrent/*.hpp', 'include/libtorrent/kademlia/*.hpp', 'include/libtorrent/aux_/*.hpp', 'include/libtorrent/extensions/*.hpp']

os.system('ctags %s 2>/dev/null' % ' '.join(paths))

files = []

for p in paths:
	files.extend(glob.glob(p))

items = []

# keeps 20 non-comment lines, used to get more context around
# todo-items
context = []

priority_count = [0, 0, 0, 0]

def html_sanitize(s):
	ret = ''
	for i in s:
		if i == '<': ret += '<'
		elif i == '>': ret += '>'
		elif i == '&': ret += '&'
		else: ret += i
	return ret

for f in files:
	h = open(f)

	state = ''
	line_no = 0
	context_lines = 0

	for l in h:
		line_no += 1
		line = l.strip()
		if 'TODO:' in line and line.startswith('//'):
			line = line.split('TODO:')[1].strip()
			state = 'todo'
			items.append({})
			items[-1]['location'] = '%s:%d' % (f, line_no)
			items[-1]['priority'] = 0
			if line[0] in '0123456789':
				items[-1]['priority'] = int(line[0])
				line = line[1:].strip()
			items[-1]['todo'] = line
			prio = items[-1]['priority']
			if prio >= 0 and prio <= 3: priority_count[prio] += 1
			continue

		if state == '':
			context.append(html_sanitize(l))
			if len(context) > 20: context.pop(0)
			continue

		if state == 'todo':
			if line.strip().startswith('//'):
				items[-1]['todo'] += '\n'
				items[-1]['todo'] += line[2:].strip()
			else:
				state = 'context'
				items[-1]['context'] = ''.join(context) + '
' + html_sanitize(l) + '
';
				context_lines = 1

				context.append(html_sanitize(l))
				if len(context) > 20: context.pop(0)
			continue

		if state == 'context':
			items[-1]['context'] += html_sanitize(l)
			context_lines += 1

			context.append(html_sanitize(l))
			if len(context) > 20: context.pop(0)
			if context_lines > 30: state = ''

	h.close()

items.sort(key = lambda x: x['priority'], reverse = True)

#for i in items:
#	print '\n\n', i['todo'], '\n'
#	print i['location'], '\n'
#	print 'prio: ', i['priority'], '\n'
#	if 'context' in i:
#		print i['context'], '\n'

out = open('docs/todo.html', 'w+')
out.write('''

libtorrent todo-list

%d important
%d relevant
%d feasible
%d notes

''' % \
	(priority_count[3], priority_count[2], priority_count[1], priority_count[0]))

prio_colors = [ '#ccc', '#ccf', '#cfc', '#fcc']

index = 0
for i in items:
	if not 'context' in i: i['context'] = ''
	out.write('
relevance %d	%s	%s
' \
		% (prio_colors[i['priority']], i['priority'], index, i['location'], i['todo'].replace('\n', ' ')))

	out.write('

%s

%s

%s

' \
		% (index, i['todo'], i['location'], i['context']))
	index += 1

out.write('
')
out.close()

<commit_msg>update todo generation tool<commit_after>import glob
import os

paths = ['src/*.cpp', 'src/kademlia/*.cpp', 'include/libtorrent/*.hpp', 'include/libtorrent/kademlia/*.hpp', 'include/libtorrent/aux_/*.hpp', 'include/libtorrent/extensions/*.hpp']

os.system('ctags %s 2>/dev/null' % ' '.join(paths))

files = []

for p in paths:
	files.extend(glob.glob(p))

items = []

# keeps 20 non-comment lines, used to get more context around
# todo-items
context = []

priority_count = [0, 0, 0, 0, 0]

def html_sanitize(s):
	ret = ''
	for i in s:
		if i == '<': ret += '<'
		elif i == '>': ret += '>'
		elif i == '&': ret += '&'
		else: ret += i
	return ret

for f in files:
	h = open(f)

	state = ''
	line_no = 0
	context_lines = 0

	for l in h:
		line_no += 1
		line = l.strip()
		if 'TODO:' in line and line.startswith('//'):
			line = line.split('TODO:')[1].strip()
			state = 'todo'
			items.append({})
			items[-1]['location'] = '%s:%d' % (f, line_no)
			items[-1]['priority'] = 0
			if line[0] in '0123456789':
				items[-1]['priority'] = int(line[0])
				line = line[1:].strip()
			items[-1]['todo'] = line
			prio = items[-1]['priority']
			if prio >= 0 and prio <= 4: priority_count[prio] += 1
			continue

		if state == '':
			context.append(html_sanitize(l))
			if len(context) > 20: context.pop(0)
			continue

		if state == 'todo':
			if line.strip().startswith('//'):
				items[-1]['todo'] += '\n'
				items[-1]['todo'] += line[2:].strip()
			else:
				state = 'context'
				items[-1]['context'] = ''.join(context) + '
' + html_sanitize(l) + '
';
				context_lines = 1

				context.append(html_sanitize(l))
				if len(context) > 20: context.pop(0)
			continue

		if state == 'context':
			items[-1]['context'] += html_sanitize(l)
			context_lines += 1

			context.append(html_sanitize(l))
			if len(context) > 20: context.pop(0)
			if context_lines > 30: state = ''

	h.close()

items.sort(key = lambda x: x['priority'], reverse = True)

#for i in items:
#	print '\n\n', i['todo'], '\n'
#	print i['location'], '\n'
#	print 'prio: ', i['priority'], '\n'
#	if 'context' in i:
#		print i['context'], '\n'

out = open('docs/todo.html', 'w+')
out.write('''

libtorrent todo-list

%d urgent
%d important
%d relevant
%d feasible
%d notes

''' % \
	(priority_count[4], priority_count[3], priority_count[2], priority_count[1], priority_count[0]))

prio_colors = [ '#ccc', '#ccf', '#cfc', '#fcc', '#f44']

index = 0
for i in items:
	if not 'context' in i: i['context'] = ''
	out.write('
relevance %d	%s	%s
' \
		% (prio_colors[i['priority']], i['priority'], index, i['location'], i['todo'].replace('\n', ' ')))

	out.write('

%s

%s

%s

' \
		% (index, i['todo'], i['location'], i['context']))
	index += 1

out.write('
')
out.close()

<|endoftext|>


<commit_before># from ddtrace import patch
# patch(requests=True)

import json
import requests
import requests.exceptions
from datadog import statsd
from bs4 import BeautifulSoup

from nb_exceptions import BlacklistedError
from utility import constants
from utility import nb_logging

from time import sleep

import rollbar
rollbar.init('00b402fc0da54ed1af8687d4c4389911')

logger = nb_logging.setup_logger('NewsblurConnector')

class NewsblurConnector:

    def __init__(self):
        self.cookies = None

    def connect(self):
        """ make connection """
        r = requests.post(constants.NB_ENDPOINT + '/api/login', constants.NB_CREDENTIALS, verify=constants.VERIFY)
        statsd.increment('nb.http_requests.post')
        self.cookies = r.cookies

    @statsd.timed('nb.NewsblurConnector.get_nb_hash_list')
    def get_nb_hash_list(self):
        """ get a list of story identifiers (hashes) from NewsBlur """
        hashes = requests.get(constants.NB_ENDPOINT + '/reader/starred_story_hashes',
                              cookies=self.cookies, verify=constants.VERIFY)
        statsd.increment('nb.http_requests.get')
        return hashes.json()['starred_story_hashes']

    @statsd.timed('nb.NewsblurConnector.get_story_list')
    def get_story_list(self, batch):
        """ get a list of stories corresponding to a list of hashes """
        req_str = constants.NB_ENDPOINT + '/reader/starred_stories?'
        for a_hash in batch:
            req_str += 'h=' + a_hash + '&'
        stories = requests.get(req_str, cookies=self.cookies, verify=constants.VERIFY)
        statsd.increment('nb.http_requests.get')
        story_list = []
        try:
            story_list = json.loads(stories.text)['stories']
        except ValueError as e:
	    rollbar.report_exc_info()
            logger.error('Failed to get stories for request %s', req_str)
            logger.error(e)
            statsd.event('Failed to get stories', e.message, alert_type='error')
            logger.debug(stories.text)
        return story_list

    # eg request_with_backoff(url, on_success)
    # eg request_with_backoff(hnurl, parse_story)
    # Hmm. Falls down on POSTs. :-( Needs more treatment
    # TODO: cf requests PreparedRequest!
    @statsd.timed('nb.NewsblurConnector.get_with_backoff')
    def get_with_backoff(self, url, on_success):
        try:
            backoff = constants.BACKOFF_START
            resp = requests.get(url, verify=constants.VERIFY)
            statsd.increment('nb.http_requests.get')
            while resp.status_code != 200:
                if resp.status_code in [403, 500, 503]:  # exponential backoff
                    logger.debug("Request for %s returned %s response", url, resp.status_code)
                    if backoff < constants.BACKOFF_MAX:
                        logger.debug("Backing off %s seconds", backoff)
                        sleep(backoff)
                        resp = requests.get(url, verify=constants.VERIFY)
                        statsd.increment('nb.http_requests.get')
                        backoff = backoff * constants.BACKOFF_FACTOR
                    else:
                        logger.debug("Giving up after %s seconds for %s", backoff, url)
                        return None
                elif resp.status_code == 520:
                    logger.info("520 response, skipping %s and waiting %s sec", url, constants.BACKOFF_ON_520)
                    logger.debug("Response headers: %s", resp.headers)
                    logger.debug("Response body: %s", resp.text)
                    sleep(constants.BACKOFF_ON_520)
                    return None
                else:
                    logger.debug(
                        "Request for %s returned unhandled %s response", url, resp.status_code)
                    raise requests.exceptions.RequestException()
        except requests.exceptions.RequestException as e:
	    rollbar.report_exc_info()
            logger.error("url is: %s", url)
            logger.error(e)
            statsd.event('Request failed', e.message, alert_type='error')
            return None

        return on_success(resp)

    # TODO: DEPRECATE in favour of request_with_backoff()
    @statsd.timed('nb.NewsblurConnector.get_comment_count')
    def get_comment_count(self, hnurl):
        sleep(1)
        try:
            backoff = 5
            story = requests.get(hnurl, verify=constants.VERIFY)
            statsd.increment('nb.http_requests.get')
            while story.status_code != 200:
                if story.status_code in [429, 500, 503]:  # exponential backoff
                    logger.debug(
                        "Request for %s returned %s response", hnurl, story.status_code)
                    if backoff < constants.BACKOFF_MAX:
                        logger.debug("Backing off %s seconds", backoff)
                        sleep(backoff)
                        backoff *= constants.BACKOFF_FACTOR
                        story = requests.get(hnurl, verify=constants.VERIFY)
                        statsd.increment('nb.http_requests.get')
                    else:
                        logger.debug("Giving up after {0} seconds for {1}".format(backoff, hnurl))
                        return None
                elif story.status_code in [520]:
                    logger.debug(
                        "520 response for %s; waiting %s sec", hnurl, constants.BACKOFF_ON_520)
                    logger.debug("Response headers: %s", story.headers)
                    logger.debug("Response body: %s", story.text)
                    sleep(constants.BACKOFF_ON_520)
                    return None
                elif story.status_code in [403]:
                    raise BlacklistedError()
                else:
                    logger.debug("Request for %s returned unhandled %s response",
                                 hnurl, story.status_code)
                    raise requests.exceptions.RequestException()
        except requests.exceptions.RequestException as e:
	    rollbar.report_exc_info()
            logger.error("hnurl: %s", hnurl)
            statsd.event('RequestException', e.message, alert_type='error')
            return None

        return self.parse_story(story.text)

    # Parse HN story to find how many comments there are
    @statsd.timed('nb.populate.parse_story')
    def parse_story(self, content):
        soup = BeautifulSoup(content, "html.parser")
        comment_count = len(soup.find_all("div", {"class": "comment"}))
        return comment_count

    @statsd.timed('nb.prune.check_if_starred')
    def check_if_starred(self, story_hash):
        hashes = requests.get(constants.NB_ENDPOINT + '/reader/starred_story_hashes', cookies=self.cookies,
                              verify=constants.VERIFY)
        statsd.increment('nb.http_requests.get')
        hashlist = hashes.json()['starred_story_hashes']

        return bool(story_hash in hashlist)

    # TODO: deprecate in favour of more generic function
    # TODO: move statsd bucket names to constants.py
    @statsd.timed('nb.prune.remove_star_with_backoff')
    def remove_star_with_backoff(self, story_hash):
        backoff = constants.BACKOFF_START
        unstar_url = constants.NB_ENDPOINT + '/reader/mark_story_hash_as_unstarred'
        try:
            resp = requests.post(unstar_url,
                                 {'story_hash': story_hash}, cookies=self.cookies, verify=constants.VERIFY)
            statsd.increment('nb.http_requests.post')
            while resp.status_code != 200:
                if resp.status_code in [403, 500, 503]:  # exponential backoff
                    logger.info("Request for %s returned %s response" % (unstar_url, resp.status_code))
                    if backoff < constants.BACKOFF_MAX:
                        logger.info("Backing off %s seconds", backoff)
                        sleep(backoff)
                        backoff = backoff * constants.BACKOFF_FACTOR
                        resp = requests.post(unstar_url,
                                             {'story_hash': story_hash}, cookies=self.cookies,
                                             verify=constants.VERIFY)
                        statsd.increment('nb.http_requests.post')
                    else:
                        logger.warn("Giving up after %s seconds for %s", backoff, unstar_url)
                        return False
                elif resp.status_code == 520:
                    logger.warn("520 response, skipping %s", unstar_url)
                    return False
                else:
                    logger.error("Request for %s returned unhandled %s response",
                                 unstar_url, resp.status_code)
                    raise requests.exceptions.RequestException()
        except requests.exceptions.RequestException as e:
            logger.info("url is: %s", unstar_url)
            logger.error(e)
            return False

        return True
<commit_msg>Update NewsblurConnector.py<commit_after># from ddtrace import patch
# patch(requests=True)

import json
import requests
import requests.exceptions
from datadog import statsd
from bs4 import BeautifulSoup

from nb_exceptions.BlacklistedError import BlacklistedError
from utility import constants
from utility import nb_logging

from time import sleep

import rollbar
rollbar.init('00b402fc0da54ed1af8687d4c4389911')

logger = nb_logging.setup_logger('NewsblurConnector')

class NewsblurConnector:

    def __init__(self):
        self.cookies = None

    def connect(self):
        """ make connection """
        r = requests.post(constants.NB_ENDPOINT + '/api/login', constants.NB_CREDENTIALS, verify=constants.VERIFY)
        statsd.increment('nb.http_requests.post')
        self.cookies = r.cookies

    @statsd.timed('nb.NewsblurConnector.get_nb_hash_list')
    def get_nb_hash_list(self):
        """ get a list of story identifiers (hashes) from NewsBlur """
        hashes = requests.get(constants.NB_ENDPOINT + '/reader/starred_story_hashes',
                              cookies=self.cookies, verify=constants.VERIFY)
        statsd.increment('nb.http_requests.get')
        return hashes.json()['starred_story_hashes']

    @statsd.timed('nb.NewsblurConnector.get_story_list')
    def get_story_list(self, batch):
        """ get a list of stories corresponding to a list of hashes """
        req_str = constants.NB_ENDPOINT + '/reader/starred_stories?'
        for a_hash in batch:
            req_str += 'h=' + a_hash + '&'
        stories = requests.get(req_str, cookies=self.cookies, verify=constants.VERIFY)
        statsd.increment('nb.http_requests.get')
        story_list = []
        try:
            story_list = json.loads(stories.text)['stories']
        except ValueError as e:
	    rollbar.report_exc_info()
            logger.error('Failed to get stories for request %s', req_str)
            logger.error(e)
            statsd.event('Failed to get stories', e.message, alert_type='error')
            logger.debug(stories.text)
        return story_list

    # eg request_with_backoff(url, on_success)
    # eg request_with_backoff(hnurl, parse_story)
    # Hmm. Falls down on POSTs. :-( Needs more treatment
    # TODO: cf requests PreparedRequest!
    @statsd.timed('nb.NewsblurConnector.get_with_backoff')
    def get_with_backoff(self, url, on_success):
        try:
            backoff = constants.BACKOFF_START
            resp = requests.get(url, verify=constants.VERIFY)
            statsd.increment('nb.http_requests.get')
            while resp.status_code != 200:
                if resp.status_code in [403, 500, 503]:  # exponential backoff
                    logger.debug("Request for %s returned %s response", url, resp.status_code)
                    if backoff < constants.BACKOFF_MAX:
                        logger.debug("Backing off %s seconds", backoff)
                        sleep(backoff)
                        resp = requests.get(url, verify=constants.VERIFY)
                        statsd.increment('nb.http_requests.get')
                        backoff = backoff * constants.BACKOFF_FACTOR
                    else:
                        logger.debug("Giving up after %s seconds for %s", backoff, url)
                        return None
                elif resp.status_code == 520:
                    logger.info("520 response, skipping %s and waiting %s sec", url, constants.BACKOFF_ON_520)
                    logger.debug("Response headers: %s", resp.headers)
                    logger.debug("Response body: %s", resp.text)
                    sleep(constants.BACKOFF_ON_520)
                    return None
                else:
                    logger.debug(
                        "Request for %s returned unhandled %s response", url, resp.status_code)
                    raise requests.exceptions.RequestException()
        except requests.exceptions.RequestException as e:
	    rollbar.report_exc_info()
            logger.error("url is: %s", url)
            logger.error(e)
            statsd.event('Request failed', e.message, alert_type='error')
            return None

        return on_success(resp)

    # TODO: DEPRECATE in favour of request_with_backoff()
    @statsd.timed('nb.NewsblurConnector.get_comment_count')
    def get_comment_count(self, hnurl):
        sleep(1)
        try:
            backoff = 5
            story = requests.get(hnurl, verify=constants.VERIFY)
            statsd.increment('nb.http_requests.get')
            while story.status_code != 200:
                if story.status_code in [429, 500, 503]:  # exponential backoff
                    logger.debug(
                        "Request for %s returned %s response", hnurl, story.status_code)
                    if backoff < constants.BACKOFF_MAX:
                        logger.debug("Backing off %s seconds", backoff)
                        sleep(backoff)
                        backoff *= constants.BACKOFF_FACTOR
                        story = requests.get(hnurl, verify=constants.VERIFY)
                        statsd.increment('nb.http_requests.get')
                    else:
                        logger.debug("Giving up after {0} seconds for {1}".format(backoff, hnurl))
                        return None
                elif story.status_code in [520]:
                    logger.debug(
                        "520 response for %s; waiting %s sec", hnurl, constants.BACKOFF_ON_520)
                    logger.debug("Response headers: %s", story.headers)
                    logger.debug("Response body: %s", story.text)
                    sleep(constants.BACKOFF_ON_520)
                    return None
                elif story.status_code in [403]:
                    raise BlacklistedError()
                else:
                    logger.debug("Request for %s returned unhandled %s response",
                                 hnurl, story.status_code)
                    raise requests.exceptions.RequestException()
        except requests.exceptions.RequestException as e:
	    rollbar.report_exc_info()
            logger.error("hnurl: %s", hnurl)
            statsd.event('RequestException', e.message, alert_type='error')
            return None

        return self.parse_story(story.text)

    # Parse HN story to find how many comments there are
    @statsd.timed('nb.populate.parse_story')
    def parse_story(self, content):
        soup = BeautifulSoup(content, "html.parser")
        comment_count = len(soup.find_all("div", {"class": "comment"}))
        return comment_count

    @statsd.timed('nb.prune.check_if_starred')
    def check_if_starred(self, story_hash):
        hashes = requests.get(constants.NB_ENDPOINT + '/reader/starred_story_hashes', cookies=self.cookies,
                              verify=constants.VERIFY)
        statsd.increment('nb.http_requests.get')
        hashlist = hashes.json()['starred_story_hashes']

        return bool(story_hash in hashlist)

    # TODO: deprecate in favour of more generic function
    # TODO: move statsd bucket names to constants.py
    @statsd.timed('nb.prune.remove_star_with_backoff')
    def remove_star_with_backoff(self, story_hash):
        backoff = constants.BACKOFF_START
        unstar_url = constants.NB_ENDPOINT + '/reader/mark_story_hash_as_unstarred'
        try:
            resp = requests.post(unstar_url,
                                 {'story_hash': story_hash}, cookies=self.cookies, verify=constants.VERIFY)
            statsd.increment('nb.http_requests.post')
            while resp.status_code != 200:
                if resp.status_code in [403, 500, 503]:  # exponential backoff
                    logger.info("Request for %s returned %s response" % (unstar_url, resp.status_code))
                    if backoff < constants.BACKOFF_MAX:
                        logger.info("Backing off %s seconds", backoff)
                        sleep(backoff)
                        backoff = backoff * constants.BACKOFF_FACTOR
                        resp = requests.post(unstar_url,
                                             {'story_hash': story_hash}, cookies=self.cookies,
                                             verify=constants.VERIFY)
                        statsd.increment('nb.http_requests.post')
                    else:
                        logger.warn("Giving up after %s seconds for %s", backoff, unstar_url)
                        return False
                elif resp.status_code == 520:
                    logger.warn("520 response, skipping %s", unstar_url)
                    return False
                else:
                    logger.error("Request for %s returned unhandled %s response",
                                 unstar_url, resp.status_code)
                    raise requests.exceptions.RequestException()
        except requests.exceptions.RequestException as e:
            logger.info("url is: %s", unstar_url)
            logger.error(e)
            return False

        return True
<|endoftext|>


<commit_before># write_geogrid.py
# Jan Mandel, June 2019

import os.path as osp
import numpy as np
import sys, os, logging
from utils import inq

def write_divide(file,divide='=',count=25):
    """
    Write a dividing line
    """
    f=open(file,'a')
    f.write(divide * count + '\n')
    f.close()

def write_table(file,lines_dict,mode='w',divider_char='=',divider_count=25,divider_before=False,divider_after=False):
    """
    write table
    """
    f=open(file,mode)
    if divider_before:
        f.write(divider_char * divider_count + '\n')
    for key,value in lines_dict.iteritems():
        f.write("%s = %s\n" % (key,value))
    if divider_after:
        f.write(divider_char * divider_count + '\n')
    f.close()

def write_geogrid_var(path_dir,varname,array,description,index,bits=32):
    """
    write geogrid dataset and index
    """
    path_dir=osp.abspath(path_dir)
    logging.info('write_geogrid_var path_dir=%s varname=%s array=%s index=%s' % (path_dir, varname, inq(array), str(index)))
    if not osp.exists(path_dir):
        os.makedirs(path_dir)

    geogrid_ds_path = osp.join(path_dir,varname)
    index['description']=description
    write_geogrid(geogrid_ds_path,array,index,bits=bits)

    # add segment of GEOGRID.TBL
    geogrid_tbl_path=osp.join(path_dir,'GEOGRID.TBL')
    write_table(geogrid_tbl_path, {'name':varname,
                   'dest_type':'continuous',
                   'interp_option':'average_4pt',
                   'abs_path':geogrid_ds_path,
                   'priority':1}, mode='a', divider_after=True)

def write_geogrid(path,array,index,bits=32):
    """
    Write geogrid dataset
    :param path: the directory where the dataset is to be stored
    :param array: numpy array of real values, 2d or 3d
    :param index: json with geogrid index, with geolocation and description already set
    :param bits: 16 or 32 (default)
    """

    logging.info('write_geogrid_var path=%s array=%s index=%s' % (path, inq(array), str(index)))
    if not osp.exists(path):
        os.makedirs(path)
    # write binary data file
    a=np.array(array)
    dims=a.shape
    if len(dims) < 3:
        dims = dims + (1,)
    xsize, ysize, zsize = dims
    scale = 2**(np.ceil(np.log2(np.max(np.abs(a))))-bits+1)
    a = np.round(a.flatten()/scale)
    if bits == 32:
        a = np.int32(a)
    elif bits == 16:
        a = np.int16(a)
    else:
        print 'unsupported word size'
        sys.exit(1)
    data_file = "00001-%05i.00001-%05i" % (xsize, ysize)
    data_path = osp.join(path,data_file)
    a.tofile(data_path)

    # write index
    index.update({'type':'continuous',
                 'signed':'yes',
                 'scale_factor':scale,
                 'wordsize':bits/8,
                 'tile_x':xsize,
                 'tile_y':ysize,
                 'tile_z':zsize,
                 'endian':sys.byteorder})
    index_path = osp.join(path,'index')
    write_table(index_path,index)

def addquotes(s):
    """
    add quotes to string
    """
    return '"'+s+'"'

if __name__ == '__main__':
    test_name = 'test_geo'
    print 'testing write_geogrid to',test_name
    write_geogrid(test_name,[[1.0,2.0],[-1.0,-2.0]],{'name':addquotes(test_name)},bits=32)
<commit_msg>fixing<commit_after># write_geogrid.py
# Jan Mandel, June 2019

import os.path as osp
import numpy as np
import sys, os, logging
from utils import inq

def write_divide(file,divide='=',count=25):
    """
    Write a dividing line
    """
    f=open(file,'a')
    f.write(divide * count + '\n')
    f.close()

def write_table(file,lines_dict,mode='w',divider_char='=',divider_count=25,divider_before=False,divider_after=False):
    """
    write table
    """
    f=open(file,mode)
    if divider_before:
        f.write(divider_char * divider_count + '\n')
    for key,value in lines_dict.iteritems():
        f.write("%s = %s\n" % (key,value))
    if divider_after:
        f.write(divider_char * divider_count + '\n')
    f.close()

def write_geogrid_var(path_dir,varname,array,description,index,bits=32):
    """
    write geogrid dataset and index
    """
    path_dir=osp.abspath(path_dir)
    logging.info('write_geogrid_var path_dir=%s varname=%s array=%s index=%s' % (path_dir, varname, inq(array), str(index)))
    if not osp.exists(path_dir):
        os.makedirs(path_dir)

    geogrid_ds_path = osp.join(path_dir,varname)
    index['description']=description
    write_geogrid(geogrid_ds_path,array,index,bits=bits)

    # add segment of GEOGRID.TBL
    geogrid_tbl_path=osp.join(path_dir,'GEOGRID.TBL')
    write_table(geogrid_tbl_path, {'name':varname,
                   'dest_type':'continuous',
                   'interp_option':'average_4pt',
                   'abs_path':geogrid_ds_path,
                   'priority':1}, mode='a', divider_after=True)

def write_geogrid(path,array,index,bits=32):
    """
    Write geogrid dataset
    :param path: the directory where the dataset is to be stored
    :param array: numpy array of real values, 2d or 3d
    :param index: json with geogrid index, with geolocation and description already set
    :param bits: 16 or 32 (default)
    """

    logging.info('write_geogrid_var path=%s array=%s index=%s' % (path, inq(array), str(index)))
    if not osp.exists(path):
        os.makedirs(path)
    # write binary data file
    a=np.array(array)
    dims=a.shape
    if len(dims) < 3:
        dims = dims + (1,)
    xsize, ysize, zsize = dims
    scale = 2**(np.ceil(np.log2(np.max(np.abs(a))))-bits+1)
    a = np.round(a.transpose().flatten()/scale)
    if bits == 32:
        a = np.int32(a)
    elif bits == 16:
        a = np.int16(a)
    else:
        print 'unsupported word size'
        sys.exit(1)
    data_file = "00001-%05i.00001-%05i" % (xsize, ysize)
    data_path = osp.join(path,data_file)
    a.tofile(data_path)

    # write index
    index.update({'type':'continuous',
                 'signed':'yes',
                 'scale_factor':scale,
                 'wordsize':bits/8,
                 'tile_x':xsize,
                 'tile_y':ysize,
                 'tile_z':zsize,
                 'endian':sys.byteorder})
    index_path = osp.join(path,'index')
    write_table(index_path,index)

def addquotes(s):
    """
    add quotes to string
    """
    return '"'+s+'"'

if __name__ == '__main__':
    test_name = 'test_geo'
    print 'testing write_geogrid to',test_name
    write_geogrid(test_name,[[1.0,2.0],[-1.0,-2.0]],{'name':addquotes(test_name)},bits=32)
<|endoftext|>


<commit_before># -*- Mode: Python -*-
# vi:si:et:sw=4:sts=4:ts=4
#
# Flumotion - a streaming media server
# Copyright (C) 2004,2005,2006 Fluendo, S.L. (www.fluendo.com).
# All rights reserved.

# This file may be distributed and/or modified under the terms of
# the GNU General Public License version 2 as published by
# the Free Software Foundation.
# This file is distributed without any warranty; without even the implied
# warranty of merchantability or fitness for a particular purpose.
# See "LICENSE.GPL" in the source distribution for more information.

# Licensees having purchased or holding a valid Flumotion Advanced
# Streaming Server license may use this file in accordance with the
# Flumotion Advanced Streaming Server Commercial License Agreement.
# See "LICENSE.Flumotion" in the source distribution for more information.

# Headers in this file shall remain intact.

from flumotion.component import feedcomponent
from flumotion.common import errors
from twisted.internet import defer

class DVB(feedcomponent.ParseLaunchComponent):

    def do_check(self):
        props = self.config['properties']
        dvb_type = self.dvb_type = props.get('dvb-type')
        if dvb_type != 'T' and dvb_type != 'S':
            msg = "Property dvb-type can only be T (for DVB-T) or S (for DVB-S)."
            return defer.fail(errors.ConfigError(msg))
        # check if the required DVB parameters are passed
        dvb_required_parameters = { "T": ["modulation", "trans-mode",
            "bandwidth", "code-rate-lp", "code-rate-hp", "guard", "hierarchy"],
            "S": ["polarity", "symbol-rate", "satellite-number"] }
        for param in dvb_required_parameters[dvb_type]:
            if not param in props:
                msg = "DVB-%s mode is missing property '%s'." % (dvb_type,
                    param)
                return defer.fail(errors.ConfigError(msg))

    def get_pipeline_string(self, props):
        dvbsrc_template = ""
        if self.dvb_type == "T":
            modulation = props.get('modulation')
            trans_mode = props.get('trans-mode')
            bandwidth = props.get('bandwidth')
            code_rate_lp = props.get('code-rate-lp')
            code_rate_hp = props.get('code-rate-hp')
            guard = props.get('guard')
            hierarchy = props.get('hierarchy')
            dvbsrc_template = '''
dvbsrc modulation="QAM_%(modulation)d" trans-mode=%(trans_mode)dk
bandwidth=%(bandwidth)d code-rate-lp=%(code_rate_lp)s
code-rate-hp=%(code_rate_hp)s guard=%(guard)d
hierarchy=%(hierarchy)d''' % dict(modulation=modulation, trans_mode=trans_mode,
                bandwidth=bandwidth, code_rate_lp=code_rate_lp,
                code_rate_hp="%d/%d" % (code_rate_hp[0], code_rate_hp[1]),
                guard=guard, hierarchy=hierarchy)
        elif self.dvb_type == "S":
            polarity = props.get('polarity')
            symbol_rate = props.get('symbol-rate')
            sat = props.get('satellite-number')
            dvbsrc_template = '''
dvbsrc pol=%(polarity)s srate=%(symbol_rate)s
diseqc-src=%(sat)d''' % dict(polarity=polarity, symbol_rate=symbol_rate,
    sat=sat)
        width = props.get('width', 720)
        height = props.get('height', int(576 * width/720.)) # assuming PAL :-/
        framerate = props.get('framerate', (25, 2))
        framerate_float = float(framerate[0]) / framerate[1]
        freq = props.get('frequency')
        pids = props.get('pids')
        template = ('dvbsrc freq=%(freq)d pids=%(pids)s'
                    ' ! flutsdemux name=demux es-pids=%(pids)s'
                    '  demux. ! queue max-size-buffers=0 max-size-time=0 '
                    ' ! video/mpeg ! mpeg2dec'
                    '    ! video/x-raw-yuv,format=(fourcc)I420'
                    '    ! videoscale'
                    '    ! video/x-raw-yuv,width=%(w)s,height=%(h)s,format=(fourcc)I420'
                    '    ! @feeder::video@'
                    '  demux. ! queue max-size-buffers=0 max-size-time=0'
                    ' ! audio/mpeg ! mad'
                    ' ! @feeder::audio@'
                    % dict(freq=freq, pids=pids, w=width, h=height,
                           fr=('%d/%d' % (framerate[0], framerate[1]))))

        return template
<commit_msg>2006-10-04  Zaheer Abbas Merali  <zaheerabbas at merali dot org><commit_after># -*- Mode: Python -*-
# vi:si:et:sw=4:sts=4:ts=4
#
# Flumotion - a streaming media server
# Copyright (C) 2004,2005,2006 Fluendo, S.L. (www.fluendo.com).
# All rights reserved.

# This file may be distributed and/or modified under the terms of
# the GNU General Public License version 2 as published by
# the Free Software Foundation.
# This file is distributed without any warranty; without even the implied
# warranty of merchantability or fitness for a particular purpose.
# See "LICENSE.GPL" in the source distribution for more information.

# Licensees having purchased or holding a valid Flumotion Advanced
# Streaming Server license may use this file in accordance with the
# Flumotion Advanced Streaming Server Commercial License Agreement.
# See "LICENSE.Flumotion" in the source distribution for more information.

# Headers in this file shall remain intact.

from flumotion.component import feedcomponent
from flumotion.common import errors
from twisted.internet import defer

class DVB(feedcomponent.ParseLaunchComponent):

    def do_check(self):
        props = self.config['properties']
        dvb_type = self.dvb_type = props.get('dvb-type')
        if dvb_type != 'T' and dvb_type != 'S':
            msg = "Property dvb-type can only be T (for DVB-T) or S (for DVB-S)."
            return defer.fail(errors.ConfigError(msg))
        # check if the required DVB parameters are passed
        dvb_required_parameters = { "T": ["modulation", "trans-mode",
            "bandwidth", "code-rate-lp", "code-rate-hp", "guard", "hierarchy"],
            "S": ["polarity", "symbol-rate", "satellite-number"] }
        for param in dvb_required_parameters[dvb_type]:
            if not param in props:
                msg = "DVB-%s mode is missing property '%s'." % (dvb_type,
                    param)
                return defer.fail(errors.ConfigError(msg))

    def get_pipeline_string(self, props):
        dvbsrc_template = ""
        if self.dvb_type == "T":
            modulation = props.get('modulation')
            trans_mode = props.get('trans-mode')
            bandwidth = props.get('bandwidth')
            code_rate_lp = props.get('code-rate-lp')
            code_rate_hp = props.get('code-rate-hp')
            guard = props.get('guard')
            hierarchy = props.get('hierarchy')
            dvbsrc_template = '''
dvbsrc modulation="QAM_%(modulation)d" trans-mode=%(trans_mode)dk
bandwidth=%(bandwidth)d code-rate-lp=%(code_rate_lp)s
code-rate-hp=%(code_rate_hp)s guard=%(guard)d
hierarchy=%(hierarchy)d''' % dict(modulation=modulation, trans_mode=trans_mode,
                bandwidth=bandwidth, code_rate_lp=code_rate_lp,
                code_rate_hp="%d/%d" % (code_rate_hp[0], code_rate_hp[1]),
                guard=guard, hierarchy=hierarchy)
        elif self.dvb_type == "S":
            polarity = props.get('polarity')
            symbol_rate = props.get('symbol-rate')
            sat = props.get('satellite-number')
            dvbsrc_template = '''
dvbsrc pol=%(polarity)s srate=%(symbol_rate)s
diseqc-src=%(sat)d''' % dict(polarity=polarity, symbol_rate=symbol_rate,
    sat=sat)
        width = props.get('width', 720)
        height = props.get('height', int(576 * width/720.)) # assuming PAL :-/
        framerate = props.get('framerate', (25, 2))
        framerate_float = float(framerate[0]) / framerate[1]
        freq = props.get('frequency')
        pids = props.get('pids')
        template = ('%s freq=%(freq)d pids=%(pids)s'
                    ' ! flutsdemux name=demux es-pids=%(pids)s'
                    '  demux. ! queue max-size-buffers=0 max-size-time=0 '
                    ' ! video/mpeg ! mpeg2dec'
                    '    ! video/x-raw-yuv,format=(fourcc)I420'
                    '    ! videoscale'
                    '    ! video/x-raw-yuv,width=%(w)s,height=%(h)s,format=(fourcc)I420'
                    '    ! @feeder::video@'
                    '  demux. ! queue max-size-buffers=0 max-size-time=0'
                    ' ! audio/mpeg ! mad'
                    ' ! @feeder::audio@'
                    % (dvbsrc_template,
                       dict(freq=freq, pids=pids, w=width, h=height,
                           fr=('%d/%d' % (framerate[0], framerate[1])))))

        return template
<|endoftext|>


<commit_before>"""writeme"""

from . import selectors
import numpy as np

class LocalCache(object):
    """Basically, a dictionary with some bonus methods.
    """
    def __init__(self, refresh_prob, selector=None):
        """writeme."""

        if selector is None:
            selector = selectors.permute_items
        self._selector = selector

        self._data = dict()
        self._refresh_prob = refresh_prob
        self._item_gen = self._selector(self._data)

    def get(self, key):
        """
        Fetch the Group for a given key.
        """
        return self._data.get(key)

    def add(self, key, entity):
        """writeme."""
        self._data[key] = entity
        self._item_gen = self._selector(self._data)

    def keys(self):
        """writeme."""
        return self._data.keys()

    def __len__(self):
        return len(self._data)

    def __getitem__(self, key):
        return self.get(key)

    def remove(self, key, prob=1):
        """Swap an existing key-value pair with a new one."""
        # Refresh on success.
        if np.random.binomial(1, p=prob):
            print "Deleting %s" % key
            self.remove(key)

    def next(self):
        """Return the next item."""
        key, entity = self._item_gen.next()
        self.remove(key, prob=self._refresh_prob)
        return key, entity

def unpack_entities(entities):
    """
    Parameters
    ----------
    entities: list of Entities
        Note that all Entities must have identical fields.

    Returns
    -------
    arrays: dict of np.ndarrays
        Values in 'arrays' are keyed by the
    """
    arrays = dict([(k, []) for k in entities[0].keys()])
    for entity in entities:
        for k in entity.keys():
            if not k in arrays:
                raise ValueError(
                    "All Entities must have the same fields: %s" % entity)
            arrays[k].append(entity[k].value)

    for k in arrays:
        arrays[k] = np.asarray(arrays[k])
    return arrays

class Queue(object):
    """
    Note: Could potentially take a list of Feature (field) names, which may be
    a subset of the fields in an entity.
    Alternatively, it'd be just as easy to introduce a Transformer that deletes
    or manages such fields (and then the unpack method never needs to change).

    The BatchQueue Pipeline:

     source
      |
    Selector
      |
    Transformer(s)
      |
    Serializer
      |
     result

    """
    def __init__(self, source, batch_size, transformers=None, selector=None,
                 serializer=None, cache_size=1000, refresh_prob=0.01):
        """writeme"""
        self._source = source
        self.batch_size = batch_size

        # If None, safely create list
        if transformers is None:
            transformers = list()
        self._transformers = transformers

        # If None, create default selector
        if selector is None:
            selector = selectors.permute_items
        self._selector = selector(self._source)

        if serializer is None:
            serializer = unpack_entities
        self._serializer = serializer

        # If the requested cache size is larger than the number of items in
        # the source, snap it down and disable auto-refresh.
        if cache_size > len(source):
            cache_size = len(source)
            refresh_prob = 0

        self.cache = LocalCache(refresh_prob=refresh_prob, selector=selector)
        self._cache_size = cache_size
        self.populate()

    def populate(self):
        """writeme"""
        while len(self.cache) <= self._cache_size:
            key, entity = self._selector.next()
            self.cache.add(key, entity)

    def next(self):
        """
        """
        item_buffer = []
        for n in range(self.batch_size):
            item = self.cache.next()[-1]
            for fx in self._transformers:
                item = fx(item)
            item_buffer.append(item)
        return self._serializer(item_buffer)

    def __iter__(self):
        return self
<commit_msg>Rearranged args for Queue init<commit_after>"""writeme"""

from . import selectors
import numpy as np

class LocalCache(object):
    """Basically, a dictionary with some bonus methods.
    """
    def __init__(self, refresh_prob, selector=None):
        """writeme."""

        if selector is None:
            selector = selectors.permute_items
        self._selector = selector

        self._data = dict()
        self._refresh_prob = refresh_prob
        self._item_gen = self._selector(self._data)

    def get(self, key):
        """
        Fetch the Group for a given key.
        """
        return self._data.get(key)

    def add(self, key, entity):
        """writeme."""
        self._data[key] = entity
        self._item_gen = self._selector(self._data)

    def keys(self):
        """writeme."""
        return self._data.keys()

    def __len__(self):
        return len(self._data)

    def __getitem__(self, key):
        return self.get(key)

    def remove(self, key, prob=1):
        """Swap an existing key-value pair with a new one."""
        # Refresh on success.
        if np.random.binomial(1, p=prob):
            print "Deleting %s" % key
            self.remove(key)

    def next(self):
        """Return the next item."""
        key, entity = self._item_gen.next()
        self.remove(key, prob=self._refresh_prob)
        return key, entity

def unpack_entities(entities):
    """
    Parameters
    ----------
    entities: list of Entities
        Note that all Entities must have identical fields.

    Returns
    -------
    arrays: dict of np.ndarrays
        Values in 'arrays' are keyed by the
    """
    arrays = dict([(k, []) for k in entities[0].keys()])
    for entity in entities:
        for k in entity.keys():
            if not k in arrays:
                raise ValueError(
                    "All Entities must have the same fields: %s" % entity)
            arrays[k].append(entity[k].value)

    for k in arrays:
        arrays[k] = np.asarray(arrays[k])
    return arrays

class Queue(object):
    """
    Note: Could potentially take a list of Feature (field) names, which may be
    a subset of the fields in an entity.
    Alternatively, it'd be just as easy to introduce a Transformer that deletes
    or manages such fields (and then the unpack method never needs to change).

    The BatchQueue Pipeline:

     source
      |
    Selector
      |
    Transformer(s)
      |
    Serializer
      |
     result

    """
    def __init__(self, source, batch_size, cache_size=1000, refresh_prob=0.01,
                 transformers=None, selector=None, serializer=None):
        """writeme"""
        self._source = source
        self.batch_size = batch_size

        # If None, safely create list
        if transformers is None:
            transformers = list()
        self._transformers = transformers

        # If None, create default selector
        if selector is None:
            selector = selectors.permute_items
        self._selector = selector(self._source)

        if serializer is None:
            serializer = unpack_entities
        self._serializer = serializer

        # If the requested cache size is larger than the number of items in
        # the source, snap it down and disable auto-refresh.
        if cache_size > len(source):
            cache_size = len(source)
            refresh_prob = 0

        self.cache = LocalCache(refresh_prob=refresh_prob, selector=selector)
        self._cache_size = cache_size
        self.populate()

    def populate(self):
        """writeme"""
        while len(self.cache) <= self._cache_size:
            key, entity = self._selector.next()
            self.cache.add(key, entity)

    def next(self):
        """
        """
        item_buffer = []
        for n in range(self.batch_size):
            item = self.cache.next()[-1]
            for fx in self._transformers:
                item = fx(item)
            item_buffer.append(item)
        return self._serializer(item_buffer)

    def __iter__(self):
        return self
<|endoftext|>


<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2012 Nebula, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from django.utils.translation import ugettext_lazy as _

from horizon import exceptions
from horizon import tabs
import logging

from openstack_dashboard import api

LOG = logging.getLogger(__name__)

class OverviewTab(tabs.Tab):
    name = _("Service")
    slug = "_service"
    template_name = '_services.html'

    def get_context_data(self, request):
        service_id = self.tab_group.kwargs['service_id']
        datacenter_id = api.get_data_center_id_for_service(request, service_id)
        service = api.services_get(request, datacenter_id, service_id)
        return {"service": service}

class LogsTab(tabs.Tab):
    name = _("Logs")
    slug = "_logs"
    template_name = '_service_logs.html'

    def get_context_data(self, request):
        service_id = self.tab_group.kwargs['service_id']
        datacenter_id = api.get_data_center_id_for_service(request, service_id)
        service = api.services_get(request, datacenter_id, service_id)
        return {"service": service}

class WinServicesTabs(tabs.TabGroup):
    slug = "services_details"
    tabs = (OverviewTab, LogsTab)
    sticky = True
<commit_msg>Experiments.<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2012 Nebula, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from django.utils.translation import ugettext_lazy as _

from horizon import exceptions
from horizon import tabs
import logging

from tabula.windc import api

LOG = logging.getLogger(__name__)

class OverviewTab(tabs.Tab):
    name = _("Service")
    slug = "_service"
    template_name = '_services.html'

    def get_context_data(self, request):
        service_id = self.tab_group.kwargs['service_id']
        datacenter_id = api.get_data_center_id_for_service(request, service_id)
        service = api.services_get(request, datacenter_id, service_id)
        return {"service": service}

class LogsTab(tabs.Tab):
    name = _("Logs")
    slug = "_logs"
    template_name = '_service_logs.html'

    def get_context_data(self, request):
        service_id = self.tab_group.kwargs['service_id']
        datacenter_id = api.get_data_center_id_for_service(request, service_id)
        service = api.services_get(request, datacenter_id, service_id)
        return {"service": service}

class WinServicesTabs(tabs.TabGroup):
    slug = "services_details"
    tabs = (OverviewTab, LogsTab)
    sticky = True
<|endoftext|>


<commit_before>"""Implements the GradingProtocol, which runs all specified tests
associated with an assignment.

The GradedTestCase interface should be implemented by TestCases that
are compatible with the GradingProtocol.
"""

from client.protocols.common import models
from client.utils import format
from client.utils import storage
import logging
import sys

log = logging.getLogger(__name__)

#####################
# Testing Mechanism #
#####################

class GradingProtocol(models.Protocol):
    """A Protocol that runs tests, formats results, and sends results
    to the server.
    """
    def run(self, messages, env=None):
        """Run gradeable tests and print results and return analytics.

        RETURNS:
        dict; a mapping of test name -> JSON-serializable object. It is up to
        each test to determine what kind of data it wants to return as
        significant for analytics. However, all tests must include the number
        passed, the number of locked tests and the number of failed tests.
        """
        if self.args.score or self.args.unlock or self.args.testing:
            return
        tests = self.assignment.specified_tests
        for test in tests:
            if self.args.suite and hasattr(test, 'suites'):
                test.run_only = self.args.suite
                try:
                    suite = test.suites[self.args.suite - 1]
                except IndexError as e:
                    sys.exit(('python3 ok: error: '
                        'Suite number must be valid.({})'.format(len(test.suites))))
                if self.args.case:
                    suite.run_only = self.args.case
        grade(tests, messages, env, verbose=self.args.verbose)

def grade(questions, messages, env=None, verbose=True):
    format.print_line('~')
    print('Running tests')
    print()
    passed = 0
    failed = 0
    locked = 0

    analytics = {}

    # The environment in which to run the tests.
    for test in questions:
        log.info('Running tests for {}'.format(test.name))
        results = test.run(env)

        # if correct once, set persistent flag
        if results['failed'] == 0 and results['locked'] == 0:
            storage.store(test.name, 'correct', True)

        passed += results['passed']
        failed += results['failed']
        locked += results['locked']
        analytics[test.name] = results

        if not verbose and (failed > 0 or locked > 0):
            # Stop at the first failed test
            break

    format.print_progress_bar('Test summary', passed, failed, locked,
                              verbose=verbose)
    print()

    messages['grading'] = analytics

protocol = GradingProtocol
<commit_msg>fix error when specifying suite and case for -q<commit_after>"""Implements the GradingProtocol, which runs all specified tests
associated with an assignment.

The GradedTestCase interface should be implemented by TestCases that
are compatible with the GradingProtocol.
"""

from client.protocols.common import models
from client.utils import format
from client.utils import storage
import logging
import sys

log = logging.getLogger(__name__)

#####################
# Testing Mechanism #
#####################

class GradingProtocol(models.Protocol):
    """A Protocol that runs tests, formats results, and sends results
    to the server.
    """
    def run(self, messages, env=None):
        """Run gradeable tests and print results and return analytics.

        RETURNS:
        dict; a mapping of test name -> JSON-serializable object. It is up to
        each test to determine what kind of data it wants to return as
        significant for analytics. However, all tests must include the number
        passed, the number of locked tests and the number of failed tests.
        """
        if self.args.score or self.args.unlock or self.args.testing:
            return
        tests = self.assignment.specified_tests
        for test in tests:
            if self.args.suite and hasattr(test, 'suites'):
                test.run_only = self.args.suite
                try:
                    suite = test.suites[int(self.args.suite) - 1]
                except IndexError as e:
                    sys.exit(('python3 ok: error: '
                        'Suite number must be valid.({})'.format(len(test.suites))))
                if self.args.case:
                    suite.run_only = self.args.case
        grade(tests, messages, env, verbose=self.args.verbose)

def grade(questions, messages, env=None, verbose=True):
    format.print_line('~')
    print('Running tests')
    print()
    passed = 0
    failed = 0
    locked = 0

    analytics = {}

    # The environment in which to run the tests.
    for test in questions:
        log.info('Running tests for {}'.format(test.name))
        results = test.run(env)

        # if correct once, set persistent flag
        if results['failed'] == 0 and results['locked'] == 0:
            storage.store(test.name, 'correct', True)

        passed += results['passed']
        failed += results['failed']
        locked += results['locked']
        analytics[test.name] = results

        if not verbose and (failed > 0 or locked > 0):
            # Stop at the first failed test
            break

    format.print_progress_bar('Test summary', passed, failed, locked,
                              verbose=verbose)
    print()

    messages['grading'] = analytics

protocol = GradingProtocol
<|endoftext|>


<commit_before>'''
Create a visual representation of the various DAGs defined
'''

import sys
import requests
import networkx as nx
import matplotlib.pyplot as plt

if __name__ == '__main__':
    g = nx.DiGraph()
    labels = {
        'edges': {},
        'nodes': {},
    }

    for routeKey, routeMap in requests.get(sys.argv[1]).json().iteritems():
        for i, node in enumerate(routeMap['Path']):
            g.add_node(node['Name'])
            labels['nodes'][node['Name']] = node['Name']
            if i - 1 >= 0:
                g.add_edge(routeMap['Path'][i-1]['Name'], routeMap['Path'][i]['Name'])
                labels['edges'][(routeMap['Path'][i-1]['Name'], routeMap['Path'][i]['Name'])] = (routeMap['Path'][i-1]['Name'], routeMap['Path'][i]['Name'])

    pos = nx.drawing.spring_layout(
        g,
        scale=10.0,
    )
    nx.draw_networkx(
        g,
        pos=pos,
        with_labels=True,
        font_size=8,
    )

    # write out the graph
    plt.savefig(
        'topology.png',
        dpi=400.0,
    )
    plt.show()  # in case people have the required libraries to make it happen
<commit_msg>Remove since d3 is taking over this task<commit_after><|endoftext|>