<commit_before>#!bin/bash/python
# -*- coding: utf-8 -*-

import socket
from typing import Tuple
from collections import deque
from packet import Packet

class Computer(object):
    """xd"""

    def __init__(self, ny_nickname: str,
                 my_socket_address: Tuple[str, int] = ('localhost', 5000),
                 next_computer_address: Tuple[str, int] = ('localhost', 6000),
                 tokenizer: bool=False):
        """
        It gets a tuple to set where to host the server,
        another tuple to set where to send his packets
        and a boolean to set whether it is the first computer on a network (defaults to false because this will only be used once)
        """
        self.ny_nickname = ny_nickname
        self.my_socket_address = my_socket_address
        self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        self.sock.bind(self.my_socket_address)

        self.next_computer_address = next_computer_address
        self.packet_queue = deque()
        if tokenizer:
            self.pass_token()

    def start(self):

        while True:
            packet = Packet(*str(self.wait_connection()).split(';'))  # get packets from socket, cast to str, split(';')
            if not packet.is_token():
                if self.ny_nickname == packet.dest_nick:
                    # if I am the destination device of this packet
                    packet.read()  # mark packet as read "OK"
                    self.connect(packet.to_bytes())  # send packet to next computer on the network
                elif self.ny_nickname == packet.origin_nick:
                    # it means the packet I had sent to some computer on the network got back to me
                    if packet.has_been_read:
                        print('Packet was read by destination machine.')
                        # pass token
                    else:
                        print('Packet was NOT READ by destination.')
                else:
                    # Packet was not sent TO me nor sent BY me.
                    self.connect(packet.to_bytes())
            elif packet.is_token():
                if len(self.packet_queue) > 0:
                    # if I want to send messages
                    self.connect(self.packet_queue.popleft().to_bytes())
                    # wait for packet to come back
                    continue
                else:
                   self.pass_token()

        pass

    def connect(self, text: bytes=b"teste"):
        self.sock.sendto(text, self.next_computer_address)

    def wait_connection(self):
        incoming = self.sock.recv(1024)
        return incoming

    def pass_token(self):
        self.connect(Packet('1234', '', '', '', '').to_bytes())

def read_file(file_path: str) -> list:
    '''
    <ip_destino_token>
    <apelido>
    <tempo_token>
    '''
    with open(file_path) as setup_file:
        return list(setup_file)

"""
To run on a single machine,
openopen two ipython sessions and copypaste this
(must cd token_ring first)

from computer import Computer
pc1 = Computer('Gian', ('0.0.0.0', 5000), ('10.32.143.194', 5000))
print(pc1.wait_connection())

from computer import Computer
pc2 = Computer('nei', ('0.0.0.0', 5000), ('10.32.143.174', 5000))
pc2.connect(b"testão")

"""
# if __name__ == "__main__":
    # setup = read_file("")
<commit_msg>Refactoring tokenizer logic.:top:<commit_after>#!bin/bash/python
# -*- coding: utf-8 -*-

import socket
from typing import Tuple
from collections import deque
from packet import Packet

class Computer(object):
    """xd"""

    def __init__(self, ny_nickname: str,
                 my_socket_address: Tuple[str, int] = ('localhost', 5000),
                 next_computer_address: Tuple[str, int] = ('localhost', 6000),
                 tokenizer: bool=False):
        """
        It gets a tuple to set where to host the server,
        another tuple to set where to send his packets
        and a boolean to set whether it is the first computer on a network (defaults to false because this will only be used once)
        """
        self.ny_nickname = ny_nickname
        self.my_socket_address = my_socket_address
        self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        self.sock.bind(self.my_socket_address)

        self.next_computer_address = next_computer_address
        self.packet_queue = deque()
        self.tokenizer = tokenizer

    def start(self):

        if self.tokenizer:
            self.pass_token()

        while True:
            packet = Packet(*str(self.wait_connection()).split(';'))  # get packets from socket, cast to str, split(';')
            if not packet.is_token():
                if self.ny_nickname == packet.dest_nick:
                    # if I am the destination device of this packet
                    packet.read()  # mark packet as read "OK"
                    self.connect(packet.to_bytes())  # send packet to next computer on the network
                elif self.ny_nickname == packet.origin_nick:
                    # it means the packet I had sent to some computer on the network got back to me
                    if packet.has_been_read:
                        print('Packet was read by destination machine.')
                        # pass token
                    else:
                        print('Packet was NOT READ by destination.')
                else:
                    # Packet was not sent TO me nor sent BY me.
                    self.connect(packet.to_bytes())
            elif packet.is_token():
                print("Recebi token")
                if len(self.packet_queue) > 0:
                    # if I want to send messages
                    self.connect(self.packet_queue.popleft().to_bytes())
                    # wait for packet to come back
                    continue
                else:
                    print("passando token")
                    self.pass_token()

        pass

    def connect(self, text: bytes=b"teste"):
        self.sock.sendto(text, self.next_computer_address)

    def wait_connection(self):
        incoming = self.sock.recv(1024)
        return incoming

    def pass_token(self):
        self.connect(Packet('1234', '', '', '', '').to_bytes())

def read_file(file_path: str) -> list:
    '''
    <ip_destino_token>
    <apelido>
    <tempo_token>to
    '''
    with open(file_path) as setup_file:
        return list(setup_file)

"""
To run on a single machine,
openopen two ipython sessions and copypaste this
(must cd token_ring first)

from computer import Computer
pc1 = Computer('Gian', ('0.0.0.0', 5000), ('10.32.143.194', 5000))
print(pc1.wait_connection())

from computer import Computer
pc2 = Computer('nei', ('0.0.0.0', 5000), ('10.32.143.174', 5000))
pc2.connect(b"testão")

"""
# if __name__ == "__main__":
    # setup = read_file("")
<|endoftext|>


<commit_before>'''
    These generic test modules can be applied to :class:`wpilib.iterativerobot.IterativeRobot`
    and :class:`wpilib.samplerobot.SampleRobot` based robots.
'''

# import basic tests
from .basic import (
    test_autonomous,
    test_disabled,
    test_operator_control,
    test_practice
)

# import common test types
from .docstring_test import test_docstrings

# simple-specific test types
from .fuzz_test import test_fuzz

<commit_msg>Remove docstring tests from default tests<commit_after>'''
    These generic test modules can be applied to :class:`wpilib.iterativerobot.IterativeRobot`
    and :class:`wpilib.samplerobot.SampleRobot` based robots.
'''

# import basic tests
from .basic import (
    test_autonomous,
    test_disabled,
    test_operator_control,
    test_practice
)

# Other test types
from .fuzz_test import test_fuzz

<|endoftext|>


<commit_before>#
# This is open-source software licensed under a BSD license.
# Please see the file LICENSE.txt for details.
#
"""
``WCSMatch`` is a global plugin for the Ginga image viewer that allows
you to roughly align images with different scales and orientations
using the images' World Coordinate System (WCS) for viewing purposes.

**Plugin Type: Global**

``WCSMatch`` is a global plugin.  Only one instance can be opened.

**Usage**

To use, simply start the plugin, and from the plugin GUI select a
channel from the drop-down menu labeled "Reference Channel".  The
image contained in that channel will be used as a reference for
synchronizing the images in the other channels.

The channels will be synchronized in viewing (pan, scale (zoom),
transforms (flips) and rotation.  The checkboxes "Match Pan",
"Match Scale", "Match Transforms" and "Match Rotation" can be
checked or not to control which attributes are synchronized between
channels.

To completely "unlock" the synchronization, simply select "None"
from the "Reference Channel" drop-down menu.

Currently, there is no way to limit the channels that are affected
by the plugin.

"""
from ginga import GingaPlugin
from ginga.gw import Widgets
from ginga.util import wcs
from ginga.misc import Bunch

__all__ = ['WCSMatch']

class WCSMatch(GingaPlugin.GlobalPlugin):

    def __init__(self, fv):
        # superclass defines some variables for us, like logger
        super(WCSMatch, self).__init__(fv)

        self.chnames = []
        self.ref_channel = None
        self.ref_image = None
        self.gui_up = False
        self._cur_opn_viewer = None
        self._match = dict(pan=True, scale=True, transforms=True,
                           rotation=True)

        fv.add_callback('add-channel', self.add_channel)
        fv.add_callback('delete-channel', self.delete_channel)

    def build_gui(self, container):
        top = Widgets.VBox()
        top.set_border_width(4)

        vbox, sw, orientation = Widgets.get_oriented_box(container)
        vbox.set_border_width(4)
        vbox.set_spacing(2)

        fr = Widgets.Frame("WCS Match")

        captions = (('Reference Channel:', 'label',
                     'ref channel', 'combobox'),
                    ('Match Pan', 'checkbutton',
                     'Match Scale', 'checkbutton'),
                    ('Match Transforms', 'checkbutton',
                    'Match Rotation', 'checkbutton'),
                    )
        w, b = Widgets.build_info(captions, orientation=orientation)
        self.w = b

        b.ref_channel.add_callback('activated', self.set_reference_channel_cb)
        self.w.match_pan.set_state(self._match['pan'])
        self.w.match_pan.set_tooltip("Match pan position of reference image")
        self.w.match_pan.add_callback('activated',
                                      self.set_match_cb, 'pan')
        self.w.match_scale.set_state(self._match['scale'])
        self.w.match_scale.add_callback('activated',
                                        self.set_match_cb, 'scale')
        self.w.match_scale.set_tooltip("Match scale of reference image")
        self.w.match_transforms.set_state(self._match['transforms'])
        self.w.match_transforms.add_callback('activated',
                                             self.set_match_cb, 'transforms')
        self.w.match_transforms.set_tooltip("Match transforms of reference image")
        self.w.match_rotation.set_state(self._match['rotation'])
        self.w.match_rotation.add_callback('activated',
                                           self.set_match_cb, 'rotation')
        self.w.match_rotation.set_tooltip("Match rotation of reference image")

        fr.set_widget(w)
        vbox.add_widget(fr, stretch=0)

        spacer = Widgets.Label('')
        vbox.add_widget(spacer, stretch=1)

        top.add_widget(sw, stretch=1)

        btns = Widgets.HBox()
        btns.set_spacing(3)

        btn = Widgets.Button("Close")
        btn.add_callback('activated', lambda w: self.close())
        btns.add_widget(btn, stretch=0)
        btn = Widgets.Button("Help")
        btn.add_callback('activated', lambda w: self.help())
        btns.add_widget(btn, stretch=0)
        btns.add_widget(Widgets.Label(''), stretch=1)
        top.add_widget(btns, stretch=0)

        container.add_widget(top, stretch=1)

        self.gui_up = True
        self._reset_channels_gui()

    def add_channel(self, viewer, channel):
        info = Bunch.Bunch(chinfo=channel)
        channel.extdata._wcsmatch_info = info

        # Add callbacks to the viewer for all the scale, pan, rotation and
        # transform settings changes
        chviewer = channel.fitsimage
        fitssettings = chviewer.get_settings()
        fitssettings.get_setting('scale').add_callback(
            'set', self.zoomset_cb, chviewer, info)
        fitssettings.get_setting('rot_deg').add_callback(
            'set', self.rotset_cb, chviewer, info)
        for name in ('flip_x', 'flip_y', 'swap_xy'):
            fitssettings.get_setting(name).add_callback(
                'set', self.xfmset_cb, chviewer, info)
        fitssettings.get_setting('pan').add_callback(
            'set', self.panset_cb, chviewer, info)
        self.fv.gui_do(self._reset_channels_gui)

    def delete_channel(self, viewer, channel):
        chname = channel.name
        self.logger.debug("deleting channel %s" % (chname))
        self.active = None
        self.info = None
        self.fv.gui_do(self._reset_channels_gui)

    def _reset_channels_gui(self):
        self.chnames = list(self.fv.get_channel_names())
        self.chnames.sort()
        self.chnames.insert(0, "None")
        if not self.gui_up:
            return
        self.w.ref_channel.clear()
        for chname in self.chnames:
            self.w.ref_channel.append_text(chname)

    # CALLBACKS

    def _update_all(self):
        if self.ref_channel is None:
            return
        chinfo = self.ref_channel
        chviewer = chinfo.fitsimage
        self.zoomset(chviewer, chinfo)
        self.rotset(chviewer, chinfo)
        self.xfmset(chviewer, chinfo)
        self.panset(chviewer, chinfo)

    def set_reference_channel_cb(self, w, idx):
        chname = self.chnames[idx]
        if chname == 'None':
            chname = None
        self.set_reference_channel(chname)

    def set_reference_channel(self, chname):
        if chname is None:
            self.ref_image = None
            self.ref_channel = None
            self.logger.info("turning off channel synchronization")
            return

        chinfo = self.fv.get_channel(chname)
        self.ref_channel = chinfo
        chviewer = chinfo.fitsimage
        self.ref_image = chviewer.get_image()

        # reset the scale base to be identical in both axes for the
        # reference image
        chviewer.set_scale_base_xy(1.0, 1.0)

        self._update_all()

        self.logger.info("set reference channel to '%s'" % (chname))

    def set_match_cb(self, w, tf, key):
        # remember, in case we are closed and reopened
        self._match.update(dict(pan=self.w.match_pan.get_state(),
                                scale=self.w.match_scale.get_state(),
                                transforms=self.w.match_transforms.get_state(),
                                rotation=self.w.match_rotation.get_state()))

        if key == 'scale' and tf is False:
            self.reset_scale_skew()

        self._update_all()

    def close(self):
        self.fv.stop_global_plugin(str(self))
        return True

    def start(self):
        pass

    def stop(self):
        self.ref_channel = None
        self.ref_image = None
        self.fv.show_status("")

    def get_other_channels(self, myname):
        return set(self.fv.get_channel_names()) - set([myname])

    def zoomset_cb(self, setting, value, chviewer, info):
        """This callback is called when a channel window is zoomed.
        """
        return self.zoomset(chviewer, info.chinfo)

    def zoomset(self, chviewer, chinfo):
        # Don't do anything if we are not active
        if not self.gui_up or self.ref_image is None:
            return

        if self._cur_opn_viewer is not None:
            return
        if not self._match['scale']:
            return

        self._cur_opn_viewer = chviewer
        try:
            self.scale_all_relative(chviewer, chinfo)

        finally:
            self._cur_opn_viewer = None

    def scale_all_relative(self, chviewer, chinfo):
        if self.ref_image is None:
            return

        # get native scale relative to reference image
        image = chviewer.get_image()
        ort = wcs.get_relative_orientation(image, self.ref_image)
        self.logger.info("scale for channel '%s' relative to ref image "
                         "%f,%f" % (chinfo.name, ort.rscale_x, ort.rscale_y))

        scale_x, scale_y = chviewer.get_scale_xy()

        chg_x, chg_y = scale_x / ort.rscale_x, scale_y / ort.rscale_y
        self.logger.info("scale changed for channel '%s' by %f,%f" % (
            chinfo.name, chg_x, chg_y))

        # for all other channels except ours
        chnames = self.get_other_channels(chinfo.name)
        for chname in chnames:
            chinfo2 = self.fv.get_channel(chname)

            # calculate scale from orientation to reference image
            image = chinfo2.fitsimage.get_image()
            if image is None:
                continue
            ort = wcs.get_relative_orientation(image, self.ref_image)

            new_scale_x, new_scale_y = (ort.rscale_x * chg_x,
                                        ort.rscale_y * chg_y)
            # apply that scale
            self.logger.info("changing scale for channel '%s' to %f,%f" % (
                chinfo2.name, new_scale_x, new_scale_y))
            chinfo2.fitsimage.scale_to(new_scale_x, new_scale_y)

    def reset_scale_skew(self):
        """Set the X/Y scaling factors to be equal in each channel.
        """
        chnames = self.fv.get_channel_names()
        for chname in chnames:
            chinfo = self.fv.get_channel(chname)

            scales = chinfo.fitsimage.get_scale_xy()
            fac = min(*scales)
            chinfo.fitsimage.scale_to(fac, fac)

    def rotset_cb(self, setting, value, chviewer, info):
        """This callback is called when a channel window is rotated.
        """
        return self.rotset(chviewer, info.chinfo)

    def rotset(self, chviewer, chinfo):
        # Don't do anything if we are not active
        if not self.gui_up or self.ref_image is None:
            return

        if self._cur_opn_viewer is not None:
            return
        if not self._match['rotation']:
            return

        self._cur_opn_viewer = chviewer
        try:
            self.rotate_all_relative(chviewer, chinfo)

        finally:
            self._cur_opn_viewer = None

    def rotate_all_relative(self, chviewer, chinfo):
        if self.ref_image is None:
            return

        # get native scale relative to reference image
        image = chviewer.get_image()
        if self.ref_image is None:
            return
        ort = wcs.get_relative_orientation(image, self.ref_image)
        self.logger.info("rotation for channel '%s' relative to ref image "
                         "%f" % (chinfo.name, ort.rrot_deg))

        rot_deg = chviewer.get_rotation()

        chg_rot_deg = rot_deg + ort.rrot_deg
        self.logger.info("rotation changed for channel '%s' by %f" % (
            chinfo.name, chg_rot_deg))

        # for all other channels except ours
        chnames = self.get_other_channels(chinfo.name)
        for chname in chnames:
            chinfo2 = self.fv.get_channel(chname)

            # Get relative rotation of their image
            image = chinfo2.fitsimage.get_image()
            if image is None:
                continue
            ort = wcs.get_relative_orientation(image, self.ref_image)

            # Apply that rotation
            new_rot_deg = ort.rrot_deg + chg_rot_deg
            self.logger.info("changing rot for channel '%s' to %f" % (
                chinfo2.name, new_rot_deg))
            chinfo2.fitsimage.rotate(new_rot_deg)

    def panset_cb(self, setting, value, chviewer, info):
        """This callback is called when a channel window is panned.
        """
        return self.panset(chviewer, info.chinfo)

    def panset(self, chviewer, chinfo):
        # Don't do anything if we are not active
        if not self.gui_up or self.ref_image is None:
            return

        if self._cur_opn_viewer is not None:
            return
        if not self._match['pan']:
            return

        self._cur_opn_viewer = chviewer
        try:
            self.pan_all_relative(chviewer, chinfo)

        finally:
            self._cur_opn_viewer = None

    def pan_all_relative(self, chviewer, chinfo):
        if self.ref_image is None:
            return

        image = chviewer.get_image()
        if self.ref_image is None:
            return

        pan_ra, pan_dec = chviewer.get_pan(coord='wcs')

        # for all other channels except ours
        chnames = self.get_other_channels(chinfo.name)
        for chname in chnames:
            chinfo2 = self.fv.get_channel(chname)

            # set pan position on their viewer
            image = chinfo2.fitsimage.get_image()
            if image is None:
                continue
            data_x, data_y = image.radectopix(pan_ra, pan_dec)
            chinfo2.fitsimage.panset_xy(data_x, data_y)

    def xfmset_cb(self, setting, value, chviewer, info):
        """This callback is called when a channel window is transformed
        (flipped, or swap axes).
        """
        return self.xfmset(chviewer, info.chinfo)

    def xfmset(self, chviewer, chinfo):
        # Don't do anything if we are not active
        if not self.gui_up or self.ref_image is None:
            return

        if self._cur_opn_viewer is not None:
            return
        if not self._match['transforms']:
            return

        self._cur_opn_viewer = chviewer
        try:
            self.transform_all_relative(chviewer, chinfo)

        finally:
            self._cur_opn_viewer = None

    def transform_all_relative(self, chviewer, chinfo):
        if self.ref_image is None:
            return

        image = chviewer.get_image()
        if self.ref_image is None:
            return

        flip_x, flip_y, swap_xy = chviewer.get_transforms()

        # for all other channels except ours
        chnames = self.get_other_channels(chinfo.name)
        for chname in chnames:
            chinfo2 = self.fv.get_channel(chname)

            # set our pan position on their viewer
            image = chinfo2.fitsimage.get_image()
            if image is None:
                continue
            chinfo2.fitsimage.transform(flip_x, flip_y, swap_xy)

    def __str__(self):
        return 'wcsmatch'

# END
<commit_msg>Fix to make GUI control match when calling via API<commit_after>#
# This is open-source software licensed under a BSD license.
# Please see the file LICENSE.txt for details.
#
"""
``WCSMatch`` is a global plugin for the Ginga image viewer that allows
you to roughly align images with different scales and orientations
using the images' World Coordinate System (WCS) for viewing purposes.

**Plugin Type: Global**

``WCSMatch`` is a global plugin.  Only one instance can be opened.

**Usage**

To use, simply start the plugin, and from the plugin GUI select a
channel from the drop-down menu labeled "Reference Channel".  The
image contained in that channel will be used as a reference for
synchronizing the images in the other channels.

The channels will be synchronized in viewing (pan, scale (zoom),
transforms (flips) and rotation.  The checkboxes "Match Pan",
"Match Scale", "Match Transforms" and "Match Rotation" can be
checked or not to control which attributes are synchronized between
channels.

To completely "unlock" the synchronization, simply select "None"
from the "Reference Channel" drop-down menu.

Currently, there is no way to limit the channels that are affected
by the plugin.

"""
from ginga import GingaPlugin
from ginga.gw import Widgets
from ginga.util import wcs
from ginga.misc import Bunch

__all__ = ['WCSMatch']

class WCSMatch(GingaPlugin.GlobalPlugin):

    def __init__(self, fv):
        # superclass defines some variables for us, like logger
        super(WCSMatch, self).__init__(fv)

        self.chnames = []
        self.ref_channel = None
        self.ref_image = None
        self.gui_up = False
        self._cur_opn_viewer = None
        self._match = dict(pan=True, scale=True, transforms=True,
                           rotation=True)

        fv.add_callback('add-channel', self.add_channel)
        fv.add_callback('delete-channel', self.delete_channel)

    def build_gui(self, container):
        top = Widgets.VBox()
        top.set_border_width(4)

        vbox, sw, orientation = Widgets.get_oriented_box(container)
        vbox.set_border_width(4)
        vbox.set_spacing(2)

        fr = Widgets.Frame("WCS Match")

        captions = (('Reference Channel:', 'label',
                     'ref channel', 'combobox'),
                    ('Match Pan', 'checkbutton',
                     'Match Scale', 'checkbutton'),
                    ('Match Transforms', 'checkbutton',
                    'Match Rotation', 'checkbutton'),
                    )
        w, b = Widgets.build_info(captions, orientation=orientation)
        self.w = b

        b.ref_channel.add_callback('activated', self._set_reference_channel_cb)
        self.w.match_pan.set_state(self._match['pan'])
        self.w.match_pan.set_tooltip("Match pan position of reference image")
        self.w.match_pan.add_callback('activated',
                                      self.set_match_cb, 'pan')
        self.w.match_scale.set_state(self._match['scale'])
        self.w.match_scale.add_callback('activated',
                                        self.set_match_cb, 'scale')
        self.w.match_scale.set_tooltip("Match scale of reference image")
        self.w.match_transforms.set_state(self._match['transforms'])
        self.w.match_transforms.add_callback('activated',
                                             self.set_match_cb, 'transforms')
        self.w.match_transforms.set_tooltip("Match transforms of reference image")
        self.w.match_rotation.set_state(self._match['rotation'])
        self.w.match_rotation.add_callback('activated',
                                           self.set_match_cb, 'rotation')
        self.w.match_rotation.set_tooltip("Match rotation of reference image")

        fr.set_widget(w)
        vbox.add_widget(fr, stretch=0)

        spacer = Widgets.Label('')
        vbox.add_widget(spacer, stretch=1)

        top.add_widget(sw, stretch=1)

        btns = Widgets.HBox()
        btns.set_spacing(3)

        btn = Widgets.Button("Close")
        btn.add_callback('activated', lambda w: self.close())
        btns.add_widget(btn, stretch=0)
        btn = Widgets.Button("Help")
        btn.add_callback('activated', lambda w: self.help())
        btns.add_widget(btn, stretch=0)
        btns.add_widget(Widgets.Label(''), stretch=1)
        top.add_widget(btns, stretch=0)

        container.add_widget(top, stretch=1)

        self.gui_up = True
        self._reset_channels_gui()

    def add_channel(self, viewer, channel):
        info = Bunch.Bunch(chinfo=channel)
        channel.extdata._wcsmatch_info = info

        # Add callbacks to the viewer for all the scale, pan, rotation and
        # transform settings changes
        chviewer = channel.fitsimage
        fitssettings = chviewer.get_settings()
        fitssettings.get_setting('scale').add_callback(
            'set', self.zoomset_cb, chviewer, info)
        fitssettings.get_setting('rot_deg').add_callback(
            'set', self.rotset_cb, chviewer, info)
        for name in ('flip_x', 'flip_y', 'swap_xy'):
            fitssettings.get_setting(name).add_callback(
                'set', self.xfmset_cb, chviewer, info)
        fitssettings.get_setting('pan').add_callback(
            'set', self.panset_cb, chviewer, info)
        self.fv.gui_do(self._reset_channels_gui)

    def delete_channel(self, viewer, channel):
        chname = channel.name
        self.logger.debug("deleting channel %s" % (chname))
        self.active = None
        self.info = None
        self.fv.gui_do(self._reset_channels_gui)

    def _reset_channels_gui(self):
        self.chnames = list(self.fv.get_channel_names())
        self.chnames.sort()
        self.chnames.insert(0, "None")
        if not self.gui_up:
            return
        self.w.ref_channel.clear()
        for chname in self.chnames:
            self.w.ref_channel.append_text(chname)

    # CALLBACKS

    def _update_all(self):
        if self.ref_channel is None:
            return
        chinfo = self.ref_channel
        chviewer = chinfo.fitsimage
        self.zoomset(chviewer, chinfo)
        self.rotset(chviewer, chinfo)
        self.xfmset(chviewer, chinfo)
        self.panset(chviewer, chinfo)

    def _set_reference_channel(self, chname):
        if chname == 'None':
            chname = None
        if chname is None:
            self.ref_image = None
            self.ref_channel = None
            self.logger.info("turning off channel synchronization")
            return

        chinfo = self.fv.get_channel(chname)
        self.ref_channel = chinfo
        chviewer = chinfo.fitsimage
        self.ref_image = chviewer.get_image()

        # reset the scale base to be identical in both axes for the
        # reference image
        chviewer.set_scale_base_xy(1.0, 1.0)

        self._update_all()

        self.logger.info("set reference channel to '%s'" % (chname))

    def _set_reference_channel_cb(self, w, idx):
        """This is the GUI callback for the control that sets the reference
        channel.
        """
        chname = self.chnames[idx]
        self._set_reference_channel(chname)

    def set_reference_channel(self, chname):
        """This is the API call to set the reference channel.
        """
        # change the GUI control to match
        idx = self.chnames.index(str(chname))
        self.w.ref_channel.set_index(idx)
        return self._set_reference_channel(chname)

    def set_match_cb(self, w, tf, key):
        # remember, in case we are closed and reopened
        self._match.update(dict(pan=self.w.match_pan.get_state(),
                                scale=self.w.match_scale.get_state(),
                                transforms=self.w.match_transforms.get_state(),
                                rotation=self.w.match_rotation.get_state()))

        if key == 'scale' and tf is False:
            self.reset_scale_skew()

        self._update_all()

    def close(self):
        self.fv.stop_global_plugin(str(self))
        return True

    def start(self):
        pass

    def stop(self):
        self.ref_channel = None
        self.ref_image = None
        self.fv.show_status("")

    def get_other_channels(self, myname):
        return set(self.fv.get_channel_names()) - set([myname])

    def zoomset_cb(self, setting, value, chviewer, info):
        """This callback is called when a channel window is zoomed.
        """
        return self.zoomset(chviewer, info.chinfo)

    def zoomset(self, chviewer, chinfo):
        # Don't do anything if we are not active
        if not self.gui_up or self.ref_image is None:
            return

        if self._cur_opn_viewer is not None:
            return
        if not self._match['scale']:
            return

        self._cur_opn_viewer = chviewer
        try:
            self.scale_all_relative(chviewer, chinfo)

        finally:
            self._cur_opn_viewer = None

    def scale_all_relative(self, chviewer, chinfo):
        if self.ref_image is None:
            return

        # get native scale relative to reference image
        image = chviewer.get_image()
        ort = wcs.get_relative_orientation(image, self.ref_image)
        self.logger.info("scale for channel '%s' relative to ref image "
                         "%f,%f" % (chinfo.name, ort.rscale_x, ort.rscale_y))

        scale_x, scale_y = chviewer.get_scale_xy()

        chg_x, chg_y = scale_x / ort.rscale_x, scale_y / ort.rscale_y
        self.logger.info("scale changed for channel '%s' by %f,%f" % (
            chinfo.name, chg_x, chg_y))

        # for all other channels except ours
        chnames = self.get_other_channels(chinfo.name)
        for chname in chnames:
            chinfo2 = self.fv.get_channel(chname)

            # calculate scale from orientation to reference image
            image = chinfo2.fitsimage.get_image()
            if image is None:
                continue
            ort = wcs.get_relative_orientation(image, self.ref_image)

            new_scale_x, new_scale_y = (ort.rscale_x * chg_x,
                                        ort.rscale_y * chg_y)
            # apply that scale
            self.logger.info("changing scale for channel '%s' to %f,%f" % (
                chinfo2.name, new_scale_x, new_scale_y))
            chinfo2.fitsimage.scale_to(new_scale_x, new_scale_y)

    def reset_scale_skew(self):
        """Set the X/Y scaling factors to be equal in each channel.
        """
        chnames = self.fv.get_channel_names()
        for chname in chnames:
            chinfo = self.fv.get_channel(chname)

            scales = chinfo.fitsimage.get_scale_xy()
            fac = min(*scales)
            chinfo.fitsimage.scale_to(fac, fac)

    def rotset_cb(self, setting, value, chviewer, info):
        """This callback is called when a channel window is rotated.
        """
        return self.rotset(chviewer, info.chinfo)

    def rotset(self, chviewer, chinfo):
        # Don't do anything if we are not active
        if not self.gui_up or self.ref_image is None:
            return

        if self._cur_opn_viewer is not None:
            return
        if not self._match['rotation']:
            return

        self._cur_opn_viewer = chviewer
        try:
            self.rotate_all_relative(chviewer, chinfo)

        finally:
            self._cur_opn_viewer = None

    def rotate_all_relative(self, chviewer, chinfo):
        if self.ref_image is None:
            return

        # get native scale relative to reference image
        image = chviewer.get_image()
        if self.ref_image is None:
            return
        ort = wcs.get_relative_orientation(image, self.ref_image)
        self.logger.info("rotation for channel '%s' relative to ref image "
                         "%f" % (chinfo.name, ort.rrot_deg))

        rot_deg = chviewer.get_rotation()

        chg_rot_deg = rot_deg + ort.rrot_deg
        self.logger.info("rotation changed for channel '%s' by %f" % (
            chinfo.name, chg_rot_deg))

        # for all other channels except ours
        chnames = self.get_other_channels(chinfo.name)
        for chname in chnames:
            chinfo2 = self.fv.get_channel(chname)

            # Get relative rotation of their image
            image = chinfo2.fitsimage.get_image()
            if image is None:
                continue
            ort = wcs.get_relative_orientation(image, self.ref_image)

            # Apply that rotation
            new_rot_deg = ort.rrot_deg + chg_rot_deg
            self.logger.info("changing rot for channel '%s' to %f" % (
                chinfo2.name, new_rot_deg))
            chinfo2.fitsimage.rotate(new_rot_deg)

    def panset_cb(self, setting, value, chviewer, info):
        """This callback is called when a channel window is panned.
        """
        return self.panset(chviewer, info.chinfo)

    def panset(self, chviewer, chinfo):
        # Don't do anything if we are not active
        if not self.gui_up or self.ref_image is None:
            return

        if self._cur_opn_viewer is not None:
            return
        if not self._match['pan']:
            return

        self._cur_opn_viewer = chviewer
        try:
            self.pan_all_relative(chviewer, chinfo)

        finally:
            self._cur_opn_viewer = None

    def pan_all_relative(self, chviewer, chinfo):
        if self.ref_image is None:
            return

        image = chviewer.get_image()
        if self.ref_image is None:
            return

        pan_ra, pan_dec = chviewer.get_pan(coord='wcs')

        # for all other channels except ours
        chnames = self.get_other_channels(chinfo.name)
        for chname in chnames:
            chinfo2 = self.fv.get_channel(chname)

            # set pan position on their viewer
            image = chinfo2.fitsimage.get_image()
            if image is None:
                continue
            data_x, data_y = image.radectopix(pan_ra, pan_dec)
            chinfo2.fitsimage.panset_xy(data_x, data_y)

    def xfmset_cb(self, setting, value, chviewer, info):
        """This callback is called when a channel window is transformed
        (flipped, or swap axes).
        """
        return self.xfmset(chviewer, info.chinfo)

    def xfmset(self, chviewer, chinfo):
        # Don't do anything if we are not active
        if not self.gui_up or self.ref_image is None:
            return

        if self._cur_opn_viewer is not None:
            return
        if not self._match['transforms']:
            return

        self._cur_opn_viewer = chviewer
        try:
            self.transform_all_relative(chviewer, chinfo)

        finally:
            self._cur_opn_viewer = None

    def transform_all_relative(self, chviewer, chinfo):
        if self.ref_image is None:
            return

        image = chviewer.get_image()
        if self.ref_image is None:
            return

        flip_x, flip_y, swap_xy = chviewer.get_transforms()

        # for all other channels except ours
        chnames = self.get_other_channels(chinfo.name)
        for chname in chnames:
            chinfo2 = self.fv.get_channel(chname)

            # set our pan position on their viewer
            image = chinfo2.fitsimage.get_image()
            if image is None:
                continue
            chinfo2.fitsimage.transform(flip_x, flip_y, swap_xy)

    def __str__(self):
        return 'wcsmatch'

# END
<|endoftext|>


<commit_before>from django.conf import settings
from django.contrib.auth.decorators import login_required
from django.http import HttpResponse
from django.views.decorators.csrf import csrf_exempt
from django.utils import timezone
from datetime import datetime
import logging
import os
import json
from statsd import statsd

from models import Submission, Grader
from models import GraderStatus, SubmissionState
import util
import grader_util
from staff_grading import staff_grading_util
from basic_check import basic_check_util
from metrics import timing_functions
import message_util
from ml_grading import ml_grading_util

log = logging.getLogger(__name__)

_INTERFACE_VERSION = 1

@csrf_exempt
@login_required
@statsd.timed('open_ended_assessment.grading_controller.controller.xqueue_interface.time', tags=['function:submit'])
def submit(request):
    '''
    Xqueue pull script posts objects here.
    Input:

    request - dict with keys xqueue_header and xqueue_body
    xqueue_header needs submission_id,submission_key,queue_name
    xqueue_body needs grader_payload, student_info, student_response, max_score
    grader_payload needs location, course_id,problem_id,grader
    student_info needs anonymous_student_id, submission_time

    Output:
    Returns status code indicating success (0) or failure (1) and message
    '''
    if request.method != 'POST':
        return util._error_response("'submit' must use HTTP POST", _INTERFACE_VERSION)
    else:
        #Minimal parsing of reply
        reply_is_valid, header, body = _is_valid_reply(request.POST.copy())

        if not reply_is_valid:
            log.error("Invalid xqueue object added: request_ip: {0} request.POST: {1}".format(
                util.get_request_ip(request),
                request.POST,
            ))
            statsd.increment("open_ended_assessment.grading_controller.controller.xqueue_interface.submit",
                tags=["success:Exception"])
            return util._error_response('Incorrect format', _INTERFACE_VERSION)
        else:
            try:
                #Retrieve individual values from xqueue body and header.
                prompt = util._value_or_default(body['grader_payload']['prompt'], "")
                rubric = util._value_or_default(body['grader_payload']['rubric'], "")
                initial_display = util._value_or_default(body['grader_payload']['initial_display'], "")
                answer = util._value_or_default(body['grader_payload']['answer'], "")
                student_id = util._value_or_default(body['student_info']['anonymous_student_id'])
                location = util._value_or_default(body['grader_payload']['location'])
                course_id = util._value_or_default(body['grader_payload']['course_id'])
                problem_id = util._value_or_default(body['grader_payload']['problem_id'], location)
                grader_settings = util._value_or_default(body['grader_payload']['grader_settings'], "")
                student_response = util._value_or_default(body['student_response'])
                xqueue_submission_id = util._value_or_default(header['submission_id'])
                xqueue_submission_key = util._value_or_default(header['submission_key'])
                state_code = SubmissionState.waiting_to_be_graded
                xqueue_queue_name = util._value_or_default(header["queue_name"])
                max_score = util._value_or_default(body['max_score'])

                submission_time_string = util._value_or_default(body['student_info']['submission_time'])
                student_submission_time = datetime.strptime(submission_time_string, "%Y%m%d%H%M%S")

                #Create submission object
                sub, created = Submission.objects.get_or_create(
                    prompt=prompt,
                    rubric=rubric,
                    student_id=student_id,
                    problem_id=problem_id,
                    state=state_code,
                    student_response=student_response,
                    student_submission_time=student_submission_time,
                    xqueue_submission_id=xqueue_submission_id,
                    xqueue_submission_key=xqueue_submission_key,
                    xqueue_queue_name=xqueue_queue_name,
                    location=location,
                    course_id=course_id,
                    max_score=max_score,
                    grader_settings=grader_settings,
                    initial_display=initial_display,
                    answer=answer,
                )

            except Exception as err:
                xqueue_submission_id = util._value_or_default(header['submission_id'])
                xqueue_submission_key = util._value_or_default(header['submission_key'])
                log.error(
                    "Error creating submission and adding to database: sender: {0}, submission_id: {1}, submission_key: {2}".format(
                        util.get_request_ip(request),
                        xqueue_submission_id,
                        xqueue_submission_key,
                    ))

                statsd.increment("open_ended_assessment.grading_controller.controller.xqueue_interface.submit",
                    tags=["success:Exception"])

                return util._error_response('Unable to create submission.', _INTERFACE_VERSION)

            #Handle submission and write to db
            success = handle_submission(sub)
            statsd.increment("open_ended_assessment.grading_controller.controller.xqueue_interface.submit",
                tags=[
                    "success:{0}".format(success),
                    "location:{0}".format(sub.location),
                    "course_id:{0}".format(course_id),
                ])

            if not success:
                return util._error_response("Failed to handle submission.", _INTERFACE_VERSION)

            return util._success_response({'message': "Saved successfully."}, _INTERFACE_VERSION)

def handle_submission(sub):
    """
    Handles a new submission.  Decides what the next grader should be and saves it.
    Input:
        sub - A Submission object from controller.models

    Output:
        True/False status code
    """
    try:
        #Run some basic sanity checks on submission
        sub.next_grader_type = "BC"
        sub.save()
        timing_functions.initialize_timing(sub.id)
        success, check_dict = basic_check_util.simple_quality_check(sub.student_response, sub.initial_display)
        if not success:
            log.exception("could not run basic checks on {0}".format(sub.student_response))

        #add additional tags needed to create a grader object
        check_dict = grader_util.add_additional_tags_to_dict(check_dict, sub.id)

        #Create and handle the grader, and return
        grader_util.create_and_handle_grader_object(check_dict)

        #If the checks result in a score of 0 (out of 1), then the submission fails basic sanity checks
        #Return to student and don't process further
        if check_dict['score'] == 0:
            return True
        else:
            sub.state = SubmissionState.waiting_to_be_graded

        #Assign whether grader should be ML or IN based on number of graded examples.
        subs_graded_by_instructor, subs_pending_instructor = staff_grading_util.count_submissions_graded_and_pending_instructor(
            sub.location)

        #TODO: abstract out logic for assigning which grader to go with.
        grader_settings_path = os.path.join(settings.GRADER_SETTINGS_DIRECTORY, sub.grader_settings)
        grader_settings = grader_util.get_grader_settings(grader_settings_path)
        if grader_settings['grader_type'] == "ML":
            success, model = ml_grading_util.get_latest_created_model(sub.location)
            if(((subs_graded_by_instructor + subs_pending_instructor) >= settings.MIN_TO_USE_ML) and success):
                sub.next_grader_type = "ML"
            else:
                sub.next_grader_type = "IN"
        elif grader_settings['grader_type'] == "PE":
            #Ensures that there will be some calibration essays before peer grading begins!
            #Calibration essays can be added using command line utility, or through normal instructor grading.
            if((subs_graded_by_instructor + subs_pending_instructor) >= settings.MIN_TO_USE_PEER):
                sub.next_grader_type = "PE"
            else:
                sub.next_grader_type = "IN"
        elif grader_settings['grader_type'] == "IN":
            sub.next_grader_type = "IN"
        else:
            log.exception("Invalid grader type specified in settings file.")
            return False

        sub.save()
        log.debug("Submission object created successfully!")

    except:
        log.exception("Submission creation failed!")
        return False

    return True

def _is_valid_reply(external_reply):
    '''
    Check if external reply is in the right format
        1) Presence of 'xqueue_header' and 'xqueue_body'
        2) Presence of specific metadata in 'xqueue_header'
            ['submission_id', 'submission_key']

    Returns:
        is_valid:       Flag indicating success (Boolean)
        header :        header of the queue item
        body:           body of the queue item
    '''
    fail = (False, -1, '')

    success, header, body = _is_valid_reply_generic(external_reply)

    if not success:
        return fail

    for tag in ['grader_payload', 'student_response', 'student_info']:
        if not body.has_key(tag):
            log.debug("{0} not found in body".format(tag))
            return fail

    try:
        body['grader_payload'] = json.loads(body['grader_payload'])
        body['student_info'] = json.loads(body['student_info'])
    except:
        log.debug("Cannot load payload or info.")
        return fail

    return True, header, body

def _is_valid_reply_generic(external_reply):
    try:
        header = json.loads(external_reply['xqueue_header'])
        body = json.loads(external_reply['xqueue_body'])
    except KeyError:
        log.debug("Cannot load header or body.")
        return False, "", ""

    if not isinstance(header, dict) or not isinstance(body, dict):
        return False, "", ""

    for tag in ['submission_id', 'submission_key', 'queue_name']:
        if not header.has_key(tag):
            log.debug("{0} not found in header".format(tag))
            return False, "", ""
    return True, header, body

def _is_valid_reply_message(external_reply):
    fail = (False, -1, '')

    success, header, body = _is_valid_reply_generic(external_reply)

    if not success:
        return fail

    for tag in ['student_info', 'submission_id', 'grader_id', 'feedback']:
        if not body.has_key(tag):
            log.debug("{0} not found in body".format(tag))
            return fail

    body['student_info'] = json.loads(body['student_info'])
    for tag in ['anonymous_student_id']:
        if not body['student_info'].has_key(tag):
            log.debug("{0} not found in student info".format(tag))
            return fail

    return True, header, body

@csrf_exempt
def submit_message(request):
    """
    Submits a message to the grading controller.

    """
    if request.method != 'POST':
        return util._error_response("'submit_message' must use HTTP POST", _INTERFACE_VERSION)

    reply_is_valid, header, body = _is_valid_reply_message(request.POST.copy())

    if not reply_is_valid:
        log.error("Invalid xqueue object added: request_ip: {0} request.POST: {1}".format(
            util.get_request_ip(request),
            request.POST,
        ))
        statsd.increment("open_ended_assessment.grading_controller.controller.xqueue_interface.submit_message",
            tags=["success:Exception"])
        return util._error_response('Incorrect format', _INTERFACE_VERSION)

    message = body['feedback']
    grader_id = body['grader_id']
    submission_id = body['submission_id']
    originator = body['student_info']['anonymous_student_id']

    try:
        if 'score' in body:
            score = int(body['score'])
        else:
            score = None
    except:
        error_message = "Score was not an integer, received \"{0}\" instead.".format(score)
        log.exception(error_message)
        return util._error_response(error_message, _INTERFACE_VERSION)

    try:
        grade = Grader.objects.get(id=grader_id)
    except:
        error_message = "Could not find a grader object for message from xqueue"
        log.exception(error_message)
        return util._error_response(error_message, _INTERFACE_VERSION)

    try:
        submission = Submission.objects.get(id=submission_id)
    except:
        error_message = "Could not find a submission object for message from xqueue"
        log.exception(error_message)
        return util._error_response(error_message, _INTERFACE_VERSION)

    if grade.submission.id != submission.id:
        error_message = "Grader id does not match submission id that was passed in"
        log.exception(error_message)
        return util._error_response(error_message, _INTERFACE_VERSION)

    if originator not in [submission.student_id, grade.grader_id]:
        error_message = "Message originator is not the grader, or the person being graded"
        log.exception(error_message)
        return util._error_response(error_message, _INTERFACE_VERSION)

    if grade.grader_type in ["ML", "IN"]:
        recipient_type = "controller"
        recipient = "controller"
    else:
        recipient_type = "human"

    if recipient_type != 'controller':
        if originator == submission.student_id:
            recipient = grade.grader_id
        elif originator == grade.grader_id:
            recipient = submission.student_id

    if recipient not in [submission.student_id, grade.grader_id, 'controller']:
        error_message = "Message recipient is not the grader, the person being graded, or the controller"
        log.exception(error_message)
        return util._error_response(error_message, _INTERFACE_VERSION)

    if originator == recipient:
        error_message = "Message recipient is the same as originator"
        log.exception(error_message)
        return util._error_response(error_message, _INTERFACE_VERSION)

    message_dict = {
        'grader_id': grader_id,
        'originator': originator,
        'submission_id': submission_id,
        'message': message,
        'recipient': recipient,
        'message_type': "feedback",
        'score': score
    }

    success, error = message_util.create_message(message_dict)

    if not success:
        return util._error_response(error, _INTERFACE_VERSION)

    return util._success_response({'message_id': error}, _INTERFACE_VERSION)

<commit_msg>Fix key error<commit_after>from django.conf import settings
from django.contrib.auth.decorators import login_required
from django.http import HttpResponse
from django.views.decorators.csrf import csrf_exempt
from django.utils import timezone
from datetime import datetime
import logging
import os
import json
from statsd import statsd

from models import Submission, Grader
from models import GraderStatus, SubmissionState
import util
import grader_util
from staff_grading import staff_grading_util
from basic_check import basic_check_util
from metrics import timing_functions
import message_util
from ml_grading import ml_grading_util

log = logging.getLogger(__name__)

_INTERFACE_VERSION = 1

@csrf_exempt
@login_required
@statsd.timed('open_ended_assessment.grading_controller.controller.xqueue_interface.time', tags=['function:submit'])
def submit(request):
    '''
    Xqueue pull script posts objects here.
    Input:

    request - dict with keys xqueue_header and xqueue_body
    xqueue_header needs submission_id,submission_key,queue_name
    xqueue_body needs grader_payload, student_info, student_response, max_score
    grader_payload needs location, course_id,problem_id,grader
    student_info needs anonymous_student_id, submission_time

    Output:
    Returns status code indicating success (0) or failure (1) and message
    '''
    if request.method != 'POST':
        return util._error_response("'submit' must use HTTP POST", _INTERFACE_VERSION)
    else:
        #Minimal parsing of reply
        reply_is_valid, header, body = _is_valid_reply(request.POST.copy())

        if not reply_is_valid:
            log.error("Invalid xqueue object added: request_ip: {0} request.POST: {1}".format(
                util.get_request_ip(request),
                request.POST,
            ))
            statsd.increment("open_ended_assessment.grading_controller.controller.xqueue_interface.submit",
                tags=["success:Exception"])
            return util._error_response('Incorrect format', _INTERFACE_VERSION)
        else:
            try:
                #Retrieve individual values from xqueue body and header.
                prompt = util._value_or_default(body['grader_payload']['prompt'], "")
                rubric = util._value_or_default(body['grader_payload']['rubric'], "")
                student_id = util._value_or_default(body['student_info']['anonymous_student_id'])
                location = util._value_or_default(body['grader_payload']['location'])
                course_id = util._value_or_default(body['grader_payload']['course_id'])
                problem_id = util._value_or_default(body['grader_payload']['problem_id'], location)
                grader_settings = util._value_or_default(body['grader_payload']['grader_settings'], "")
                student_response = util._value_or_default(body['student_response'])
                xqueue_submission_id = util._value_or_default(header['submission_id'])
                xqueue_submission_key = util._value_or_default(header['submission_key'])
                state_code = SubmissionState.waiting_to_be_graded
                xqueue_queue_name = util._value_or_default(header["queue_name"])
                max_score = util._value_or_default(body['max_score'])

                submission_time_string = util._value_or_default(body['student_info']['submission_time'])
                student_submission_time = datetime.strptime(submission_time_string, "%Y%m%d%H%M%S")

                initial_display=""
                if 'initial_display' in body['grader_payload'].keys():
                    initial_display = util._value_or_default(body['grader_payload']['initial_display'], "")
                answer=""
                if 'answer' in body['grader_payload'].keys():
                    answer = util._value_or_default(body['grader_payload']['answer'], "")

                #Create submission object
                sub, created = Submission.objects.get_or_create(
                    prompt=prompt,
                    rubric=rubric,
                    student_id=student_id,
                    problem_id=problem_id,
                    state=state_code,
                    student_response=student_response,
                    student_submission_time=student_submission_time,
                    xqueue_submission_id=xqueue_submission_id,
                    xqueue_submission_key=xqueue_submission_key,
                    xqueue_queue_name=xqueue_queue_name,
                    location=location,
                    course_id=course_id,
                    max_score=max_score,
                    grader_settings=grader_settings,
                    initial_display=initial_display,
                    answer=answer,
                )

            except Exception as err:
                xqueue_submission_id = util._value_or_default(header['submission_id'])
                xqueue_submission_key = util._value_or_default(header['submission_key'])
                log.exception(
                    "Error creating submission and adding to database: sender: {0}, submission_id: {1}, submission_key: {2}".format(
                        util.get_request_ip(request),
                        xqueue_submission_id,
                        xqueue_submission_key,
                    ))

                statsd.increment("open_ended_assessment.grading_controller.controller.xqueue_interface.submit",
                    tags=["success:Exception"])

                return util._error_response('Unable to create submission.', _INTERFACE_VERSION)

            #Handle submission and write to db
            success = handle_submission(sub)
            statsd.increment("open_ended_assessment.grading_controller.controller.xqueue_interface.submit",
                tags=[
                    "success:{0}".format(success),
                    "location:{0}".format(sub.location),
                    "course_id:{0}".format(course_id),
                ])

            if not success:
                return util._error_response("Failed to handle submission.", _INTERFACE_VERSION)

            return util._success_response({'message': "Saved successfully."}, _INTERFACE_VERSION)

def handle_submission(sub):
    """
    Handles a new submission.  Decides what the next grader should be and saves it.
    Input:
        sub - A Submission object from controller.models

    Output:
        True/False status code
    """
    try:
        #Run some basic sanity checks on submission
        sub.next_grader_type = "BC"
        sub.save()
        timing_functions.initialize_timing(sub.id)
        success, check_dict = basic_check_util.simple_quality_check(sub.student_response, sub.initial_display)
        if not success:
            log.exception("could not run basic checks on {0}".format(sub.student_response))

        #add additional tags needed to create a grader object
        check_dict = grader_util.add_additional_tags_to_dict(check_dict, sub.id)

        #Create and handle the grader, and return
        grader_util.create_and_handle_grader_object(check_dict)

        #If the checks result in a score of 0 (out of 1), then the submission fails basic sanity checks
        #Return to student and don't process further
        if check_dict['score'] == 0:
            return True
        else:
            sub.state = SubmissionState.waiting_to_be_graded

        #Assign whether grader should be ML or IN based on number of graded examples.
        subs_graded_by_instructor, subs_pending_instructor = staff_grading_util.count_submissions_graded_and_pending_instructor(
            sub.location)

        #TODO: abstract out logic for assigning which grader to go with.
        grader_settings_path = os.path.join(settings.GRADER_SETTINGS_DIRECTORY, sub.grader_settings)
        grader_settings = grader_util.get_grader_settings(grader_settings_path)
        if grader_settings['grader_type'] == "ML":
            success, model = ml_grading_util.get_latest_created_model(sub.location)
            if(((subs_graded_by_instructor + subs_pending_instructor) >= settings.MIN_TO_USE_ML) and success):
                sub.next_grader_type = "ML"
            else:
                sub.next_grader_type = "IN"
        elif grader_settings['grader_type'] == "PE":
            #Ensures that there will be some calibration essays before peer grading begins!
            #Calibration essays can be added using command line utility, or through normal instructor grading.
            if((subs_graded_by_instructor + subs_pending_instructor) >= settings.MIN_TO_USE_PEER):
                sub.next_grader_type = "PE"
            else:
                sub.next_grader_type = "IN"
        elif grader_settings['grader_type'] == "IN":
            sub.next_grader_type = "IN"
        else:
            log.exception("Invalid grader type specified in settings file.")
            return False

        sub.save()
        log.debug("Submission object created successfully!")

    except:
        log.exception("Submission creation failed!")
        return False

    return True

def _is_valid_reply(external_reply):
    '''
    Check if external reply is in the right format
        1) Presence of 'xqueue_header' and 'xqueue_body'
        2) Presence of specific metadata in 'xqueue_header'
            ['submission_id', 'submission_key']

    Returns:
        is_valid:       Flag indicating success (Boolean)
        header :        header of the queue item
        body:           body of the queue item
    '''
    fail = (False, -1, '')

    success, header, body = _is_valid_reply_generic(external_reply)

    if not success:
        return fail

    for tag in ['grader_payload', 'student_response', 'student_info']:
        if not body.has_key(tag):
            log.debug("{0} not found in body".format(tag))
            return fail

    try:
        body['grader_payload'] = json.loads(body['grader_payload'])
        body['student_info'] = json.loads(body['student_info'])
    except:
        log.debug("Cannot load payload or info.")
        return fail

    return True, header, body

def _is_valid_reply_generic(external_reply):
    try:
        header = json.loads(external_reply['xqueue_header'])
        body = json.loads(external_reply['xqueue_body'])
    except KeyError:
        log.debug("Cannot load header or body.")
        return False, "", ""

    if not isinstance(header, dict) or not isinstance(body, dict):
        return False, "", ""

    for tag in ['submission_id', 'submission_key', 'queue_name']:
        if not header.has_key(tag):
            log.debug("{0} not found in header".format(tag))
            return False, "", ""
    return True, header, body

def _is_valid_reply_message(external_reply):
    fail = (False, -1, '')

    success, header, body = _is_valid_reply_generic(external_reply)

    if not success:
        return fail

    for tag in ['student_info', 'submission_id', 'grader_id', 'feedback']:
        if not body.has_key(tag):
            log.debug("{0} not found in body".format(tag))
            return fail

    body['student_info'] = json.loads(body['student_info'])
    for tag in ['anonymous_student_id']:
        if not body['student_info'].has_key(tag):
            log.debug("{0} not found in student info".format(tag))
            return fail

    return True, header, body

@csrf_exempt
def submit_message(request):
    """
    Submits a message to the grading controller.

    """
    if request.method != 'POST':
        return util._error_response("'submit_message' must use HTTP POST", _INTERFACE_VERSION)

    reply_is_valid, header, body = _is_valid_reply_message(request.POST.copy())

    if not reply_is_valid:
        log.error("Invalid xqueue object added: request_ip: {0} request.POST: {1}".format(
            util.get_request_ip(request),
            request.POST,
        ))
        statsd.increment("open_ended_assessment.grading_controller.controller.xqueue_interface.submit_message",
            tags=["success:Exception"])
        return util._error_response('Incorrect format', _INTERFACE_VERSION)

    message = body['feedback']
    grader_id = body['grader_id']
    submission_id = body['submission_id']
    originator = body['student_info']['anonymous_student_id']

    try:
        if 'score' in body:
            score = int(body['score'])
        else:
            score = None
    except:
        error_message = "Score was not an integer, received \"{0}\" instead.".format(score)
        log.exception(error_message)
        return util._error_response(error_message, _INTERFACE_VERSION)

    try:
        grade = Grader.objects.get(id=grader_id)
    except:
        error_message = "Could not find a grader object for message from xqueue"
        log.exception(error_message)
        return util._error_response(error_message, _INTERFACE_VERSION)

    try:
        submission = Submission.objects.get(id=submission_id)
    except:
        error_message = "Could not find a submission object for message from xqueue"
        log.exception(error_message)
        return util._error_response(error_message, _INTERFACE_VERSION)

    if grade.submission.id != submission.id:
        error_message = "Grader id does not match submission id that was passed in"
        log.exception(error_message)
        return util._error_response(error_message, _INTERFACE_VERSION)

    if originator not in [submission.student_id, grade.grader_id]:
        error_message = "Message originator is not the grader, or the person being graded"
        log.exception(error_message)
        return util._error_response(error_message, _INTERFACE_VERSION)

    if grade.grader_type in ["ML", "IN"]:
        recipient_type = "controller"
        recipient = "controller"
    else:
        recipient_type = "human"

    if recipient_type != 'controller':
        if originator == submission.student_id:
            recipient = grade.grader_id
        elif originator == grade.grader_id:
            recipient = submission.student_id

    if recipient not in [submission.student_id, grade.grader_id, 'controller']:
        error_message = "Message recipient is not the grader, the person being graded, or the controller"
        log.exception(error_message)
        return util._error_response(error_message, _INTERFACE_VERSION)

    if originator == recipient:
        error_message = "Message recipient is the same as originator"
        log.exception(error_message)
        return util._error_response(error_message, _INTERFACE_VERSION)

    message_dict = {
        'grader_id': grader_id,
        'originator': originator,
        'submission_id': submission_id,
        'message': message,
        'recipient': recipient,
        'message_type': "feedback",
        'score': score
    }

    success, error = message_util.create_message(message_dict)

    if not success:
        return util._error_response(error, _INTERFACE_VERSION)

    return util._success_response({'message_id': error}, _INTERFACE_VERSION)

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from django.contrib.auth.models import AnonymousUser
from django.test import RequestFactory
from django.utils.translation import override

from haystack import indexes

from .conf import settings
from .utils import _get_language_from_alias_func

language_from_alias = _get_language_from_alias_func(settings.ALDRYN_SEARCH_LANGUAGE_FROM_ALIAS)

class AbstractIndex(indexes.SearchIndex):
    text = indexes.CharField(document=True, use_template=False)

    def _get_backend(self, using):
        """
        We set the backend to allow easy access for things like document search.
        """
        self._backend = super(AbstractIndex, self)._get_backend(using)
        self._backend_alias = using
        return self._backend

    def index_queryset(self, using=None):
        self._get_backend(using)
        language = self.get_current_language(using)
        filter_kwargs = self.get_index_kwargs(language)
        qs = self.get_index_queryset(language)
        return (qs.filter(**filter_kwargs))

    def get_index_queryset(self, language):
        return self.get_model().objects.all()

    def prepare(self, obj):
        current_language = self.get_current_language(using=self._backend_alias, obj=obj)

        with override(current_language):
            self.prepared_data = super(AbstractIndex, self).prepare(obj)

            request_factory = RequestFactory(HTTP_HOST=settings.ALLOWED_HOSTS[0])
            request = request_factory.get("/")
            request.session = {}
            request.LANGUAGE_CODE = current_language
            # Needed for plugin rendering.
            request.current_page = None
            request.user = AnonymousUser()

            self.prepared_data['text'] = self.get_search_data(obj, current_language, request)
            self.prepare_fields(obj, current_language, request)
            return self.prepared_data

    def get_language(self, obj):
        """
        Equivalent to self.prepare_language.
        """
        return None

    def get_current_language(self, using=None, obj=None):
        """
        Helper method bound to ALWAYS return a language.

        When obj is not None, this calls self.get_language to try and get a language from obj,
        this is useful when the object itself defines it's language in a "language" field.

        If no language was found or obj is None, then we call self.get_default_language to try and get a fallback language.
        """
        language = self.get_language(obj) if obj else None
        return language or self.get_default_language(using)

    def get_default_language(self, using):
        """
        When using multiple languages, this allows us to specify a fallback based on the
        backend being used.
        """
        language = None

        if using and language_from_alias:
            language = language_from_alias(using)
        return language or settings.ALDRYN_SEARCH_DEFAULT_LANGUAGE

    def get_index_kwargs(self, language):
        """
        This is called to filter the index queryset.
        """
        return {}

    def get_model(self):
        raise NotImplementedError()

    def get_search_data(self, obj, language, request):
        """
        Returns a string that will be used to populate the text field (primary field).
        """
        raise NotImplementedError()

    def prepare_fields(self, obj, language, request):
        """
        This is called to prepare any extra fields.
        """
        pass

class AldrynIndexBase(AbstractIndex):
    # For some apps it makes sense to turn on the title indexing.
    INDEX_TITLE = False

    language = indexes.CharField()
    description = indexes.CharField(indexed=False, stored=True, null=True)
    pub_date = indexes.DateTimeField(null=True)
    login_required = indexes.BooleanField(default=False)
    url = indexes.CharField(stored=True, indexed=False)
    title = indexes.CharField(stored=True, indexed=False)
    site_id = indexes.IntegerField(stored=True, indexed=True, null=True)

    def get_url(self, obj):
        """
        Equivalent to self.prepare_url.
        """
        return obj.get_absolute_url()

    def get_title(self, obj):
        """
        Equivalent to self.prepare_title.
        """
        return None

    def get_description(self, obj):
        """
        Equivalent to self.prepare_description.
        """
        return None

    def prepare_fields(self, obj, language, request):
        self.prepared_data['language'] = language
        # We set the following fields here because on some models,
        # the value of these fields is dependent on the active language
        # this being the case we extrapolate the language hacks.
        self.prepared_data['url'] = self.get_url(obj)
        self.prepared_data['title'] = self.get_title(obj)
        self.prepared_data['description'] = self.get_description(obj)

        if self.INDEX_TITLE:
            self.prepared_data['text'] = self.prepared_data['title'] + " " + self.prepared_data['text']
<commit_msg>don't override get_model<commit_after># -*- coding: utf-8 -*-
from django.contrib.auth.models import AnonymousUser
from django.test import RequestFactory
from django.utils.translation import override

from haystack import indexes

from .conf import settings
from .utils import _get_language_from_alias_func

language_from_alias = _get_language_from_alias_func(settings.ALDRYN_SEARCH_LANGUAGE_FROM_ALIAS)

class AbstractIndex(indexes.SearchIndex):
    text = indexes.CharField(document=True, use_template=False)

    def _get_backend(self, using):
        """
        We set the backend to allow easy access for things like document search.
        """
        self._backend = super(AbstractIndex, self)._get_backend(using)
        self._backend_alias = using
        return self._backend

    def index_queryset(self, using=None):
        self._get_backend(using)
        language = self.get_current_language(using)
        filter_kwargs = self.get_index_kwargs(language)
        qs = self.get_index_queryset(language)
        return (qs.filter(**filter_kwargs))

    def get_index_queryset(self, language):
        return self.get_model().objects.all()

    def prepare(self, obj):
        current_language = self.get_current_language(using=self._backend_alias, obj=obj)

        with override(current_language):
            self.prepared_data = super(AbstractIndex, self).prepare(obj)

            request_factory = RequestFactory(HTTP_HOST=settings.ALLOWED_HOSTS[0])
            request = request_factory.get("/")
            request.session = {}
            request.LANGUAGE_CODE = current_language
            # Needed for plugin rendering.
            request.current_page = None
            request.user = AnonymousUser()

            self.prepared_data['text'] = self.get_search_data(obj, current_language, request)
            self.prepare_fields(obj, current_language, request)
            return self.prepared_data

    def get_language(self, obj):
        """
        Equivalent to self.prepare_language.
        """
        return None

    def get_current_language(self, using=None, obj=None):
        """
        Helper method bound to ALWAYS return a language.

        When obj is not None, this calls self.get_language to try and get a language from obj,
        this is useful when the object itself defines it's language in a "language" field.

        If no language was found or obj is None, then we call self.get_default_language to try and get a fallback language.
        """
        language = self.get_language(obj) if obj else None
        return language or self.get_default_language(using)

    def get_default_language(self, using):
        """
        When using multiple languages, this allows us to specify a fallback based on the
        backend being used.
        """
        language = None

        if using and language_from_alias:
            language = language_from_alias(using)
        return language or settings.ALDRYN_SEARCH_DEFAULT_LANGUAGE

    def get_index_kwargs(self, language):
        """
        This is called to filter the index queryset.
        """
        return {}

    def get_search_data(self, obj, language, request):
        """
        Returns a string that will be used to populate the text field (primary field).
        """
        raise NotImplementedError()

    def prepare_fields(self, obj, language, request):
        """
        This is called to prepare any extra fields.
        """
        pass

class AldrynIndexBase(AbstractIndex):
    # For some apps it makes sense to turn on the title indexing.
    INDEX_TITLE = False

    language = indexes.CharField()
    description = indexes.CharField(indexed=False, stored=True, null=True)
    pub_date = indexes.DateTimeField(null=True)
    login_required = indexes.BooleanField(default=False)
    url = indexes.CharField(stored=True, indexed=False)
    title = indexes.CharField(stored=True, indexed=False)
    site_id = indexes.IntegerField(stored=True, indexed=True, null=True)

    def get_url(self, obj):
        """
        Equivalent to self.prepare_url.
        """
        return obj.get_absolute_url()

    def get_title(self, obj):
        """
        Equivalent to self.prepare_title.
        """
        return None

    def get_description(self, obj):
        """
        Equivalent to self.prepare_description.
        """
        return None

    def prepare_fields(self, obj, language, request):
        self.prepared_data['language'] = language
        # We set the following fields here because on some models,
        # the value of these fields is dependent on the active language
        # this being the case we extrapolate the language hacks.
        self.prepared_data['url'] = self.get_url(obj)
        self.prepared_data['title'] = self.get_title(obj)
        self.prepared_data['description'] = self.get_description(obj)

        if self.INDEX_TITLE:
            self.prepared_data['text'] = self.prepared_data['title'] + " " + self.prepared_data['text']
<|endoftext|>


<commit_before>from joblib import Parallel, delayed

import concept_dependency_graph as cdg
from simple_mdp import create_custom_dependency
from helpers import expected_reward, compute_optimal_actions

import student as st
from drqn import *

def construct_drqn_inputs(act_hist, ob_hist, n_concepts):
    n_timesteps = len(act_hist)
    inputs = np.zeros((1, n_timesteps, 2 * n_concepts))
    for t in xrange(n_timesteps):
        action, ob = act_hist[t], ob_hist[t]
        exer = np.zeros(n_concepts)
        exer[action] = 1
        if ob == 1:
            inputs[0, t, :n_concepts] = exer
        else:
            inputs[0, t, n_concepts:] = exer
    return inputs

def test_drqn_single(dgraph, student, horizon, model, DEBUG=False):
    '''
    Performs a single trajectory with MCTS and returns the final true student knowledge.
    '''
    n_concepts = dgraph.n

    # create the model and simulators
    student.reset()
    student.knowledge[0] = 1  # initialize the first concept to be known
    sim = st.StudentExactSim(student, dgraph)

    # initialize state (or alternatively choose random first action)
    act_hist = [0]
    ob_hist = [0]
    for i in range(horizon - 1):
        # print('Step {}'.format(i))
        inputs = construct_drqn_inputs(act_hist, ob_hist, n_concepts)
        best_action, _ = model.predict(inputs, last_timestep_only=True)
        best_action = best_action[0]
        concept = best_action
        conceptvec = np.zeros(n_concepts)
        conceptvec[concept] = 1
        action = st.StudentAction(concept, conceptvec)
        # print(best_action.concept)

        # debug check for whether action is optimal
        if DEBUG:
            opt_acts = compute_optimal_actions(sim.dgraph, sim.student.knowledge) # put function code into shared file
            is_opt = action.concept in opt_acts
            if not is_opt:
                print('ERROR {} executed non-optimal action {}'.format(sim.student.knowledge,
                                                                       action.concept))

        # act in the real environment
        (ob, reward) = sim.advance_simulator(action)
        act_hist.append(action.concept)
        ob_hist.append(ob)

        # print('Next state: {}'.format(str(new_root.state)))
    return sim.student.knowledge

def test_drqn_chunk(n_trajectories, dgraph, student, model_id, horizon):
    '''
    Runs a bunch of trajectories and returns the avg posttest score.
    For parallelization to run in a separate thread/process.
    '''

    model = DRQNModel(model_id=model_id, timesteps=horizon)
    model.init_evaluator()

    acc = 0.0
    for i in xrange(n_trajectories):
        print('traj i {}'.format(i))
        k = test_drqn_single(dgraph, student, horizon, model)
        acc += np.mean(k)

    acc /= n_trajectories
    return acc

def test_drqn(model_id=""):
    '''
    Test DRQN
    '''
    n_concepts = 4
    learn_prob = 0.15
    horizon = 6
    n_trajectories = 100
    n_jobs = 8
    traj_per_job = n_trajectories // n_jobs

    from simple_mdp import create_custom_dependency
    # dgraph = create_custom_dependency()

    dgraph = cdg.ConceptDependencyGraph()
    dgraph.init_default_tree(n_concepts)

    # student = st.Student(n=n_concepts, p_trans_satisfied=learn_prob, p_trans_not_satisfied=0.0, p_get_ex_correct_if_concepts_learned=1.0)
    student = st.Student2(n_concepts)
    if model_id == "":
        model_id = "test_model_drqn"

    print('Testing model: {}'.format(model_id))
    print('horizon: {}'.format(horizon))

    # accs = Parallel(n_jobs=n_jobs)(delayed(test_drqn_chunk)(traj_per_job, dgraph, student, model_id, horizon) for _ in range(n_jobs))
    # avg = sum(accs) / (n_jobs)
    avg = test_drqn_chunk(n_trajectories, dgraph, student, model_id, horizon)

    test_data = dg.generate_data(dgraph, student=student, n_students=1000, seqlen=horizon, policy='expert', filename=None, verbose=False)
    print('Average posttest true: {}'.format(expected_reward(test_data)))
    print('Average posttest drqn: {}'.format(avg))
<commit_msg>added parallel threading option<commit_after>from joblib import Parallel, delayed

import concept_dependency_graph as cdg
from simple_mdp import create_custom_dependency
from helpers import expected_reward, compute_optimal_actions

import student as st
from drqn import *

def construct_drqn_inputs(act_hist, ob_hist, n_concepts):
    n_timesteps = len(act_hist)
    inputs = np.zeros((1, n_timesteps, 2 * n_concepts))
    for t in xrange(n_timesteps):
        action, ob = act_hist[t], ob_hist[t]
        exer = np.zeros(n_concepts)
        exer[action] = 1
        if ob == 1:
            inputs[0, t, :n_concepts] = exer
        else:
            inputs[0, t, n_concepts:] = exer
    return inputs

def test_drqn_single(dgraph, student, horizon, model, DEBUG=False):
    '''
    Performs a single trajectory with MCTS and returns the final true student knowledge.
    '''
    n_concepts = dgraph.n

    # create the model and simulators
    student.reset()
    student.knowledge[0] = 1  # initialize the first concept to be known
    sim = st.StudentExactSim(student, dgraph)

    # initialize state (or alternatively choose random first action)
    act_hist = [0]
    ob_hist = [0]
    for i in range(horizon - 1):
        # print('Step {}'.format(i))
        inputs = construct_drqn_inputs(act_hist, ob_hist, n_concepts)
        best_action, _ = model.predict(inputs, last_timestep_only=True)
        best_action = best_action[0]
        concept = best_action
        conceptvec = np.zeros(n_concepts)
        conceptvec[concept] = 1
        action = st.StudentAction(concept, conceptvec)
        # print(best_action.concept)

        # debug check for whether action is optimal
        if DEBUG:
            opt_acts = compute_optimal_actions(sim.dgraph, sim.student.knowledge) # put function code into shared file
            is_opt = action.concept in opt_acts
            if not is_opt:
                print('ERROR {} executed non-optimal action {}'.format(sim.student.knowledge,
                                                                       action.concept))

        # act in the real environment
        (ob, reward) = sim.advance_simulator(action)
        act_hist.append(action.concept)
        ob_hist.append(ob)

        # print('Next state: {}'.format(str(new_root.state)))
    return sim.student.knowledge

def test_drqn_chunk(n_trajectories, dgraph, student, model_id, horizon):
    '''
    Runs a bunch of trajectories and returns the avg posttest score.
    For parallelization to run in a separate thread/process.
    '''

    model = DRQNModel(model_id=model_id, timesteps=horizon)
    model.init_evaluator()

    acc = 0.0
    for i in xrange(n_trajectories):
        print('traj i {}'.format(i))
        k = test_drqn_single(dgraph, student, horizon, model)
        acc += np.mean(k)

    acc /= n_trajectories
    return acc

def test_drqn(model_id="", parallel=False):
    '''
    Test DRQN
    '''
    n_concepts = 4
    learn_prob = 0.15
    horizon = 6
    n_trajectories = 100
    n_jobs = 8
    traj_per_job = n_trajectories // n_jobs

    from simple_mdp import create_custom_dependency
    # dgraph = create_custom_dependency()

    dgraph = cdg.ConceptDependencyGraph()
    dgraph.init_default_tree(n_concepts)

    # student = st.Student(n=n_concepts, p_trans_satisfied=learn_prob, p_trans_not_satisfied=0.0, p_get_ex_correct_if_concepts_learned=1.0)
    student = st.Student2(n_concepts)
    if model_id == "":
        model_id = "test_model_drqn"

    print('Testing model: {}'.format(model_id))
    print('horizon: {}'.format(horizon))

    if parallel:
        accs = Parallel(n_jobs=n_jobs)(delayed(test_drqn_chunk)(traj_per_job, dgraph, student, model_id, horizon) for _ in range(n_jobs))
        avg = sum(accs) / (n_jobs)
    else:
        avg = test_drqn_chunk(n_trajectories, dgraph, student, model_id, horizon)

    test_data = dg.generate_data(dgraph, student=student, n_students=1000, seqlen=horizon, policy='expert', filename=None, verbose=False)
    print('Average posttest true: {}'.format(expected_reward(test_data)))
    print('Average posttest drqn: {}'.format(avg))
<|endoftext|>