<commit_before>import os.path

from astropy import units as u
from astropy.time import Time

from collections import OrderedDict

from ..utils.logger import get_logger
from ..utils.config import load_config
from ..utils import error
from ..utils import images

class Observation(object):

    def __init__(self, obs_config=dict(), cameras=None):
        """An object which describes a single observation.

        Each observation can have a number of different `Exposure`s based on the config settings.
        For each type of exposure ('primary' or 'secondary') there are `[type]_nexp` `Exposure`
            objects created. Each of these `Exposure`s has a list of cameras. (See `Exposure` for details)

        Example::

              - analyze: false
                primary_exptime: 300
                primary_filter: null
                primary_nexp: 3
                secondary_exptime: 300
                secondary_filter: null
                secondary_nexp: 3

        Args:
            obs_config (dictionary): a dictionary describing the observation as read from
                the YAML file, see Example.
            cameras(list[panoptes.camera]): A list of `panoptes.camera` objects to use for
                this observation.

        """
        self.config = load_config()
        self.logger = get_logger(self)

        self.cameras = cameras

        self.logger.debug("Camears for observation: {}".format(cameras))
        self.exposures = self._create_exposures(obs_config)

        self._images_exist = False
        self._done_exposing = False

        self.reset_exposures()

##################################################################################################
# Properties
##################################################################################################

    @property
    def is_exposing(self):
        return self._is_exposing

    @property
    def done_exposing(self):
        """ Bool indicating whether or not any exposures are left """
        self.logger.debug("Checking if observation has exposures")

        return self._done_exposing

    @property
    def complete(self):
        return self.done_exposing

##################################################################################################
# Methods
##################################################################################################

    def get_exposure_iter(self):
        """ Yields the next exposure """

        for num, exposure in enumerate(self.exposures):
            self.logger.debug("Getting next exposure ({})".format(exposure))

            if num == len(self.exposures) - 1:
                self._done_exposing = True

            self.current_exposure = exposure
            yield exposure

    def reset_exposures(self):
        """ Resets the exposures iterator """
        self.exposure_iterator = self.get_exposure_iter()
        self._done_exposing = False
        self.current_exposure = None
        self._is_exposing = False

    def take_exposure(self):
        """ Take the next exposure """
        try:
            exposure = next(self.exposure_iterator)
            # One start_time for this round of exposures
            start_time = Time.now().isot

            img_files = []

            # Take a picture with each camera
            for cam_name, cam in self.cameras.items():
                self.logger.debug("Exposing for camera: {}".format(cam_name))
                # Start exposure
                img_file = cam.take_exposure(seconds=exposure.exptime)
                self._is_exposing = True

                obs_info = {
                    'camera_id': cam.uid,
                    'img_file': img_file,
                    'filter': exposure.filter_type,
                    'start_time': start_time,
                    'guide_image': cam.is_guide,
                }
                self.logger.debug("{}".format(obs_info))
                exposure.images[cam_name] = obs_info
                img_files.append(img_file)

        except error.InvalidCommand as e:
            self.logger.warning("{} is already running a command.".format(cam.name))
            self._is_exposing = False
        except Exception as e:
            self.logger.warning("Can't take exposure from Observation: {}".format(e))
            self._is_exposing = False
        finally:
            return img_files

    def estimate_duration(self, overhead=0 * u.s):
        """Method to estimate the duration of a single observation.

        A quick and dirty estimation of the time it takes to execute the
        observation.  Does not take overheads such as slewing, image readout,
        or image download in to consideration.

        Args:
            overhead (astropy.units.Quantity): The overhead time for the observation in
            units which are reducible to seconds.  This is the overhead which occurs
            for each exposure.

        Returns:
            astropy.units.Quantity: The duration (with units of seconds).
        """
        duration = max([(self.primary_exptime + overhead) * self.primary_nexp,
                        (self.secondary_exptime + overhead) * self.secondary_nexp])
        self.logger.debug('Observation duration estimated as {}'.format(duration))
        return duration

##################################################################################################
# Private Methods
##################################################################################################

    def _create_exposures(self, obs_config):
        self.logger.debug("Creating exposures")

        primary_exptime = obs_config.get('primary_exptime', 120) * u.s
        primary_filter = obs_config.get('primary_filter', None)
        primary_nexp = obs_config.get('primary_nexp', 30)
        # analyze = obs_config.get('primary_analyze', False)

        primary_exposures = [self.Exposure(
            exptime=primary_exptime,
            filter_type=primary_filter,
        ) for n in range(primary_nexp)]
        self.logger.debug("Primary exposures: {}".format(primary_exposures))
        self.num_exposures = primary_nexp

        # secondary_exptime (assumes units of seconds, defaults to 120 seconds)
        # secondary_exptime = obs_config.get('secondary_exptime', 120) * u.s
        # secondary_nexp = obs_config.get('secondary_nexp', 0)
        # secondary_filter = obs_config.get('secondary_filter', None)

        # secondary_exposures = [Exposure(
        #     exptime=secondary_exptime,
        #     filter_type=secondary_filter,
        #     analyze=False,
        #     cameras=[c for c in cameras.values() if not c.is_primary],
        # ) for n in range(secondary_nexp)]

        # if secondary_nexp > primary_nexp:
        #     self.num_exposures = secondary_nexp

        # self.logger.debug("Secondary exposures: {}".format(secondary_exposures))

        return primary_exposures

##################################################################################################
# Private Class
##################################################################################################

    class Exposure(object):

        """ An individual exposure taken by an `Observation` """

        def __init__(self, exptime=120, filter_type=None):
            self.logger = get_logger(self)

            self.exptime = exptime
            self.filter_type = filter_type
            self.images = OrderedDict()
            self._images_exist = False

        @property
        def has_images(self):
            return len(self.images) > 0

        @property
        def images_exist(self):
            """ Whether or not the images indicated by `self.images` exists.

            The `images` attribute is set when the exposure starts, so this is
            effectively a test for if the exposure has ended correctly.
            """
            self._images_exist = all(os.path.exists(f) for f in self.get_images())

            if self._images_exist:
                self._is_exposing = False

            return self._images_exist

        def get_images(self):
            """ Get all the images for this exposure """
            return [f.get('img_file') for f in list(self.images.values())]

        def get_guide_image_info(self):
            """ Gets the most recent image from the camera marked as `guide` """
            for cam_name, img_info in self.images.items():
                if self.cameras[cam_name].is_guide:
                    return img_info

        def process_images(self, fits_headers={}, **kwargs):
            """ Process the raw data images

            Args:
                fits_headers{dict, optional}:   Key/value headers for the fits file.
            """
            assert self.images_exist, self.logger.warning("No images to process")
            start_time = Time.now()

            self.logger.debug("Processing images: {}".format(self.images))

            for cam_name, img_info in self.images.items():
                self.logger.debug("Cam {} Info {}".format(cam_name, img_info))

                fits_headers = {
                    'detname': img_info.get('camera_id', ''),
                }
                kwargs['primary'] = self.cameras[cam_name].is_primary

                processsed_info = images.process_cr2(img_info.get('img_file'), fits_headers=fits_headers, **kwargs)

                self.logger.debug("Processed image info: {}".format(processsed_info))

                img_info.update(processsed_info)
                self.logger.debug("Done processing")

            # End total processing time
            end_time = Time.now()
            self.logger.debug("Processing time: {}".format((end_time - start_time).to(u.s)))
<commit_msg>Fixing `Exposure` so it knows about primary and guide images<commit_after>import os.path

from astropy import units as u
from astropy.time import Time

from collections import OrderedDict

from ..utils.logger import get_logger
from ..utils.config import load_config
from ..utils import error
from ..utils import images

class Observation(object):

    def __init__(self, obs_config=dict(), cameras=None):
        """An object which describes a single observation.

        Each observation can have a number of different `Exposure`s based on the config settings.
        For each type of exposure ('primary' or 'secondary') there are `[type]_nexp` `Exposure`
            objects created. Each of these `Exposure`s has a list of cameras. (See `Exposure` for details)

        Example::

              - analyze: false
                primary_exptime: 300
                primary_filter: null
                primary_nexp: 3
                secondary_exptime: 300
                secondary_filter: null
                secondary_nexp: 3

        Args:
            obs_config (dictionary): a dictionary describing the observation as read from
                the YAML file, see Example.
            cameras(list[panoptes.camera]): A list of `panoptes.camera` objects to use for
                this observation.

        """
        self.config = load_config()
        self.logger = get_logger(self)

        self.cameras = cameras

        self.logger.debug("Camears for observation: {}".format(cameras))
        self.exposures = self._create_exposures(obs_config)

        self._images_exist = False
        self._done_exposing = False

        self.reset_exposures()

##################################################################################################
# Properties
##################################################################################################

    @property
    def is_exposing(self):
        return self._is_exposing

    @property
    def done_exposing(self):
        """ Bool indicating whether or not any exposures are left """
        self.logger.debug("Checking if observation has exposures")

        return self._done_exposing

    @property
    def complete(self):
        return self.done_exposing

##################################################################################################
# Methods
##################################################################################################

    def get_exposure_iter(self):
        """ Yields the next exposure """

        for num, exposure in enumerate(self.exposures):
            self.logger.debug("Getting next exposure ({})".format(exposure))

            if num == len(self.exposures) - 1:
                self._done_exposing = True

            self.current_exposure = exposure
            yield exposure

    def reset_exposures(self):
        """ Resets the exposures iterator """
        self.exposure_iterator = self.get_exposure_iter()
        self._done_exposing = False
        self.current_exposure = None
        self._is_exposing = False

    def take_exposure(self):
        """ Take the next exposure """
        try:
            exposure = next(self.exposure_iterator)
            # One start_time for this round of exposures
            start_time = Time.now().isot

            img_files = []

            # Take a picture with each camera
            for cam_name, cam in self.cameras.items():
                self.logger.debug("Exposing for camera: {}".format(cam_name))
                # Start exposure
                img_file = cam.take_exposure(seconds=exposure.exptime)
                self._is_exposing = True

                obs_info = {
                    'camera_id': cam.uid,
                    'img_file': img_file,
                    'filter': exposure.filter_type,
                    'start_time': start_time,
                    'guide_image': cam.is_guide,
                    'primary': cam.is_primary,
                }
                self.logger.debug("{}".format(obs_info))
                exposure.images[cam_name] = obs_info
                img_files.append(img_file)

        except error.InvalidCommand as e:
            self.logger.warning("{} is already running a command.".format(cam.name))
            self._is_exposing = False
        except Exception as e:
            self.logger.warning("Can't take exposure from Observation: {}".format(e))
            self._is_exposing = False
        finally:
            return img_files

    def estimate_duration(self, overhead=0 * u.s):
        """Method to estimate the duration of a single observation.

        A quick and dirty estimation of the time it takes to execute the
        observation.  Does not take overheads such as slewing, image readout,
        or image download in to consideration.

        Args:
            overhead (astropy.units.Quantity): The overhead time for the observation in
            units which are reducible to seconds.  This is the overhead which occurs
            for each exposure.

        Returns:
            astropy.units.Quantity: The duration (with units of seconds).
        """
        duration = max([(self.primary_exptime + overhead) * self.primary_nexp,
                        (self.secondary_exptime + overhead) * self.secondary_nexp])
        self.logger.debug('Observation duration estimated as {}'.format(duration))
        return duration

##################################################################################################
# Private Methods
##################################################################################################

    def _create_exposures(self, obs_config):
        self.logger.debug("Creating exposures")

        primary_exptime = obs_config.get('primary_exptime', 120) * u.s
        primary_filter = obs_config.get('primary_filter', None)
        primary_nexp = obs_config.get('primary_nexp', 30)
        # analyze = obs_config.get('primary_analyze', False)

        primary_exposures = [self.Exposure(
            exptime=primary_exptime,
            filter_type=primary_filter,
        ) for n in range(primary_nexp)]
        self.logger.debug("Primary exposures: {}".format(primary_exposures))
        self.num_exposures = primary_nexp

        # secondary_exptime (assumes units of seconds, defaults to 120 seconds)
        # secondary_exptime = obs_config.get('secondary_exptime', 120) * u.s
        # secondary_nexp = obs_config.get('secondary_nexp', 0)
        # secondary_filter = obs_config.get('secondary_filter', None)

        # secondary_exposures = [Exposure(
        #     exptime=secondary_exptime,
        #     filter_type=secondary_filter,
        #     analyze=False,
        #     cameras=[c for c in cameras.values() if not c.is_primary],
        # ) for n in range(secondary_nexp)]

        # if secondary_nexp > primary_nexp:
        #     self.num_exposures = secondary_nexp

        # self.logger.debug("Secondary exposures: {}".format(secondary_exposures))

        return primary_exposures

##################################################################################################
# Private Class
##################################################################################################

    class Exposure(object):

        """ An individual exposure taken by an `Observation` """

        def __init__(self, exptime=120, filter_type=None):
            self.logger = get_logger(self)

            self.exptime = exptime
            self.filter_type = filter_type
            self.images = OrderedDict()
            self._images_exist = False

        @property
        def has_images(self):
            return len(self.images) > 0

        @property
        def images_exist(self):
            """ Whether or not the images indicated by `self.images` exists.

            The `images` attribute is set when the exposure starts, so this is
            effectively a test for if the exposure has ended correctly.
            """
            self._images_exist = all(os.path.exists(f) for f in self.get_images())

            if self._images_exist:
                self._is_exposing = False

            return self._images_exist

        def get_images(self):
            """ Get all the images for this exposure """
            return [f.get('img_file') for f in list(self.images.values())]

        def get_guide_image_info(self):
            """ Gets the most recent image from the camera marked as `guide` """
            for cam_name, img_info in self.images.items():
                if self.cameras[cam_name].is_guide:
                    return img_info

        def process_images(self, fits_headers={}, **kwargs):
            """ Process the raw data images

            Args:
                fits_headers{dict, optional}:   Key/value headers for the fits file.
            """
            assert self.images_exist, self.logger.warning("No images to process")
            start_time = Time.now()

            self.logger.debug("Processing images: {}".format(self.images))

            for cam_name, img_info in self.images.items():
                self.logger.debug("Cam {} Info {}".format(cam_name, img_info))

                fits_headers = {
                    'detname': img_info.get('camera_id', ''),
                }
                kwargs['primary'] = img_info.get('primary', False)

                processsed_info = images.process_cr2(img_info.get('img_file'), fits_headers=fits_headers, **kwargs)

                self.logger.debug("Processed image info: {}".format(processsed_info))

                img_info.update(processsed_info)
                self.logger.debug("Done processing")

            # End total processing time
            end_time = Time.now()
            self.logger.debug("Processing time: {}".format((end_time - start_time).to(u.s)))
<|endoftext|>


<commit_before>#!/usr/bin/python2.5
#
# Copyright 2008 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Views for Programs.
"""

__authors__ = [
    '"Sverre Rabbelier" <sverre@rabbelier.nl>',
    '"Lennard de Rijk" <ljvderijk@gmail.com>',
  ]

from django import forms

from soc.logic import dicts
from soc.logic.models import program as program_logic
from soc.views import helper
from soc.views import out_of_band
from soc.views.helper import access
from soc.views.helper import decorators
from soc.views.helper import redirects
from soc.views.helper import widgets
from soc.views.models import presence_with_tos
from soc.views.models import document as document_view
from soc.views.models import sponsor as sponsor_view
from soc.views.sitemap import sidebar

import soc.logic.models.program

class View(presence_with_tos.View):
  """View methods for the Program model.
  """

  def __init__(self, params=None):
    """Defines the fields and methods required for the base View class
    to provide the user with list, public, create, edit and delete views.

    Params:
      params: a dict with params for this View
    """

    rights = access.Checker(params)
    rights['any_access'] = ['allow']
    rights['show'] = ['allow']
    rights['create'] = ['checkIsHost']
    rights['edit'] = ['checkIsHostForProgram']
    rights['delete'] = ['checkIsDeveloper']

    new_params = {}
    new_params['logic'] = soc.logic.models.program.logic
    new_params['rights'] = rights

    new_params['scope_view'] = sponsor_view
    new_params['scope_redirect'] = redirects.getCreateRedirect

    new_params['name'] = "Program"
    new_params['sidebar_grouping'] = 'Programs'

    new_params['extra_dynaexclude'] = ['timeline']

    # TODO add clean field to check for uniqueness in link_id and scope_path
    new_params['create_extra_dynafields'] = {
        'description': forms.fields.CharField(widget=helper.widgets.TinyMCE(
            attrs={'rows':10, 'cols':40})),

        'scope_path': forms.CharField(widget=forms.HiddenInput, required=True),

        'workflow': forms.ChoiceField(choices=[('gsoc','Project-based'),
            ('ghop','Task-based')], required=True),
        }

    new_params['edit_extra_dynafields'] = {
        'workflow': forms.CharField(widget=widgets.ReadOnlyInput(),
                                   required=True),
        }

    params = dicts.merge(params, new_params)

    super(View, self).__init__(params=params)

  def _editPost(self, request, entity, fields):
    """See base._editPost().
    """

    if not entity:
      # there is no existing entity so create a new timeline
      fields['timeline'] = self._createTimelineForType(fields)
    else:
      # use the timeline from the entity
      fields['timeline'] = entity.timeline

    super(View, self)._editPost(request, entity, fields)

  def _createTimelineForType(self, fields):
    """Creates and stores a timeline model for the given type of program.
    """

    workflow = fields['workflow']

    timeline_logic = program_logic.logic.TIMELINE_LOGIC[workflow]

    key_fields = self._logic.getKeyFieldsFromFields(fields)
    key_name = self._logic.getKeyNameFromFields(key_fields)

    properties = {'scope_path': key_name}

    timeline = timeline_logic.updateOrCreateFromFields(properties, properties)
    return timeline

  @decorators.merge_params
  def getExtraMenus(self, id, user, params=None):
    """Returns the extra menu's for this view.

    A menu item is generated for each program that is currently
    running. The public page for each program is added as menu item,
    as well as all public documents for that program.

    Args:
      params: a dict with params for this View.
    """

    logic = params['logic']
    rights = params['rights']

    # only get all invisible and visible programs
    fields = {'status':['invisible', 'visible']}
    entities = logic.getForFields(fields)

    #TODO(ljvderijk) Add timeline dependent entries

    menus = []

    rights.setCurrentUser(id, user)
    filter_args = {}

    for entity in entities:
      items = []

      if entity.status == 'visible':
        # show the documents for this program, even for not logged in users
        items += document_view.view.getMenusForScope(entity, params)

      try:
        # check if the current user is a host for this program
        rights.doCachedCheck('checkIsHostForProgram',
            {'scope_path': entity.scope_path,
            'link_id': entity.link_id,
            'workflow': entity.workflow}, [])

        if entity.status == 'invisible':
          # still add the document links so hosts can see how it looks like
          items += document_view.view.getMenusForScope(entity, params)

        # add link to edit Program Profile
        items += [(redirects.getEditRedirect(entity, params),
            'Edit Program Profile','any_access')]
        # add link to edit Program Timeline
        items += [(redirects.getEditRedirect(entity, {'url_name': 'timeline'}),
            "Edit Program Timeline", 'any_access')]
        # add link to create a new Program Document
        items += [(redirects.getCreateDocumentRedirect(entity, 'program'),
            "Create a New Document", 'any_access')]

      except out_of_band.Error:
        pass

      menu = {}
      menu['heading'] = entity.short_name
      menu['items'] = sidebar.getSidebarMenu(id, user, items, params=params)
      menu['group'] = 'Programs'
      menus.append(menu)

    return menus

view = View()

create = view.create
delete = view.delete
edit = view.edit
list = view.list
public = view.public
export = view.export
home = view.home
pick = view.pick
<commit_msg>Do not return empty menu's in program.<commit_after>#!/usr/bin/python2.5
#
# Copyright 2008 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Views for Programs.
"""

__authors__ = [
    '"Sverre Rabbelier" <sverre@rabbelier.nl>',
    '"Lennard de Rijk" <ljvderijk@gmail.com>',
  ]

from django import forms

from soc.logic import dicts
from soc.logic.models import program as program_logic
from soc.views import helper
from soc.views import out_of_band
from soc.views.helper import access
from soc.views.helper import decorators
from soc.views.helper import redirects
from soc.views.helper import widgets
from soc.views.models import presence_with_tos
from soc.views.models import document as document_view
from soc.views.models import sponsor as sponsor_view
from soc.views.sitemap import sidebar

import soc.logic.models.program

class View(presence_with_tos.View):
  """View methods for the Program model.
  """

  def __init__(self, params=None):
    """Defines the fields and methods required for the base View class
    to provide the user with list, public, create, edit and delete views.

    Params:
      params: a dict with params for this View
    """

    rights = access.Checker(params)
    rights['any_access'] = ['allow']
    rights['show'] = ['allow']
    rights['create'] = ['checkIsHost']
    rights['edit'] = ['checkIsHostForProgram']
    rights['delete'] = ['checkIsDeveloper']

    new_params = {}
    new_params['logic'] = soc.logic.models.program.logic
    new_params['rights'] = rights

    new_params['scope_view'] = sponsor_view
    new_params['scope_redirect'] = redirects.getCreateRedirect

    new_params['name'] = "Program"
    new_params['sidebar_grouping'] = 'Programs'

    new_params['extra_dynaexclude'] = ['timeline']

    # TODO add clean field to check for uniqueness in link_id and scope_path
    new_params['create_extra_dynafields'] = {
        'description': forms.fields.CharField(widget=helper.widgets.TinyMCE(
            attrs={'rows':10, 'cols':40})),

        'scope_path': forms.CharField(widget=forms.HiddenInput, required=True),

        'workflow': forms.ChoiceField(choices=[('gsoc','Project-based'),
            ('ghop','Task-based')], required=True),
        }

    new_params['edit_extra_dynafields'] = {
        'workflow': forms.CharField(widget=widgets.ReadOnlyInput(),
                                   required=True),
        }

    params = dicts.merge(params, new_params)

    super(View, self).__init__(params=params)

  def _editPost(self, request, entity, fields):
    """See base._editPost().
    """

    if not entity:
      # there is no existing entity so create a new timeline
      fields['timeline'] = self._createTimelineForType(fields)
    else:
      # use the timeline from the entity
      fields['timeline'] = entity.timeline

    super(View, self)._editPost(request, entity, fields)

  def _createTimelineForType(self, fields):
    """Creates and stores a timeline model for the given type of program.
    """

    workflow = fields['workflow']

    timeline_logic = program_logic.logic.TIMELINE_LOGIC[workflow]

    key_fields = self._logic.getKeyFieldsFromFields(fields)
    key_name = self._logic.getKeyNameFromFields(key_fields)

    properties = {'scope_path': key_name}

    timeline = timeline_logic.updateOrCreateFromFields(properties, properties)
    return timeline

  @decorators.merge_params
  def getExtraMenus(self, id, user, params=None):
    """Returns the extra menu's for this view.

    A menu item is generated for each program that is currently
    running. The public page for each program is added as menu item,
    as well as all public documents for that program.

    Args:
      params: a dict with params for this View.
    """

    logic = params['logic']
    rights = params['rights']

    # only get all invisible and visible programs
    fields = {'status':['invisible', 'visible']}
    entities = logic.getForFields(fields)

    #TODO(ljvderijk) Add timeline dependent entries

    menus = []

    rights.setCurrentUser(id, user)
    filter_args = {}

    for entity in entities:
      items = []

      if entity.status == 'visible':
        # show the documents for this program, even for not logged in users
        items += document_view.view.getMenusForScope(entity, params)

      try:
        # check if the current user is a host for this program
        rights.doCachedCheck('checkIsHostForProgram',
            {'scope_path': entity.scope_path,
            'link_id': entity.link_id,
            'workflow': entity.workflow}, [])

        if entity.status == 'invisible':
          # still add the document links so hosts can see how it looks like
          items += document_view.view.getMenusForScope(entity, params)

        # add link to edit Program Profile
        items += [(redirects.getEditRedirect(entity, params),
            'Edit Program Profile','any_access')]
        # add link to edit Program Timeline
        items += [(redirects.getEditRedirect(entity, {'url_name': 'timeline'}),
            "Edit Program Timeline", 'any_access')]
        # add link to create a new Program Document
        items += [(redirects.getCreateDocumentRedirect(entity, 'program'),
            "Create a New Document", 'any_access')]

      except out_of_band.Error:
        pass

      items = sidebar.getSidebarMenu(id, user, items, params=params)
      if not items:
        continue

      menu = {}
      menu['heading'] = entity.short_name
      menu['items'] = items
      menu['group'] = 'Programs'
      menus.append(menu)

    return menus

view = View()

create = view.create
delete = view.delete
edit = view.edit
list = view.list
public = view.public
export = view.export
home = view.home
pick = view.pick
<|endoftext|>


<commit_before>#!/usr/bin/env python3

# Copyright (c) 2017-present, Facebook, Inc.
# All rights reserved.
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree. An additional grant
# of patent rights can be found in the PATENTS file in the same directory.

import os

import torch
from torch import nn

from parlai.core.torch_agent import TorchAgent, Output
from parlai.core.thread_utils import SharedTable
from parlai.core.utils import round_sigfigs, padded_3d

class TorchRankerAgent(TorchAgent):
    @staticmethod
    def add_cmdline_args(argparser):
        TorchAgent.add_cmdline_args(argparser)
        agent = argparser.add_argument_group('TorchRankerAgent')
        agent.add_argument(
            '-tc', '--train-candidates', type=str, default='batch',
            choices=['batch', 'inline', 'fixed', 'vocab'],
            help='The source of candidates during training')
        agent.add_argument(
            '-ec', '--eval-candidates', type=str, default='batch',
            choices=['batch', 'inline', 'fixed', 'vocab'],
            help='The source of candidates during training')
        agent.add_argument(
            '-cands', '--fixed-candidates-path', type=str,
            help='A text file of fixed candidates to use for all examples, one '
                 'candidate per line')
        agent.add_argument(
            '-candvecs', '--fixed-candidate-vecs-path', type=str,
            help='A torch file containing vectorized fixed candidates to use for all '
                 'examples. If a file already exists at -fixed-candidate-vecs-path, '
                 'those vectors will be loaded. If not, they will be computed and '
                 'written to that path')

    def __init__(self, opt, shared):
        if shared:
            super().__init__(opt, shared)
            self.model = shared['model']
            self.metrics = shared['metrics']
        else:
            # Must call _get_model_file first so that paths are updated if
            # necessary (e.g., .dict file)
            model_file = self._get_model_file(opt)
            self.metrics = {'loss': 0.0, 'examples': 0, 'rank': 0}
            opt['rank_candidates'] = True

            super().__init__(opt, shared)

            print('Building model of type {}'.format(self.id))
            self.build_model()
            if model_file:
                print('Loading existing model parameters from ' + model_file)
                self.load(model_file)

        self.rank_loss = nn.CrossEntropyLoss(reduce=True, size_average=False)
        self.set_fixed_candidates(shared)

        if self.use_cuda:
            self.model.cuda()
            self.rank_loss.cuda()

        optim_params = [p for p in self.model.parameters() if p.requires_grad]
        self._init_optim(optim_params)

    def score_candidates(self, batch, cand_vecs):
        """Given a batch and candidate set, return scores (for ranking)"""
        raise NotImplementedError(
            'Abstract class: user must implement score()')

    def build_model(self):
        """Build a new model (implemented by children classes)"""
        raise NotImplementedError(
            'Abstract class: user must implement build_model()')

    def train_step(self, batch):
        """Train on a single batch of examples."""
        if batch.text_vec is None:
            return
        batchsize = batch.text_vec.size(0)
        self.model.train()
        self.optimizer.zero_grad()

        cands, cand_vecs, label_inds = self._build_candidates(
            batch, source=self.opt['train_candidates'], mode='train')
        scores = self.score_candidates(batch, cand_vecs)
        loss = self.rank_loss(scores, label_inds)

        self.metrics['loss'] += loss.item()
        self.metrics['examples'] += batchsize
        _, ranks = scores.sort(1, descending=True)
        for b in range(batchsize):
            rank = (ranks[b] == label_inds[b]).nonzero().item()
            self.metrics['rank'] += 1 + rank

        loss.backward()
        self.update_params()

        cand_preds = []
        for i, ordering in enumerate(ranks):
            if cand_vecs.dim() == 2:
                cand_list = cands
            elif cand_vecs.dim() == 3:
                cand_list = cands[i]
            cand_preds.append([cand_list[rank] for rank in ordering])
        preds = [cand_preds[i][0] for i in range(batchsize)]
        return Output(preds, cand_preds)

    def eval_step(self, batch):
        """Evaluate a single batch of examples."""
        if batch.text_vec is None:
            return
        batchsize = batch.text_vec.size(0)
        self.model.eval()

        cands, cand_vecs, label_inds = self._build_candidates(
            batch, source=self.opt['eval_candidates'], mode='eval')

        scores = self.score_candidates(batch, cand_vecs)
        _, ranks = scores.sort(1, descending=True)

        if label_inds is not None:
            loss = self.rank_loss(scores, label_inds)
            self.metrics['loss'] += loss.item()
            self.metrics['examples'] += batchsize
            for b in range(batchsize):
                rank = (ranks[b] == label_inds[b]).nonzero().item()
                self.metrics['rank'] += 1 + rank

        cand_preds = []
        for i, ordering in enumerate(ranks):
            if cand_vecs.dim() == 2:
                cand_list = cands
            elif cand_vecs.dim() == 3:
                cand_list = cands[i]
            cand_preds.append([cand_list[rank] for rank in ordering])
        preds = [cand_preds[i][0] for i in range(batchsize)]
        return Output(preds, cand_preds)

    def _build_candidates(self, batch, source, mode):
        """Build a candidate set for this batch

        :param batch: a Batch object (defined in torch_agent.py)
        :param source: the source from which candidates should be built, one of
            ['batch', 'inline', 'fixed']
        :param mode: 'train' or 'eval'

        :return: tuple of tensors (label_inds, cands, cand_vecs)
            label_inds: A [bsz] LongTensor of the indices of the labels for each
                example from its respective candidate set
            cands: A [num_cands] list of (text) candidates
                OR a [batchsize] list of such lists if source=='inline'
            cand_vecs: A padded [num_cands, seqlen] LongTensor of vectorized candidates
                OR a [batchsize, num_cands, seqlen] LongTensor if source=='inline'

        Possible sources of candidates:
            * batch: the set of all labels in this batch
                Use all labels in the batch as the candidate set (with all but the
                example's label being treated as negatives). If labels are single
                tokens, filter to unique labels. (Ideally, we would always filter to
                unique labels, but we abstain for the sake of speed.)
                Note: with this setting, the candidate set is identical for all
                examplesin a batch.
            * inline: batch_size lists, one list per example
                If each example comes with a list of possible candidates, use those.
                Note: With this setting, each example will have its own candidate set.
            * fixed: one global candidate list, provide in a file from the user
                If self.fixed_candidates is not None, use a set of fixed candidates for
                all examples.
                Note: this setting is not recommended for training unless the
                universe of possible candidates is very small.
        """
        label_vecs = batch.label_vec  # [bsz] list of lists of LongTensors
        label_inds = None
        batchsize = batch.text_vec.shape[0]

        if label_vecs is not None:
            assert label_vecs.dim() == 2

        if source == 'batch':
            self._warn_once(
                flag=(mode + '_batch_candidates'),
                msg=('[ Executing {} mode with batch labels as set of candidates. ]'
                     ''.format(mode)))
            if batchsize == 1:
                self._warn_once(
                    flag=(mode + '_batchsize_1'),
                    msg=("[ Warning: using candidate source 'batch' and observed a "
                         "batch of size 1. This may be due to uneven batch sizes at "
                         "the end of an epoch. ]"))
            if label_vecs is None:
                raise ValueError(
                    "If using candidate source 'batch', then batch.label_vec cannot be "
                    "None.")

            # Labels are single tokens, enforce uniqueness
            if label_vecs.size(1) == 1:
                cand_vecs, label_inds = label_vecs.unique(return_inverse=True)
                cands = [batch.labels[ind] for ind in label_inds]
            # To save computation, treat all label_vecs as unique
            else:
                cands = batch.labels
                cand_vecs = label_vecs
                label_inds = label_vecs.new(range(batchsize))

        elif source == 'inline':
            self._warn_once(
                flag=(mode + '_batch_candidates'),
                msg=('[ Executing {} mode with provided inline set of candidates ]'
                     ''.format(mode)))
            if batch.candidate_vecs is None:
                raise ValueError(
                    "If using candidate source 'inline', then batch.candidate_vecs "
                    "cannot be None.")

            cands = batch.candidates
            cand_vecs = padded_3d(batch.candidate_vecs)
            if label_vecs is not None:
                label_inds = label_vecs.new_empty(batchsize)
                for i, label_vec in enumerate(label_vecs):
                    label_vec_pad = label_vec.new_zeros(cand_vecs[i].size(1))
                    label_vec_pad[0:label_vec.size(0)] = label_vec
                    label_inds[i] = (cand_vecs[i] == label_vec_pad).all(1).nonzero()[0]

        elif source == 'fixed':
            self._warn_once(
                flag=(mode + '_batch_candidates'),
                msg=('[ Executing {} mode with a common set of fixed candidates. ]'
                     ''.format(mode)))
            if self.fixed_candidates is None:
                raise ValueError(
                    "If using candidate source 'fixed', then you must provide the path "
                    "to a file of candidates with the flag --fixed-candidates-path")

            cands = self.fixed_candidates
            cand_vecs = self.fixed_candidate_vecs
            if label_vecs is not None:
                label_inds = label_vecs.new_empty((batchsize))
                for i, label in enumerate(label_vecs):
                    label_inds[i] = (cand_vecs == label).all(1).nonzero()

        return (cands, cand_vecs, label_inds)

    def share(self):
        """Share model parameters."""
        shared = super().share()
        shared['model'] = self.model
        if self.opt.get('numthreads', 1) > 1 and isinstance(self.metrics, dict):
            torch.set_num_threads(1)
            # move metrics and model to shared memory
            self.metrics = SharedTable(self.metrics)
            self.model.share_memory()
        shared['metrics'] = self.metrics
        shared['fixed_candidates'] = self.fixed_candidates
        shared['fixed_candidate_vecs'] = self.fixed_candidate_vecs
        return shared

    def update_params(self):
        """Do optim step and clip gradients if needed."""
        if self.clip > 0:
            torch.nn.utils.clip_grad_norm_(self.model.parameters(), self.clip)
        self.optimizer.step()

    def reset_metrics(self):
        """Reset metrics."""
        super().reset_metrics()
        self.metrics['loss'] = 0.0
        self.metrics['examples'] = 0
        self.metrics['rank'] = 0

    def report(self):
        """Report loss and mean_rank from model's perspective."""
        m = {}
        examples = self.metrics['examples']
        if examples > 0:
            m['examples'] = examples
            m['loss'] = self.metrics['loss']
            m['mean_loss'] = self.metrics['loss'] / examples
            m['mean_rank'] = self.metrics['rank'] / examples
        for k, v in m.items():
            # clean up: rounds to sigfigs and converts tensors to floats
            m[k] = round_sigfigs(v, 4)
        return m

    def _get_model_file(self, opt):
        model_file = None

        # first check load path in case we need to override paths
        if opt.get('init_model') and os.path.isfile(opt['init_model']):
            # check first for 'init_model' for loading model from file
            model_file = opt['init_model']

        if opt.get('model_file') and os.path.isfile(opt['model_file']):
            # next check for 'model_file', this would override init_model
            model_file = opt['model_file']

        if model_file is not None:
            # if we are loading a model, should load its dict too
            if (os.path.isfile(model_file + '.dict') or
                    opt['dict_file'] is None):
                opt['dict_file'] = model_file + '.dict'

        return model_file

    def set_fixed_candidates(self, shared):
        """Load a set of fixed candidates and their vectors (or vectorize them here)

        self.fixed_candidates will contain a [num_cands] list of strings
        self.fixed_candidate_vecs will contain a [num_cands, seq_len] LongTensor

        Note: TorchRankerAgent by default converts candidates to vectors by vectorizing
        in the common sense (i.e., replacing each token with its index in the
        dictionary). If a child model wants to actually perform encoding, it can
        overwrite the vectorize_fixed_candidates() method to produce encoded vectors
        instead of just vectorized ones.
        """
        if shared:
            self.fixed_candidates = shared['fixed_candidates']
            self.fixed_candidate_vecs = shared['fixed_candidate_vecs']
        else:
            opt = self.opt
            if opt['fixed_candidates_path']:
                print("[ Loading fixed candidate set text from {} ]".format(
                    opt['fixed_candidates_path']))
                with open(opt['fixed_candidates_path'], 'r') as f:
                    self.fixed_candidates = [line.strip() for line in f.readlines()]

                if (opt['fixed_candidate_vecs_path'] and
                        os.path.isfile(opt['fixed_candidate_vecs_path'])):
                    print("[ Loading fixed candidate set vectors from {} ]".format(
                        opt['fixed_candidate_vecs_path']))
                    self.fixed_candidate_vecs = torch.load(
                        opt['fixed_candidate_vecs_path'])
                else:
                    cands = self.fixed_candidates
                    cand_batches = [cands[i:i + 512] for i in range(0, len(cands), 512)]
                    print("[ Vectorizing fixed candidates set from {} ({} batch(es) of "
                          "up to 512) ]".format(opt['fixed_candidates_path'],
                                          len(cand_batches)))
                    cand_vecs = []
                    for batch in cand_batches:
                        cand_vecs.extend(self.vectorize_fixed_candidates(batch))
                    self.fixed_candidate_vecs = padded_3d([cand_vecs]).squeeze(0)
                    if opt['fixed_candidate_vecs_path']:
                        print("[ Saving fixed candidate set vectors to {} ]".format(
                            opt['fixed_candidate_vecs_path']))
                        torch.save(self.fixed_candidate_vecs,
                                   opt['fixed_candidate_vecs_path'])

                if self.use_cuda:
                    self.fixed_candidate_vecs = self.fixed_candidate_vecs.cuda()
            else:
                self.fixed_candidates = None
                self.fixed_candidate_vecs = None

    def vectorize_fixed_candidates(self, cands_batch):
        """Convert a batch of candidates from text to vectors

        Args:
            cands_batch: a [batchsize] list of candiadates (strings)

        By default, candidates are simply vectorized (tokens replaced by token ids).
        A child class may choose to overwrite this method to perform vecotrization as
        well as encoding if so desired.
        """
        return [self._vectorize_text(cand, truncate=self.truncate, truncate_left=False)
                for cand in cands_batch]

    def _warn_once(self, flag, msg):
        """
        Args:
            flag: The name of the flag
            msg: The message to display
        """
        warn_flag = '__warned_' + flag
        if not hasattr(self, warn_flag):
            setattr(self, warn_flag, True)
            print(msg)
<commit_msg>Calculate predictions but not rankings in train_step<commit_after>#!/usr/bin/env python3

# Copyright (c) 2017-present, Facebook, Inc.
# All rights reserved.
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree. An additional grant
# of patent rights can be found in the PATENTS file in the same directory.

import os

import torch
from torch import nn

from parlai.core.torch_agent import TorchAgent, Output
from parlai.core.thread_utils import SharedTable
from parlai.core.utils import round_sigfigs, padded_3d

class TorchRankerAgent(TorchAgent):
    @staticmethod
    def add_cmdline_args(argparser):
        TorchAgent.add_cmdline_args(argparser)
        agent = argparser.add_argument_group('TorchRankerAgent')
        agent.add_argument(
            '-tc', '--train-candidates', type=str, default='batch',
            choices=['batch', 'inline', 'fixed', 'vocab'],
            help='The source of candidates during training')
        agent.add_argument(
            '-ec', '--eval-candidates', type=str, default='batch',
            choices=['batch', 'inline', 'fixed', 'vocab'],
            help='The source of candidates during training')
        agent.add_argument(
            '-cands', '--fixed-candidates-path', type=str,
            help='A text file of fixed candidates to use for all examples, one '
                 'candidate per line')
        agent.add_argument(
            '-candvecs', '--fixed-candidate-vecs-path', type=str,
            help='A torch file containing vectorized fixed candidates to use for all '
                 'examples. If a file already exists at -fixed-candidate-vecs-path, '
                 'those vectors will be loaded. If not, they will be computed and '
                 'written to that path')

    def __init__(self, opt, shared):
        if shared:
            super().__init__(opt, shared)
            self.model = shared['model']
            self.metrics = shared['metrics']
        else:
            # Must call _get_model_file first so that paths are updated if
            # necessary (e.g., .dict file)
            model_file = self._get_model_file(opt)
            self.metrics = {'loss': 0.0, 'examples': 0, 'rank': 0}
            opt['rank_candidates'] = True

            super().__init__(opt, shared)

            print('Building model of type {}'.format(self.id))
            self.build_model()
            if model_file:
                print('Loading existing model parameters from ' + model_file)
                self.load(model_file)

        self.rank_loss = nn.CrossEntropyLoss(reduce=True, size_average=False)
        self.set_fixed_candidates(shared)

        if self.use_cuda:
            self.model.cuda()
            self.rank_loss.cuda()

        optim_params = [p for p in self.model.parameters() if p.requires_grad]
        self._init_optim(optim_params)

    def score_candidates(self, batch, cand_vecs):
        """Given a batch and candidate set, return scores (for ranking)"""
        raise NotImplementedError(
            'Abstract class: user must implement score()')

    def build_model(self):
        """Build a new model (implemented by children classes)"""
        raise NotImplementedError(
            'Abstract class: user must implement build_model()')

    def train_step(self, batch):
        """Train on a single batch of examples."""
        if batch.text_vec is None:
            return
        batchsize = batch.text_vec.size(0)
        self.model.train()
        self.optimizer.zero_grad()

        cands, cand_vecs, label_inds = self._build_candidates(
            batch, source=self.opt['train_candidates'], mode='train')
        scores = self.score_candidates(batch, cand_vecs)
        loss = self.rank_loss(scores, label_inds)

        # Update metrics
        self.metrics['loss'] += loss.item()
        self.metrics['examples'] += batchsize
        _, ranks = scores.sort(1, descending=True)
        for b in range(batchsize):
            rank = (ranks[b] == label_inds[b]).nonzero().item()
            self.metrics['rank'] += 1 + rank

        loss.backward()
        self.update_params()

        # Get predictions but not full rankings for the sake of speed
        preds = [self._v2t(cands[row[0]]) for row in ranks]
        return Output(preds)

    def eval_step(self, batch):
        """Evaluate a single batch of examples."""
        if batch.text_vec is None:
            return
        batchsize = batch.text_vec.size(0)
        self.model.eval()

        cands, cand_vecs, label_inds = self._build_candidates(
            batch, source=self.opt['eval_candidates'], mode='eval')

        scores = self.score_candidates(batch, cand_vecs)
        _, ranks = scores.sort(1, descending=True)

        # Update metrics
        if label_inds is not None:
            loss = self.rank_loss(scores, label_inds)
            self.metrics['loss'] += loss.item()
            self.metrics['examples'] += batchsize
            for b in range(batchsize):
                rank = (ranks[b] == label_inds[b]).nonzero().item()
                self.metrics['rank'] += 1 + rank

        cand_preds = []
        for i, ordering in enumerate(ranks):
            if cand_vecs.dim() == 2:
                cand_list = cands
            elif cand_vecs.dim() == 3:
                cand_list = cands[i]
            cand_preds.append([cand_list[rank] for rank in ordering])
        preds = [cand_preds[i][0] for i in range(batchsize)]
        return Output(preds, cand_preds)

    def _build_candidates(self, batch, source, mode):
        """Build a candidate set for this batch

        :param batch: a Batch object (defined in torch_agent.py)
        :param source: the source from which candidates should be built, one of
            ['batch', 'inline', 'fixed']
        :param mode: 'train' or 'eval'

        :return: tuple of tensors (label_inds, cands, cand_vecs)
            label_inds: A [bsz] LongTensor of the indices of the labels for each
                example from its respective candidate set
            cands: A [num_cands] list of (text) candidates
                OR a [batchsize] list of such lists if source=='inline'
            cand_vecs: A padded [num_cands, seqlen] LongTensor of vectorized candidates
                OR a [batchsize, num_cands, seqlen] LongTensor if source=='inline'

        Possible sources of candidates:
            * batch: the set of all labels in this batch
                Use all labels in the batch as the candidate set (with all but the
                example's label being treated as negatives). If labels are single
                tokens, filter to unique labels. (Ideally, we would always filter to
                unique labels, but we abstain for the sake of speed.)
                Note: with this setting, the candidate set is identical for all
                examplesin a batch.
            * inline: batch_size lists, one list per example
                If each example comes with a list of possible candidates, use those.
                Note: With this setting, each example will have its own candidate set.
            * fixed: one global candidate list, provide in a file from the user
                If self.fixed_candidates is not None, use a set of fixed candidates for
                all examples.
                Note: this setting is not recommended for training unless the
                universe of possible candidates is very small.
        """
        label_vecs = batch.label_vec  # [bsz] list of lists of LongTensors
        label_inds = None
        batchsize = batch.text_vec.shape[0]

        if label_vecs is not None:
            assert label_vecs.dim() == 2

        if source == 'batch':
            self._warn_once(
                flag=(mode + '_batch_candidates'),
                msg=('[ Executing {} mode with batch labels as set of candidates. ]'
                     ''.format(mode)))
            if batchsize == 1:
                self._warn_once(
                    flag=(mode + '_batchsize_1'),
                    msg=("[ Warning: using candidate source 'batch' and observed a "
                         "batch of size 1. This may be due to uneven batch sizes at "
                         "the end of an epoch. ]"))
            if label_vecs is None:
                raise ValueError(
                    "If using candidate source 'batch', then batch.label_vec cannot be "
                    "None.")

            # Labels are single tokens, enforce uniqueness
            if label_vecs.size(1) == 1:
                cand_vecs, label_inds = label_vecs.unique(return_inverse=True)
                cands = [batch.labels[ind] for ind in label_inds]
            # To save computation, treat all label_vecs as unique
            else:
                cands = batch.labels
                cand_vecs = label_vecs
                label_inds = label_vecs.new(range(batchsize))

        elif source == 'inline':
            self._warn_once(
                flag=(mode + '_batch_candidates'),
                msg=('[ Executing {} mode with provided inline set of candidates ]'
                     ''.format(mode)))
            if batch.candidate_vecs is None:
                raise ValueError(
                    "If using candidate source 'inline', then batch.candidate_vecs "
                    "cannot be None.")

            cands = batch.candidates
            cand_vecs = padded_3d(batch.candidate_vecs)
            if label_vecs is not None:
                label_inds = label_vecs.new_empty(batchsize)
                for i, label_vec in enumerate(label_vecs):
                    label_vec_pad = label_vec.new_zeros(cand_vecs[i].size(1))
                    label_vec_pad[0:label_vec.size(0)] = label_vec
                    label_inds[i] = (cand_vecs[i] == label_vec_pad).all(1).nonzero()[0]

        elif source == 'fixed':
            self._warn_once(
                flag=(mode + '_batch_candidates'),
                msg=('[ Executing {} mode with a common set of fixed candidates. ]'
                     ''.format(mode)))
            if self.fixed_candidates is None:
                raise ValueError(
                    "If using candidate source 'fixed', then you must provide the path "
                    "to a file of candidates with the flag --fixed-candidates-path")

            cands = self.fixed_candidates
            cand_vecs = self.fixed_candidate_vecs
            if label_vecs is not None:
                label_inds = label_vecs.new_empty((batchsize))
                for i, label in enumerate(label_vecs):
                    label_inds[i] = (cand_vecs == label).all(1).nonzero()

        return (cands, cand_vecs, label_inds)

    def share(self):
        """Share model parameters."""
        shared = super().share()
        shared['model'] = self.model
        if self.opt.get('numthreads', 1) > 1 and isinstance(self.metrics, dict):
            torch.set_num_threads(1)
            # move metrics and model to shared memory
            self.metrics = SharedTable(self.metrics)
            self.model.share_memory()
        shared['metrics'] = self.metrics
        shared['fixed_candidates'] = self.fixed_candidates
        shared['fixed_candidate_vecs'] = self.fixed_candidate_vecs
        return shared

    def update_params(self):
        """Do optim step and clip gradients if needed."""
        if self.clip > 0:
            torch.nn.utils.clip_grad_norm_(self.model.parameters(), self.clip)
        self.optimizer.step()

    def reset_metrics(self):
        """Reset metrics."""
        super().reset_metrics()
        self.metrics['examples'] = 0
        self.metrics['loss'] = 0.0
        self.metrics['rank'] = 0

    def report(self):
        """Report loss and mean_rank from model's perspective."""
        m = {}
        examples = self.metrics['examples']
        if examples > 0:
            m['examples'] = examples
            m['loss'] = self.metrics['loss']
            m['mean_loss'] = self.metrics['loss'] / examples
            m['mean_rank'] = self.metrics['rank'] / examples
        for k, v in m.items():
            # clean up: rounds to sigfigs and converts tensors to floats
            m[k] = round_sigfigs(v, 4)
        return m

    def _get_model_file(self, opt):
        model_file = None

        # first check load path in case we need to override paths
        if opt.get('init_model') and os.path.isfile(opt['init_model']):
            # check first for 'init_model' for loading model from file
            model_file = opt['init_model']

        if opt.get('model_file') and os.path.isfile(opt['model_file']):
            # next check for 'model_file', this would override init_model
            model_file = opt['model_file']

        if model_file is not None:
            # if we are loading a model, should load its dict too
            if (os.path.isfile(model_file + '.dict') or
                    opt['dict_file'] is None):
                opt['dict_file'] = model_file + '.dict'

        return model_file

    def set_fixed_candidates(self, shared):
        """Load a set of fixed candidates and their vectors (or vectorize them here)

        self.fixed_candidates will contain a [num_cands] list of strings
        self.fixed_candidate_vecs will contain a [num_cands, seq_len] LongTensor

        Note: TorchRankerAgent by default converts candidates to vectors by vectorizing
        in the common sense (i.e., replacing each token with its index in the
        dictionary). If a child model wants to actually perform encoding, it can
        overwrite the vectorize_fixed_candidates() method to produce encoded vectors
        instead of just vectorized ones.
        """
        if shared:
            self.fixed_candidates = shared['fixed_candidates']
            self.fixed_candidate_vecs = shared['fixed_candidate_vecs']
        else:
            opt = self.opt
            if opt['fixed_candidates_path']:
                print("[ Loading fixed candidate set text from {} ]".format(
                    opt['fixed_candidates_path']))
                with open(opt['fixed_candidates_path'], 'r') as f:
                    self.fixed_candidates = [line.strip() for line in f.readlines()]

                if (opt['fixed_candidate_vecs_path'] and
                        os.path.isfile(opt['fixed_candidate_vecs_path'])):
                    print("[ Loading fixed candidate set vectors from {} ]".format(
                        opt['fixed_candidate_vecs_path']))
                    self.fixed_candidate_vecs = torch.load(
                        opt['fixed_candidate_vecs_path'])
                else:
                    cands = self.fixed_candidates
                    cand_batches = [cands[i:i + 512] for i in range(0, len(cands), 512)]
                    print("[ Vectorizing fixed candidates set from {} ({} batch(es) of "
                          "up to 512) ]".format(opt['fixed_candidates_path'],
                                          len(cand_batches)))
                    cand_vecs = []
                    for batch in cand_batches:
                        cand_vecs.extend(self.vectorize_fixed_candidates(batch))
                    self.fixed_candidate_vecs = padded_3d([cand_vecs]).squeeze(0)
                    if opt['fixed_candidate_vecs_path']:
                        print("[ Saving fixed candidate set vectors to {} ]".format(
                            opt['fixed_candidate_vecs_path']))
                        torch.save(self.fixed_candidate_vecs,
                                   opt['fixed_candidate_vecs_path'])

                if self.use_cuda:
                    self.fixed_candidate_vecs = self.fixed_candidate_vecs.cuda()
            else:
                self.fixed_candidates = None
                self.fixed_candidate_vecs = None

    def vectorize_fixed_candidates(self, cands_batch):
        """Convert a batch of candidates from text to vectors

        Args:
            cands_batch: a [batchsize] list of candiadates (strings)

        By default, candidates are simply vectorized (tokens replaced by token ids).
        A child class may choose to overwrite this method to perform vecotrization as
        well as encoding if so desired.
        """
        return [self._vectorize_text(cand, truncate=self.truncate, truncate_left=False)
                for cand in cands_batch]

    def _warn_once(self, flag, msg):
        """
        Args:
            flag: The name of the flag
            msg: The message to display
        """
        warn_flag = '__warned_' + flag
        if not hasattr(self, warn_flag):
            setattr(self, warn_flag, True)
            print(msg)
<|endoftext|>


<commit_before># coding=utf-8

from app import db
from werkzeug.security import generate_password_hash, check_password_hash
from mongoengine import NULLIFY, PULL

from app.helpers.filestorage import *

from app.structures.models.user import User

#Some constants for doing submission status stuff
SUBMISSION_UNGRADED = 0
SUBMISSION_TESTING = 1
SUBMISSION_TESTED = 2
SUBMISSION_GRADING = 3
SUBMISSION_GRADED = 4

class Submission(db.Document):
  '''
  A submission contains all the information about one attempt at a given problem
  by a student.
  '''
  #bookkeeping
  submitter = db.ReferenceField('User')
  problem = db.ReferenceField('Problem')
  isLatest = db.BooleanField(default=True)

  submissionTime = db.DateTimeField(required=True)
  isLate = db.BooleanField(default=False)
  grade = db.ReferenceField('GBGrade')
  status = db.IntField(default=0)
  gradedBy = db.ReferenceField('User')

  #Comment fields
  comments = db.StringField(default="")
  autoGraderComments = db.StringField(default="")

  #partnerinfo
  partner = db.ReferenceField('User')
  partnerSubmission = db.ReferenceField('Submission')

  meta = {"cascade": True}

  def cleanup(self):
    try:
      if self.partnerInfo:
        self.partnerInfo.delete()
      self.grade.delete()
    except:
      pass

  def getStatus(self):
    if self.status == SUBMISSION_UNGRADED:
      return "info", "Submitted (Waiting for Auto-grader)"
    elif self.status == SUBMISSION_TESTING:
      return "warning", "Autograde in progress"
    elif self.status == SUBMISSION_TESTED:
      return "info", "Submitted (Auto-graded, Waiting for Grader)"
    elif self.status == SUBMISSION_GRADING:
      return "warning", "Grading in progress"
    elif self.status == SUBMISSION_GRADED:
      return "success", "Graded"

  def setGrades(self, gradeDict):
    pass

class StudentSubmissionList(db.EmbeddedDocument):
  '''
  A list of all the submissions a student has made for a specific problem.
  '''
  submissions = db.ListField(db.ReferenceField('Submission'))

  meta = {"cascade": True}

  def cleanup(self):
    for s in self.submissions:
      s.cleanup()
      s.delete()
    self.submissions = []

  def addSubmission(self, sub):
    if len(self.submissions) > 0:
      self.submissions[-1].isLatest = False
      self.submissions[-1].save()
    self.submissions.append(sub)

class Problem(db.Document):
  '''
  One problem that a student can submit files to.
  '''
  name = db.StringField()
  gradeColumn = db.ReferenceField('GBColumn', reverse_delete_rule=NULLIFY)
  duedate = db.DateTimeField()

  #The problem rubric
  rubric = db.MapField(db.DecimalField())

  #List of the names of the files we test with
  testfiles = db.ListField(db.StringField())

  #Can students have partners
  allowPartners = db.BooleanField(default=True)

  #Which files must a student submit
  requiredFiles = db.StringField(default=None)
  strictFiles = db.StringField(default=None)
  bannedFiles = db.StringField(default=None)

  #URLs for grader notes and for problem specification
  gradeNotes = db.StringField()
  problemPage = db.StringField()

  #Settings for releasing grades
  releaseAutoComments = db.BooleanField(default=True)
  autoGradeOnly = db.BooleanField(default=False)

  #Settings for opening and closing a problem
  isOpen = db.BooleanField(default=True)

  #Map usernames to submission lists
  studentSubmissions = db.MapField(db.EmbeddedDocumentField('StudentSubmissionList'))

  meta = {"cascade": True}

  def __init__(self, name, **data):
    super(Problem, self).__init__(**data)
    self.name = name

  def getStatusCount(self):
    c,a = self.getParents()
    users = User.objects.filter(courseStudent=c)
    ungraded = 0
    ip = 0
    done = 0
    for u in users:
      sub = self.getLatestSubmission(u)
      if sub == None:
        continue
      if sub.status < 3:
        ungraded += 1
      elif sub.status == 3:
        ip += 1
      else:
        done += 1
    return ungraded, ip, done

  def cleanup(self):
    if self.gradeColumn != None:
      self.gradeColumn.cleanup()
      self.gradeColumn.delete()
    for k in self.studentSubmissions:
      self.studentSubmissions[k].cleanup()

  def totalPoints(self):
    total = 0
    for k in self.rubric:
      total += self.rubric[k]
    return total

  def getSubmissionNumber(self, user):
    '''gets the number of the latest submission'''
    if user.username in self.studentSubmissions:
      return len(self.studentSubmissions[user.username].submissions)
    else:
      return 0

  def getSubmission(self, user, subnum):
    '''Returns a single submission'''
    return self.studentSubmissions[user.username].submissions[int(subnum)-1]

  def getLatestSubmission(self, user):
    '''Gets the latest submission for a user'''
    if self.getSubmissionNumber(user) == 0:
      return None
    else:
      return self.getSubmission(user, self.getSubmissionNumber(user))

  def getParents(self):
    a = AssignmentGroup.objects.get(problems=self)
    c = Course.objects.get(assignments=a)
    return c,a

  def getFiles(self, user, subnum):
    from os import listdir
    from os.path import isfile, join
    c, a = self.getParents()
    filePath = getSubmissionPath(c, a, self, user, subnum)
    return [ f for f in listdir(filePath) if isfile(join(filePath,f)) ]

  def getFilePath(self, user, subnum):
    c, a = self.getParents()
    return getSubmissionPath(c, a, self, user, subnum)

  def getTestFilePath(self):
    c, a = self.getParents()
    ensurePathExists(getTestPath(c, a, self))
    return getTestPath(c, a, self)

  def getRequiredFiles(self):
    import re
    if self.requiredFiles != None and len(self.requiredFiles) > 0:
      return re.split(', *', self.requiredFiles)
    else:
      return []

  def getStrictFiles(self):
    import re
    if self.strictFiles != None and len(self.strictFiles) > 0:
      return re.split(', *', self.strictFiles)
    else:
      return []

  def getSubmissionInfo(self, sub):
    for key, value in self.studentSubmissions.iteritems():
      if sub in value.submissions:
        return User.objects.get(username=key), (value.submissions.index(sub)+1)
    return None, -1

class AssignmentGroup(db.Document):
  '''
  A logical grouping of problems (e.g. One week's homework)
  '''
  name = db.StringField(required=True)
  gradeEntry = db.ReferenceField('GBGroup', reverse_delete_rule=NULLIFY)
  problems = db.ListField(db.ReferenceField('Problem', reverse_delete_rule=PULL))

  meta = {"cascade": True}

  def __init__(self, name, **data):
    super(AssignmentGroup, self).__init__(**data)
    self.name = name

  def cleanup(self):
    if self.gradeEntry != None:
      self.gradeEntry.cleanup()
    for p in self.problems:
      p.cleanup()
      p.delete()

class Course(db.Document):
  '''
  One sememsters class
  '''
  #Identification information
  name = db.StringField(required=True)
  semester = db.StringField(required=True)

  #Information for grading and submission
  gradeBook = db.EmbeddedDocumentField('GradeBook')
  assignments = db.ListField(db.ReferenceField('AssignmentGroup', reverse_delete_rule=PULL))

  #Is this course still being taught at this time
  isActive = db.BooleanField(default=True)

  #Do we show usernames during grading
  anonymousGrading = db.BooleanField(default=False)
  #Map real usernames to identifiers
  anonIds = db.MapField(db.StringField())
  #How do we calculate late grades (Defaults to one that just highlights late grades)
  lateGradePolicy = db.StringField(default="Highlighter")

  homepage = db.StringField(default="#")

  meta = {"cascade": True, 'ordering': ["+semester", "+name"]}

  def cleanup(self):
    self.gradeBook.cleanup()

    for a in self.assignments:
      a.cleanup()
      a.delete()

  def ensureIDs(self):
    users = User.objects.filter(courseStudent=self)
    for u in users:
      if u.username in self.anonIds:
        continue
      else:
        from random import choice, randint
        from string import ascii_lowercase
        while True:
          ID = choice(ascii_lowercase) + choice(ascii_lowercase)+ str(randint(0,9))
          if not ID in self.anonIds.values():
            self.anonIds[u.username] = ID
            break
    self.save()

  def getIdentifier(self, username):
    if not username in self.anonIds:
      self.ensureIDs()
    return self.anonIds[username]
<commit_msg>Added two API functions for helping programmatically set grades.<commit_after># coding=utf-8

from app import db
from werkzeug.security import generate_password_hash, check_password_hash
from mongoengine import NULLIFY, PULL

from app.helpers.filestorage import *

from app.structures.models.user import User

#Some constants for doing submission status stuff
SUBMISSION_UNGRADED = 0
SUBMISSION_TESTING = 1
SUBMISSION_TESTED = 2
SUBMISSION_GRADING = 3
SUBMISSION_GRADED = 4

class Submission(db.Document):
  '''
  A submission contains all the information about one attempt at a given problem
  by a student.
  '''
  #bookkeeping
  submitter = db.ReferenceField('User')
  problem = db.ReferenceField('Problem')
  isLatest = db.BooleanField(default=True)

  submissionTime = db.DateTimeField(required=True)
  isLate = db.BooleanField(default=False)
  grade = db.ReferenceField('GBGrade')
  status = db.IntField(default=0)
  gradedBy = db.ReferenceField('User')

  #Comment fields
  comments = db.StringField(default="")
  autoGraderComments = db.StringField(default="")

  #partnerinfo
  partner = db.ReferenceField('User')
  partnerSubmission = db.ReferenceField('Submission')

  meta = {"cascade": True}

  def cleanup(self):
    try:
      if self.partnerInfo:
        self.partnerInfo.delete()
      self.grade.delete()
    except:
      pass

  def getStatus(self):
    if self.status == SUBMISSION_UNGRADED:
      return "info", "Submitted (Waiting for Auto-grader)"
    elif self.status == SUBMISSION_TESTING:
      return "warning", "Autograde in progress"
    elif self.status == SUBMISSION_TESTED:
      return "info", "Submitted (Auto-graded, Waiting for Grader)"
    elif self.status == SUBMISSION_GRADING:
      return "warning", "Grading in progress"
    elif self.status == SUBMISSION_GRADED:
      return "success", "Graded"

  def setGrades(self, gradeDict, zeroOthers=False):
    for k in gradeDict.iterkeys():
      if not k in problem.rubric:
        raise Exception("Requested rubric section " + k + " is not in the problem rubric")

    #If we validated all the keys then apply the grades
    #iterate across the rubric so that we can zero non-supplied grades if needed
    for k in problem.rubric.iterkeys():
      if not k in gradeDict and zeroOthers:
        self.grade[k] = 0.0
      else:
        self.grade[k] = gradeDict[k]

    #save the grade when we are done
    self.grade.save()

  def setGradedBy(self, grader):
    if type(grader) is str:
      grader = User.objects.get(username=grader)

    if type(grader) is User:
      self.gradedBy = grader
    else:
      raise Exception("Invalid type for gradedBy field")

class StudentSubmissionList(db.EmbeddedDocument):
  '''
  A list of all the submissions a student has made for a specific problem.
  '''
  submissions = db.ListField(db.ReferenceField('Submission'))

  meta = {"cascade": True}

  def cleanup(self):
    for s in self.submissions:
      s.cleanup()
      s.delete()
    self.submissions = []

  def addSubmission(self, sub):
    if len(self.submissions) > 0:
      self.submissions[-1].isLatest = False
      self.submissions[-1].save()
    self.submissions.append(sub)

class Problem(db.Document):
  '''
  One problem that a student can submit files to.
  '''
  name = db.StringField()
  gradeColumn = db.ReferenceField('GBColumn', reverse_delete_rule=NULLIFY)
  duedate = db.DateTimeField()

  #The problem rubric
  rubric = db.MapField(db.DecimalField())

  #List of the names of the files we test with
  testfiles = db.ListField(db.StringField())

  #Can students have partners
  allowPartners = db.BooleanField(default=True)

  #Which files must a student submit
  requiredFiles = db.StringField(default=None)
  strictFiles = db.StringField(default=None)
  bannedFiles = db.StringField(default=None)

  #URLs for grader notes and for problem specification
  gradeNotes = db.StringField()
  problemPage = db.StringField()

  #Settings for releasing grades
  releaseAutoComments = db.BooleanField(default=True)
  autoGradeOnly = db.BooleanField(default=False)

  #Settings for opening and closing a problem
  isOpen = db.BooleanField(default=True)

  #Map usernames to submission lists
  studentSubmissions = db.MapField(db.EmbeddedDocumentField('StudentSubmissionList'))

  meta = {"cascade": True}

  def __init__(self, name, **data):
    super(Problem, self).__init__(**data)
    self.name = name

  def getStatusCount(self):
    c,a = self.getParents()
    users = User.objects.filter(courseStudent=c)
    ungraded = 0
    ip = 0
    done = 0
    for u in users:
      sub = self.getLatestSubmission(u)
      if sub == None:
        continue
      if sub.status < 3:
        ungraded += 1
      elif sub.status == 3:
        ip += 1
      else:
        done += 1
    return ungraded, ip, done

  def cleanup(self):
    if self.gradeColumn != None:
      self.gradeColumn.cleanup()
      self.gradeColumn.delete()
    for k in self.studentSubmissions:
      self.studentSubmissions[k].cleanup()

  def totalPoints(self):
    total = 0
    for k in self.rubric:
      total += self.rubric[k]
    return total

  def getSubmissionNumber(self, user):
    '''gets the number of the latest submission'''
    if user.username in self.studentSubmissions:
      return len(self.studentSubmissions[user.username].submissions)
    else:
      return 0

  def getSubmission(self, user, subnum):
    '''Returns a single submission'''
    return self.studentSubmissions[user.username].submissions[int(subnum)-1]

  def getLatestSubmission(self, user):
    '''Gets the latest submission for a user'''
    if self.getSubmissionNumber(user) == 0:
      return None
    else:
      return self.getSubmission(user, self.getSubmissionNumber(user))

  def getParents(self):
    a = AssignmentGroup.objects.get(problems=self)
    c = Course.objects.get(assignments=a)
    return c,a

  def getFiles(self, user, subnum):
    from os import listdir
    from os.path import isfile, join
    c, a = self.getParents()
    filePath = getSubmissionPath(c, a, self, user, subnum)
    return [ f for f in listdir(filePath) if isfile(join(filePath,f)) ]

  def getFilePath(self, user, subnum):
    c, a = self.getParents()
    return getSubmissionPath(c, a, self, user, subnum)

  def getTestFilePath(self):
    c, a = self.getParents()
    ensurePathExists(getTestPath(c, a, self))
    return getTestPath(c, a, self)

  def getRequiredFiles(self):
    import re
    if self.requiredFiles != None and len(self.requiredFiles) > 0:
      return re.split(', *', self.requiredFiles)
    else:
      return []

  def getStrictFiles(self):
    import re
    if self.strictFiles != None and len(self.strictFiles) > 0:
      return re.split(', *', self.strictFiles)
    else:
      return []

  def getSubmissionInfo(self, sub):
    for key, value in self.studentSubmissions.iteritems():
      if sub in value.submissions:
        return User.objects.get(username=key), (value.submissions.index(sub)+1)
    return None, -1

class AssignmentGroup(db.Document):
  '''
  A logical grouping of problems (e.g. One week's homework)
  '''
  name = db.StringField(required=True)
  gradeEntry = db.ReferenceField('GBGroup', reverse_delete_rule=NULLIFY)
  problems = db.ListField(db.ReferenceField('Problem', reverse_delete_rule=PULL))

  meta = {"cascade": True}

  def __init__(self, name, **data):
    super(AssignmentGroup, self).__init__(**data)
    self.name = name

  def cleanup(self):
    if self.gradeEntry != None:
      self.gradeEntry.cleanup()
    for p in self.problems:
      p.cleanup()
      p.delete()

class Course(db.Document):
  '''
  One sememsters class
  '''
  #Identification information
  name = db.StringField(required=True)
  semester = db.StringField(required=True)

  #Information for grading and submission
  gradeBook = db.EmbeddedDocumentField('GradeBook')
  assignments = db.ListField(db.ReferenceField('AssignmentGroup', reverse_delete_rule=PULL))

  #Is this course still being taught at this time
  isActive = db.BooleanField(default=True)

  #Do we show usernames during grading
  anonymousGrading = db.BooleanField(default=False)
  #Map real usernames to identifiers
  anonIds = db.MapField(db.StringField())
  #How do we calculate late grades (Defaults to one that just highlights late grades)
  lateGradePolicy = db.StringField(default="Highlighter")

  homepage = db.StringField(default="#")

  meta = {"cascade": True, 'ordering': ["+semester", "+name"]}

  def cleanup(self):
    self.gradeBook.cleanup()

    for a in self.assignments:
      a.cleanup()
      a.delete()

  def ensureIDs(self):
    users = User.objects.filter(courseStudent=self)
    for u in users:
      if u.username in self.anonIds:
        continue
      else:
        from random import choice, randint
        from string import ascii_lowercase
        while True:
          ID = choice(ascii_lowercase) + choice(ascii_lowercase)+ str(randint(0,9))
          if not ID in self.anonIds.values():
            self.anonIds[u.username] = ID
            break
    self.save()

  def getIdentifier(self, username):
    if not username in self.anonIds:
      self.ensureIDs()
    return self.anonIds[username]
<|endoftext|>